import{s as kl,n as gl,o as _l}from"../chunks/scheduler.9bc65507.js";import{S as Nl,i as Bl,g as s,s as i,u as w,m as Al,A as El,h as a,f as e,c as o,j as c,v as M,y as r,n as Sl,k as p,o as h,a as n,w as d,d as y,t as u,x as m}from"../chunks/index.78c82d43.js";import{I as gt}from"../chunks/IconCopyLink.bedaa44d.js";import{C as _t}from"../chunks/CodeBlock.48618ec2.js";function Vl(Ae){let C,Nt,Jt,Bt,U,I,It,k,ke,Ct,Ee="Custom hardware for training",At,g,Se='The hardware you use to run model training and inference can have a big effect on performance. For a deep dive into GPUs make sure to check out Tim Dettmer’s excellent <a href="https://timdettmers.com/2020/09/07/which-gpu-for-deep-learning/" rel="nofollow">blog post</a>.',Et,_,Ve="Let’s have a look at some practical advice for GPU setups.",St,f,b,bt,N,ge,Ut,Ge="GPU",Vt,B,$e='<li>bigger GPUs</li> <li>more GPUs</li> <li>more CPU and NVMe (offloaded to by <a href="main_classes/deepspeed#nvme-support">DeepSpeed-Infinity</a>)</li>',Gt,A,Le="Let’s start at the case where you have a single GPU.",$t,v,x,xt,E,_e,ft,Ze="Power and Cooling",Lt,S,Xe="If you bought an expensive high end GPU make sure you give it the correct power and sufficient cooling.",Zt,V,Re="<strong>Power</strong>:",Xt,G,We="Some high end consumer GPU cards have 2 and sometimes 3 PCI-E 8-Pin power sockets. Make sure you have as many independent 12V PCI-E 8-Pin cables plugged into the card as there are sockets. Do not use the 2 splits at one end of the same cable (also known as pigtail cable). That is if you have 2 sockets on the GPU, you want 2 PCI-E 8-Pin cables going from your PSU to the card and not one that has 2 PCI-E 8-Pin connectors at the end! You won’t get the full performance out of your card otherwise.",Rt,$,He="Each PCI-E 8-Pin power cable needs to be plugged into a 12V rail on the PSU side and can supply up to 150W of power.",Wt,L,Qe="Some other cards may use a PCI-E 12-Pin connectors, and these can deliver up to 500-600W of power.",Ht,Z,Ye="Low end cards may use 6-Pin connectors, which supply up to 75W of power.",Qt,X,Fe="Additionally you want the high-end PSU that has stable voltage. Some lower quality ones may not give the card the stable voltage it needs to function at its peak.",Yt,R,De="And of course the PSU needs to have enough unused Watts to power the card.",Ft,W,ze="<strong>Cooling</strong>:",Dt,H,Oe="When a GPU gets overheated it will start throttling down and will not deliver full performance and it can even shutdown if it gets too hot.",zt,Q,qe="It’s hard to tell the exact best temperature to strive for when a GPU is heavily loaded, but probably anything under +80C is good, but lower is better - perhaps 70-75C is an excellent range to be in. The throttling down is likely to start at around 84-90C. But other than throttling performance a prolonged very high temperature is likely to reduce the lifespan of a GPU.",Ot,Y,Ke="Next let’s have a look at one of the most important aspects when having multiple GPUs: connectivity.",qt,T,j,jt,F,Ne,vt,tl="Multi-GPU Connectivity",Kt,D,el="If you use multiple GPUs the way cards are inter-connected can have a huge impact on the total training time. If the GPUs are on the same physical node, you can run:",te,z,ee,O,ll="and it will tell you how the GPUs are inter-connected. On a machine with dual-GPU and which are connected with NVLink, you will most likely see something like:",le,q,ne,K,nl="on a different machine w/o NVLink we may see:",se,tt,ae,et,sl="The report includes this legend:",ie,lt,oe,nt,al="So the first report <code>NV2</code> tells us the GPUs are interconnected with 2 NVLinks, and the second report <code>PHB</code> we have a typical consumer-level PCIe+Bridge setup.",re,st,il="Check what type of connectivity you have on your setup. Some of these will make the communication between cards faster (e.g. NVLink), others slower (e.g. PHB).",pe,at,ol="Depending on the type of scalability solution used, the connectivity speed could have a major or a minor impact. If the GPUs need to sync rarely, as in DDP, the impact of a slower connection will be less significant. If the GPUs need to send messages to each other often, as in ZeRO-DP, then faster connectivity becomes super important to achieve faster training.",he,J,P,Pt,it,Be,Tt,rl="NVlink",ce,ot,pl='<a href="https://en.wikipedia.org/wiki/NVLink" rel="nofollow">NVLink</a> is a wire-based serial multi-lane near-range communications link developed by Nvidia.',we,rt,hl='Each new generation provides a faster bandwidth, e.g. here is a quote from <a href="https://www.nvidia.com/content/dam/en-zz/Solutions/geforce/ampere/pdf/NVIDIA-ampere-GA102-GPU-Architecture-Whitepaper-V1.pdf" rel="nofollow">Nvidia Ampere GA102 GPU Architecture</a>:',Me,pt,cl=`<p>Third-Generation NVLink®
GA102 GPUs utilize NVIDIA’s third-generation NVLink interface, which includes four x4 links,
with each link providing 14.0625 GB/sec bandwidth in each direction between two GPUs. Four
links provide 56.25 GB/sec bandwidth in each direction, and 112.5 GB/sec total bandwidth
between two GPUs. Two RTX 3090 GPUs can be connected together for SLI using NVLink.
(Note that 3-Way and 4-Way SLI configurations are not supported.)</p>`,de,ht,wl="So the higher <code>X</code> you get in the report of <code>NVX</code> in the output of <code>nvidia-smi topo -m</code> the better. The generation will depend on your GPU architecture.",ye,ct,Ml="Let’s compare the execution of a gpt2 language model training over a small sample of wikitext.",ue,wt,dl="The results are:",me,Mt,yl='<thead><tr><th>NVlink</th> <th align="right">Time</th></tr></thead> <tbody><tr><td>Y</td> <td align="right">101s</td></tr> <tr><td>N</td> <td align="right">131s</td></tr></tbody>',Ce,dt,ul="You can see that NVLink completes the training ~23% faster. In the second benchmark we use <code>NCCL_P2P_DISABLE=1</code> to tell the GPUs not to use NVLink.",Ue,yt,ml="Here is the full benchmark code and outputs:",fe,ut,ve,mt,Cl=`Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks (<code>NV2</code> in <code>nvidia-smi topo -m</code>)
Software: <code>pytorch-1.8-to-be</code> + <code>cuda-11.0</code> / <code>transformers==4.3.0.dev0</code>`,Te,kt,Je;return k=new gt({}),N=new gt({}),E=new gt({}),F=new gt({}),z=new _t({props:{code:"bnZpZGlhLXNtaSUyMHRvcG8lMjAtbQ==",highlighted:'<span class="hljs-symbol">nvidia</span>-<span class="hljs-keyword">smi</span> topo -m'}}),q=new _t({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwTlYyJTIwJTIwJTIwJTIwJTIwMC0yMyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyME5WMiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTIzJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      NV2     <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A
<span class="hljs-attribute">GPU1</span>    NV2      X      <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A`}}),tt=new _t({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwUEhCJTIwJTIwJTIwJTIwJTIwMC0xMSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyMFBIQiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTExJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      PHB     <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A
<span class="hljs-attribute">GPU1</span>    PHB      X      <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A`}}),lt=new _t({props:{code:"JTIwJTIwWCUyMCUyMCUyMCUyMCUzRCUyMFNlbGYlMEElMjAlMjBTWVMlMjAlMjAlM0QlMjBDb25uZWN0aW9uJTIwdHJhdmVyc2luZyUyMFBDSWUlMjBhcyUyMHdlbGwlMjBhcyUyMHRoZSUyMFNNUCUyMGludGVyY29ubmVjdCUyMGJldHdlZW4lMjBOVU1BJTIwbm9kZXMlMjAoZS5nLiUyQyUyMFFQSSUyRlVQSSklMEElMjAlMjBOT0RFJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjB0aGUlMjBpbnRlcmNvbm5lY3QlMjBiZXR3ZWVuJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2VzJTIwd2l0aGluJTIwYSUyME5VTUElMjBub2RlJTBBJTIwJTIwUEhCJTIwJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjBhJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2UlMjAodHlwaWNhbGx5JTIwdGhlJTIwQ1BVKSUwQSUyMCUyMFBYQiUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwbXVsdGlwbGUlMjBQQ0llJTIwYnJpZGdlcyUyMCh3aXRob3V0JTIwdHJhdmVyc2luZyUyMHRoZSUyMFBDSWUlMjBIb3N0JTIwQnJpZGdlKSUwQSUyMCUyMFBJWCUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYXQlMjBtb3N0JTIwYSUyMHNpbmdsZSUyMFBDSWUlMjBicmlkZ2UlMEElMjAlMjBOViUyMyUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYSUyMGJvbmRlZCUyMHNldCUyMG9mJTIwJTIzJTIwTlZMaW5rcw==",highlighted:`  X    = Self
  SYS  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> interconnect between PCIe Host Bridges <span class="hljs-keyword">within</span> <span class="hljs-keyword">a</span> NUMA node
  PHB  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">a</span> PCIe Host Bridge (typically <span class="hljs-keyword">the</span> CPU)
  PXB  = Connection traversing multiple PCIe bridges (<span class="hljs-keyword">without</span> traversing <span class="hljs-keyword">the</span> PCIe Host Bridge)
  PIX  = Connection traversing <span class="hljs-keyword">at</span> most <span class="hljs-keyword">a</span> single PCIe bridge
  NV<span class="hljs-comment">#  = Connection traversing a bonded set of # NVLinks</span>`}}),it=new gt({}),ut=new _t({props:{code:"JTIzJTIwRERQJTIwdyUyRiUyME5WTGluayUwQSUwQXJtJTIwLXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUzQiUyMENVREFfVklTSUJMRV9ERVZJQ0VTJTNEMCUyQzElMjBweXRob24lMjAtbSUyMHRvcmNoLmRpc3RyaWJ1dGVkLmxhdW5jaCUyMCU1QyUwQS0tbnByb2NfcGVyX25vZGUlMjAyJTIwZXhhbXBsZXMlMkZweXRvcmNoJTJGbGFuZ3VhZ2UtbW9kZWxpbmclMkZydW5fY2xtLnB5JTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBncHQyJTIwJTVDJTBBLS1kYXRhc2V0X25hbWUlMjB3aWtpdGV4dCUyMC0tZGF0YXNldF9jb25maWdfbmFtZSUyMHdpa2l0ZXh0LTItcmF3LXYxJTIwLS1kb190cmFpbiUyMCU1QyUwQS0tb3V0cHV0X2RpciUyMCUyRnRtcCUyRnRlc3QtY2xtJTIwLS1wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjA0JTIwLS1tYXhfc3RlcHMlMjAyMDAlMEElMEElN0IndHJhaW5fcnVudGltZSclM0ElMjAxMDEuOTAwMyUyQyUyMCd0cmFpbl9zYW1wbGVzX3Blcl9zZWNvbmQnJTNBJTIwMS45NjMlMkMlMjAnZXBvY2gnJTNBJTIwMC42OSU3RCUwQSUwQSUyMyUyMEREUCUyMHclMkZvJTIwTlZMaW5rJTBBJTBBcm0lMjAtciUyMCUyRnRtcCUyRnRlc3QtY2xtJTNCJTIwQ1VEQV9WSVNJQkxFX0RFVklDRVMlM0QwJTJDMSUyME5DQ0xfUDJQX0RJU0FCTEUlM0QxJTIwcHl0aG9uJTIwLW0lMjB0b3JjaC5kaXN0cmlidXRlZC5sYXVuY2glMjAlNUMlMEEtLW5wcm9jX3Blcl9ub2RlJTIwMiUyMGV4YW1wbGVzJTJGcHl0b3JjaCUyRmxhbmd1YWdlLW1vZGVsaW5nJTJGcnVuX2NsbS5weSUyMC0tbW9kZWxfbmFtZV9vcl9wYXRoJTIwZ3B0MiUyMCU1QyUwQS0tZGF0YXNldF9uYW1lJTIwd2lraXRleHQlMjAtLWRhdGFzZXRfY29uZmlnX25hbWUlMjB3aWtpdGV4dC0yLXJhdy12MSUyMC0tZG9fdHJhaW4lMEEtLW91dHB1dF9kaXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwNCUyMC0tbWF4X3N0ZXBzJTIwMjAwJTBBJTBBJTdCJ3RyYWluX3J1bnRpbWUnJTNBJTIwMTMxLjQzNjclMkMlMjAndHJhaW5fc2FtcGxlc19wZXJfc2Vjb25kJyUzQSUyMDEuNTIyJTJDJTIwJ2Vwb2NoJyUzQSUyMDAuNjklN0Q=",highlighted:`<span class="hljs-comment"># DDP w/ NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train \\
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 101.9003, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.963, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}

<span class="hljs-comment"># DDP w/o NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 NCCL_P2P_DISABLE=1 python -m torch.distributed.launch \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 131.4367, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.522, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}`}}),{c(){C=s("meta"),Nt=i(),Jt=s("p"),Bt=i(),U=s("h1"),I=s("a"),It=s("span"),w(k.$$.fragment),ke=i(),Ct=s("span"),Ct.textContent=Ee,At=i(),g=s("p"),g.innerHTML=Se,Et=i(),_=s("p"),_.textContent=Ve,St=i(),f=s("h2"),b=s("a"),bt=s("span"),w(N.$$.fragment),ge=i(),Ut=s("span"),Ut.textContent=Ge,Vt=Al(`

When you train bigger models you have essentially three options:
`),B=s("ul"),B.innerHTML=$e,Gt=i(),A=s("p"),A.textContent=Le,$t=i(),v=s("h3"),x=s("a"),xt=s("span"),w(E.$$.fragment),_e=i(),ft=s("span"),ft.textContent=Ze,Lt=i(),S=s("p"),S.textContent=Xe,Zt=i(),V=s("p"),V.innerHTML=Re,Xt=i(),G=s("p"),G.textContent=We,Rt=i(),$=s("p"),$.textContent=He,Wt=i(),L=s("p"),L.textContent=Qe,Ht=i(),Z=s("p"),Z.textContent=Ye,Qt=i(),X=s("p"),X.textContent=Fe,Yt=i(),R=s("p"),R.textContent=De,Ft=i(),W=s("p"),W.innerHTML=ze,Dt=i(),H=s("p"),H.textContent=Oe,zt=i(),Q=s("p"),Q.textContent=qe,Ot=i(),Y=s("p"),Y.textContent=Ke,qt=i(),T=s("h3"),j=s("a"),jt=s("span"),w(F.$$.fragment),Ne=i(),vt=s("span"),vt.textContent=tl,Kt=i(),D=s("p"),D.textContent=el,te=i(),w(z.$$.fragment),ee=i(),O=s("p"),O.textContent=ll,le=i(),w(q.$$.fragment),ne=i(),K=s("p"),K.textContent=nl,se=i(),w(tt.$$.fragment),ae=i(),et=s("p"),et.textContent=sl,ie=i(),w(lt.$$.fragment),oe=i(),nt=s("p"),nt.innerHTML=al,re=i(),st=s("p"),st.textContent=il,pe=i(),at=s("p"),at.textContent=ol,he=i(),J=s("h4"),P=s("a"),Pt=s("span"),w(it.$$.fragment),Be=i(),Tt=s("span"),Tt.textContent=rl,ce=i(),ot=s("p"),ot.innerHTML=pl,we=i(),rt=s("p"),rt.innerHTML=hl,Me=i(),pt=s("blockquote"),pt.innerHTML=cl,de=i(),ht=s("p"),ht.innerHTML=wl,ye=i(),ct=s("p"),ct.textContent=Ml,ue=i(),wt=s("p"),wt.textContent=dl,me=i(),Mt=s("table"),Mt.innerHTML=yl,Ce=i(),dt=s("p"),dt.innerHTML=ul,Ue=i(),yt=s("p"),yt.textContent=ml,fe=i(),w(ut.$$.fragment),ve=i(),mt=s("p"),mt.innerHTML=Cl,Te=i(),kt=s("p"),this.h()},l(t){const l=El("svelte-1phssyn",document.head);C=a(l,"META",{name:!0,content:!0}),l.forEach(e),Nt=o(t),Jt=a(t,"P",{}),c(Jt).forEach(e),Bt=o(t),U=a(t,"H1",{class:!0});var Ie=c(U);I=a(Ie,"A",{id:!0,class:!0,href:!0});var Ul=c(I);It=a(Ul,"SPAN",{});var fl=c(It);M(k.$$.fragment,fl),fl.forEach(e),Ul.forEach(e),ke=o(Ie),Ct=a(Ie,"SPAN",{"data-svelte-h":!0}),r(Ct)!=="svelte-abpmmt"&&(Ct.textContent=Ee),Ie.forEach(e),At=o(t),g=a(t,"P",{"data-svelte-h":!0}),r(g)!=="svelte-iifysf"&&(g.innerHTML=Se),Et=o(t),_=a(t,"P",{"data-svelte-h":!0}),r(_)!=="svelte-1dshoiu"&&(_.textContent=Ve),St=o(t),f=a(t,"H2",{class:!0});var be=c(f);b=a(be,"A",{id:!0,class:!0,href:!0});var vl=c(b);bt=a(vl,"SPAN",{});var Tl=c(bt);M(N.$$.fragment,Tl),Tl.forEach(e),vl.forEach(e),ge=o(be),Ut=a(be,"SPAN",{"data-svelte-h":!0}),r(Ut)!=="svelte-1giv785"&&(Ut.textContent=Ge),be.forEach(e),Vt=Sl(t,`

When you train bigger models you have essentially three options:
`),B=a(t,"UL",{"data-svelte-h":!0}),r(B)!=="svelte-pz3vuj"&&(B.innerHTML=$e),Gt=o(t),A=a(t,"P",{"data-svelte-h":!0}),r(A)!=="svelte-z1tcso"&&(A.textContent=Le),$t=o(t),v=a(t,"H3",{class:!0});var xe=c(v);x=a(xe,"A",{id:!0,class:!0,href:!0});var Jl=c(x);xt=a(Jl,"SPAN",{});var Il=c(xt);M(E.$$.fragment,Il),Il.forEach(e),Jl.forEach(e),_e=o(xe),ft=a(xe,"SPAN",{"data-svelte-h":!0}),r(ft)!=="svelte-yfil20"&&(ft.textContent=Ze),xe.forEach(e),Lt=o(t),S=a(t,"P",{"data-svelte-h":!0}),r(S)!=="svelte-ppjikm"&&(S.textContent=Xe),Zt=o(t),V=a(t,"P",{"data-svelte-h":!0}),r(V)!=="svelte-11nr6dq"&&(V.innerHTML=Re),Xt=o(t),G=a(t,"P",{"data-svelte-h":!0}),r(G)!=="svelte-11nhv2k"&&(G.textContent=We),Rt=o(t),$=a(t,"P",{"data-svelte-h":!0}),r($)!=="svelte-150xmew"&&($.textContent=He),Wt=o(t),L=a(t,"P",{"data-svelte-h":!0}),r(L)!=="svelte-12gfejp"&&(L.textContent=Qe),Ht=o(t),Z=a(t,"P",{"data-svelte-h":!0}),r(Z)!=="svelte-uh6lop"&&(Z.textContent=Ye),Qt=o(t),X=a(t,"P",{"data-svelte-h":!0}),r(X)!=="svelte-s9j0io"&&(X.textContent=Fe),Yt=o(t),R=a(t,"P",{"data-svelte-h":!0}),r(R)!=="svelte-8g3p2u"&&(R.textContent=De),Ft=o(t),W=a(t,"P",{"data-svelte-h":!0}),r(W)!=="svelte-1gvtzcc"&&(W.innerHTML=ze),Dt=o(t),H=a(t,"P",{"data-svelte-h":!0}),r(H)!=="svelte-hba1wh"&&(H.textContent=Oe),zt=o(t),Q=a(t,"P",{"data-svelte-h":!0}),r(Q)!=="svelte-1ho6s7e"&&(Q.textContent=qe),Ot=o(t),Y=a(t,"P",{"data-svelte-h":!0}),r(Y)!=="svelte-1c43r32"&&(Y.textContent=Ke),qt=o(t),T=a(t,"H3",{class:!0});var je=c(T);j=a(je,"A",{id:!0,class:!0,href:!0});var bl=c(j);jt=a(bl,"SPAN",{});var xl=c(jt);M(F.$$.fragment,xl),xl.forEach(e),bl.forEach(e),Ne=o(je),vt=a(je,"SPAN",{"data-svelte-h":!0}),r(vt)!=="svelte-7i4hfm"&&(vt.textContent=tl),je.forEach(e),Kt=o(t),D=a(t,"P",{"data-svelte-h":!0}),r(D)!=="svelte-1b7hx6s"&&(D.textContent=el),te=o(t),M(z.$$.fragment,t),ee=o(t),O=a(t,"P",{"data-svelte-h":!0}),r(O)!=="svelte-1m7rqth"&&(O.textContent=ll),le=o(t),M(q.$$.fragment,t),ne=o(t),K=a(t,"P",{"data-svelte-h":!0}),r(K)!=="svelte-hggxkn"&&(K.textContent=nl),se=o(t),M(tt.$$.fragment,t),ae=o(t),et=a(t,"P",{"data-svelte-h":!0}),r(et)!=="svelte-869n3z"&&(et.textContent=sl),ie=o(t),M(lt.$$.fragment,t),oe=o(t),nt=a(t,"P",{"data-svelte-h":!0}),r(nt)!=="svelte-xobw9p"&&(nt.innerHTML=al),re=o(t),st=a(t,"P",{"data-svelte-h":!0}),r(st)!=="svelte-abx4jy"&&(st.textContent=il),pe=o(t),at=a(t,"P",{"data-svelte-h":!0}),r(at)!=="svelte-1hcsw7p"&&(at.textContent=ol),he=o(t),J=a(t,"H4",{class:!0});var Pe=c(J);P=a(Pe,"A",{id:!0,class:!0,href:!0});var jl=c(P);Pt=a(jl,"SPAN",{});var Pl=c(Pt);M(it.$$.fragment,Pl),Pl.forEach(e),jl.forEach(e),Be=o(Pe),Tt=a(Pe,"SPAN",{"data-svelte-h":!0}),r(Tt)!=="svelte-ar78ut"&&(Tt.textContent=rl),Pe.forEach(e),ce=o(t),ot=a(t,"P",{"data-svelte-h":!0}),r(ot)!=="svelte-1gqi5fx"&&(ot.innerHTML=pl),we=o(t),rt=a(t,"P",{"data-svelte-h":!0}),r(rt)!=="svelte-gh3n8e"&&(rt.innerHTML=hl),Me=o(t),pt=a(t,"BLOCKQUOTE",{"data-svelte-h":!0}),r(pt)!=="svelte-14khom1"&&(pt.innerHTML=cl),de=o(t),ht=a(t,"P",{"data-svelte-h":!0}),r(ht)!=="svelte-1naftur"&&(ht.innerHTML=wl),ye=o(t),ct=a(t,"P",{"data-svelte-h":!0}),r(ct)!=="svelte-smgvyt"&&(ct.textContent=Ml),ue=o(t),wt=a(t,"P",{"data-svelte-h":!0}),r(wt)!=="svelte-j6eczh"&&(wt.textContent=dl),me=o(t),Mt=a(t,"TABLE",{"data-svelte-h":!0}),r(Mt)!=="svelte-1hvmx1i"&&(Mt.innerHTML=yl),Ce=o(t),dt=a(t,"P",{"data-svelte-h":!0}),r(dt)!=="svelte-192o6bz"&&(dt.innerHTML=ul),Ue=o(t),yt=a(t,"P",{"data-svelte-h":!0}),r(yt)!=="svelte-16jny0z"&&(yt.textContent=ml),fe=o(t),M(ut.$$.fragment,t),ve=o(t),mt=a(t,"P",{"data-svelte-h":!0}),r(mt)!=="svelte-pittpg"&&(mt.innerHTML=Cl),Te=o(t),kt=a(t,"P",{}),c(kt).forEach(e),this.h()},h(){p(C,"name","hf:doc:metadata"),p(C,"content",JSON.stringify(Gl)),p(I,"id","custom-hardware-for-training"),p(I,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(I,"href","#custom-hardware-for-training"),p(U,"class","relative group"),p(b,"id","gpu"),p(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(b,"href","#gpu"),p(f,"class","relative group"),p(x,"id","power-and-cooling"),p(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(x,"href","#power-and-cooling"),p(v,"class","relative group"),p(j,"id","multigpu-connectivity"),p(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(j,"href","#multigpu-connectivity"),p(T,"class","relative group"),p(P,"id","nvlink"),p(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(P,"href","#nvlink"),p(J,"class","relative group")},m(t,l){h(document.head,C),n(t,Nt,l),n(t,Jt,l),n(t,Bt,l),n(t,U,l),h(U,I),h(I,It),d(k,It,null),h(U,ke),h(U,Ct),n(t,At,l),n(t,g,l),n(t,Et,l),n(t,_,l),n(t,St,l),n(t,f,l),h(f,b),h(b,bt),d(N,bt,null),h(f,ge),h(f,Ut),n(t,Vt,l),n(t,B,l),n(t,Gt,l),n(t,A,l),n(t,$t,l),n(t,v,l),h(v,x),h(x,xt),d(E,xt,null),h(v,_e),h(v,ft),n(t,Lt,l),n(t,S,l),n(t,Zt,l),n(t,V,l),n(t,Xt,l),n(t,G,l),n(t,Rt,l),n(t,$,l),n(t,Wt,l),n(t,L,l),n(t,Ht,l),n(t,Z,l),n(t,Qt,l),n(t,X,l),n(t,Yt,l),n(t,R,l),n(t,Ft,l),n(t,W,l),n(t,Dt,l),n(t,H,l),n(t,zt,l),n(t,Q,l),n(t,Ot,l),n(t,Y,l),n(t,qt,l),n(t,T,l),h(T,j),h(j,jt),d(F,jt,null),h(T,Ne),h(T,vt),n(t,Kt,l),n(t,D,l),n(t,te,l),d(z,t,l),n(t,ee,l),n(t,O,l),n(t,le,l),d(q,t,l),n(t,ne,l),n(t,K,l),n(t,se,l),d(tt,t,l),n(t,ae,l),n(t,et,l),n(t,ie,l),d(lt,t,l),n(t,oe,l),n(t,nt,l),n(t,re,l),n(t,st,l),n(t,pe,l),n(t,at,l),n(t,he,l),n(t,J,l),h(J,P),h(P,Pt),d(it,Pt,null),h(J,Be),h(J,Tt),n(t,ce,l),n(t,ot,l),n(t,we,l),n(t,rt,l),n(t,Me,l),n(t,pt,l),n(t,de,l),n(t,ht,l),n(t,ye,l),n(t,ct,l),n(t,ue,l),n(t,wt,l),n(t,me,l),n(t,Mt,l),n(t,Ce,l),n(t,dt,l),n(t,Ue,l),n(t,yt,l),n(t,fe,l),d(ut,t,l),n(t,ve,l),n(t,mt,l),n(t,Te,l),n(t,kt,l),Je=!0},p:gl,i(t){Je||(y(k.$$.fragment,t),y(N.$$.fragment,t),y(E.$$.fragment,t),y(F.$$.fragment,t),y(z.$$.fragment,t),y(q.$$.fragment,t),y(tt.$$.fragment,t),y(lt.$$.fragment,t),y(it.$$.fragment,t),y(ut.$$.fragment,t),Je=!0)},o(t){u(k.$$.fragment,t),u(N.$$.fragment,t),u(E.$$.fragment,t),u(F.$$.fragment,t),u(z.$$.fragment,t),u(q.$$.fragment,t),u(tt.$$.fragment,t),u(lt.$$.fragment,t),u(it.$$.fragment,t),u(ut.$$.fragment,t),Je=!1},d(t){t&&(e(Nt),e(Jt),e(Bt),e(U),e(At),e(g),e(Et),e(_),e(St),e(f),e(Vt),e(B),e(Gt),e(A),e($t),e(v),e(Lt),e(S),e(Zt),e(V),e(Xt),e(G),e(Rt),e($),e(Wt),e(L),e(Ht),e(Z),e(Qt),e(X),e(Yt),e(R),e(Ft),e(W),e(Dt),e(H),e(zt),e(Q),e(Ot),e(Y),e(qt),e(T),e(Kt),e(D),e(te),e(ee),e(O),e(le),e(ne),e(K),e(se),e(ae),e(et),e(ie),e(oe),e(nt),e(re),e(st),e(pe),e(at),e(he),e(J),e(ce),e(ot),e(we),e(rt),e(Me),e(pt),e(de),e(ht),e(ye),e(ct),e(ue),e(wt),e(me),e(Mt),e(Ce),e(dt),e(Ue),e(yt),e(fe),e(ve),e(mt),e(Te),e(kt)),e(C),m(k),m(N),m(E),m(F),m(z,t),m(q,t),m(tt,t),m(lt,t),m(it),m(ut,t)}}}const Gl={local:"custom-hardware-for-training",sections:[{local:"gpu",sections:[{local:"power-and-cooling",title:"Power and Cooling"},{local:"multigpu-connectivity",sections:[{local:"nvlink",title:"NVlink"}],title:"Multi-GPU Connectivity"}],title:"GPU"}],title:"Custom hardware for training"};function $l(Ae){return _l(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wl extends Nl{constructor(C){super(),Bl(this,C,$l,Vl,kl,{})}}export{Wl as component};
