import{s as Lt,n as Zt,o as Xt}from"../chunks/scheduler.9bc65507.js";import{S as Rt,i as Qt,g as i,s,r,A as Wt,h as o,f as l,c as a,j as St,u as M,x as p,k as Et,y as Ht,a as n,v as w,d as y,t as h,w as c}from"../chunks/index.707bf1b6.js";import{C as pe}from"../chunks/CodeBlock.54a9f38d.js";import{H as re}from"../chunks/Heading.342b1fa6.js";function Yt(nt){let m,Me,ie,we,d,ye,U,st='The hardware you use to run model training and inference can have a big effect on performance. For a deep dive into GPUs make sure to check out Tim Dettmer’s excellent <a href="https://timdettmers.com/2020/09/07/which-gpu-for-deep-learning/" rel="nofollow">blog post</a>.',he,u,at="Let’s have a look at some practical advice for GPU setups.",ce,C,me,T,it="When you train bigger models you have essentially three options:",de,f,ot='<li>bigger GPUs</li> <li>more GPUs</li> <li>more CPU and NVMe (offloaded to by <a href="main_classes/deepspeed#nvme-support">DeepSpeed-Infinity</a>)</li>',Ue,J,pt="Let’s start at the case where you have a single GPU.",ue,I,Ce,v,rt="If you bought an expensive high end GPU make sure you give it the correct power and sufficient cooling.",Te,b,Mt="<strong>Power</strong>:",fe,j,wt="Some high end consumer GPU cards have 2 and sometimes 3 PCI-E 8-Pin power sockets. Make sure you have as many independent 12V PCI-E 8-Pin cables plugged into the card as there are sockets. Do not use the 2 splits at one end of the same cable (also known as pigtail cable). That is if you have 2 sockets on the GPU, you want 2 PCI-E 8-Pin cables going from your PSU to the card and not one that has 2 PCI-E 8-Pin connectors at the end! You won’t get the full performance out of your card otherwise.",Je,x,yt="Each PCI-E 8-Pin power cable needs to be plugged into a 12V rail on the PSU side and can supply up to 150W of power.",Ie,P,ht="Some other cards may use a PCI-E 12-Pin connectors, and these can deliver up to 500-600W of power.",ve,k,ct="Low end cards may use 6-Pin connectors, which supply up to 75W of power.",be,B,mt="Additionally you want the high-end PSU that has stable voltage. Some lower quality ones may not give the card the stable voltage it needs to function at its peak.",je,g,dt="And of course the PSU needs to have enough unused Watts to power the card.",xe,_,Ut="<strong>Cooling</strong>:",Pe,N,ut="When a GPU gets overheated it will start throttling down and will not deliver full performance and it can even shutdown if it gets too hot.",ke,V,Ct="It’s hard to tell the exact best temperature to strive for when a GPU is heavily loaded, but probably anything under +80C is good, but lower is better - perhaps 70-75C is an excellent range to be in. The throttling down is likely to start at around 84-90C. But other than throttling performance a prolonged very high temperature is likely to reduce the lifespan of a GPU.",Be,G,Tt="Next let’s have a look at one of the most important aspects when having multiple GPUs: connectivity.",ge,A,_e,$,ft="If you use multiple GPUs the way cards are inter-connected can have a huge impact on the total training time. If the GPUs are on the same physical node, you can run:",Ne,S,Ve,E,Jt="and it will tell you how the GPUs are inter-connected. On a machine with dual-GPU and which are connected with NVLink, you will most likely see something like:",Ge,L,Ae,Z,It="on a different machine w/o NVLink we may see:",$e,X,Se,R,vt="The report includes this legend:",Ee,Q,Le,W,bt="So the first report <code>NV2</code> tells us the GPUs are interconnected with 2 NVLinks, and the second report <code>PHB</code> we have a typical consumer-level PCIe+Bridge setup.",Ze,H,jt="Check what type of connectivity you have on your setup. Some of these will make the communication between cards faster (e.g. NVLink), others slower (e.g. PHB).",Xe,Y,xt="Depending on the type of scalability solution used, the connectivity speed could have a major or a minor impact. If the GPUs need to sync rarely, as in DDP, the impact of a slower connection will be less significant. If the GPUs need to send messages to each other often, as in ZeRO-DP, then faster connectivity becomes super important to achieve faster training.",Re,F,Qe,z,Pt='<a href="https://en.wikipedia.org/wiki/NVLink" rel="nofollow">NVLink</a> is a wire-based serial multi-lane near-range communications link developed by Nvidia.',We,D,kt='Each new generation provides a faster bandwidth, e.g. here is a quote from <a href="https://www.nvidia.com/content/dam/en-zz/Solutions/geforce/ampere/pdf/NVIDIA-ampere-GA102-GPU-Architecture-Whitepaper-V1.pdf" rel="nofollow">Nvidia Ampere GA102 GPU Architecture</a>:',He,O,Bt=`<p>Third-Generation NVLink®
GA102 GPUs utilize NVIDIA’s third-generation NVLink interface, which includes four x4 links,
with each link providing 14.0625 GB/sec bandwidth in each direction between two GPUs. Four
links provide 56.25 GB/sec bandwidth in each direction, and 112.5 GB/sec total bandwidth
between two GPUs. Two RTX 3090 GPUs can be connected together for SLI using NVLink.
(Note that 3-Way and 4-Way SLI configurations are not supported.)</p>`,Ye,q,gt="So the higher <code>X</code> you get in the report of <code>NVX</code> in the output of <code>nvidia-smi topo -m</code> the better. The generation will depend on your GPU architecture.",Fe,K,_t="Let’s compare the execution of a gpt2 language model training over a small sample of wikitext.",ze,ee,Nt="The results are:",De,te,Vt='<thead><tr><th>NVlink</th> <th align="right">Time</th></tr></thead> <tbody><tr><td>Y</td> <td align="right">101s</td></tr> <tr><td>N</td> <td align="right">131s</td></tr></tbody>',Oe,le,Gt="You can see that NVLink completes the training ~23% faster. In the second benchmark we use <code>NCCL_P2P_DISABLE=1</code> to tell the GPUs not to use NVLink.",qe,ne,At="Here is the full benchmark code and outputs:",Ke,se,et,ae,$t=`Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks (<code>NV2</code> in <code>nvidia-smi topo -m</code>)
Software: <code>pytorch-1.8-to-be</code> + <code>cuda-11.0</code> / <code>transformers==4.3.0.dev0</code>`,tt,oe,lt;return d=new re({props:{title:"Custom hardware for training",local:"custom-hardware-for-training",headingTag:"h1"}}),C=new re({props:{title:"GPU",local:"gpu",headingTag:"h2"}}),I=new re({props:{title:"Power and Cooling",local:"power-and-cooling",headingTag:"h3"}}),A=new re({props:{title:"Multi-GPU Connectivity",local:"multi-gpu-connectivity",headingTag:"h3"}}),S=new pe({props:{code:"bnZpZGlhLXNtaSUyMHRvcG8lMjAtbQ==",highlighted:'<span class="hljs-symbol">nvidia</span>-<span class="hljs-keyword">smi</span> topo -m',wrap:!1}}),L=new pe({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwTlYyJTIwJTIwJTIwJTIwJTIwMC0yMyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyME5WMiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTIzJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      NV2     <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A
<span class="hljs-attribute">GPU1</span>    NV2      X      <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A`,wrap:!1}}),X=new pe({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwUEhCJTIwJTIwJTIwJTIwJTIwMC0xMSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyMFBIQiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTExJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      PHB     <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A
<span class="hljs-attribute">GPU1</span>    PHB      X      <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A`,wrap:!1}}),Q=new pe({props:{code:"JTIwJTIwWCUyMCUyMCUyMCUyMCUzRCUyMFNlbGYlMEElMjAlMjBTWVMlMjAlMjAlM0QlMjBDb25uZWN0aW9uJTIwdHJhdmVyc2luZyUyMFBDSWUlMjBhcyUyMHdlbGwlMjBhcyUyMHRoZSUyMFNNUCUyMGludGVyY29ubmVjdCUyMGJldHdlZW4lMjBOVU1BJTIwbm9kZXMlMjAoZS5nLiUyQyUyMFFQSSUyRlVQSSklMEElMjAlMjBOT0RFJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjB0aGUlMjBpbnRlcmNvbm5lY3QlMjBiZXR3ZWVuJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2VzJTIwd2l0aGluJTIwYSUyME5VTUElMjBub2RlJTBBJTIwJTIwUEhCJTIwJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjBhJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2UlMjAodHlwaWNhbGx5JTIwdGhlJTIwQ1BVKSUwQSUyMCUyMFBYQiUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwbXVsdGlwbGUlMjBQQ0llJTIwYnJpZGdlcyUyMCh3aXRob3V0JTIwdHJhdmVyc2luZyUyMHRoZSUyMFBDSWUlMjBIb3N0JTIwQnJpZGdlKSUwQSUyMCUyMFBJWCUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYXQlMjBtb3N0JTIwYSUyMHNpbmdsZSUyMFBDSWUlMjBicmlkZ2UlMEElMjAlMjBOViUyMyUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYSUyMGJvbmRlZCUyMHNldCUyMG9mJTIwJTIzJTIwTlZMaW5rcw==",highlighted:`  X    = Self
  SYS  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> interconnect between PCIe Host Bridges <span class="hljs-keyword">within</span> <span class="hljs-keyword">a</span> NUMA node
  PHB  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">a</span> PCIe Host Bridge (typically <span class="hljs-keyword">the</span> CPU)
  PXB  = Connection traversing multiple PCIe bridges (<span class="hljs-keyword">without</span> traversing <span class="hljs-keyword">the</span> PCIe Host Bridge)
  PIX  = Connection traversing <span class="hljs-keyword">at</span> most <span class="hljs-keyword">a</span> single PCIe bridge
  NV<span class="hljs-comment">#  = Connection traversing a bonded set of # NVLinks</span>`,wrap:!1}}),F=new re({props:{title:"NVlink",local:"nvlink",headingTag:"h4"}}),se=new pe({props:{code:"JTIzJTIwRERQJTIwdyUyRiUyME5WTGluayUwQSUwQXJtJTIwLXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUzQiUyMENVREFfVklTSUJMRV9ERVZJQ0VTJTNEMCUyQzElMjB0b3JjaHJ1biUyMCU1QyUwQS0tbnByb2NfcGVyX25vZGUlMjAyJTIwZXhhbXBsZXMlMkZweXRvcmNoJTJGbGFuZ3VhZ2UtbW9kZWxpbmclMkZydW5fY2xtLnB5JTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBncHQyJTIwJTVDJTBBLS1kYXRhc2V0X25hbWUlMjB3aWtpdGV4dCUyMC0tZGF0YXNldF9jb25maWdfbmFtZSUyMHdpa2l0ZXh0LTItcmF3LXYxJTIwLS1kb190cmFpbiUyMCU1QyUwQS0tb3V0cHV0X2RpciUyMCUyRnRtcCUyRnRlc3QtY2xtJTIwLS1wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjA0JTIwLS1tYXhfc3RlcHMlMjAyMDAlMEElMEElN0IndHJhaW5fcnVudGltZSclM0ElMjAxMDEuOTAwMyUyQyUyMCd0cmFpbl9zYW1wbGVzX3Blcl9zZWNvbmQnJTNBJTIwMS45NjMlMkMlMjAnZXBvY2gnJTNBJTIwMC42OSU3RCUwQSUwQSUyMyUyMEREUCUyMHclMkZvJTIwTlZMaW5rJTBBJTBBcm0lMjAtciUyMCUyRnRtcCUyRnRlc3QtY2xtJTNCJTIwQ1VEQV9WSVNJQkxFX0RFVklDRVMlM0QwJTJDMSUyME5DQ0xfUDJQX0RJU0FCTEUlM0QxJTIwdG9yY2hydW4lMjAlNUMlMEEtLW5wcm9jX3Blcl9ub2RlJTIwMiUyMGV4YW1wbGVzJTJGcHl0b3JjaCUyRmxhbmd1YWdlLW1vZGVsaW5nJTJGcnVuX2NsbS5weSUyMC0tbW9kZWxfbmFtZV9vcl9wYXRoJTIwZ3B0MiUyMCU1QyUwQS0tZGF0YXNldF9uYW1lJTIwd2lraXRleHQlMjAtLWRhdGFzZXRfY29uZmlnX25hbWUlMjB3aWtpdGV4dC0yLXJhdy12MSUyMC0tZG9fdHJhaW4lMEEtLW91dHB1dF9kaXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwNCUyMC0tbWF4X3N0ZXBzJTIwMjAwJTBBJTBBJTdCJ3RyYWluX3J1bnRpbWUnJTNBJTIwMTMxLjQzNjclMkMlMjAndHJhaW5fc2FtcGxlc19wZXJfc2Vjb25kJyUzQSUyMDEuNTIyJTJDJTIwJ2Vwb2NoJyUzQSUyMDAuNjklN0Q=",highlighted:`<span class="hljs-comment"># DDP w/ NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 torchrun \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train \\
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 101.9003, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.963, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}

<span class="hljs-comment"># DDP w/o NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 NCCL_P2P_DISABLE=1 torchrun \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 131.4367, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.522, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}`,wrap:!1}}),{c(){m=i("meta"),Me=s(),ie=i("p"),we=s(),r(d.$$.fragment),ye=s(),U=i("p"),U.innerHTML=st,he=s(),u=i("p"),u.textContent=at,ce=s(),r(C.$$.fragment),me=s(),T=i("p"),T.textContent=it,de=s(),f=i("ul"),f.innerHTML=ot,Ue=s(),J=i("p"),J.textContent=pt,ue=s(),r(I.$$.fragment),Ce=s(),v=i("p"),v.textContent=rt,Te=s(),b=i("p"),b.innerHTML=Mt,fe=s(),j=i("p"),j.textContent=wt,Je=s(),x=i("p"),x.textContent=yt,Ie=s(),P=i("p"),P.textContent=ht,ve=s(),k=i("p"),k.textContent=ct,be=s(),B=i("p"),B.textContent=mt,je=s(),g=i("p"),g.textContent=dt,xe=s(),_=i("p"),_.innerHTML=Ut,Pe=s(),N=i("p"),N.textContent=ut,ke=s(),V=i("p"),V.textContent=Ct,Be=s(),G=i("p"),G.textContent=Tt,ge=s(),r(A.$$.fragment),_e=s(),$=i("p"),$.textContent=ft,Ne=s(),r(S.$$.fragment),Ve=s(),E=i("p"),E.textContent=Jt,Ge=s(),r(L.$$.fragment),Ae=s(),Z=i("p"),Z.textContent=It,$e=s(),r(X.$$.fragment),Se=s(),R=i("p"),R.textContent=vt,Ee=s(),r(Q.$$.fragment),Le=s(),W=i("p"),W.innerHTML=bt,Ze=s(),H=i("p"),H.textContent=jt,Xe=s(),Y=i("p"),Y.textContent=xt,Re=s(),r(F.$$.fragment),Qe=s(),z=i("p"),z.innerHTML=Pt,We=s(),D=i("p"),D.innerHTML=kt,He=s(),O=i("blockquote"),O.innerHTML=Bt,Ye=s(),q=i("p"),q.innerHTML=gt,Fe=s(),K=i("p"),K.textContent=_t,ze=s(),ee=i("p"),ee.textContent=Nt,De=s(),te=i("table"),te.innerHTML=Vt,Oe=s(),le=i("p"),le.innerHTML=Gt,qe=s(),ne=i("p"),ne.textContent=At,Ke=s(),r(se.$$.fragment),et=s(),ae=i("p"),ae.innerHTML=$t,tt=s(),oe=i("p"),this.h()},l(e){const t=Wt("svelte-u9bgzb",document.head);m=o(t,"META",{name:!0,content:!0}),t.forEach(l),Me=a(e),ie=o(e,"P",{}),St(ie).forEach(l),we=a(e),M(d.$$.fragment,e),ye=a(e),U=o(e,"P",{"data-svelte-h":!0}),p(U)!=="svelte-iifysf"&&(U.innerHTML=st),he=a(e),u=o(e,"P",{"data-svelte-h":!0}),p(u)!=="svelte-1dshoiu"&&(u.textContent=at),ce=a(e),M(C.$$.fragment,e),me=a(e),T=o(e,"P",{"data-svelte-h":!0}),p(T)!=="svelte-eleqzp"&&(T.textContent=it),de=a(e),f=o(e,"UL",{"data-svelte-h":!0}),p(f)!=="svelte-pz3vuj"&&(f.innerHTML=ot),Ue=a(e),J=o(e,"P",{"data-svelte-h":!0}),p(J)!=="svelte-z1tcso"&&(J.textContent=pt),ue=a(e),M(I.$$.fragment,e),Ce=a(e),v=o(e,"P",{"data-svelte-h":!0}),p(v)!=="svelte-ppjikm"&&(v.textContent=rt),Te=a(e),b=o(e,"P",{"data-svelte-h":!0}),p(b)!=="svelte-11nr6dq"&&(b.innerHTML=Mt),fe=a(e),j=o(e,"P",{"data-svelte-h":!0}),p(j)!=="svelte-11nhv2k"&&(j.textContent=wt),Je=a(e),x=o(e,"P",{"data-svelte-h":!0}),p(x)!=="svelte-150xmew"&&(x.textContent=yt),Ie=a(e),P=o(e,"P",{"data-svelte-h":!0}),p(P)!=="svelte-12gfejp"&&(P.textContent=ht),ve=a(e),k=o(e,"P",{"data-svelte-h":!0}),p(k)!=="svelte-uh6lop"&&(k.textContent=ct),be=a(e),B=o(e,"P",{"data-svelte-h":!0}),p(B)!=="svelte-s9j0io"&&(B.textContent=mt),je=a(e),g=o(e,"P",{"data-svelte-h":!0}),p(g)!=="svelte-8g3p2u"&&(g.textContent=dt),xe=a(e),_=o(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-1gvtzcc"&&(_.innerHTML=Ut),Pe=a(e),N=o(e,"P",{"data-svelte-h":!0}),p(N)!=="svelte-hba1wh"&&(N.textContent=ut),ke=a(e),V=o(e,"P",{"data-svelte-h":!0}),p(V)!=="svelte-1ho6s7e"&&(V.textContent=Ct),Be=a(e),G=o(e,"P",{"data-svelte-h":!0}),p(G)!=="svelte-1c43r32"&&(G.textContent=Tt),ge=a(e),M(A.$$.fragment,e),_e=a(e),$=o(e,"P",{"data-svelte-h":!0}),p($)!=="svelte-1b7hx6s"&&($.textContent=ft),Ne=a(e),M(S.$$.fragment,e),Ve=a(e),E=o(e,"P",{"data-svelte-h":!0}),p(E)!=="svelte-1m7rqth"&&(E.textContent=Jt),Ge=a(e),M(L.$$.fragment,e),Ae=a(e),Z=o(e,"P",{"data-svelte-h":!0}),p(Z)!=="svelte-hggxkn"&&(Z.textContent=It),$e=a(e),M(X.$$.fragment,e),Se=a(e),R=o(e,"P",{"data-svelte-h":!0}),p(R)!=="svelte-869n3z"&&(R.textContent=vt),Ee=a(e),M(Q.$$.fragment,e),Le=a(e),W=o(e,"P",{"data-svelte-h":!0}),p(W)!=="svelte-xobw9p"&&(W.innerHTML=bt),Ze=a(e),H=o(e,"P",{"data-svelte-h":!0}),p(H)!=="svelte-abx4jy"&&(H.textContent=jt),Xe=a(e),Y=o(e,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-1hcsw7p"&&(Y.textContent=xt),Re=a(e),M(F.$$.fragment,e),Qe=a(e),z=o(e,"P",{"data-svelte-h":!0}),p(z)!=="svelte-1gqi5fx"&&(z.innerHTML=Pt),We=a(e),D=o(e,"P",{"data-svelte-h":!0}),p(D)!=="svelte-gh3n8e"&&(D.innerHTML=kt),He=a(e),O=o(e,"BLOCKQUOTE",{"data-svelte-h":!0}),p(O)!=="svelte-14khom1"&&(O.innerHTML=Bt),Ye=a(e),q=o(e,"P",{"data-svelte-h":!0}),p(q)!=="svelte-1naftur"&&(q.innerHTML=gt),Fe=a(e),K=o(e,"P",{"data-svelte-h":!0}),p(K)!=="svelte-smgvyt"&&(K.textContent=_t),ze=a(e),ee=o(e,"P",{"data-svelte-h":!0}),p(ee)!=="svelte-j6eczh"&&(ee.textContent=Nt),De=a(e),te=o(e,"TABLE",{"data-svelte-h":!0}),p(te)!=="svelte-1hvmx1i"&&(te.innerHTML=Vt),Oe=a(e),le=o(e,"P",{"data-svelte-h":!0}),p(le)!=="svelte-192o6bz"&&(le.innerHTML=Gt),qe=a(e),ne=o(e,"P",{"data-svelte-h":!0}),p(ne)!=="svelte-16jny0z"&&(ne.textContent=At),Ke=a(e),M(se.$$.fragment,e),et=a(e),ae=o(e,"P",{"data-svelte-h":!0}),p(ae)!=="svelte-pittpg"&&(ae.innerHTML=$t),tt=a(e),oe=o(e,"P",{}),St(oe).forEach(l),this.h()},h(){Et(m,"name","hf:doc:metadata"),Et(m,"content",Ft)},m(e,t){Ht(document.head,m),n(e,Me,t),n(e,ie,t),n(e,we,t),w(d,e,t),n(e,ye,t),n(e,U,t),n(e,he,t),n(e,u,t),n(e,ce,t),w(C,e,t),n(e,me,t),n(e,T,t),n(e,de,t),n(e,f,t),n(e,Ue,t),n(e,J,t),n(e,ue,t),w(I,e,t),n(e,Ce,t),n(e,v,t),n(e,Te,t),n(e,b,t),n(e,fe,t),n(e,j,t),n(e,Je,t),n(e,x,t),n(e,Ie,t),n(e,P,t),n(e,ve,t),n(e,k,t),n(e,be,t),n(e,B,t),n(e,je,t),n(e,g,t),n(e,xe,t),n(e,_,t),n(e,Pe,t),n(e,N,t),n(e,ke,t),n(e,V,t),n(e,Be,t),n(e,G,t),n(e,ge,t),w(A,e,t),n(e,_e,t),n(e,$,t),n(e,Ne,t),w(S,e,t),n(e,Ve,t),n(e,E,t),n(e,Ge,t),w(L,e,t),n(e,Ae,t),n(e,Z,t),n(e,$e,t),w(X,e,t),n(e,Se,t),n(e,R,t),n(e,Ee,t),w(Q,e,t),n(e,Le,t),n(e,W,t),n(e,Ze,t),n(e,H,t),n(e,Xe,t),n(e,Y,t),n(e,Re,t),w(F,e,t),n(e,Qe,t),n(e,z,t),n(e,We,t),n(e,D,t),n(e,He,t),n(e,O,t),n(e,Ye,t),n(e,q,t),n(e,Fe,t),n(e,K,t),n(e,ze,t),n(e,ee,t),n(e,De,t),n(e,te,t),n(e,Oe,t),n(e,le,t),n(e,qe,t),n(e,ne,t),n(e,Ke,t),w(se,e,t),n(e,et,t),n(e,ae,t),n(e,tt,t),n(e,oe,t),lt=!0},p:Zt,i(e){lt||(y(d.$$.fragment,e),y(C.$$.fragment,e),y(I.$$.fragment,e),y(A.$$.fragment,e),y(S.$$.fragment,e),y(L.$$.fragment,e),y(X.$$.fragment,e),y(Q.$$.fragment,e),y(F.$$.fragment,e),y(se.$$.fragment,e),lt=!0)},o(e){h(d.$$.fragment,e),h(C.$$.fragment,e),h(I.$$.fragment,e),h(A.$$.fragment,e),h(S.$$.fragment,e),h(L.$$.fragment,e),h(X.$$.fragment,e),h(Q.$$.fragment,e),h(F.$$.fragment,e),h(se.$$.fragment,e),lt=!1},d(e){e&&(l(Me),l(ie),l(we),l(ye),l(U),l(he),l(u),l(ce),l(me),l(T),l(de),l(f),l(Ue),l(J),l(ue),l(Ce),l(v),l(Te),l(b),l(fe),l(j),l(Je),l(x),l(Ie),l(P),l(ve),l(k),l(be),l(B),l(je),l(g),l(xe),l(_),l(Pe),l(N),l(ke),l(V),l(Be),l(G),l(ge),l(_e),l($),l(Ne),l(Ve),l(E),l(Ge),l(Ae),l(Z),l($e),l(Se),l(R),l(Ee),l(Le),l(W),l(Ze),l(H),l(Xe),l(Y),l(Re),l(Qe),l(z),l(We),l(D),l(He),l(O),l(Ye),l(q),l(Fe),l(K),l(ze),l(ee),l(De),l(te),l(Oe),l(le),l(qe),l(ne),l(Ke),l(et),l(ae),l(tt),l(oe)),l(m),c(d,e),c(C,e),c(I,e),c(A,e),c(S,e),c(L,e),c(X,e),c(Q,e),c(F,e),c(se,e)}}}const Ft='{"title":"Custom hardware for training","local":"custom-hardware-for-training","sections":[{"title":"GPU","local":"gpu","sections":[{"title":"Power and Cooling","local":"power-and-cooling","sections":[],"depth":3},{"title":"Multi-GPU Connectivity","local":"multi-gpu-connectivity","sections":[{"title":"NVlink","local":"nvlink","sections":[],"depth":4}],"depth":3}],"depth":2}],"depth":1}';function zt(nt){return Xt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class el extends Rt{constructor(m){super(),Qt(this,m,zt,Yt,Lt,{})}}export{el as component};
