import{s as $t,n as Lt,o as Xt}from"../chunks/scheduler.9bc65507.js";import{S as Ht,i as Rt,g as a,s,r,A as Wt,h as o,f as l,c as i,j as St,u as M,x as p,k as Zt,y as Qt,a as n,v as w,d as y,t as h,w as m}from"../chunks/index.707bf1b6.js";import{C as pe}from"../chunks/CodeBlock.54a9f38d.js";import{H as re}from"../chunks/Heading.342b1fa6.js";function Yt(nt){let c,Me,ae,we,d,ye,u,st='The hardware you use to run model training and inference can have a big effect on performance. For a deep dive into GPUs make sure to check out Tim Dettmer’s excellent <a href="https://timdettmers.com/2020/09/07/which-gpu-for-deep-learning/" rel="nofollow">blog post</a>.',he,U,it="Let’s have a look at some practical advice for GPU setups.",me,C,ce,f,at="When you train bigger models you have essentially three options:",de,T,ot='<li>bigger GPUs</li> <li>more GPUs</li> <li>more CPU and NVMe (offloaded to by <a href="main_classes/deepspeed#nvme-support">DeepSpeed-Infinity</a>)</li>',ue,J,pt="Let’s start at the case where you have a single GPU.",Ue,I,Ce,v,rt="If you bought an expensive high end GPU make sure you give it the correct power and sufficient cooling.",fe,b,Mt="<strong>Power</strong>:",Te,j,wt="Some high end consumer GPU cards have 2 and sometimes 3 PCI-E 8-Pin power sockets. Make sure you have as many independent 12V PCI-E 8-Pin cables plugged into the card as there are sockets. Do not use the 2 splits at one end of the same cable (also known as pigtail cable). That is if you have 2 sockets on the GPU, you want 2 PCI-E 8-Pin cables going from your PSU to the card and not one that has 2 PCI-E 8-Pin connectors at the end! You won’t get the full performance out of your card otherwise.",Je,x,yt="Each PCI-E 8-Pin power cable needs to be plugged into a 12V rail on the PSU side and can supply up to 150W of power.",Ie,P,ht="Some other cards may use a PCI-E 12-Pin connectors, and these can deliver up to 500-600W of power.",ve,k,mt="Low end cards may use 6-Pin connectors, which supply up to 75W of power.",be,B,ct="Additionally you want the high-end PSU that has stable voltage. Some lower quality ones may not give the card the stable voltage it needs to function at its peak.",je,g,dt="And of course the PSU needs to have enough unused Watts to power the card.",xe,_,ut="<strong>Cooling</strong>:",Pe,N,Ut="When a GPU gets overheated it will start throttling down and will not deliver full performance and it can even shutdown if it gets too hot.",ke,G,Ct="It’s hard to tell the exact best temperature to strive for when a GPU is heavily loaded, but probably anything under +80C is good, but lower is better - perhaps 70-75C is an excellent range to be in. The throttling down is likely to start at around 84-90C. But other than throttling performance a prolonged very high temperature is likely to reduce the lifespan of a GPU.",Be,V,ft="Next let’s have a look at one of the most important aspects when having multiple GPUs: connectivity.",ge,A,_e,E,Tt="If you use multiple GPUs the way cards are inter-connected can have a huge impact on the total training time. If the GPUs are on the same physical node, you can run:",Ne,S,Ge,Z,Jt="and it will tell you how the GPUs are inter-connected. On a machine with dual-GPU and which are connected with NVLink, you will most likely see something like:",Ve,$,Ae,L,It="on a different machine w/o NVLink we may see:",Ee,X,Se,H,vt="The report includes this legend:",Ze,R,$e,W,bt="So the first report <code>NV2</code> tells us the GPUs are interconnected with 2 NVLinks, and the second report <code>PHB</code> we have a typical consumer-level PCIe+Bridge setup.",Le,Q,jt="Check what type of connectivity you have on your setup. Some of these will make the communication between cards faster (e.g. NVLink), others slower (e.g. PHB).",Xe,Y,xt="Depending on the type of scalability solution used, the connectivity speed could have a major or a minor impact. If the GPUs need to sync rarely, as in DDP, the impact of a slower connection will be less significant. If the GPUs need to send messages to each other often, as in ZeRO-DP, then faster connectivity becomes super important to achieve faster training.",He,F,Re,z,Pt='<a href="https://en.wikipedia.org/wiki/NVLink" rel="nofollow">NVLink</a> is a wire-based serial multi-lane near-range communications link developed by Nvidia.',We,D,kt='Each new generation provides a faster bandwidth, e.g. here is a quote from <a href="https://www.nvidia.com/content/dam/en-zz/Solutions/geforce/ampere/pdf/NVIDIA-ampere-GA102-GPU-Architecture-Whitepaper-V1.pdf" rel="nofollow">Nvidia Ampere GA102 GPU Architecture</a>:',Qe,O,Bt=`<p>Third-Generation NVLink®
GA102 GPUs utilize NVIDIA’s third-generation NVLink interface, which includes four x4 links,
with each link providing 14.0625 GB/sec bandwidth in each direction between two GPUs. Four
links provide 56.25 GB/sec bandwidth in each direction, and 112.5 GB/sec total bandwidth
between two GPUs. Two RTX 3090 GPUs can be connected together for SLI using NVLink.
(Note that 3-Way and 4-Way SLI configurations are not supported.)</p>`,Ye,q,gt="So the higher <code>X</code> you get in the report of <code>NVX</code> in the output of <code>nvidia-smi topo -m</code> the better. The generation will depend on your GPU architecture.",Fe,K,_t="Let’s compare the execution of a openai-community/gpt2 language model training over a small sample of wikitext.",ze,ee,Nt="The results are:",De,te,Gt='<thead><tr><th>NVlink</th> <th align="right">Time</th></tr></thead> <tbody><tr><td>Y</td> <td align="right">101s</td></tr> <tr><td>N</td> <td align="right">131s</td></tr></tbody>',Oe,le,Vt="You can see that NVLink completes the training ~23% faster. In the second benchmark we use <code>NCCL_P2P_DISABLE=1</code> to tell the GPUs not to use NVLink.",qe,ne,At="Here is the full benchmark code and outputs:",Ke,se,et,ie,Et=`Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks (<code>NV2</code> in <code>nvidia-smi topo -m</code>)
Software: <code>pytorch-1.8-to-be</code> + <code>cuda-11.0</code> / <code>transformers==4.3.0.dev0</code>`,tt,oe,lt;return d=new re({props:{title:"Custom hardware for training",local:"custom-hardware-for-training",headingTag:"h1"}}),C=new re({props:{title:"GPU",local:"gpu",headingTag:"h2"}}),I=new re({props:{title:"Power and Cooling",local:"power-and-cooling",headingTag:"h3"}}),A=new re({props:{title:"Multi-GPU Connectivity",local:"multi-gpu-connectivity",headingTag:"h3"}}),S=new pe({props:{code:"bnZpZGlhLXNtaSUyMHRvcG8lMjAtbQ==",highlighted:"nvidia-smi topo -m",wrap:!1}}),$=new pe({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwTlYyJTIwJTIwJTIwJTIwJTIwMC0yMyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyME5WMiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTIzJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      NV2     <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A
<span class="hljs-attribute">GPU1</span>    NV2      X      <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A`,wrap:!1}}),X=new pe({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwUEhCJTIwJTIwJTIwJTIwJTIwMC0xMSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyMFBIQiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTExJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      PHB     <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A
<span class="hljs-attribute">GPU1</span>    PHB      X      <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A`,wrap:!1}}),R=new pe({props:{code:"JTIwJTIwWCUyMCUyMCUyMCUyMCUzRCUyMFNlbGYlMEElMjAlMjBTWVMlMjAlMjAlM0QlMjBDb25uZWN0aW9uJTIwdHJhdmVyc2luZyUyMFBDSWUlMjBhcyUyMHdlbGwlMjBhcyUyMHRoZSUyMFNNUCUyMGludGVyY29ubmVjdCUyMGJldHdlZW4lMjBOVU1BJTIwbm9kZXMlMjAoZS5nLiUyQyUyMFFQSSUyRlVQSSklMEElMjAlMjBOT0RFJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjB0aGUlMjBpbnRlcmNvbm5lY3QlMjBiZXR3ZWVuJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2VzJTIwd2l0aGluJTIwYSUyME5VTUElMjBub2RlJTBBJTIwJTIwUEhCJTIwJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjBhJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2UlMjAodHlwaWNhbGx5JTIwdGhlJTIwQ1BVKSUwQSUyMCUyMFBYQiUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwbXVsdGlwbGUlMjBQQ0llJTIwYnJpZGdlcyUyMCh3aXRob3V0JTIwdHJhdmVyc2luZyUyMHRoZSUyMFBDSWUlMjBIb3N0JTIwQnJpZGdlKSUwQSUyMCUyMFBJWCUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYXQlMjBtb3N0JTIwYSUyMHNpbmdsZSUyMFBDSWUlMjBicmlkZ2UlMEElMjAlMjBOViUyMyUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYSUyMGJvbmRlZCUyMHNldCUyMG9mJTIwJTIzJTIwTlZMaW5rcw==",highlighted:`  X    = Self
  SYS  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> interconnect between PCIe Host Bridges <span class="hljs-keyword">within</span> <span class="hljs-keyword">a</span> NUMA node
  PHB  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">a</span> PCIe Host Bridge (typically <span class="hljs-keyword">the</span> CPU)
  PXB  = Connection traversing multiple PCIe bridges (<span class="hljs-keyword">without</span> traversing <span class="hljs-keyword">the</span> PCIe Host Bridge)
  PIX  = Connection traversing <span class="hljs-keyword">at</span> most <span class="hljs-keyword">a</span> single PCIe bridge
  NV<span class="hljs-comment">#  = Connection traversing a bonded set of # NVLinks</span>`,wrap:!1}}),F=new re({props:{title:"NVlink",local:"nvlink",headingTag:"h4"}}),se=new pe({props:{code:"JTIzJTIwRERQJTIwdyUyRiUyME5WTGluayUwQSUwQXJtJTIwLXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUzQiUyMENVREFfVklTSUJMRV9ERVZJQ0VTJTNEMCUyQzElMjB0b3JjaHJ1biUyMCU1QyUwQS0tbnByb2NfcGVyX25vZGUlMjAyJTIwZXhhbXBsZXMlMkZweXRvcmNoJTJGbGFuZ3VhZ2UtbW9kZWxpbmclMkZydW5fY2xtLnB5JTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBvcGVuYWktY29tbXVuaXR5JTJGZ3B0MiUyMCU1QyUwQS0tZGF0YXNldF9uYW1lJTIwd2lraXRleHQlMjAtLWRhdGFzZXRfY29uZmlnX25hbWUlMjB3aWtpdGV4dC0yLXJhdy12MSUyMC0tZG9fdHJhaW4lMjAlNUMlMEEtLW91dHB1dF9kaXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwNCUyMC0tbWF4X3N0ZXBzJTIwMjAwJTBBJTBBJTdCJ3RyYWluX3J1bnRpbWUnJTNBJTIwMTAxLjkwMDMlMkMlMjAndHJhaW5fc2FtcGxlc19wZXJfc2Vjb25kJyUzQSUyMDEuOTYzJTJDJTIwJ2Vwb2NoJyUzQSUyMDAuNjklN0QlMEElMEElMjMlMjBERFAlMjB3JTJGbyUyME5WTGluayUwQSUwQXJtJTIwLXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUzQiUyMENVREFfVklTSUJMRV9ERVZJQ0VTJTNEMCUyQzElMjBOQ0NMX1AyUF9ESVNBQkxFJTNEMSUyMHRvcmNocnVuJTIwJTVDJTBBLS1ucHJvY19wZXJfbm9kZSUyMDIlMjBleGFtcGxlcyUyRnB5dG9yY2glMkZsYW5ndWFnZS1tb2RlbGluZyUyRnJ1bl9jbG0ucHklMjAtLW1vZGVsX25hbWVfb3JfcGF0aCUyMG9wZW5haS1jb21tdW5pdHklMkZncHQyJTIwJTVDJTBBLS1kYXRhc2V0X25hbWUlMjB3aWtpdGV4dCUyMC0tZGF0YXNldF9jb25maWdfbmFtZSUyMHdpa2l0ZXh0LTItcmF3LXYxJTIwLS1kb190cmFpbiUwQS0tb3V0cHV0X2RpciUyMCUyRnRtcCUyRnRlc3QtY2xtJTIwLS1wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjA0JTIwLS1tYXhfc3RlcHMlMjAyMDAlMEElMEElN0IndHJhaW5fcnVudGltZSclM0ElMjAxMzEuNDM2NyUyQyUyMCd0cmFpbl9zYW1wbGVzX3Blcl9zZWNvbmQnJTNBJTIwMS41MjIlMkMlMjAnZXBvY2gnJTNBJTIwMC42OSU3RA==",highlighted:`<span class="hljs-comment"># DDP w/ NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 torchrun \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path openai-community/gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train \\
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 101.9003, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.963, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}

<span class="hljs-comment"># DDP w/o NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 NCCL_P2P_DISABLE=1 torchrun \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path openai-community/gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 131.4367, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.522, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}`,wrap:!1}}),{c(){c=a("meta"),Me=s(),ae=a("p"),we=s(),r(d.$$.fragment),ye=s(),u=a("p"),u.innerHTML=st,he=s(),U=a("p"),U.textContent=it,me=s(),r(C.$$.fragment),ce=s(),f=a("p"),f.textContent=at,de=s(),T=a("ul"),T.innerHTML=ot,ue=s(),J=a("p"),J.textContent=pt,Ue=s(),r(I.$$.fragment),Ce=s(),v=a("p"),v.textContent=rt,fe=s(),b=a("p"),b.innerHTML=Mt,Te=s(),j=a("p"),j.textContent=wt,Je=s(),x=a("p"),x.textContent=yt,Ie=s(),P=a("p"),P.textContent=ht,ve=s(),k=a("p"),k.textContent=mt,be=s(),B=a("p"),B.textContent=ct,je=s(),g=a("p"),g.textContent=dt,xe=s(),_=a("p"),_.innerHTML=ut,Pe=s(),N=a("p"),N.textContent=Ut,ke=s(),G=a("p"),G.textContent=Ct,Be=s(),V=a("p"),V.textContent=ft,ge=s(),r(A.$$.fragment),_e=s(),E=a("p"),E.textContent=Tt,Ne=s(),r(S.$$.fragment),Ge=s(),Z=a("p"),Z.textContent=Jt,Ve=s(),r($.$$.fragment),Ae=s(),L=a("p"),L.textContent=It,Ee=s(),r(X.$$.fragment),Se=s(),H=a("p"),H.textContent=vt,Ze=s(),r(R.$$.fragment),$e=s(),W=a("p"),W.innerHTML=bt,Le=s(),Q=a("p"),Q.textContent=jt,Xe=s(),Y=a("p"),Y.textContent=xt,He=s(),r(F.$$.fragment),Re=s(),z=a("p"),z.innerHTML=Pt,We=s(),D=a("p"),D.innerHTML=kt,Qe=s(),O=a("blockquote"),O.innerHTML=Bt,Ye=s(),q=a("p"),q.innerHTML=gt,Fe=s(),K=a("p"),K.textContent=_t,ze=s(),ee=a("p"),ee.textContent=Nt,De=s(),te=a("table"),te.innerHTML=Gt,Oe=s(),le=a("p"),le.innerHTML=Vt,qe=s(),ne=a("p"),ne.textContent=At,Ke=s(),r(se.$$.fragment),et=s(),ie=a("p"),ie.innerHTML=Et,tt=s(),oe=a("p"),this.h()},l(e){const t=Wt("svelte-u9bgzb",document.head);c=o(t,"META",{name:!0,content:!0}),t.forEach(l),Me=i(e),ae=o(e,"P",{}),St(ae).forEach(l),we=i(e),M(d.$$.fragment,e),ye=i(e),u=o(e,"P",{"data-svelte-h":!0}),p(u)!=="svelte-iifysf"&&(u.innerHTML=st),he=i(e),U=o(e,"P",{"data-svelte-h":!0}),p(U)!=="svelte-1dshoiu"&&(U.textContent=it),me=i(e),M(C.$$.fragment,e),ce=i(e),f=o(e,"P",{"data-svelte-h":!0}),p(f)!=="svelte-eleqzp"&&(f.textContent=at),de=i(e),T=o(e,"UL",{"data-svelte-h":!0}),p(T)!=="svelte-pz3vuj"&&(T.innerHTML=ot),ue=i(e),J=o(e,"P",{"data-svelte-h":!0}),p(J)!=="svelte-z1tcso"&&(J.textContent=pt),Ue=i(e),M(I.$$.fragment,e),Ce=i(e),v=o(e,"P",{"data-svelte-h":!0}),p(v)!=="svelte-ppjikm"&&(v.textContent=rt),fe=i(e),b=o(e,"P",{"data-svelte-h":!0}),p(b)!=="svelte-11nr6dq"&&(b.innerHTML=Mt),Te=i(e),j=o(e,"P",{"data-svelte-h":!0}),p(j)!=="svelte-11nhv2k"&&(j.textContent=wt),Je=i(e),x=o(e,"P",{"data-svelte-h":!0}),p(x)!=="svelte-150xmew"&&(x.textContent=yt),Ie=i(e),P=o(e,"P",{"data-svelte-h":!0}),p(P)!=="svelte-12gfejp"&&(P.textContent=ht),ve=i(e),k=o(e,"P",{"data-svelte-h":!0}),p(k)!=="svelte-uh6lop"&&(k.textContent=mt),be=i(e),B=o(e,"P",{"data-svelte-h":!0}),p(B)!=="svelte-s9j0io"&&(B.textContent=ct),je=i(e),g=o(e,"P",{"data-svelte-h":!0}),p(g)!=="svelte-8g3p2u"&&(g.textContent=dt),xe=i(e),_=o(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-1gvtzcc"&&(_.innerHTML=ut),Pe=i(e),N=o(e,"P",{"data-svelte-h":!0}),p(N)!=="svelte-hba1wh"&&(N.textContent=Ut),ke=i(e),G=o(e,"P",{"data-svelte-h":!0}),p(G)!=="svelte-1ho6s7e"&&(G.textContent=Ct),Be=i(e),V=o(e,"P",{"data-svelte-h":!0}),p(V)!=="svelte-1c43r32"&&(V.textContent=ft),ge=i(e),M(A.$$.fragment,e),_e=i(e),E=o(e,"P",{"data-svelte-h":!0}),p(E)!=="svelte-1b7hx6s"&&(E.textContent=Tt),Ne=i(e),M(S.$$.fragment,e),Ge=i(e),Z=o(e,"P",{"data-svelte-h":!0}),p(Z)!=="svelte-1m7rqth"&&(Z.textContent=Jt),Ve=i(e),M($.$$.fragment,e),Ae=i(e),L=o(e,"P",{"data-svelte-h":!0}),p(L)!=="svelte-hggxkn"&&(L.textContent=It),Ee=i(e),M(X.$$.fragment,e),Se=i(e),H=o(e,"P",{"data-svelte-h":!0}),p(H)!=="svelte-869n3z"&&(H.textContent=vt),Ze=i(e),M(R.$$.fragment,e),$e=i(e),W=o(e,"P",{"data-svelte-h":!0}),p(W)!=="svelte-xobw9p"&&(W.innerHTML=bt),Le=i(e),Q=o(e,"P",{"data-svelte-h":!0}),p(Q)!=="svelte-abx4jy"&&(Q.textContent=jt),Xe=i(e),Y=o(e,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-1hcsw7p"&&(Y.textContent=xt),He=i(e),M(F.$$.fragment,e),Re=i(e),z=o(e,"P",{"data-svelte-h":!0}),p(z)!=="svelte-1gqi5fx"&&(z.innerHTML=Pt),We=i(e),D=o(e,"P",{"data-svelte-h":!0}),p(D)!=="svelte-gh3n8e"&&(D.innerHTML=kt),Qe=i(e),O=o(e,"BLOCKQUOTE",{"data-svelte-h":!0}),p(O)!=="svelte-14khom1"&&(O.innerHTML=Bt),Ye=i(e),q=o(e,"P",{"data-svelte-h":!0}),p(q)!=="svelte-1naftur"&&(q.innerHTML=gt),Fe=i(e),K=o(e,"P",{"data-svelte-h":!0}),p(K)!=="svelte-137n5ps"&&(K.textContent=_t),ze=i(e),ee=o(e,"P",{"data-svelte-h":!0}),p(ee)!=="svelte-j6eczh"&&(ee.textContent=Nt),De=i(e),te=o(e,"TABLE",{"data-svelte-h":!0}),p(te)!=="svelte-1hvmx1i"&&(te.innerHTML=Gt),Oe=i(e),le=o(e,"P",{"data-svelte-h":!0}),p(le)!=="svelte-192o6bz"&&(le.innerHTML=Vt),qe=i(e),ne=o(e,"P",{"data-svelte-h":!0}),p(ne)!=="svelte-16jny0z"&&(ne.textContent=At),Ke=i(e),M(se.$$.fragment,e),et=i(e),ie=o(e,"P",{"data-svelte-h":!0}),p(ie)!=="svelte-pittpg"&&(ie.innerHTML=Et),tt=i(e),oe=o(e,"P",{}),St(oe).forEach(l),this.h()},h(){Zt(c,"name","hf:doc:metadata"),Zt(c,"content",Ft)},m(e,t){Qt(document.head,c),n(e,Me,t),n(e,ae,t),n(e,we,t),w(d,e,t),n(e,ye,t),n(e,u,t),n(e,he,t),n(e,U,t),n(e,me,t),w(C,e,t),n(e,ce,t),n(e,f,t),n(e,de,t),n(e,T,t),n(e,ue,t),n(e,J,t),n(e,Ue,t),w(I,e,t),n(e,Ce,t),n(e,v,t),n(e,fe,t),n(e,b,t),n(e,Te,t),n(e,j,t),n(e,Je,t),n(e,x,t),n(e,Ie,t),n(e,P,t),n(e,ve,t),n(e,k,t),n(e,be,t),n(e,B,t),n(e,je,t),n(e,g,t),n(e,xe,t),n(e,_,t),n(e,Pe,t),n(e,N,t),n(e,ke,t),n(e,G,t),n(e,Be,t),n(e,V,t),n(e,ge,t),w(A,e,t),n(e,_e,t),n(e,E,t),n(e,Ne,t),w(S,e,t),n(e,Ge,t),n(e,Z,t),n(e,Ve,t),w($,e,t),n(e,Ae,t),n(e,L,t),n(e,Ee,t),w(X,e,t),n(e,Se,t),n(e,H,t),n(e,Ze,t),w(R,e,t),n(e,$e,t),n(e,W,t),n(e,Le,t),n(e,Q,t),n(e,Xe,t),n(e,Y,t),n(e,He,t),w(F,e,t),n(e,Re,t),n(e,z,t),n(e,We,t),n(e,D,t),n(e,Qe,t),n(e,O,t),n(e,Ye,t),n(e,q,t),n(e,Fe,t),n(e,K,t),n(e,ze,t),n(e,ee,t),n(e,De,t),n(e,te,t),n(e,Oe,t),n(e,le,t),n(e,qe,t),n(e,ne,t),n(e,Ke,t),w(se,e,t),n(e,et,t),n(e,ie,t),n(e,tt,t),n(e,oe,t),lt=!0},p:Lt,i(e){lt||(y(d.$$.fragment,e),y(C.$$.fragment,e),y(I.$$.fragment,e),y(A.$$.fragment,e),y(S.$$.fragment,e),y($.$$.fragment,e),y(X.$$.fragment,e),y(R.$$.fragment,e),y(F.$$.fragment,e),y(se.$$.fragment,e),lt=!0)},o(e){h(d.$$.fragment,e),h(C.$$.fragment,e),h(I.$$.fragment,e),h(A.$$.fragment,e),h(S.$$.fragment,e),h($.$$.fragment,e),h(X.$$.fragment,e),h(R.$$.fragment,e),h(F.$$.fragment,e),h(se.$$.fragment,e),lt=!1},d(e){e&&(l(Me),l(ae),l(we),l(ye),l(u),l(he),l(U),l(me),l(ce),l(f),l(de),l(T),l(ue),l(J),l(Ue),l(Ce),l(v),l(fe),l(b),l(Te),l(j),l(Je),l(x),l(Ie),l(P),l(ve),l(k),l(be),l(B),l(je),l(g),l(xe),l(_),l(Pe),l(N),l(ke),l(G),l(Be),l(V),l(ge),l(_e),l(E),l(Ne),l(Ge),l(Z),l(Ve),l(Ae),l(L),l(Ee),l(Se),l(H),l(Ze),l($e),l(W),l(Le),l(Q),l(Xe),l(Y),l(He),l(Re),l(z),l(We),l(D),l(Qe),l(O),l(Ye),l(q),l(Fe),l(K),l(ze),l(ee),l(De),l(te),l(Oe),l(le),l(qe),l(ne),l(Ke),l(et),l(ie),l(tt),l(oe)),l(c),m(d,e),m(C,e),m(I,e),m(A,e),m(S,e),m($,e),m(X,e),m(R,e),m(F,e),m(se,e)}}}const Ft='{"title":"Custom hardware for training","local":"custom-hardware-for-training","sections":[{"title":"GPU","local":"gpu","sections":[{"title":"Power and Cooling","local":"power-and-cooling","sections":[],"depth":3},{"title":"Multi-GPU Connectivity","local":"multi-gpu-connectivity","sections":[{"title":"NVlink","local":"nvlink","sections":[],"depth":4}],"depth":3}],"depth":2}],"depth":1}';function zt(nt){return Xt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class el extends Ht{constructor(c){super(),Rt(this,c,zt,Yt,$t,{})}}export{el as component};
