import{S as fn,i as dn,s as wn,e as o,k as h,w as M,t as r,M as un,c as n,d as t,m as c,a as s,x as U,h as p,b as f,G as l,g as i,y as v,L as yn,q as T,o as b,B as I,v as mn}from"../chunks/vendor-hf-doc-builder.js";import{I as ct}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as ft}from"../chunks/CodeBlock-hf-doc-builder.js";function Mn(Ka){let P,dt,C,j,Ae,X,ml,Se,Ml,wt,N,Ul,Z,vl,Tl,ut,ie,bl,yt,E,G,ge,R,Il,$e,Pl,mt,y,Le,Cl,El,Xe,Jl,kl,W,_l,re,jl,Nl,Mt,pe,Gl,Ut,J,B,Ze,x,Bl,Re,Vl,vt,he,Al,Tt,D,We,Sl,gl,bt,ce,$l,It,fe,Ll,Pt,de,Xl,Ct,we,Zl,Et,ue,Rl,Jt,ye,Wl,kt,Q,xe,xl,Dl,_t,me,Ql,jt,Me,Yl,Nt,Ue,Hl,Gt,k,V,De,Y,Fl,Qe,zl,Bt,ve,Ol,Vt,H,At,Te,ql,St,F,gt,be,Kl,$t,z,Lt,Ie,ea,Xt,O,Zt,m,ta,Ye,la,aa,He,oa,na,Rt,Pe,sa,Wt,Ce,ia,xt,_,A,Fe,q,ra,ze,pa,Dt,K,ee,ha,ca,Qt,S,fa,te,da,wa,Yt,Ee,Oe,ua,Ht,w,ya,qe,ma,Ma,Ke,Ua,va,et,Ta,ba,Ft,Je,Ia,zt,ke,Pa,Ot,g,tt,le,lt,Ca,Ea,_e,Ja,ka,ae,oe,at,_a,ja,je,Na,Ga,ne,ot,Ba,Va,Ne,Aa,qt,$,Sa,nt,ga,$a,Kt,Ge,La,el,se,tl,d,Xa,st,Za,Ra,it,Wa,xa,rt,Da,Qa,pt,Ya,Ha,ht,Fa,ll;return X=new ct({}),R=new ct({}),x=new ct({}),Y=new ct({}),H=new ft({props:{code:"bnZpZGlhLXNtaSUyMHRvcG8lMjAtbQ==",highlighted:'<span class="hljs-symbol">nvidia</span>-<span class="hljs-keyword">smi</span> topo -m'}}),F=new ft({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwTlYyJTIwJTIwJTIwJTIwJTIwMC0yMyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyME5WMiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTIzJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      NV2     <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A
<span class="hljs-attribute">GPU1</span>    NV2      X      <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A`}}),z=new ft({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwUEhCJTIwJTIwJTIwJTIwJTIwMC0xMSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyMFBIQiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTExJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      PHB     <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A
<span class="hljs-attribute">GPU1</span>    PHB      X      <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A`}}),O=new ft({props:{code:"JTIwJTIwWCUyMCUyMCUyMCUyMCUzRCUyMFNlbGYlMEElMjAlMjBTWVMlMjAlMjAlM0QlMjBDb25uZWN0aW9uJTIwdHJhdmVyc2luZyUyMFBDSWUlMjBhcyUyMHdlbGwlMjBhcyUyMHRoZSUyMFNNUCUyMGludGVyY29ubmVjdCUyMGJldHdlZW4lMjBOVU1BJTIwbm9kZXMlMjAoZS5nLiUyQyUyMFFQSSUyRlVQSSklMEElMjAlMjBOT0RFJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjB0aGUlMjBpbnRlcmNvbm5lY3QlMjBiZXR3ZWVuJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2VzJTIwd2l0aGluJTIwYSUyME5VTUElMjBub2RlJTBBJTIwJTIwUEhCJTIwJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjBhJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2UlMjAodHlwaWNhbGx5JTIwdGhlJTIwQ1BVKSUwQSUyMCUyMFBYQiUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwbXVsdGlwbGUlMjBQQ0llJTIwYnJpZGdlcyUyMCh3aXRob3V0JTIwdHJhdmVyc2luZyUyMHRoZSUyMFBDSWUlMjBIb3N0JTIwQnJpZGdlKSUwQSUyMCUyMFBJWCUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYXQlMjBtb3N0JTIwYSUyMHNpbmdsZSUyMFBDSWUlMjBicmlkZ2UlMEElMjAlMjBOViUyMyUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYSUyMGJvbmRlZCUyMHNldCUyMG9mJTIwJTIzJTIwTlZMaW5rcw==",highlighted:`  X    = Self
  SYS  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> interconnect between PCIe Host Bridges <span class="hljs-keyword">within</span> <span class="hljs-keyword">a</span> NUMA node
  PHB  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">a</span> PCIe Host Bridge (typically <span class="hljs-keyword">the</span> CPU)
  PXB  = Connection traversing multiple PCIe bridges (<span class="hljs-keyword">without</span> traversing <span class="hljs-keyword">the</span> PCIe Host Bridge)
  PIX  = Connection traversing <span class="hljs-keyword">at</span> most <span class="hljs-keyword">a</span> single PCIe bridge
  NV<span class="hljs-comment">#  = Connection traversing a bonded set of # NVLinks</span>`}}),q=new ct({}),se=new ft({props:{code:"JTIzJTIwRERQJTIwdyUyRiUyME5WTGluayUwQSUwQXJtJTIwLXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUzQiUyMENVREFfVklTSUJMRV9ERVZJQ0VTJTNEMCUyQzElMjBweXRob24lMjAtbSUyMHRvcmNoLmRpc3RyaWJ1dGVkLmxhdW5jaCUyMCU1QyUwQS0tbnByb2NfcGVyX25vZGUlMjAyJTIwZXhhbXBsZXMlMkZweXRvcmNoJTJGbGFuZ3VhZ2UtbW9kZWxpbmclMkZydW5fY2xtLnB5JTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBncHQyJTIwJTVDJTBBLS1kYXRhc2V0X25hbWUlMjB3aWtpdGV4dCUyMC0tZGF0YXNldF9jb25maWdfbmFtZSUyMHdpa2l0ZXh0LTItcmF3LXYxJTIwLS1kb190cmFpbiUyMCU1QyUwQS0tb3V0cHV0X2RpciUyMCUyRnRtcCUyRnRlc3QtY2xtJTIwLS1wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjA0JTIwLS1tYXhfc3RlcHMlMjAyMDAlMEElMEElN0IndHJhaW5fcnVudGltZSclM0ElMjAxMDEuOTAwMyUyQyUyMCd0cmFpbl9zYW1wbGVzX3Blcl9zZWNvbmQnJTNBJTIwMS45NjMlMkMlMjAnZXBvY2gnJTNBJTIwMC42OSU3RCUwQSUwQSUyMyUyMEREUCUyMHclMkZvJTIwTlZMaW5rJTBBJTBBcm0lMjAtciUyMCUyRnRtcCUyRnRlc3QtY2xtJTNCJTIwQ1VEQV9WSVNJQkxFX0RFVklDRVMlM0QwJTJDMSUyME5DQ0xfUDJQX0RJU0FCTEUlM0QxJTIwcHl0aG9uJTIwLW0lMjB0b3JjaC5kaXN0cmlidXRlZC5sYXVuY2glMjAlNUMlMEEtLW5wcm9jX3Blcl9ub2RlJTIwMiUyMGV4YW1wbGVzJTJGcHl0b3JjaCUyRmxhbmd1YWdlLW1vZGVsaW5nJTJGcnVuX2NsbS5weSUyMC0tbW9kZWxfbmFtZV9vcl9wYXRoJTIwZ3B0MiUyMCU1QyUwQS0tZGF0YXNldF9uYW1lJTIwd2lraXRleHQlMjAtLWRhdGFzZXRfY29uZmlnX25hbWUlMjB3aWtpdGV4dC0yLXJhdy12MSUyMC0tZG9fdHJhaW4lMEEtLW91dHB1dF9kaXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwNCUyMC0tbWF4X3N0ZXBzJTIwMjAwJTBBJTBBJTdCJ3RyYWluX3J1bnRpbWUnJTNBJTIwMTMxLjQzNjclMkMlMjAndHJhaW5fc2FtcGxlc19wZXJfc2Vjb25kJyUzQSUyMDEuNTIyJTJDJTIwJ2Vwb2NoJyUzQSUyMDAuNjklN0Q=",highlighted:`<span class="hljs-comment"># DDP w/ NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train \\
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 101.9003, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.963, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}

<span class="hljs-comment"># DDP w/o NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 NCCL_P2P_DISABLE=1 python -m torch.distributed.launch \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 131.4367, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.522, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}`}}),{c(){P=o("meta"),dt=h(),C=o("h1"),j=o("a"),Ae=o("span"),M(X.$$.fragment),ml=h(),Se=o("span"),Ml=r("Custom hardware for training"),wt=h(),N=o("p"),Ul=r("The hardware you use to run model training and inference can have a big effect on performance. For a deep dive into GPUs make sure to check out Tim Dettmer\u2019s excellent "),Z=o("a"),vl=r("blog post"),Tl=r("."),ut=h(),ie=o("p"),bl=r("Let\u2019s have a look at some practical advice for GPU setups."),yt=h(),E=o("h2"),G=o("a"),ge=o("span"),M(R.$$.fragment),Il=h(),$e=o("span"),Pl=r("GPU"),mt=r(`

When you train bigger models you have essentially three options:
`),y=o("ul"),Le=o("li"),Cl=r("bigger GPUs"),El=h(),Xe=o("li"),Jl=r("more GPUs"),kl=h(),W=o("li"),_l=r("more CPU and NVMe (offloaded to by "),re=o("a"),jl=r("DeepSpeed-Infinity"),Nl=r(")"),Mt=h(),pe=o("p"),Gl=r("Let\u2019s start at the case where you have a single GPU."),Ut=h(),J=o("h3"),B=o("a"),Ze=o("span"),M(x.$$.fragment),Bl=h(),Re=o("span"),Vl=r("Power and Cooling"),vt=h(),he=o("p"),Al=r("If you bought an expensive high end GPU make sure you give it the correct power and sufficient cooling."),Tt=h(),D=o("p"),We=o("strong"),Sl=r("Power"),gl=r(":"),bt=h(),ce=o("p"),$l=r("Some high end consumer GPU cards have 2 and sometimes 3 PCI-E 8-Pin power sockets. Make sure you have as many independent 12V PCI-E 8-Pin cables plugged into the card as there are sockets. Do not use the 2 splits at one end of the same cable (also known as pigtail cable). That is if you have 2 sockets on the GPU, you want 2 PCI-E 8-Pin cables going from your PSU to the card and not one that has 2 PCI-E 8-Pin connectors at the end! You won\u2019t get the full performance out of your card otherwise."),It=h(),fe=o("p"),Ll=r("Each PCI-E 8-Pin power cable needs to be plugged into a 12V rail on the PSU side and can supply up to 150W of power."),Pt=h(),de=o("p"),Xl=r("Some other cards may use a PCI-E 12-Pin connectors, and these can deliver up to 500-600W of power."),Ct=h(),we=o("p"),Zl=r("Low end cards may use 6-Pin connectors, which supply up to 75W of power."),Et=h(),ue=o("p"),Rl=r("Additionally you want the high-end PSU that has stable voltage. Some lower quality ones may not give the card the stable voltage it needs to function at its peak."),Jt=h(),ye=o("p"),Wl=r("And of course the PSU needs to have enough unused Watts to power the card."),kt=h(),Q=o("p"),xe=o("strong"),xl=r("Cooling"),Dl=r(":"),_t=h(),me=o("p"),Ql=r("When a GPU gets overheated it will start throttling down and will not deliver full performance and it can even shutdown if it gets too hot."),jt=h(),Me=o("p"),Yl=r("It\u2019s hard to tell the exact best temperature to strive for when a GPU is heavily loaded, but probably anything under +80C is good, but lower is better - perhaps 70-75C is an excellent range to be in. The throttling down is likely to start at around 84-90C. But other than throttling performance a prolonged very high temperature is likely to reduce the lifespan of a GPU."),Nt=h(),Ue=o("p"),Hl=r("Next let\u2019s have a look at one of the most important aspects when having multiple GPUs: connectivity."),Gt=h(),k=o("h3"),V=o("a"),De=o("span"),M(Y.$$.fragment),Fl=h(),Qe=o("span"),zl=r("Multi-GPU Connectivity"),Bt=h(),ve=o("p"),Ol=r("If you use multiple GPUs the way cards are inter-connected can have a huge impact on the total training time. If the GPUs are on the same physical node, you can run:"),Vt=h(),M(H.$$.fragment),At=h(),Te=o("p"),ql=r("and it will tell you how the GPUs are inter-connected. On a machine with dual-GPU and which are connected with NVLink, you will most likely see something like:"),St=h(),M(F.$$.fragment),gt=h(),be=o("p"),Kl=r("on a different machine w/o NVLink we may see:"),$t=h(),M(z.$$.fragment),Lt=h(),Ie=o("p"),ea=r("The report includes this legend:"),Xt=h(),M(O.$$.fragment),Zt=h(),m=o("p"),ta=r("So the first report "),Ye=o("code"),la=r("NV2"),aa=r(" tells us the GPUs are interconnected with 2 NVLinks, and the second report "),He=o("code"),oa=r("PHB"),na=r(" we have a typical consumer-level PCIe+Bridge setup."),Rt=h(),Pe=o("p"),sa=r("Check what type of connectivity you have on your setup. Some of these will make the communication between cards faster (e.g. NVLink), others slower (e.g. PHB)."),Wt=h(),Ce=o("p"),ia=r("Depending on the type of scalability solution used, the connectivity speed could have a major or a minor impact. If the GPUs need to sync rarely, as in DDP, the impact of a slower connection will be less significant. If the GPUs need to send messages to each other often, as in ZeRO-DP, then faster connectivity becomes super important to achieve faster training."),xt=h(),_=o("h4"),A=o("a"),Fe=o("span"),M(q.$$.fragment),ra=h(),ze=o("span"),pa=r("NVlink"),Dt=h(),K=o("p"),ee=o("a"),ha=r("NVLink"),ca=r(" is a wire-based serial multi-lane near-range communications link developed by Nvidia."),Qt=h(),S=o("p"),fa=r("Each new generation provides a faster bandwidth, e.g. here is a quote from "),te=o("a"),da=r("Nvidia Ampere GA102 GPU Architecture"),wa=r(":"),Yt=h(),Ee=o("blockquote"),Oe=o("p"),ua=r(`Third-Generation NVLink\xAE
GA102 GPUs utilize NVIDIA\u2019s third-generation NVLink interface, which includes four x4 links,
with each link providing 14.0625 GB/sec bandwidth in each direction between two GPUs. Four
links provide 56.25 GB/sec bandwidth in each direction, and 112.5 GB/sec total bandwidth
between two GPUs. Two RTX 3090 GPUs can be connected together for SLI using NVLink.
(Note that 3-Way and 4-Way SLI configurations are not supported.)`),Ht=h(),w=o("p"),ya=r("So the higher "),qe=o("code"),ma=r("X"),Ma=r(" you get in the report of "),Ke=o("code"),Ua=r("NVX"),va=r(" in the output of "),et=o("code"),Ta=r("nvidia-smi topo -m"),ba=r(" the better. The generation will depend on your GPU architecture."),Ft=h(),Je=o("p"),Ia=r("Let\u2019s compare the execution of a gpt2 language model training over a small sample of wikitext."),zt=h(),ke=o("p"),Pa=r("The results are:"),Ot=h(),g=o("table"),tt=o("thead"),le=o("tr"),lt=o("th"),Ca=r("NVlink"),Ea=h(),_e=o("th"),Ja=r("Time"),ka=h(),ae=o("tbody"),oe=o("tr"),at=o("td"),_a=r("Y"),ja=h(),je=o("td"),Na=r("101s"),Ga=h(),ne=o("tr"),ot=o("td"),Ba=r("N"),Va=h(),Ne=o("td"),Aa=r("131s"),qt=h(),$=o("p"),Sa=r("You can see that NVLink completes the training ~23% faster. In the second benchmark we use "),nt=o("code"),ga=r("NCCL_P2P_DISABLE=1"),$a=r(" to tell the GPUs not to use NVLink."),Kt=h(),Ge=o("p"),La=r("Here is the full benchmark code and outputs:"),el=h(),M(se.$$.fragment),tl=h(),d=o("p"),Xa=r("Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks ("),st=o("code"),Za=r("NV2"),Ra=r(" in "),it=o("code"),Wa=r("nvidia-smi topo -m"),xa=r(`)
Software: `),rt=o("code"),Da=r("pytorch-1.8-to-be"),Qa=r(" + "),pt=o("code"),Ya=r("cuda-11.0"),Ha=r(" / "),ht=o("code"),Fa=r("transformers==4.3.0.dev0"),this.h()},l(e){const a=un('[data-svelte="svelte-1phssyn"]',document.head);P=n(a,"META",{name:!0,content:!0}),a.forEach(t),dt=c(e),C=n(e,"H1",{class:!0});var al=s(C);j=n(al,"A",{id:!0,class:!0,href:!0});var eo=s(j);Ae=n(eo,"SPAN",{});var to=s(Ae);U(X.$$.fragment,to),to.forEach(t),eo.forEach(t),ml=c(al),Se=n(al,"SPAN",{});var lo=s(Se);Ml=p(lo,"Custom hardware for training"),lo.forEach(t),al.forEach(t),wt=c(e),N=n(e,"P",{});var ol=s(N);Ul=p(ol,"The hardware you use to run model training and inference can have a big effect on performance. For a deep dive into GPUs make sure to check out Tim Dettmer\u2019s excellent "),Z=n(ol,"A",{href:!0,rel:!0});var ao=s(Z);vl=p(ao,"blog post"),ao.forEach(t),Tl=p(ol,"."),ol.forEach(t),ut=c(e),ie=n(e,"P",{});var oo=s(ie);bl=p(oo,"Let\u2019s have a look at some practical advice for GPU setups."),oo.forEach(t),yt=c(e),E=n(e,"H2",{class:!0});var nl=s(E);G=n(nl,"A",{id:!0,class:!0,href:!0});var no=s(G);ge=n(no,"SPAN",{});var so=s(ge);U(R.$$.fragment,so),so.forEach(t),no.forEach(t),Il=c(nl),$e=n(nl,"SPAN",{});var io=s($e);Pl=p(io,"GPU"),io.forEach(t),nl.forEach(t),mt=p(e,`

When you train bigger models you have essentially three options:
`),y=n(e,"UL",{});var Be=s(y);Le=n(Be,"LI",{});var ro=s(Le);Cl=p(ro,"bigger GPUs"),ro.forEach(t),El=c(Be),Xe=n(Be,"LI",{});var po=s(Xe);Jl=p(po,"more GPUs"),po.forEach(t),kl=c(Be),W=n(Be,"LI",{});var sl=s(W);_l=p(sl,"more CPU and NVMe (offloaded to by "),re=n(sl,"A",{href:!0});var ho=s(re);jl=p(ho,"DeepSpeed-Infinity"),ho.forEach(t),Nl=p(sl,")"),sl.forEach(t),Be.forEach(t),Mt=c(e),pe=n(e,"P",{});var co=s(pe);Gl=p(co,"Let\u2019s start at the case where you have a single GPU."),co.forEach(t),Ut=c(e),J=n(e,"H3",{class:!0});var il=s(J);B=n(il,"A",{id:!0,class:!0,href:!0});var fo=s(B);Ze=n(fo,"SPAN",{});var wo=s(Ze);U(x.$$.fragment,wo),wo.forEach(t),fo.forEach(t),Bl=c(il),Re=n(il,"SPAN",{});var uo=s(Re);Vl=p(uo,"Power and Cooling"),uo.forEach(t),il.forEach(t),vt=c(e),he=n(e,"P",{});var yo=s(he);Al=p(yo,"If you bought an expensive high end GPU make sure you give it the correct power and sufficient cooling."),yo.forEach(t),Tt=c(e),D=n(e,"P",{});var za=s(D);We=n(za,"STRONG",{});var mo=s(We);Sl=p(mo,"Power"),mo.forEach(t),gl=p(za,":"),za.forEach(t),bt=c(e),ce=n(e,"P",{});var Mo=s(ce);$l=p(Mo,"Some high end consumer GPU cards have 2 and sometimes 3 PCI-E 8-Pin power sockets. Make sure you have as many independent 12V PCI-E 8-Pin cables plugged into the card as there are sockets. Do not use the 2 splits at one end of the same cable (also known as pigtail cable). That is if you have 2 sockets on the GPU, you want 2 PCI-E 8-Pin cables going from your PSU to the card and not one that has 2 PCI-E 8-Pin connectors at the end! You won\u2019t get the full performance out of your card otherwise."),Mo.forEach(t),It=c(e),fe=n(e,"P",{});var Uo=s(fe);Ll=p(Uo,"Each PCI-E 8-Pin power cable needs to be plugged into a 12V rail on the PSU side and can supply up to 150W of power."),Uo.forEach(t),Pt=c(e),de=n(e,"P",{});var vo=s(de);Xl=p(vo,"Some other cards may use a PCI-E 12-Pin connectors, and these can deliver up to 500-600W of power."),vo.forEach(t),Ct=c(e),we=n(e,"P",{});var To=s(we);Zl=p(To,"Low end cards may use 6-Pin connectors, which supply up to 75W of power."),To.forEach(t),Et=c(e),ue=n(e,"P",{});var bo=s(ue);Rl=p(bo,"Additionally you want the high-end PSU that has stable voltage. Some lower quality ones may not give the card the stable voltage it needs to function at its peak."),bo.forEach(t),Jt=c(e),ye=n(e,"P",{});var Io=s(ye);Wl=p(Io,"And of course the PSU needs to have enough unused Watts to power the card."),Io.forEach(t),kt=c(e),Q=n(e,"P",{});var Oa=s(Q);xe=n(Oa,"STRONG",{});var Po=s(xe);xl=p(Po,"Cooling"),Po.forEach(t),Dl=p(Oa,":"),Oa.forEach(t),_t=c(e),me=n(e,"P",{});var Co=s(me);Ql=p(Co,"When a GPU gets overheated it will start throttling down and will not deliver full performance and it can even shutdown if it gets too hot."),Co.forEach(t),jt=c(e),Me=n(e,"P",{});var Eo=s(Me);Yl=p(Eo,"It\u2019s hard to tell the exact best temperature to strive for when a GPU is heavily loaded, but probably anything under +80C is good, but lower is better - perhaps 70-75C is an excellent range to be in. The throttling down is likely to start at around 84-90C. But other than throttling performance a prolonged very high temperature is likely to reduce the lifespan of a GPU."),Eo.forEach(t),Nt=c(e),Ue=n(e,"P",{});var Jo=s(Ue);Hl=p(Jo,"Next let\u2019s have a look at one of the most important aspects when having multiple GPUs: connectivity."),Jo.forEach(t),Gt=c(e),k=n(e,"H3",{class:!0});var rl=s(k);V=n(rl,"A",{id:!0,class:!0,href:!0});var ko=s(V);De=n(ko,"SPAN",{});var _o=s(De);U(Y.$$.fragment,_o),_o.forEach(t),ko.forEach(t),Fl=c(rl),Qe=n(rl,"SPAN",{});var jo=s(Qe);zl=p(jo,"Multi-GPU Connectivity"),jo.forEach(t),rl.forEach(t),Bt=c(e),ve=n(e,"P",{});var No=s(ve);Ol=p(No,"If you use multiple GPUs the way cards are inter-connected can have a huge impact on the total training time. If the GPUs are on the same physical node, you can run:"),No.forEach(t),Vt=c(e),U(H.$$.fragment,e),At=c(e),Te=n(e,"P",{});var Go=s(Te);ql=p(Go,"and it will tell you how the GPUs are inter-connected. On a machine with dual-GPU and which are connected with NVLink, you will most likely see something like:"),Go.forEach(t),St=c(e),U(F.$$.fragment,e),gt=c(e),be=n(e,"P",{});var Bo=s(be);Kl=p(Bo,"on a different machine w/o NVLink we may see:"),Bo.forEach(t),$t=c(e),U(z.$$.fragment,e),Lt=c(e),Ie=n(e,"P",{});var Vo=s(Ie);ea=p(Vo,"The report includes this legend:"),Vo.forEach(t),Xt=c(e),U(O.$$.fragment,e),Zt=c(e),m=n(e,"P",{});var Ve=s(m);ta=p(Ve,"So the first report "),Ye=n(Ve,"CODE",{});var Ao=s(Ye);la=p(Ao,"NV2"),Ao.forEach(t),aa=p(Ve," tells us the GPUs are interconnected with 2 NVLinks, and the second report "),He=n(Ve,"CODE",{});var So=s(He);oa=p(So,"PHB"),So.forEach(t),na=p(Ve," we have a typical consumer-level PCIe+Bridge setup."),Ve.forEach(t),Rt=c(e),Pe=n(e,"P",{});var go=s(Pe);sa=p(go,"Check what type of connectivity you have on your setup. Some of these will make the communication between cards faster (e.g. NVLink), others slower (e.g. PHB)."),go.forEach(t),Wt=c(e),Ce=n(e,"P",{});var $o=s(Ce);ia=p($o,"Depending on the type of scalability solution used, the connectivity speed could have a major or a minor impact. If the GPUs need to sync rarely, as in DDP, the impact of a slower connection will be less significant. If the GPUs need to send messages to each other often, as in ZeRO-DP, then faster connectivity becomes super important to achieve faster training."),$o.forEach(t),xt=c(e),_=n(e,"H4",{class:!0});var pl=s(_);A=n(pl,"A",{id:!0,class:!0,href:!0});var Lo=s(A);Fe=n(Lo,"SPAN",{});var Xo=s(Fe);U(q.$$.fragment,Xo),Xo.forEach(t),Lo.forEach(t),ra=c(pl),ze=n(pl,"SPAN",{});var Zo=s(ze);pa=p(Zo,"NVlink"),Zo.forEach(t),pl.forEach(t),Dt=c(e),K=n(e,"P",{});var qa=s(K);ee=n(qa,"A",{href:!0,rel:!0});var Ro=s(ee);ha=p(Ro,"NVLink"),Ro.forEach(t),ca=p(qa," is a wire-based serial multi-lane near-range communications link developed by Nvidia."),qa.forEach(t),Qt=c(e),S=n(e,"P",{});var hl=s(S);fa=p(hl,"Each new generation provides a faster bandwidth, e.g. here is a quote from "),te=n(hl,"A",{href:!0,rel:!0});var Wo=s(te);da=p(Wo,"Nvidia Ampere GA102 GPU Architecture"),Wo.forEach(t),wa=p(hl,":"),hl.forEach(t),Yt=c(e),Ee=n(e,"BLOCKQUOTE",{});var xo=s(Ee);Oe=n(xo,"P",{});var Do=s(Oe);ua=p(Do,`Third-Generation NVLink\xAE
GA102 GPUs utilize NVIDIA\u2019s third-generation NVLink interface, which includes four x4 links,
with each link providing 14.0625 GB/sec bandwidth in each direction between two GPUs. Four
links provide 56.25 GB/sec bandwidth in each direction, and 112.5 GB/sec total bandwidth
between two GPUs. Two RTX 3090 GPUs can be connected together for SLI using NVLink.
(Note that 3-Way and 4-Way SLI configurations are not supported.)`),Do.forEach(t),xo.forEach(t),Ht=c(e),w=n(e,"P",{});var L=s(w);ya=p(L,"So the higher "),qe=n(L,"CODE",{});var Qo=s(qe);ma=p(Qo,"X"),Qo.forEach(t),Ma=p(L," you get in the report of "),Ke=n(L,"CODE",{});var Yo=s(Ke);Ua=p(Yo,"NVX"),Yo.forEach(t),va=p(L," in the output of "),et=n(L,"CODE",{});var Ho=s(et);Ta=p(Ho,"nvidia-smi topo -m"),Ho.forEach(t),ba=p(L," the better. The generation will depend on your GPU architecture."),L.forEach(t),Ft=c(e),Je=n(e,"P",{});var Fo=s(Je);Ia=p(Fo,"Let\u2019s compare the execution of a gpt2 language model training over a small sample of wikitext."),Fo.forEach(t),zt=c(e),ke=n(e,"P",{});var zo=s(ke);Pa=p(zo,"The results are:"),zo.forEach(t),Ot=c(e),g=n(e,"TABLE",{});var cl=s(g);tt=n(cl,"THEAD",{});var Oo=s(tt);le=n(Oo,"TR",{});var fl=s(le);lt=n(fl,"TH",{});var qo=s(lt);Ca=p(qo,"NVlink"),qo.forEach(t),Ea=c(fl),_e=n(fl,"TH",{align:!0});var Ko=s(_e);Ja=p(Ko,"Time"),Ko.forEach(t),fl.forEach(t),Oo.forEach(t),ka=c(cl),ae=n(cl,"TBODY",{});var dl=s(ae);oe=n(dl,"TR",{});var wl=s(oe);at=n(wl,"TD",{});var en=s(at);_a=p(en,"Y"),en.forEach(t),ja=c(wl),je=n(wl,"TD",{align:!0});var tn=s(je);Na=p(tn,"101s"),tn.forEach(t),wl.forEach(t),Ga=c(dl),ne=n(dl,"TR",{});var ul=s(ne);ot=n(ul,"TD",{});var ln=s(ot);Ba=p(ln,"N"),ln.forEach(t),Va=c(ul),Ne=n(ul,"TD",{align:!0});var an=s(Ne);Aa=p(an,"131s"),an.forEach(t),ul.forEach(t),dl.forEach(t),cl.forEach(t),qt=c(e),$=n(e,"P",{});var yl=s($);Sa=p(yl,"You can see that NVLink completes the training ~23% faster. In the second benchmark we use "),nt=n(yl,"CODE",{});var on=s(nt);ga=p(on,"NCCL_P2P_DISABLE=1"),on.forEach(t),$a=p(yl," to tell the GPUs not to use NVLink."),yl.forEach(t),Kt=c(e),Ge=n(e,"P",{});var nn=s(Ge);La=p(nn,"Here is the full benchmark code and outputs:"),nn.forEach(t),el=c(e),U(se.$$.fragment,e),tl=c(e),d=n(e,"P",{});var u=s(d);Xa=p(u,"Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks ("),st=n(u,"CODE",{});var sn=s(st);Za=p(sn,"NV2"),sn.forEach(t),Ra=p(u," in "),it=n(u,"CODE",{});var rn=s(it);Wa=p(rn,"nvidia-smi topo -m"),rn.forEach(t),xa=p(u,`)
Software: `),rt=n(u,"CODE",{});var pn=s(rt);Da=p(pn,"pytorch-1.8-to-be"),pn.forEach(t),Qa=p(u," + "),pt=n(u,"CODE",{});var hn=s(pt);Ya=p(hn,"cuda-11.0"),hn.forEach(t),Ha=p(u," / "),ht=n(u,"CODE",{});var cn=s(ht);Fa=p(cn,"transformers==4.3.0.dev0"),cn.forEach(t),u.forEach(t),this.h()},h(){f(P,"name","hf:doc:metadata"),f(P,"content",JSON.stringify(Un)),f(j,"id","custom-hardware-for-training"),f(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(j,"href","#custom-hardware-for-training"),f(C,"class","relative group"),f(Z,"href","https://timdettmers.com/2020/09/07/which-gpu-for-deep-learning/"),f(Z,"rel","nofollow"),f(G,"id","gpu"),f(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(G,"href","#gpu"),f(E,"class","relative group"),f(re,"href","main_classes/deepspeed#nvme-support"),f(B,"id","power-and-cooling"),f(B,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(B,"href","#power-and-cooling"),f(J,"class","relative group"),f(V,"id","multigpu-connectivity"),f(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(V,"href","#multigpu-connectivity"),f(k,"class","relative group"),f(A,"id","nvlink"),f(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(A,"href","#nvlink"),f(_,"class","relative group"),f(ee,"href","https://en.wikipedia.org/wiki/NVLink"),f(ee,"rel","nofollow"),f(te,"href","https://www.nvidia.com/content/dam/en-zz/Solutions/geforce/ampere/pdf/NVIDIA-ampere-GA102-GPU-Architecture-Whitepaper-V1.pdf"),f(te,"rel","nofollow"),f(_e,"align","right"),f(je,"align","right"),f(Ne,"align","right")},m(e,a){l(document.head,P),i(e,dt,a),i(e,C,a),l(C,j),l(j,Ae),v(X,Ae,null),l(C,ml),l(C,Se),l(Se,Ml),i(e,wt,a),i(e,N,a),l(N,Ul),l(N,Z),l(Z,vl),l(N,Tl),i(e,ut,a),i(e,ie,a),l(ie,bl),i(e,yt,a),i(e,E,a),l(E,G),l(G,ge),v(R,ge,null),l(E,Il),l(E,$e),l($e,Pl),i(e,mt,a),i(e,y,a),l(y,Le),l(Le,Cl),l(y,El),l(y,Xe),l(Xe,Jl),l(y,kl),l(y,W),l(W,_l),l(W,re),l(re,jl),l(W,Nl),i(e,Mt,a),i(e,pe,a),l(pe,Gl),i(e,Ut,a),i(e,J,a),l(J,B),l(B,Ze),v(x,Ze,null),l(J,Bl),l(J,Re),l(Re,Vl),i(e,vt,a),i(e,he,a),l(he,Al),i(e,Tt,a),i(e,D,a),l(D,We),l(We,Sl),l(D,gl),i(e,bt,a),i(e,ce,a),l(ce,$l),i(e,It,a),i(e,fe,a),l(fe,Ll),i(e,Pt,a),i(e,de,a),l(de,Xl),i(e,Ct,a),i(e,we,a),l(we,Zl),i(e,Et,a),i(e,ue,a),l(ue,Rl),i(e,Jt,a),i(e,ye,a),l(ye,Wl),i(e,kt,a),i(e,Q,a),l(Q,xe),l(xe,xl),l(Q,Dl),i(e,_t,a),i(e,me,a),l(me,Ql),i(e,jt,a),i(e,Me,a),l(Me,Yl),i(e,Nt,a),i(e,Ue,a),l(Ue,Hl),i(e,Gt,a),i(e,k,a),l(k,V),l(V,De),v(Y,De,null),l(k,Fl),l(k,Qe),l(Qe,zl),i(e,Bt,a),i(e,ve,a),l(ve,Ol),i(e,Vt,a),v(H,e,a),i(e,At,a),i(e,Te,a),l(Te,ql),i(e,St,a),v(F,e,a),i(e,gt,a),i(e,be,a),l(be,Kl),i(e,$t,a),v(z,e,a),i(e,Lt,a),i(e,Ie,a),l(Ie,ea),i(e,Xt,a),v(O,e,a),i(e,Zt,a),i(e,m,a),l(m,ta),l(m,Ye),l(Ye,la),l(m,aa),l(m,He),l(He,oa),l(m,na),i(e,Rt,a),i(e,Pe,a),l(Pe,sa),i(e,Wt,a),i(e,Ce,a),l(Ce,ia),i(e,xt,a),i(e,_,a),l(_,A),l(A,Fe),v(q,Fe,null),l(_,ra),l(_,ze),l(ze,pa),i(e,Dt,a),i(e,K,a),l(K,ee),l(ee,ha),l(K,ca),i(e,Qt,a),i(e,S,a),l(S,fa),l(S,te),l(te,da),l(S,wa),i(e,Yt,a),i(e,Ee,a),l(Ee,Oe),l(Oe,ua),i(e,Ht,a),i(e,w,a),l(w,ya),l(w,qe),l(qe,ma),l(w,Ma),l(w,Ke),l(Ke,Ua),l(w,va),l(w,et),l(et,Ta),l(w,ba),i(e,Ft,a),i(e,Je,a),l(Je,Ia),i(e,zt,a),i(e,ke,a),l(ke,Pa),i(e,Ot,a),i(e,g,a),l(g,tt),l(tt,le),l(le,lt),l(lt,Ca),l(le,Ea),l(le,_e),l(_e,Ja),l(g,ka),l(g,ae),l(ae,oe),l(oe,at),l(at,_a),l(oe,ja),l(oe,je),l(je,Na),l(ae,Ga),l(ae,ne),l(ne,ot),l(ot,Ba),l(ne,Va),l(ne,Ne),l(Ne,Aa),i(e,qt,a),i(e,$,a),l($,Sa),l($,nt),l(nt,ga),l($,$a),i(e,Kt,a),i(e,Ge,a),l(Ge,La),i(e,el,a),v(se,e,a),i(e,tl,a),i(e,d,a),l(d,Xa),l(d,st),l(st,Za),l(d,Ra),l(d,it),l(it,Wa),l(d,xa),l(d,rt),l(rt,Da),l(d,Qa),l(d,pt),l(pt,Ya),l(d,Ha),l(d,ht),l(ht,Fa),ll=!0},p:yn,i(e){ll||(T(X.$$.fragment,e),T(R.$$.fragment,e),T(x.$$.fragment,e),T(Y.$$.fragment,e),T(H.$$.fragment,e),T(F.$$.fragment,e),T(z.$$.fragment,e),T(O.$$.fragment,e),T(q.$$.fragment,e),T(se.$$.fragment,e),ll=!0)},o(e){b(X.$$.fragment,e),b(R.$$.fragment,e),b(x.$$.fragment,e),b(Y.$$.fragment,e),b(H.$$.fragment,e),b(F.$$.fragment,e),b(z.$$.fragment,e),b(O.$$.fragment,e),b(q.$$.fragment,e),b(se.$$.fragment,e),ll=!1},d(e){t(P),e&&t(dt),e&&t(C),I(X),e&&t(wt),e&&t(N),e&&t(ut),e&&t(ie),e&&t(yt),e&&t(E),I(R),e&&t(mt),e&&t(y),e&&t(Mt),e&&t(pe),e&&t(Ut),e&&t(J),I(x),e&&t(vt),e&&t(he),e&&t(Tt),e&&t(D),e&&t(bt),e&&t(ce),e&&t(It),e&&t(fe),e&&t(Pt),e&&t(de),e&&t(Ct),e&&t(we),e&&t(Et),e&&t(ue),e&&t(Jt),e&&t(ye),e&&t(kt),e&&t(Q),e&&t(_t),e&&t(me),e&&t(jt),e&&t(Me),e&&t(Nt),e&&t(Ue),e&&t(Gt),e&&t(k),I(Y),e&&t(Bt),e&&t(ve),e&&t(Vt),I(H,e),e&&t(At),e&&t(Te),e&&t(St),I(F,e),e&&t(gt),e&&t(be),e&&t($t),I(z,e),e&&t(Lt),e&&t(Ie),e&&t(Xt),I(O,e),e&&t(Zt),e&&t(m),e&&t(Rt),e&&t(Pe),e&&t(Wt),e&&t(Ce),e&&t(xt),e&&t(_),I(q),e&&t(Dt),e&&t(K),e&&t(Qt),e&&t(S),e&&t(Yt),e&&t(Ee),e&&t(Ht),e&&t(w),e&&t(Ft),e&&t(Je),e&&t(zt),e&&t(ke),e&&t(Ot),e&&t(g),e&&t(qt),e&&t($),e&&t(Kt),e&&t(Ge),e&&t(el),I(se,e),e&&t(tl),e&&t(d)}}}const Un={local:"custom-hardware-for-training",sections:[{local:"gpu",sections:[{local:"power-and-cooling",title:"Power and Cooling"},{local:"multigpu-connectivity",sections:[{local:"nvlink",title:"NVlink"}],title:"Multi-GPU Connectivity"}],title:"GPU"}],title:"Custom hardware for training"};function vn(Ka){return mn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pn extends fn{constructor(P){super();dn(this,P,vn,Mn,wn,{})}}export{Pn as default,Un as metadata};
