import{S as yt,i as wt,s as Tt,e as r,k as c,w as de,t as n,M as _t,c as l,d as t,m as h,a as s,x as fe,h as i,b as p,G as a,g as m,y as ue,L as bt,q as ge,o as ve,B as ye,v as Pt}from"../../chunks/vendor-hf-doc-builder.js";import{I as gt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as vt}from"../../chunks/CodeBlock-hf-doc-builder.js";function Mt(Xe){let g,Z,v,w,V,P,we,z,Te,X,y,T,x,M,_e,F,be,j,_,Pe,E,Me,Ee,Y,L,Ge,Q,B,H,Ae,K,J,ke,ee,b,$e,G,Oe,Ce,te,d,Ne,A,Le,Be,k,Je,Re,ae,R,Se,oe,$,re,S,We,ne,f,Ie,q,Ue,Ve,D,ze,xe,le,O,ie,u,Fe,C,He,qe,N,De,Ze,se;return P=new gt({}),M=new gt({}),$=new vt({props:{code:"d2dldCUyMC0tY29udGVudC1kaXNwb3NpdGlvbiUyMGh0dHBzJTNBJTJGJTJGYXBpLm5nYy5udmlkaWEuY29tJTJGdjIlMkZtb2RlbHMlMkZudmlkaWElMkZtZWdhdHJvbl9sbV8zNDVtJTJGdmVyc2lvbnMlMkZ2MC4wJTJGemlwJTIwLU8lMEFtZWdhdHJvbl9ncHQyXzM0NW1fdjBfMC56aXA=",highlighted:`wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/megatron_lm_345m/versions/v0.0/zip -O
megatron_gpt2_345m_v0_0.zip`}}),O=new vt({props:{code:"cHl0aG9uMyUyMCUyNFBBVEhfVE9fVFJBTlNGT1JNRVJTJTJGbW9kZWxzJTJGbWVnYXRyb25fZ3B0MiUyRmNvbnZlcnRfbWVnYXRyb25fZ3B0Ml9jaGVja3BvaW50LnB5JTIwbWVnYXRyb25fZ3B0Ml8zNDVtX3YwXzAuemlw",highlighted:'python3 <span class="hljs-variable">$PATH_TO_TRANSFORMERS</span>/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py megatron_gpt2_345m_v0_0.zip'}}),{c(){g=r("meta"),Z=c(),v=r("h1"),w=r("a"),V=r("span"),de(P.$$.fragment),we=c(),z=r("span"),Te=n("MegatronGPT2"),X=c(),y=r("h2"),T=r("a"),x=r("span"),de(M.$$.fragment),_e=c(),F=r("span"),be=n("Overview"),j=c(),_=r("p"),Pe=n("The MegatronGPT2 model was proposed in "),E=r("a"),Me=n(`Megatron-LM: Training Multi-Billion Parameter Language Models Using Model
Parallelism`),Ee=n(` by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley,
Jared Casper and Bryan Catanzaro.`),Y=c(),L=r("p"),Ge=n("The abstract from the paper is the following:"),Q=c(),B=r("p"),H=r("em"),Ae=n(`Recent work in language modeling demonstrates that training large transformer models advances the state of the art in
Natural Language Processing applications. However, very large models can be quite difficult to train due to memory
constraints. In this work, we present our techniques for training very large transformer models and implement a simple,
efficient intra-layer model parallel approach that enables training transformer models with billions of parameters. Our
approach does not require a new compiler or library changes, is orthogonal and complimentary to pipeline model
parallelism, and can be fully implemented with the insertion of a few communication operations in native PyTorch. We
illustrate this approach by converging transformer based models up to 8.3 billion parameters using 512 GPUs. We sustain
15.1 PetaFLOPs across the entire application with 76% scaling efficiency when compared to a strong single GPU baseline
that sustains 39 TeraFLOPs, which is 30% of peak FLOPs. To demonstrate that large language models can further advance
the state of the art (SOTA), we train an 8.3 billion parameter transformer language model similar to GPT-2 and a 3.9
billion parameter model similar to BERT. We show that careful attention to the placement of layer normalization in
BERT-like models is critical to achieving increased performance as the model size grows. Using the GPT-2 model we
achieve SOTA results on the WikiText103 (10.8 compared to SOTA perplexity of 15.8) and LAMBADA (66.5% compared to SOTA
accuracy of 63.2%) datasets. Our BERT model achieves SOTA results on the RACE dataset (90.9% compared to SOTA accuracy
of 89.4%).`),K=c(),J=r("p"),ke=n("Tips:"),ee=c(),b=r("p"),$e=n("We have provided pretrained "),G=r("a"),Oe=n("GPT2-345M"),Ce=n(` checkpoints
for use to evaluate or finetuning downstream tasks.`),te=c(),d=r("p"),Ne=n("To access these checkpoints, first "),A=r("a"),Le=n("sign up"),Be=n(` for and setup the NVIDIA GPU Cloud (NGC)
Registry CLI. Further documentation for downloading models can be found in the `),k=r("a"),Je=n("NGC documentation"),Re=n("."),ae=c(),R=r("p"),Se=n("Alternatively, you can directly download the checkpoints using:"),oe=c(),de($.$$.fragment),re=c(),S=r("p"),We=n(`Once you have obtained the checkpoint from NVIDIA GPU Cloud (NGC), you have to convert it to a format that will easily
be loaded by Hugging Face Transformers GPT2 implementation.`),ne=c(),f=r("p"),Ie=n("The following command allows you to do the conversion. We assume that the folder "),q=r("code"),Ue=n("models/megatron_gpt2"),Ve=n(` contains
`),D=r("code"),ze=n("megatron_gpt2_345m_v0_0.zip"),xe=n(" and that the command is run from that folder:"),le=c(),de(O.$$.fragment),ie=c(),u=r("p"),Fe=n("This model was contributed by "),C=r("a"),He=n("jdemouth"),qe=n(". The original code can be found "),N=r("a"),De=n("here"),Ze=n(`. That repository contains a multi-GPU and multi-node implementation of the
Megatron Language models. In particular, it contains a hybrid model parallel approach using \u201Ctensor parallel\u201D and
\u201Cpipeline parallel\u201D techniques.`),this.h()},l(e){const o=_t('[data-svelte="svelte-1phssyn"]',document.head);g=l(o,"META",{name:!0,content:!0}),o.forEach(t),Z=h(e),v=l(e,"H1",{class:!0});var me=s(v);w=l(me,"A",{id:!0,class:!0,href:!0});var je=s(w);V=l(je,"SPAN",{});var Ye=s(V);fe(P.$$.fragment,Ye),Ye.forEach(t),je.forEach(t),we=h(me),z=l(me,"SPAN",{});var Qe=s(z);Te=i(Qe,"MegatronGPT2"),Qe.forEach(t),me.forEach(t),X=h(e),y=l(e,"H2",{class:!0});var pe=s(y);T=l(pe,"A",{id:!0,class:!0,href:!0});var Ke=s(T);x=l(Ke,"SPAN",{});var et=s(x);fe(M.$$.fragment,et),et.forEach(t),Ke.forEach(t),_e=h(pe),F=l(pe,"SPAN",{});var tt=s(F);be=i(tt,"Overview"),tt.forEach(t),pe.forEach(t),j=h(e),_=l(e,"P",{});var ce=s(_);Pe=i(ce,"The MegatronGPT2 model was proposed in "),E=l(ce,"A",{href:!0,rel:!0});var at=s(E);Me=i(at,`Megatron-LM: Training Multi-Billion Parameter Language Models Using Model
Parallelism`),at.forEach(t),Ee=i(ce,` by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley,
Jared Casper and Bryan Catanzaro.`),ce.forEach(t),Y=h(e),L=l(e,"P",{});var ot=s(L);Ge=i(ot,"The abstract from the paper is the following:"),ot.forEach(t),Q=h(e),B=l(e,"P",{});var rt=s(B);H=l(rt,"EM",{});var nt=s(H);Ae=i(nt,`Recent work in language modeling demonstrates that training large transformer models advances the state of the art in
Natural Language Processing applications. However, very large models can be quite difficult to train due to memory
constraints. In this work, we present our techniques for training very large transformer models and implement a simple,
efficient intra-layer model parallel approach that enables training transformer models with billions of parameters. Our
approach does not require a new compiler or library changes, is orthogonal and complimentary to pipeline model
parallelism, and can be fully implemented with the insertion of a few communication operations in native PyTorch. We
illustrate this approach by converging transformer based models up to 8.3 billion parameters using 512 GPUs. We sustain
15.1 PetaFLOPs across the entire application with 76% scaling efficiency when compared to a strong single GPU baseline
that sustains 39 TeraFLOPs, which is 30% of peak FLOPs. To demonstrate that large language models can further advance
the state of the art (SOTA), we train an 8.3 billion parameter transformer language model similar to GPT-2 and a 3.9
billion parameter model similar to BERT. We show that careful attention to the placement of layer normalization in
BERT-like models is critical to achieving increased performance as the model size grows. Using the GPT-2 model we
achieve SOTA results on the WikiText103 (10.8 compared to SOTA perplexity of 15.8) and LAMBADA (66.5% compared to SOTA
accuracy of 63.2%) datasets. Our BERT model achieves SOTA results on the RACE dataset (90.9% compared to SOTA accuracy
of 89.4%).`),nt.forEach(t),rt.forEach(t),K=h(e),J=l(e,"P",{});var lt=s(J);ke=i(lt,"Tips:"),lt.forEach(t),ee=h(e),b=l(e,"P",{});var he=s(b);$e=i(he,"We have provided pretrained "),G=l(he,"A",{href:!0,rel:!0});var it=s(G);Oe=i(it,"GPT2-345M"),it.forEach(t),Ce=i(he,` checkpoints
for use to evaluate or finetuning downstream tasks.`),he.forEach(t),te=h(e),d=l(e,"P",{});var W=s(d);Ne=i(W,"To access these checkpoints, first "),A=l(W,"A",{href:!0,rel:!0});var st=s(A);Le=i(st,"sign up"),st.forEach(t),Be=i(W,` for and setup the NVIDIA GPU Cloud (NGC)
Registry CLI. Further documentation for downloading models can be found in the `),k=l(W,"A",{href:!0,rel:!0});var mt=s(k);Je=i(mt,"NGC documentation"),mt.forEach(t),Re=i(W,"."),W.forEach(t),ae=h(e),R=l(e,"P",{});var pt=s(R);Se=i(pt,"Alternatively, you can directly download the checkpoints using:"),pt.forEach(t),oe=h(e),fe($.$$.fragment,e),re=h(e),S=l(e,"P",{});var ct=s(S);We=i(ct,`Once you have obtained the checkpoint from NVIDIA GPU Cloud (NGC), you have to convert it to a format that will easily
be loaded by Hugging Face Transformers GPT2 implementation.`),ct.forEach(t),ne=h(e),f=l(e,"P",{});var I=s(f);Ie=i(I,"The following command allows you to do the conversion. We assume that the folder "),q=l(I,"CODE",{});var ht=s(q);Ue=i(ht,"models/megatron_gpt2"),ht.forEach(t),Ve=i(I,` contains
`),D=l(I,"CODE",{});var dt=s(D);ze=i(dt,"megatron_gpt2_345m_v0_0.zip"),dt.forEach(t),xe=i(I," and that the command is run from that folder:"),I.forEach(t),le=h(e),fe(O.$$.fragment,e),ie=h(e),u=l(e,"P",{});var U=s(u);Fe=i(U,"This model was contributed by "),C=l(U,"A",{href:!0,rel:!0});var ft=s(C);He=i(ft,"jdemouth"),ft.forEach(t),qe=i(U,". The original code can be found "),N=l(U,"A",{href:!0,rel:!0});var ut=s(N);De=i(ut,"here"),ut.forEach(t),Ze=i(U,`. That repository contains a multi-GPU and multi-node implementation of the
Megatron Language models. In particular, it contains a hybrid model parallel approach using \u201Ctensor parallel\u201D and
\u201Cpipeline parallel\u201D techniques.`),U.forEach(t),this.h()},h(){p(g,"name","hf:doc:metadata"),p(g,"content",JSON.stringify(Et)),p(w,"id","megatrongpt2"),p(w,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(w,"href","#megatrongpt2"),p(v,"class","relative group"),p(T,"id","overview"),p(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(T,"href","#overview"),p(y,"class","relative group"),p(E,"href","https://arxiv.org/abs/1909.08053"),p(E,"rel","nofollow"),p(G,"href","https://ngc.nvidia.com/catalog/models/nvidia:megatron_lm_345m"),p(G,"rel","nofollow"),p(A,"href","https://ngc.nvidia.com/signup"),p(A,"rel","nofollow"),p(k,"href","https://docs.nvidia.com/dgx/ngc-registry-cli-user-guide/index.html#topic_6_4_1"),p(k,"rel","nofollow"),p(C,"href","https://huggingface.co/jdemouth"),p(C,"rel","nofollow"),p(N,"href","https://github.com/NVIDIA/Megatron-LM"),p(N,"rel","nofollow")},m(e,o){a(document.head,g),m(e,Z,o),m(e,v,o),a(v,w),a(w,V),ue(P,V,null),a(v,we),a(v,z),a(z,Te),m(e,X,o),m(e,y,o),a(y,T),a(T,x),ue(M,x,null),a(y,_e),a(y,F),a(F,be),m(e,j,o),m(e,_,o),a(_,Pe),a(_,E),a(E,Me),a(_,Ee),m(e,Y,o),m(e,L,o),a(L,Ge),m(e,Q,o),m(e,B,o),a(B,H),a(H,Ae),m(e,K,o),m(e,J,o),a(J,ke),m(e,ee,o),m(e,b,o),a(b,$e),a(b,G),a(G,Oe),a(b,Ce),m(e,te,o),m(e,d,o),a(d,Ne),a(d,A),a(A,Le),a(d,Be),a(d,k),a(k,Je),a(d,Re),m(e,ae,o),m(e,R,o),a(R,Se),m(e,oe,o),ue($,e,o),m(e,re,o),m(e,S,o),a(S,We),m(e,ne,o),m(e,f,o),a(f,Ie),a(f,q),a(q,Ue),a(f,Ve),a(f,D),a(D,ze),a(f,xe),m(e,le,o),ue(O,e,o),m(e,ie,o),m(e,u,o),a(u,Fe),a(u,C),a(C,He),a(u,qe),a(u,N),a(N,De),a(u,Ze),se=!0},p:bt,i(e){se||(ge(P.$$.fragment,e),ge(M.$$.fragment,e),ge($.$$.fragment,e),ge(O.$$.fragment,e),se=!0)},o(e){ve(P.$$.fragment,e),ve(M.$$.fragment,e),ve($.$$.fragment,e),ve(O.$$.fragment,e),se=!1},d(e){t(g),e&&t(Z),e&&t(v),ye(P),e&&t(X),e&&t(y),ye(M),e&&t(j),e&&t(_),e&&t(Y),e&&t(L),e&&t(Q),e&&t(B),e&&t(K),e&&t(J),e&&t(ee),e&&t(b),e&&t(te),e&&t(d),e&&t(ae),e&&t(R),e&&t(oe),ye($,e),e&&t(re),e&&t(S),e&&t(ne),e&&t(f),e&&t(le),ye(O,e),e&&t(ie),e&&t(u)}}}const Et={local:"megatrongpt2",sections:[{local:"overview",title:"Overview"}],title:"MegatronGPT2"};function Gt(Xe){return Pt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ot extends yt{constructor(g){super();wt(this,g,Gt,Mt,Tt,{})}}export{Ot as default,Et as metadata};
