import{S as Ft,i as jt,s as Nt,e as o,k as p,w as te,t as s,M as Jt,c as l,d as t,m as u,a as n,x as ae,h as i,b as h,G as a,g as c,y as oe,L as Zt,q as le,o as ne,B as re,v as Bt}from"../../chunks/vendor-hf-doc-builder.js";import{I as Ue}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as Gt}from"../../chunks/CodeBlock-hf-doc-builder.js";function xt(st){let v,se,g,b,R,I,Ie,X,Ae,ie,w,k,C,A,Le,P,Fe,he,d,je,J,Ne,Je,z,Ze,Be,ce,Z,Ge,pe,f,W,xe,Se,V,qe,Re,H,Xe,ue,T,Ce,B,Pe,ze,de,E,We,L,Ve,He,fe,y,M,O,F,Oe,Y,Ye,me,$,Qe,Q,De,Ke,ve,j,ge,_,U,D,N,et,K,tt,we,m,at,ee,ot,lt,G,nt,rt,ye;return I=new Ue({}),A=new Ue({}),F=new Ue({}),j=new Gt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTS5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZmxhbi11bDIlMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZmxhbi11bDIlMjIpJTBBJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMkElMjBzdGVwJTIwYnklMjBzdGVwJTIwcmVjaXBlJTIwdG8lMjBtYWtlJTIwYm9sb2duZXNlJTIwcGFzdGElM0ElMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyklMEFwcmludCh0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;google/flan-ul2&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google/flan-ul2&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;A step by step recipe to make bolognese pasta:&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&#x27;In a large skillet, brown the ground beef and onion over medium heat. Add the garlic&#x27;</span>]`}}),N=new Ue({}),{c(){v=o("meta"),se=p(),g=o("h1"),b=o("a"),R=o("span"),te(I.$$.fragment),Ie=p(),X=o("span"),Ae=s("FLAN-UL2"),ie=p(),w=o("h2"),k=o("a"),C=o("span"),te(A.$$.fragment),Le=p(),P=o("span"),Fe=s("Overview"),he=p(),d=o("p"),je=s("Flan-UL2 is an encoder decoder model based on the T5 architecture. It uses the same configuration as the "),J=o("a"),Ne=s("UL2"),Je=s(` model released earlier last year.
It was fine tuned using the \u201CFlan\u201D prompt tuning and dataset collection. Similiar to `),z=o("code"),Ze=s("Flan-T5"),Be=s(",  one can directly use FLAN-UL2 weights without finetuning the model:"),ce=p(),Z=o("p"),Ge=s("According ot the original blog here are the notable improvements:"),pe=p(),f=o("ul"),W=o("li"),xe=s("The original UL2 model was only trained with receptive field of 512, which made it non-ideal for N-shot prompting where N is large."),Se=p(),V=o("li"),qe=s("The Flan-UL2 checkpoint uses a receptive field of 2048 which makes it more usable for few-shot in-context learning."),Re=p(),H=o("li"),Xe=s(`The original UL2 model also had mode switch tokens that was rather mandatory to get good performance. However, they were a little cumbersome as this requires often some changes during inference or finetuning. In this update/change, we continue training UL2 20B for an additional 100k steps (with small batch) to forget \u201Cmode tokens\u201D before applying Flan instruction tuning. This Flan-UL2 checkpoint does not require mode tokens anymore.
Google has released the following variants:`),ue=p(),T=o("p"),Ce=s("One can refer to "),B=o("a"),Pe=s("T5\u2019s documentation page"),ze=s(" for all tips, code examples and notebooks. As well as the FLAN-T5 model card for more details regarding training and evaluation of the model."),de=p(),E=o("p"),We=s("The original checkpoints can be found "),L=o("a"),Ve=s("here"),He=s("."),fe=p(),y=o("h2"),M=o("a"),O=o("span"),te(F.$$.fragment),Oe=p(),Y=o("span"),Ye=s("Running on low resource devices"),me=p(),$=o("p"),Qe=s("The model is pretty heavy (~40GB in half precision) so if you just want to run the model, make sure you load your model in 8bit, and use "),Q=o("code"),De=s('device_map="auto"'),Ke=s(" to make sure  you don\u2019t have any OOM issue!"),ve=p(),te(j.$$.fragment),ge=p(),_=o("h2"),U=o("a"),D=o("span"),te(N.$$.fragment),et=p(),K=o("span"),tt=s("Inference"),we=p(),m=o("p"),at=s("The inference protocol is exaclty the same as any "),ee=o("code"),ot=s("T5"),lt=s(" model, please have a look at the "),G=o("a"),nt=s("T5\u2019s documentation page"),rt=s(" for more details."),this.h()},l(e){const r=Jt('[data-svelte="svelte-1phssyn"]',document.head);v=l(r,"META",{name:!0,content:!0}),r.forEach(t),se=u(e),g=l(e,"H1",{class:!0});var _e=n(g);b=l(_e,"A",{id:!0,class:!0,href:!0});var it=n(b);R=l(it,"SPAN",{});var ht=n(R);ae(I.$$.fragment,ht),ht.forEach(t),it.forEach(t),Ie=u(_e),X=l(_e,"SPAN",{});var ct=n(X);Ae=i(ct,"FLAN-UL2"),ct.forEach(t),_e.forEach(t),ie=u(e),w=l(e,"H2",{class:!0});var be=n(w);k=l(be,"A",{id:!0,class:!0,href:!0});var pt=n(k);C=l(pt,"SPAN",{});var ut=n(C);ae(A.$$.fragment,ut),ut.forEach(t),pt.forEach(t),Le=u(be),P=l(be,"SPAN",{});var dt=n(P);Fe=i(dt,"Overview"),dt.forEach(t),be.forEach(t),he=u(e),d=l(e,"P",{});var x=n(d);je=i(x,"Flan-UL2 is an encoder decoder model based on the T5 architecture. It uses the same configuration as the "),J=l(x,"A",{href:!0});var ft=n(J);Ne=i(ft,"UL2"),ft.forEach(t),Je=i(x,` model released earlier last year.
It was fine tuned using the \u201CFlan\u201D prompt tuning and dataset collection. Similiar to `),z=l(x,"CODE",{});var mt=n(z);Ze=i(mt,"Flan-T5"),mt.forEach(t),Be=i(x,",  one can directly use FLAN-UL2 weights without finetuning the model:"),x.forEach(t),ce=u(e),Z=l(e,"P",{});var vt=n(Z);Ge=i(vt,"According ot the original blog here are the notable improvements:"),vt.forEach(t),pe=u(e),f=l(e,"UL",{});var S=n(f);W=l(S,"LI",{});var gt=n(W);xe=i(gt,"The original UL2 model was only trained with receptive field of 512, which made it non-ideal for N-shot prompting where N is large."),gt.forEach(t),Se=u(S),V=l(S,"LI",{});var wt=n(V);qe=i(wt,"The Flan-UL2 checkpoint uses a receptive field of 2048 which makes it more usable for few-shot in-context learning."),wt.forEach(t),Re=u(S),H=l(S,"LI",{});var yt=n(H);Xe=i(yt,`The original UL2 model also had mode switch tokens that was rather mandatory to get good performance. However, they were a little cumbersome as this requires often some changes during inference or finetuning. In this update/change, we continue training UL2 20B for an additional 100k steps (with small batch) to forget \u201Cmode tokens\u201D before applying Flan instruction tuning. This Flan-UL2 checkpoint does not require mode tokens anymore.
Google has released the following variants:`),yt.forEach(t),S.forEach(t),ue=u(e),T=l(e,"P",{});var ke=n(T);Ce=i(ke,"One can refer to "),B=l(ke,"A",{href:!0});var _t=n(B);Pe=i(_t,"T5\u2019s documentation page"),_t.forEach(t),ze=i(ke," for all tips, code examples and notebooks. As well as the FLAN-T5 model card for more details regarding training and evaluation of the model."),ke.forEach(t),de=u(e),E=l(e,"P",{});var Te=n(E);We=i(Te,"The original checkpoints can be found "),L=l(Te,"A",{href:!0,rel:!0});var bt=n(L);Ve=i(bt,"here"),bt.forEach(t),He=i(Te,"."),Te.forEach(t),fe=u(e),y=l(e,"H2",{class:!0});var Ee=n(y);M=l(Ee,"A",{id:!0,class:!0,href:!0});var kt=n(M);O=l(kt,"SPAN",{});var Tt=n(O);ae(F.$$.fragment,Tt),Tt.forEach(t),kt.forEach(t),Oe=u(Ee),Y=l(Ee,"SPAN",{});var Et=n(Y);Ye=i(Et,"Running on low resource devices"),Et.forEach(t),Ee.forEach(t),me=u(e),$=l(e,"P",{});var Me=n($);Qe=i(Me,"The model is pretty heavy (~40GB in half precision) so if you just want to run the model, make sure you load your model in 8bit, and use "),Q=l(Me,"CODE",{});var Mt=n(Q);De=i(Mt,'device_map="auto"'),Mt.forEach(t),Ke=i(Me," to make sure  you don\u2019t have any OOM issue!"),Me.forEach(t),ve=u(e),ae(j.$$.fragment,e),ge=u(e),_=l(e,"H2",{class:!0});var $e=n(_);U=l($e,"A",{id:!0,class:!0,href:!0});var $t=n(U);D=l($t,"SPAN",{});var Ut=n(D);ae(N.$$.fragment,Ut),Ut.forEach(t),$t.forEach(t),et=u($e),K=l($e,"SPAN",{});var It=n(K);tt=i(It,"Inference"),It.forEach(t),$e.forEach(t),we=u(e),m=l(e,"P",{});var q=n(m);at=i(q,"The inference protocol is exaclty the same as any "),ee=l(q,"CODE",{});var At=n(ee);ot=i(At,"T5"),At.forEach(t),lt=i(q," model, please have a look at the "),G=l(q,"A",{href:!0});var Lt=n(G);nt=i(Lt,"T5\u2019s documentation page"),Lt.forEach(t),rt=i(q," for more details."),q.forEach(t),this.h()},h(){h(v,"name","hf:doc:metadata"),h(v,"content",JSON.stringify(St)),h(b,"id","flanul2"),h(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(b,"href","#flanul2"),h(g,"class","relative group"),h(k,"id","overview"),h(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(k,"href","#overview"),h(w,"class","relative group"),h(J,"href","ul2"),h(B,"href","t5"),h(L,"href","https://github.com/google-research/t5x/blob/main/docs/models.md#flan-ul2-checkpoints"),h(L,"rel","nofollow"),h(M,"id","running-on-low-resource-devices"),h(M,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(M,"href","#running-on-low-resource-devices"),h(y,"class","relative group"),h(U,"id","inference"),h(U,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(U,"href","#inference"),h(_,"class","relative group"),h(G,"href","t5")},m(e,r){a(document.head,v),c(e,se,r),c(e,g,r),a(g,b),a(b,R),oe(I,R,null),a(g,Ie),a(g,X),a(X,Ae),c(e,ie,r),c(e,w,r),a(w,k),a(k,C),oe(A,C,null),a(w,Le),a(w,P),a(P,Fe),c(e,he,r),c(e,d,r),a(d,je),a(d,J),a(J,Ne),a(d,Je),a(d,z),a(z,Ze),a(d,Be),c(e,ce,r),c(e,Z,r),a(Z,Ge),c(e,pe,r),c(e,f,r),a(f,W),a(W,xe),a(f,Se),a(f,V),a(V,qe),a(f,Re),a(f,H),a(H,Xe),c(e,ue,r),c(e,T,r),a(T,Ce),a(T,B),a(B,Pe),a(T,ze),c(e,de,r),c(e,E,r),a(E,We),a(E,L),a(L,Ve),a(E,He),c(e,fe,r),c(e,y,r),a(y,M),a(M,O),oe(F,O,null),a(y,Oe),a(y,Y),a(Y,Ye),c(e,me,r),c(e,$,r),a($,Qe),a($,Q),a(Q,De),a($,Ke),c(e,ve,r),oe(j,e,r),c(e,ge,r),c(e,_,r),a(_,U),a(U,D),oe(N,D,null),a(_,et),a(_,K),a(K,tt),c(e,we,r),c(e,m,r),a(m,at),a(m,ee),a(ee,ot),a(m,lt),a(m,G),a(G,nt),a(m,rt),ye=!0},p:Zt,i(e){ye||(le(I.$$.fragment,e),le(A.$$.fragment,e),le(F.$$.fragment,e),le(j.$$.fragment,e),le(N.$$.fragment,e),ye=!0)},o(e){ne(I.$$.fragment,e),ne(A.$$.fragment,e),ne(F.$$.fragment,e),ne(j.$$.fragment,e),ne(N.$$.fragment,e),ye=!1},d(e){t(v),e&&t(se),e&&t(g),re(I),e&&t(ie),e&&t(w),re(A),e&&t(he),e&&t(d),e&&t(ce),e&&t(Z),e&&t(pe),e&&t(f),e&&t(ue),e&&t(T),e&&t(de),e&&t(E),e&&t(fe),e&&t(y),re(F),e&&t(me),e&&t($),e&&t(ve),re(j,e),e&&t(ge),e&&t(_),re(N),e&&t(we),e&&t(m)}}}const St={local:"flanul2",sections:[{local:"overview",title:"Overview"},{local:"running-on-low-resource-devices",title:"Running on low resource devices"},{local:"inference",title:"Inference"}],title:"FLAN-UL2"};function qt(st){return Bt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pt extends Ft{constructor(v){super();jt(this,v,qt,xt,Nt,{})}}export{Pt as default,St as metadata};
