import{S as Ut,i as Nt,s as jt,e as a,k as c,w as te,t as l,M as xt,c as n,d as t,m as u,a as r,x as oe,h as i,b as h,G as o,g as p,y as ae,L as Pt,q as ne,o as re,B as se,v as It}from"../../chunks/vendor-hf-doc-builder.js";import{I as Te}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as Mt}from"../../chunks/CodeBlock-hf-doc-builder.js";function Ot(lt){let v,le,g,y,B,q,qe,H,Fe,ie,w,b,G,F,Se,R,Ue,he,f,Ne,x,je,xe,D,Pe,Ie,pe,P,Me,ce,d,J,Oe,ze,K,Ce,Be,Q,He,ue,$,Ge,I,Re,De,fe,A,Je,S,Ke,Qe,de,_,E,V,U,Ve,W,We,me,L,Xe,X,Ye,Ze,ve,N,ge,k,T,Y,j,et,Z,tt,we,m,ot,ee,at,nt,M,rt,st,_e;return q=new Te({}),F=new Te({}),U=new Te({}),N=new Mt({props:{code:`from transformers import AutoModelForSeq2SeqLM, AutoTokenizer

model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-ul2", load_in_8bit=True, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained("google/flan-ul2")

inputs = tokenizer("A step by step recipe to make bolognese pasta:", return_tensors="pt")
outputs = model.generate(**inputs)
print(tokenizer.batch_decode(outputs, skip_special_tokens=True))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;google/flan-ul2&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google/flan-ul2&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;A step by step recipe to make bolognese pasta:&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&#x27;In a large skillet, brown the ground beef and onion over medium heat. Add the garlic&#x27;</span>]`}}),j=new Te({}),{c(){v=a("meta"),le=c(),g=a("h1"),y=a("a"),B=a("span"),te(q.$$.fragment),qe=c(),H=a("span"),Fe=l("FLAN-UL2"),ie=c(),w=a("h2"),b=a("a"),G=a("span"),te(F.$$.fragment),Se=c(),R=a("span"),Ue=l("Overview"),he=c(),f=a("p"),Ne=l("Flan-UL2 is an encoder decoder model based on the T5 architecture. It uses the same configuration as the "),x=a("a"),je=l("UL2"),xe=l(` model released earlier last year.
It was fine tuned using the \u201CFlan\u201D prompt tuning and dataset collection. Similiar to `),D=a("code"),Pe=l("Flan-T5"),Ie=l(",  one can directly use FLAN-UL2 weights without finetuning the model:"),pe=c(),P=a("p"),Me=l("According ot the original blog here are the notable improvements:"),ce=c(),d=a("ul"),J=a("li"),Oe=l("The original UL2 model was only trained with receptive field of 512, which made it non-ideal for N-shot prompting where N is large."),ze=c(),K=a("li"),Ce=l("The Flan-UL2 checkpoint uses a receptive field of 2048 which makes it more usable for few-shot in-context learning."),Be=c(),Q=a("li"),He=l(`The original UL2 model also had mode switch tokens that was rather mandatory to get good performance. However, they were a little cumbersome as this requires often some changes during inference or finetuning. In this update/change, we continue training UL2 20B for an additional 100k steps (with small batch) to forget \u201Cmode tokens\u201D before applying Flan instruction tuning. This Flan-UL2 checkpoint does not require mode tokens anymore.
Google has released the following variants:`),ue=c(),$=a("p"),Ge=l("One can refer to "),I=a("a"),Re=l("T5\u2019s documentation page"),De=l(" for all tips, code examples and notebooks. As well as the FLAN-T5 model card for more details regarding training and evaluation of the model."),fe=c(),A=a("p"),Je=l("The original checkpoints can be found "),S=a("a"),Ke=l("here"),Qe=l("."),de=c(),_=a("h2"),E=a("a"),V=a("span"),te(U.$$.fragment),Ve=c(),W=a("span"),We=l("Running on low resource devices"),me=c(),L=a("p"),Xe=l("The model is pretty heavy (~40GB in half precision) so if you just want to run the model, make sure you load your model in 8bit, and use "),X=a("code"),Ye=l('device_map="auto"'),Ze=l(" to make sure  you don\u2019t have any OOM issue!"),ve=c(),te(N.$$.fragment),ge=c(),k=a("h2"),T=a("a"),Y=a("span"),te(j.$$.fragment),et=c(),Z=a("span"),tt=l("Inference"),we=c(),m=a("p"),ot=l("The inference protocol is exaclty the same as any "),ee=a("code"),at=l("T5"),nt=l(" model, please have a look at the "),M=a("a"),rt=l("T5\u2019s documentation page"),st=l(" for more details."),this.h()},l(e){const s=xt('[data-svelte="svelte-1phssyn"]',document.head);v=n(s,"META",{name:!0,content:!0}),s.forEach(t),le=u(e),g=n(e,"H1",{class:!0});var ke=r(g);y=n(ke,"A",{id:!0,class:!0,href:!0});var it=r(y);B=n(it,"SPAN",{});var ht=r(B);oe(q.$$.fragment,ht),ht.forEach(t),it.forEach(t),qe=u(ke),H=n(ke,"SPAN",{});var pt=r(H);Fe=i(pt,"FLAN-UL2"),pt.forEach(t),ke.forEach(t),ie=u(e),w=n(e,"H2",{class:!0});var ye=r(w);b=n(ye,"A",{id:!0,class:!0,href:!0});var ct=r(b);G=n(ct,"SPAN",{});var ut=r(G);oe(F.$$.fragment,ut),ut.forEach(t),ct.forEach(t),Se=u(ye),R=n(ye,"SPAN",{});var ft=r(R);Ue=i(ft,"Overview"),ft.forEach(t),ye.forEach(t),he=u(e),f=n(e,"P",{});var O=r(f);Ne=i(O,"Flan-UL2 is an encoder decoder model based on the T5 architecture. It uses the same configuration as the "),x=n(O,"A",{href:!0});var dt=r(x);je=i(dt,"UL2"),dt.forEach(t),xe=i(O,` model released earlier last year.
It was fine tuned using the \u201CFlan\u201D prompt tuning and dataset collection. Similiar to `),D=n(O,"CODE",{});var mt=r(D);Pe=i(mt,"Flan-T5"),mt.forEach(t),Ie=i(O,",  one can directly use FLAN-UL2 weights without finetuning the model:"),O.forEach(t),pe=u(e),P=n(e,"P",{});var vt=r(P);Me=i(vt,"According ot the original blog here are the notable improvements:"),vt.forEach(t),ce=u(e),d=n(e,"UL",{});var z=r(d);J=n(z,"LI",{});var gt=r(J);Oe=i(gt,"The original UL2 model was only trained with receptive field of 512, which made it non-ideal for N-shot prompting where N is large."),gt.forEach(t),ze=u(z),K=n(z,"LI",{});var wt=r(K);Ce=i(wt,"The Flan-UL2 checkpoint uses a receptive field of 2048 which makes it more usable for few-shot in-context learning."),wt.forEach(t),Be=u(z),Q=n(z,"LI",{});var _t=r(Q);He=i(_t,`The original UL2 model also had mode switch tokens that was rather mandatory to get good performance. However, they were a little cumbersome as this requires often some changes during inference or finetuning. In this update/change, we continue training UL2 20B for an additional 100k steps (with small batch) to forget \u201Cmode tokens\u201D before applying Flan instruction tuning. This Flan-UL2 checkpoint does not require mode tokens anymore.
Google has released the following variants:`),_t.forEach(t),z.forEach(t),ue=u(e),$=n(e,"P",{});var be=r($);Ge=i(be,"One can refer to "),I=n(be,"A",{href:!0});var kt=r(I);Re=i(kt,"T5\u2019s documentation page"),kt.forEach(t),De=i(be," for all tips, code examples and notebooks. As well as the FLAN-T5 model card for more details regarding training and evaluation of the model."),be.forEach(t),fe=u(e),A=n(e,"P",{});var $e=r(A);Je=i($e,"The original checkpoints can be found "),S=n($e,"A",{href:!0,rel:!0});var yt=r(S);Ke=i(yt,"here"),yt.forEach(t),Qe=i($e,"."),$e.forEach(t),de=u(e),_=n(e,"H2",{class:!0});var Ae=r(_);E=n(Ae,"A",{id:!0,class:!0,href:!0});var bt=r(E);V=n(bt,"SPAN",{});var $t=r(V);oe(U.$$.fragment,$t),$t.forEach(t),bt.forEach(t),Ve=u(Ae),W=n(Ae,"SPAN",{});var At=r(W);We=i(At,"Running on low resource devices"),At.forEach(t),Ae.forEach(t),me=u(e),L=n(e,"P",{});var Ee=r(L);Xe=i(Ee,"The model is pretty heavy (~40GB in half precision) so if you just want to run the model, make sure you load your model in 8bit, and use "),X=n(Ee,"CODE",{});var Et=r(X);Ye=i(Et,'device_map="auto"'),Et.forEach(t),Ze=i(Ee," to make sure  you don\u2019t have any OOM issue!"),Ee.forEach(t),ve=u(e),oe(N.$$.fragment,e),ge=u(e),k=n(e,"H2",{class:!0});var Le=r(k);T=n(Le,"A",{id:!0,class:!0,href:!0});var Lt=r(T);Y=n(Lt,"SPAN",{});var Tt=r(Y);oe(j.$$.fragment,Tt),Tt.forEach(t),Lt.forEach(t),et=u(Le),Z=n(Le,"SPAN",{});var qt=r(Z);tt=i(qt,"Inference"),qt.forEach(t),Le.forEach(t),we=u(e),m=n(e,"P",{});var C=r(m);ot=i(C,"The inference protocol is exaclty the same as any "),ee=n(C,"CODE",{});var Ft=r(ee);at=i(Ft,"T5"),Ft.forEach(t),nt=i(C," model, please have a look at the "),M=n(C,"A",{href:!0});var St=r(M);rt=i(St,"T5\u2019s documentation page"),St.forEach(t),st=i(C," for more details."),C.forEach(t),this.h()},h(){h(v,"name","hf:doc:metadata"),h(v,"content",JSON.stringify(zt)),h(y,"id","flanul2"),h(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(y,"href","#flanul2"),h(g,"class","relative group"),h(b,"id","overview"),h(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(b,"href","#overview"),h(w,"class","relative group"),h(x,"href","ul2"),h(I,"href","t5"),h(S,"href","https://github.com/google-research/t5x/blob/main/docs/models.md#flan-ul2-checkpoints"),h(S,"rel","nofollow"),h(E,"id","running-on-low-resource-devices"),h(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(E,"href","#running-on-low-resource-devices"),h(_,"class","relative group"),h(T,"id","inference"),h(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(T,"href","#inference"),h(k,"class","relative group"),h(M,"href","t5")},m(e,s){o(document.head,v),p(e,le,s),p(e,g,s),o(g,y),o(y,B),ae(q,B,null),o(g,qe),o(g,H),o(H,Fe),p(e,ie,s),p(e,w,s),o(w,b),o(b,G),ae(F,G,null),o(w,Se),o(w,R),o(R,Ue),p(e,he,s),p(e,f,s),o(f,Ne),o(f,x),o(x,je),o(f,xe),o(f,D),o(D,Pe),o(f,Ie),p(e,pe,s),p(e,P,s),o(P,Me),p(e,ce,s),p(e,d,s),o(d,J),o(J,Oe),o(d,ze),o(d,K),o(K,Ce),o(d,Be),o(d,Q),o(Q,He),p(e,ue,s),p(e,$,s),o($,Ge),o($,I),o(I,Re),o($,De),p(e,fe,s),p(e,A,s),o(A,Je),o(A,S),o(S,Ke),o(A,Qe),p(e,de,s),p(e,_,s),o(_,E),o(E,V),ae(U,V,null),o(_,Ve),o(_,W),o(W,We),p(e,me,s),p(e,L,s),o(L,Xe),o(L,X),o(X,Ye),o(L,Ze),p(e,ve,s),ae(N,e,s),p(e,ge,s),p(e,k,s),o(k,T),o(T,Y),ae(j,Y,null),o(k,et),o(k,Z),o(Z,tt),p(e,we,s),p(e,m,s),o(m,ot),o(m,ee),o(ee,at),o(m,nt),o(m,M),o(M,rt),o(m,st),_e=!0},p:Pt,i(e){_e||(ne(q.$$.fragment,e),ne(F.$$.fragment,e),ne(U.$$.fragment,e),ne(N.$$.fragment,e),ne(j.$$.fragment,e),_e=!0)},o(e){re(q.$$.fragment,e),re(F.$$.fragment,e),re(U.$$.fragment,e),re(N.$$.fragment,e),re(j.$$.fragment,e),_e=!1},d(e){t(v),e&&t(le),e&&t(g),se(q),e&&t(ie),e&&t(w),se(F),e&&t(he),e&&t(f),e&&t(pe),e&&t(P),e&&t(ce),e&&t(d),e&&t(ue),e&&t($),e&&t(fe),e&&t(A),e&&t(de),e&&t(_),se(U),e&&t(me),e&&t(L),e&&t(ve),se(N,e),e&&t(ge),e&&t(k),se(j),e&&t(we),e&&t(m)}}}const zt={local:"flanul2",sections:[{local:"overview",title:"Overview"},{local:"running-on-low-resource-devices",title:"Running on low resource devices"},{local:"inference",title:"Inference"}],title:"FLAN-UL2"};function Ct(lt){return It(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Rt extends Ut{constructor(v){super();Nt(this,v,Ct,Ot,jt,{})}}export{Rt as default,zt as metadata};
