import{S as Vr,i as Ar,s as Rr,e as s,k as u,w as d,t as n,M as Xr,c as l,d as t,m as c,a as o,x as f,h as r,b as h,f as Ws,N as Wr,G as a,g as p,y as m,q as g,o as y,B as b,v as zr}from"../chunks/vendor-hf-doc-builder.js";import{T as Cr}from"../chunks/Tip-hf-doc-builder.js";import{I}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as _e}from"../chunks/CodeBlock-hf-doc-builder.js";import{D as Br}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";function Nr(ya){let v,Y,w,k,V,U,$,A,ke;return{c(){v=s("p"),Y=n("If you\u2019re interested in basic LLM usage, our high-level "),w=s("a"),k=s("code"),V=n("Pipeline"),U=n(" interface is a great starting point. However, LLMs often require advanced features like quantization and fine control of the token selection step, which is best done through "),$=s("a"),A=n("generate()"),ke=n(". Autoregressive generation with LLMs is also resource-intensive and should be executed on a GPU for adequate throughput."),this.h()},l(G){v=l(G,"P",{});var M=o(v);Y=r(M,"If you\u2019re interested in basic LLM usage, our high-level "),w=l(M,"A",{href:!0});var je=o(w);k=l(je,"CODE",{});var R=o(k);V=r(R,"Pipeline"),R.forEach(t),je.forEach(t),U=r(M," interface is a great starting point. However, LLMs often require advanced features like quantization and fine control of the token selection step, which is best done through "),$=l(M,"A",{href:!0});var Ke=o($);A=r(Ke,"generate()"),Ke.forEach(t),ke=r(M,". Autoregressive generation with LLMs is also resource-intensive and should be executed on a GPU for adequate throughput."),M.forEach(t),this.h()},h(){h(w,"href","pipeline_tutorial"),h($,"href","/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate")},m(G,M){p(G,v,M),a(v,Y),a(v,w),a(w,k),a(k,V),a(v,U),a(v,$),a($,A),a(v,ke)},d(G){G&&t(v)}}}function Fr(ya){let v,Y,w,k,V,U,$,A,ke,G,M,je,R,Ke,ba,P,Vs,et,As,Rs,va,tt,Xs,wa,L,Lt,zs,Cs,xt,Bs,Ns,Wt,Fs,Ma,at,qs,_a,Je,ka,X,D,Vt,Ee,Ss,At,Qs,ja,O,Hs,st,Ys,Ps,Ja,Te,j,Ds,Rt,Os,an,Ea,lt,Ks,Ta,Ue,J,el,Xt,tl,sn,Ua,K,al,zt,sl,ll,Za,ee,ol,ot,nl,rl,Ia,nt,il,$a,te,Ga,rt,pl,La,Ze,xa,ae,hl,Ct,ul,cl,Wa,se,it,Bt,dl,fl,ml,le,Nt,gl,yl,pt,bl,vl,Va,ht,wl,Aa,oe,Ml,ut,_l,kl,Ra,Ie,Xa,x,jl,Ft,Jl,El,ct,Tl,Ul,za,ne,Zl,dt,Il,$l,Ca,$e,Ba,ft,Gl,Na,z,re,qt,Ge,Ll,St,xl,Fa,ie,Wl,mt,Vl,Al,qa,Le,Sa,C,pe,Qt,xe,Rl,Ht,Xl,Qa,_,zl,gt,Cl,Bl,Yt,Nl,Fl,Pt,ql,Sl,Dt,Ql,Hl,We,Yl,Pl,Ha,Ve,Ya,B,he,Ot,Ae,Dl,Kt,Ol,Pa,E,Kl,yt,eo,to,ea,ao,so,ta,lo,oo,Re,no,ro,Da,Xe,Oa,N,ue,aa,ze,io,sa,po,Ka,ce,ho,Ce,uo,co,es,Be,ts,F,de,la,Ne,fo,oa,mo,as,bt,go,ss,q,fe,na,Fe,yo,ra,bo,ls,me,vt,wt,vo,wo,Mo,Z,_o,Mt,ko,jo,_t,Jo,Eo,kt,To,Uo,os,S,ge,ia,qe,Zo,pa,Io,ns,ye,jt,Se,$o,Go,Lo,Jt,Qe,xo,Wo,rs,Q,be,ha,He,Vo,ua,Ao,is,Et,Tt,Ut,Ro,Xo,ps,H,ve,ca,Ye,zo,da,Co,hs,we,Zt,Pe,fa,Bo,No,Fo,It,De,ma,qo,So,us;return U=new I({}),M=new Br({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/llm_tutorial.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/llm_tutorial.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/llm_tutorial.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/llm_tutorial.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/llm_tutorial.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/llm_tutorial.ipynb"}]}}),Je=new _e({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGJpdHNhbmRieXRlcyUzRSUzRDAuMzkuMCUyMC1x",highlighted:"pip install transformers bitsandbytes&gt;=0.39.0 -q"}}),Ee=new I({}),te=new Cr({props:{$$slots:{default:[Nr]},$$scope:{ctx:ya}}}),Ze=new _e({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyb3BlbmxtLXJlc2VhcmNoJTJGb3Blbl9sbGFtYV83YiUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;openlm-research/open_llama_7b&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>
<span class="hljs-meta">... </span>)`}}),Ie=new _e({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJvcGVubG0tcmVzZWFyY2glMkZvcGVuX2xsYW1hXzdiJTIyKSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplciglNUIlMjJBJTIwbGlzdCUyMG9mJTIwY29sb3JzJTNBJTIwcmVkJTJDJTIwYmx1ZSUyMiU1RCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;openlm-research/open_llama_7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model_inputs = tokenizer([<span class="hljs-string">&quot;A list of colors: red, blue&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)`}}),$e=new _e({props:{code:"Z2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqbW9kZWxfaW5wdXRzKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUoZ2VuZXJhdGVkX2lkcyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**model_inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;A list of colors: red, blue, green, yellow, black, white, and brown&#x27;</span>`}}),Ge=new I({}),Le=new _e({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm9wZW5sbS1yZXNlYXJjaCUyRm9wZW5fbGxhbWFfN2IlMjIpJTBBdG9rZW5pemVyLnBhZF90b2tlbiUyMCUzRCUyMHRva2VuaXplci5lb3NfdG9rZW4lMjAlMjAlMjMlMjBMbGFtYSUyMGhhcyUyMG5vJTIwcGFkJTIwdG9rZW4lMjBieSUyMGRlZmF1bHQlMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJvcGVubG0tcmVzZWFyY2glMkZvcGVuX2xsYW1hXzdiJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;openlm-research/open_llama_7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token  <span class="hljs-comment"># Llama has no pad token by default</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;openlm-research/open_llama_7b&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>
<span class="hljs-meta">... </span>)`}}),xe=new I({}),Ve=new _e({props:{code:"bW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCU1QiUyMkElMjBzZXF1ZW5jZSUyMG9mJTIwbnVtYmVycyUzQSUyMDElMkMlMjAyJTIyJTVEJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oJTIyY3VkYSUyMiklMEElMEElMjMlMjBCeSUyMGRlZmF1bHQlMkMlMjB0aGUlMjBvdXRwdXQlMjB3aWxsJTIwY29udGFpbiUyMHVwJTIwdG8lMjAyMCUyMHRva2VucyUwQWdlbmVyYXRlZF9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKm1vZGVsX2lucHV0cyklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKGdlbmVyYXRlZF9pZHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklNUIwJTVEJTBBJTBBJTIzJTIwU2V0dGluZyUyMCU2MG1heF9uZXdfdG9rZW5zJTYwJTIwYWxsb3dzJTIweW91JTIwdG8lMjBjb250cm9sJTIwdGhlJTIwbWF4aW11bSUyMGxlbmd0aCUwQWdlbmVyYXRlZF9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKm1vZGVsX2lucHV0cyUyQyUyMG1heF9uZXdfdG9rZW5zJTNENTApJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShnZW5lcmF0ZWRfaWRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>model_inputs = tokenizer([<span class="hljs-string">&quot;A sequence of numbers: 1, 2&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># By default, the output will contain up to 20 tokens</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**model_inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;A sequence of numbers: 1, 2, 3, 4, 5&#x27;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Setting \`max_new_tokens\` allows you to control the maximum length</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**model_inputs, max_new_tokens=<span class="hljs-number">50</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;A sequence of numbers: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,&#x27;</span>`}}),Ae=new I({}),Xe=new _e({props:{code:"JTIzJTIwU2V0JTIwc2VlZCUyMG9yJTIwcmVwcm9kdWNpYmlsaXR5JTIwLS0lMjB5b3UlMjBkb24ndCUyMG5lZWQlMjB0aGlzJTIwdW5sZXNzJTIweW91JTIwd2FudCUyMGZ1bGwlMjByZXByb2R1Y2liaWxpdHklMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwc2V0X3NlZWQlMEFzZXRfc2VlZCgwKSUwQSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplciglNUIlMjJJJTIwYW0lMjBhJTIwY2F0LiUyMiU1RCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTIzJTIwTExNJTIwJTJCJTIwZ3JlZWR5JTIwZGVjb2RpbmclMjAlM0QlMjByZXBldGl0aXZlJTJDJTIwYm9yaW5nJTIwb3V0cHV0JTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqbW9kZWxfaW5wdXRzKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUoZ2VuZXJhdGVkX2lkcyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQlMEElMEElMjMlMjBXaXRoJTIwc2FtcGxpbmclMkMlMjB0aGUlMjBvdXRwdXQlMjBiZWNvbWVzJTIwbW9yZSUyMGNyZWF0aXZlISUwQWdlbmVyYXRlZF9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKm1vZGVsX2lucHV0cyUyQyUyMGRvX3NhbXBsZSUzRFRydWUpJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShnZW5lcmF0ZWRfaWRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Set seed or reproducibility -- you don&#x27;t need this unless you want full reproducibility</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> set_seed
<span class="hljs-meta">&gt;&gt;&gt; </span>set_seed(<span class="hljs-number">0</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model_inputs = tokenizer([<span class="hljs-string">&quot;I am a cat.&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># LLM + greedy decoding = repetitive, boring output</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**model_inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;I am a cat. I am a cat. I am a cat. I am a cat&#x27;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># With sampling, the output becomes more creative!</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**model_inputs, do_sample=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;I am a cat.\\nI just need to be. I am always.\\nEvery time&#x27;</span>`}}),ze=new I({}),Be=new _e({props:{code:"JTIzJTIwVGhlJTIwdG9rZW5pemVyJTIwaW5pdGlhbGl6ZWQlMjBhYm92ZSUyMGhhcyUyMHJpZ2h0LXBhZGRpbmclMjBhY3RpdmUlMjBieSUyMGRlZmF1bHQlM0ElMjB0aGUlMjAxc3QlMjBzZXF1ZW5jZSUyQyUwQSUyMyUyMHdoaWNoJTIwaXMlMjBzaG9ydGVyJTJDJTIwaGFzJTIwcGFkZGluZyUyMG9uJTIwdGhlJTIwcmlnaHQlMjBzaWRlLiUyMEdlbmVyYXRpb24lMjBmYWlscy4lMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTBBJTIwJTIwJTIwJTIwJTVCJTIyMSUyQyUyMDIlMkMlMjAzJTIyJTJDJTIwJTIyQSUyQyUyMEIlMkMlMjBDJTJDJTIwRCUyQyUyMEUlMjIlNUQlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMEEpLnRvKCUyMmN1ZGElMjIpJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqbW9kZWxfaW5wdXRzKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUoZ2VuZXJhdGVkX2lkcyU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklNUIwJTVEJTBBJTBBJTIzJTIwV2l0aCUyMGxlZnQtcGFkZGluZyUyQyUyMGl0JTIwd29ya3MlMjBhcyUyMGV4cGVjdGVkISUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm9wZW5sbS1yZXNlYXJjaCUyRm9wZW5fbGxhbWFfN2IlMjIlMkMlMjBwYWRkaW5nX3NpZGUlM0QlMjJsZWZ0JTIyKSUwQXRva2VuaXplci5wYWRfdG9rZW4lMjAlM0QlMjB0b2tlbml6ZXIuZW9zX3Rva2VuJTIwJTIwJTIzJTIwTGxhbWElMjBoYXMlMjBubyUyMHBhZCUyMHRva2VuJTIwYnklMjBkZWZhdWx0JTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUwQSUyMCUyMCUyMCUyMCU1QiUyMjElMkMlMjAyJTJDJTIwMyUyMiUyQyUyMCUyMkElMkMlMjBCJTJDJTIwQyUyQyUyMEQlMkMlMjBFJTIyJTVEJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyJTBBKS50byglMjJjdWRhJTIyKSUwQWdlbmVyYXRlZF9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKm1vZGVsX2lucHV0cyklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKGdlbmVyYXRlZF9pZHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklNUIwJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># The tokenizer initialized above has right-padding active by default: the 1st sequence,</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># which is shorter, has padding on the right side. Generation fails.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model_inputs = tokenizer(
<span class="hljs-meta">... </span>    [<span class="hljs-string">&quot;1, 2, 3&quot;</span>, <span class="hljs-string">&quot;A, B, C, D, E&quot;</span>], padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
<span class="hljs-meta">... </span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**model_inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generated_ids[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;&#x27;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># With left-padding, it works as expected!</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;openlm-research/open_llama_7b&quot;</span>, padding_side=<span class="hljs-string">&quot;left&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token  <span class="hljs-comment"># Llama has no pad token by default</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model_inputs = tokenizer(
<span class="hljs-meta">... </span>    [<span class="hljs-string">&quot;1, 2, 3&quot;</span>, <span class="hljs-string">&quot;A, B, C, D, E&quot;</span>], padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
<span class="hljs-meta">... </span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**model_inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;1, 2, 3, 4, 5, 6,&#x27;</span>`}}),Ne=new I({}),Fe=new I({}),qe=new I({}),He=new I({}),Ye=new I({}),{c(){v=s("meta"),Y=u(),w=s("h1"),k=s("a"),V=s("span"),d(U.$$.fragment),$=u(),A=s("span"),ke=n("Generation with LLMs"),G=u(),d(M.$$.fragment),je=u(),R=s("p"),Ke=n("LLMs, or Large Language Models, are the key component behind text generation. In a nutshell, they consist of large pretrained transformer models trained to predict the next word (or, more precisely, token) given some input text. Since they predict one token at a time, you need to do something more elaborate to generate new sentences other than just calling the model \u2014 you need to do autoregressive generation."),ba=u(),P=s("p"),Vs=n("Autoregressive generation is the inference-time procedure of iteratively calling a model with its own generated outputs, given a few initial inputs. In \u{1F917} Transformers, this is handled by the "),et=s("a"),As=n("generate()"),Rs=n(" method, which is available to all models with generative capabilities."),va=u(),tt=s("p"),Xs=n("This tutorial will show you how to:"),wa=u(),L=s("ul"),Lt=s("li"),zs=n("Generate text with an LLM"),Cs=u(),xt=s("li"),Bs=n("Avoid common pitfalls"),Ns=u(),Wt=s("li"),Fs=n("Next steps to help you get the most out your LLM"),Ma=u(),at=s("p"),qs=n("Before you begin, make sure you have all the necessary libraries installed:"),_a=u(),d(Je.$$.fragment),ka=u(),X=s("h2"),D=s("a"),Vt=s("span"),d(Ee.$$.fragment),Ss=u(),At=s("span"),Qs=n("Generate text"),ja=u(),O=s("p"),Hs=n("A language model trained for "),st=s("a"),Ys=n("causal language modeling"),Ps=n(" takes a sequence of text tokens as input and returns the probability distribution for the next token."),Ja=u(),Te=s("figure"),j=s("video"),Ds=n(`&lt;/video>
    `),Rt=s("figcaption"),Os=n('"Forward pass of an LLM"'),Ea=u(),lt=s("p"),Ks=n("A critical aspect of autoregressive generation with LLMs is how to select the next token from this probability distribution. Anything goes in this step as long as you end up with a token for the next iteration. This means it can be as simple as selecting the most likely token from the probability distribution or as complex as applying a dozen transformations before sampling from the resulting distribution."),Ta=u(),Ue=s("figure"),J=s("video"),el=n(`&lt;/video>
    `),Xt=s("figcaption"),tl=n('"Autoregressive generation iteratively selects the next token from a probability distribution to generate text"'),Ua=u(),K=s("p"),al=n("The process depicted above is repeated iteratively until some stopping condition is reached. Ideally, the stopping condition is dictated by the model, which should learn when to output an end-of-sequence ("),zt=s("code"),sl=n("EOS"),ll=n(") token. If this is not the case, generation stops when some predefined maximum length is reached."),Za=u(),ee=s("p"),ol=n("Properly setting up the token selection step and the stopping condition is essential to make your model behave as you\u2019d expect on your task. That is why we have a "),ot=s("a"),nl=n("GenerationConfig"),rl=n(" file associated with each model, which contains a good default generative parameterization and is loaded alongside your model."),Ia=u(),nt=s("p"),il=n("Let\u2019s talk code!"),$a=u(),d(te.$$.fragment),Ga=u(),rt=s("p"),pl=n("First, you need to load the model."),La=u(),d(Ze.$$.fragment),xa=u(),ae=s("p"),hl=n("You\u2019ll notice two flags in the "),Ct=s("code"),ul=n("from_pretrained"),cl=n(" call:"),Wa=u(),se=s("ul"),it=s("li"),Bt=s("code"),dl=n("device_map"),fl=n(" ensures the model is moved to your GPU(s)"),ml=u(),le=s("li"),Nt=s("code"),gl=n("load_in_4bit"),yl=n(" applies "),pt=s("a"),bl=n("4-bit dynamic quantization"),vl=n(" to massively reduce the resource requirements"),Va=u(),ht=s("p"),wl=n("There are other ways to initialize a model, but this is a good baseline to begin with an LLM."),Aa=u(),oe=s("p"),Ml=n("Next, you need to preprocess your text input with a "),ut=s("a"),_l=n("tokenizer"),kl=n("."),Ra=u(),d(Ie.$$.fragment),Xa=u(),x=s("p"),jl=n("The "),Ft=s("code"),Jl=n("model_inputs"),El=n(" variable holds the tokenized text input, as well as the attention mask. While "),ct=s("a"),Tl=n("generate()"),Ul=n(" does its best effort to infer the attention mask when it is not passed, we recommend passing it whenever possible for optimal results."),za=u(),ne=s("p"),Zl=n("Finally, call the "),dt=s("a"),Il=n("generate()"),$l=n(" method to returns the generated tokens, which should be converted to text before printing."),Ca=u(),d($e.$$.fragment),Ba=u(),ft=s("p"),Gl=n("And that\u2019s it! In a few lines of code, you can harness the power of an LLM."),Na=u(),z=s("h2"),re=s("a"),qt=s("span"),d(Ge.$$.fragment),Ll=u(),St=s("span"),xl=n("Common pitfalls"),Fa=u(),ie=s("p"),Wl=n("There are many "),mt=s("a"),Vl=n("generation strategies"),Al=n(", and sometimes the default values may not be appropriate for your use case. If your outputs aren\u2019t aligned with what you\u2019re expecting, we\u2019ve created a list of the most common pitfalls and how to avoid them."),qa=u(),d(Le.$$.fragment),Sa=u(),C=s("h3"),pe=s("a"),Qt=s("span"),d(xe.$$.fragment),Rl=u(),Ht=s("span"),Xl=n("Generated output is too short/long"),Qa=u(),_=s("p"),zl=n("If not specified in the "),gt=s("a"),Cl=n("GenerationConfig"),Bl=n(" file, "),Yt=s("code"),Nl=n("generate"),Fl=n(" returns up to 20 tokens by default. We highly recommend manually setting "),Pt=s("code"),ql=n("max_new_tokens"),Sl=n(" in your "),Dt=s("code"),Ql=n("generate"),Hl=n(" call to control the maximum number of new tokens it can return. Keep in mind LLMs (more precisely, "),We=s("a"),Yl=n("decoder-only models"),Pl=n(") also return the input prompt as part of the output."),Ha=u(),d(Ve.$$.fragment),Ya=u(),B=s("h3"),he=s("a"),Ot=s("span"),d(Ae.$$.fragment),Dl=u(),Kt=s("span"),Ol=n("Incorrect generation mode"),Pa=u(),E=s("p"),Kl=n("By default, and unless specified in the "),yt=s("a"),eo=n("GenerationConfig"),to=n(" file, "),ea=s("code"),ao=n("generate"),so=n(" selects the most likely token at each iteration (greedy decoding). Depending on your task, this may be undesirable; creative tasks like chatbots or writing an essay benefit from sampling. On the other hand, input-grounded tasks like audio transcription or translation benefit from greedy decoding. Enable sampling with "),ta=s("code"),lo=n("do_sample=True"),oo=n(", and you can learn more about this topic in this "),Re=s("a"),no=n("blog post"),ro=n("."),Da=u(),d(Xe.$$.fragment),Oa=u(),N=s("h3"),ue=s("a"),aa=s("span"),d(ze.$$.fragment),io=u(),sa=s("span"),po=n("Wrong padding side"),Ka=u(),ce=s("p"),ho=n("LLMs are "),Ce=s("a"),uo=n("decoder-only"),co=n(" architectures, meaning they continue to iterate on your input prompt. If your inputs do not have the same length, they need to be padded. Since LLMs are not trained to continue from pad tokens, your input needs to be left-padded. Make sure you also don\u2019t forget to pass the attention mask to generate!"),es=u(),d(Be.$$.fragment),ts=u(),F=s("h2"),de=s("a"),la=s("span"),d(Ne.$$.fragment),fo=u(),oa=s("span"),mo=n("Further resources"),as=u(),bt=s("p"),go=n("While the autoregressive generation process is relatively straightforward, making the most out of your LLM can be a challenging endeavor because there are many moving parts. For your next steps to help you dive deeper into LLM usage and understanding:"),ss=u(),q=s("h3"),fe=s("a"),na=s("span"),d(Fe.$$.fragment),yo=u(),ra=s("span"),bo=n("Advanced generate usage"),ls=u(),me=s("ol"),vt=s("li"),wt=s("a"),vo=n("Guide"),wo=n(" on how to control different generation methods, how to set up the generation configuration file, and how to stream the output;"),Mo=u(),Z=s("li"),_o=n("API reference on "),Mt=s("a"),ko=n("GenerationConfig"),jo=n(", "),_t=s("a"),Jo=n("generate()"),Eo=n(", and "),kt=s("a"),To=n("generate-related classes"),Uo=n("."),os=u(),S=s("h3"),ge=s("a"),ia=s("span"),d(qe.$$.fragment),Zo=u(),pa=s("span"),Io=n("LLM leaderboards"),ns=u(),ye=s("ol"),jt=s("li"),Se=s("a"),$o=n("Open LLM Leaderboard"),Go=n(", which focuses on the quality of the open-source models;"),Lo=u(),Jt=s("li"),Qe=s("a"),xo=n("Open LLM-Perf Leaderboard"),Wo=n(", which focuses on LLM throughput."),rs=u(),Q=s("h3"),be=s("a"),ha=s("span"),d(He.$$.fragment),Vo=u(),ua=s("span"),Ao=n("Latency and throughput"),is=u(),Et=s("ol"),Tt=s("li"),Ut=s("a"),Ro=n("Guide"),Xo=n(" on dynamic quantization, which shows you how to drastically reduce your memory requirements."),ps=u(),H=s("h3"),ve=s("a"),ca=s("span"),d(Ye.$$.fragment),zo=u(),da=s("span"),Co=n("Related libraries"),hs=u(),we=s("ol"),Zt=s("li"),Pe=s("a"),fa=s("code"),Bo=n("text-generation-inference"),No=n(", a production-ready server for LLMs;"),Fo=u(),It=s("li"),De=s("a"),ma=s("code"),qo=n("optimum"),So=n(", an extension of \u{1F917} Transformers that optimizes for specific hardware devices."),this.h()},l(e){const i=Xr('[data-svelte="svelte-1phssyn"]',document.head);v=l(i,"META",{name:!0,content:!0}),i.forEach(t),Y=c(e),w=l(e,"H1",{class:!0});var Oe=o(w);k=l(Oe,"A",{id:!0,class:!0,href:!0});var ln=o(k);V=l(ln,"SPAN",{});var on=o(V);f(U.$$.fragment,on),on.forEach(t),ln.forEach(t),$=c(Oe),A=l(Oe,"SPAN",{});var nn=o(A);ke=r(nn,"Generation with LLMs"),nn.forEach(t),Oe.forEach(t),G=c(e),f(M.$$.fragment,e),je=c(e),R=l(e,"P",{});var rn=o(R);Ke=r(rn,"LLMs, or Large Language Models, are the key component behind text generation. In a nutshell, they consist of large pretrained transformer models trained to predict the next word (or, more precisely, token) given some input text. Since they predict one token at a time, you need to do something more elaborate to generate new sentences other than just calling the model \u2014 you need to do autoregressive generation."),rn.forEach(t),ba=c(e),P=l(e,"P",{});var cs=o(P);Vs=r(cs,"Autoregressive generation is the inference-time procedure of iteratively calling a model with its own generated outputs, given a few initial inputs. In \u{1F917} Transformers, this is handled by the "),et=l(cs,"A",{href:!0});var pn=o(et);As=r(pn,"generate()"),pn.forEach(t),Rs=r(cs," method, which is available to all models with generative capabilities."),cs.forEach(t),va=c(e),tt=l(e,"P",{});var hn=o(tt);Xs=r(hn,"This tutorial will show you how to:"),hn.forEach(t),wa=c(e),L=l(e,"UL",{});var $t=o(L);Lt=l($t,"LI",{});var un=o(Lt);zs=r(un,"Generate text with an LLM"),un.forEach(t),Cs=c($t),xt=l($t,"LI",{});var cn=o(xt);Bs=r(cn,"Avoid common pitfalls"),cn.forEach(t),Ns=c($t),Wt=l($t,"LI",{});var dn=o(Wt);Fs=r(dn,"Next steps to help you get the most out your LLM"),dn.forEach(t),$t.forEach(t),Ma=c(e),at=l(e,"P",{});var fn=o(at);qs=r(fn,"Before you begin, make sure you have all the necessary libraries installed:"),fn.forEach(t),_a=c(e),f(Je.$$.fragment,e),ka=c(e),X=l(e,"H2",{class:!0});var ds=o(X);D=l(ds,"A",{id:!0,class:!0,href:!0});var mn=o(D);Vt=l(mn,"SPAN",{});var gn=o(Vt);f(Ee.$$.fragment,gn),gn.forEach(t),mn.forEach(t),Ss=c(ds),At=l(ds,"SPAN",{});var yn=o(At);Qs=r(yn,"Generate text"),yn.forEach(t),ds.forEach(t),ja=c(e),O=l(e,"P",{});var fs=o(O);Hs=r(fs,"A language model trained for "),st=l(fs,"A",{href:!0});var bn=o(st);Ys=r(bn,"causal language modeling"),bn.forEach(t),Ps=r(fs," takes a sequence of text tokens as input and returns the probability distribution for the next token."),fs.forEach(t),Ja=c(e),Te=l(e,"FIGURE",{class:!0});var vn=o(Te);j=l(vn,"VIDEO",{style:!0,src:!0});var Qo=o(j);Ds=r(Qo,`&lt;/video>
    `),Rt=l(Qo,"FIGCAPTION",{});var wn=o(Rt);Os=r(wn,'"Forward pass of an LLM"'),wn.forEach(t),Qo.forEach(t),vn.forEach(t),Ea=c(e),lt=l(e,"P",{});var Mn=o(lt);Ks=r(Mn,"A critical aspect of autoregressive generation with LLMs is how to select the next token from this probability distribution. Anything goes in this step as long as you end up with a token for the next iteration. This means it can be as simple as selecting the most likely token from the probability distribution or as complex as applying a dozen transformations before sampling from the resulting distribution."),Mn.forEach(t),Ta=c(e),Ue=l(e,"FIGURE",{class:!0});var _n=o(Ue);J=l(_n,"VIDEO",{style:!0,src:!0});var Ho=o(J);el=r(Ho,`&lt;/video>
    `),Xt=l(Ho,"FIGCAPTION",{});var kn=o(Xt);tl=r(kn,'"Autoregressive generation iteratively selects the next token from a probability distribution to generate text"'),kn.forEach(t),Ho.forEach(t),_n.forEach(t),Ua=c(e),K=l(e,"P",{});var ms=o(K);al=r(ms,"The process depicted above is repeated iteratively until some stopping condition is reached. Ideally, the stopping condition is dictated by the model, which should learn when to output an end-of-sequence ("),zt=l(ms,"CODE",{});var jn=o(zt);sl=r(jn,"EOS"),jn.forEach(t),ll=r(ms,") token. If this is not the case, generation stops when some predefined maximum length is reached."),ms.forEach(t),Za=c(e),ee=l(e,"P",{});var gs=o(ee);ol=r(gs,"Properly setting up the token selection step and the stopping condition is essential to make your model behave as you\u2019d expect on your task. That is why we have a "),ot=l(gs,"A",{href:!0});var Jn=o(ot);nl=r(Jn,"GenerationConfig"),Jn.forEach(t),rl=r(gs," file associated with each model, which contains a good default generative parameterization and is loaded alongside your model."),gs.forEach(t),Ia=c(e),nt=l(e,"P",{});var En=o(nt);il=r(En,"Let\u2019s talk code!"),En.forEach(t),$a=c(e),f(te.$$.fragment,e),Ga=c(e),rt=l(e,"P",{});var Tn=o(rt);pl=r(Tn,"First, you need to load the model."),Tn.forEach(t),La=c(e),f(Ze.$$.fragment,e),xa=c(e),ae=l(e,"P",{});var ys=o(ae);hl=r(ys,"You\u2019ll notice two flags in the "),Ct=l(ys,"CODE",{});var Un=o(Ct);ul=r(Un,"from_pretrained"),Un.forEach(t),cl=r(ys," call:"),ys.forEach(t),Wa=c(e),se=l(e,"UL",{});var bs=o(se);it=l(bs,"LI",{});var Yo=o(it);Bt=l(Yo,"CODE",{});var Zn=o(Bt);dl=r(Zn,"device_map"),Zn.forEach(t),fl=r(Yo," ensures the model is moved to your GPU(s)"),Yo.forEach(t),ml=c(bs),le=l(bs,"LI",{});var ga=o(le);Nt=l(ga,"CODE",{});var In=o(Nt);gl=r(In,"load_in_4bit"),In.forEach(t),yl=r(ga," applies "),pt=l(ga,"A",{href:!0});var $n=o(pt);bl=r($n,"4-bit dynamic quantization"),$n.forEach(t),vl=r(ga," to massively reduce the resource requirements"),ga.forEach(t),bs.forEach(t),Va=c(e),ht=l(e,"P",{});var Gn=o(ht);wl=r(Gn,"There are other ways to initialize a model, but this is a good baseline to begin with an LLM."),Gn.forEach(t),Aa=c(e),oe=l(e,"P",{});var vs=o(oe);Ml=r(vs,"Next, you need to preprocess your text input with a "),ut=l(vs,"A",{href:!0});var Ln=o(ut);_l=r(Ln,"tokenizer"),Ln.forEach(t),kl=r(vs,"."),vs.forEach(t),Ra=c(e),f(Ie.$$.fragment,e),Xa=c(e),x=l(e,"P",{});var Gt=o(x);jl=r(Gt,"The "),Ft=l(Gt,"CODE",{});var xn=o(Ft);Jl=r(xn,"model_inputs"),xn.forEach(t),El=r(Gt," variable holds the tokenized text input, as well as the attention mask. While "),ct=l(Gt,"A",{href:!0});var Wn=o(ct);Tl=r(Wn,"generate()"),Wn.forEach(t),Ul=r(Gt," does its best effort to infer the attention mask when it is not passed, we recommend passing it whenever possible for optimal results."),Gt.forEach(t),za=c(e),ne=l(e,"P",{});var ws=o(ne);Zl=r(ws,"Finally, call the "),dt=l(ws,"A",{href:!0});var Vn=o(dt);Il=r(Vn,"generate()"),Vn.forEach(t),$l=r(ws," method to returns the generated tokens, which should be converted to text before printing."),ws.forEach(t),Ca=c(e),f($e.$$.fragment,e),Ba=c(e),ft=l(e,"P",{});var An=o(ft);Gl=r(An,"And that\u2019s it! In a few lines of code, you can harness the power of an LLM."),An.forEach(t),Na=c(e),z=l(e,"H2",{class:!0});var Ms=o(z);re=l(Ms,"A",{id:!0,class:!0,href:!0});var Rn=o(re);qt=l(Rn,"SPAN",{});var Xn=o(qt);f(Ge.$$.fragment,Xn),Xn.forEach(t),Rn.forEach(t),Ll=c(Ms),St=l(Ms,"SPAN",{});var zn=o(St);xl=r(zn,"Common pitfalls"),zn.forEach(t),Ms.forEach(t),Fa=c(e),ie=l(e,"P",{});var _s=o(ie);Wl=r(_s,"There are many "),mt=l(_s,"A",{href:!0});var Cn=o(mt);Vl=r(Cn,"generation strategies"),Cn.forEach(t),Al=r(_s,", and sometimes the default values may not be appropriate for your use case. If your outputs aren\u2019t aligned with what you\u2019re expecting, we\u2019ve created a list of the most common pitfalls and how to avoid them."),_s.forEach(t),qa=c(e),f(Le.$$.fragment,e),Sa=c(e),C=l(e,"H3",{class:!0});var ks=o(C);pe=l(ks,"A",{id:!0,class:!0,href:!0});var Bn=o(pe);Qt=l(Bn,"SPAN",{});var Nn=o(Qt);f(xe.$$.fragment,Nn),Nn.forEach(t),Bn.forEach(t),Rl=c(ks),Ht=l(ks,"SPAN",{});var Fn=o(Ht);Xl=r(Fn,"Generated output is too short/long"),Fn.forEach(t),ks.forEach(t),Qa=c(e),_=l(e,"P",{});var T=o(_);zl=r(T,"If not specified in the "),gt=l(T,"A",{href:!0});var qn=o(gt);Cl=r(qn,"GenerationConfig"),qn.forEach(t),Bl=r(T," file, "),Yt=l(T,"CODE",{});var Sn=o(Yt);Nl=r(Sn,"generate"),Sn.forEach(t),Fl=r(T," returns up to 20 tokens by default. We highly recommend manually setting "),Pt=l(T,"CODE",{});var Qn=o(Pt);ql=r(Qn,"max_new_tokens"),Qn.forEach(t),Sl=r(T," in your "),Dt=l(T,"CODE",{});var Hn=o(Dt);Ql=r(Hn,"generate"),Hn.forEach(t),Hl=r(T," call to control the maximum number of new tokens it can return. Keep in mind LLMs (more precisely, "),We=l(T,"A",{href:!0,rel:!0});var Yn=o(We);Yl=r(Yn,"decoder-only models"),Yn.forEach(t),Pl=r(T,") also return the input prompt as part of the output."),T.forEach(t),Ha=c(e),f(Ve.$$.fragment,e),Ya=c(e),B=l(e,"H3",{class:!0});var js=o(B);he=l(js,"A",{id:!0,class:!0,href:!0});var Pn=o(he);Ot=l(Pn,"SPAN",{});var Dn=o(Ot);f(Ae.$$.fragment,Dn),Dn.forEach(t),Pn.forEach(t),Dl=c(js),Kt=l(js,"SPAN",{});var On=o(Kt);Ol=r(On,"Incorrect generation mode"),On.forEach(t),js.forEach(t),Pa=c(e),E=l(e,"P",{});var W=o(E);Kl=r(W,"By default, and unless specified in the "),yt=l(W,"A",{href:!0});var Kn=o(yt);eo=r(Kn,"GenerationConfig"),Kn.forEach(t),to=r(W," file, "),ea=l(W,"CODE",{});var er=o(ea);ao=r(er,"generate"),er.forEach(t),so=r(W," selects the most likely token at each iteration (greedy decoding). Depending on your task, this may be undesirable; creative tasks like chatbots or writing an essay benefit from sampling. On the other hand, input-grounded tasks like audio transcription or translation benefit from greedy decoding. Enable sampling with "),ta=l(W,"CODE",{});var tr=o(ta);lo=r(tr,"do_sample=True"),tr.forEach(t),oo=r(W,", and you can learn more about this topic in this "),Re=l(W,"A",{href:!0,rel:!0});var ar=o(Re);no=r(ar,"blog post"),ar.forEach(t),ro=r(W,"."),W.forEach(t),Da=c(e),f(Xe.$$.fragment,e),Oa=c(e),N=l(e,"H3",{class:!0});var Js=o(N);ue=l(Js,"A",{id:!0,class:!0,href:!0});var sr=o(ue);aa=l(sr,"SPAN",{});var lr=o(aa);f(ze.$$.fragment,lr),lr.forEach(t),sr.forEach(t),io=c(Js),sa=l(Js,"SPAN",{});var or=o(sa);po=r(or,"Wrong padding side"),or.forEach(t),Js.forEach(t),Ka=c(e),ce=l(e,"P",{});var Es=o(ce);ho=r(Es,"LLMs are "),Ce=l(Es,"A",{href:!0,rel:!0});var nr=o(Ce);uo=r(nr,"decoder-only"),nr.forEach(t),co=r(Es," architectures, meaning they continue to iterate on your input prompt. If your inputs do not have the same length, they need to be padded. Since LLMs are not trained to continue from pad tokens, your input needs to be left-padded. Make sure you also don\u2019t forget to pass the attention mask to generate!"),Es.forEach(t),es=c(e),f(Be.$$.fragment,e),ts=c(e),F=l(e,"H2",{class:!0});var Ts=o(F);de=l(Ts,"A",{id:!0,class:!0,href:!0});var rr=o(de);la=l(rr,"SPAN",{});var ir=o(la);f(Ne.$$.fragment,ir),ir.forEach(t),rr.forEach(t),fo=c(Ts),oa=l(Ts,"SPAN",{});var pr=o(oa);mo=r(pr,"Further resources"),pr.forEach(t),Ts.forEach(t),as=c(e),bt=l(e,"P",{});var hr=o(bt);go=r(hr,"While the autoregressive generation process is relatively straightforward, making the most out of your LLM can be a challenging endeavor because there are many moving parts. For your next steps to help you dive deeper into LLM usage and understanding:"),hr.forEach(t),ss=c(e),q=l(e,"H3",{class:!0});var Us=o(q);fe=l(Us,"A",{id:!0,class:!0,href:!0});var ur=o(fe);na=l(ur,"SPAN",{});var cr=o(na);f(Fe.$$.fragment,cr),cr.forEach(t),ur.forEach(t),yo=c(Us),ra=l(Us,"SPAN",{});var dr=o(ra);bo=r(dr,"Advanced generate usage"),dr.forEach(t),Us.forEach(t),ls=c(e),me=l(e,"OL",{});var Zs=o(me);vt=l(Zs,"LI",{});var Po=o(vt);wt=l(Po,"A",{href:!0});var fr=o(wt);vo=r(fr,"Guide"),fr.forEach(t),wo=r(Po," on how to control different generation methods, how to set up the generation configuration file, and how to stream the output;"),Po.forEach(t),Mo=c(Zs),Z=l(Zs,"LI",{});var Me=o(Z);_o=r(Me,"API reference on "),Mt=l(Me,"A",{href:!0});var mr=o(Mt);ko=r(mr,"GenerationConfig"),mr.forEach(t),jo=r(Me,", "),_t=l(Me,"A",{href:!0});var gr=o(_t);Jo=r(gr,"generate()"),gr.forEach(t),Eo=r(Me,", and "),kt=l(Me,"A",{href:!0});var yr=o(kt);To=r(yr,"generate-related classes"),yr.forEach(t),Uo=r(Me,"."),Me.forEach(t),Zs.forEach(t),os=c(e),S=l(e,"H3",{class:!0});var Is=o(S);ge=l(Is,"A",{id:!0,class:!0,href:!0});var br=o(ge);ia=l(br,"SPAN",{});var vr=o(ia);f(qe.$$.fragment,vr),vr.forEach(t),br.forEach(t),Zo=c(Is),pa=l(Is,"SPAN",{});var wr=o(pa);Io=r(wr,"LLM leaderboards"),wr.forEach(t),Is.forEach(t),ns=c(e),ye=l(e,"OL",{});var $s=o(ye);jt=l($s,"LI",{});var Do=o(jt);Se=l(Do,"A",{href:!0,rel:!0});var Mr=o(Se);$o=r(Mr,"Open LLM Leaderboard"),Mr.forEach(t),Go=r(Do,", which focuses on the quality of the open-source models;"),Do.forEach(t),Lo=c($s),Jt=l($s,"LI",{});var Oo=o(Jt);Qe=l(Oo,"A",{href:!0,rel:!0});var _r=o(Qe);xo=r(_r,"Open LLM-Perf Leaderboard"),_r.forEach(t),Wo=r(Oo,", which focuses on LLM throughput."),Oo.forEach(t),$s.forEach(t),rs=c(e),Q=l(e,"H3",{class:!0});var Gs=o(Q);be=l(Gs,"A",{id:!0,class:!0,href:!0});var kr=o(be);ha=l(kr,"SPAN",{});var jr=o(ha);f(He.$$.fragment,jr),jr.forEach(t),kr.forEach(t),Vo=c(Gs),ua=l(Gs,"SPAN",{});var Jr=o(ua);Ao=r(Jr,"Latency and throughput"),Jr.forEach(t),Gs.forEach(t),is=c(e),Et=l(e,"OL",{});var Er=o(Et);Tt=l(Er,"LI",{});var Ko=o(Tt);Ut=l(Ko,"A",{href:!0});var Tr=o(Ut);Ro=r(Tr,"Guide"),Tr.forEach(t),Xo=r(Ko," on dynamic quantization, which shows you how to drastically reduce your memory requirements."),Ko.forEach(t),Er.forEach(t),ps=c(e),H=l(e,"H3",{class:!0});var Ls=o(H);ve=l(Ls,"A",{id:!0,class:!0,href:!0});var Ur=o(ve);ca=l(Ur,"SPAN",{});var Zr=o(ca);f(Ye.$$.fragment,Zr),Zr.forEach(t),Ur.forEach(t),zo=c(Ls),da=l(Ls,"SPAN",{});var Ir=o(da);Co=r(Ir,"Related libraries"),Ir.forEach(t),Ls.forEach(t),hs=c(e),we=l(e,"OL",{});var xs=o(we);Zt=l(xs,"LI",{});var en=o(Zt);Pe=l(en,"A",{href:!0,rel:!0});var $r=o(Pe);fa=l($r,"CODE",{});var Gr=o(fa);Bo=r(Gr,"text-generation-inference"),Gr.forEach(t),$r.forEach(t),No=r(en,", a production-ready server for LLMs;"),en.forEach(t),Fo=c(xs),It=l(xs,"LI",{});var tn=o(It);De=l(tn,"A",{href:!0,rel:!0});var Lr=o(De);ma=l(Lr,"CODE",{});var xr=o(ma);qo=r(xr,"optimum"),xr.forEach(t),Lr.forEach(t),So=r(tn,", an extension of \u{1F917} Transformers that optimizes for specific hardware devices."),tn.forEach(t),xs.forEach(t),this.h()},h(){h(v,"name","hf:doc:metadata"),h(v,"content",JSON.stringify(qr)),h(k,"id","generation-with-llms"),h(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(k,"href","#generation-with-llms"),h(w,"class","relative group"),h(et,"href","/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate"),h(D,"id","generate-text"),h(D,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(D,"href","#generate-text"),h(X,"class","relative group"),h(st,"href","tasks/language_modeling"),Ws(j,"max-width","90%"),Ws(j,"margin","auto"),j.autoplay=!0,j.loop=!0,j.muted=!0,j.playsInline=!0,Wr(j.src,an="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/assisted-generation/gif_1_1080p.mov")||h(j,"src",an),h(Te,"class","image table text-center m-0 w-full"),Ws(J,"max-width","90%"),Ws(J,"margin","auto"),J.autoplay=!0,J.loop=!0,J.muted=!0,J.playsInline=!0,Wr(J.src,sn="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/assisted-generation/gif_2_1080p.mov")||h(J,"src",sn),h(Ue,"class","image table text-center m-0 w-full"),h(ot,"href","/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig"),h(pt,"href","main_classes/quantization"),h(ut,"href","tokenizer_summary"),h(ct,"href","/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate"),h(dt,"href","/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate"),h(re,"id","common-pitfalls"),h(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(re,"href","#common-pitfalls"),h(z,"class","relative group"),h(mt,"href","generation_strategies"),h(pe,"id","generated-output-is-too-shortlong"),h(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(pe,"href","#generated-output-is-too-shortlong"),h(C,"class","relative group"),h(gt,"href","/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig"),h(We,"href","https://huggingface.co/learn/nlp-course/chapter1/6?fw=pt"),h(We,"rel","nofollow"),h(he,"id","incorrect-generation-mode"),h(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(he,"href","#incorrect-generation-mode"),h(B,"class","relative group"),h(yt,"href","/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig"),h(Re,"href","https://huggingface.co/blog/how-to-generate"),h(Re,"rel","nofollow"),h(ue,"id","wrong-padding-side"),h(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ue,"href","#wrong-padding-side"),h(N,"class","relative group"),h(Ce,"href","https://huggingface.co/learn/nlp-course/chapter1/6?fw=pt"),h(Ce,"rel","nofollow"),h(de,"id","further-resources"),h(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(de,"href","#further-resources"),h(F,"class","relative group"),h(fe,"id","advanced-generate-usage"),h(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(fe,"href","#advanced-generate-usage"),h(q,"class","relative group"),h(wt,"href","generation_strategies"),h(Mt,"href","/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig"),h(_t,"href","/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate"),h(kt,"href","internal/generation_utils"),h(ge,"id","llm-leaderboards"),h(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ge,"href","#llm-leaderboards"),h(S,"class","relative group"),h(Se,"href","https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"),h(Se,"rel","nofollow"),h(Qe,"href","https://huggingface.co/spaces/optimum/llm-perf-leaderboard"),h(Qe,"rel","nofollow"),h(be,"id","latency-and-throughput"),h(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(be,"href","#latency-and-throughput"),h(Q,"class","relative group"),h(Ut,"href","main_classes/quantization"),h(ve,"id","related-libraries"),h(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ve,"href","#related-libraries"),h(H,"class","relative group"),h(Pe,"href","https://github.com/huggingface/text-generation-inference"),h(Pe,"rel","nofollow"),h(De,"href","https://github.com/huggingface/optimum"),h(De,"rel","nofollow")},m(e,i){a(document.head,v),p(e,Y,i),p(e,w,i),a(w,k),a(k,V),m(U,V,null),a(w,$),a(w,A),a(A,ke),p(e,G,i),m(M,e,i),p(e,je,i),p(e,R,i),a(R,Ke),p(e,ba,i),p(e,P,i),a(P,Vs),a(P,et),a(et,As),a(P,Rs),p(e,va,i),p(e,tt,i),a(tt,Xs),p(e,wa,i),p(e,L,i),a(L,Lt),a(Lt,zs),a(L,Cs),a(L,xt),a(xt,Bs),a(L,Ns),a(L,Wt),a(Wt,Fs),p(e,Ma,i),p(e,at,i),a(at,qs),p(e,_a,i),m(Je,e,i),p(e,ka,i),p(e,X,i),a(X,D),a(D,Vt),m(Ee,Vt,null),a(X,Ss),a(X,At),a(At,Qs),p(e,ja,i),p(e,O,i),a(O,Hs),a(O,st),a(st,Ys),a(O,Ps),p(e,Ja,i),p(e,Te,i),a(Te,j),a(j,Ds),a(j,Rt),a(Rt,Os),p(e,Ea,i),p(e,lt,i),a(lt,Ks),p(e,Ta,i),p(e,Ue,i),a(Ue,J),a(J,el),a(J,Xt),a(Xt,tl),p(e,Ua,i),p(e,K,i),a(K,al),a(K,zt),a(zt,sl),a(K,ll),p(e,Za,i),p(e,ee,i),a(ee,ol),a(ee,ot),a(ot,nl),a(ee,rl),p(e,Ia,i),p(e,nt,i),a(nt,il),p(e,$a,i),m(te,e,i),p(e,Ga,i),p(e,rt,i),a(rt,pl),p(e,La,i),m(Ze,e,i),p(e,xa,i),p(e,ae,i),a(ae,hl),a(ae,Ct),a(Ct,ul),a(ae,cl),p(e,Wa,i),p(e,se,i),a(se,it),a(it,Bt),a(Bt,dl),a(it,fl),a(se,ml),a(se,le),a(le,Nt),a(Nt,gl),a(le,yl),a(le,pt),a(pt,bl),a(le,vl),p(e,Va,i),p(e,ht,i),a(ht,wl),p(e,Aa,i),p(e,oe,i),a(oe,Ml),a(oe,ut),a(ut,_l),a(oe,kl),p(e,Ra,i),m(Ie,e,i),p(e,Xa,i),p(e,x,i),a(x,jl),a(x,Ft),a(Ft,Jl),a(x,El),a(x,ct),a(ct,Tl),a(x,Ul),p(e,za,i),p(e,ne,i),a(ne,Zl),a(ne,dt),a(dt,Il),a(ne,$l),p(e,Ca,i),m($e,e,i),p(e,Ba,i),p(e,ft,i),a(ft,Gl),p(e,Na,i),p(e,z,i),a(z,re),a(re,qt),m(Ge,qt,null),a(z,Ll),a(z,St),a(St,xl),p(e,Fa,i),p(e,ie,i),a(ie,Wl),a(ie,mt),a(mt,Vl),a(ie,Al),p(e,qa,i),m(Le,e,i),p(e,Sa,i),p(e,C,i),a(C,pe),a(pe,Qt),m(xe,Qt,null),a(C,Rl),a(C,Ht),a(Ht,Xl),p(e,Qa,i),p(e,_,i),a(_,zl),a(_,gt),a(gt,Cl),a(_,Bl),a(_,Yt),a(Yt,Nl),a(_,Fl),a(_,Pt),a(Pt,ql),a(_,Sl),a(_,Dt),a(Dt,Ql),a(_,Hl),a(_,We),a(We,Yl),a(_,Pl),p(e,Ha,i),m(Ve,e,i),p(e,Ya,i),p(e,B,i),a(B,he),a(he,Ot),m(Ae,Ot,null),a(B,Dl),a(B,Kt),a(Kt,Ol),p(e,Pa,i),p(e,E,i),a(E,Kl),a(E,yt),a(yt,eo),a(E,to),a(E,ea),a(ea,ao),a(E,so),a(E,ta),a(ta,lo),a(E,oo),a(E,Re),a(Re,no),a(E,ro),p(e,Da,i),m(Xe,e,i),p(e,Oa,i),p(e,N,i),a(N,ue),a(ue,aa),m(ze,aa,null),a(N,io),a(N,sa),a(sa,po),p(e,Ka,i),p(e,ce,i),a(ce,ho),a(ce,Ce),a(Ce,uo),a(ce,co),p(e,es,i),m(Be,e,i),p(e,ts,i),p(e,F,i),a(F,de),a(de,la),m(Ne,la,null),a(F,fo),a(F,oa),a(oa,mo),p(e,as,i),p(e,bt,i),a(bt,go),p(e,ss,i),p(e,q,i),a(q,fe),a(fe,na),m(Fe,na,null),a(q,yo),a(q,ra),a(ra,bo),p(e,ls,i),p(e,me,i),a(me,vt),a(vt,wt),a(wt,vo),a(vt,wo),a(me,Mo),a(me,Z),a(Z,_o),a(Z,Mt),a(Mt,ko),a(Z,jo),a(Z,_t),a(_t,Jo),a(Z,Eo),a(Z,kt),a(kt,To),a(Z,Uo),p(e,os,i),p(e,S,i),a(S,ge),a(ge,ia),m(qe,ia,null),a(S,Zo),a(S,pa),a(pa,Io),p(e,ns,i),p(e,ye,i),a(ye,jt),a(jt,Se),a(Se,$o),a(jt,Go),a(ye,Lo),a(ye,Jt),a(Jt,Qe),a(Qe,xo),a(Jt,Wo),p(e,rs,i),p(e,Q,i),a(Q,be),a(be,ha),m(He,ha,null),a(Q,Vo),a(Q,ua),a(ua,Ao),p(e,is,i),p(e,Et,i),a(Et,Tt),a(Tt,Ut),a(Ut,Ro),a(Tt,Xo),p(e,ps,i),p(e,H,i),a(H,ve),a(ve,ca),m(Ye,ca,null),a(H,zo),a(H,da),a(da,Co),p(e,hs,i),p(e,we,i),a(we,Zt),a(Zt,Pe),a(Pe,fa),a(fa,Bo),a(Zt,No),a(we,Fo),a(we,It),a(It,De),a(De,ma),a(ma,qo),a(It,So),us=!0},p(e,[i]){const Oe={};i&2&&(Oe.$$scope={dirty:i,ctx:e}),te.$set(Oe)},i(e){us||(g(U.$$.fragment,e),g(M.$$.fragment,e),g(Je.$$.fragment,e),g(Ee.$$.fragment,e),g(te.$$.fragment,e),g(Ze.$$.fragment,e),g(Ie.$$.fragment,e),g($e.$$.fragment,e),g(Ge.$$.fragment,e),g(Le.$$.fragment,e),g(xe.$$.fragment,e),g(Ve.$$.fragment,e),g(Ae.$$.fragment,e),g(Xe.$$.fragment,e),g(ze.$$.fragment,e),g(Be.$$.fragment,e),g(Ne.$$.fragment,e),g(Fe.$$.fragment,e),g(qe.$$.fragment,e),g(He.$$.fragment,e),g(Ye.$$.fragment,e),us=!0)},o(e){y(U.$$.fragment,e),y(M.$$.fragment,e),y(Je.$$.fragment,e),y(Ee.$$.fragment,e),y(te.$$.fragment,e),y(Ze.$$.fragment,e),y(Ie.$$.fragment,e),y($e.$$.fragment,e),y(Ge.$$.fragment,e),y(Le.$$.fragment,e),y(xe.$$.fragment,e),y(Ve.$$.fragment,e),y(Ae.$$.fragment,e),y(Xe.$$.fragment,e),y(ze.$$.fragment,e),y(Be.$$.fragment,e),y(Ne.$$.fragment,e),y(Fe.$$.fragment,e),y(qe.$$.fragment,e),y(He.$$.fragment,e),y(Ye.$$.fragment,e),us=!1},d(e){t(v),e&&t(Y),e&&t(w),b(U),e&&t(G),b(M,e),e&&t(je),e&&t(R),e&&t(ba),e&&t(P),e&&t(va),e&&t(tt),e&&t(wa),e&&t(L),e&&t(Ma),e&&t(at),e&&t(_a),b(Je,e),e&&t(ka),e&&t(X),b(Ee),e&&t(ja),e&&t(O),e&&t(Ja),e&&t(Te),e&&t(Ea),e&&t(lt),e&&t(Ta),e&&t(Ue),e&&t(Ua),e&&t(K),e&&t(Za),e&&t(ee),e&&t(Ia),e&&t(nt),e&&t($a),b(te,e),e&&t(Ga),e&&t(rt),e&&t(La),b(Ze,e),e&&t(xa),e&&t(ae),e&&t(Wa),e&&t(se),e&&t(Va),e&&t(ht),e&&t(Aa),e&&t(oe),e&&t(Ra),b(Ie,e),e&&t(Xa),e&&t(x),e&&t(za),e&&t(ne),e&&t(Ca),b($e,e),e&&t(Ba),e&&t(ft),e&&t(Na),e&&t(z),b(Ge),e&&t(Fa),e&&t(ie),e&&t(qa),b(Le,e),e&&t(Sa),e&&t(C),b(xe),e&&t(Qa),e&&t(_),e&&t(Ha),b(Ve,e),e&&t(Ya),e&&t(B),b(Ae),e&&t(Pa),e&&t(E),e&&t(Da),b(Xe,e),e&&t(Oa),e&&t(N),b(ze),e&&t(Ka),e&&t(ce),e&&t(es),b(Be,e),e&&t(ts),e&&t(F),b(Ne),e&&t(as),e&&t(bt),e&&t(ss),e&&t(q),b(Fe),e&&t(ls),e&&t(me),e&&t(os),e&&t(S),b(qe),e&&t(ns),e&&t(ye),e&&t(rs),e&&t(Q),b(He),e&&t(is),e&&t(Et),e&&t(ps),e&&t(H),b(Ye),e&&t(hs),e&&t(we)}}}const qr={local:"generation-with-llms",sections:[{local:"generate-text",title:"Generate text"},{local:"common-pitfalls",sections:[{local:"generated-output-is-too-shortlong",title:"Generated output is too short/long"},{local:"incorrect-generation-mode",title:"Incorrect generation mode"},{local:"wrong-padding-side",title:"Wrong padding side"}],title:"Common pitfalls"},{local:"further-resources",sections:[{local:"advanced-generate-usage",title:"Advanced generate usage"},{local:"llm-leaderboards",title:"LLM leaderboards"},{local:"latency-and-throughput",title:"Latency and throughput"},{local:"related-libraries",title:"Related libraries"}],title:"Further resources"}],title:"Generation with LLMs"};function Sr(ya){return zr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Or extends Vr{constructor(v){super();Ar(this,v,Sr,Fr,Rr,{})}}export{Or as default,qr as metadata};
