import{S as wr,i as br,s as jr,e as p,k as d,w as q,t as n,M as vr,c as m,d as t,m as g,a as f,x as P,h as l,b as w,G as s,g as i,y as F,q as C,o as D,B as L,v as xr,L as hs}from"../../chunks/vendor-hf-doc-builder.js";import{T as Xt}from"../../chunks/Tip-hf-doc-builder.js";import{Y as yn}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Ze}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as J}from"../../chunks/CodeBlock-hf-doc-builder.js";import{F as aa,M as Ne}from"../../chunks/Markdown-hf-doc-builder.js";function Er(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U,b;return{c(){a=p("p"),$=n("You can finetune other architectures for language modeling such as "),o=p("a"),_=n("GPT-Neo"),k=n(", "),j=p("a"),v=n("GPT-J"),A=n(", and "),y=p("a"),E=n("BERT"),z=n(", following the same steps in this guide! See the text generation "),I=p("a"),R=n("task page"),O=n(" and fill mask "),H=p("a"),U=n("task page"),b=n(" for more information about their associated models, datasets, and metrics."),this.h()},l(T){a=m(T,"P",{});var S=f(a);$=l(S,"You can finetune other architectures for language modeling such as "),o=m(S,"A",{href:!0,rel:!0});var N=f(o);_=l(N,"GPT-Neo"),N.forEach(t),k=l(S,", "),j=m(S,"A",{href:!0,rel:!0});var Y=f(j);v=l(Y,"GPT-J"),Y.forEach(t),A=l(S,", and "),y=m(S,"A",{href:!0,rel:!0});var Q=f(y);E=l(Q,"BERT"),Q.forEach(t),z=l(S,", following the same steps in this guide! See the text generation "),I=m(S,"A",{href:!0,rel:!0});var G=f(I);R=l(G,"task page"),G.forEach(t),O=l(S," and fill mask "),H=m(S,"A",{href:!0,rel:!0});var B=f(H);U=l(B,"task page"),B.forEach(t),b=l(S," for more information about their associated models, datasets, and metrics."),S.forEach(t),this.h()},h(){w(o,"href","https://huggingface.co/EleutherAI/gpt-neo-125M"),w(o,"rel","nofollow"),w(j,"href","https://huggingface.co/EleutherAI/gpt-j-6B"),w(j,"rel","nofollow"),w(y,"href","https://huggingface.co/bert-base-uncased"),w(y,"rel","nofollow"),w(I,"href","https://huggingface.co/tasks/text-generation"),w(I,"rel","nofollow"),w(H,"href","https://huggingface.co/tasks/fill-mask"),w(H,"rel","nofollow")},m(T,S){i(T,a,S),s(a,$),s(a,o),s(o,_),s(a,k),s(a,j),s(j,v),s(a,A),s(a,y),s(y,E),s(a,z),s(a,I),s(I,R),s(a,O),s(a,H),s(H,U),s(a,b)},d(T){T&&t(a)}}}function Tr(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U;return v=new J({props:{code:`from transformers import DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token
<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>)`}}),H=new J({props:{code:`from transformers import DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token
<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=<span class="hljs-number">0.15</span>)`}}),{c(){a=p("p"),$=n("For causal language modeling, use the end-of-sequence token as the padding token and set "),o=p("code"),_=n("mlm=False"),k=n(". This will use the inputs as labels shifted to the right by one element:"),j=d(),q(v.$$.fragment),A=d(),y=p("p"),E=n("For masked language modeling, use the end-of-sequence token as the padding token and specify "),z=p("code"),I=n("mlm_probability"),R=n(" to randomly mask tokens each time you iterate over the data:"),O=d(),q(H.$$.fragment)},l(b){a=m(b,"P",{});var T=f(a);$=l(T,"For causal language modeling, use the end-of-sequence token as the padding token and set "),o=m(T,"CODE",{});var S=f(o);_=l(S,"mlm=False"),S.forEach(t),k=l(T,". This will use the inputs as labels shifted to the right by one element:"),T.forEach(t),j=g(b),P(v.$$.fragment,b),A=g(b),y=m(b,"P",{});var N=f(y);E=l(N,"For masked language modeling, use the end-of-sequence token as the padding token and specify "),z=m(N,"CODE",{});var Y=f(z);I=l(Y,"mlm_probability"),Y.forEach(t),R=l(N," to randomly mask tokens each time you iterate over the data:"),N.forEach(t),O=g(b),P(H.$$.fragment,b)},m(b,T){i(b,a,T),s(a,$),s(a,o),s(o,_),s(a,k),i(b,j,T),F(v,b,T),i(b,A,T),i(b,y,T),s(y,E),s(y,z),s(z,I),s(y,R),i(b,O,T),F(H,b,T),U=!0},p:hs,i(b){U||(C(v.$$.fragment,b),C(H.$$.fragment,b),U=!0)},o(b){D(v.$$.fragment,b),D(H.$$.fragment,b),U=!1},d(b){b&&t(a),b&&t(j),L(v,b),b&&t(A),b&&t(y),b&&t(O),L(H,b)}}}function Ar(K){let a,$;return a=new Ne({props:{$$slots:{default:[Tr]},$$scope:{ctx:K}}}),{c(){q(a.$$.fragment)},l(o){P(a.$$.fragment,o)},m(o,_){F(a,o,_),$=!0},p(o,_){const k={};_&2&&(k.$$scope={dirty:_,ctx:o}),a.$set(k)},i(o){$||(C(a.$$.fragment,o),$=!0)},o(o){D(a.$$.fragment,o),$=!1},d(o){L(a,o)}}}function Mr(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U;return v=new J({props:{code:`from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),H=new J({props:{code:`from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=<span class="hljs-number">0.15</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){a=p("p"),$=n("For causal language modeling, use the end-of-sequence token as the padding token and set "),o=p("code"),_=n("mlm=False"),k=n(". This will use the inputs as labels shifted to the right by one element:"),j=d(),q(v.$$.fragment),A=d(),y=p("p"),E=n("For masked language modeling, use the end-of-sequence token as the padding token and specify "),z=p("code"),I=n("mlm_probability"),R=n(" to randomly mask tokens each time you iterate over the data:"),O=d(),q(H.$$.fragment)},l(b){a=m(b,"P",{});var T=f(a);$=l(T,"For causal language modeling, use the end-of-sequence token as the padding token and set "),o=m(T,"CODE",{});var S=f(o);_=l(S,"mlm=False"),S.forEach(t),k=l(T,". This will use the inputs as labels shifted to the right by one element:"),T.forEach(t),j=g(b),P(v.$$.fragment,b),A=g(b),y=m(b,"P",{});var N=f(y);E=l(N,"For masked language modeling, use the end-of-sequence token as the padding token and specify "),z=m(N,"CODE",{});var Y=f(z);I=l(Y,"mlm_probability"),Y.forEach(t),R=l(N," to randomly mask tokens each time you iterate over the data:"),N.forEach(t),O=g(b),P(H.$$.fragment,b)},m(b,T){i(b,a,T),s(a,$),s(a,o),s(o,_),s(a,k),i(b,j,T),F(v,b,T),i(b,A,T),i(b,y,T),s(y,E),s(y,z),s(z,I),s(y,R),i(b,O,T),F(H,b,T),U=!0},p:hs,i(b){U||(C(v.$$.fragment,b),C(H.$$.fragment,b),U=!0)},o(b){D(v.$$.fragment,b),D(H.$$.fragment,b),U=!1},d(b){b&&t(a),b&&t(j),L(v,b),b&&t(A),b&&t(y),b&&t(O),L(H,b)}}}function zr(K){let a,$;return a=new Ne({props:{$$slots:{default:[Mr]},$$scope:{ctx:K}}}),{c(){q(a.$$.fragment)},l(o){P(a.$$.fragment,o)},m(o,_){F(a,o,_),$=!0},p(o,_){const k={};_&2&&(k.$$scope={dirty:_,ctx:o}),a.$set(k)},i(o){$||(C(a.$$.fragment,o),$=!0)},o(o){D(a.$$.fragment,o),$=!1},d(o){L(a,o)}}}function qr(K){let a,$,o,_,k,j,v,A;return{c(){a=p("p"),$=n("If you aren\u2019t familiar with finetuning a model with the "),o=p("a"),_=n("Trainer"),k=n(", take a look at the basic tutorial "),j=p("a"),v=n("here"),A=n("!"),this.h()},l(y){a=m(y,"P",{});var E=f(a);$=l(E,"If you aren\u2019t familiar with finetuning a model with the "),o=m(E,"A",{href:!0});var z=f(o);_=l(z,"Trainer"),z.forEach(t),k=l(E,", take a look at the basic tutorial "),j=m(E,"A",{href:!0});var I=f(j);v=l(I,"here"),I.forEach(t),A=l(E,"!"),E.forEach(t),this.h()},h(){w(o,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.Trainer"),w(j,"href","../training#train-with-pytorch-trainer")},m(y,E){i(y,a,E),s(a,$),s(a,o),s(o,_),s(a,k),s(a,j),s(j,v),s(a,A)},d(y){y&&t(a)}}}function Pr(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U,b,T,S,N,Y,Q,G,B,u,x,X,V,Z,ae,te,ge,ee,ve,le,_e,ue,xe,re,$e,ne,Ee,se,ke,fe,ce,be,oe,ie,pe;return a=new Xt({props:{$$slots:{default:[qr]},$$scope:{ctx:K}}}),o=new J({props:{code:`from transformers import AutoModelForCausalLM, TrainingArguments, Trainer

model = AutoModelForCausalLM.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),ee=new J({props:{code:`training_args = TrainingArguments(
    output_dir="my_awesome_eli5_clm-model",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    weight_decay=0.01,
    push_to_hub=True,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=lm_dataset["train"],
    eval_dataset=lm_dataset["test"],
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_eli5_clm-model&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),ne=new J({props:{code:`import math

eval_results = trainer.evaluate()
print(f"Perplexity: {math.exp(eval_results['eval_loss']):.2f}")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> math

<span class="hljs-meta">&gt;&gt;&gt; </span>eval_results = trainer.evaluate()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Perplexity: <span class="hljs-subst">{math.exp(eval_results[<span class="hljs-string">&#x27;eval_loss&#x27;</span>]):<span class="hljs-number">.2</span>f}</span>&quot;</span>)
Perplexity: <span class="hljs-number">49.61</span>`}}),ie=new J({props:{code:"trainer.push_to_hub()",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()'}}),{c(){q(a.$$.fragment),$=n(`
You're ready to start training your model now! Load DistilGPT2 with [AutoModelForCausalLM](/docs/transformers/v4.25.1/en/model_doc/auto#transformers.AutoModelForCausalLM):

	`),q(o.$$.fragment),_=d(),k=p("p"),j=n("At this point, only three steps remain:"),v=d(),A=p("ol"),y=p("li"),E=n("Define your training hyperparameters in "),z=p("a"),I=n("TrainingArguments"),R=n(". The only required parameter is "),O=p("code"),H=n("output_dir"),U=n(" which specifies where to save your model. You\u2019ll push this model to the Hub by setting "),b=p("code"),T=n("push_to_hub=True"),S=n(" (you need to be signed in to Hugging Face to upload your model)."),N=d(),Y=p("li"),Q=n("Pass the training arguments to "),G=p("a"),B=n("Trainer"),u=n(" along with the model, datasets, and data collator."),x=d(),X=p("li"),V=n("Call "),Z=p("a"),ae=n("train()"),te=n(" to finetune your model."),ge=d(),q(ee.$$.fragment),ve=d(),le=p("p"),_e=n("Once training is completed, use the "),ue=p("a"),xe=n("evaluate()"),re=n(" method to evaluate your model and get its perplexity:"),$e=d(),q(ne.$$.fragment),Ee=d(),se=p("p"),ke=n("Then share your model to the Hub with the "),fe=p("a"),ce=n("push_to_hub()"),be=n(" method so everyone can use your model:"),oe=d(),q(ie.$$.fragment),this.h()},l(c){P(a.$$.fragment,c),$=l(c,`
You're ready to start training your model now! Load DistilGPT2 with [AutoModelForCausalLM](/docs/transformers/v4.25.1/en/model_doc/auto#transformers.AutoModelForCausalLM):

	`),P(o.$$.fragment,c),_=g(c),k=m(c,"P",{});var W=f(k);j=l(W,"At this point, only three steps remain:"),W.forEach(t),v=g(c),A=m(c,"OL",{});var r=f(A);y=m(r,"LI",{});var M=f(y);E=l(M,"Define your training hyperparameters in "),z=m(M,"A",{href:!0});var je=f(z);I=l(je,"TrainingArguments"),je.forEach(t),R=l(M,". The only required parameter is "),O=m(M,"CODE",{});var me=f(O);H=l(me,"output_dir"),me.forEach(t),U=l(M," which specifies where to save your model. You\u2019ll push this model to the Hub by setting "),b=m(M,"CODE",{});var ze=f(b);T=l(ze,"push_to_hub=True"),ze.forEach(t),S=l(M," (you need to be signed in to Hugging Face to upload your model)."),M.forEach(t),N=g(r),Y=m(r,"LI",{});var he=f(Y);Q=l(he,"Pass the training arguments to "),G=m(he,"A",{href:!0});var de=f(G);B=l(de,"Trainer"),de.forEach(t),u=l(he," along with the model, datasets, and data collator."),he.forEach(t),x=g(r),X=m(r,"LI",{});var Te=f(X);V=l(Te,"Call "),Z=m(Te,"A",{href:!0});var Ae=f(Z);ae=l(Ae,"train()"),Ae.forEach(t),te=l(Te," to finetune your model."),Te.forEach(t),r.forEach(t),ge=g(c),P(ee.$$.fragment,c),ve=g(c),le=m(c,"P",{});var ye=f(le);_e=l(ye,"Once training is completed, use the "),ue=m(ye,"A",{href:!0});var Me=f(ue);xe=l(Me,"evaluate()"),Me.forEach(t),re=l(ye," method to evaluate your model and get its perplexity:"),ye.forEach(t),$e=g(c),P(ne.$$.fragment,c),Ee=g(c),se=m(c,"P",{});var we=f(se);ke=l(we,"Then share your model to the Hub with the "),fe=m(we,"A",{href:!0});var qe=f(fe);ce=l(qe,"push_to_hub()"),qe.forEach(t),be=l(we," method so everyone can use your model:"),we.forEach(t),oe=g(c),P(ie.$$.fragment,c),this.h()},h(){w(z,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.TrainingArguments"),w(G,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.Trainer"),w(Z,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.Trainer.train"),w(ue,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.Trainer.evaluate"),w(fe,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.Trainer.push_to_hub")},m(c,W){F(a,c,W),i(c,$,W),F(o,c,W),i(c,_,W),i(c,k,W),s(k,j),i(c,v,W),i(c,A,W),s(A,y),s(y,E),s(y,z),s(z,I),s(y,R),s(y,O),s(O,H),s(y,U),s(y,b),s(b,T),s(y,S),s(A,N),s(A,Y),s(Y,Q),s(Y,G),s(G,B),s(Y,u),s(A,x),s(A,X),s(X,V),s(X,Z),s(Z,ae),s(X,te),i(c,ge,W),F(ee,c,W),i(c,ve,W),i(c,le,W),s(le,_e),s(le,ue),s(ue,xe),s(le,re),i(c,$e,W),F(ne,c,W),i(c,Ee,W),i(c,se,W),s(se,ke),s(se,fe),s(fe,ce),s(se,be),i(c,oe,W),F(ie,c,W),pe=!0},p(c,W){const r={};W&2&&(r.$$scope={dirty:W,ctx:c}),a.$set(r)},i(c){pe||(C(a.$$.fragment,c),C(o.$$.fragment,c),C(ee.$$.fragment,c),C(ne.$$.fragment,c),C(ie.$$.fragment,c),pe=!0)},o(c){D(a.$$.fragment,c),D(o.$$.fragment,c),D(ee.$$.fragment,c),D(ne.$$.fragment,c),D(ie.$$.fragment,c),pe=!1},d(c){L(a,c),c&&t($),L(o,c),c&&t(_),c&&t(k),c&&t(v),c&&t(A),c&&t(ge),L(ee,c),c&&t(ve),c&&t(le),c&&t($e),L(ne,c),c&&t(Ee),c&&t(se),c&&t(oe),L(ie,c)}}}function Fr(K){let a,$;return a=new Ne({props:{$$slots:{default:[Pr]},$$scope:{ctx:K}}}),{c(){q(a.$$.fragment)},l(o){P(a.$$.fragment,o)},m(o,_){F(a,o,_),$=!0},p(o,_){const k={};_&2&&(k.$$scope={dirty:_,ctx:o}),a.$set(k)},i(o){$||(C(a.$$.fragment,o),$=!0)},o(o){D(a.$$.fragment,o),$=!1},d(o){L(a,o)}}}function Cr(K){let a,$,o,_,k;return{c(){a=p("p"),$=n("If you aren\u2019t familiar with finetuning a model with Keras, take a look at the basic tutorial "),o=p("a"),_=n("here"),k=n("!"),this.h()},l(j){a=m(j,"P",{});var v=f(a);$=l(v,"If you aren\u2019t familiar with finetuning a model with Keras, take a look at the basic tutorial "),o=m(v,"A",{href:!0});var A=f(o);_=l(A,"here"),A.forEach(t),k=l(v,"!"),v.forEach(t),this.h()},h(){w(o,"href","../training#train-a-tensorflow-model-with-keras")},m(j,v){i(j,a,v),s(a,$),s(a,o),s(o,_),s(a,k)},d(j){j&&t(a)}}}function Dr(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U,b,T,S,N,Y,Q,G,B,u,x,X,V,Z,ae,te,ge,ee,ve,le,_e,ue,xe,re,$e,ne,Ee,se,ke,fe,ce,be,oe,ie,pe,c,W;return a=new Xt({props:{$$slots:{default:[Cr]},$$scope:{ctx:K}}}),o=new J({props:{code:`from transformers import create_optimizer, AdamWeightDecay

optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer, AdamWeightDecay

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamWeightDecay(learning_rate=<span class="hljs-number">2e-5</span>, weight_decay_rate=<span class="hljs-number">0.01</span>)`}}),z=new J({props:{code:`from transformers import TFAutoModelForCausalLM

model = TFAutoModelForCausalLM.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),Q=new J({props:{code:`tf_train_set = model.prepare_tf_dataset(
    lm_dataset["train"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_test_set = model.prepare_tf_dataset(
    lm_dataset["test"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_test_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),te=new J({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),re=new J({props:{code:`from transformers.keras_callbacks import PushToHubCallback

callback = PushToHubCallback(
    output_dir="my_awesome_eli5_clm-model",
    tokenizer=tokenizer,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>callback = PushToHubCallback(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_eli5_clm-model&quot;</span>,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>)`}}),oe=new J({props:{code:"model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3, callbacks=[callback])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=<span class="hljs-number">3</span>, callbacks=[callback])'}}),{c(){q(a.$$.fragment),$=n(`
To finetune a model in TensorFlow, start by setting up an optimizer function, learning rate schedule, and some training hyperparameters:

	`),q(o.$$.fragment),_=d(),k=p("p"),j=n("Then you can load DistilGPT2 with "),v=p("a"),A=n("TFAutoModelForCausalLM"),y=n(":"),E=d(),q(z.$$.fragment),I=d(),R=p("p"),O=n("Convert your datasets to the "),H=p("code"),U=n("tf.data.Dataset"),b=n(" format with "),T=p("a"),S=n("prepare_tf_dataset()"),N=n(":"),Y=d(),q(Q.$$.fragment),G=d(),B=p("p"),u=n("Configure the model for training with "),x=p("a"),X=p("code"),V=n("compile"),Z=n(":"),ae=d(),q(te.$$.fragment),ge=d(),ee=p("p"),ve=n("This can be done by specifying where to push your model and tokenizer in the "),le=p("a"),_e=n("PushToHubCallback"),ue=n(":"),xe=d(),q(re.$$.fragment),$e=d(),ne=p("p"),Ee=n("Finally, you\u2019re ready to start training your model! Call "),se=p("a"),ke=p("code"),fe=n("fit"),ce=n(" with your training and validation datasets, the number of epochs, and your callback to finetune the model:"),be=d(),q(oe.$$.fragment),ie=d(),pe=p("p"),c=n("Once training is completed, your model is automatically uploaded to the Hub so everyone can use it!"),this.h()},l(r){P(a.$$.fragment,r),$=l(r,`
To finetune a model in TensorFlow, start by setting up an optimizer function, learning rate schedule, and some training hyperparameters:

	`),P(o.$$.fragment,r),_=g(r),k=m(r,"P",{});var M=f(k);j=l(M,"Then you can load DistilGPT2 with "),v=m(M,"A",{href:!0});var je=f(v);A=l(je,"TFAutoModelForCausalLM"),je.forEach(t),y=l(M,":"),M.forEach(t),E=g(r),P(z.$$.fragment,r),I=g(r),R=m(r,"P",{});var me=f(R);O=l(me,"Convert your datasets to the "),H=m(me,"CODE",{});var ze=f(H);U=l(ze,"tf.data.Dataset"),ze.forEach(t),b=l(me," format with "),T=m(me,"A",{href:!0});var he=f(T);S=l(he,"prepare_tf_dataset()"),he.forEach(t),N=l(me,":"),me.forEach(t),Y=g(r),P(Q.$$.fragment,r),G=g(r),B=m(r,"P",{});var de=f(B);u=l(de,"Configure the model for training with "),x=m(de,"A",{href:!0,rel:!0});var Te=f(x);X=m(Te,"CODE",{});var Ae=f(X);V=l(Ae,"compile"),Ae.forEach(t),Te.forEach(t),Z=l(de,":"),de.forEach(t),ae=g(r),P(te.$$.fragment,r),ge=g(r),ee=m(r,"P",{});var ye=f(ee);ve=l(ye,"This can be done by specifying where to push your model and tokenizer in the "),le=m(ye,"A",{href:!0});var Me=f(le);_e=l(Me,"PushToHubCallback"),Me.forEach(t),ue=l(ye,":"),ye.forEach(t),xe=g(r),P(re.$$.fragment,r),$e=g(r),ne=m(r,"P",{});var we=f(ne);Ee=l(we,"Finally, you\u2019re ready to start training your model! Call "),se=m(we,"A",{href:!0,rel:!0});var qe=f(se);ke=m(qe,"CODE",{});var Ce=f(ke);fe=l(Ce,"fit"),Ce.forEach(t),qe.forEach(t),ce=l(we," with your training and validation datasets, the number of epochs, and your callback to finetune the model:"),we.forEach(t),be=g(r),P(oe.$$.fragment,r),ie=g(r),pe=m(r,"P",{});var Ye=f(pe);c=l(Ye,"Once training is completed, your model is automatically uploaded to the Hub so everyone can use it!"),Ye.forEach(t),this.h()},h(){w(v,"href","/docs/transformers/v4.25.1/en/model_doc/auto#transformers.TFAutoModelForCausalLM"),w(T,"href","/docs/transformers/v4.25.1/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset"),w(x,"href","https://keras.io/api/models/model_training_apis/#compile-method"),w(x,"rel","nofollow"),w(le,"href","/docs/transformers/v4.25.1/en/main_classes/keras_callbacks#transformers.PushToHubCallback"),w(se,"href","https://keras.io/api/models/model_training_apis/#fit-method"),w(se,"rel","nofollow")},m(r,M){F(a,r,M),i(r,$,M),F(o,r,M),i(r,_,M),i(r,k,M),s(k,j),s(k,v),s(v,A),s(k,y),i(r,E,M),F(z,r,M),i(r,I,M),i(r,R,M),s(R,O),s(R,H),s(H,U),s(R,b),s(R,T),s(T,S),s(R,N),i(r,Y,M),F(Q,r,M),i(r,G,M),i(r,B,M),s(B,u),s(B,x),s(x,X),s(X,V),s(B,Z),i(r,ae,M),F(te,r,M),i(r,ge,M),i(r,ee,M),s(ee,ve),s(ee,le),s(le,_e),s(ee,ue),i(r,xe,M),F(re,r,M),i(r,$e,M),i(r,ne,M),s(ne,Ee),s(ne,se),s(se,ke),s(ke,fe),s(ne,ce),i(r,be,M),F(oe,r,M),i(r,ie,M),i(r,pe,M),s(pe,c),W=!0},p(r,M){const je={};M&2&&(je.$$scope={dirty:M,ctx:r}),a.$set(je)},i(r){W||(C(a.$$.fragment,r),C(o.$$.fragment,r),C(z.$$.fragment,r),C(Q.$$.fragment,r),C(te.$$.fragment,r),C(re.$$.fragment,r),C(oe.$$.fragment,r),W=!0)},o(r){D(a.$$.fragment,r),D(o.$$.fragment,r),D(z.$$.fragment,r),D(Q.$$.fragment,r),D(te.$$.fragment,r),D(re.$$.fragment,r),D(oe.$$.fragment,r),W=!1},d(r){L(a,r),r&&t($),L(o,r),r&&t(_),r&&t(k),r&&t(E),L(z,r),r&&t(I),r&&t(R),r&&t(Y),L(Q,r),r&&t(G),r&&t(B),r&&t(ae),L(te,r),r&&t(ge),r&&t(ee),r&&t(xe),L(re,r),r&&t($e),r&&t(ne),r&&t(be),L(oe,r),r&&t(ie),r&&t(pe)}}}function Lr(K){let a,$;return a=new Ne({props:{$$slots:{default:[Dr]},$$scope:{ctx:K}}}),{c(){q(a.$$.fragment)},l(o){P(a.$$.fragment,o)},m(o,_){F(a,o,_),$=!0},p(o,_){const k={};_&2&&(k.$$scope={dirty:_,ctx:o}),a.$set(k)},i(o){$||(C(a.$$.fragment,o),$=!0)},o(o){D(a.$$.fragment,o),$=!1},d(o){L(a,o)}}}function Ir(K){let a,$,o,_,k,j,v,A;return{c(){a=p("p"),$=n(`For a more in-depth example of how to finetune a model for causal language modeling, take a look at the corresponding
`),o=p("a"),_=n("PyTorch notebook"),k=n(`
or `),j=p("a"),v=n("TensorFlow notebook"),A=n("."),this.h()},l(y){a=m(y,"P",{});var E=f(a);$=l(E,`For a more in-depth example of how to finetune a model for causal language modeling, take a look at the corresponding
`),o=m(E,"A",{href:!0,rel:!0});var z=f(o);_=l(z,"PyTorch notebook"),z.forEach(t),k=l(E,`
or `),j=m(E,"A",{href:!0,rel:!0});var I=f(j);v=l(I,"TensorFlow notebook"),I.forEach(t),A=l(E,"."),E.forEach(t),this.h()},h(){w(o,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb"),w(o,"rel","nofollow"),w(j,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb"),w(j,"rel","nofollow")},m(y,E){i(y,a,E),s(a,$),s(a,o),s(o,_),s(a,k),s(a,j),s(j,v),s(a,A)},d(y){y&&t(a)}}}function Or(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U,b,T,S,N,Y,Q,G,B;return v=new J({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("my_awesome_eli5_clm-model")
inputs = tokenizer(prompt, return_tensors="pt").input_ids`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;my_awesome_eli5_clm-model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids`}}),T=new J({props:{code:`from transformers import AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained("my_awesome_eli5_clm-model")
outputs = model.generate(inputs, max_new_tokens=100, do_sample=True, top_k=50, top_p=0.95)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;my_awesome_eli5_clm-model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(inputs, max_new_tokens=<span class="hljs-number">100</span>, do_sample=<span class="hljs-literal">True</span>, top_k=<span class="hljs-number">50</span>, top_p=<span class="hljs-number">0.95</span>)`}}),G=new J({props:{code:"tokenizer.batch_decode(outputs, skip_special_tokens=True)",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&quot;Somatic hypermutation allows the immune system to react to drugs with the ability to adapt to a different environmental situation. In other words, a system of &#x27;hypermutation&#x27; can help the immune system to adapt to a different environmental situation or in some cases even a single life. In contrast, researchers at the University of Massachusetts-Boston have found that &#x27;hypermutation&#x27; is much stronger in mice than in humans but can be found in humans, and that it&#x27;s not completely unknown to the immune system. A study on how the immune system&quot;</span>]`}}),{c(){a=p("p"),$=n("Tokenize the text and return the "),o=p("code"),_=n("input_ids"),k=n(" as PyTorch tensors:"),j=d(),q(v.$$.fragment),A=d(),y=p("p"),E=n("Use the "),z=p("a"),I=n("generate()"),R=n(" method to generate text. For more details about the different text generation strategies and parameters for controlling generation, check out the "),O=p("a"),H=n("Text Generation"),U=n(" API."),b=d(),q(T.$$.fragment),S=d(),N=p("p"),Y=n("Decode the generated token ids back into text:"),Q=d(),q(G.$$.fragment),this.h()},l(u){a=m(u,"P",{});var x=f(a);$=l(x,"Tokenize the text and return the "),o=m(x,"CODE",{});var X=f(o);_=l(X,"input_ids"),X.forEach(t),k=l(x," as PyTorch tensors:"),x.forEach(t),j=g(u),P(v.$$.fragment,u),A=g(u),y=m(u,"P",{});var V=f(y);E=l(V,"Use the "),z=m(V,"A",{href:!0});var Z=f(z);I=l(Z,"generate()"),Z.forEach(t),R=l(V," method to generate text. For more details about the different text generation strategies and parameters for controlling generation, check out the "),O=m(V,"A",{href:!0});var ae=f(O);H=l(ae,"Text Generation"),ae.forEach(t),U=l(V," API."),V.forEach(t),b=g(u),P(T.$$.fragment,u),S=g(u),N=m(u,"P",{});var te=f(N);Y=l(te,"Decode the generated token ids back into text:"),te.forEach(t),Q=g(u),P(G.$$.fragment,u),this.h()},h(){w(z,"href","/docs/transformers/v4.25.1/en/main_classes/text_generation#transformers.GenerationMixin.generate"),w(O,"href","./main_classes/text_generation")},m(u,x){i(u,a,x),s(a,$),s(a,o),s(o,_),s(a,k),i(u,j,x),F(v,u,x),i(u,A,x),i(u,y,x),s(y,E),s(y,z),s(z,I),s(y,R),s(y,O),s(O,H),s(y,U),i(u,b,x),F(T,u,x),i(u,S,x),i(u,N,x),s(N,Y),i(u,Q,x),F(G,u,x),B=!0},p:hs,i(u){B||(C(v.$$.fragment,u),C(T.$$.fragment,u),C(G.$$.fragment,u),B=!0)},o(u){D(v.$$.fragment,u),D(T.$$.fragment,u),D(G.$$.fragment,u),B=!1},d(u){u&&t(a),u&&t(j),L(v,u),u&&t(A),u&&t(y),u&&t(b),L(T,u),u&&t(S),u&&t(N),u&&t(Q),L(G,u)}}}function Hr(K){let a,$;return a=new Ne({props:{$$slots:{default:[Or]},$$scope:{ctx:K}}}),{c(){q(a.$$.fragment)},l(o){P(a.$$.fragment,o)},m(o,_){F(a,o,_),$=!0},p(o,_){const k={};_&2&&(k.$$scope={dirty:_,ctx:o}),a.$set(k)},i(o){$||(C(a.$$.fragment,o),$=!0)},o(o){D(a.$$.fragment,o),$=!1},d(o){L(a,o)}}}function Sr(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U,b,T,S,N,Y,Q,G,B;return v=new J({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("my_awesome_eli5_clm-model")
inputs = tokenizer(prompt, return_tensors="tf").input_ids`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;my_awesome_eli5_clm-model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>).input_ids`}}),T=new J({props:{code:`from transformers import TFAutoModelForCausalLM

model = TFAutoModelForCausalLM.from_pretrained("my_awesome_eli5_clm-model")
outputs = model.generate(input_ids=inputs, max_new_tokens=100, do_sample=True, top_k=50, top_p=0.95)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;my_awesome_eli5_clm-model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(input_ids=inputs, max_new_tokens=<span class="hljs-number">100</span>, do_sample=<span class="hljs-literal">True</span>, top_k=<span class="hljs-number">50</span>, top_p=<span class="hljs-number">0.95</span>)`}}),G=new J({props:{code:"tokenizer.batch_decode(outputs, skip_special_tokens=True)",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;Somatic hypermutation allows the immune system to detect the presence of other viruses as they become more prevalent. Therefore, researchers have identified a high proportion of human viruses. The proportion of virus-associated viruses in our study increases with age. Therefore, we propose a simple algorithm to detect the presence of these new viruses in our samples as a sign of improved immunity. A first study based on this algorithm, which will be published in Science on Friday, aims to show that this finding could translate into the development of a better vaccine that is more effective for&#x27;</span>]`}}),{c(){a=p("p"),$=n("Tokenize the text and return the "),o=p("code"),_=n("input_ids"),k=n(" as TensorFlow tensors:"),j=d(),q(v.$$.fragment),A=d(),y=p("p"),E=n("Use the "),z=p("a"),I=n("generate()"),R=n(" method to create the summarization. For more details about the different text generation strategies and parameters for controlling generation, check out the "),O=p("a"),H=n("Text Generation"),U=n(" API."),b=d(),q(T.$$.fragment),S=d(),N=p("p"),Y=n("Decode the generated token ids back into text:"),Q=d(),q(G.$$.fragment),this.h()},l(u){a=m(u,"P",{});var x=f(a);$=l(x,"Tokenize the text and return the "),o=m(x,"CODE",{});var X=f(o);_=l(X,"input_ids"),X.forEach(t),k=l(x," as TensorFlow tensors:"),x.forEach(t),j=g(u),P(v.$$.fragment,u),A=g(u),y=m(u,"P",{});var V=f(y);E=l(V,"Use the "),z=m(V,"A",{href:!0});var Z=f(z);I=l(Z,"generate()"),Z.forEach(t),R=l(V," method to create the summarization. For more details about the different text generation strategies and parameters for controlling generation, check out the "),O=m(V,"A",{href:!0});var ae=f(O);H=l(ae,"Text Generation"),ae.forEach(t),U=l(V," API."),V.forEach(t),b=g(u),P(T.$$.fragment,u),S=g(u),N=m(u,"P",{});var te=f(N);Y=l(te,"Decode the generated token ids back into text:"),te.forEach(t),Q=g(u),P(G.$$.fragment,u),this.h()},h(){w(z,"href","/docs/transformers/v4.25.1/en/main_classes/text_generation#transformers.TFGenerationMixin.generate"),w(O,"href","./main_classes/text_generation")},m(u,x){i(u,a,x),s(a,$),s(a,o),s(o,_),s(a,k),i(u,j,x),F(v,u,x),i(u,A,x),i(u,y,x),s(y,E),s(y,z),s(z,I),s(y,R),s(y,O),s(O,H),s(y,U),i(u,b,x),F(T,u,x),i(u,S,x),i(u,N,x),s(N,Y),i(u,Q,x),F(G,u,x),B=!0},p:hs,i(u){B||(C(v.$$.fragment,u),C(T.$$.fragment,u),C(G.$$.fragment,u),B=!0)},o(u){D(v.$$.fragment,u),D(T.$$.fragment,u),D(G.$$.fragment,u),B=!1},d(u){u&&t(a),u&&t(j),L(v,u),u&&t(A),u&&t(y),u&&t(b),L(T,u),u&&t(S),u&&t(N),u&&t(Q),L(G,u)}}}function Gr(K){let a,$;return a=new Ne({props:{$$slots:{default:[Sr]},$$scope:{ctx:K}}}),{c(){q(a.$$.fragment)},l(o){P(a.$$.fragment,o)},m(o,_){F(a,o,_),$=!0},p(o,_){const k={};_&2&&(k.$$scope={dirty:_,ctx:o}),a.$set(k)},i(o){$||(C(a.$$.fragment,o),$=!0)},o(o){D(a.$$.fragment,o),$=!1},d(o){L(a,o)}}}function Rr(K){let a,$,o,_,k,j,v,A;return{c(){a=p("p"),$=n("If you aren\u2019t familiar with finetuning a model with the "),o=p("a"),_=n("Trainer"),k=n(", take a look at the basic tutorial "),j=p("a"),v=n("here"),A=n("!"),this.h()},l(y){a=m(y,"P",{});var E=f(a);$=l(E,"If you aren\u2019t familiar with finetuning a model with the "),o=m(E,"A",{href:!0});var z=f(o);_=l(z,"Trainer"),z.forEach(t),k=l(E,", take a look at the basic tutorial "),j=m(E,"A",{href:!0});var I=f(j);v=l(I,"here"),I.forEach(t),A=l(E,"!"),E.forEach(t),this.h()},h(){w(o,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.Trainer"),w(j,"href","../training#train-with-pytorch-trainer")},m(y,E){i(y,a,E),s(a,$),s(a,o),s(o,_),s(a,k),s(a,j),s(j,v),s(a,A)},d(y){y&&t(a)}}}function Wr(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U,b,T,S,N,Y,Q,G,B,u,x,X,V,Z,ae,te,ge,ee,ve,le,_e,ue,xe,re,$e,ne,Ee,se,ke,fe,ce,be,oe,ie,pe;return a=new Xt({props:{$$slots:{default:[Rr]},$$scope:{ctx:K}}}),o=new J({props:{code:`from transformers import AutoModelForMaskedLM

model = AutoModelForMaskedLM.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),ee=new J({props:{code:`training_args = TrainingArguments(
    output_dir="my_awesome_eli5_mlm_model",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    num_train_epochs=3,
    weight_decay=0.01,
    push_to_hub=True,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=lm_dataset["train"],
    eval_dataset=lm_dataset["test"],
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_eli5_mlm_model&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),ne=new J({props:{code:`import math

eval_results = trainer.evaluate()
print(f"Perplexity: {math.exp(eval_results['eval_loss']):.2f}")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> math

<span class="hljs-meta">&gt;&gt;&gt; </span>eval_results = trainer.evaluate()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Perplexity: <span class="hljs-subst">{math.exp(eval_results[<span class="hljs-string">&#x27;eval_loss&#x27;</span>]):<span class="hljs-number">.2</span>f}</span>&quot;</span>)
Perplexity: <span class="hljs-number">8.76</span>`}}),ie=new J({props:{code:"trainer.push_to_hub()",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()'}}),{c(){q(a.$$.fragment),$=n(`
You're ready to start training your model now! Load DistilRoBERTa with [AutoModelForMaskedLM](/docs/transformers/v4.25.1/en/model_doc/auto#transformers.AutoModelForMaskedLM):

	`),q(o.$$.fragment),_=d(),k=p("p"),j=n("At this point, only three steps remain:"),v=d(),A=p("ol"),y=p("li"),E=n("Define your training hyperparameters in "),z=p("a"),I=n("TrainingArguments"),R=n(". The only required parameter is "),O=p("code"),H=n("output_dir"),U=n(" which specifies where to save your model. You\u2019ll push this model to the Hub by setting "),b=p("code"),T=n("push_to_hub=True"),S=n(" (you need to be signed in to Hugging Face to upload your model)."),N=d(),Y=p("li"),Q=n("Pass the training arguments to "),G=p("a"),B=n("Trainer"),u=n(" along with the model, datasets, and data collator."),x=d(),X=p("li"),V=n("Call "),Z=p("a"),ae=n("train()"),te=n(" to finetune your model."),ge=d(),q(ee.$$.fragment),ve=d(),le=p("p"),_e=n("Once training is completed, use the "),ue=p("a"),xe=n("evaluate()"),re=n(" method to evaluate your model and get its perplexity:"),$e=d(),q(ne.$$.fragment),Ee=d(),se=p("p"),ke=n("Then share your model to the Hub with the "),fe=p("a"),ce=n("push_to_hub()"),be=n(" method so everyone can use your model:"),oe=d(),q(ie.$$.fragment),this.h()},l(c){P(a.$$.fragment,c),$=l(c,`
You're ready to start training your model now! Load DistilRoBERTa with [AutoModelForMaskedLM](/docs/transformers/v4.25.1/en/model_doc/auto#transformers.AutoModelForMaskedLM):

	`),P(o.$$.fragment,c),_=g(c),k=m(c,"P",{});var W=f(k);j=l(W,"At this point, only three steps remain:"),W.forEach(t),v=g(c),A=m(c,"OL",{});var r=f(A);y=m(r,"LI",{});var M=f(y);E=l(M,"Define your training hyperparameters in "),z=m(M,"A",{href:!0});var je=f(z);I=l(je,"TrainingArguments"),je.forEach(t),R=l(M,". The only required parameter is "),O=m(M,"CODE",{});var me=f(O);H=l(me,"output_dir"),me.forEach(t),U=l(M," which specifies where to save your model. You\u2019ll push this model to the Hub by setting "),b=m(M,"CODE",{});var ze=f(b);T=l(ze,"push_to_hub=True"),ze.forEach(t),S=l(M," (you need to be signed in to Hugging Face to upload your model)."),M.forEach(t),N=g(r),Y=m(r,"LI",{});var he=f(Y);Q=l(he,"Pass the training arguments to "),G=m(he,"A",{href:!0});var de=f(G);B=l(de,"Trainer"),de.forEach(t),u=l(he," along with the model, datasets, and data collator."),he.forEach(t),x=g(r),X=m(r,"LI",{});var Te=f(X);V=l(Te,"Call "),Z=m(Te,"A",{href:!0});var Ae=f(Z);ae=l(Ae,"train()"),Ae.forEach(t),te=l(Te," to finetune your model."),Te.forEach(t),r.forEach(t),ge=g(c),P(ee.$$.fragment,c),ve=g(c),le=m(c,"P",{});var ye=f(le);_e=l(ye,"Once training is completed, use the "),ue=m(ye,"A",{href:!0});var Me=f(ue);xe=l(Me,"evaluate()"),Me.forEach(t),re=l(ye," method to evaluate your model and get its perplexity:"),ye.forEach(t),$e=g(c),P(ne.$$.fragment,c),Ee=g(c),se=m(c,"P",{});var we=f(se);ke=l(we,"Then share your model to the Hub with the "),fe=m(we,"A",{href:!0});var qe=f(fe);ce=l(qe,"push_to_hub()"),qe.forEach(t),be=l(we," method so everyone can use your model:"),we.forEach(t),oe=g(c),P(ie.$$.fragment,c),this.h()},h(){w(z,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.TrainingArguments"),w(G,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.Trainer"),w(Z,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.Trainer.train"),w(ue,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.Trainer.evaluate"),w(fe,"href","/docs/transformers/v4.25.1/en/main_classes/trainer#transformers.Trainer.push_to_hub")},m(c,W){F(a,c,W),i(c,$,W),F(o,c,W),i(c,_,W),i(c,k,W),s(k,j),i(c,v,W),i(c,A,W),s(A,y),s(y,E),s(y,z),s(z,I),s(y,R),s(y,O),s(O,H),s(y,U),s(y,b),s(b,T),s(y,S),s(A,N),s(A,Y),s(Y,Q),s(Y,G),s(G,B),s(Y,u),s(A,x),s(A,X),s(X,V),s(X,Z),s(Z,ae),s(X,te),i(c,ge,W),F(ee,c,W),i(c,ve,W),i(c,le,W),s(le,_e),s(le,ue),s(ue,xe),s(le,re),i(c,$e,W),F(ne,c,W),i(c,Ee,W),i(c,se,W),s(se,ke),s(se,fe),s(fe,ce),s(se,be),i(c,oe,W),F(ie,c,W),pe=!0},p(c,W){const r={};W&2&&(r.$$scope={dirty:W,ctx:c}),a.$set(r)},i(c){pe||(C(a.$$.fragment,c),C(o.$$.fragment,c),C(ee.$$.fragment,c),C(ne.$$.fragment,c),C(ie.$$.fragment,c),pe=!0)},o(c){D(a.$$.fragment,c),D(o.$$.fragment,c),D(ee.$$.fragment,c),D(ne.$$.fragment,c),D(ie.$$.fragment,c),pe=!1},d(c){L(a,c),c&&t($),L(o,c),c&&t(_),c&&t(k),c&&t(v),c&&t(A),c&&t(ge),L(ee,c),c&&t(ve),c&&t(le),c&&t($e),L(ne,c),c&&t(Ee),c&&t(se),c&&t(oe),L(ie,c)}}}function Nr(K){let a,$;return a=new Ne({props:{$$slots:{default:[Wr]},$$scope:{ctx:K}}}),{c(){q(a.$$.fragment)},l(o){P(a.$$.fragment,o)},m(o,_){F(a,o,_),$=!0},p(o,_){const k={};_&2&&(k.$$scope={dirty:_,ctx:o}),a.$set(k)},i(o){$||(C(a.$$.fragment,o),$=!0)},o(o){D(a.$$.fragment,o),$=!1},d(o){L(a,o)}}}function Yr(K){let a,$,o,_,k;return{c(){a=p("p"),$=n("If you aren\u2019t familiar with finetuning a model with Keras, take a look at the basic tutorial "),o=p("a"),_=n("here"),k=n("!"),this.h()},l(j){a=m(j,"P",{});var v=f(a);$=l(v,"If you aren\u2019t familiar with finetuning a model with Keras, take a look at the basic tutorial "),o=m(v,"A",{href:!0});var A=f(o);_=l(A,"here"),A.forEach(t),k=l(v,"!"),v.forEach(t),this.h()},h(){w(o,"href","../training#train-a-tensorflow-model-with-keras")},m(j,v){i(j,a,v),s(a,$),s(a,o),s(o,_),s(a,k)},d(j){j&&t(a)}}}function Br(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U,b,T,S,N,Y,Q,G,B,u,x,X,V,Z,ae,te,ge,ee,ve,le,_e,ue,xe,re,$e,ne,Ee,se,ke,fe,ce,be,oe,ie,pe,c,W;return a=new Xt({props:{$$slots:{default:[Yr]},$$scope:{ctx:K}}}),o=new J({props:{code:`from transformers import create_optimizer, AdamWeightDecay

optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer, AdamWeightDecay

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamWeightDecay(learning_rate=<span class="hljs-number">2e-5</span>, weight_decay_rate=<span class="hljs-number">0.01</span>)`}}),z=new J({props:{code:`from transformers import TFAutoModelForMaskedLM

model = TFAutoModelForMaskedLM.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),Q=new J({props:{code:`tf_train_set = model.prepare_tf_dataset(
    lm_dataset["train"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_test_set = model.prepare_tf_dataset(
    lm_dataset["test"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_test_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),te=new J({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),re=new J({props:{code:`from transformers.keras_callbacks import PushToHubCallback

callback = PushToHubCallback(
    output_dir="my_awesome_eli5_mlm_model",
    tokenizer=tokenizer,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>callback = PushToHubCallback(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_eli5_mlm_model&quot;</span>,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>)`}}),oe=new J({props:{code:"model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3, callbacks=[callback])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=<span class="hljs-number">3</span>, callbacks=[callback])'}}),{c(){q(a.$$.fragment),$=n(`
To finetune a model in TensorFlow, start by setting up an optimizer function, learning rate schedule, and some training hyperparameters:

	`),q(o.$$.fragment),_=d(),k=p("p"),j=n("Then you can load DistilRoBERTa with "),v=p("a"),A=n("TFAutoModelForMaskedLM"),y=n(":"),E=d(),q(z.$$.fragment),I=d(),R=p("p"),O=n("Convert your datasets to the "),H=p("code"),U=n("tf.data.Dataset"),b=n(" format with "),T=p("a"),S=n("prepare_tf_dataset()"),N=n(":"),Y=d(),q(Q.$$.fragment),G=d(),B=p("p"),u=n("Configure the model for training with "),x=p("a"),X=p("code"),V=n("compile"),Z=n(":"),ae=d(),q(te.$$.fragment),ge=d(),ee=p("p"),ve=n("This can be done by specifying where to push your model and tokenizer in the "),le=p("a"),_e=n("PushToHubCallback"),ue=n(":"),xe=d(),q(re.$$.fragment),$e=d(),ne=p("p"),Ee=n("Finally, you\u2019re ready to start training your model! Call "),se=p("a"),ke=p("code"),fe=n("fit"),ce=n(" with your training and validation datasets, the number of epochs, and your callback to finetune the model:"),be=d(),q(oe.$$.fragment),ie=d(),pe=p("p"),c=n("Once training is completed, your model is automatically uploaded to the Hub so everyone can use it!"),this.h()},l(r){P(a.$$.fragment,r),$=l(r,`
To finetune a model in TensorFlow, start by setting up an optimizer function, learning rate schedule, and some training hyperparameters:

	`),P(o.$$.fragment,r),_=g(r),k=m(r,"P",{});var M=f(k);j=l(M,"Then you can load DistilRoBERTa with "),v=m(M,"A",{href:!0});var je=f(v);A=l(je,"TFAutoModelForMaskedLM"),je.forEach(t),y=l(M,":"),M.forEach(t),E=g(r),P(z.$$.fragment,r),I=g(r),R=m(r,"P",{});var me=f(R);O=l(me,"Convert your datasets to the "),H=m(me,"CODE",{});var ze=f(H);U=l(ze,"tf.data.Dataset"),ze.forEach(t),b=l(me," format with "),T=m(me,"A",{href:!0});var he=f(T);S=l(he,"prepare_tf_dataset()"),he.forEach(t),N=l(me,":"),me.forEach(t),Y=g(r),P(Q.$$.fragment,r),G=g(r),B=m(r,"P",{});var de=f(B);u=l(de,"Configure the model for training with "),x=m(de,"A",{href:!0,rel:!0});var Te=f(x);X=m(Te,"CODE",{});var Ae=f(X);V=l(Ae,"compile"),Ae.forEach(t),Te.forEach(t),Z=l(de,":"),de.forEach(t),ae=g(r),P(te.$$.fragment,r),ge=g(r),ee=m(r,"P",{});var ye=f(ee);ve=l(ye,"This can be done by specifying where to push your model and tokenizer in the "),le=m(ye,"A",{href:!0});var Me=f(le);_e=l(Me,"PushToHubCallback"),Me.forEach(t),ue=l(ye,":"),ye.forEach(t),xe=g(r),P(re.$$.fragment,r),$e=g(r),ne=m(r,"P",{});var we=f(ne);Ee=l(we,"Finally, you\u2019re ready to start training your model! Call "),se=m(we,"A",{href:!0,rel:!0});var qe=f(se);ke=m(qe,"CODE",{});var Ce=f(ke);fe=l(Ce,"fit"),Ce.forEach(t),qe.forEach(t),ce=l(we," with your training and validation datasets, the number of epochs, and your callback to finetune the model:"),we.forEach(t),be=g(r),P(oe.$$.fragment,r),ie=g(r),pe=m(r,"P",{});var Ye=f(pe);c=l(Ye,"Once training is completed, your model is automatically uploaded to the Hub so everyone can use it!"),Ye.forEach(t),this.h()},h(){w(v,"href","/docs/transformers/v4.25.1/en/model_doc/auto#transformers.TFAutoModelForMaskedLM"),w(T,"href","/docs/transformers/v4.25.1/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset"),w(x,"href","https://keras.io/api/models/model_training_apis/#compile-method"),w(x,"rel","nofollow"),w(le,"href","/docs/transformers/v4.25.1/en/main_classes/keras_callbacks#transformers.PushToHubCallback"),w(se,"href","https://keras.io/api/models/model_training_apis/#fit-method"),w(se,"rel","nofollow")},m(r,M){F(a,r,M),i(r,$,M),F(o,r,M),i(r,_,M),i(r,k,M),s(k,j),s(k,v),s(v,A),s(k,y),i(r,E,M),F(z,r,M),i(r,I,M),i(r,R,M),s(R,O),s(R,H),s(H,U),s(R,b),s(R,T),s(T,S),s(R,N),i(r,Y,M),F(Q,r,M),i(r,G,M),i(r,B,M),s(B,u),s(B,x),s(x,X),s(X,V),s(B,Z),i(r,ae,M),F(te,r,M),i(r,ge,M),i(r,ee,M),s(ee,ve),s(ee,le),s(le,_e),s(ee,ue),i(r,xe,M),F(re,r,M),i(r,$e,M),i(r,ne,M),s(ne,Ee),s(ne,se),s(se,ke),s(ke,fe),s(ne,ce),i(r,be,M),F(oe,r,M),i(r,ie,M),i(r,pe,M),s(pe,c),W=!0},p(r,M){const je={};M&2&&(je.$$scope={dirty:M,ctx:r}),a.$set(je)},i(r){W||(C(a.$$.fragment,r),C(o.$$.fragment,r),C(z.$$.fragment,r),C(Q.$$.fragment,r),C(te.$$.fragment,r),C(re.$$.fragment,r),C(oe.$$.fragment,r),W=!0)},o(r){D(a.$$.fragment,r),D(o.$$.fragment,r),D(z.$$.fragment,r),D(Q.$$.fragment,r),D(te.$$.fragment,r),D(re.$$.fragment,r),D(oe.$$.fragment,r),W=!1},d(r){L(a,r),r&&t($),L(o,r),r&&t(_),r&&t(k),r&&t(E),L(z,r),r&&t(I),r&&t(R),r&&t(Y),L(Q,r),r&&t(G),r&&t(B),r&&t(ae),L(te,r),r&&t(ge),r&&t(ee),r&&t(xe),L(re,r),r&&t($e),r&&t(ne),r&&t(be),L(oe,r),r&&t(ie),r&&t(pe)}}}function Ur(K){let a,$;return a=new Ne({props:{$$slots:{default:[Br]},$$scope:{ctx:K}}}),{c(){q(a.$$.fragment)},l(o){P(a.$$.fragment,o)},m(o,_){F(a,o,_),$=!0},p(o,_){const k={};_&2&&(k.$$scope={dirty:_,ctx:o}),a.$set(k)},i(o){$||(C(a.$$.fragment,o),$=!0)},o(o){D(a.$$.fragment,o),$=!1},d(o){L(a,o)}}}function Jr(K){let a,$,o,_,k,j,v,A;return{c(){a=p("p"),$=n(`For a more in-depth example of how to finetune a model for masked language modeling, take a look at the corresponding
`),o=p("a"),_=n("PyTorch notebook"),k=n(`
or `),j=p("a"),v=n("TensorFlow notebook"),A=n("."),this.h()},l(y){a=m(y,"P",{});var E=f(a);$=l(E,`For a more in-depth example of how to finetune a model for masked language modeling, take a look at the corresponding
`),o=m(E,"A",{href:!0,rel:!0});var z=f(o);_=l(z,"PyTorch notebook"),z.forEach(t),k=l(E,`
or `),j=m(E,"A",{href:!0,rel:!0});var I=f(j);v=l(I,"TensorFlow notebook"),I.forEach(t),A=l(E,"."),E.forEach(t),this.h()},h(){w(o,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb"),w(o,"rel","nofollow"),w(j,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb"),w(j,"rel","nofollow")},m(y,E){i(y,a,E),s(a,$),s(a,o),s(o,_),s(a,k),s(a,j),s(j,v),s(a,A)},d(y){y&&t(a)}}}function Kr(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U,b,T,S,N,Y,Q,G,B;return E=new J({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("my_awesome_eli5_mlm_model")
inputs = tokenizer(text, return_tensors="pt")
mask_token_index = torch.where(inputs["input_ids"] == tokenizer.mask_token_id)[1]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;my_awesome_eli5_mlm_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_token_index = torch.where(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>] == tokenizer.mask_token_id)[<span class="hljs-number">1</span>]`}}),T=new J({props:{code:`from transformers import AutoModelForMaskedLM

model = AutoModelForMaskedLM.from_pretrained("stevhliu/my_awesome_eli5_mlm_model")
logits = model(**inputs).logits
mask_token_logits = logits[0, mask_token_index, :]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_eli5_mlm_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>logits = model(**inputs).logits
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_token_logits = logits[<span class="hljs-number">0</span>, mask_token_index, :]`}}),G=new J({props:{code:`top_3_tokens = torch.topk(mask_token_logits, 3, dim=1).indices[0].tolist()

for token in top_3_tokens:
    print(text.replace(tokenizer.mask_token, tokenizer.decode([token])))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>top_3_tokens = torch.topk(mask_token_logits, <span class="hljs-number">3</span>, dim=<span class="hljs-number">1</span>).indices[<span class="hljs-number">0</span>].tolist()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> token <span class="hljs-keyword">in</span> top_3_tokens:
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(text.replace(tokenizer.mask_token, tokenizer.decode([token])))
The Milky Way <span class="hljs-keyword">is</span> a spiral galaxy.
The Milky Way <span class="hljs-keyword">is</span> a massive galaxy.
The Milky Way <span class="hljs-keyword">is</span> a small galaxy.`}}),{c(){a=p("p"),$=n("Tokenize the text and return the "),o=p("code"),_=n("input_ids"),k=n(" as PyTorch tensors. You\u2019ll also need to specify the position of the "),j=p("code"),v=n("<mask>"),A=n(" token:"),y=d(),q(E.$$.fragment),z=d(),I=p("p"),R=n("Pass your inputs to the model and return the "),O=p("code"),H=n("logits"),U=n(" of the masked token:"),b=d(),q(T.$$.fragment),S=d(),N=p("p"),Y=n("Then return the three masked tokens with the highest probability and print them out:"),Q=d(),q(G.$$.fragment)},l(u){a=m(u,"P",{});var x=f(a);$=l(x,"Tokenize the text and return the "),o=m(x,"CODE",{});var X=f(o);_=l(X,"input_ids"),X.forEach(t),k=l(x," as PyTorch tensors. You\u2019ll also need to specify the position of the "),j=m(x,"CODE",{});var V=f(j);v=l(V,"<mask>"),V.forEach(t),A=l(x," token:"),x.forEach(t),y=g(u),P(E.$$.fragment,u),z=g(u),I=m(u,"P",{});var Z=f(I);R=l(Z,"Pass your inputs to the model and return the "),O=m(Z,"CODE",{});var ae=f(O);H=l(ae,"logits"),ae.forEach(t),U=l(Z," of the masked token:"),Z.forEach(t),b=g(u),P(T.$$.fragment,u),S=g(u),N=m(u,"P",{});var te=f(N);Y=l(te,"Then return the three masked tokens with the highest probability and print them out:"),te.forEach(t),Q=g(u),P(G.$$.fragment,u)},m(u,x){i(u,a,x),s(a,$),s(a,o),s(o,_),s(a,k),s(a,j),s(j,v),s(a,A),i(u,y,x),F(E,u,x),i(u,z,x),i(u,I,x),s(I,R),s(I,O),s(O,H),s(I,U),i(u,b,x),F(T,u,x),i(u,S,x),i(u,N,x),s(N,Y),i(u,Q,x),F(G,u,x),B=!0},p:hs,i(u){B||(C(E.$$.fragment,u),C(T.$$.fragment,u),C(G.$$.fragment,u),B=!0)},o(u){D(E.$$.fragment,u),D(T.$$.fragment,u),D(G.$$.fragment,u),B=!1},d(u){u&&t(a),u&&t(y),L(E,u),u&&t(z),u&&t(I),u&&t(b),L(T,u),u&&t(S),u&&t(N),u&&t(Q),L(G,u)}}}function Qr(K){let a,$;return a=new Ne({props:{$$slots:{default:[Kr]},$$scope:{ctx:K}}}),{c(){q(a.$$.fragment)},l(o){P(a.$$.fragment,o)},m(o,_){F(a,o,_),$=!0},p(o,_){const k={};_&2&&(k.$$scope={dirty:_,ctx:o}),a.$set(k)},i(o){$||(C(a.$$.fragment,o),$=!0)},o(o){D(a.$$.fragment,o),$=!1},d(o){L(a,o)}}}function Vr(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U,b,T,S,N,Y,Q,G,B;return E=new J({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("my_awesome_eli5_mlm_model")
inputs = tokenizer(text, return_tensors="tf")
mask_token_index = tf.where(inputs["input_ids"] == tokenizer.mask_token_id)[0, 1]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;my_awesome_eli5_mlm_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_token_index = tf.where(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>] == tokenizer.mask_token_id)[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]`}}),T=new J({props:{code:`from transformers import TFAutoModelForMaskedLM

model = TFAutoModelForMaskedLM.from_pretrained("stevhliu/my_awesome_eli5_mlm_model")
logits = model(**inputs).logits
mask_token_logits = logits[0, mask_token_index, :]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_eli5_mlm_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>logits = model(**inputs).logits
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_token_logits = logits[<span class="hljs-number">0</span>, mask_token_index, :]`}}),G=new J({props:{code:`top_3_tokens = tf.math.top_k(mask_token_logits, 3).indices.numpy()

for token in top_3_tokens:
    print(text.replace(tokenizer.mask_token, tokenizer.decode([token])))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>top_3_tokens = tf.math.top_k(mask_token_logits, <span class="hljs-number">3</span>).indices.numpy()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> token <span class="hljs-keyword">in</span> top_3_tokens:
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(text.replace(tokenizer.mask_token, tokenizer.decode([token])))
The Milky Way <span class="hljs-keyword">is</span> a spiral galaxy.
The Milky Way <span class="hljs-keyword">is</span> a massive galaxy.
The Milky Way <span class="hljs-keyword">is</span> a small galaxy.`}}),{c(){a=p("p"),$=n("Tokenize the text and return the "),o=p("code"),_=n("input_ids"),k=n(" as TensorFlow tensors. You\u2019ll also need to specify the position of the "),j=p("code"),v=n("<mask>"),A=n(" token:"),y=d(),q(E.$$.fragment),z=d(),I=p("p"),R=n("Pass your inputs to the model and return the "),O=p("code"),H=n("logits"),U=n(" of the masked token:"),b=d(),q(T.$$.fragment),S=d(),N=p("p"),Y=n("Then return the three masked tokens with the highest probability and print them out:"),Q=d(),q(G.$$.fragment)},l(u){a=m(u,"P",{});var x=f(a);$=l(x,"Tokenize the text and return the "),o=m(x,"CODE",{});var X=f(o);_=l(X,"input_ids"),X.forEach(t),k=l(x," as TensorFlow tensors. You\u2019ll also need to specify the position of the "),j=m(x,"CODE",{});var V=f(j);v=l(V,"<mask>"),V.forEach(t),A=l(x," token:"),x.forEach(t),y=g(u),P(E.$$.fragment,u),z=g(u),I=m(u,"P",{});var Z=f(I);R=l(Z,"Pass your inputs to the model and return the "),O=m(Z,"CODE",{});var ae=f(O);H=l(ae,"logits"),ae.forEach(t),U=l(Z," of the masked token:"),Z.forEach(t),b=g(u),P(T.$$.fragment,u),S=g(u),N=m(u,"P",{});var te=f(N);Y=l(te,"Then return the three masked tokens with the highest probability and print them out:"),te.forEach(t),Q=g(u),P(G.$$.fragment,u)},m(u,x){i(u,a,x),s(a,$),s(a,o),s(o,_),s(a,k),s(a,j),s(j,v),s(a,A),i(u,y,x),F(E,u,x),i(u,z,x),i(u,I,x),s(I,R),s(I,O),s(O,H),s(I,U),i(u,b,x),F(T,u,x),i(u,S,x),i(u,N,x),s(N,Y),i(u,Q,x),F(G,u,x),B=!0},p:hs,i(u){B||(C(E.$$.fragment,u),C(T.$$.fragment,u),C(G.$$.fragment,u),B=!0)},o(u){D(E.$$.fragment,u),D(T.$$.fragment,u),D(G.$$.fragment,u),B=!1},d(u){u&&t(a),u&&t(y),L(E,u),u&&t(z),u&&t(I),u&&t(b),L(T,u),u&&t(S),u&&t(N),u&&t(Q),L(G,u)}}}function Xr(K){let a,$;return a=new Ne({props:{$$slots:{default:[Vr]},$$scope:{ctx:K}}}),{c(){q(a.$$.fragment)},l(o){P(a.$$.fragment,o)},m(o,_){F(a,o,_),$=!0},p(o,_){const k={};_&2&&(k.$$scope={dirty:_,ctx:o}),a.$set(k)},i(o){$||(C(a.$$.fragment,o),$=!0)},o(o){D(a.$$.fragment,o),$=!1},d(o){L(a,o)}}}function Zr(K){let a,$,o,_,k,j,v,A,y,E,z,I,R,O,H,U,b,T,S,N,Y,Q,G,B,u,x,X,V,Z,ae,te,ge,ee,ve,le,_e,ue,xe,re,$e,ne,Ee,se,ke,fe,ce,be,oe,ie,pe,c,W,r,M,je,me,ze,he,de,Te,Ae,ye,Me,we,qe,Ce,Ye,na,bt,la,Le,wn,cs,bn,jn,jt,vn,xn,oa,vt,ra,Zt,En,ia,xt,pa,Ie,Tn,ds,An,Mn,gs,zn,qn,ma,Be,et,_s,Et,Pn,$s,Fn,fa,Tt,ua,tt,Cn,ks,Dn,Ln,ha,At,ca,Mt,da,st,In,ys,On,Hn,ga,zt,_a,Pe,Sn,ws,Gn,Rn,bs,Wn,Nn,js,Yn,Bn,qt,vs,Un,Jn,$a,Pt,ka,Oe,Kn,xs,Qn,Vn,Es,Xn,Zn,ya,es,el,wa,Ft,ba,Fe,tl,Ct,sl,al,Ts,nl,ll,As,ol,rl,Ms,il,pl,ja,Dt,va,ts,ml,xa,at,zs,fl,ul,Lt,hl,qs,cl,dl,Ea,It,Ta,nt,gl,Ps,_l,$l,Aa,Ot,Ma,He,kl,ss,yl,wl,Fs,bl,jl,za,lt,qa,Ue,ot,Cs,Ht,vl,Ds,xl,Pa,rt,El,St,Tl,Al,Fa,Je,it,Ls,Gt,Ml,Is,zl,Ca,pt,Da,mt,La,Ke,ft,Os,Rt,ql,Hs,Pl,Ia,as,Fl,Oa,ns,Cl,Ha,Wt,Sa,Se,Dl,ls,Ll,Il,Ss,Ol,Hl,Ga,Nt,Ra,ut,Wa,Qe,ht,Gs,Yt,Sl,Rs,Gl,Na,ct,Rl,Bt,Wl,Nl,Ya,Ve,dt,Ws,Ut,Yl,Ns,Bl,Ba,gt,Ua,_t,Ja,Xe,$t,Ys,Jt,Ul,Bs,Jl,Ka,os,Kl,Qa,kt,Ql,Us,Vl,Xl,Va,Kt,Xa,De,Zl,rs,eo,to,Js,so,ao,Ks,no,lo,Za,Qt,en,yt,tn;return j=new Ze({}),O=new yn({props:{id:"Vpjb1lu0MDk"}}),S=new yn({props:{id:"mqElG5QJWUg"}}),ce=new Xt({props:{$$slots:{default:[Er]},$$scope:{ctx:K}}}),c=new J({props:{code:"pip install transformers datasets evaluate",highlighted:"pip install transformers datasets evaluate"}}),me=new J({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_login()`}}),Ae=new Ze({}),bt=new J({props:{code:`from datasets import load_dataset

eli5 = load_dataset("eli5", split="train_asks[:5000]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = load_dataset(<span class="hljs-string">&quot;eli5&quot;</span>, split=<span class="hljs-string">&quot;train_asks[:5000]&quot;</span>)`}}),vt=new J({props:{code:"eli5 = eli5.train_test_split(test_size=0.2)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = eli5.train_test_split(test_size=<span class="hljs-number">0.2</span>)'}}),xt=new J({props:{code:'eli5["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>eli5[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers&#x27;</span>: {<span class="hljs-string">&#x27;a_id&#x27;</span>: [<span class="hljs-string">&#x27;c3d1aib&#x27;</span>, <span class="hljs-string">&#x27;c3d4lya&#x27;</span>],
  <span class="hljs-string">&#x27;score&#x27;</span>: [<span class="hljs-number">6</span>, <span class="hljs-number">3</span>],
  <span class="hljs-string">&#x27;text&#x27;</span>: [<span class="hljs-string">&quot;The velocity needed to remain in orbit is equal to the square root of Newton&#x27;s constant times the mass of earth divided by the distance from the center of the earth. I don&#x27;t know the altitude of that specific mission, but they&#x27;re usually around 300 km. That means he&#x27;s going 7-8 km/s.\\n\\nIn space there are no other forces acting on either the shuttle or the guy, so they stay in the same position relative to each other. If he were to become unable to return to the ship, he would presumably run out of oxygen, or slowly fall into the atmosphere and burn up.&quot;</span>,
   <span class="hljs-string">&quot;Hope you don&#x27;t mind me asking another question, but why aren&#x27;t there any stars visible in this photo?&quot;</span>]},
 <span class="hljs-string">&#x27;answers_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: []},
 <span class="hljs-string">&#x27;document&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
 <span class="hljs-string">&#x27;q_id&#x27;</span>: <span class="hljs-string">&#x27;nyxfp&#x27;</span>,
 <span class="hljs-string">&#x27;selftext&#x27;</span>: <span class="hljs-string">&#x27;_URL_0_\\n\\nThis was on the front page earlier and I have a few questions about it. Is it possible to calculate how fast the astronaut would be orbiting the earth? Also how does he stay close to the shuttle so that he can return safely, i.e is he orbiting at the same speed and can therefore stay next to it? And finally if his propulsion system failed, would he eventually re-enter the atmosphere and presumably die?&#x27;</span>,
 <span class="hljs-string">&#x27;selftext_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: [<span class="hljs-string">&#x27;http://apod.nasa.gov/apod/image/1201/freeflyer_nasa_3000.jpg&#x27;</span>]},
 <span class="hljs-string">&#x27;subreddit&#x27;</span>: <span class="hljs-string">&#x27;askscience&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;Few questions about this space walk photograph.&#x27;</span>,
 <span class="hljs-string">&#x27;title_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: []}}`}}),Et=new Ze({}),Tt=new yn({props:{id:"ma1TrR7gE7I"}}),At=new J({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),Mt=new yn({props:{id:"8PmhEIXhBvI"}}),zt=new J({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),Pt=new J({props:{code:`eli5 = eli5.flatten()
eli5["train"][0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = eli5.flatten()
<span class="hljs-meta">&gt;&gt;&gt; </span>eli5[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers.a_id&#x27;</span>: [<span class="hljs-string">&#x27;c3d1aib&#x27;</span>, <span class="hljs-string">&#x27;c3d4lya&#x27;</span>],
 <span class="hljs-string">&#x27;answers.score&#x27;</span>: [<span class="hljs-number">6</span>, <span class="hljs-number">3</span>],
 <span class="hljs-string">&#x27;answers.text&#x27;</span>: [<span class="hljs-string">&quot;The velocity needed to remain in orbit is equal to the square root of Newton&#x27;s constant times the mass of earth divided by the distance from the center of the earth. I don&#x27;t know the altitude of that specific mission, but they&#x27;re usually around 300 km. That means he&#x27;s going 7-8 km/s.\\n\\nIn space there are no other forces acting on either the shuttle or the guy, so they stay in the same position relative to each other. If he were to become unable to return to the ship, he would presumably run out of oxygen, or slowly fall into the atmosphere and burn up.&quot;</span>,
  <span class="hljs-string">&quot;Hope you don&#x27;t mind me asking another question, but why aren&#x27;t there any stars visible in this photo?&quot;</span>],
 <span class="hljs-string">&#x27;answers_urls.url&#x27;</span>: [],
 <span class="hljs-string">&#x27;document&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
 <span class="hljs-string">&#x27;q_id&#x27;</span>: <span class="hljs-string">&#x27;nyxfp&#x27;</span>,
 <span class="hljs-string">&#x27;selftext&#x27;</span>: <span class="hljs-string">&#x27;_URL_0_\\n\\nThis was on the front page earlier and I have a few questions about it. Is it possible to calculate how fast the astronaut would be orbiting the earth? Also how does he stay close to the shuttle so that he can return safely, i.e is he orbiting at the same speed and can therefore stay next to it? And finally if his propulsion system failed, would he eventually re-enter the atmosphere and presumably die?&#x27;</span>,
 <span class="hljs-string">&#x27;selftext_urls.url&#x27;</span>: [<span class="hljs-string">&#x27;http://apod.nasa.gov/apod/image/1201/freeflyer_nasa_3000.jpg&#x27;</span>],
 <span class="hljs-string">&#x27;subreddit&#x27;</span>: <span class="hljs-string">&#x27;askscience&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;Few questions about this space walk photograph.&#x27;</span>,
 <span class="hljs-string">&#x27;title_urls.url&#x27;</span>: []}`}}),Ft=new J({props:{code:`def preprocess_function(examples):
    return tokenizer([" ".join(x) for x in examples["answers.text"]], truncation=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer([<span class="hljs-string">&quot; &quot;</span>.join(x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;answers.text&quot;</span>]], truncation=<span class="hljs-literal">True</span>)`}}),Dt=new J({props:{code:`tokenized_eli5 = eli5.map(
    preprocess_function,
    batched=True,
    num_proc=4,
    remove_columns=eli5["train"].column_names,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_eli5 = eli5.<span class="hljs-built_in">map</span>(
<span class="hljs-meta">... </span>    preprocess_function,
<span class="hljs-meta">... </span>    batched=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    num_proc=<span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    remove_columns=eli5[<span class="hljs-string">&quot;train&quot;</span>].column_names,
<span class="hljs-meta">... </span>)`}}),It=new J({props:{code:`block_size = 128


def group_texts(examples):
    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
    total_length = len(concatenated_examples[list(examples.keys())[0]])
    total_length = (total_length // block_size) * block_size
    result = {
        k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
        for k, t in concatenated_examples.items()
    }
    result["labels"] = result["input_ids"].copy()
    return result`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>block_size = <span class="hljs-number">128</span>


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">group_texts</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    concatenated_examples = {k: <span class="hljs-built_in">sum</span>(examples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> examples.keys()}
<span class="hljs-meta">... </span>    total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-built_in">list</span>(examples.keys())[<span class="hljs-number">0</span>]])
<span class="hljs-meta">... </span>    total_length = (total_length // block_size) * block_size
<span class="hljs-meta">... </span>    result = {
<span class="hljs-meta">... </span>        k: [t[i : i + block_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, block_size)]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
<span class="hljs-meta">... </span>    }
<span class="hljs-meta">... </span>    result[<span class="hljs-string">&quot;labels&quot;</span>] = result[<span class="hljs-string">&quot;input_ids&quot;</span>].copy()
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> result`}}),Ot=new J({props:{code:"lm_dataset = tokenized_eli5.map(group_texts, batched=True, num_proc=4)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lm_dataset = tokenized_eli5.<span class="hljs-built_in">map</span>(group_texts, batched=<span class="hljs-literal">True</span>, num_proc=<span class="hljs-number">4</span>)'}}),lt=new aa({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[zr],pytorch:[Ar]},$$scope:{ctx:K}}}),Ht=new Ze({}),Gt=new Ze({}),pt=new aa({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Lr],pytorch:[Fr]},$$scope:{ctx:K}}}),mt=new Xt({props:{$$slots:{default:[Ir]},$$scope:{ctx:K}}}),Rt=new Ze({}),Wt=new J({props:{code:'prompt = "Somatic hypermutation allows the immune system to"',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;Somatic hypermutation allows the immune system to&quot;</span>'}}),Nt=new J({props:{code:`from transformers import pipeline

generator = pipeline("text-generation", model="my_awesome_eli5_clm-model")
generator(prompt)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>generator = pipeline(<span class="hljs-string">&quot;text-generation&quot;</span>, model=<span class="hljs-string">&quot;my_awesome_eli5_clm-model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generator(prompt)
[{<span class="hljs-string">&#x27;generated_text&#x27;</span>: <span class="hljs-string">&quot;Somatic hypermutation allows the immune system to be able to effectively reverse the damage caused by an infection.\\n\\n\\nThe damage caused by an infection is caused by the immune system&#x27;s ability to perform its own self-correcting tasks.&quot;</span>}]`}}),ut=new aa({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Gr],pytorch:[Hr]},$$scope:{ctx:K}}}),Yt=new Ze({}),Ut=new Ze({}),gt=new aa({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Ur],pytorch:[Nr]},$$scope:{ctx:K}}}),_t=new Xt({props:{$$slots:{default:[Jr]},$$scope:{ctx:K}}}),Jt=new Ze({}),Kt=new J({props:{code:'text = "The Milky Way is a <mask> galaxy."',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>text = <span class="hljs-string">&quot;The Milky Way is a &lt;mask&gt; galaxy.&quot;</span>'}}),Qt=new J({props:{code:`from transformers import pipeline

mask_filler = pipeline("fill-mask", "stevhliu/my_awesome_eli5_mlm_model")
mask_filler(text, top_k=3)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>mask_filler = pipeline(<span class="hljs-string">&quot;fill-mask&quot;</span>, <span class="hljs-string">&quot;stevhliu/my_awesome_eli5_mlm_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_filler(text, top_k=<span class="hljs-number">3</span>)
[{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.5150994658470154</span>,
  <span class="hljs-string">&#x27;token&#x27;</span>: <span class="hljs-number">21300</span>,
  <span class="hljs-string">&#x27;token_str&#x27;</span>: <span class="hljs-string">&#x27; spiral&#x27;</span>,
  <span class="hljs-string">&#x27;sequence&#x27;</span>: <span class="hljs-string">&#x27;The Milky Way is a spiral galaxy.&#x27;</span>},
 {<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.07087188959121704</span>,
  <span class="hljs-string">&#x27;token&#x27;</span>: <span class="hljs-number">2232</span>,
  <span class="hljs-string">&#x27;token_str&#x27;</span>: <span class="hljs-string">&#x27; massive&#x27;</span>,
  <span class="hljs-string">&#x27;sequence&#x27;</span>: <span class="hljs-string">&#x27;The Milky Way is a massive galaxy.&#x27;</span>},
 {<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.06434620916843414</span>,
  <span class="hljs-string">&#x27;token&#x27;</span>: <span class="hljs-number">650</span>,
  <span class="hljs-string">&#x27;token_str&#x27;</span>: <span class="hljs-string">&#x27; small&#x27;</span>,
  <span class="hljs-string">&#x27;sequence&#x27;</span>: <span class="hljs-string">&#x27;The Milky Way is a small galaxy.&#x27;</span>}]`}}),yt=new aa({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Xr],pytorch:[Qr]},$$scope:{ctx:K}}}),{c(){a=p("meta"),$=d(),o=p("h1"),_=p("a"),k=p("span"),q(j.$$.fragment),v=d(),A=p("span"),y=n("Language modeling"),E=d(),z=p("p"),I=n("Language modeling tasks predicts words in a sentence, making these types of models great at generating text. You can use these models for creative applications like choosing your own text adventure or an intelligent coding assistant like Copilot or CodeParrot. There are two types of language modeling, causal and masked."),R=d(),q(O.$$.fragment),H=d(),U=p("p"),b=n("Causal language modeling predicts the next token in a sequence of tokens, and the model can only attend to tokens on the left. This means the model cannot see future tokens. GPT-2 is an example of a causal language model."),T=d(),q(S.$$.fragment),N=d(),Y=p("p"),Q=n("Masked language modeling predicts a masked token in a sequence, and the model can attend to tokens bidirectionally. This means the model has full access to the tokens on the left and right. BERT is an example of a masked language model."),G=d(),B=p("p"),u=n("This guide will show you how to:"),x=d(),X=p("ol"),V=p("li"),Z=n("Finetune "),ae=p("a"),te=n("DistilGPT2"),ge=n(" for causal language modeling and "),ee=p("a"),ve=n("DistilRoBERTa"),le=n(" for masked language modeling on the "),_e=p("a"),ue=n("r/askscience"),xe=n(" subset of the "),re=p("a"),$e=n("ELI5"),ne=n(" dataset."),Ee=d(),se=p("li"),ke=n("Use your finetuned model for inference."),fe=d(),q(ce.$$.fragment),be=d(),oe=p("p"),ie=n("Before you begin, make sure you have all the necessary libraries installed:"),pe=d(),q(c.$$.fragment),W=d(),r=p("p"),M=n("We encourage you to login to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to login:"),je=d(),q(me.$$.fragment),ze=d(),he=p("h2"),de=p("a"),Te=p("span"),q(Ae.$$.fragment),ye=d(),Me=p("span"),we=n("Load ELI5 dataset"),qe=d(),Ce=p("p"),Ye=n("Start by loading a smaller subset of the r/askscience subset of the ELI5 dataset from the \u{1F917} Datasets library. This\u2019ll give you a chance to experiment and make sure everythings works before spending more time training on the full dataset."),na=d(),q(bt.$$.fragment),la=d(),Le=p("p"),wn=n("Split the dataset\u2019s "),cs=p("code"),bn=n("train_asks"),jn=n(" split into a train and test set with the "),jt=p("a"),vn=n("train_test_split"),xn=n(" method:"),oa=d(),q(vt.$$.fragment),ra=d(),Zt=p("p"),En=n("Then take a look at an example:"),ia=d(),q(xt.$$.fragment),pa=d(),Ie=p("p"),Tn=n("While this may look like a lot, you\u2019re only really interested in the "),ds=p("code"),An=n("text"),Mn=n(" field. What\u2019s cool about language modeling tasks is you don\u2019t need labels (also known as an unsupervised task) because the next word "),gs=p("em"),zn=n("is"),qn=n(" the label."),ma=d(),Be=p("h2"),et=p("a"),_s=p("span"),q(Et.$$.fragment),Pn=d(),$s=p("span"),Fn=n("Preprocess"),fa=d(),q(Tt.$$.fragment),ua=d(),tt=p("p"),Cn=n("For causal language modeling, the next step is to load a DistilGPT2 tokenizer to process the "),ks=p("code"),Dn=n("text"),Ln=n(" subfield:"),ha=d(),q(At.$$.fragment),ca=d(),q(Mt.$$.fragment),da=d(),st=p("p"),In=n("For masked language modeling, the next step is to load a DistilRoBERTa tokenizer to process the "),ys=p("code"),On=n("text"),Hn=n(" subfield:"),ga=d(),q(zt.$$.fragment),_a=d(),Pe=p("p"),Sn=n("You\u2019ll notice from the example above, the "),ws=p("code"),Gn=n("text"),Rn=n(" field is actually nested inside "),bs=p("code"),Wn=n("answers"),Nn=n(". This means you\u2019ll need to extract the "),js=p("code"),Yn=n("text"),Bn=n(" subfield from its nested structure with the "),qt=p("a"),vs=p("code"),Un=n("flatten"),Jn=n(" method:"),$a=d(),q(Pt.$$.fragment),ka=d(),Oe=p("p"),Kn=n("Each subfield is now a separate column as indicated by the "),xs=p("code"),Qn=n("answers"),Vn=n(" prefix, and the "),Es=p("code"),Xn=n("text"),Zn=n(" field is a list now. Instead of tokenizing each sentence separately, convert the list to a string so you can jointly tokenize them."),ya=d(),es=p("p"),el=n("Here is how you can create a preprocessing function to convert the list to a string, and truncate sequences to be no longer than DistilGPT2\u2019s maximum input length:"),wa=d(),q(Ft.$$.fragment),ba=d(),Fe=p("p"),tl=n("To apply the preprocessing function over the entire dataset, use \u{1F917} Datasets "),Ct=p("a"),sl=n("with_transform"),al=n(" method. You can speed up the "),Ts=p("code"),nl=n("map"),ll=n(" function by setting "),As=p("code"),ol=n("batched=True"),rl=n(" to process multiple elements of the dataset at once, and increasing the number of processes with "),Ms=p("code"),il=n("num_proc"),pl=n(". Remove any columns you don\u2019t need:"),ja=d(),q(Dt.$$.fragment),va=d(),ts=p("p"),ml=n("Now you\u2019ll need a second preprocessing function to capture text truncated from the lengthier examples to avoid losing any information. This preprocessing function should:"),xa=d(),at=p("ul"),zs=p("li"),fl=n("Concatenate all the text."),ul=d(),Lt=p("li"),hl=n("Split the concatenated text into smaller chunks defined by "),qs=p("code"),cl=n("block_size"),dl=n("."),Ea=d(),q(It.$$.fragment),Ta=d(),nt=p("p"),gl=n("Apply the "),Ps=p("code"),_l=n("group_texts"),$l=n(" function over the entire dataset:"),Aa=d(),q(Ot.$$.fragment),Ma=d(),He=p("p"),kl=n("Now create a batch of examples using "),ss=p("a"),yl=n("DataCollatorForLanguageModeling"),wl=n(". It\u2019s more efficient to "),Fs=p("em"),bl=n("dynamically pad"),jl=n(" the sentences to the longest length in a batch during collation, instead of padding the whole dataset to the maximium length."),za=d(),q(lt.$$.fragment),qa=d(),Ue=p("h2"),ot=p("a"),Cs=p("span"),q(Ht.$$.fragment),vl=d(),Ds=p("span"),xl=n("Causal language modeling"),Pa=d(),rt=p("p"),El=n("Causal language models are frequently used for text generation. This section shows you how to finetune "),St=p("a"),Tl=n("DistilGPT2"),Al=n(" to generate new text."),Fa=d(),Je=p("h3"),it=p("a"),Ls=p("span"),q(Gt.$$.fragment),Ml=d(),Is=p("span"),zl=n("Train"),Ca=d(),q(pt.$$.fragment),Da=d(),q(mt.$$.fragment),La=d(),Ke=p("h3"),ft=p("a"),Os=p("span"),q(Rt.$$.fragment),ql=d(),Hs=p("span"),Pl=n("Inference"),Ia=d(),as=p("p"),Fl=n("Great, now that you\u2019ve finetuned a model, you can use it for inference!"),Oa=d(),ns=p("p"),Cl=n("Come up with a prompt you\u2019d like to generate text from:"),Ha=d(),q(Wt.$$.fragment),Sa=d(),Se=p("p"),Dl=n("The simplest way to try out your finetuned model for inference is to use it in a "),ls=p("a"),Ll=n("pipeline()"),Il=n(". Instantiate a "),Ss=p("code"),Ol=n("pipeline"),Hl=n(" for text generation with your model, and pass your text to it:"),Ga=d(),q(Nt.$$.fragment),Ra=d(),q(ut.$$.fragment),Wa=d(),Qe=p("h2"),ht=p("a"),Gs=p("span"),q(Yt.$$.fragment),Sl=d(),Rs=p("span"),Gl=n("Masked language modeling"),Na=d(),ct=p("p"),Rl=n("Masked language modeling are good for tasks that require a good contextual understanding of an entire sequence. This section shows you how to finetune "),Bt=p("a"),Wl=n("DistilRoBERTa"),Nl=n(" to predict a masked word."),Ya=d(),Ve=p("h3"),dt=p("a"),Ws=p("span"),q(Ut.$$.fragment),Yl=d(),Ns=p("span"),Bl=n("Train"),Ba=d(),q(gt.$$.fragment),Ua=d(),q(_t.$$.fragment),Ja=d(),Xe=p("h3"),$t=p("a"),Ys=p("span"),q(Jt.$$.fragment),Ul=d(),Bs=p("span"),Jl=n("Inference"),Ka=d(),os=p("p"),Kl=n("Great, now that you\u2019ve finetuned a model, you can use it for inference!"),Qa=d(),kt=p("p"),Ql=n("Come up with some text you\u2019d like the model to fill in the blank with, and use the special "),Us=p("code"),Vl=n("<mask>"),Xl=n(" token to indicate the blank:"),Va=d(),q(Kt.$$.fragment),Xa=d(),De=p("p"),Zl=n("The simplest way to try out your finetuned model for inference is to use it in a "),rs=p("a"),eo=n("pipeline()"),to=n(". Instantiate a "),Js=p("code"),so=n("pipeline"),ao=n(" for fill-mask with your model, and pass your text to it. If you like, you can use the "),Ks=p("code"),no=n("top_k"),lo=n(" parameter to specify how many predictions to return:"),Za=d(),q(Qt.$$.fragment),en=d(),q(yt.$$.fragment),this.h()},l(e){const h=vr('[data-svelte="svelte-1phssyn"]',document.head);a=m(h,"META",{name:!0,content:!0}),h.forEach(t),$=g(e),o=m(e,"H1",{class:!0});var Vt=f(o);_=m(Vt,"A",{id:!0,class:!0,href:!0});var Qs=f(_);k=m(Qs,"SPAN",{});var Vs=f(k);P(j.$$.fragment,Vs),Vs.forEach(t),Qs.forEach(t),v=g(Vt),A=m(Vt,"SPAN",{});var Xs=f(A);y=l(Xs,"Language modeling"),Xs.forEach(t),Vt.forEach(t),E=g(e),z=m(e,"P",{});var Zs=f(z);I=l(Zs,"Language modeling tasks predicts words in a sentence, making these types of models great at generating text. You can use these models for creative applications like choosing your own text adventure or an intelligent coding assistant like Copilot or CodeParrot. There are two types of language modeling, causal and masked."),Zs.forEach(t),R=g(e),P(O.$$.fragment,e),H=g(e),U=m(e,"P",{});var ea=f(U);b=l(ea,"Causal language modeling predicts the next token in a sequence of tokens, and the model can only attend to tokens on the left. This means the model cannot see future tokens. GPT-2 is an example of a causal language model."),ea.forEach(t),T=g(e),P(S.$$.fragment,e),N=g(e),Y=m(e,"P",{});var ta=f(Y);Q=l(ta,"Masked language modeling predicts a masked token in a sequence, and the model can attend to tokens bidirectionally. This means the model has full access to the tokens on the left and right. BERT is an example of a masked language model."),ta.forEach(t),G=g(e),B=m(e,"P",{});var sa=f(B);u=l(sa,"This guide will show you how to:"),sa.forEach(t),x=g(e),X=m(e,"OL",{});var sn=f(X);V=m(sn,"LI",{});var Ge=f(V);Z=l(Ge,"Finetune "),ae=m(Ge,"A",{href:!0,rel:!0});var oo=f(ae);te=l(oo,"DistilGPT2"),oo.forEach(t),ge=l(Ge," for causal language modeling and "),ee=m(Ge,"A",{href:!0,rel:!0});var ro=f(ee);ve=l(ro,"DistilRoBERTa"),ro.forEach(t),le=l(Ge," for masked language modeling on the "),_e=m(Ge,"A",{href:!0,rel:!0});var io=f(_e);ue=l(io,"r/askscience"),io.forEach(t),xe=l(Ge," subset of the "),re=m(Ge,"A",{href:!0,rel:!0});var po=f(re);$e=l(po,"ELI5"),po.forEach(t),ne=l(Ge," dataset."),Ge.forEach(t),Ee=g(sn),se=m(sn,"LI",{});var mo=f(se);ke=l(mo,"Use your finetuned model for inference."),mo.forEach(t),sn.forEach(t),fe=g(e),P(ce.$$.fragment,e),be=g(e),oe=m(e,"P",{});var fo=f(oe);ie=l(fo,"Before you begin, make sure you have all the necessary libraries installed:"),fo.forEach(t),pe=g(e),P(c.$$.fragment,e),W=g(e),r=m(e,"P",{});var uo=f(r);M=l(uo,"We encourage you to login to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to login:"),uo.forEach(t),je=g(e),P(me.$$.fragment,e),ze=g(e),he=m(e,"H2",{class:!0});var an=f(he);de=m(an,"A",{id:!0,class:!0,href:!0});var ho=f(de);Te=m(ho,"SPAN",{});var co=f(Te);P(Ae.$$.fragment,co),co.forEach(t),ho.forEach(t),ye=g(an),Me=m(an,"SPAN",{});var go=f(Me);we=l(go,"Load ELI5 dataset"),go.forEach(t),an.forEach(t),qe=g(e),Ce=m(e,"P",{});var _o=f(Ce);Ye=l(_o,"Start by loading a smaller subset of the r/askscience subset of the ELI5 dataset from the \u{1F917} Datasets library. This\u2019ll give you a chance to experiment and make sure everythings works before spending more time training on the full dataset."),_o.forEach(t),na=g(e),P(bt.$$.fragment,e),la=g(e),Le=m(e,"P",{});var is=f(Le);wn=l(is,"Split the dataset\u2019s "),cs=m(is,"CODE",{});var $o=f(cs);bn=l($o,"train_asks"),$o.forEach(t),jn=l(is," split into a train and test set with the "),jt=m(is,"A",{href:!0,rel:!0});var ko=f(jt);vn=l(ko,"train_test_split"),ko.forEach(t),xn=l(is," method:"),is.forEach(t),oa=g(e),P(vt.$$.fragment,e),ra=g(e),Zt=m(e,"P",{});var yo=f(Zt);En=l(yo,"Then take a look at an example:"),yo.forEach(t),ia=g(e),P(xt.$$.fragment,e),pa=g(e),Ie=m(e,"P",{});var ps=f(Ie);Tn=l(ps,"While this may look like a lot, you\u2019re only really interested in the "),ds=m(ps,"CODE",{});var wo=f(ds);An=l(wo,"text"),wo.forEach(t),Mn=l(ps," field. What\u2019s cool about language modeling tasks is you don\u2019t need labels (also known as an unsupervised task) because the next word "),gs=m(ps,"EM",{});var bo=f(gs);zn=l(bo,"is"),bo.forEach(t),qn=l(ps," the label."),ps.forEach(t),ma=g(e),Be=m(e,"H2",{class:!0});var nn=f(Be);et=m(nn,"A",{id:!0,class:!0,href:!0});var jo=f(et);_s=m(jo,"SPAN",{});var vo=f(_s);P(Et.$$.fragment,vo),vo.forEach(t),jo.forEach(t),Pn=g(nn),$s=m(nn,"SPAN",{});var xo=f($s);Fn=l(xo,"Preprocess"),xo.forEach(t),nn.forEach(t),fa=g(e),P(Tt.$$.fragment,e),ua=g(e),tt=m(e,"P",{});var ln=f(tt);Cn=l(ln,"For causal language modeling, the next step is to load a DistilGPT2 tokenizer to process the "),ks=m(ln,"CODE",{});var Eo=f(ks);Dn=l(Eo,"text"),Eo.forEach(t),Ln=l(ln," subfield:"),ln.forEach(t),ha=g(e),P(At.$$.fragment,e),ca=g(e),P(Mt.$$.fragment,e),da=g(e),st=m(e,"P",{});var on=f(st);In=l(on,"For masked language modeling, the next step is to load a DistilRoBERTa tokenizer to process the "),ys=m(on,"CODE",{});var To=f(ys);On=l(To,"text"),To.forEach(t),Hn=l(on," subfield:"),on.forEach(t),ga=g(e),P(zt.$$.fragment,e),_a=g(e),Pe=m(e,"P",{});var Re=f(Pe);Sn=l(Re,"You\u2019ll notice from the example above, the "),ws=m(Re,"CODE",{});var Ao=f(ws);Gn=l(Ao,"text"),Ao.forEach(t),Rn=l(Re," field is actually nested inside "),bs=m(Re,"CODE",{});var Mo=f(bs);Wn=l(Mo,"answers"),Mo.forEach(t),Nn=l(Re,". This means you\u2019ll need to extract the "),js=m(Re,"CODE",{});var zo=f(js);Yn=l(zo,"text"),zo.forEach(t),Bn=l(Re," subfield from its nested structure with the "),qt=m(Re,"A",{href:!0,rel:!0});var qo=f(qt);vs=m(qo,"CODE",{});var Po=f(vs);Un=l(Po,"flatten"),Po.forEach(t),qo.forEach(t),Jn=l(Re," method:"),Re.forEach(t),$a=g(e),P(Pt.$$.fragment,e),ka=g(e),Oe=m(e,"P",{});var ms=f(Oe);Kn=l(ms,"Each subfield is now a separate column as indicated by the "),xs=m(ms,"CODE",{});var Fo=f(xs);Qn=l(Fo,"answers"),Fo.forEach(t),Vn=l(ms," prefix, and the "),Es=m(ms,"CODE",{});var Co=f(Es);Xn=l(Co,"text"),Co.forEach(t),Zn=l(ms," field is a list now. Instead of tokenizing each sentence separately, convert the list to a string so you can jointly tokenize them."),ms.forEach(t),ya=g(e),es=m(e,"P",{});var Do=f(es);el=l(Do,"Here is how you can create a preprocessing function to convert the list to a string, and truncate sequences to be no longer than DistilGPT2\u2019s maximum input length:"),Do.forEach(t),wa=g(e),P(Ft.$$.fragment,e),ba=g(e),Fe=m(e,"P",{});var We=f(Fe);tl=l(We,"To apply the preprocessing function over the entire dataset, use \u{1F917} Datasets "),Ct=m(We,"A",{href:!0,rel:!0});var Lo=f(Ct);sl=l(Lo,"with_transform"),Lo.forEach(t),al=l(We," method. You can speed up the "),Ts=m(We,"CODE",{});var Io=f(Ts);nl=l(Io,"map"),Io.forEach(t),ll=l(We," function by setting "),As=m(We,"CODE",{});var Oo=f(As);ol=l(Oo,"batched=True"),Oo.forEach(t),rl=l(We," to process multiple elements of the dataset at once, and increasing the number of processes with "),Ms=m(We,"CODE",{});var Ho=f(Ms);il=l(Ho,"num_proc"),Ho.forEach(t),pl=l(We,". Remove any columns you don\u2019t need:"),We.forEach(t),ja=g(e),P(Dt.$$.fragment,e),va=g(e),ts=m(e,"P",{});var So=f(ts);ml=l(So,"Now you\u2019ll need a second preprocessing function to capture text truncated from the lengthier examples to avoid losing any information. This preprocessing function should:"),So.forEach(t),xa=g(e),at=m(e,"UL",{});var rn=f(at);zs=m(rn,"LI",{});var Go=f(zs);fl=l(Go,"Concatenate all the text."),Go.forEach(t),ul=g(rn),Lt=m(rn,"LI",{});var pn=f(Lt);hl=l(pn,"Split the concatenated text into smaller chunks defined by "),qs=m(pn,"CODE",{});var Ro=f(qs);cl=l(Ro,"block_size"),Ro.forEach(t),dl=l(pn,"."),pn.forEach(t),rn.forEach(t),Ea=g(e),P(It.$$.fragment,e),Ta=g(e),nt=m(e,"P",{});var mn=f(nt);gl=l(mn,"Apply the "),Ps=m(mn,"CODE",{});var Wo=f(Ps);_l=l(Wo,"group_texts"),Wo.forEach(t),$l=l(mn," function over the entire dataset:"),mn.forEach(t),Aa=g(e),P(Ot.$$.fragment,e),Ma=g(e),He=m(e,"P",{});var fs=f(He);kl=l(fs,"Now create a batch of examples using "),ss=m(fs,"A",{href:!0});var No=f(ss);yl=l(No,"DataCollatorForLanguageModeling"),No.forEach(t),wl=l(fs,". It\u2019s more efficient to "),Fs=m(fs,"EM",{});var Yo=f(Fs);bl=l(Yo,"dynamically pad"),Yo.forEach(t),jl=l(fs," the sentences to the longest length in a batch during collation, instead of padding the whole dataset to the maximium length."),fs.forEach(t),za=g(e),P(lt.$$.fragment,e),qa=g(e),Ue=m(e,"H2",{class:!0});var fn=f(Ue);ot=m(fn,"A",{id:!0,class:!0,href:!0});var Bo=f(ot);Cs=m(Bo,"SPAN",{});var Uo=f(Cs);P(Ht.$$.fragment,Uo),Uo.forEach(t),Bo.forEach(t),vl=g(fn),Ds=m(fn,"SPAN",{});var Jo=f(Ds);xl=l(Jo,"Causal language modeling"),Jo.forEach(t),fn.forEach(t),Pa=g(e),rt=m(e,"P",{});var un=f(rt);El=l(un,"Causal language models are frequently used for text generation. This section shows you how to finetune "),St=m(un,"A",{href:!0,rel:!0});var Ko=f(St);Tl=l(Ko,"DistilGPT2"),Ko.forEach(t),Al=l(un," to generate new text."),un.forEach(t),Fa=g(e),Je=m(e,"H3",{class:!0});var hn=f(Je);it=m(hn,"A",{id:!0,class:!0,href:!0});var Qo=f(it);Ls=m(Qo,"SPAN",{});var Vo=f(Ls);P(Gt.$$.fragment,Vo),Vo.forEach(t),Qo.forEach(t),Ml=g(hn),Is=m(hn,"SPAN",{});var Xo=f(Is);zl=l(Xo,"Train"),Xo.forEach(t),hn.forEach(t),Ca=g(e),P(pt.$$.fragment,e),Da=g(e),P(mt.$$.fragment,e),La=g(e),Ke=m(e,"H3",{class:!0});var cn=f(Ke);ft=m(cn,"A",{id:!0,class:!0,href:!0});var Zo=f(ft);Os=m(Zo,"SPAN",{});var er=f(Os);P(Rt.$$.fragment,er),er.forEach(t),Zo.forEach(t),ql=g(cn),Hs=m(cn,"SPAN",{});var tr=f(Hs);Pl=l(tr,"Inference"),tr.forEach(t),cn.forEach(t),Ia=g(e),as=m(e,"P",{});var sr=f(as);Fl=l(sr,"Great, now that you\u2019ve finetuned a model, you can use it for inference!"),sr.forEach(t),Oa=g(e),ns=m(e,"P",{});var ar=f(ns);Cl=l(ar,"Come up with a prompt you\u2019d like to generate text from:"),ar.forEach(t),Ha=g(e),P(Wt.$$.fragment,e),Sa=g(e),Se=m(e,"P",{});var us=f(Se);Dl=l(us,"The simplest way to try out your finetuned model for inference is to use it in a "),ls=m(us,"A",{href:!0});var nr=f(ls);Ll=l(nr,"pipeline()"),nr.forEach(t),Il=l(us,". Instantiate a "),Ss=m(us,"CODE",{});var lr=f(Ss);Ol=l(lr,"pipeline"),lr.forEach(t),Hl=l(us," for text generation with your model, and pass your text to it:"),us.forEach(t),Ga=g(e),P(Nt.$$.fragment,e),Ra=g(e),P(ut.$$.fragment,e),Wa=g(e),Qe=m(e,"H2",{class:!0});var dn=f(Qe);ht=m(dn,"A",{id:!0,class:!0,href:!0});var or=f(ht);Gs=m(or,"SPAN",{});var rr=f(Gs);P(Yt.$$.fragment,rr),rr.forEach(t),or.forEach(t),Sl=g(dn),Rs=m(dn,"SPAN",{});var ir=f(Rs);Gl=l(ir,"Masked language modeling"),ir.forEach(t),dn.forEach(t),Na=g(e),ct=m(e,"P",{});var gn=f(ct);Rl=l(gn,"Masked language modeling are good for tasks that require a good contextual understanding of an entire sequence. This section shows you how to finetune "),Bt=m(gn,"A",{href:!0,rel:!0});var pr=f(Bt);Wl=l(pr,"DistilRoBERTa"),pr.forEach(t),Nl=l(gn," to predict a masked word."),gn.forEach(t),Ya=g(e),Ve=m(e,"H3",{class:!0});var _n=f(Ve);dt=m(_n,"A",{id:!0,class:!0,href:!0});var mr=f(dt);Ws=m(mr,"SPAN",{});var fr=f(Ws);P(Ut.$$.fragment,fr),fr.forEach(t),mr.forEach(t),Yl=g(_n),Ns=m(_n,"SPAN",{});var ur=f(Ns);Bl=l(ur,"Train"),ur.forEach(t),_n.forEach(t),Ba=g(e),P(gt.$$.fragment,e),Ua=g(e),P(_t.$$.fragment,e),Ja=g(e),Xe=m(e,"H3",{class:!0});var $n=f(Xe);$t=m($n,"A",{id:!0,class:!0,href:!0});var hr=f($t);Ys=m(hr,"SPAN",{});var cr=f(Ys);P(Jt.$$.fragment,cr),cr.forEach(t),hr.forEach(t),Ul=g($n),Bs=m($n,"SPAN",{});var dr=f(Bs);Jl=l(dr,"Inference"),dr.forEach(t),$n.forEach(t),Ka=g(e),os=m(e,"P",{});var gr=f(os);Kl=l(gr,"Great, now that you\u2019ve finetuned a model, you can use it for inference!"),gr.forEach(t),Qa=g(e),kt=m(e,"P",{});var kn=f(kt);Ql=l(kn,"Come up with some text you\u2019d like the model to fill in the blank with, and use the special "),Us=m(kn,"CODE",{});var _r=f(Us);Vl=l(_r,"<mask>"),_r.forEach(t),Xl=l(kn," token to indicate the blank:"),kn.forEach(t),Va=g(e),P(Kt.$$.fragment,e),Xa=g(e),De=m(e,"P",{});var wt=f(De);Zl=l(wt,"The simplest way to try out your finetuned model for inference is to use it in a "),rs=m(wt,"A",{href:!0});var $r=f(rs);eo=l($r,"pipeline()"),$r.forEach(t),to=l(wt,". Instantiate a "),Js=m(wt,"CODE",{});var kr=f(Js);so=l(kr,"pipeline"),kr.forEach(t),ao=l(wt," for fill-mask with your model, and pass your text to it. If you like, you can use the "),Ks=m(wt,"CODE",{});var yr=f(Ks);no=l(yr,"top_k"),yr.forEach(t),lo=l(wt," parameter to specify how many predictions to return:"),wt.forEach(t),Za=g(e),P(Qt.$$.fragment,e),en=g(e),P(yt.$$.fragment,e),this.h()},h(){w(a,"name","hf:doc:metadata"),w(a,"content",JSON.stringify(ei)),w(_,"id","language-modeling"),w(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),w(_,"href","#language-modeling"),w(o,"class","relative group"),w(ae,"href","https://huggingface.co/distilgpt2"),w(ae,"rel","nofollow"),w(ee,"href","https://huggingface.co/distilroberta-base"),w(ee,"rel","nofollow"),w(_e,"href","https://www.reddit.com/r/askscience/"),w(_e,"rel","nofollow"),w(re,"href","https://huggingface.co/datasets/eli5"),w(re,"rel","nofollow"),w(de,"id","load-eli5-dataset"),w(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),w(de,"href","#load-eli5-dataset"),w(he,"class","relative group"),w(jt,"href","https://huggingface.co/docs/datasets/v2.7.1/en/package_reference/main_classes#datasets.Dataset.train_test_split"),w(jt,"rel","nofollow"),w(et,"id","preprocess"),w(et,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),w(et,"href","#preprocess"),w(Be,"class","relative group"),w(qt,"href","https://huggingface.co/docs/datasets/process.html#flatten"),w(qt,"rel","nofollow"),w(Ct,"href","https://huggingface.co/docs/datasets/v2.7.1/en/package_reference/main_classes#datasets.Dataset.with_transform"),w(Ct,"rel","nofollow"),w(ss,"href","/docs/transformers/v4.25.1/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling"),w(ot,"id","causal-language-modeling"),w(ot,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),w(ot,"href","#causal-language-modeling"),w(Ue,"class","relative group"),w(St,"href","https://huggingface.co/distilgpt2"),w(St,"rel","nofollow"),w(it,"id","train"),w(it,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),w(it,"href","#train"),w(Je,"class","relative group"),w(ft,"id","inference"),w(ft,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),w(ft,"href","#inference"),w(Ke,"class","relative group"),w(ls,"href","/docs/transformers/v4.25.1/en/main_classes/pipelines#transformers.pipeline"),w(ht,"id","masked-language-modeling"),w(ht,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),w(ht,"href","#masked-language-modeling"),w(Qe,"class","relative group"),w(Bt,"href","https://huggingface.co/distilroberta-base"),w(Bt,"rel","nofollow"),w(dt,"id","train"),w(dt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),w(dt,"href","#train"),w(Ve,"class","relative group"),w($t,"id","inference"),w($t,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),w($t,"href","#inference"),w(Xe,"class","relative group"),w(rs,"href","/docs/transformers/v4.25.1/en/main_classes/pipelines#transformers.pipeline")},m(e,h){s(document.head,a),i(e,$,h),i(e,o,h),s(o,_),s(_,k),F(j,k,null),s(o,v),s(o,A),s(A,y),i(e,E,h),i(e,z,h),s(z,I),i(e,R,h),F(O,e,h),i(e,H,h),i(e,U,h),s(U,b),i(e,T,h),F(S,e,h),i(e,N,h),i(e,Y,h),s(Y,Q),i(e,G,h),i(e,B,h),s(B,u),i(e,x,h),i(e,X,h),s(X,V),s(V,Z),s(V,ae),s(ae,te),s(V,ge),s(V,ee),s(ee,ve),s(V,le),s(V,_e),s(_e,ue),s(V,xe),s(V,re),s(re,$e),s(V,ne),s(X,Ee),s(X,se),s(se,ke),i(e,fe,h),F(ce,e,h),i(e,be,h),i(e,oe,h),s(oe,ie),i(e,pe,h),F(c,e,h),i(e,W,h),i(e,r,h),s(r,M),i(e,je,h),F(me,e,h),i(e,ze,h),i(e,he,h),s(he,de),s(de,Te),F(Ae,Te,null),s(he,ye),s(he,Me),s(Me,we),i(e,qe,h),i(e,Ce,h),s(Ce,Ye),i(e,na,h),F(bt,e,h),i(e,la,h),i(e,Le,h),s(Le,wn),s(Le,cs),s(cs,bn),s(Le,jn),s(Le,jt),s(jt,vn),s(Le,xn),i(e,oa,h),F(vt,e,h),i(e,ra,h),i(e,Zt,h),s(Zt,En),i(e,ia,h),F(xt,e,h),i(e,pa,h),i(e,Ie,h),s(Ie,Tn),s(Ie,ds),s(ds,An),s(Ie,Mn),s(Ie,gs),s(gs,zn),s(Ie,qn),i(e,ma,h),i(e,Be,h),s(Be,et),s(et,_s),F(Et,_s,null),s(Be,Pn),s(Be,$s),s($s,Fn),i(e,fa,h),F(Tt,e,h),i(e,ua,h),i(e,tt,h),s(tt,Cn),s(tt,ks),s(ks,Dn),s(tt,Ln),i(e,ha,h),F(At,e,h),i(e,ca,h),F(Mt,e,h),i(e,da,h),i(e,st,h),s(st,In),s(st,ys),s(ys,On),s(st,Hn),i(e,ga,h),F(zt,e,h),i(e,_a,h),i(e,Pe,h),s(Pe,Sn),s(Pe,ws),s(ws,Gn),s(Pe,Rn),s(Pe,bs),s(bs,Wn),s(Pe,Nn),s(Pe,js),s(js,Yn),s(Pe,Bn),s(Pe,qt),s(qt,vs),s(vs,Un),s(Pe,Jn),i(e,$a,h),F(Pt,e,h),i(e,ka,h),i(e,Oe,h),s(Oe,Kn),s(Oe,xs),s(xs,Qn),s(Oe,Vn),s(Oe,Es),s(Es,Xn),s(Oe,Zn),i(e,ya,h),i(e,es,h),s(es,el),i(e,wa,h),F(Ft,e,h),i(e,ba,h),i(e,Fe,h),s(Fe,tl),s(Fe,Ct),s(Ct,sl),s(Fe,al),s(Fe,Ts),s(Ts,nl),s(Fe,ll),s(Fe,As),s(As,ol),s(Fe,rl),s(Fe,Ms),s(Ms,il),s(Fe,pl),i(e,ja,h),F(Dt,e,h),i(e,va,h),i(e,ts,h),s(ts,ml),i(e,xa,h),i(e,at,h),s(at,zs),s(zs,fl),s(at,ul),s(at,Lt),s(Lt,hl),s(Lt,qs),s(qs,cl),s(Lt,dl),i(e,Ea,h),F(It,e,h),i(e,Ta,h),i(e,nt,h),s(nt,gl),s(nt,Ps),s(Ps,_l),s(nt,$l),i(e,Aa,h),F(Ot,e,h),i(e,Ma,h),i(e,He,h),s(He,kl),s(He,ss),s(ss,yl),s(He,wl),s(He,Fs),s(Fs,bl),s(He,jl),i(e,za,h),F(lt,e,h),i(e,qa,h),i(e,Ue,h),s(Ue,ot),s(ot,Cs),F(Ht,Cs,null),s(Ue,vl),s(Ue,Ds),s(Ds,xl),i(e,Pa,h),i(e,rt,h),s(rt,El),s(rt,St),s(St,Tl),s(rt,Al),i(e,Fa,h),i(e,Je,h),s(Je,it),s(it,Ls),F(Gt,Ls,null),s(Je,Ml),s(Je,Is),s(Is,zl),i(e,Ca,h),F(pt,e,h),i(e,Da,h),F(mt,e,h),i(e,La,h),i(e,Ke,h),s(Ke,ft),s(ft,Os),F(Rt,Os,null),s(Ke,ql),s(Ke,Hs),s(Hs,Pl),i(e,Ia,h),i(e,as,h),s(as,Fl),i(e,Oa,h),i(e,ns,h),s(ns,Cl),i(e,Ha,h),F(Wt,e,h),i(e,Sa,h),i(e,Se,h),s(Se,Dl),s(Se,ls),s(ls,Ll),s(Se,Il),s(Se,Ss),s(Ss,Ol),s(Se,Hl),i(e,Ga,h),F(Nt,e,h),i(e,Ra,h),F(ut,e,h),i(e,Wa,h),i(e,Qe,h),s(Qe,ht),s(ht,Gs),F(Yt,Gs,null),s(Qe,Sl),s(Qe,Rs),s(Rs,Gl),i(e,Na,h),i(e,ct,h),s(ct,Rl),s(ct,Bt),s(Bt,Wl),s(ct,Nl),i(e,Ya,h),i(e,Ve,h),s(Ve,dt),s(dt,Ws),F(Ut,Ws,null),s(Ve,Yl),s(Ve,Ns),s(Ns,Bl),i(e,Ba,h),F(gt,e,h),i(e,Ua,h),F(_t,e,h),i(e,Ja,h),i(e,Xe,h),s(Xe,$t),s($t,Ys),F(Jt,Ys,null),s(Xe,Ul),s(Xe,Bs),s(Bs,Jl),i(e,Ka,h),i(e,os,h),s(os,Kl),i(e,Qa,h),i(e,kt,h),s(kt,Ql),s(kt,Us),s(Us,Vl),s(kt,Xl),i(e,Va,h),F(Kt,e,h),i(e,Xa,h),i(e,De,h),s(De,Zl),s(De,rs),s(rs,eo),s(De,to),s(De,Js),s(Js,so),s(De,ao),s(De,Ks),s(Ks,no),s(De,lo),i(e,Za,h),F(Qt,e,h),i(e,en,h),F(yt,e,h),tn=!0},p(e,[h]){const Vt={};h&2&&(Vt.$$scope={dirty:h,ctx:e}),ce.$set(Vt);const Qs={};h&2&&(Qs.$$scope={dirty:h,ctx:e}),lt.$set(Qs);const Vs={};h&2&&(Vs.$$scope={dirty:h,ctx:e}),pt.$set(Vs);const Xs={};h&2&&(Xs.$$scope={dirty:h,ctx:e}),mt.$set(Xs);const Zs={};h&2&&(Zs.$$scope={dirty:h,ctx:e}),ut.$set(Zs);const ea={};h&2&&(ea.$$scope={dirty:h,ctx:e}),gt.$set(ea);const ta={};h&2&&(ta.$$scope={dirty:h,ctx:e}),_t.$set(ta);const sa={};h&2&&(sa.$$scope={dirty:h,ctx:e}),yt.$set(sa)},i(e){tn||(C(j.$$.fragment,e),C(O.$$.fragment,e),C(S.$$.fragment,e),C(ce.$$.fragment,e),C(c.$$.fragment,e),C(me.$$.fragment,e),C(Ae.$$.fragment,e),C(bt.$$.fragment,e),C(vt.$$.fragment,e),C(xt.$$.fragment,e),C(Et.$$.fragment,e),C(Tt.$$.fragment,e),C(At.$$.fragment,e),C(Mt.$$.fragment,e),C(zt.$$.fragment,e),C(Pt.$$.fragment,e),C(Ft.$$.fragment,e),C(Dt.$$.fragment,e),C(It.$$.fragment,e),C(Ot.$$.fragment,e),C(lt.$$.fragment,e),C(Ht.$$.fragment,e),C(Gt.$$.fragment,e),C(pt.$$.fragment,e),C(mt.$$.fragment,e),C(Rt.$$.fragment,e),C(Wt.$$.fragment,e),C(Nt.$$.fragment,e),C(ut.$$.fragment,e),C(Yt.$$.fragment,e),C(Ut.$$.fragment,e),C(gt.$$.fragment,e),C(_t.$$.fragment,e),C(Jt.$$.fragment,e),C(Kt.$$.fragment,e),C(Qt.$$.fragment,e),C(yt.$$.fragment,e),tn=!0)},o(e){D(j.$$.fragment,e),D(O.$$.fragment,e),D(S.$$.fragment,e),D(ce.$$.fragment,e),D(c.$$.fragment,e),D(me.$$.fragment,e),D(Ae.$$.fragment,e),D(bt.$$.fragment,e),D(vt.$$.fragment,e),D(xt.$$.fragment,e),D(Et.$$.fragment,e),D(Tt.$$.fragment,e),D(At.$$.fragment,e),D(Mt.$$.fragment,e),D(zt.$$.fragment,e),D(Pt.$$.fragment,e),D(Ft.$$.fragment,e),D(Dt.$$.fragment,e),D(It.$$.fragment,e),D(Ot.$$.fragment,e),D(lt.$$.fragment,e),D(Ht.$$.fragment,e),D(Gt.$$.fragment,e),D(pt.$$.fragment,e),D(mt.$$.fragment,e),D(Rt.$$.fragment,e),D(Wt.$$.fragment,e),D(Nt.$$.fragment,e),D(ut.$$.fragment,e),D(Yt.$$.fragment,e),D(Ut.$$.fragment,e),D(gt.$$.fragment,e),D(_t.$$.fragment,e),D(Jt.$$.fragment,e),D(Kt.$$.fragment,e),D(Qt.$$.fragment,e),D(yt.$$.fragment,e),tn=!1},d(e){t(a),e&&t($),e&&t(o),L(j),e&&t(E),e&&t(z),e&&t(R),L(O,e),e&&t(H),e&&t(U),e&&t(T),L(S,e),e&&t(N),e&&t(Y),e&&t(G),e&&t(B),e&&t(x),e&&t(X),e&&t(fe),L(ce,e),e&&t(be),e&&t(oe),e&&t(pe),L(c,e),e&&t(W),e&&t(r),e&&t(je),L(me,e),e&&t(ze),e&&t(he),L(Ae),e&&t(qe),e&&t(Ce),e&&t(na),L(bt,e),e&&t(la),e&&t(Le),e&&t(oa),L(vt,e),e&&t(ra),e&&t(Zt),e&&t(ia),L(xt,e),e&&t(pa),e&&t(Ie),e&&t(ma),e&&t(Be),L(Et),e&&t(fa),L(Tt,e),e&&t(ua),e&&t(tt),e&&t(ha),L(At,e),e&&t(ca),L(Mt,e),e&&t(da),e&&t(st),e&&t(ga),L(zt,e),e&&t(_a),e&&t(Pe),e&&t($a),L(Pt,e),e&&t(ka),e&&t(Oe),e&&t(ya),e&&t(es),e&&t(wa),L(Ft,e),e&&t(ba),e&&t(Fe),e&&t(ja),L(Dt,e),e&&t(va),e&&t(ts),e&&t(xa),e&&t(at),e&&t(Ea),L(It,e),e&&t(Ta),e&&t(nt),e&&t(Aa),L(Ot,e),e&&t(Ma),e&&t(He),e&&t(za),L(lt,e),e&&t(qa),e&&t(Ue),L(Ht),e&&t(Pa),e&&t(rt),e&&t(Fa),e&&t(Je),L(Gt),e&&t(Ca),L(pt,e),e&&t(Da),L(mt,e),e&&t(La),e&&t(Ke),L(Rt),e&&t(Ia),e&&t(as),e&&t(Oa),e&&t(ns),e&&t(Ha),L(Wt,e),e&&t(Sa),e&&t(Se),e&&t(Ga),L(Nt,e),e&&t(Ra),L(ut,e),e&&t(Wa),e&&t(Qe),L(Yt),e&&t(Na),e&&t(ct),e&&t(Ya),e&&t(Ve),L(Ut),e&&t(Ba),L(gt,e),e&&t(Ua),L(_t,e),e&&t(Ja),e&&t(Xe),L(Jt),e&&t(Ka),e&&t(os),e&&t(Qa),e&&t(kt),e&&t(Va),L(Kt,e),e&&t(Xa),e&&t(De),e&&t(Za),L(Qt,e),e&&t(en),L(yt,e)}}}const ei={local:"language-modeling",sections:[{local:"load-eli5-dataset",title:"Load ELI5 dataset"},{local:"preprocess",title:"Preprocess"},{local:"causal-language-modeling",sections:[{local:"train",title:"Train"},{local:"inference",title:"Inference"}],title:"Causal language modeling"},{local:"masked-language-modeling",sections:[{local:"train",title:"Train"},{local:"inference",title:"Inference"}],title:"Masked language modeling"}],title:"Language modeling"};function ti(K){return xr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ii extends wr{constructor(a){super();br(this,a,ti,Zr,jr,{})}}export{ii as default,ei as metadata};
