import{s as qs,o as Ss,n as ee}from"../chunks/scheduler.9bc65507.js";import{S as Ps,i as Ds,g as w,s as i,r as u,A as Ks,h as $,f as t,c as m,j as Ys,u as g,x as k,k as As,y as Os,a as l,v as M,d as y,t as b,w as j,m as Qs,n as Ls}from"../chunks/index.707bf1b6.js";import{T as js}from"../chunks/Tip.c2ecdbf4.js";import{Y as Ns}from"../chunks/Youtube.e1129c6f.js";import{C as R}from"../chunks/CodeBlock.54a9f38d.js";import{D as et}from"../chunks/DocNotebookDropdown.41f65cb5.js";import{F as ws,M as We}from"../chunks/Markdown.fef84341.js";import{H as Be}from"../chunks/Heading.342b1fa6.js";function st(Z){let s,c,a='<a href="../model_doc/albert">ALBERT</a>, <a href="../model_doc/bart">BART</a>, <a href="../model_doc/bert">BERT</a>, <a href="../model_doc/big_bird">BigBird</a>, <a href="../model_doc/camembert">CamemBERT</a>, <a href="../model_doc/convbert">ConvBERT</a>, <a href="../model_doc/data2vec-text">Data2VecText</a>, <a href="../model_doc/deberta">DeBERTa</a>, <a href="../model_doc/deberta-v2">DeBERTa-v2</a>, <a href="../model_doc/distilbert">DistilBERT</a>, <a href="../model_doc/electra">ELECTRA</a>, <a href="../model_doc/ernie">ERNIE</a>, <a href="../model_doc/esm">ESM</a>, <a href="../model_doc/flaubert">FlauBERT</a>, <a href="../model_doc/fnet">FNet</a>, <a href="../model_doc/funnel">Funnel Transformer</a>, <a href="../model_doc/ibert">I-BERT</a>, <a href="../model_doc/layoutlm">LayoutLM</a>, <a href="../model_doc/longformer">Longformer</a>, <a href="../model_doc/luke">LUKE</a>, <a href="../model_doc/mbart">mBART</a>, <a href="../model_doc/mega">MEGA</a>, <a href="../model_doc/megatron-bert">Megatron-BERT</a>, <a href="../model_doc/mobilebert">MobileBERT</a>, <a href="../model_doc/mpnet">MPNet</a>, <a href="../model_doc/mra">MRA</a>, <a href="../model_doc/mvp">MVP</a>, <a href="../model_doc/nezha">Nezha</a>, <a href="../model_doc/nystromformer">Nyströmformer</a>, <a href="../model_doc/perceiver">Perceiver</a>, <a href="../model_doc/qdqbert">QDQBert</a>, <a href="../model_doc/reformer">Reformer</a>, <a href="../model_doc/rembert">RemBERT</a>, <a href="../model_doc/roberta">RoBERTa</a>, <a href="../model_doc/roberta-prelayernorm">RoBERTa-PreLayerNorm</a>, <a href="../model_doc/roc_bert">RoCBert</a>, <a href="../model_doc/roformer">RoFormer</a>, <a href="../model_doc/squeezebert">SqueezeBERT</a>, <a href="../model_doc/tapas">TAPAS</a>, <a href="../model_doc/wav2vec2">Wav2Vec2</a>, <a href="../model_doc/xlm">XLM</a>, <a href="../model_doc/xlm-roberta">XLM-RoBERTa</a>, <a href="../model_doc/xlm-roberta-xl">XLM-RoBERTa-XL</a>, <a href="../model_doc/xmod">X-MOD</a>, <a href="../model_doc/yoso">YOSO</a>';return{c(){s=Qs(`You can finetune other architectures for masked language modeling following the same steps in this guide.
Choose one of the following architectures:

`),c=w("p"),c.innerHTML=a},l(o){s=Ls(o,`You can finetune other architectures for masked language modeling following the same steps in this guide.
Choose one of the following architectures:

`),c=$(o,"P",{"data-svelte-h":!0}),k(c)!=="svelte-e35b6o"&&(c.innerHTML=a)},m(o,d){l(o,s,d),l(o,c,d)},p:ee,d(o){o&&(t(s),t(c))}}}function tt(Z){let s,c="Use the end-of-sequence token as the padding token and specify <code>mlm_probability</code> to randomly mask tokens each time you iterate over the data:",a,o,d;return o=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvckxhbmd1YWdlTW9kZWxpbmclMEElMEF0b2tlbml6ZXIucGFkX3Rva2VuJTIwJTNEJTIwdG9rZW5pemVyLmVvc190b2tlbiUwQWRhdGFfY29sbGF0b3IlMjAlM0QlMjBEYXRhQ29sbGF0b3JGb3JMYW5ndWFnZU1vZGVsaW5nKHRva2VuaXplciUzRHRva2VuaXplciUyQyUyMG1sbV9wcm9iYWJpbGl0eSUzRDAuMTUp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token
<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=<span class="hljs-number">0.15</span>)`,wrap:!1}}),{c(){s=w("p"),s.innerHTML=c,a=i(),u(o.$$.fragment)},l(f){s=$(f,"P",{"data-svelte-h":!0}),k(s)!=="svelte-z8dnpj"&&(s.innerHTML=c),a=m(f),g(o.$$.fragment,f)},m(f,v){l(f,s,v),l(f,a,v),M(o,f,v),d=!0},p:ee,i(f){d||(y(o.$$.fragment,f),d=!0)},o(f){b(o.$$.fragment,f),d=!1},d(f){f&&(t(s),t(a)),j(o,f)}}}function at(Z){let s,c;return s=new We({props:{$$slots:{default:[tt]},$$scope:{ctx:Z}}}),{c(){u(s.$$.fragment)},l(a){g(s.$$.fragment,a)},m(a,o){M(s,a,o),c=!0},p(a,o){const d={};o&2&&(d.$$scope={dirty:o,ctx:a}),s.$set(d)},i(a){c||(y(s.$$.fragment,a),c=!0)},o(a){b(s.$$.fragment,a),c=!1},d(a){j(s,a)}}}function lt(Z){let s,c="Use the end-of-sequence token as the padding token and specify <code>mlm_probability</code> to randomly mask tokens each time you iterate over the data:",a,o,d;return o=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvckxhbmd1YWdlTW9kZWxpbmclMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yTGFuZ3VhZ2VNb2RlbGluZyh0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMjBtbG1fcHJvYmFiaWxpdHklM0QwLjE1JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJ0ZiUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=<span class="hljs-number">0.15</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),{c(){s=w("p"),s.innerHTML=c,a=i(),u(o.$$.fragment)},l(f){s=$(f,"P",{"data-svelte-h":!0}),k(s)!=="svelte-z8dnpj"&&(s.innerHTML=c),a=m(f),g(o.$$.fragment,f)},m(f,v){l(f,s,v),l(f,a,v),M(o,f,v),d=!0},p:ee,i(f){d||(y(o.$$.fragment,f),d=!0)},o(f){b(o.$$.fragment,f),d=!1},d(f){f&&(t(s),t(a)),j(o,f)}}}function nt(Z){let s,c;return s=new We({props:{$$slots:{default:[lt]},$$scope:{ctx:Z}}}),{c(){u(s.$$.fragment)},l(a){g(s.$$.fragment,a)},m(a,o){M(s,a,o),c=!0},p(a,o){const d={};o&2&&(d.$$scope={dirty:o,ctx:a}),s.$set(d)},i(a){c||(y(s.$$.fragment,a),c=!0)},o(a){b(s.$$.fragment,a),c=!1},d(a){j(s,a)}}}function rt(Z){let s,c='If you aren’t familiar with finetuning a model with the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a>, take a look at the basic tutorial <a href="../training#train-with-pytorch-trainer">here</a>!';return{c(){s=w("p"),s.innerHTML=c},l(a){s=$(a,"P",{"data-svelte-h":!0}),k(s)!=="svelte-15s4um0"&&(s.innerHTML=c)},m(a,o){l(a,s,o)},p:ee,d(a){a&&t(s)}}}function ot(Z){let s,c,a,o='You’re ready to start training your model now! Load DistilRoBERTa with <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoModelForMaskedLM">AutoModelForMaskedLM</a>:',d,f,v,W,G="At this point, only three steps remain:",C,U,E='<li>Define your training hyperparameters in <a href="/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>. The only required parameter is <code>output_dir</code> which specifies where to save your model. You’ll push this model to the Hub by setting <code>push_to_hub=True</code> (you need to be signed in to Hugging Face to upload your model).</li> <li>Pass the training arguments to <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> along with the model, datasets, and data collator.</li> <li>Call <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.train">train()</a> to finetune your model.</li>',I,x,B,r,T='Once training is completed, use the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.evaluate">evaluate()</a> method to evaluate your model and get its perplexity:',H,z,X,Y,A='Then share your model to the Hub with the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.push_to_hub">push_to_hub()</a> method so everyone can use your model:',N,F,V;return s=new js({props:{$$slots:{default:[rt]},$$scope:{ctx:Z}}}),f=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvck1hc2tlZExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JNYXNrZWRMTS5mcm9tX3ByZXRyYWluZWQoJTIyZGlzdGlsYmVydCUyRmRpc3RpbHJvYmVydGEtYmFzZSUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;distilbert/distilroberta-base&quot;</span>)`,wrap:!1}}),x=new R({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJteV9hd2Vzb21lX2VsaTVfbWxtX21vZGVsJTIyJTJDJTBBJTIwJTIwJTIwJTIwZXZhbHVhdGlvbl9zdHJhdGVneSUzRCUyMmVwb2NoJTIyJTJDJTBBJTIwJTIwJTIwJTIwbGVhcm5pbmdfcmF0ZSUzRDJlLTUlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMyUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRDAuMDElMkMlMEElMjAlMjAlMjAlMjBwdXNoX3RvX2h1YiUzRFRydWUlMkMlMEEpJTBBJTBBdHJhaW5lciUyMCUzRCUyMFRyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRGxtX2RhdGFzZXQlNUIlMjJ0cmFpbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfZGF0YXNldCUzRGxtX2RhdGFzZXQlNUIlMjJ0ZXN0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZGF0YV9jb2xsYXRvciUzRGRhdGFfY29sbGF0b3IlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_eli5_mlm_model&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`,wrap:!1}}),z=new R({props:{code:"aW1wb3J0JTIwbWF0aCUwQSUwQWV2YWxfcmVzdWx0cyUyMCUzRCUyMHRyYWluZXIuZXZhbHVhdGUoKSUwQXByaW50KGYlMjJQZXJwbGV4aXR5JTNBJTIwJTdCbWF0aC5leHAoZXZhbF9yZXN1bHRzJTVCJ2V2YWxfbG9zcyclNUQpJTNBLjJmJTdEJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> math

<span class="hljs-meta">&gt;&gt;&gt; </span>eval_results = trainer.evaluate()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Perplexity: <span class="hljs-subst">{math.exp(eval_results[<span class="hljs-string">&#x27;eval_loss&#x27;</span>]):<span class="hljs-number">.2</span>f}</span>&quot;</span>)
Perplexity: <span class="hljs-number">8.76</span>`,wrap:!1}}),F=new R({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()',wrap:!1}}),{c(){u(s.$$.fragment),c=i(),a=w("p"),a.innerHTML=o,d=i(),u(f.$$.fragment),v=i(),W=w("p"),W.textContent=G,C=i(),U=w("ol"),U.innerHTML=E,I=i(),u(x.$$.fragment),B=i(),r=w("p"),r.innerHTML=T,H=i(),u(z.$$.fragment),X=i(),Y=w("p"),Y.innerHTML=A,N=i(),u(F.$$.fragment)},l(h){g(s.$$.fragment,h),c=m(h),a=$(h,"P",{"data-svelte-h":!0}),k(a)!=="svelte-1m3teom"&&(a.innerHTML=o),d=m(h),g(f.$$.fragment,h),v=m(h),W=$(h,"P",{"data-svelte-h":!0}),k(W)!=="svelte-l42k0i"&&(W.textContent=G),C=m(h),U=$(h,"OL",{"data-svelte-h":!0}),k(U)!=="svelte-9nzdq0"&&(U.innerHTML=E),I=m(h),g(x.$$.fragment,h),B=m(h),r=$(h,"P",{"data-svelte-h":!0}),k(r)!=="svelte-18h3zoe"&&(r.innerHTML=T),H=m(h),g(z.$$.fragment,h),X=m(h),Y=$(h,"P",{"data-svelte-h":!0}),k(Y)!=="svelte-1ofhil3"&&(Y.innerHTML=A),N=m(h),g(F.$$.fragment,h)},m(h,J){M(s,h,J),l(h,c,J),l(h,a,J),l(h,d,J),M(f,h,J),l(h,v,J),l(h,W,J),l(h,C,J),l(h,U,J),l(h,I,J),M(x,h,J),l(h,B,J),l(h,r,J),l(h,H,J),M(z,h,J),l(h,X,J),l(h,Y,J),l(h,N,J),M(F,h,J),V=!0},p(h,J){const L={};J&2&&(L.$$scope={dirty:J,ctx:h}),s.$set(L)},i(h){V||(y(s.$$.fragment,h),y(f.$$.fragment,h),y(x.$$.fragment,h),y(z.$$.fragment,h),y(F.$$.fragment,h),V=!0)},o(h){b(s.$$.fragment,h),b(f.$$.fragment,h),b(x.$$.fragment,h),b(z.$$.fragment,h),b(F.$$.fragment,h),V=!1},d(h){h&&(t(c),t(a),t(d),t(v),t(W),t(C),t(U),t(I),t(B),t(r),t(H),t(X),t(Y),t(N)),j(s,h),j(f,h),j(x,h),j(z,h),j(F,h)}}}function pt(Z){let s,c;return s=new We({props:{$$slots:{default:[ot]},$$scope:{ctx:Z}}}),{c(){u(s.$$.fragment)},l(a){g(s.$$.fragment,a)},m(a,o){M(s,a,o),c=!0},p(a,o){const d={};o&2&&(d.$$scope={dirty:o,ctx:a}),s.$set(d)},i(a){c||(y(s.$$.fragment,a),c=!0)},o(a){b(s.$$.fragment,a),c=!1},d(a){j(s,a)}}}function it(Z){let s,c='If you aren’t familiar with finetuning a model with Keras, take a look at the basic tutorial <a href="../training#train-a-tensorflow-model-with-keras">here</a>!';return{c(){s=w("p"),s.innerHTML=c},l(a){s=$(a,"P",{"data-svelte-h":!0}),k(s)!=="svelte-1rd4nl8"&&(s.innerHTML=c)},m(a,o){l(a,s,o)},p:ee,d(a){a&&t(s)}}}function mt(Z){let s,c,a,o,d,f='Then you can load DistilRoBERTa with <a href="/docs/transformers/main/en/model_doc/auto#transformers.TFAutoModelForMaskedLM">TFAutoModelForMaskedLM</a>:',v,W,G,C,U='Convert your datasets to the <code>tf.data.Dataset</code> format with <a href="/docs/transformers/main/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset">prepare_tf_dataset()</a>:',E,I,x,B,r='Configure the model for training with <a href="https://keras.io/api/models/model_training_apis/#compile-method" rel="nofollow"><code>compile</code></a>. Note that Transformers models all have a default task-relevant loss function, so you don’t need to specify one unless you want to:',T,H,z,X,Y='This can be done by specifying where to push your model and tokenizer in the <a href="/docs/transformers/main/en/main_classes/keras_callbacks#transformers.PushToHubCallback">PushToHubCallback</a>:',A,N,F,V,h='Finally, you’re ready to start training your model! Call <a href="https://keras.io/api/models/model_training_apis/#fit-method" rel="nofollow"><code>fit</code></a> with your training and validation datasets, the number of epochs, and your callback to finetune the model:',J,L,S,Q,se="Once training is completed, your model is automatically uploaded to the Hub so everyone can use it!",q;return s=new js({props:{$$slots:{default:[it]},$$scope:{ctx:Z}}}),a=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMkMlMjBBZGFtV2VpZ2h0RGVjYXklMEElMEFvcHRpbWl6ZXIlMjAlM0QlMjBBZGFtV2VpZ2h0RGVjYXkobGVhcm5pbmdfcmF0ZSUzRDJlLTUlMkMlMjB3ZWlnaHRfZGVjYXlfcmF0ZSUzRDAuMDEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer, AdamWeightDecay

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamWeightDecay(learning_rate=<span class="hljs-number">2e-5</span>, weight_decay_rate=<span class="hljs-number">0.01</span>)`,wrap:!1}}),W=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yTWFza2VkTE0lMEElMEFtb2RlbCUyMCUzRCUyMFRGQXV0b01vZGVsRm9yTWFza2VkTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmRpc3RpbGJlcnQlMkZkaXN0aWxyb2JlcnRhLWJhc2UlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;distilbert/distilroberta-base&quot;</span>)`,wrap:!1}}),I=new R({props:{code:"dGZfdHJhaW5fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMGxtX2RhdGFzZXQlNUIlMjJ0cmFpbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwY29sbGF0ZV9mbiUzRGRhdGFfY29sbGF0b3IlMkMlMEEpJTBBJTBBdGZfdGVzdF9zZXQlMjAlM0QlMjBtb2RlbC5wcmVwYXJlX3RmX2RhdGFzZXQoJTBBJTIwJTIwJTIwJTIwbG1fZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBzaHVmZmxlJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBjb2xsYXRlX2ZuJTNEZGF0YV9jb2xsYXRvciUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_test_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`,wrap:!1}}),H=new R({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplciklMjAlMjAlMjMlMjBObyUyMGxvc3MlMjBhcmd1bWVudCE=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)  <span class="hljs-comment"># No loss argument!</span>`,wrap:!1}}),N=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBQdXNoVG9IdWJDYWxsYmFjayUwQSUwQWNhbGxiYWNrJTIwJTNEJTIwUHVzaFRvSHViQ2FsbGJhY2soJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm15X2F3ZXNvbWVfZWxpNV9tbG1fbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>callback = PushToHubCallback(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_eli5_mlm_model&quot;</span>,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>)`,wrap:!1}}),L=new R({props:{code:"bW9kZWwuZml0KHglM0R0Zl90cmFpbl9zZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl90ZXN0X3NldCUyQyUyMGVwb2NocyUzRDMlMkMlMjBjYWxsYmFja3MlM0QlNUJjYWxsYmFjayU1RCk=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=<span class="hljs-number">3</span>, callbacks=[callback])',wrap:!1}}),{c(){u(s.$$.fragment),c=Qs(`
To finetune a model in TensorFlow, start by setting up an optimizer function, learning rate schedule, and some training hyperparameters:

	`),u(a.$$.fragment),o=i(),d=w("p"),d.innerHTML=f,v=i(),u(W.$$.fragment),G=i(),C=w("p"),C.innerHTML=U,E=i(),u(I.$$.fragment),x=i(),B=w("p"),B.innerHTML=r,T=i(),u(H.$$.fragment),z=i(),X=w("p"),X.innerHTML=Y,A=i(),u(N.$$.fragment),F=i(),V=w("p"),V.innerHTML=h,J=i(),u(L.$$.fragment),S=i(),Q=w("p"),Q.textContent=se},l(p){g(s.$$.fragment,p),c=Ls(p,`
To finetune a model in TensorFlow, start by setting up an optimizer function, learning rate schedule, and some training hyperparameters:

	`),g(a.$$.fragment,p),o=m(p),d=$(p,"P",{"data-svelte-h":!0}),k(d)!=="svelte-o4qgcw"&&(d.innerHTML=f),v=m(p),g(W.$$.fragment,p),G=m(p),C=$(p,"P",{"data-svelte-h":!0}),k(C)!=="svelte-9ymftz"&&(C.innerHTML=U),E=m(p),g(I.$$.fragment,p),x=m(p),B=$(p,"P",{"data-svelte-h":!0}),k(B)!=="svelte-17cxx5e"&&(B.innerHTML=r),T=m(p),g(H.$$.fragment,p),z=m(p),X=$(p,"P",{"data-svelte-h":!0}),k(X)!=="svelte-1aajtcp"&&(X.innerHTML=Y),A=m(p),g(N.$$.fragment,p),F=m(p),V=$(p,"P",{"data-svelte-h":!0}),k(V)!=="svelte-1pfsro2"&&(V.innerHTML=h),J=m(p),g(L.$$.fragment,p),S=m(p),Q=$(p,"P",{"data-svelte-h":!0}),k(Q)!=="svelte-2s71om"&&(Q.textContent=se)},m(p,_){M(s,p,_),l(p,c,_),M(a,p,_),l(p,o,_),l(p,d,_),l(p,v,_),M(W,p,_),l(p,G,_),l(p,C,_),l(p,E,_),M(I,p,_),l(p,x,_),l(p,B,_),l(p,T,_),M(H,p,_),l(p,z,_),l(p,X,_),l(p,A,_),M(N,p,_),l(p,F,_),l(p,V,_),l(p,J,_),M(L,p,_),l(p,S,_),l(p,Q,_),q=!0},p(p,_){const Ce={};_&2&&(Ce.$$scope={dirty:_,ctx:p}),s.$set(Ce)},i(p){q||(y(s.$$.fragment,p),y(a.$$.fragment,p),y(W.$$.fragment,p),y(I.$$.fragment,p),y(H.$$.fragment,p),y(N.$$.fragment,p),y(L.$$.fragment,p),q=!0)},o(p){b(s.$$.fragment,p),b(a.$$.fragment,p),b(W.$$.fragment,p),b(I.$$.fragment,p),b(H.$$.fragment,p),b(N.$$.fragment,p),b(L.$$.fragment,p),q=!1},d(p){p&&(t(c),t(o),t(d),t(v),t(G),t(C),t(E),t(x),t(B),t(T),t(z),t(X),t(A),t(F),t(V),t(J),t(S),t(Q)),j(s,p),j(a,p),j(W,p),j(I,p),j(H,p),j(N,p),j(L,p)}}}function ct(Z){let s,c;return s=new We({props:{$$slots:{default:[mt]},$$scope:{ctx:Z}}}),{c(){u(s.$$.fragment)},l(a){g(s.$$.fragment,a)},m(a,o){M(s,a,o),c=!0},p(a,o){const d={};o&2&&(d.$$scope={dirty:o,ctx:a}),s.$set(d)},i(a){c||(y(s.$$.fragment,a),c=!0)},o(a){b(s.$$.fragment,a),c=!1},d(a){j(s,a)}}}function ht(Z){let s,c=`For a more in-depth example of how to finetune a model for masked language modeling, take a look at the corresponding
<a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb" rel="nofollow">PyTorch notebook</a>
or <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb" rel="nofollow">TensorFlow notebook</a>.`;return{c(){s=w("p"),s.innerHTML=c},l(a){s=$(a,"P",{"data-svelte-h":!0}),k(s)!=="svelte-1c4g6cv"&&(s.innerHTML=c)},m(a,o){l(a,s,o)},p:ee,d(a){a&&t(s)}}}function dt(Z){let s,c="Tokenize the text and return the <code>input_ids</code> as PyTorch tensors. You’ll also need to specify the position of the <code>&lt;mask&gt;</code> token:",a,o,d,f,v="Pass your inputs to the model and return the <code>logits</code> of the masked token:",W,G,C,U,E="Then return the three masked tokens with the highest probability and print them out:",I,x,B;return o=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJ1c2VybmFtZSUyRm15X2F3ZXNvbWVfZWxpNV9tbG1fbW9kZWwlMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHRleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQW1hc2tfdG9rZW5faW5kZXglMjAlM0QlMjB0b3JjaC53aGVyZShpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQlMjAlM0QlM0QlMjB0b2tlbml6ZXIubWFza190b2tlbl9pZCklNUIxJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;username/my_awesome_eli5_mlm_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_token_index = torch.where(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>] == tokenizer.mask_token_id)[<span class="hljs-number">1</span>]`,wrap:!1}}),G=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvck1hc2tlZExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JNYXNrZWRMTS5mcm9tX3ByZXRyYWluZWQoJTIydXNlcm5hbWUlMkZteV9hd2Vzb21lX2VsaTVfbWxtX21vZGVsJTIyKSUwQWxvZ2l0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKS5sb2dpdHMlMEFtYXNrX3Rva2VuX2xvZ2l0cyUyMCUzRCUyMGxvZ2l0cyU1QjAlMkMlMjBtYXNrX3Rva2VuX2luZGV4JTJDJTIwJTNBJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;username/my_awesome_eli5_mlm_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>logits = model(**inputs).logits
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_token_logits = logits[<span class="hljs-number">0</span>, mask_token_index, :]`,wrap:!1}}),x=new R({props:{code:"dG9wXzNfdG9rZW5zJTIwJTNEJTIwdG9yY2gudG9wayhtYXNrX3Rva2VuX2xvZ2l0cyUyQyUyMDMlMkMlMjBkaW0lM0QxKS5pbmRpY2VzJTVCMCU1RC50b2xpc3QoKSUwQSUwQWZvciUyMHRva2VuJTIwaW4lMjB0b3BfM190b2tlbnMlM0ElMEElMjAlMjAlMjAlMjBwcmludCh0ZXh0LnJlcGxhY2UodG9rZW5pemVyLm1hc2tfdG9rZW4lMkMlMjB0b2tlbml6ZXIuZGVjb2RlKCU1QnRva2VuJTVEKSkp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>top_3_tokens = torch.topk(mask_token_logits, <span class="hljs-number">3</span>, dim=<span class="hljs-number">1</span>).indices[<span class="hljs-number">0</span>].tolist()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> token <span class="hljs-keyword">in</span> top_3_tokens:
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(text.replace(tokenizer.mask_token, tokenizer.decode([token])))
The Milky Way <span class="hljs-keyword">is</span> a spiral galaxy.
The Milky Way <span class="hljs-keyword">is</span> a massive galaxy.
The Milky Way <span class="hljs-keyword">is</span> a small galaxy.`,wrap:!1}}),{c(){s=w("p"),s.innerHTML=c,a=i(),u(o.$$.fragment),d=i(),f=w("p"),f.innerHTML=v,W=i(),u(G.$$.fragment),C=i(),U=w("p"),U.textContent=E,I=i(),u(x.$$.fragment)},l(r){s=$(r,"P",{"data-svelte-h":!0}),k(s)!=="svelte-1rt88cg"&&(s.innerHTML=c),a=m(r),g(o.$$.fragment,r),d=m(r),f=$(r,"P",{"data-svelte-h":!0}),k(f)!=="svelte-1abk23t"&&(f.innerHTML=v),W=m(r),g(G.$$.fragment,r),C=m(r),U=$(r,"P",{"data-svelte-h":!0}),k(U)!=="svelte-jux2mn"&&(U.textContent=E),I=m(r),g(x.$$.fragment,r)},m(r,T){l(r,s,T),l(r,a,T),M(o,r,T),l(r,d,T),l(r,f,T),l(r,W,T),M(G,r,T),l(r,C,T),l(r,U,T),l(r,I,T),M(x,r,T),B=!0},p:ee,i(r){B||(y(o.$$.fragment,r),y(G.$$.fragment,r),y(x.$$.fragment,r),B=!0)},o(r){b(o.$$.fragment,r),b(G.$$.fragment,r),b(x.$$.fragment,r),B=!1},d(r){r&&(t(s),t(a),t(d),t(f),t(W),t(C),t(U),t(I)),j(o,r),j(G,r),j(x,r)}}}function ft(Z){let s,c;return s=new We({props:{$$slots:{default:[dt]},$$scope:{ctx:Z}}}),{c(){u(s.$$.fragment)},l(a){g(s.$$.fragment,a)},m(a,o){M(s,a,o),c=!0},p(a,o){const d={};o&2&&(d.$$scope={dirty:o,ctx:a}),s.$set(d)},i(a){c||(y(s.$$.fragment,a),c=!0)},o(a){b(s.$$.fragment,a),c=!1},d(a){j(s,a)}}}function ut(Z){let s,c="Tokenize the text and return the <code>input_ids</code> as TensorFlow tensors. You’ll also need to specify the position of the <code>&lt;mask&gt;</code> token:",a,o,d,f,v="Pass your inputs to the model and return the <code>logits</code> of the masked token:",W,G,C,U,E="Then return the three masked tokens with the highest probability and print them out:",I,x,B;return o=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJ1c2VybmFtZSUyRm15X2F3ZXNvbWVfZWxpNV9tbG1fbW9kZWwlMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHRleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKSUwQW1hc2tfdG9rZW5faW5kZXglMjAlM0QlMjB0Zi53aGVyZShpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQlMjAlM0QlM0QlMjB0b2tlbml6ZXIubWFza190b2tlbl9pZCklNUIwJTJDJTIwMSU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;username/my_awesome_eli5_mlm_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_token_index = tf.where(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>] == tokenizer.mask_token_id)[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]`,wrap:!1}}),G=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yTWFza2VkTE0lMEElMEFtb2RlbCUyMCUzRCUyMFRGQXV0b01vZGVsRm9yTWFza2VkTE0uZnJvbV9wcmV0cmFpbmVkKCUyMnVzZXJuYW1lJTJGbXlfYXdlc29tZV9lbGk1X21sbV9tb2RlbCUyMiklMEFsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRzJTBBbWFza190b2tlbl9sb2dpdHMlMjAlM0QlMjBsb2dpdHMlNUIwJTJDJTIwbWFza190b2tlbl9pbmRleCUyQyUyMCUzQSU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;username/my_awesome_eli5_mlm_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>logits = model(**inputs).logits
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_token_logits = logits[<span class="hljs-number">0</span>, mask_token_index, :]`,wrap:!1}}),x=new R({props:{code:"dG9wXzNfdG9rZW5zJTIwJTNEJTIwdGYubWF0aC50b3BfayhtYXNrX3Rva2VuX2xvZ2l0cyUyQyUyMDMpLmluZGljZXMubnVtcHkoKSUwQSUwQWZvciUyMHRva2VuJTIwaW4lMjB0b3BfM190b2tlbnMlM0ElMEElMjAlMjAlMjAlMjBwcmludCh0ZXh0LnJlcGxhY2UodG9rZW5pemVyLm1hc2tfdG9rZW4lMkMlMjB0b2tlbml6ZXIuZGVjb2RlKCU1QnRva2VuJTVEKSkp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>top_3_tokens = tf.math.top_k(mask_token_logits, <span class="hljs-number">3</span>).indices.numpy()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> token <span class="hljs-keyword">in</span> top_3_tokens:
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(text.replace(tokenizer.mask_token, tokenizer.decode([token])))
The Milky Way <span class="hljs-keyword">is</span> a spiral galaxy.
The Milky Way <span class="hljs-keyword">is</span> a massive galaxy.
The Milky Way <span class="hljs-keyword">is</span> a small galaxy.`,wrap:!1}}),{c(){s=w("p"),s.innerHTML=c,a=i(),u(o.$$.fragment),d=i(),f=w("p"),f.innerHTML=v,W=i(),u(G.$$.fragment),C=i(),U=w("p"),U.textContent=E,I=i(),u(x.$$.fragment)},l(r){s=$(r,"P",{"data-svelte-h":!0}),k(s)!=="svelte-h6kxlc"&&(s.innerHTML=c),a=m(r),g(o.$$.fragment,r),d=m(r),f=$(r,"P",{"data-svelte-h":!0}),k(f)!=="svelte-1abk23t"&&(f.innerHTML=v),W=m(r),g(G.$$.fragment,r),C=m(r),U=$(r,"P",{"data-svelte-h":!0}),k(U)!=="svelte-jux2mn"&&(U.textContent=E),I=m(r),g(x.$$.fragment,r)},m(r,T){l(r,s,T),l(r,a,T),M(o,r,T),l(r,d,T),l(r,f,T),l(r,W,T),M(G,r,T),l(r,C,T),l(r,U,T),l(r,I,T),M(x,r,T),B=!0},p:ee,i(r){B||(y(o.$$.fragment,r),y(G.$$.fragment,r),y(x.$$.fragment,r),B=!0)},o(r){b(o.$$.fragment,r),b(G.$$.fragment,r),b(x.$$.fragment,r),B=!1},d(r){r&&(t(s),t(a),t(d),t(f),t(W),t(C),t(U),t(I)),j(o,r),j(G,r),j(x,r)}}}function gt(Z){let s,c;return s=new We({props:{$$slots:{default:[ut]},$$scope:{ctx:Z}}}),{c(){u(s.$$.fragment)},l(a){g(s.$$.fragment,a)},m(a,o){M(s,a,o),c=!0},p(a,o){const d={};o&2&&(d.$$scope={dirty:o,ctx:a}),s.$set(d)},i(a){c||(y(s.$$.fragment,a),c=!0)},o(a){b(s.$$.fragment,a),c=!1},d(a){j(s,a)}}}function Mt(Z){let s,c,a,o,d,f,v,W,G,C,U,E=`Masked language modeling predicts a masked token in a sequence, and the model can attend to tokens bidirectionally. This
means the model has full access to the tokens on the left and right. Masked language modeling is great for tasks that
require a good contextual understanding of an entire sequence. BERT is an example of a masked language model.`,I,x,B="This guide will show you how to:",r,T,H='<li>Finetune <a href="https://huggingface.co/distilbert/distilroberta-base" rel="nofollow">DistilRoBERTa</a> on the <a href="https://www.reddit.com/r/askscience/" rel="nofollow">r/askscience</a> subset of the <a href="https://huggingface.co/datasets/eli5" rel="nofollow">ELI5</a> dataset.</li> <li>Use your finetuned model for inference.</li>',z,X,Y,A,N="Before you begin, make sure you have all the necessary libraries installed:",F,V,h,J,L="We encourage you to log in to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to log in:",S,Q,se,q,p,_,Ce='Start by loading the first 5000 examples from the <a href="https://huggingface.co/datasets/eli5_category" rel="nofollow">ELI5-Category</a> dataset with the 🤗 Datasets library. This’ll give you a chance to experiment and make sure everything works before spending more time training on the full dataset.',Xe,te,Ve,ae,$s='Split the dataset’s <code>train</code> split into a train and test set with the <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.train_test_split" rel="nofollow">train_test_split</a> method:',ze,le,Fe,ne,ks="Then take a look at an example:",Ee,re,He,oe,Ts="While this may look like a lot, you’re only really interested in the <code>text</code> field. What’s cool about language modeling tasks is you don’t need labels (also known as an unsupervised task) because the next word <em>is</em> the label.",Ye,pe,Ae,ie,Ne,me,xs="For masked language modeling, the next step is to load a DistilRoBERTa tokenizer to process the <code>text</code> subfield:",Qe,ce,Le,he,_s='You’ll notice from the example above, the <code>text</code> field is actually nested inside <code>answers</code>. This means you’ll need to extract the <code>text</code> subfield from its nested structure with the <a href="https://huggingface.co/docs/datasets/process#flatten" rel="nofollow"><code>flatten</code></a> method:',qe,de,Se,fe,Js=`Each subfield is now a separate column as indicated by the <code>answers</code> prefix, and the <code>text</code> field is a list now. Instead
of tokenizing each sentence separately, convert the list to a string so you can jointly tokenize them.`,Pe,ue,Us="Here is a first preprocessing function to join the list of strings for each example and tokenize the result:",De,ge,Ke,Me,Zs='To apply this preprocessing function over the entire dataset, use the 🤗 Datasets <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.map" rel="nofollow">map</a> method. You can speed up the <code>map</code> function by setting <code>batched=True</code> to process multiple elements of the dataset at once, and increasing the number of processes with <code>num_proc</code>. Remove any columns you don’t need:',Oe,ye,es,be,vs="This dataset contains the token sequences, but some of these are longer than the maximum input length for the model.",ss,je,Gs="You can now use a second preprocessing function to",ts,we,Rs="<li>concatenate all the sequences</li> <li>split the concatenated sequences into shorter chunks defined by <code>block_size</code>, which should be both shorter than the maximum input length and short enough for your GPU RAM.</li>",as,$e,ls,ke,Ws="Apply the <code>group_texts</code> function over the entire dataset:",ns,Te,rs,xe,Cs='Now create a batch of examples using <a href="/docs/transformers/main/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling">DataCollatorForLanguageModeling</a>. It’s more efficient to <em>dynamically pad</em> the sentences to the longest length in a batch during collation, instead of padding the whole dataset to the maximum length.',os,P,ps,_e,is,D,ms,K,cs,Je,hs,Ue,Is="Great, now that you’ve finetuned a model, you can use it for inference!",ds,Ze,Bs="Come up with some text you’d like the model to fill in the blank with, and use the special <code>&lt;mask&gt;</code> token to indicate the blank:",fs,ve,us,Ge,Xs='The simplest way to try out your finetuned model for inference is to use it in a <a href="/docs/transformers/main/en/main_classes/pipelines#transformers.pipeline">pipeline()</a>. Instantiate a <code>pipeline</code> for fill-mask with your model, and pass your text to it. If you like, you can use the <code>top_k</code> parameter to specify how many predictions to return:',gs,Re,Ms,O,ys,Ie,bs;return d=new Be({props:{title:"Masked language modeling",local:"masked-language-modeling",headingTag:"h1"}}),v=new et({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/masked_language_modeling.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/masked_language_modeling.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/masked_language_modeling.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/masked_language_modeling.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/masked_language_modeling.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/masked_language_modeling.ipynb"}]}}),G=new Ns({props:{id:"mqElG5QJWUg"}}),X=new js({props:{$$slots:{default:[st]},$$scope:{ctx:Z}}}),V=new R({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwZXZhbHVhdGU=",highlighted:"pip install transformers datasets evaluate",wrap:!1}}),Q=new R({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_login()`,wrap:!1}}),q=new Be({props:{title:"Load ELI5 dataset",local:"load-eli5-dataset",headingTag:"h2"}}),te=new R({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZWxpNSUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJlbGk1X2NhdGVnb3J5JTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiU1QiUzQTUwMDAlNUQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = load_dataset(<span class="hljs-string">&quot;eli5_category&quot;</span>, split=<span class="hljs-string">&quot;train[:5000]&quot;</span>)`,wrap:!1}}),le=new R({props:{code:"ZWxpNSUyMCUzRCUyMGVsaTUudHJhaW5fdGVzdF9zcGxpdCh0ZXN0X3NpemUlM0QwLjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = eli5.train_test_split(test_size=<span class="hljs-number">0.2</span>)',wrap:!1}}),re=new R({props:{code:"ZWxpNSU1QiUyMnRyYWluJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>eli5[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;q_id&#x27;</span>: <span class="hljs-string">&#x27;7h191n&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;What does the tax bill that was passed today mean? How will it affect Americans in each tax bracket?&#x27;</span>,
 <span class="hljs-string">&#x27;selftext&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
 <span class="hljs-string">&#x27;category&#x27;</span>: <span class="hljs-string">&#x27;Economics&#x27;</span>,
 <span class="hljs-string">&#x27;subreddit&#x27;</span>: <span class="hljs-string">&#x27;explainlikeimfive&#x27;</span>,
 <span class="hljs-string">&#x27;answers&#x27;</span>: {<span class="hljs-string">&#x27;a_id&#x27;</span>: [<span class="hljs-string">&#x27;dqnds8l&#x27;</span>, <span class="hljs-string">&#x27;dqnd1jl&#x27;</span>, <span class="hljs-string">&#x27;dqng3i1&#x27;</span>, <span class="hljs-string">&#x27;dqnku5x&#x27;</span>],
  <span class="hljs-string">&#x27;text&#x27;</span>: [<span class="hljs-string">&quot;The tax bill is 500 pages long and there were a lot of changes still going on right to the end. It&#x27;s not just an adjustment to the income tax brackets, it&#x27;s a whole bunch of changes. As such there is no good answer to your question. The big take aways are: - Big reduction in corporate income tax rate will make large companies very happy. - Pass through rate change will make certain styles of business (law firms, hedge funds) extremely happy - Income tax changes are moderate, and are set to expire (though it&#x27;s the kind of thing that might just always get re-applied without being made permanent) - People in high tax states (California, New York) lose out, and many of them will end up with their taxes raised.&quot;</span>,
   <span class="hljs-string">&#x27;None yet. It has to be reconciled with a vastly different house bill and then passed again.&#x27;</span>,
   <span class="hljs-string">&#x27;Also: does this apply to 2017 taxes? Or does it start with 2018 taxes?&#x27;</span>,
   <span class="hljs-string">&#x27;This article explains both the House and senate bills, including the proposed changes to your income taxes based on your income level. URL_0&#x27;</span>],
  <span class="hljs-string">&#x27;score&#x27;</span>: [<span class="hljs-number">21</span>, <span class="hljs-number">19</span>, <span class="hljs-number">5</span>, <span class="hljs-number">3</span>],
  <span class="hljs-string">&#x27;text_urls&#x27;</span>: [[],
   [],
   [],
   [<span class="hljs-string">&#x27;https://www.investopedia.com/news/trumps-tax-reform-what-can-be-done/&#x27;</span>]]},
 <span class="hljs-string">&#x27;title_urls&#x27;</span>: [<span class="hljs-string">&#x27;url&#x27;</span>],
 <span class="hljs-string">&#x27;selftext_urls&#x27;</span>: [<span class="hljs-string">&#x27;url&#x27;</span>]}`,wrap:!1}}),pe=new Be({props:{title:"Preprocess",local:"preprocess",headingTag:"h2"}}),ie=new Ns({props:{id:"8PmhEIXhBvI"}}),ce=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0JTJGZGlzdGlscm9iZXJ0YS1iYXNlJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert/distilroberta-base&quot;</span>)`,wrap:!1}}),de=new R({props:{code:"ZWxpNSUyMCUzRCUyMGVsaTUuZmxhdHRlbigpJTBBZWxpNSU1QiUyMnRyYWluJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = eli5.flatten()
<span class="hljs-meta">&gt;&gt;&gt; </span>eli5[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;q_id&#x27;</span>: <span class="hljs-string">&#x27;7h191n&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;What does the tax bill that was passed today mean? How will it affect Americans in each tax bracket?&#x27;</span>,
 <span class="hljs-string">&#x27;selftext&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
 <span class="hljs-string">&#x27;category&#x27;</span>: <span class="hljs-string">&#x27;Economics&#x27;</span>,
 <span class="hljs-string">&#x27;subreddit&#x27;</span>: <span class="hljs-string">&#x27;explainlikeimfive&#x27;</span>,
 <span class="hljs-string">&#x27;answers.a_id&#x27;</span>: [<span class="hljs-string">&#x27;dqnds8l&#x27;</span>, <span class="hljs-string">&#x27;dqnd1jl&#x27;</span>, <span class="hljs-string">&#x27;dqng3i1&#x27;</span>, <span class="hljs-string">&#x27;dqnku5x&#x27;</span>],
 <span class="hljs-string">&#x27;answers.text&#x27;</span>: [<span class="hljs-string">&quot;The tax bill is 500 pages long and there were a lot of changes still going on right to the end. It&#x27;s not just an adjustment to the income tax brackets, it&#x27;s a whole bunch of changes. As such there is no good answer to your question. The big take aways are: - Big reduction in corporate income tax rate will make large companies very happy. - Pass through rate change will make certain styles of business (law firms, hedge funds) extremely happy - Income tax changes are moderate, and are set to expire (though it&#x27;s the kind of thing that might just always get re-applied without being made permanent) - People in high tax states (California, New York) lose out, and many of them will end up with their taxes raised.&quot;</span>,
  <span class="hljs-string">&#x27;None yet. It has to be reconciled with a vastly different house bill and then passed again.&#x27;</span>,
  <span class="hljs-string">&#x27;Also: does this apply to 2017 taxes? Or does it start with 2018 taxes?&#x27;</span>,
  <span class="hljs-string">&#x27;This article explains both the House and senate bills, including the proposed changes to your income taxes based on your income level. URL_0&#x27;</span>],
 <span class="hljs-string">&#x27;answers.score&#x27;</span>: [<span class="hljs-number">21</span>, <span class="hljs-number">19</span>, <span class="hljs-number">5</span>, <span class="hljs-number">3</span>],
 <span class="hljs-string">&#x27;answers.text_urls&#x27;</span>: [[],
  [],
  [],
  [<span class="hljs-string">&#x27;https://www.investopedia.com/news/trumps-tax-reform-what-can-be-done/&#x27;</span>]],
 <span class="hljs-string">&#x27;title_urls&#x27;</span>: [<span class="hljs-string">&#x27;url&#x27;</span>],
 <span class="hljs-string">&#x27;selftext_urls&#x27;</span>: [<span class="hljs-string">&#x27;url&#x27;</span>]}`,wrap:!1}}),ge=new R({props:{code:"ZGVmJTIwcHJlcHJvY2Vzc19mdW5jdGlvbihleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjB0b2tlbml6ZXIoJTVCJTIyJTIwJTIyLmpvaW4oeCklMjBmb3IlMjB4JTIwaW4lMjBleGFtcGxlcyU1QiUyMmFuc3dlcnMudGV4dCUyMiU1RCU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer([<span class="hljs-string">&quot; &quot;</span>.join(x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;answers.text&quot;</span>]])`,wrap:!1}}),ye=new R({props:{code:"dG9rZW5pemVkX2VsaTUlMjAlM0QlMjBlbGk1Lm1hcCglMEElMjAlMjAlMjAlMjBwcmVwcm9jZXNzX2Z1bmN0aW9uJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hlZCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBudW1fcHJvYyUzRDQlMkMlMEElMjAlMjAlMjAlMjByZW1vdmVfY29sdW1ucyUzRGVsaTUlNUIlMjJ0cmFpbiUyMiU1RC5jb2x1bW5fbmFtZXMlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_eli5 = eli5.<span class="hljs-built_in">map</span>(
<span class="hljs-meta">... </span>    preprocess_function,
<span class="hljs-meta">... </span>    batched=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    num_proc=<span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    remove_columns=eli5[<span class="hljs-string">&quot;train&quot;</span>].column_names,
<span class="hljs-meta">... </span>)`,wrap:!1}}),$e=new R({props:{code:"YmxvY2tfc2l6ZSUyMCUzRCUyMDEyOCUwQSUwQSUwQWRlZiUyMGdyb3VwX3RleHRzKGV4YW1wbGVzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMyUyMENvbmNhdGVuYXRlJTIwYWxsJTIwdGV4dHMuJTBBJTIwJTIwJTIwJTIwY29uY2F0ZW5hdGVkX2V4YW1wbGVzJTIwJTNEJTIwJTdCayUzQSUyMHN1bShleGFtcGxlcyU1QmslNUQlMkMlMjAlNUIlNUQpJTIwZm9yJTIwayUyMGluJTIwZXhhbXBsZXMua2V5cygpJTdEJTBBJTIwJTIwJTIwJTIwdG90YWxfbGVuZ3RoJTIwJTNEJTIwbGVuKGNvbmNhdGVuYXRlZF9leGFtcGxlcyU1Qmxpc3QoZXhhbXBsZXMua2V5cygpKSU1QjAlNUQlNUQpJTBBJTIwJTIwJTIwJTIwJTIzJTIwV2UlMjBkcm9wJTIwdGhlJTIwc21hbGwlMjByZW1haW5kZXIlMkMlMjB3ZSUyMGNvdWxkJTIwYWRkJTIwcGFkZGluZyUyMGlmJTIwdGhlJTIwbW9kZWwlMjBzdXBwb3J0ZWQlMjBpdCUyMGluc3RlYWQlMjBvZiUyMHRoaXMlMjBkcm9wJTJDJTIweW91JTIwY2FuJTBBJTIwJTIwJTIwJTIwJTIzJTIwY3VzdG9taXplJTIwdGhpcyUyMHBhcnQlMjB0byUyMHlvdXIlMjBuZWVkcy4lMEElMjAlMjAlMjAlMjBpZiUyMHRvdGFsX2xlbmd0aCUyMCUzRSUzRCUyMGJsb2NrX3NpemUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3RhbF9sZW5ndGglMjAlM0QlMjAodG90YWxfbGVuZ3RoJTIwJTJGJTJGJTIwYmxvY2tfc2l6ZSklMjAqJTIwYmxvY2tfc2l6ZSUwQSUyMCUyMCUyMCUyMCUyMyUyMFNwbGl0JTIwYnklMjBjaHVua3MlMjBvZiUyMGJsb2NrX3NpemUuJTBBJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwayUzQSUyMCU1QnQlNUJpJTIwJTNBJTIwaSUyMCUyQiUyMGJsb2NrX3NpemUlNUQlMjBmb3IlMjBpJTIwaW4lMjByYW5nZSgwJTJDJTIwdG90YWxfbGVuZ3RoJTJDJTIwYmxvY2tfc2l6ZSklNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBrJTJDJTIwdCUyMGluJTIwY29uY2F0ZW5hdGVkX2V4YW1wbGVzLml0ZW1zKCklMEElMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjByZXR1cm4lMjByZXN1bHQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>block_size = <span class="hljs-number">128</span>


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">group_texts</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-comment"># Concatenate all texts.</span>
<span class="hljs-meta">... </span>    concatenated_examples = {k: <span class="hljs-built_in">sum</span>(examples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> examples.keys()}
<span class="hljs-meta">... </span>    total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-built_in">list</span>(examples.keys())[<span class="hljs-number">0</span>]])
<span class="hljs-meta">... </span>    <span class="hljs-comment"># We drop the small remainder, we could add padding if the model supported it instead of this drop, you can</span>
<span class="hljs-meta">... </span>    <span class="hljs-comment"># customize this part to your needs.</span>
<span class="hljs-meta">... </span>    <span class="hljs-keyword">if</span> total_length &gt;= block_size:
<span class="hljs-meta">... </span>        total_length = (total_length // block_size) * block_size
<span class="hljs-meta">... </span>    <span class="hljs-comment"># Split by chunks of block_size.</span>
<span class="hljs-meta">... </span>    result = {
<span class="hljs-meta">... </span>        k: [t[i : i + block_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, block_size)]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
<span class="hljs-meta">... </span>    }
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> result`,wrap:!1}}),Te=new R({props:{code:"bG1fZGF0YXNldCUyMCUzRCUyMHRva2VuaXplZF9lbGk1Lm1hcChncm91cF90ZXh0cyUyQyUyMGJhdGNoZWQlM0RUcnVlJTJDJTIwbnVtX3Byb2MlM0Q0KQ==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lm_dataset = tokenized_eli5.<span class="hljs-built_in">map</span>(group_texts, batched=<span class="hljs-literal">True</span>, num_proc=<span class="hljs-number">4</span>)',wrap:!1}}),P=new ws({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[nt],pytorch:[at]},$$scope:{ctx:Z}}}),_e=new Be({props:{title:"Train",local:"train",headingTag:"h2"}}),D=new ws({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[ct],pytorch:[pt]},$$scope:{ctx:Z}}}),K=new js({props:{$$slots:{default:[ht]},$$scope:{ctx:Z}}}),Je=new Be({props:{title:"Inference",local:"inference",headingTag:"h2"}}),ve=new R({props:{code:"dGV4dCUyMCUzRCUyMCUyMlRoZSUyME1pbGt5JTIwV2F5JTIwaXMlMjBhJTIwJTNDbWFzayUzRSUyMGdhbGF4eS4lMjI=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>text = <span class="hljs-string">&quot;The Milky Way is a &lt;mask&gt; galaxy.&quot;</span>',wrap:!1}}),Re=new R({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBbWFza19maWxsZXIlMjAlM0QlMjBwaXBlbGluZSglMjJmaWxsLW1hc2slMjIlMkMlMjAlMjJ1c2VybmFtZSUyRm15X2F3ZXNvbWVfZWxpNV9tbG1fbW9kZWwlMjIpJTBBbWFza19maWxsZXIodGV4dCUyQyUyMHRvcF9rJTNEMyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>mask_filler = pipeline(<span class="hljs-string">&quot;fill-mask&quot;</span>, <span class="hljs-string">&quot;username/my_awesome_eli5_mlm_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_filler(text, top_k=<span class="hljs-number">3</span>)
[{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.5150994658470154</span>,
  <span class="hljs-string">&#x27;token&#x27;</span>: <span class="hljs-number">21300</span>,
  <span class="hljs-string">&#x27;token_str&#x27;</span>: <span class="hljs-string">&#x27; spiral&#x27;</span>,
  <span class="hljs-string">&#x27;sequence&#x27;</span>: <span class="hljs-string">&#x27;The Milky Way is a spiral galaxy.&#x27;</span>},
 {<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.07087188959121704</span>,
  <span class="hljs-string">&#x27;token&#x27;</span>: <span class="hljs-number">2232</span>,
  <span class="hljs-string">&#x27;token_str&#x27;</span>: <span class="hljs-string">&#x27; massive&#x27;</span>,
  <span class="hljs-string">&#x27;sequence&#x27;</span>: <span class="hljs-string">&#x27;The Milky Way is a massive galaxy.&#x27;</span>},
 {<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.06434620916843414</span>,
  <span class="hljs-string">&#x27;token&#x27;</span>: <span class="hljs-number">650</span>,
  <span class="hljs-string">&#x27;token_str&#x27;</span>: <span class="hljs-string">&#x27; small&#x27;</span>,
  <span class="hljs-string">&#x27;sequence&#x27;</span>: <span class="hljs-string">&#x27;The Milky Way is a small galaxy.&#x27;</span>}]`,wrap:!1}}),O=new ws({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[gt],pytorch:[ft]},$$scope:{ctx:Z}}}),{c(){s=w("meta"),c=i(),a=w("p"),o=i(),u(d.$$.fragment),f=i(),u(v.$$.fragment),W=i(),u(G.$$.fragment),C=i(),U=w("p"),U.textContent=E,I=i(),x=w("p"),x.textContent=B,r=i(),T=w("ol"),T.innerHTML=H,z=i(),u(X.$$.fragment),Y=i(),A=w("p"),A.textContent=N,F=i(),u(V.$$.fragment),h=i(),J=w("p"),J.textContent=L,S=i(),u(Q.$$.fragment),se=i(),u(q.$$.fragment),p=i(),_=w("p"),_.innerHTML=Ce,Xe=i(),u(te.$$.fragment),Ve=i(),ae=w("p"),ae.innerHTML=$s,ze=i(),u(le.$$.fragment),Fe=i(),ne=w("p"),ne.textContent=ks,Ee=i(),u(re.$$.fragment),He=i(),oe=w("p"),oe.innerHTML=Ts,Ye=i(),u(pe.$$.fragment),Ae=i(),u(ie.$$.fragment),Ne=i(),me=w("p"),me.innerHTML=xs,Qe=i(),u(ce.$$.fragment),Le=i(),he=w("p"),he.innerHTML=_s,qe=i(),u(de.$$.fragment),Se=i(),fe=w("p"),fe.innerHTML=Js,Pe=i(),ue=w("p"),ue.textContent=Us,De=i(),u(ge.$$.fragment),Ke=i(),Me=w("p"),Me.innerHTML=Zs,Oe=i(),u(ye.$$.fragment),es=i(),be=w("p"),be.textContent=vs,ss=i(),je=w("p"),je.textContent=Gs,ts=i(),we=w("ul"),we.innerHTML=Rs,as=i(),u($e.$$.fragment),ls=i(),ke=w("p"),ke.innerHTML=Ws,ns=i(),u(Te.$$.fragment),rs=i(),xe=w("p"),xe.innerHTML=Cs,os=i(),u(P.$$.fragment),ps=i(),u(_e.$$.fragment),is=i(),u(D.$$.fragment),ms=i(),u(K.$$.fragment),cs=i(),u(Je.$$.fragment),hs=i(),Ue=w("p"),Ue.textContent=Is,ds=i(),Ze=w("p"),Ze.innerHTML=Bs,fs=i(),u(ve.$$.fragment),us=i(),Ge=w("p"),Ge.innerHTML=Xs,gs=i(),u(Re.$$.fragment),Ms=i(),u(O.$$.fragment),ys=i(),Ie=w("p"),this.h()},l(e){const n=Ks("svelte-u9bgzb",document.head);s=$(n,"META",{name:!0,content:!0}),n.forEach(t),c=m(e),a=$(e,"P",{}),Ys(a).forEach(t),o=m(e),g(d.$$.fragment,e),f=m(e),g(v.$$.fragment,e),W=m(e),g(G.$$.fragment,e),C=m(e),U=$(e,"P",{"data-svelte-h":!0}),k(U)!=="svelte-ed5ap0"&&(U.textContent=E),I=m(e),x=$(e,"P",{"data-svelte-h":!0}),k(x)!=="svelte-1aff4p7"&&(x.textContent=B),r=m(e),T=$(e,"OL",{"data-svelte-h":!0}),k(T)!=="svelte-k2329c"&&(T.innerHTML=H),z=m(e),g(X.$$.fragment,e),Y=m(e),A=$(e,"P",{"data-svelte-h":!0}),k(A)!=="svelte-1c9nexd"&&(A.textContent=N),F=m(e),g(V.$$.fragment,e),h=m(e),J=$(e,"P",{"data-svelte-h":!0}),k(J)!=="svelte-27hn0u"&&(J.textContent=L),S=m(e),g(Q.$$.fragment,e),se=m(e),g(q.$$.fragment,e),p=m(e),_=$(e,"P",{"data-svelte-h":!0}),k(_)!=="svelte-1172m8d"&&(_.innerHTML=Ce),Xe=m(e),g(te.$$.fragment,e),Ve=m(e),ae=$(e,"P",{"data-svelte-h":!0}),k(ae)!=="svelte-rugbz4"&&(ae.innerHTML=$s),ze=m(e),g(le.$$.fragment,e),Fe=m(e),ne=$(e,"P",{"data-svelte-h":!0}),k(ne)!=="svelte-1m91ua0"&&(ne.textContent=ks),Ee=m(e),g(re.$$.fragment,e),He=m(e),oe=$(e,"P",{"data-svelte-h":!0}),k(oe)!=="svelte-dtsbii"&&(oe.innerHTML=Ts),Ye=m(e),g(pe.$$.fragment,e),Ae=m(e),g(ie.$$.fragment,e),Ne=m(e),me=$(e,"P",{"data-svelte-h":!0}),k(me)!=="svelte-i99oeh"&&(me.innerHTML=xs),Qe=m(e),g(ce.$$.fragment,e),Le=m(e),he=$(e,"P",{"data-svelte-h":!0}),k(he)!=="svelte-1275j7o"&&(he.innerHTML=_s),qe=m(e),g(de.$$.fragment,e),Se=m(e),fe=$(e,"P",{"data-svelte-h":!0}),k(fe)!=="svelte-1mdv3gu"&&(fe.innerHTML=Js),Pe=m(e),ue=$(e,"P",{"data-svelte-h":!0}),k(ue)!=="svelte-njkc6i"&&(ue.textContent=Us),De=m(e),g(ge.$$.fragment,e),Ke=m(e),Me=$(e,"P",{"data-svelte-h":!0}),k(Me)!=="svelte-1xbgtq4"&&(Me.innerHTML=Zs),Oe=m(e),g(ye.$$.fragment,e),es=m(e),be=$(e,"P",{"data-svelte-h":!0}),k(be)!=="svelte-pz0l04"&&(be.textContent=vs),ss=m(e),je=$(e,"P",{"data-svelte-h":!0}),k(je)!=="svelte-5guq64"&&(je.textContent=Gs),ts=m(e),we=$(e,"UL",{"data-svelte-h":!0}),k(we)!=="svelte-vz17je"&&(we.innerHTML=Rs),as=m(e),g($e.$$.fragment,e),ls=m(e),ke=$(e,"P",{"data-svelte-h":!0}),k(ke)!=="svelte-1o69amy"&&(ke.innerHTML=Ws),ns=m(e),g(Te.$$.fragment,e),rs=m(e),xe=$(e,"P",{"data-svelte-h":!0}),k(xe)!=="svelte-1bhjz4l"&&(xe.innerHTML=Cs),os=m(e),g(P.$$.fragment,e),ps=m(e),g(_e.$$.fragment,e),is=m(e),g(D.$$.fragment,e),ms=m(e),g(K.$$.fragment,e),cs=m(e),g(Je.$$.fragment,e),hs=m(e),Ue=$(e,"P",{"data-svelte-h":!0}),k(Ue)!=="svelte-633ppb"&&(Ue.textContent=Is),ds=m(e),Ze=$(e,"P",{"data-svelte-h":!0}),k(Ze)!=="svelte-9jago5"&&(Ze.innerHTML=Bs),fs=m(e),g(ve.$$.fragment,e),us=m(e),Ge=$(e,"P",{"data-svelte-h":!0}),k(Ge)!=="svelte-79y659"&&(Ge.innerHTML=Xs),gs=m(e),g(Re.$$.fragment,e),Ms=m(e),g(O.$$.fragment,e),ys=m(e),Ie=$(e,"P",{}),Ys(Ie).forEach(t),this.h()},h(){As(s,"name","hf:doc:metadata"),As(s,"content",yt)},m(e,n){Os(document.head,s),l(e,c,n),l(e,a,n),l(e,o,n),M(d,e,n),l(e,f,n),M(v,e,n),l(e,W,n),M(G,e,n),l(e,C,n),l(e,U,n),l(e,I,n),l(e,x,n),l(e,r,n),l(e,T,n),l(e,z,n),M(X,e,n),l(e,Y,n),l(e,A,n),l(e,F,n),M(V,e,n),l(e,h,n),l(e,J,n),l(e,S,n),M(Q,e,n),l(e,se,n),M(q,e,n),l(e,p,n),l(e,_,n),l(e,Xe,n),M(te,e,n),l(e,Ve,n),l(e,ae,n),l(e,ze,n),M(le,e,n),l(e,Fe,n),l(e,ne,n),l(e,Ee,n),M(re,e,n),l(e,He,n),l(e,oe,n),l(e,Ye,n),M(pe,e,n),l(e,Ae,n),M(ie,e,n),l(e,Ne,n),l(e,me,n),l(e,Qe,n),M(ce,e,n),l(e,Le,n),l(e,he,n),l(e,qe,n),M(de,e,n),l(e,Se,n),l(e,fe,n),l(e,Pe,n),l(e,ue,n),l(e,De,n),M(ge,e,n),l(e,Ke,n),l(e,Me,n),l(e,Oe,n),M(ye,e,n),l(e,es,n),l(e,be,n),l(e,ss,n),l(e,je,n),l(e,ts,n),l(e,we,n),l(e,as,n),M($e,e,n),l(e,ls,n),l(e,ke,n),l(e,ns,n),M(Te,e,n),l(e,rs,n),l(e,xe,n),l(e,os,n),M(P,e,n),l(e,ps,n),M(_e,e,n),l(e,is,n),M(D,e,n),l(e,ms,n),M(K,e,n),l(e,cs,n),M(Je,e,n),l(e,hs,n),l(e,Ue,n),l(e,ds,n),l(e,Ze,n),l(e,fs,n),M(ve,e,n),l(e,us,n),l(e,Ge,n),l(e,gs,n),M(Re,e,n),l(e,Ms,n),M(O,e,n),l(e,ys,n),l(e,Ie,n),bs=!0},p(e,[n]){const Vs={};n&2&&(Vs.$$scope={dirty:n,ctx:e}),X.$set(Vs);const zs={};n&2&&(zs.$$scope={dirty:n,ctx:e}),P.$set(zs);const Fs={};n&2&&(Fs.$$scope={dirty:n,ctx:e}),D.$set(Fs);const Es={};n&2&&(Es.$$scope={dirty:n,ctx:e}),K.$set(Es);const Hs={};n&2&&(Hs.$$scope={dirty:n,ctx:e}),O.$set(Hs)},i(e){bs||(y(d.$$.fragment,e),y(v.$$.fragment,e),y(G.$$.fragment,e),y(X.$$.fragment,e),y(V.$$.fragment,e),y(Q.$$.fragment,e),y(q.$$.fragment,e),y(te.$$.fragment,e),y(le.$$.fragment,e),y(re.$$.fragment,e),y(pe.$$.fragment,e),y(ie.$$.fragment,e),y(ce.$$.fragment,e),y(de.$$.fragment,e),y(ge.$$.fragment,e),y(ye.$$.fragment,e),y($e.$$.fragment,e),y(Te.$$.fragment,e),y(P.$$.fragment,e),y(_e.$$.fragment,e),y(D.$$.fragment,e),y(K.$$.fragment,e),y(Je.$$.fragment,e),y(ve.$$.fragment,e),y(Re.$$.fragment,e),y(O.$$.fragment,e),bs=!0)},o(e){b(d.$$.fragment,e),b(v.$$.fragment,e),b(G.$$.fragment,e),b(X.$$.fragment,e),b(V.$$.fragment,e),b(Q.$$.fragment,e),b(q.$$.fragment,e),b(te.$$.fragment,e),b(le.$$.fragment,e),b(re.$$.fragment,e),b(pe.$$.fragment,e),b(ie.$$.fragment,e),b(ce.$$.fragment,e),b(de.$$.fragment,e),b(ge.$$.fragment,e),b(ye.$$.fragment,e),b($e.$$.fragment,e),b(Te.$$.fragment,e),b(P.$$.fragment,e),b(_e.$$.fragment,e),b(D.$$.fragment,e),b(K.$$.fragment,e),b(Je.$$.fragment,e),b(ve.$$.fragment,e),b(Re.$$.fragment,e),b(O.$$.fragment,e),bs=!1},d(e){e&&(t(c),t(a),t(o),t(f),t(W),t(C),t(U),t(I),t(x),t(r),t(T),t(z),t(Y),t(A),t(F),t(h),t(J),t(S),t(se),t(p),t(_),t(Xe),t(Ve),t(ae),t(ze),t(Fe),t(ne),t(Ee),t(He),t(oe),t(Ye),t(Ae),t(Ne),t(me),t(Qe),t(Le),t(he),t(qe),t(Se),t(fe),t(Pe),t(ue),t(De),t(Ke),t(Me),t(Oe),t(es),t(be),t(ss),t(je),t(ts),t(we),t(as),t(ls),t(ke),t(ns),t(rs),t(xe),t(os),t(ps),t(is),t(ms),t(cs),t(hs),t(Ue),t(ds),t(Ze),t(fs),t(us),t(Ge),t(gs),t(Ms),t(ys),t(Ie)),t(s),j(d,e),j(v,e),j(G,e),j(X,e),j(V,e),j(Q,e),j(q,e),j(te,e),j(le,e),j(re,e),j(pe,e),j(ie,e),j(ce,e),j(de,e),j(ge,e),j(ye,e),j($e,e),j(Te,e),j(P,e),j(_e,e),j(D,e),j(K,e),j(Je,e),j(ve,e),j(Re,e),j(O,e)}}}const yt='{"title":"Masked language modeling","local":"masked-language-modeling","sections":[{"title":"Load ELI5 dataset","local":"load-eli5-dataset","sections":[],"depth":2},{"title":"Preprocess","local":"preprocess","sections":[],"depth":2},{"title":"Train","local":"train","sections":[],"depth":2},{"title":"Inference","local":"inference","sections":[],"depth":2}],"depth":1}';function bt(Z){return Ss(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ut extends Ps{constructor(s){super(),Ds(this,s,bt,Mt,qs,{})}}export{Ut as component};
