import{s as si,f as ai,o as ni,n as Se}from"../chunks/scheduler.37c15a92.js";import{S as ii,i as oi,g as J,s as i,r as d,A as Mi,h as u,f as t,c as o,j as Aa,u as h,x as T,k as fe,y as ri,a as s,v as w,t as p,b as Ce,d as r,w as m,p as _e}from"../chunks/index.2bf4358c.js";import{T as Fe}from"../chunks/Tip.363c041f.js";import{Y as Ra}from"../chunks/Youtube.1e50a667.js";import{C as f}from"../chunks/CodeBlock.4f5fc1ad.js";import{C as ti}from"../chunks/CourseFloatingBanner.15ba07e6.js";import{F as pi}from"../chunks/FrameworkSwitchCourse.8d4d4ab6.js";import{H as Ee}from"../chunks/Heading.8ada512a.js";function yi(I){let a,c;return a=new ti({props:{chapter:7,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter7/section3_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter7/section3_tf.ipynb"}]}}),{c(){d(a.$$.fragment)},l(n){h(a.$$.fragment,n)},m(n,y){w(a,n,y),c=!0},i(n){c||(r(a.$$.fragment,n),c=!0)},o(n){p(a.$$.fragment,n),c=!1},d(n){m(a,n)}}}function ci(I){let a,c;return a=new ti({props:{chapter:7,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter7/section3_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter7/section3_pt.ipynb"}]}}),{c(){d(a.$$.fragment)},l(n){h(a.$$.fragment,n)},m(n,y){w(a,n,y),c=!0},i(n){c||(r(a.$$.fragment,n),c=!0)},o(n){p(a.$$.fragment,n),c=!1},d(n){m(a,n)}}}function di(I){let a,c='🙋 If the terms “masked language modeling” and “pretrained model” sound unfamiliar to you, go check out <a href="/course/chapter1">Chapter 1</a>, where we explain all these core concepts, complete with videos!';return{c(){a=J("p"),a.innerHTML=c},l(n){a=u(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-kl6xhw"&&(a.innerHTML=c)},m(n,y){s(n,a,y)},p:Se,d(n){n&&t(a)}}}function hi(I){let a,c="Let’s go ahead and download DistilBERT using the <code>AutoModelForMaskedLM</code> class:",n,y,_,U,R="We can see how many parameters this model has by calling the <code>summary()</code> method:",W,C,X,Z,B;return y=new f({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yTWFza2VkTE0lMEElMEFtb2RlbF9jaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQlMjIlMEFtb2RlbCUyMCUzRCUyMFRGQXV0b01vZGVsRm9yTWFza2VkTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2NoZWNrcG9pbnQp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForMaskedLM

model_checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>
model = TFAutoModelForMaskedLM.from_pretrained(model_checkpoint)`,wrap:!1}}),C=new f({props:{code:"bW9kZWwuc3VtbWFyeSgp",highlighted:"model.summary()",wrap:!1}}),Z=new f({props:{code:"TW9kZWwlM0ElMjAlMjJ0Zl9kaXN0aWxfYmVydF9mb3JfbWFza2VkX2xtJTIyJTBBX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX18lMEFMYXllciUyMCh0eXBlKSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME91dHB1dCUyMFNoYXBlJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwUGFyYW0lMjAlMjMlMjAlMjAlMjAlMEElM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMEFkaXN0aWxiZXJ0JTIwKFRGRGlzdGlsQmVydE1haW4lMjBtdWx0aXBsZSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDY2MzYyODgwJTIwJTIwJTBBX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX18lMEF2b2NhYl90cmFuc2Zvcm0lMjAoRGVuc2UpJTIwJTIwJTIwJTIwJTIwJTIwbXVsdGlwbGUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjA1OTA1OTIlMjAlMjAlMjAlMjAlMEFfX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fXyUwQXZvY2FiX2xheWVyX25vcm0lMjAoTGF5ZXJOb3JtYSUyMG11bHRpcGxlJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwMTUzNiUyMCUyMCUyMCUyMCUyMCUyMCUwQV9fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fJTBBdm9jYWJfcHJvamVjdG9yJTIwKFRGRGlzdGlsQmVyJTIwbXVsdGlwbGUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAyMzg2NjE3MCUyMCUyMCUwQSUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUwQVRvdGFsJTIwcGFyYW1zJTNBJTIwNjYlMkM5ODUlMkM1MzAlMEFUcmFpbmFibGUlMjBwYXJhbXMlM0ElMjA2NiUyQzk4NSUyQzUzMCUwQU5vbi10cmFpbmFibGUlMjBwYXJhbXMlM0ElMjAwJTBBX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX18=",highlighted:`Model: <span class="hljs-string">&quot;tf_distil_bert_for_masked_lm&quot;</span>
_________________________________________________________________
Layer (<span class="hljs-built_in">type</span>)                 Output Shape              Param <span class="hljs-comment">#   </span>
=================================================================
distilbert (TFDistilBertMain multiple                  <span class="hljs-number">66362880</span>  
_________________________________________________________________
vocab_transform (Dense)      multiple                  <span class="hljs-number">590592</span>    
_________________________________________________________________
vocab_layer_norm (LayerNorma multiple                  <span class="hljs-number">1536</span>      
_________________________________________________________________
vocab_projector (TFDistilBer multiple                  <span class="hljs-number">23866170</span>  
=================================================================
Total params: <span class="hljs-number">66</span>,<span class="hljs-number">985</span>,<span class="hljs-number">530</span>
Trainable params: <span class="hljs-number">66</span>,<span class="hljs-number">985</span>,<span class="hljs-number">530</span>
Non-trainable params: <span class="hljs-number">0</span>
_________________________________________________________________`,wrap:!1}}),{c(){a=J("p"),a.innerHTML=c,n=i(),d(y.$$.fragment),_=i(),U=J("p"),U.innerHTML=R,W=i(),d(C.$$.fragment),X=i(),d(Z.$$.fragment)},l(j){a=u(j,"P",{"data-svelte-h":!0}),T(a)!=="svelte-1cnwv98"&&(a.innerHTML=c),n=o(j),h(y.$$.fragment,j),_=o(j),U=u(j,"P",{"data-svelte-h":!0}),T(U)!=="svelte-t7jivq"&&(U.innerHTML=R),W=o(j),h(C.$$.fragment,j),X=o(j),h(Z.$$.fragment,j)},m(j,G){s(j,a,G),s(j,n,G),w(y,j,G),s(j,_,G),s(j,U,G),s(j,W,G),w(C,j,G),s(j,X,G),w(Z,j,G),B=!0},i(j){B||(r(y.$$.fragment,j),r(C.$$.fragment,j),r(Z.$$.fragment,j),B=!0)},o(j){p(y.$$.fragment,j),p(C.$$.fragment,j),p(Z.$$.fragment,j),B=!1},d(j){j&&(t(a),t(n),t(_),t(U),t(W),t(X)),m(y,j),m(C,j),m(Z,j)}}}function wi(I){let a,c="Let’s go ahead and download DistilBERT using the <code>AutoModelForMaskedLM</code> class:",n,y,_,U,R="We can see how many parameters this model has by calling the <code>num_parameters()</code> method:",W,C,X,Z,B;return y=new f({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvck1hc2tlZExNJTBBJTBBbW9kZWxfY2hlY2twb2ludCUyMCUzRCUyMCUyMmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JNYXNrZWRMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfY2hlY2twb2ludCk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM

model_checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>
model = AutoModelForMaskedLM.from_pretrained(model_checkpoint)`,wrap:!1}}),C=new f({props:{code:"ZGlzdGlsYmVydF9udW1fcGFyYW1ldGVycyUyMCUzRCUyMG1vZGVsLm51bV9wYXJhbWV0ZXJzKCklMjAlMkYlMjAxXzAwMF8wMDAlMEFwcmludChmJTIyJyUzRSUzRSUzRSUyMERpc3RpbEJFUlQlMjBudW1iZXIlMjBvZiUyMHBhcmFtZXRlcnMlM0ElMjAlN0Jyb3VuZChkaXN0aWxiZXJ0X251bV9wYXJhbWV0ZXJzKSU3RE0nJTIyKSUwQXByaW50KGYlMjInJTNFJTNFJTNFJTIwQkVSVCUyMG51bWJlciUyMG9mJTIwcGFyYW1ldGVycyUzQSUyMDExME0nJTIyKQ==",highlighted:`distilbert_num_parameters = model.num_parameters() / <span class="hljs-number">1_000_000</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; DistilBERT number of parameters: <span class="hljs-subst">{<span class="hljs-built_in">round</span>(distilbert_num_parameters)}</span>M&#x27;&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; BERT number of parameters: 110M&#x27;&quot;</span>)`,wrap:!1}}),Z=new f({props:{code:"JyUzRSUzRSUzRSUyMERpc3RpbEJFUlQlMjBudW1iZXIlMjBvZiUyMHBhcmFtZXRlcnMlM0ElMjA2N00nJTBBJyUzRSUzRSUzRSUyMEJFUlQlMjBudW1iZXIlMjBvZiUyMHBhcmFtZXRlcnMlM0ElMjAxMTBNJw==",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; DistilBERT number of parameters: 67M&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; BERT number of parameters: 110M&#x27;</span>`,wrap:!1}}),{c(){a=J("p"),a.innerHTML=c,n=i(),d(y.$$.fragment),_=i(),U=J("p"),U.innerHTML=R,W=i(),d(C.$$.fragment),X=i(),d(Z.$$.fragment)},l(j){a=u(j,"P",{"data-svelte-h":!0}),T(a)!=="svelte-1cnwv98"&&(a.innerHTML=c),n=o(j),h(y.$$.fragment,j),_=o(j),U=u(j,"P",{"data-svelte-h":!0}),T(U)!=="svelte-1ezrcnz"&&(U.innerHTML=R),W=o(j),h(C.$$.fragment,j),X=o(j),h(Z.$$.fragment,j)},m(j,G){s(j,a,G),s(j,n,G),w(y,j,G),s(j,_,G),s(j,U,G),s(j,W,G),w(C,j,G),s(j,X,G),w(Z,j,G),B=!0},i(j){B||(r(y.$$.fragment,j),r(C.$$.fragment,j),r(Z.$$.fragment,j),B=!0)},o(j){p(y.$$.fragment,j),p(C.$$.fragment,j),p(Z.$$.fragment,j),B=!1},d(j){j&&(t(a),t(n),t(_),t(U),t(W),t(X)),m(y,j),m(C,j),m(Z,j)}}}function mi(I){let a,c;return a=new f({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBaW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIybnAlMjIpJTBBdG9rZW5fbG9naXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpLmxvZ2l0cyUwQSUyMyUyMEZpbmQlMjB0aGUlMjBsb2NhdGlvbiUyMG9mJTIwJTVCTUFTSyU1RCUyMGFuZCUyMGV4dHJhY3QlMjBpdHMlMjBsb2dpdHMlMEFtYXNrX3Rva2VuX2luZGV4JTIwJTNEJTIwbnAuYXJnd2hlcmUoaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTIwJTNEJTNEJTIwdG9rZW5pemVyLm1hc2tfdG9rZW5faWQpJTVCMCUyQyUyMDElNUQlMEFtYXNrX3Rva2VuX2xvZ2l0cyUyMCUzRCUyMHRva2VuX2xvZ2l0cyU1QjAlMkMlMjBtYXNrX3Rva2VuX2luZGV4JTJDJTIwJTNBJTVEJTBBJTIzJTIwUGljayUyMHRoZSUyMCU1Qk1BU0slNUQlMjBjYW5kaWRhdGVzJTIwd2l0aCUyMHRoZSUyMGhpZ2hlc3QlMjBsb2dpdHMlMEElMjMlMjBXZSUyMG5lZ2F0ZSUyMHRoZSUyMGFycmF5JTIwYmVmb3JlJTIwYXJnc29ydCUyMHRvJTIwZ2V0JTIwdGhlJTIwbGFyZ2VzdCUyQyUyMG5vdCUyMHRoZSUyMHNtYWxsZXN0JTJDJTIwbG9naXRzJTBBdG9wXzVfdG9rZW5zJTIwJTNEJTIwbnAuYXJnc29ydCgtbWFza190b2tlbl9sb2dpdHMpJTVCJTNBNSU1RC50b2xpc3QoKSUwQSUwQWZvciUyMHRva2VuJTIwaW4lMjB0b3BfNV90b2tlbnMlM0ElMEElMjAlMjAlMjAlMjBwcmludChmJTIyJTNFJTNFJTNFJTIwJTdCdGV4dC5yZXBsYWNlKHRva2VuaXplci5tYXNrX3Rva2VuJTJDJTIwdG9rZW5pemVyLmRlY29kZSglNUJ0b2tlbiU1RCkpJTdEJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)
token_logits = model(**inputs).logits
<span class="hljs-comment"># Find the location of [MASK] and extract its logits</span>
mask_token_index = np.argwhere(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>] == tokenizer.mask_token_id)[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
mask_token_logits = token_logits[<span class="hljs-number">0</span>, mask_token_index, :]
<span class="hljs-comment"># Pick the [MASK] candidates with the highest logits</span>
<span class="hljs-comment"># We negate the array before argsort to get the largest, not the smallest, logits</span>
top_5_tokens = np.argsort(-mask_token_logits)[:<span class="hljs-number">5</span>].tolist()

<span class="hljs-keyword">for</span> token <span class="hljs-keyword">in</span> top_5_tokens:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; <span class="hljs-subst">{text.replace(tokenizer.mask_token, tokenizer.decode([token]))}</span>&quot;</span>)`,wrap:!1}}),{c(){d(a.$$.fragment)},l(n){h(a.$$.fragment,n)},m(n,y){w(a,n,y),c=!0},i(n){c||(r(a.$$.fragment,n),c=!0)},o(n){p(a.$$.fragment,n),c=!1},d(n){m(a,n)}}}function Ji(I){let a,c;return a=new f({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBdG9rZW5fbG9naXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpLmxvZ2l0cyUwQSUyMyUyMEZpbmQlMjB0aGUlMjBsb2NhdGlvbiUyMG9mJTIwJTVCTUFTSyU1RCUyMGFuZCUyMGV4dHJhY3QlMjBpdHMlMjBsb2dpdHMlMEFtYXNrX3Rva2VuX2luZGV4JTIwJTNEJTIwdG9yY2gud2hlcmUoaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTIwJTNEJTNEJTIwdG9rZW5pemVyLm1hc2tfdG9rZW5faWQpJTVCMSU1RCUwQW1hc2tfdG9rZW5fbG9naXRzJTIwJTNEJTIwdG9rZW5fbG9naXRzJTVCMCUyQyUyMG1hc2tfdG9rZW5faW5kZXglMkMlMjAlM0ElNUQlMEElMjMlMjBQaWNrJTIwdGhlJTIwJTVCTUFTSyU1RCUyMGNhbmRpZGF0ZXMlMjB3aXRoJTIwdGhlJTIwaGlnaGVzdCUyMGxvZ2l0cyUwQXRvcF81X3Rva2VucyUyMCUzRCUyMHRvcmNoLnRvcGsobWFza190b2tlbl9sb2dpdHMlMkMlMjA1JTJDJTIwZGltJTNEMSkuaW5kaWNlcyU1QjAlNUQudG9saXN0KCklMEElMEFmb3IlMjB0b2tlbiUyMGluJTIwdG9wXzVfdG9rZW5zJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiclM0UlM0UlM0UlMjAlN0J0ZXh0LnJlcGxhY2UodG9rZW5pemVyLm1hc2tfdG9rZW4lMkMlMjB0b2tlbml6ZXIuZGVjb2RlKCU1QnRva2VuJTVEKSklN0QnJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> torch

inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
token_logits = model(**inputs).logits
<span class="hljs-comment"># Find the location of [MASK] and extract its logits</span>
mask_token_index = torch.where(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>] == tokenizer.mask_token_id)[<span class="hljs-number">1</span>]
mask_token_logits = token_logits[<span class="hljs-number">0</span>, mask_token_index, :]
<span class="hljs-comment"># Pick the [MASK] candidates with the highest logits</span>
top_5_tokens = torch.topk(mask_token_logits, <span class="hljs-number">5</span>, dim=<span class="hljs-number">1</span>).indices[<span class="hljs-number">0</span>].tolist()

<span class="hljs-keyword">for</span> token <span class="hljs-keyword">in</span> top_5_tokens:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; <span class="hljs-subst">{text.replace(tokenizer.mask_token, tokenizer.decode([token]))}</span>&#x27;&quot;</span>)`,wrap:!1}}),{c(){d(a.$$.fragment)},l(n){h(a.$$.fragment,n)},m(n,y){w(a,n,y),c=!0},i(n){c||(r(a.$$.fragment,n),c=!0)},o(n){p(a.$$.fragment,n),c=!1},d(n){m(a,n)}}}function ui(I){let a,c="✏️ <strong>Try it out!</strong> Create a random sample of the <code>unsupervised</code> split and verify that the labels are neither <code>0</code> nor <code>1</code>. While you’re at it, you could also check that the labels in the <code>train</code> and <code>test</code> splits are indeed <code>0</code> or <code>1</code> — this is a useful sanity check that every NLP practitioner should perform at the start of a new project!";return{c(){a=J("p"),a.innerHTML=c},l(n){a=u(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-fd14wo"&&(a.innerHTML=c)},m(n,y){s(n,a,y)},p:Se,d(n){n&&t(a)}}}function Ui(I){let a,c='✏️ <strong>Try it out!</strong> Some Transformer models, like <a href="https://huggingface.co/google/bigbird-roberta-base" rel="nofollow">BigBird</a> and <a href="hf.co/allenai/longformer-base-4096">Longformer</a>, have a much longer context length than BERT and other early Transformer models. Instantiate the tokenizer for one of these checkpoints and verify that the <code>model_max_length</code> agrees with what’s quoted on its model card.';return{c(){a=J("p"),a.innerHTML=c},l(n){a=u(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-snuco1"&&(a.innerHTML=c)},m(n,y){s(n,a,y)},p:Se,d(n){n&&t(a)}}}function Ti(I){let a,c="Note that using a small chunk size can be detrimental in real-world scenarios, so you should use a size that corresponds to the use case you will apply your model to.";return{c(){a=J("p"),a.textContent=c},l(n){a=u(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-1w9d6if"&&(a.textContent=c)},m(n,y){s(n,a,y)},p:Se,d(n){n&&t(a)}}}function ji(I){let a,c="✏️ <strong>Try it out!</strong> Run the code snippet above several times to see the random masking happen in front of your very eyes! Also replace the <code>tokenizer.decode()</code> method with <code>tokenizer.convert_ids_to_tokens()</code> to see that sometimes a single token from a given word is masked, and not the others.";return{c(){a=J("p"),a.innerHTML=c},l(n){a=u(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-1f3fue2"&&(a.innerHTML=c)},m(n,y){s(n,a,y)},p:Se,d(n){n&&t(a)}}}function On(I){let a,c="One side effect of random masking is that our evaluation metrics will not be deterministic when using the <code>Trainer</code>, since we use the same data collator for the training and test sets. We’ll see later, when we look at fine-tuning with 🤗 Accelerate, how we can use the flexibility of a custom evaluation loop to freeze the randomness.";return{c(){a=J("p"),a.innerHTML=c},l(n){a=u(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-w1yods"&&(a.innerHTML=c)},m(n,y){s(n,a,y)},d(n){n&&t(a)}}}function fi(I){let a,c;return a=new f({props:{code:"aW1wb3J0JTIwY29sbGVjdGlvbnMlMEFpbXBvcnQlMjBudW1weSUyMGFzJTIwbnAlMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzLmRhdGEuZGF0YV9jb2xsYXRvciUyMGltcG9ydCUyMHRmX2RlZmF1bHRfZGF0YV9jb2xsYXRvciUwQSUwQXd3bV9wcm9iYWJpbGl0eSUyMCUzRCUyMDAuMiUwQSUwQSUwQWRlZiUyMHdob2xlX3dvcmRfbWFza2luZ19kYXRhX2NvbGxhdG9yKGZlYXR1cmVzKSUzQSUwQSUyMCUyMCUyMCUyMGZvciUyMGZlYXR1cmUlMjBpbiUyMGZlYXR1cmVzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd29yZF9pZHMlMjAlM0QlMjBmZWF0dXJlLnBvcCglMjJ3b3JkX2lkcyUyMiklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBDcmVhdGUlMjBhJTIwbWFwJTIwYmV0d2VlbiUyMHdvcmRzJTIwYW5kJTIwY29ycmVzcG9uZGluZyUyMHRva2VuJTIwaW5kaWNlcyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hcHBpbmclMjAlM0QlMjBjb2xsZWN0aW9ucy5kZWZhdWx0ZGljdChsaXN0KSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGN1cnJlbnRfd29yZF9pbmRleCUyMCUzRCUyMC0xJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY3VycmVudF93b3JkJTIwJTNEJTIwTm9uZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMGlkeCUyQyUyMHdvcmRfaWQlMjBpbiUyMGVudW1lcmF0ZSh3b3JkX2lkcyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHdvcmRfaWQlMjBpcyUyMG5vdCUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHdvcmRfaWQlMjAhJTNEJTIwY3VycmVudF93b3JkJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY3VycmVudF93b3JkJTIwJTNEJTIwd29yZF9pZCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGN1cnJlbnRfd29yZF9pbmRleCUyMCUyQiUzRCUyMDElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXBwaW5nJTVCY3VycmVudF93b3JkX2luZGV4JTVELmFwcGVuZChpZHgpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwUmFuZG9tbHklMjBtYXNrJTIwd29yZHMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXNrJTIwJTNEJTIwbnAucmFuZG9tLmJpbm9taWFsKDElMkMlMjB3d21fcHJvYmFiaWxpdHklMkMlMjAobGVuKG1hcHBpbmcpJTJDKSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbnB1dF9pZHMlMjAlM0QlMjBmZWF0dXJlJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFiZWxzJTIwJTNEJTIwZmVhdHVyZSU1QiUyMmxhYmVscyUyMiU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG5ld19sYWJlbHMlMjAlM0QlMjAlNUItMTAwJTVEJTIwKiUyMGxlbihsYWJlbHMpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwd29yZF9pZCUyMGluJTIwbnAud2hlcmUobWFzayklNUIwJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd29yZF9pZCUyMCUzRCUyMHdvcmRfaWQuaXRlbSgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwaWR4JTIwaW4lMjBtYXBwaW5nJTVCd29yZF9pZCU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG5ld19sYWJlbHMlNUJpZHglNUQlMjAlM0QlMjBsYWJlbHMlNUJpZHglNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbnB1dF9pZHMlNUJpZHglNUQlMjAlM0QlMjB0b2tlbml6ZXIubWFza190b2tlbl9pZCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmUlNUIlMjJsYWJlbHMlMjIlNUQlMjAlM0QlMjBuZXdfbGFiZWxzJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwdGZfZGVmYXVsdF9kYXRhX2NvbGxhdG9yKGZlYXR1cmVzKQ==",highlighted:`<span class="hljs-keyword">import</span> collections
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-keyword">from</span> transformers.data.data_collator <span class="hljs-keyword">import</span> tf_default_data_collator

wwm_probability = <span class="hljs-number">0.2</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">whole_word_masking_data_collator</span>(<span class="hljs-params">features</span>):
    <span class="hljs-keyword">for</span> feature <span class="hljs-keyword">in</span> features:
        word_ids = feature.pop(<span class="hljs-string">&quot;word_ids&quot;</span>)

        <span class="hljs-comment"># Create a map between words and corresponding token indices</span>
        mapping = collections.defaultdict(<span class="hljs-built_in">list</span>)
        current_word_index = -<span class="hljs-number">1</span>
        current_word = <span class="hljs-literal">None</span>
        <span class="hljs-keyword">for</span> idx, word_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word_ids):
            <span class="hljs-keyword">if</span> word_id <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                <span class="hljs-keyword">if</span> word_id != current_word:
                    current_word = word_id
                    current_word_index += <span class="hljs-number">1</span>
                mapping[current_word_index].append(idx)

        <span class="hljs-comment"># Randomly mask words</span>
        mask = np.random.binomial(<span class="hljs-number">1</span>, wwm_probability, (<span class="hljs-built_in">len</span>(mapping),))
        input_ids = feature[<span class="hljs-string">&quot;input_ids&quot;</span>]
        labels = feature[<span class="hljs-string">&quot;labels&quot;</span>]
        new_labels = [-<span class="hljs-number">100</span>] * <span class="hljs-built_in">len</span>(labels)
        <span class="hljs-keyword">for</span> word_id <span class="hljs-keyword">in</span> np.where(mask)[<span class="hljs-number">0</span>]:
            word_id = word_id.item()
            <span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> mapping[word_id]:
                new_labels[idx] = labels[idx]
                input_ids[idx] = tokenizer.mask_token_id
        feature[<span class="hljs-string">&quot;labels&quot;</span>] = new_labels

    <span class="hljs-keyword">return</span> tf_default_data_collator(features)`,wrap:!1}}),{c(){d(a.$$.fragment)},l(n){h(a.$$.fragment,n)},m(n,y){w(a,n,y),c=!0},i(n){c||(r(a.$$.fragment,n),c=!0)},o(n){p(a.$$.fragment,n),c=!1},d(n){m(a,n)}}}function bi(I){let a,c;return a=new f({props:{code:"aW1wb3J0JTIwY29sbGVjdGlvbnMlMEFpbXBvcnQlMjBudW1weSUyMGFzJTIwbnAlMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwZGVmYXVsdF9kYXRhX2NvbGxhdG9yJTBBJTBBd3dtX3Byb2JhYmlsaXR5JTIwJTNEJTIwMC4yJTBBJTBBJTBBZGVmJTIwd2hvbGVfd29yZF9tYXNraW5nX2RhdGFfY29sbGF0b3IoZmVhdHVyZXMpJTNBJTBBJTIwJTIwJTIwJTIwZm9yJTIwZmVhdHVyZSUyMGluJTIwZmVhdHVyZXMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkcyUyMCUzRCUyMGZlYXR1cmUucG9wKCUyMndvcmRfaWRzJTIyKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMENyZWF0ZSUyMGElMjBtYXAlMjBiZXR3ZWVuJTIwd29yZHMlMjBhbmQlMjBjb3JyZXNwb25kaW5nJTIwdG9rZW4lMjBpbmRpY2VzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWFwcGluZyUyMCUzRCUyMGNvbGxlY3Rpb25zLmRlZmF1bHRkaWN0KGxpc3QpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY3VycmVudF93b3JkX2luZGV4JTIwJTNEJTIwLTElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjdXJyZW50X3dvcmQlMjAlM0QlMjBOb25lJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwaWR4JTJDJTIwd29yZF9pZCUyMGluJTIwZW51bWVyYXRlKHdvcmRfaWRzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZCUyMGlzJTIwbm90JTIwTm9uZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZCUyMCElM0QlMjBjdXJyZW50X3dvcmQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjdXJyZW50X3dvcmQlMjAlM0QlMjB3b3JkX2lkJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY3VycmVudF93b3JkX2luZGV4JTIwJTJCJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hcHBpbmclNUJjdXJyZW50X3dvcmRfaW5kZXglNUQuYXBwZW5kKGlkeCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBSYW5kb21seSUyMG1hc2slMjB3b3JkcyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hc2slMjAlM0QlMjBucC5yYW5kb20uYmlub21pYWwoMSUyQyUyMHd3bV9wcm9iYWJpbGl0eSUyQyUyMChsZW4obWFwcGluZyklMkMpKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0X2lkcyUyMCUzRCUyMGZlYXR1cmUlNUIlMjJpbnB1dF9pZHMlMjIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbHMlMjAlM0QlMjBmZWF0dXJlJTVCJTIybGFiZWxzJTIyJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbmV3X2xhYmVscyUyMCUzRCUyMCU1Qi0xMDAlNUQlMjAqJTIwbGVuKGxhYmVscyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjB3b3JkX2lkJTIwaW4lMjBucC53aGVyZShtYXNrKSU1QjAlNUQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkJTIwJTNEJTIwd29yZF9pZC5pdGVtKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBpZHglMjBpbiUyMG1hcHBpbmclNUJ3b3JkX2lkJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbmV3X2xhYmVscyU1QmlkeCU1RCUyMCUzRCUyMGxhYmVscyU1QmlkeCU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0X2lkcyU1QmlkeCU1RCUyMCUzRCUyMHRva2VuaXplci5tYXNrX3Rva2VuX2lkJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmVhdHVyZSU1QiUyMmxhYmVscyUyMiU1RCUyMCUzRCUyMG5ld19sYWJlbHMlMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBkZWZhdWx0X2RhdGFfY29sbGF0b3IoZmVhdHVyZXMp",highlighted:`<span class="hljs-keyword">import</span> collections
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> default_data_collator

wwm_probability = <span class="hljs-number">0.2</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">whole_word_masking_data_collator</span>(<span class="hljs-params">features</span>):
    <span class="hljs-keyword">for</span> feature <span class="hljs-keyword">in</span> features:
        word_ids = feature.pop(<span class="hljs-string">&quot;word_ids&quot;</span>)

        <span class="hljs-comment"># Create a map between words and corresponding token indices</span>
        mapping = collections.defaultdict(<span class="hljs-built_in">list</span>)
        current_word_index = -<span class="hljs-number">1</span>
        current_word = <span class="hljs-literal">None</span>
        <span class="hljs-keyword">for</span> idx, word_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word_ids):
            <span class="hljs-keyword">if</span> word_id <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                <span class="hljs-keyword">if</span> word_id != current_word:
                    current_word = word_id
                    current_word_index += <span class="hljs-number">1</span>
                mapping[current_word_index].append(idx)

        <span class="hljs-comment"># Randomly mask words</span>
        mask = np.random.binomial(<span class="hljs-number">1</span>, wwm_probability, (<span class="hljs-built_in">len</span>(mapping),))
        input_ids = feature[<span class="hljs-string">&quot;input_ids&quot;</span>]
        labels = feature[<span class="hljs-string">&quot;labels&quot;</span>]
        new_labels = [-<span class="hljs-number">100</span>] * <span class="hljs-built_in">len</span>(labels)
        <span class="hljs-keyword">for</span> word_id <span class="hljs-keyword">in</span> np.where(mask)[<span class="hljs-number">0</span>]:
            word_id = word_id.item()
            <span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> mapping[word_id]:
                new_labels[idx] = labels[idx]
                input_ids[idx] = tokenizer.mask_token_id
        feature[<span class="hljs-string">&quot;labels&quot;</span>] = new_labels

    <span class="hljs-keyword">return</span> default_data_collator(features)`,wrap:!1}}),{c(){d(a.$$.fragment)},l(n){h(a.$$.fragment,n)},m(n,y){w(a,n,y),c=!0},i(n){c||(r(a.$$.fragment,n),c=!0)},o(n){p(a.$$.fragment,n),c=!1},d(n){m(a,n)}}}function Ii(I){let a,c="✏️ <strong>Try it out!</strong> Run the code snippet above several times to see the random masking happen in front of your very eyes! Also replace the <code>tokenizer.decode()</code> method with <code>tokenizer.convert_ids_to_tokens()</code> to see that the tokens from a given word are always masked together.";return{c(){a=J("p"),a.innerHTML=c},l(n){a=u(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-wmwcem"&&(a.innerHTML=c)},m(n,y){s(n,a,y)},p:Se,d(n){n&&t(a)}}}function gi(I){let a,c="Once we’re logged in, we can specify the arguments for the <code>Trainer</code>:",n,y,_,U,R="Here we tweaked a few of the default options, including <code>logging_steps</code> to ensure we track the training loss with each epoch. We’ve also used <code>fp16=True</code> to enable mixed-precision training, which gives us another boost in speed. By default, the <code>Trainer</code> will remove any columns that are not part of the model’s <code>forward()</code> method. This means that if you’re using the whole word masking collator, you’ll also need to set <code>remove_unused_columns=False</code> to ensure we don’t lose the <code>word_ids</code> column during training.",W,C,X='Note that you can specify the name of the repository you want to push to with the <code>hub_model_id</code> argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the <a href="https://huggingface.co/huggingface-course" rel="nofollow"><code>huggingface-course</code> organization</a>, we added <code>hub_model_id=&quot;huggingface-course/distilbert-finetuned-imdb&quot;</code> to <code>TrainingArguments</code>. By default, the repository used will be in your namespace and named after the output directory you set, so in our case it will be <code>&quot;lewtun/distilbert-finetuned-imdb&quot;</code>.',Z,B,j="We now have all the ingredients to instantiate the <code>Trainer</code>. Here we just use the standard <code>data_collator</code>, but you can try the whole word masking collator and compare the results as an exercise:",G,v,Q,g,$="We’re now ready to run <code>trainer.train()</code> — but before doing so let’s briefly look at <em>perplexity</em>, which is a common metric to evaluate the performance of language models.",S;return y=new f({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTBBJTBBYmF0Y2hfc2l6ZSUyMCUzRCUyMDY0JTBBJTIzJTIwU2hvdyUyMHRoZSUyMHRyYWluaW5nJTIwbG9zcyUyMHdpdGglMjBldmVyeSUyMGVwb2NoJTBBbG9nZ2luZ19zdGVwcyUyMCUzRCUyMGxlbihkb3duc2FtcGxlZF9kYXRhc2V0JTVCJTIydHJhaW4lMjIlNUQpJTIwJTJGJTJGJTIwYmF0Y2hfc2l6ZSUwQW1vZGVsX25hbWUlMjAlM0QlMjBtb2RlbF9jaGVja3BvaW50LnNwbGl0KCUyMiUyRiUyMiklNUItMSU1RCUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEZiUyMiU3Qm1vZGVsX25hbWUlN0QtZmluZXR1bmVkLWltZGIlMjIlMkMlMEElMjAlMjAlMjAlMjBvdmVyd3JpdGVfb3V0cHV0X2RpciUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBldmFsdWF0aW9uX3N0cmF0ZWd5JTNEJTIyZXBvY2glMjIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMmUtNSUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRDAuMDElMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0RiYXRjaF9zaXplJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0RiYXRjaF9zaXplJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwZnAxNiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNEbG9nZ2luZ19zdGVwcyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

batch_size = <span class="hljs-number">64</span>
<span class="hljs-comment"># Show the training loss with every epoch</span>
logging_steps = <span class="hljs-built_in">len</span>(downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size
model_name = model_checkpoint.split(<span class="hljs-string">&quot;/&quot;</span>)[-<span class="hljs-number">1</span>]

training_args = TrainingArguments(
    output_dir=<span class="hljs-string">f&quot;<span class="hljs-subst">{model_name}</span>-finetuned-imdb&quot;</span>,
    overwrite_output_dir=<span class="hljs-literal">True</span>,
    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    push_to_hub=<span class="hljs-literal">True</span>,
    fp16=<span class="hljs-literal">True</span>,
    logging_steps=logging_steps,
)`,wrap:!1}}),v=new f({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZG93bnNhbXBsZWRfZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEZG93bnNhbXBsZWRfZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2NvbGxhdG9yJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>],
    data_collator=data_collator,
    tokenizer=tokenizer,
)`,wrap:!1}}),{c(){a=J("p"),a.innerHTML=c,n=i(),d(y.$$.fragment),_=i(),U=J("p"),U.innerHTML=R,W=i(),C=J("p"),C.innerHTML=X,Z=i(),B=J("p"),B.innerHTML=j,G=i(),d(v.$$.fragment),Q=i(),g=J("p"),g.innerHTML=$},l(k){a=u(k,"P",{"data-svelte-h":!0}),T(a)!=="svelte-jgpv5a"&&(a.innerHTML=c),n=o(k),h(y.$$.fragment,k),_=o(k),U=u(k,"P",{"data-svelte-h":!0}),T(U)!=="svelte-zfa7rd"&&(U.innerHTML=R),W=o(k),C=u(k,"P",{"data-svelte-h":!0}),T(C)!=="svelte-1ot1vnd"&&(C.innerHTML=X),Z=o(k),B=u(k,"P",{"data-svelte-h":!0}),T(B)!=="svelte-47mng"&&(B.innerHTML=j),G=o(k),h(v.$$.fragment,k),Q=o(k),g=u(k,"P",{"data-svelte-h":!0}),T(g)!=="svelte-c9apqh"&&(g.innerHTML=$)},m(k,V){s(k,a,V),s(k,n,V),w(y,k,V),s(k,_,V),s(k,U,V),s(k,W,V),s(k,C,V),s(k,Z,V),s(k,B,V),s(k,G,V),w(v,k,V),s(k,Q,V),s(k,g,V),S=!0},i(k){S||(r(y.$$.fragment,k),r(v.$$.fragment,k),S=!0)},o(k){p(y.$$.fragment,k),p(v.$$.fragment,k),S=!1},d(k){k&&(t(a),t(n),t(_),t(U),t(W),t(C),t(Z),t(B),t(G),t(Q),t(g)),m(y,k),m(v,k)}}}function Ci(I){let a,c="Once we’re logged in, we can create our <code>tf.data</code> datasets. To do so, we’ll use the <code>prepare_tf_dataset()</code> method, which uses our model to automatically infer which columns should go into the dataset. If you want to control exactly which columns to use, you can use the <code>Dataset.to_tf_dataset()</code> method instead. To keep things simple, we’ll just use the standard data collator here, but you can also try the whole word masking collator and compare the results as an exercise:",n,y,_,U,R="Next, we set up our training hyperparameters and compile our model. We use the <code>create_optimizer()</code> function from the 🤗 Transformers library, which gives us an <code>AdamW</code> optimizer with linear learning rate decay. We also use the model’s built-in loss, which is the default when no loss is specified as an argument to <code>compile()</code>, and we set the training precision to <code>&quot;mixed_float16&quot;</code>. Note that if you’re using a Colab GPU or other GPU that does not have accelerated float16 support, you should probably comment out that line.",W,C,X='In addition, we set up a <code>PushToHubCallback</code> that will save the model to the Hub after each epoch. You can specify the name of the repository you want to push to with the <code>hub_model_id</code> argument (in particular, you will have to use this argument to push to an organization). For instance, to push the model to the <a href="https://huggingface.co/huggingface-course" rel="nofollow"><code>huggingface-course</code> organization</a>, we added <code>hub_model_id=&quot;huggingface-course/distilbert-finetuned-imdb&quot;</code>. By default, the repository used will be in your namespace and named after the output directory you set, so in our case it will be <code>&quot;lewtun/distilbert-finetuned-imdb&quot;</code>.',Z,B,j,G,v="We’re now ready to run <code>model.fit()</code> — but before doing so let’s briefly look at <em>perplexity</em>, which is a common metric to evaluate the performance of language models.",Q;return y=new f({props:{code:"dGZfdHJhaW5fZGF0YXNldCUyMCUzRCUyMG1vZGVsLnByZXBhcmVfdGZfZGF0YXNldCglMEElMjAlMjAlMjAlMjBkb3duc2FtcGxlZF9kYXRhc2V0JTVCJTIydHJhaW4lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBjb2xsYXRlX2ZuJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDMyJTJDJTBBKSUwQSUwQXRmX2V2YWxfZGF0YXNldCUyMCUzRCUyMG1vZGVsLnByZXBhcmVfdGZfZGF0YXNldCglMEElMjAlMjAlMjAlMjBkb3duc2FtcGxlZF9kYXRhc2V0JTVCJTIydGVzdCUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBJTIwJTIwJTIwJTIwc2h1ZmZsZSUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDMyJTJDJTBBKQ==",highlighted:`tf_train_dataset = model.prepare_tf_dataset(
    downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">32</span>,
)

tf_eval_dataset = model.prepare_tf_dataset(
    downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">32</span>,
)`,wrap:!1}}),B=new f({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEFmcm9tJTIwdHJhbnNmb3JtZXJzLmtlcmFzX2NhbGxiYWNrcyUyMGltcG9ydCUyMFB1c2hUb0h1YkNhbGxiYWNrJTBBaW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFudW1fdHJhaW5fc3RlcHMlMjAlM0QlMjBsZW4odGZfdHJhaW5fZGF0YXNldCklMEFvcHRpbWl6ZXIlMkMlMjBzY2hlZHVsZSUyMCUzRCUyMGNyZWF0ZV9vcHRpbWl6ZXIoJTBBJTIwJTIwJTIwJTIwaW5pdF9sciUzRDJlLTUlMkMlMEElMjAlMjAlMjAlMjBudW1fd2FybXVwX3N0ZXBzJTNEMV8wMDAlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fc3RlcHMlM0RudW1fdHJhaW5fc3RlcHMlMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRfZGVjYXlfcmF0ZSUzRDAuMDElMkMlMEEpJTBBbW9kZWwuY29tcGlsZShvcHRpbWl6ZXIlM0RvcHRpbWl6ZXIpJTBBJTBBJTIzJTIwVHJhaW4lMjBpbiUyMG1peGVkLXByZWNpc2lvbiUyMGZsb2F0MTYlMEF0Zi5rZXJhcy5taXhlZF9wcmVjaXNpb24uc2V0X2dsb2JhbF9wb2xpY3koJTIybWl4ZWRfZmxvYXQxNiUyMiklMEElMEFtb2RlbF9uYW1lJTIwJTNEJTIwbW9kZWxfY2hlY2twb2ludC5zcGxpdCglMjIlMkYlMjIpJTVCLTElNUQlMEFjYWxsYmFjayUyMCUzRCUyMFB1c2hUb0h1YkNhbGxiYWNrKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0RmJTIyJTdCbW9kZWxfbmFtZSU3RC1maW5ldHVuZWQtaW1kYiUyMiUyQyUyMHRva2VuaXplciUzRHRva2VuaXplciUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer
<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

num_train_steps = <span class="hljs-built_in">len</span>(tf_train_dataset)
optimizer, schedule = create_optimizer(
    init_lr=<span class="hljs-number">2e-5</span>,
    num_warmup_steps=<span class="hljs-number">1_000</span>,
    num_train_steps=num_train_steps,
    weight_decay_rate=<span class="hljs-number">0.01</span>,
)
model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)

<span class="hljs-comment"># Train in mixed-precision float16</span>
tf.keras.mixed_precision.set_global_policy(<span class="hljs-string">&quot;mixed_float16&quot;</span>)

model_name = model_checkpoint.split(<span class="hljs-string">&quot;/&quot;</span>)[-<span class="hljs-number">1</span>]
callback = PushToHubCallback(
    output_dir=<span class="hljs-string">f&quot;<span class="hljs-subst">{model_name}</span>-finetuned-imdb&quot;</span>, tokenizer=tokenizer
)`,wrap:!1}}),{c(){a=J("p"),a.innerHTML=c,n=i(),d(y.$$.fragment),_=i(),U=J("p"),U.innerHTML=R,W=i(),C=J("p"),C.innerHTML=X,Z=i(),d(B.$$.fragment),j=i(),G=J("p"),G.innerHTML=v},l(g){a=u(g,"P",{"data-svelte-h":!0}),T(a)!=="svelte-smif6r"&&(a.innerHTML=c),n=o(g),h(y.$$.fragment,g),_=o(g),U=u(g,"P",{"data-svelte-h":!0}),T(U)!=="svelte-1mjs8eb"&&(U.innerHTML=R),W=o(g),C=u(g,"P",{"data-svelte-h":!0}),T(C)!=="svelte-lexz8a"&&(C.innerHTML=X),Z=o(g),h(B.$$.fragment,g),j=o(g),G=u(g,"P",{"data-svelte-h":!0}),T(G)!=="svelte-9vk4aq"&&(G.innerHTML=v)},m(g,$){s(g,a,$),s(g,n,$),w(y,g,$),s(g,_,$),s(g,U,$),s(g,W,$),s(g,C,$),s(g,Z,$),w(B,g,$),s(g,j,$),s(g,G,$),Q=!0},i(g){Q||(r(y.$$.fragment,g),r(B.$$.fragment,g),Q=!0)},o(g){p(y.$$.fragment,g),p(B.$$.fragment,g),Q=!1},d(g){g&&(t(a),t(n),t(_),t(U),t(W),t(C),t(Z),t(j),t(G)),m(y,g),m(B,g)}}}function _i(I){let a,c="Assuming our test set consists mostly of sentences that are grammatically correct, then one way to measure the quality of our language model is to calculate the probabilities it assigns to the next word in all the sentences of the test set. High probabilities indicates that the model indicates that the model is not “surprised” or “perplexed” by the unseen examples, and suggests it has learned the basic patterns of grammar in the language. There are various mathematical definitions of perplexity, but the one we’ll use defines it as the exponential of the cross-entropy loss. Thus, we can calculate the perplexity of our pretrained model by using the <code>model.evaluate()</code> method to compute the cross-entropy loss on the test set and then taking the exponential of the result:",n,y,_;return y=new f({props:{code:"aW1wb3J0JTIwbWF0aCUwQSUwQWV2YWxfbG9zcyUyMCUzRCUyMG1vZGVsLmV2YWx1YXRlKHRmX2V2YWxfZGF0YXNldCklMEFwcmludChmJTIyUGVycGxleGl0eSUzQSUyMCU3Qm1hdGguZXhwKGV2YWxfbG9zcyklM0EuMmYlN0QlMjIp",highlighted:`<span class="hljs-keyword">import</span> math

eval_loss = model.evaluate(tf_eval_dataset)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Perplexity: <span class="hljs-subst">{math.exp(eval_loss):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,wrap:!1}}),{c(){a=J("p"),a.innerHTML=c,n=i(),d(y.$$.fragment)},l(U){a=u(U,"P",{"data-svelte-h":!0}),T(a)!=="svelte-xzdc5c"&&(a.innerHTML=c),n=o(U),h(y.$$.fragment,U)},m(U,R){s(U,a,R),s(U,n,R),w(y,U,R),_=!0},i(U){_||(r(y.$$.fragment,U),_=!0)},o(U){p(y.$$.fragment,U),_=!1},d(U){U&&(t(a),t(n)),m(y,U)}}}function Gi(I){let a,c="Assuming our test set consists mostly of sentences that are grammatically correct, then one way to measure the quality of our language model is to calculate the probabilities it assigns to the next word in all the sentences of the test set. High probabilities indicates that the model is not “surprised” or “perplexed” by the unseen examples, and suggests it has learned the basic patterns of grammar in the language. There are various mathematical definitions of perplexity, but the one we’ll use defines it as the exponential of the cross-entropy loss. Thus, we can calculate the perplexity of our pretrained model by using the <code>Trainer.evaluate()</code> function to compute the cross-entropy loss on the test set and then taking the exponential of the result:",n,y,_;return y=new f({props:{code:"aW1wb3J0JTIwbWF0aCUwQSUwQWV2YWxfcmVzdWx0cyUyMCUzRCUyMHRyYWluZXIuZXZhbHVhdGUoKSUwQXByaW50KGYlMjIlM0UlM0UlM0UlMjBQZXJwbGV4aXR5JTNBJTIwJTdCbWF0aC5leHAoZXZhbF9yZXN1bHRzJTVCJ2V2YWxfbG9zcyclNUQpJTNBLjJmJTdEJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> math

eval_results = trainer.evaluate()
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; Perplexity: <span class="hljs-subst">{math.exp(eval_results[<span class="hljs-string">&#x27;eval_loss&#x27;</span>]):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,wrap:!1}}),{c(){a=J("p"),a.innerHTML=c,n=i(),d(y.$$.fragment)},l(U){a=u(U,"P",{"data-svelte-h":!0}),T(a)!=="svelte-1nciy2w"&&(a.innerHTML=c),n=o(U),h(y.$$.fragment,U)},m(U,R){s(U,a,R),s(U,n,R),w(y,U,R),_=!0},i(U){_||(r(y.$$.fragment,U),_=!0)},o(U){p(y.$$.fragment,U),_=!1},d(U){U&&(t(a),t(n)),m(y,U)}}}function ki(I){let a,c;return a=new f({props:{code:"bW9kZWwuZml0KHRmX3RyYWluX2RhdGFzZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl9ldmFsX2RhdGFzZXQlMkMlMjBjYWxsYmFja3MlM0QlNUJjYWxsYmFjayU1RCk=",highlighted:"model.fit(tf_train_dataset, validation_data=tf_eval_dataset, callbacks=[callback])",wrap:!1}}),{c(){d(a.$$.fragment)},l(n){h(a.$$.fragment,n)},m(n,y){w(a,n,y),c=!0},i(n){c||(r(a.$$.fragment,n),c=!0)},o(n){p(a.$$.fragment,n),c=!1},d(n){m(a,n)}}}function Bi(I){let a,c;return a=new f({props:{code:"dHJhaW5lci50cmFpbigp",highlighted:"trainer.train()",wrap:!1}}),{c(){d(a.$$.fragment)},l(n){h(a.$$.fragment,n)},m(n,y){w(a,n,y),c=!0},i(n){c||(r(a.$$.fragment,n),c=!0)},o(n){p(a.$$.fragment,n),c=!1},d(n){m(a,n)}}}function Zi(I){let a,c;return a=new f({props:{code:"ZXZhbF9sb3NzJTIwJTNEJTIwbW9kZWwuZXZhbHVhdGUodGZfZXZhbF9kYXRhc2V0KSUwQXByaW50KGYlMjJQZXJwbGV4aXR5JTNBJTIwJTdCbWF0aC5leHAoZXZhbF9sb3NzKSUzQS4yZiU3RCUyMik=",highlighted:`eval_loss = model.evaluate(tf_eval_dataset)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Perplexity: <span class="hljs-subst">{math.exp(eval_loss):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,wrap:!1}}),{c(){d(a.$$.fragment)},l(n){h(a.$$.fragment,n)},m(n,y){w(a,n,y),c=!0},i(n){c||(r(a.$$.fragment,n),c=!0)},o(n){p(a.$$.fragment,n),c=!1},d(n){m(a,n)}}}function Ri(I){let a,c;return a=new f({props:{code:"ZXZhbF9yZXN1bHRzJTIwJTNEJTIwdHJhaW5lci5ldmFsdWF0ZSgpJTBBcHJpbnQoZiUyMiUzRSUzRSUzRSUyMFBlcnBsZXhpdHklM0ElMjAlN0JtYXRoLmV4cChldmFsX3Jlc3VsdHMlNUInZXZhbF9sb3NzJyU1RCklM0EuMmYlN0QlMjIp",highlighted:`eval_results = trainer.evaluate()
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; Perplexity: <span class="hljs-subst">{math.exp(eval_results[<span class="hljs-string">&#x27;eval_loss&#x27;</span>]):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,wrap:!1}}),{c(){d(a.$$.fragment)},l(n){h(a.$$.fragment,n)},m(n,y){w(a,n,y),c=!0},i(n){c||(r(a.$$.fragment,n),c=!0)},o(n){p(a.$$.fragment,n),c=!1},d(n){m(a,n)}}}function ei(I){let a,c="Once training is finished, we can push the model card with the training information to the Hub (the checkpoints are saved during training itself):",n,y,_;return y=new f({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:"trainer.push_to_hub()",wrap:!1}}),{c(){a=J("p"),a.textContent=c,n=i(),d(y.$$.fragment)},l(U){a=u(U,"P",{"data-svelte-h":!0}),T(a)!=="svelte-13hhldr"&&(a.textContent=c),n=o(U),h(y.$$.fragment,U)},m(U,R){s(U,a,R),s(U,n,R),w(y,U,R),_=!0},i(U){_||(r(y.$$.fragment,U),_=!0)},o(U){p(y.$$.fragment,U),_=!1},d(U){U&&(t(a),t(n)),m(y,U)}}}function Xi(I){let a,c="✏️ <strong>Your turn!</strong> Run the training above after changing the data collator to the whole word masking collator. Do you get better results?";return{c(){a=J("p"),a.innerHTML=c},l(n){a=u(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-1jdek3j"&&(a.innerHTML=c)},m(n,y){s(n,a,y)},p:Se,d(n){n&&t(a)}}}function li(I){let a,c="In our use case we didn’t need to do anything special with the training loop, but in some cases you might need to implement some custom logic. For these applications, you can use 🤗 Accelerate — let’s take a look!",n,y,_,U,R='As we saw with the <code>Trainer</code>, fine-tuning a masked language model is very similar to the text classification example from <a href="/course/chapter3">Chapter 3</a>. In fact, the only subtlety is the use of a special data collator, and we’ve already covered that earlier in this section!',W,C,X="However, we saw that <code>DataCollatorForLanguageModeling</code> also applies random masking with each evaluation, so we’ll see some fluctuations in our perplexity scores with each training run. One way to eliminate this source of randomness is to apply the masking <em>once</em> on the whole test set, and then use the default data collator in 🤗 Transformers to collect the batches during evaluation. To see how this works, let’s implement a simple function that applies the masking on a batch, similar to our first encounter with <code>DataCollatorForLanguageModeling</code>:",Z,B,j,G,v="Next, we’ll apply this function to our test set and drop the unmasked columns so we can replace them with the masked ones. You can use whole word masking by replacing the <code>data_collator</code> above with the appropriate one, in which case you should remove the first line here:",Q,g,$,S,k="We can then set up the dataloaders as usual, but we’ll use the <code>default_data_collator</code> from 🤗 Transformers for the evaluation set:",V,be,Ge,H,Yt="Form here, we follow the standard steps with 🤗 Accelerate. The first order of business is to load a fresh version of the pretrained model:",ke,z,He,se,Ie="Then we need to specify the optimizer; we’ll use the standard <code>AdamW</code>:",Le,ae,ne,ie,pe="With these objects, we can now prepare everything for training with the <code>Accelerator</code> object:",Be,L,Ze,q,Ft="Now that our model, optimizer, and dataloaders are configured, we can specify the learning rate scheduler as follows:",Re,Y,qe,oe,ge="There is just one last thing to do before training: create a model repository on the Hugging Face Hub! We can use the 🤗 Hub library to first generate the full name of our repo:",De,Me,F,x,ye,D,Et="then create and clone the repository using the <code>Repository</code> class from 🤗 Hub:",Xe,K,We,P,St="With that done, it’s just a simple matter of writing out the full training and evaluation loop:",$e,O,Ve,ee,Ke,re,te="Cool, we’ve been able to evaluate perplexity with each epoch and ensure that multiple training runs are reproducible!",E;return y=new Ee({props:{title:"Fine-tuning DistilBERT with 🤗 Accelerate",local:"fine-tuning-distilbert-with-accelerate",headingTag:"h2"}}),B=new f({props:{code:"ZGVmJTIwaW5zZXJ0X3JhbmRvbV9tYXNrKGJhdGNoKSUzQSUwQSUyMCUyMCUyMCUyMGZlYXR1cmVzJTIwJTNEJTIwJTVCZGljdCh6aXAoYmF0Y2glMkMlMjB0KSklMjBmb3IlMjB0JTIwaW4lMjB6aXAoKmJhdGNoLnZhbHVlcygpKSU1RCUwQSUyMCUyMCUyMCUyMG1hc2tlZF9pbnB1dHMlMjAlM0QlMjBkYXRhX2NvbGxhdG9yKGZlYXR1cmVzKSUwQSUyMCUyMCUyMCUyMCUyMyUyMENyZWF0ZSUyMGElMjBuZXclMjAlMjJtYXNrZWQlMjIlMjBjb2x1bW4lMjBmb3IlMjBlYWNoJTIwY29sdW1uJTIwaW4lMjB0aGUlMjBkYXRhc2V0JTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTdCJTIybWFza2VkXyUyMiUyMCUyQiUyMGslM0ElMjB2Lm51bXB5KCklMjBmb3IlMjBrJTJDJTIwdiUyMGluJTIwbWFza2VkX2lucHV0cy5pdGVtcygpJTdE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">insert_random_mask</span>(<span class="hljs-params">batch</span>):
    features = [<span class="hljs-built_in">dict</span>(<span class="hljs-built_in">zip</span>(batch, t)) <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(*batch.values())]
    masked_inputs = data_collator(features)
    <span class="hljs-comment"># Create a new &quot;masked&quot; column for each column in the dataset</span>
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;masked_&quot;</span> + k: v.numpy() <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> masked_inputs.items()}`,wrap:!1}}),g=new f({props:{code:"ZG93bnNhbXBsZWRfZGF0YXNldCUyMCUzRCUyMGRvd25zYW1wbGVkX2RhdGFzZXQucmVtb3ZlX2NvbHVtbnMoJTVCJTIyd29yZF9pZHMlMjIlNUQpJTBBZXZhbF9kYXRhc2V0JTIwJTNEJTIwZG93bnNhbXBsZWRfZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQubWFwKCUwQSUyMCUyMCUyMCUyMGluc2VydF9yYW5kb21fbWFzayUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoZWQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcmVtb3ZlX2NvbHVtbnMlM0Rkb3duc2FtcGxlZF9kYXRhc2V0JTVCJTIydGVzdCUyMiU1RC5jb2x1bW5fbmFtZXMlMkMlMEEpJTBBZXZhbF9kYXRhc2V0JTIwJTNEJTIwZXZhbF9kYXRhc2V0LnJlbmFtZV9jb2x1bW5zKCUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1hc2tlZF9pbnB1dF9pZHMlMjIlM0ElMjAlMjJpbnB1dF9pZHMlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJtYXNrZWRfYXR0ZW50aW9uX21hc2slMjIlM0ElMjAlMjJhdHRlbnRpb25fbWFzayUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1hc2tlZF9sYWJlbHMlMjIlM0ElMjAlMjJsYWJlbHMlMjIlMkMlMEElMjAlMjAlMjAlMjAlN0QlMEEp",highlighted:`downsampled_dataset = downsampled_dataset.remove_columns([<span class="hljs-string">&quot;word_ids&quot;</span>])
eval_dataset = downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>].<span class="hljs-built_in">map</span>(
    insert_random_mask,
    batched=<span class="hljs-literal">True</span>,
    remove_columns=downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>].column_names,
)
eval_dataset = eval_dataset.rename_columns(
    {
        <span class="hljs-string">&quot;masked_input_ids&quot;</span>: <span class="hljs-string">&quot;input_ids&quot;</span>,
        <span class="hljs-string">&quot;masked_attention_mask&quot;</span>: <span class="hljs-string">&quot;attention_mask&quot;</span>,
        <span class="hljs-string">&quot;masked_labels&quot;</span>: <span class="hljs-string">&quot;labels&quot;</span>,
    }
)`,wrap:!1}}),be=new f({props:{code:"ZnJvbSUyMHRvcmNoLnV0aWxzLmRhdGElMjBpbXBvcnQlMjBEYXRhTG9hZGVyJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGRlZmF1bHRfZGF0YV9jb2xsYXRvciUwQSUwQWJhdGNoX3NpemUlMjAlM0QlMjA2NCUwQXRyYWluX2RhdGFsb2FkZXIlMjAlM0QlMjBEYXRhTG9hZGVyKCUwQSUyMCUyMCUyMCUyMGRvd25zYW1wbGVkX2RhdGFzZXQlNUIlMjJ0cmFpbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRGJhdGNoX3NpemUlMkMlMEElMjAlMjAlMjAlMjBjb2xsYXRlX2ZuJTNEZGF0YV9jb2xsYXRvciUyQyUwQSklMEFldmFsX2RhdGFsb2FkZXIlMjAlM0QlMjBEYXRhTG9hZGVyKCUwQSUyMCUyMCUyMCUyMGV2YWxfZGF0YXNldCUyQyUyMGJhdGNoX3NpemUlM0RiYXRjaF9zaXplJTJDJTIwY29sbGF0ZV9mbiUzRGRlZmF1bHRfZGF0YV9jb2xsYXRvciUwQSk=",highlighted:`<span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> default_data_collator

batch_size = <span class="hljs-number">64</span>
train_dataloader = DataLoader(
    downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=batch_size,
    collate_fn=data_collator,
)
eval_dataloader = DataLoader(
    eval_dataset, batch_size=batch_size, collate_fn=default_data_collator
)`,wrap:!1}}),z=new f({props:{code:"bW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JNYXNrZWRMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfY2hlY2twb2ludCk=",highlighted:'model = <span class="hljs-module-access"><span class="hljs-module"><span class="hljs-identifier">AutoModelForMaskedLM</span>.</span></span>from<span class="hljs-constructor">_pretrained(<span class="hljs-params">model_checkpoint</span>)</span>',wrap:!1}}),ae=new f({props:{code:"ZnJvbSUyMHRvcmNoLm9wdGltJTIwaW1wb3J0JTIwQWRhbVclMEElMEFvcHRpbWl6ZXIlMjAlM0QlMjBBZGFtVyhtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMjBsciUzRDVlLTUp",highlighted:`<span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW

optimizer = AdamW(model.parameters(), lr=<span class="hljs-number">5e-5</span>)`,wrap:!1}}),L=new f({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoKSUwQW1vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwdHJhaW5fZGF0YWxvYWRlciUyQyUyMGV2YWxfZGF0YWxvYWRlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUoJTBBJTIwJTIwJTIwJTIwbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjB0cmFpbl9kYXRhbG9hZGVyJTJDJTIwZXZhbF9kYXRhbG9hZGVyJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`,wrap:!1}}),Y=new f({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGdldF9zY2hlZHVsZXIlMEElMEFudW1fdHJhaW5fZXBvY2hzJTIwJTNEJTIwMyUwQW51bV91cGRhdGVfc3RlcHNfcGVyX2Vwb2NoJTIwJTNEJTIwbGVuKHRyYWluX2RhdGFsb2FkZXIpJTBBbnVtX3RyYWluaW5nX3N0ZXBzJTIwJTNEJTIwbnVtX3RyYWluX2Vwb2NocyUyMColMjBudW1fdXBkYXRlX3N0ZXBzX3Blcl9lcG9jaCUwQSUwQWxyX3NjaGVkdWxlciUyMCUzRCUyMGdldF9zY2hlZHVsZXIoJTBBJTIwJTIwJTIwJTIwJTIybGluZWFyJTIyJTJDJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyJTNEb3B0aW1pemVyJTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5pbmdfc3RlcHMlM0RudW1fdHJhaW5pbmdfc3RlcHMlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

num_train_epochs = <span class="hljs-number">3</span>
num_update_steps_per_epoch = <span class="hljs-built_in">len</span>(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    <span class="hljs-string">&quot;linear&quot;</span>,
    optimizer=optimizer,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_training_steps=num_training_steps,
)`,wrap:!1}}),Me=new f({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGdldF9mdWxsX3JlcG9fbmFtZSUwQSUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZC1maW5ldHVuZWQtaW1kYi1hY2NlbGVyYXRlJTIyJTBBcmVwb19uYW1lJTIwJTNEJTIwZ2V0X2Z1bGxfcmVwb19uYW1lKG1vZGVsX25hbWUpJTBBcmVwb19uYW1l",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> get_full_repo_name

model_name = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-imdb-accelerate&quot;</span>
repo_name = get_full_repo_name(model_name)
repo_name`,wrap:!1}}),x=new f({props:{code:"J2xld3R1biUyRmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkLWZpbmV0dW5lZC1pbWRiLWFjY2VsZXJhdGUn",highlighted:'<span class="hljs-string">&#x27;lewtun/distilbert-base-uncased-finetuned-imdb-accelerate&#x27;</span>',wrap:!1}}),K=new f({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMFJlcG9zaXRvcnklMEElMEFvdXRwdXRfZGlyJTIwJTNEJTIwbW9kZWxfbmFtZSUwQXJlcG8lMjAlM0QlMjBSZXBvc2l0b3J5KG91dHB1dF9kaXIlMkMlMjBjbG9uZV9mcm9tJTNEcmVwb19uYW1lKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository

output_dir = model_name
repo = Repository(output_dir, clone_from=repo_name)`,wrap:!1}}),O=new f({props:{code:"ZnJvbSUyMHRxZG0uYXV0byUyMGltcG9ydCUyMHRxZG0lMEFpbXBvcnQlMjB0b3JjaCUwQWltcG9ydCUyMG1hdGglMEElMEFwcm9ncmVzc19iYXIlMjAlM0QlMjB0cWRtKHJhbmdlKG51bV90cmFpbmluZ19zdGVwcykpJTBBJTBBZm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKG51bV90cmFpbl9lcG9jaHMpJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwVHJhaW5pbmclMEElMjAlMjAlMjAlMjBtb2RlbC50cmFpbigpJTBBJTIwJTIwJTIwJTIwZm9yJTIwYmF0Y2glMjBpbiUyMHRyYWluX2RhdGFsb2FkZXIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwb3V0cHV0cy5sb3NzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IuYmFja3dhcmQobG9zcyklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplci56ZXJvX2dyYWQoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByb2dyZXNzX2Jhci51cGRhdGUoMSklMEElMEElMjAlMjAlMjAlMjAlMjMlMjBFdmFsdWF0aW9uJTBBJTIwJTIwJTIwJTIwbW9kZWwuZXZhbCgpJTBBJTIwJTIwJTIwJTIwbG9zc2VzJTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwc3RlcCUyQyUyMGJhdGNoJTIwaW4lMjBlbnVtZXJhdGUoZXZhbF9kYXRhbG9hZGVyKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKipiYXRjaCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwb3V0cHV0cy5sb3NzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbG9zc2VzLmFwcGVuZChhY2NlbGVyYXRvci5nYXRoZXIobG9zcy5yZXBlYXQoYmF0Y2hfc2l6ZSkpKSUwQSUwQSUyMCUyMCUyMCUyMGxvc3NlcyUyMCUzRCUyMHRvcmNoLmNhdChsb3NzZXMpJTBBJTIwJTIwJTIwJTIwbG9zc2VzJTIwJTNEJTIwbG9zc2VzJTVCJTNBJTIwbGVuKGV2YWxfZGF0YXNldCklNUQlMEElMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwZXJwbGV4aXR5JTIwJTNEJTIwbWF0aC5leHAodG9yY2gubWVhbihsb3NzZXMpKSUwQSUyMCUyMCUyMCUyMGV4Y2VwdCUyME92ZXJmbG93RXJyb3IlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwZXJwbGV4aXR5JTIwJTNEJTIwZmxvYXQoJTIyaW5mJTIyKSUwQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlM0UlM0UlM0UlMjBFcG9jaCUyMCU3QmVwb2NoJTdEJTNBJTIwUGVycGxleGl0eSUzQSUyMCU3QnBlcnBsZXhpdHklN0QlMjIpJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwU2F2ZSUyMGFuZCUyMHVwbG9hZCUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLndhaXRfZm9yX2V2ZXJ5b25lKCklMEElMjAlMjAlMjAlMjB1bndyYXBwZWRfbW9kZWwlMjAlM0QlMjBhY2NlbGVyYXRvci51bndyYXBfbW9kZWwobW9kZWwpJTBBJTIwJTIwJTIwJTIwdW53cmFwcGVkX21vZGVsLnNhdmVfcHJldHJhaW5lZChvdXRwdXRfZGlyJTJDJTIwc2F2ZV9mdW5jdGlvbiUzRGFjY2VsZXJhdG9yLnNhdmUpJTBBJTIwJTIwJTIwJTIwaWYlMjBhY2NlbGVyYXRvci5pc19tYWluX3Byb2Nlc3MlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b2tlbml6ZXIuc2F2ZV9wcmV0cmFpbmVkKG91dHB1dF9kaXIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVwby5wdXNoX3RvX2h1YiglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb21taXRfbWVzc2FnZSUzRGYlMjJUcmFpbmluZyUyMGluJTIwcHJvZ3Jlc3MlMjBlcG9jaCUyMCU3QmVwb2NoJTdEJTIyJTJDJTIwYmxvY2tpbmclM0RGYWxzZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCk=",highlighted:`<span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> math

progress_bar = tqdm(<span class="hljs-built_in">range</span>(num_training_steps))

<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_train_epochs):
    <span class="hljs-comment"># Training</span>
    model.train()
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(<span class="hljs-number">1</span>)

    <span class="hljs-comment"># Evaluation</span>
    model.<span class="hljs-built_in">eval</span>()
    losses = []
    <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(eval_dataloader):
        <span class="hljs-keyword">with</span> torch.no_grad():
            outputs = model(**batch)

        loss = outputs.loss
        losses.append(accelerator.gather(loss.repeat(batch_size)))

    losses = torch.cat(losses)
    losses = losses[: <span class="hljs-built_in">len</span>(eval_dataset)]
    <span class="hljs-keyword">try</span>:
        perplexity = math.exp(torch.mean(losses))
    <span class="hljs-keyword">except</span> OverflowError:
        perplexity = <span class="hljs-built_in">float</span>(<span class="hljs-string">&quot;inf&quot;</span>)

    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; Epoch <span class="hljs-subst">{epoch}</span>: Perplexity: <span class="hljs-subst">{perplexity}</span>&quot;</span>)

    <span class="hljs-comment"># Save and upload</span>
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    <span class="hljs-keyword">if</span> accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=<span class="hljs-string">f&quot;Training in progress epoch <span class="hljs-subst">{epoch}</span>&quot;</span>, blocking=<span class="hljs-literal">False</span>
        )`,wrap:!1}}),ee=new f({props:{code:"RXBvY2glMjAwJTNBJTIwUGVycGxleGl0eSUzQSUyMDExLjM5NzU0NTMwNzkwMDQ3MiUwQUVwb2NoJTIwMSUzQSUyMFBlcnBsZXhpdHklM0ElMjAxMC45MDQ5MDkzMzA5ODMwOTIlMEFFcG9jaCUyMDIlM0ElMjBQZXJwbGV4aXR5JTNBJTIwMTAuNzI5NTAzNTA1MzQwNDA5",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>Epoch <span class="hljs-number">0</span>: Perplexity: <span class="hljs-number">11.397545307900472</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>Epoch <span class="hljs-number">1</span>: Perplexity: <span class="hljs-number">10.904909330983092</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>Epoch <span class="hljs-number">2</span>: Perplexity: <span class="hljs-number">10.729503505340409</span>`,wrap:!1}}),{c(){a=J("p"),a.textContent=c,n=i(),d(y.$$.fragment),_=i(),U=J("p"),U.innerHTML=R,W=i(),C=J("p"),C.innerHTML=X,Z=i(),d(B.$$.fragment),j=i(),G=J("p"),G.innerHTML=v,Q=i(),d(g.$$.fragment),$=i(),S=J("p"),S.innerHTML=k,V=i(),d(be.$$.fragment),Ge=i(),H=J("p"),H.textContent=Yt,ke=i(),d(z.$$.fragment),He=i(),se=J("p"),se.innerHTML=Ie,Le=i(),d(ae.$$.fragment),ne=i(),ie=J("p"),ie.innerHTML=pe,Be=i(),d(L.$$.fragment),Ze=i(),q=J("p"),q.textContent=Ft,Re=i(),d(Y.$$.fragment),qe=i(),oe=J("p"),oe.textContent=ge,De=i(),d(Me.$$.fragment),F=i(),d(x.$$.fragment),ye=i(),D=J("p"),D.innerHTML=Et,Xe=i(),d(K.$$.fragment),We=i(),P=J("p"),P.textContent=St,$e=i(),d(O.$$.fragment),Ve=i(),d(ee.$$.fragment),Ke=i(),re=J("p"),re.textContent=te},l(M){a=u(M,"P",{"data-svelte-h":!0}),T(a)!=="svelte-18f0f90"&&(a.textContent=c),n=o(M),h(y.$$.fragment,M),_=o(M),U=u(M,"P",{"data-svelte-h":!0}),T(U)!=="svelte-17r4mi9"&&(U.innerHTML=R),W=o(M),C=u(M,"P",{"data-svelte-h":!0}),T(C)!=="svelte-gxymov"&&(C.innerHTML=X),Z=o(M),h(B.$$.fragment,M),j=o(M),G=u(M,"P",{"data-svelte-h":!0}),T(G)!=="svelte-14pqtt6"&&(G.innerHTML=v),Q=o(M),h(g.$$.fragment,M),$=o(M),S=u(M,"P",{"data-svelte-h":!0}),T(S)!=="svelte-8xq44y"&&(S.innerHTML=k),V=o(M),h(be.$$.fragment,M),Ge=o(M),H=u(M,"P",{"data-svelte-h":!0}),T(H)!=="svelte-1ixsyix"&&(H.textContent=Yt),ke=o(M),h(z.$$.fragment,M),He=o(M),se=u(M,"P",{"data-svelte-h":!0}),T(se)!=="svelte-1uukzr7"&&(se.innerHTML=Ie),Le=o(M),h(ae.$$.fragment,M),ne=o(M),ie=u(M,"P",{"data-svelte-h":!0}),T(ie)!=="svelte-xxic1e"&&(ie.innerHTML=pe),Be=o(M),h(L.$$.fragment,M),Ze=o(M),q=u(M,"P",{"data-svelte-h":!0}),T(q)!=="svelte-cqa387"&&(q.textContent=Ft),Re=o(M),h(Y.$$.fragment,M),qe=o(M),oe=u(M,"P",{"data-svelte-h":!0}),T(oe)!=="svelte-1asoup2"&&(oe.textContent=ge),De=o(M),h(Me.$$.fragment,M),F=o(M),h(x.$$.fragment,M),ye=o(M),D=u(M,"P",{"data-svelte-h":!0}),T(D)!=="svelte-om9ony"&&(D.innerHTML=Et),Xe=o(M),h(K.$$.fragment,M),We=o(M),P=u(M,"P",{"data-svelte-h":!0}),T(P)!=="svelte-1ut1bsc"&&(P.textContent=St),$e=o(M),h(O.$$.fragment,M),Ve=o(M),h(ee.$$.fragment,M),Ke=o(M),re=u(M,"P",{"data-svelte-h":!0}),T(re)!=="svelte-63yh5i"&&(re.textContent=te)},m(M,b){s(M,a,b),s(M,n,b),w(y,M,b),s(M,_,b),s(M,U,b),s(M,W,b),s(M,C,b),s(M,Z,b),w(B,M,b),s(M,j,b),s(M,G,b),s(M,Q,b),w(g,M,b),s(M,$,b),s(M,S,b),s(M,V,b),w(be,M,b),s(M,Ge,b),s(M,H,b),s(M,ke,b),w(z,M,b),s(M,He,b),s(M,se,b),s(M,Le,b),w(ae,M,b),s(M,ne,b),s(M,ie,b),s(M,Be,b),w(L,M,b),s(M,Ze,b),s(M,q,b),s(M,Re,b),w(Y,M,b),s(M,qe,b),s(M,oe,b),s(M,De,b),w(Me,M,b),s(M,F,b),w(x,M,b),s(M,ye,b),s(M,D,b),s(M,Xe,b),w(K,M,b),s(M,We,b),s(M,P,b),s(M,$e,b),w(O,M,b),s(M,Ve,b),w(ee,M,b),s(M,Ke,b),s(M,re,b),E=!0},i(M){E||(r(y.$$.fragment,M),r(B.$$.fragment,M),r(g.$$.fragment,M),r(be.$$.fragment,M),r(z.$$.fragment,M),r(ae.$$.fragment,M),r(L.$$.fragment,M),r(Y.$$.fragment,M),r(Me.$$.fragment,M),r(x.$$.fragment,M),r(K.$$.fragment,M),r(O.$$.fragment,M),r(ee.$$.fragment,M),E=!0)},o(M){p(y.$$.fragment,M),p(B.$$.fragment,M),p(g.$$.fragment,M),p(be.$$.fragment,M),p(z.$$.fragment,M),p(ae.$$.fragment,M),p(L.$$.fragment,M),p(Y.$$.fragment,M),p(Me.$$.fragment,M),p(x.$$.fragment,M),p(K.$$.fragment,M),p(O.$$.fragment,M),p(ee.$$.fragment,M),E=!1},d(M){M&&(t(a),t(n),t(_),t(U),t(W),t(C),t(Z),t(j),t(G),t(Q),t($),t(S),t(V),t(Ge),t(H),t(ke),t(He),t(se),t(Le),t(ne),t(ie),t(Be),t(Ze),t(q),t(Re),t(qe),t(oe),t(De),t(F),t(ye),t(D),t(Xe),t(We),t(P),t($e),t(Ve),t(Ke),t(re)),m(y,M),m(B,M),m(g,M),m(be,M),m(z,M),m(ae,M),m(L,M),m(Y,M),m(Me,M),m(x,M),m(K,M),m(O,M),m(ee,M)}}}function Wi(I){let a,c='✏️ <strong>Try it out!</strong> To quantify the benefits of domain adaptation, fine-tune a classifier on the IMDb labels for both the pretrained and fine-tuned DistilBERT checkpoints. If you need a refresher on text classification, check out <a href="/course/chapter3">Chapter 3</a>.';return{c(){a=J("p"),a.innerHTML=c},l(n){a=u(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-13mk4ny"&&(a.innerHTML=c)},m(n,y){s(n,a,y)},p:Se,d(n){n&&t(a)}}}function $i(I){let a,c,n,y,_,U,R,W,C,X,Z,B,j="For many NLP applications involving Transformer models, you can simply take a pretrained model from the Hugging Face Hub and fine-tune it directly on your data for the task at hand. Provided that the corpus used for pretraining is not too different from the corpus used for fine-tuning, transfer learning will usually produce good results.",G,v,Q="However, there are a few cases where you’ll want to first fine-tune the language models on your data, before training a task-specific head. For example, if your dataset contains legal contracts or scientific articles, a vanilla Transformer model like BERT will typically treat the domain-specific words in your corpus as rare tokens, and the resulting performance may be less than satisfactory. By fine-tuning the language model on in-domain data you can boost the performance of many downstream tasks, which means you usually only have to do this step once!",g,$,S='This process of fine-tuning a pretrained language model on in-domain data is usually called <em>domain adaptation</em>. It was popularized in 2018 by <a href="https://arxiv.org/abs/1801.06146" rel="nofollow">ULMFiT</a>, which was one of the first neural architectures (based on LSTMs) to make transfer learning really work for NLP. An example of domain adaptation with ULMFiT is shown in the image below; in this section we’ll do something similar, but with a Transformer instead of an LSTM!',k,V,be='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/ulmfit.svg" alt="ULMFiT."/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/ulmfit-dark.svg" alt="ULMFiT."/>',Ge,H,Yt='By the end of this section you’ll have a <a href="https://huggingface.co/huggingface-course/distilbert-base-uncased-finetuned-imdb?text=This+is+a+great+%5BMASK%5D." rel="nofollow">masked language model</a> on the Hub that can autocomplete sentences as shown below:',ke,z,He,se,Ie,Le="Let’s dive in!",ae,ne,ie,pe,Be,L,Ze,q,Ft='To get started, let’s pick a suitable pretrained model for masked language modeling. As shown in the following screenshot, you can find a list of candidates by applying the “Fill-Mask” filter on the <a href="https://huggingface.co/models?pipeline_tag=fill-mask&amp;sort=downloads" rel="nofollow">Hugging Face Hub</a>:',Re,Y,qe='<img src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/mlm-models.png" alt="Hub models." width="80%"/>',oe,ge,De=`Although the BERT and RoBERTa family of models are the most downloaded, we’ll use a model called <a href="https://huggingface.co/distilbert-base-uncased" rel="nofollow">DistilBERT</a>
that can be trained much faster with little to no loss in downstream performance. This model was trained using a special technique called <a href="https://en.wikipedia.org/wiki/Knowledge_distillation" rel="nofollow"><em>knowledge distillation</em></a>, where a large “teacher model” like BERT is used to guide the training of a “student model” that has far fewer parameters. An explanation of the details of knowledge distillation would take us too far afield in this section, but if you’re interested you can read all about it in <a href="https://www.oreilly.com/library/view/natural-language-processing/9781098136789/" rel="nofollow"><em>Natural Language Processing with Transformers</em></a> (colloquially known as the Transformers textbook).`,Me,F,x,ye,D,Et="With around 67 million parameters, DistilBERT is approximately two times smaller than the BERT base model, which roughly translates into a two-fold speedup in training — nice! Let’s now see what kinds of tokens this model predicts are the most likely completions of a small sample of text:",Xe,K,We,P,St='As humans, we can imagine many possibilities for the <code>[MASK]</code> token, such as “day”, “ride”, or “painting”. For pretrained models, the predictions depend on the corpus the model was trained on, since it learns to pick up the statistical patterns present in the data. Like BERT, DistilBERT was pretrained on the <a href="https://huggingface.co/datasets/wikipedia" rel="nofollow">English Wikipedia</a> and <a href="https://huggingface.co/datasets/bookcorpus" rel="nofollow">BookCorpus</a> datasets, so we expect the predictions for <code>[MASK]</code> to reflect these domains. To predict the mask we need DistilBERT’s tokenizer to produce the inputs for the model, so let’s download that from the Hub as well:',$e,O,Ve,ee,Ke="With a tokenizer and a model, we can now pass our text example to the model, extract the logits, and print out the top 5 candidates:",re,te,E,M,b,Lt,Pe,Qa="We can see from the outputs that the model’s predictions refer to everyday terms, which is perhaps not surprising given the foundation of English Wikipedia. Let’s see how we can change this domain to something a bit more niche — highly polarized movie reviews!",qt,Oe,Dt,el,Ya='To showcase domain adaptation, we’ll use the famous <a href="https://huggingface.co/datasets/imdb" rel="nofollow">Large Movie Review Dataset</a> (or IMDb for short), which is a corpus of movie reviews that is often used to benchmark sentiment analysis models. By fine-tuning DistilBERT on this corpus, we expect the language model will adapt its vocabulary from the factual data of Wikipedia that it was pretrained on to the more subjective elements of movie reviews. We can get the data from the Hugging Face Hub with the <code>load_dataset()</code> function from 🤗 Datasets:',Kt,ll,Pt,tl,Ot,sl,Fa="We can see that the <code>train</code> and <code>test</code> splits each consist of 25,000 reviews, while there is an unlabeled split called <code>unsupervised</code> that contains 50,000 reviews. Let’s take a look at a few samples to get an idea of what kind of text we’re dealing with. As we’ve done in previous chapters of the course, we’ll chain the <code>Dataset.shuffle()</code> and <code>Dataset.select()</code> functions to create a random sample:",es,al,ls,nl,ts,il,Ea="Yep, these are certainly movie reviews, and if you’re old enough you may even understand the comment in the last review about owning a VHS version 😜! Although we won’t need the labels for language modeling, we can already see that a <code>0</code> denotes a negative review, while a <code>1</code> corresponds to a positive one.",ss,ve,as,ol,Sa='Now that we’ve had a quick look at the data, let’s dive into preparing it for masked language modeling. As we’ll see, there are some additional steps that one needs to take compared to the sequence classification tasks we saw in <a href="/course/chapter3">Chapter 3</a>. Let’s go!',ns,Ml,is,rl,os,pl,Ha="For both auto-regressive and masked language modeling, a common preprocessing step is to concatenate all the examples and then split the whole corpus into chunks of equal size. This is quite different from our usual approach, where we simply tokenize individual examples. Why concatenate everything together? The reason is that individual examples might get truncated if they’re too long, and that would result in losing information that might be useful for the language modeling task!",Ms,yl,La='So to get started, we’ll first tokenize our corpus as usual, but <em>without</em> setting the <code>truncation=True</code> option in our tokenizer. We’ll also grab the word IDs if they are available ((which they will be if we’re using a fast tokenizer, as described in <a href="/course/chapter6/3">Chapter 6</a>), as we will need them later on to do whole word masking. We’ll wrap this in a simple function, and while we’re at it we’ll remove the <code>text</code> and <code>label</code> columns since we don’t need them any longer:',rs,cl,ps,dl,ys,hl,qa="Since DistilBERT is a BERT-like model, we can see that the encoded texts consist of the <code>input_ids</code> and <code>attention_mask</code> that we’ve seen in other chapters, as well as the <code>word_ids</code> we added.",cs,wl,Da="Now that we’ve tokenized our movie reviews, the next step is to group them all together and split the result into chunks. But how big should these chunks be? This will ultimately be determined by the amount of GPU memory that you have available, but a good starting point is to see what the model’s maximum context size is. This can be inferred by inspecting the <code>model_max_length</code> attribute of the tokenizer:",ds,ml,hs,Jl,ws,ul,Ka="This value is derived from the <em>tokenizer_config.json</em> file associated with a checkpoint; in this case we can see that the context size is 512 tokens, just like with BERT.",ms,ze,Js,Ul,Pa="So, in order to run our experiments on GPUs like those found on Google Colab, we’ll pick something a bit smaller that can fit in memory:",us,Tl,Us,xe,Ts,jl,Oa="Now comes the fun part. To show how the concatenation works, let’s take a few reviews from our tokenized training set and print out the number of tokens per review:",js,fl,fs,bl,bs,Il,en="We can then concatenate all these examples with a simple dictionary comprehension, as follows:",Is,gl,gs,Cl,Cs,_l,ln="Great, the total length checks out — so now let’s split the concatenated reviews into chunks of the size given by <code>chunk_size</code>. To do so, we iterate over the features in <code>concatenated_examples</code> and use a list comprehension to create slices of each feature. The result is a dictionary of chunks for each feature:",_s,Gl,Gs,kl,ks,Bl,tn="As you can see in this example, the last chunk will generally be smaller than the maximum chunk size. There are two main strategies for dealing with this:",Bs,Zl,sn="<li>Drop the last chunk if it’s smaller than <code>chunk_size</code>.</li> <li>Pad the last chunk until its length equals <code>chunk_size</code>.</li>",Zs,Rl,an="We’ll take the first approach here, so let’s wrap all of the above logic in a single function that we can apply to our tokenized datasets:",Rs,Xl,Xs,Wl,nn="Note that in the last step of <code>group_texts()</code> we create a new <code>labels</code> column which is a copy of the <code>input_ids</code> one. As we’ll see shortly, that’s because in masked language modeling the objective is to predict randomly masked tokens in the input batch, and by creating a <code>labels</code> column we provide the ground truth for our language model to learn from.",Ws,$l,on="Let’s now apply <code>group_texts()</code> to our tokenized datasets using our trusty <code>Dataset.map()</code> function:",$s,Vl,Vs,vl,vs,zl,Mn="You can see that grouping and then chunking the texts has produced many more examples than our original 25,000 for the <code>train</code> and <code>test</code> splits. That’s because we now have examples involving <em>contiguous tokens</em> that span across multiple examples from the original corpus. You can see this explicitly by looking for the special <code>[SEP]</code> and <code>[CLS]</code> tokens in one of the chunks:",zs,xl,xs,Nl,Ns,Al,rn="In this example you can see two overlapping movie reviews, one about a high school movie and the other about homelessness. Let’s also check out what the labels look like for masked language modeling:",As,Ql,Qs,Yl,Ys,Fl,pn="As expected from our <code>group_texts()</code> function above, this looks identical to the decoded <code>input_ids</code> — but then how can our model possibly learn anything? We’re missing a key step: inserting <code>[MASK]</code> tokens at random positions in the inputs! Let’s see how we can do this on the fly during fine-tuning using a special data collator.",Fs,El,Es,Sl,yn='Fine-tuning a masked language model is almost identical to fine-tuning a sequence classification model, like we did in <a href="/course/chapter3">Chapter 3</a>. The only difference is that we need a special data collator that can randomly mask some of the tokens in each batch of texts. Fortunately, 🤗 Transformers comes prepared with a dedicated <code>DataCollatorForLanguageModeling</code> for just this task. We just have to pass it the tokenizer and an <code>mlm_probability</code> argument that specifies what fraction of the tokens to mask. We’ll pick 15%, which is the amount used for BERT and a common choice in the literature:',Ss,Hl,Hs,Ll,cn="To see how the random masking works, let’s feed a few examples to the data collator. Since it expects a list of <code>dict</code>s, where each <code>dict</code> represents a single chunk of contiguous text, we first iterate over the dataset before feeding the batch to the collator. We remove the <code>&quot;word_ids&quot;</code> key for this data collator as it does not expect it:",Ls,ql,qs,Dl,Ds,Kl,dn="Nice, it worked! We can see that the <code>[MASK]</code> token has been randomly inserted at various locations in our text. These will be the tokens which our model will have to predict during training — and the beauty of the data collator is that it will randomize the <code>[MASK]</code> insertion with every batch!",Ks,Ne,Ps,$t,Pl,hn="When training models for masked language modeling, one technique that can be used is to mask whole words together, not just individual tokens. This approach is called <em>whole word masking</em>. If we want to use whole word masking, we will need to build a data collator ourselves. A data collator is just a function that takes a list of samples and converts them into a batch, so let’s do this now! We’ll use the word IDs computed earlier to make a map between word indices and the corresponding tokens, then randomly decide which words to mask and apply that mask on the inputs. Note that the labels are all <code>-100</code> except for the ones corresponding to mask words.",Os,ce,de,Vt,Ol,wn="Next, we can try it on the same samples as before:",ea,et,la,lt,ta,Ae,sa,tt,mn='Now that we have two data collators, the rest of the fine-tuning steps are standard. Training can take a while on Google Colab if you’re not lucky enough to score a mythical P100 GPU 😭, so we’ll first downsample the size of the training set to a few thousand examples. Don’t worry, we’ll still get a pretty decent language model! A quick way to downsample a dataset in 🤗 Datasets is via the <code>Dataset.train_test_split()</code> function that we saw in <a href="/course/chapter5">Chapter 5</a>:',aa,st,na,at,ia,nt,Jn="This has automatically created new <code>train</code> and <code>test</code> splits, with the training set size set to 10,000 examples and the validation set to 10% of that — feel free to increase this if you have a beefy GPU! The next thing we need to do is log in to the Hugging Face Hub. If you’re running this code in a notebook, you can do so with the following utility function:",oa,it,Ma,ot,un="which will display a widget where you can enter your credentials. Alternatively, you can run:",ra,Mt,pa,rt,Un="in your favorite terminal and log in there.",ya,he,we,vt,pt,ca,yt,da,ct,Tn="Unlike other tasks like text classification or question answering where we’re given a labeled corpus to train on, with language modeling we don’t have any explicit labels. So how do we determine what makes a good language model? Like with the autocorrect feature in your phone, a good language model is one that assigns high probabilities to sentences that are grammatically correct, and low probabilities to nonsense sentences. To give you a better idea of what this looks like, you can find whole sets of “autocorrect fails” online, where the model in a person’s phone has produced some rather funny (and often inappropriate) completions!",ha,me,Je,zt,dt,wa,ht,jn="A lower perplexity score means a better language model, and we can see here that our starting model has a somewhat large value. Let’s see if we can lower it by fine-tuning! To do that, we first run the training loop:",ma,ue,Ue,xt,wt,fn="and then compute the resulting perplexity on the test set as before:",Ja,Te,je,Nt,mt,ua,Jt,bn="Nice — this is quite a reduction in perplexity, which tells us the model has learned something about the domain of movie reviews!",Ua,At,Qe,Ta,Qt,ut,ja,Ut,In="You can interact with your fine-tuned model either by using its widget on the Hub or locally with the <code>pipeline</code> from 🤗 Transformers. Let’s use the latter to download our model using the <code>fill-mask</code> pipeline:",fa,Tt,ba,jt,gn="We can then feed the pipeline our sample text of “This is a great [MASK]” and see what the top 5 predictions are:",Ia,ft,ga,bt,Ca,It,Cn="Neat — our model has clearly adapted its weights to predict words that are more strongly associated with movies!",_a,gt,Ga,Ct,_n='This wraps up our first experiment with training a language model. In <a href="/course/en/chapter7/6">section 6</a> you’ll learn how to train an auto-regressive model like GPT-2 from scratch; head over there if you’d like to see how you can pretrain your very own Transformer model!',ka,Ye,Ba,Ht,Za;_=new pi({props:{fw:I[0]}}),R=new Ee({props:{title:"Fine-tuning a masked language model",local:"fine-tuning-a-masked-language-model",headingTag:"h1"}});const Gn=[ci,yi],_t=[];function kn(e,l){return e[0]==="pt"?0:1}C=kn(I),X=_t[C]=Gn[C](I),ne=new Ra({props:{id:"mqElG5QJWUg"}}),pe=new Fe({props:{$$slots:{default:[di]},$$scope:{ctx:I}}}),L=new Ee({props:{title:"Picking a pretrained model for masked language modeling",local:"picking-a-pretrained-model-for-masked-language-modeling",headingTag:"h2"}});const Bn=[wi,hi],Gt=[];function Zn(e,l){return e[0]==="pt"?0:1}F=Zn(I),x=Gt[F]=Bn[F](I),K=new f({props:{code:"dGV4dCUyMCUzRCUyMCUyMlRoaXMlMjBpcyUyMGElMjBncmVhdCUyMCU1Qk1BU0slNUQuJTIy",highlighted:'text = <span class="hljs-string">&quot;This is a great [MASK].&quot;</span>',wrap:!1}}),O=new f({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9jaGVja3BvaW50KQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)`,wrap:!1}});const Rn=[Ji,mi],kt=[];function Xn(e,l){return e[0]==="pt"?0:1}te=Xn(I),E=kt[te]=Rn[te](I),b=new f({props:{code:"JyUzRSUzRSUzRSUyMFRoaXMlMjBpcyUyMGElMjBncmVhdCUyMGRlYWwuJyUwQSclM0UlM0UlM0UlMjBUaGlzJTIwaXMlMjBhJTIwZ3JlYXQlMjBzdWNjZXNzLiclMEEnJTNFJTNFJTNFJTIwVGhpcyUyMGlzJTIwYSUyMGdyZWF0JTIwYWR2ZW50dXJlLiclMEEnJTNFJTNFJTNFJTIwVGhpcyUyMGlzJTIwYSUyMGdyZWF0JTIwaWRlYS4nJTBBJyUzRSUzRSUzRSUyMFRoaXMlMjBpcyUyMGElMjBncmVhdCUyMGZlYXQuJw==",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great deal.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great success.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great adventure.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great idea.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great feat.&#x27;</span>`,wrap:!1}}),Oe=new Ee({props:{title:"The dataset",local:"the-dataset",headingTag:"h2"}}),ll=new f({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBaW1kYl9kYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmltZGIlMjIpJTBBaW1kYl9kYXRhc2V0",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

imdb_dataset = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>)
imdb_dataset`,wrap:!1}}),tl=new f({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ3RleHQnJTJDJTIwJ2xhYmVsJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwMjUwMDAlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTIwJTIwJTIwJTIwdGVzdCUzQSUyMERhdGFzZXQoJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmVhdHVyZXMlM0ElMjAlNUIndGV4dCclMkMlMjAnbGFiZWwnJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3Jvd3MlM0ElMjAyNTAwMCUwQSUyMCUyMCUyMCUyMCU3RCklMEElMjAlMjAlMjAlMjB1bnN1cGVydmlzZWQlM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ3RleHQnJTJDJTIwJ2xhYmVsJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwNTAwMDAlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTdEKQ==",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;text&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;text&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    unsupervised: Dataset({
        features: [<span class="hljs-string">&#x27;text&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>],
        num_rows: <span class="hljs-number">50000</span>
    })
})`,wrap:!1}}),al=new f({props:{code:"c2FtcGxlJTIwJTNEJTIwaW1kYl9kYXRhc2V0JTVCJTIydHJhaW4lMjIlNUQuc2h1ZmZsZShzZWVkJTNENDIpLnNlbGVjdChyYW5nZSgzKSklMEElMEFmb3IlMjByb3clMjBpbiUyMHNhbXBsZSUzQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlNUNuJyUzRSUzRSUzRSUyMFJldmlldyUzQSUyMCU3QnJvdyU1Qid0ZXh0JyU1RCU3RCclMjIpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiclM0UlM0UlM0UlMjBMYWJlbCUzQSUyMCU3QnJvdyU1QidsYWJlbCclNUQlN0QnJTIyKQ==",highlighted:`sample = imdb_dataset[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>))

<span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> sample:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; Review: <span class="hljs-subst">{row[<span class="hljs-string">&#x27;text&#x27;</span>]}</span>&#x27;&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Label: <span class="hljs-subst">{row[<span class="hljs-string">&#x27;label&#x27;</span>]}</span>&#x27;&quot;</span>)`,wrap:!1}}),nl=new f({props:{code:" ",highlighted:`
<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: This is your typical Priyadarshan movie--a bunch of loony characters out on some silly mission. His signature climax has the entire cast of the film coming together and fighting each other in some crazy moshpit over hidden money. Whether it is a winning lottery ticket in Malamaal Weekly, black money in Hera Pheri, &quot;kodokoo&quot; in Phir Hera Pheri, etc., etc., the director is becoming ridiculously predictable. Don\\&#x27;t get me wrong; as clichéd and preposterous his movies may be, I usually end up enjoying the comedy. However, in most his previous movies there has actually been some good humor, (Hungama and Hera Pheri being noteworthy ones). Now, the hilarity of his films is fading as he is using the same formula over and over again.&lt;br /&gt;&lt;br /&gt;Songs are good. Tanushree Datta looks awesome. Rajpal Yadav is irritating, and Tusshar is not a whole lot better. Kunal Khemu is OK, and Sharman Joshi is the best.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Label: 0&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: Okay, the story makes no sense, the characters lack any dimensionally, the best dialogue is ad-libs about the low quality of movie, the cinematography is dismal, and only editing saves a bit of the muddle, but Sam&quot; Peckinpah directed the film. Somehow, his direction is not enough. For those who appreciate Peckinpah and his great work, this movie is a disappointment. Even a great cast cannot redeem the time the viewer wastes with this minimal effort.&lt;br /&gt;&lt;br /&gt;The proper response to the movie is the contempt that the director San Peckinpah, James Caan, Robert Duvall, Burt Young, Bo Hopkins, Arthur Hill, and even Gig Young bring to their work. Watch the great Peckinpah films. Skip this mess.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Label: 0&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: I saw this movie at the theaters when I was about 6 or 7 years old. I loved it then, and have recently come to own a VHS version. &lt;br /&gt;&lt;br /&gt;My 4 and 6 year old children love this movie and have been asking again and again to watch it. &lt;br /&gt;&lt;br /&gt;I have enjoyed watching it again too. Though I have to admit it is not as good on a little TV.&lt;br /&gt;&lt;br /&gt;I do not have older children so I do not know what they would think of it. &lt;br /&gt;&lt;br /&gt;The songs are very cute. My daughter keeps singing them over and over.&lt;br /&gt;&lt;br /&gt;Hope this helps.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Label: 1&#x27;</span>`,wrap:!1}}),ve=new Fe({props:{$$slots:{default:[ui]},$$scope:{ctx:I}}}),Ml=new Ee({props:{title:"Preprocessing the data",local:"preprocessing-the-data",headingTag:"h2"}}),rl=new Ra({props:{id:"8PmhEIXhBvI"}}),cl=new f({props:{code:"ZGVmJTIwdG9rZW5pemVfZnVuY3Rpb24oZXhhbXBsZXMpJTNBJTBBJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydGV4dCUyMiU1RCklMEElMjAlMjAlMjAlMjBpZiUyMHRva2VuaXplci5pc19mYXN0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTVCJTIyd29yZF9pZHMlMjIlNUQlMjAlM0QlMjAlNUJyZXN1bHQud29yZF9pZHMoaSklMjBmb3IlMjBpJTIwaW4lMjByYW5nZShsZW4ocmVzdWx0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKSklNUQlMEElMjAlMjAlMjAlMjByZXR1cm4lMjByZXN1bHQlMEElMEElMEElMjMlMjBVc2UlMjBiYXRjaGVkJTNEVHJ1ZSUyMHRvJTIwYWN0aXZhdGUlMjBmYXN0JTIwbXVsdGl0aHJlYWRpbmchJTBBdG9rZW5pemVkX2RhdGFzZXRzJTIwJTNEJTIwaW1kYl9kYXRhc2V0Lm1hcCglMEElMjAlMjAlMjAlMjB0b2tlbml6ZV9mdW5jdGlvbiUyQyUyMGJhdGNoZWQlM0RUcnVlJTJDJTIwcmVtb3ZlX2NvbHVtbnMlM0QlNUIlMjJ0ZXh0JTIyJTJDJTIwJTIybGFiZWwlMjIlNUQlMEEpJTBBdG9rZW5pemVkX2RhdGFzZXRz",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_function</span>(<span class="hljs-params">examples</span>):
    result = tokenizer(examples[<span class="hljs-string">&quot;text&quot;</span>])
    <span class="hljs-keyword">if</span> tokenizer.is_fast:
        result[<span class="hljs-string">&quot;word_ids&quot;</span>] = [result.word_ids(i) <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(result[<span class="hljs-string">&quot;input_ids&quot;</span>]))]
    <span class="hljs-keyword">return</span> result


<span class="hljs-comment"># Use batched=True to activate fast multithreading!</span>
tokenized_datasets = imdb_dataset.<span class="hljs-built_in">map</span>(
    tokenize_function, batched=<span class="hljs-literal">True</span>, remove_columns=[<span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;label&quot;</span>]
)
tokenized_datasets`,wrap:!1}}),dl=new f({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ2F0dGVudGlvbl9tYXNrJyUyQyUyMCdpbnB1dF9pZHMnJTJDJTIwJ3dvcmRfaWRzJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwMjUwMDAlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTIwJTIwJTIwJTIwdGVzdCUzQSUyMERhdGFzZXQoJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmVhdHVyZXMlM0ElMjAlNUInYXR0ZW50aW9uX21hc2snJTJDJTIwJ2lucHV0X2lkcyclMkMlMjAnd29yZF9pZHMnJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3Jvd3MlM0ElMjAyNTAwMCUwQSUyMCUyMCUyMCUyMCU3RCklMEElMjAlMjAlMjAlMjB1bnN1cGVydmlzZWQlM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ2F0dGVudGlvbl9tYXNrJyUyQyUyMCdpbnB1dF9pZHMnJTJDJTIwJ3dvcmRfaWRzJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwNTAwMDAlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTdEKQ==",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    unsupervised: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">50000</span>
    })
})`,wrap:!1}}),ml=new f({props:{code:"dG9rZW5pemVyLm1vZGVsX21heF9sZW5ndGg=",highlighted:"tokenizer.model_max_length",wrap:!1}}),Jl=new f({props:{code:"NTEy",highlighted:'<span class="hljs-number">512</span>',wrap:!1}}),ze=new Fe({props:{$$slots:{default:[Ui]},$$scope:{ctx:I}}}),Tl=new f({props:{code:"Y2h1bmtfc2l6ZSUyMCUzRCUyMDEyOA==",highlighted:'chunk_size = <span class="hljs-number">128</span>',wrap:!1}}),xe=new Fe({props:{warning:!0,$$slots:{default:[Ti]},$$scope:{ctx:I}}}),fl=new f({props:{code:"JTIzJTIwU2xpY2luZyUyMHByb2R1Y2VzJTIwYSUyMGxpc3QlMjBvZiUyMGxpc3RzJTIwZm9yJTIwZWFjaCUyMGZlYXR1cmUlMEF0b2tlbml6ZWRfc2FtcGxlcyUyMCUzRCUyMHRva2VuaXplZF9kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVEJTVCJTNBMyU1RCUwQSUwQWZvciUyMGlkeCUyQyUyMHNhbXBsZSUyMGluJTIwZW51bWVyYXRlKHRva2VuaXplZF9zYW1wbGVzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUzQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjInJTNFJTNFJTNFJTIwUmV2aWV3JTIwJTdCaWR4JTdEJTIwbGVuZ3RoJTNBJTIwJTdCbGVuKHNhbXBsZSklN0QnJTIyKQ==",highlighted:`<span class="hljs-comment"># Slicing produces a list of lists for each feature</span>
tokenized_samples = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][:<span class="hljs-number">3</span>]

<span class="hljs-keyword">for</span> idx, sample <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(tokenized_samples[<span class="hljs-string">&quot;input_ids&quot;</span>]):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Review <span class="hljs-subst">{idx}</span> length: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(sample)}</span>&#x27;&quot;</span>)`,wrap:!1}}),bl=new f({props:{code:"JyUzRSUzRSUzRSUyMFJldmlldyUyMDAlMjBsZW5ndGglM0ElMjAyMDAnJTBBJyUzRSUzRSUzRSUyMFJldmlldyUyMDElMjBsZW5ndGglM0ElMjA1NTknJTBBJyUzRSUzRSUzRSUyMFJldmlldyUyMDIlMjBsZW5ndGglM0ElMjAxOTIn",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; Review 0 length: 200&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Review 1 length: 559&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Review 2 length: 192&#x27;</span>`,wrap:!1}}),gl=new f({props:{code:"Y29uY2F0ZW5hdGVkX2V4YW1wbGVzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwayUzQSUyMHN1bSh0b2tlbml6ZWRfc2FtcGxlcyU1QmslNUQlMkMlMjAlNUIlNUQpJTIwZm9yJTIwayUyMGluJTIwdG9rZW5pemVkX3NhbXBsZXMua2V5cygpJTBBJTdEJTBBdG90YWxfbGVuZ3RoJTIwJTNEJTIwbGVuKGNvbmNhdGVuYXRlZF9leGFtcGxlcyU1QiUyMmlucHV0X2lkcyUyMiU1RCklMEFwcmludChmJTIyJyUzRSUzRSUzRSUyMENvbmNhdGVuYXRlZCUyMHJldmlld3MlMjBsZW5ndGglM0ElMjAlN0J0b3RhbF9sZW5ndGglN0QnJTIyKQ==",highlighted:`concatenated_examples = {
    k: <span class="hljs-built_in">sum</span>(tokenized_samples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> tokenized_samples.keys()
}
total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Concatenated reviews length: <span class="hljs-subst">{total_length}</span>&#x27;&quot;</span>)`,wrap:!1}}),Cl=new f({props:{code:"JyUzRSUzRSUzRSUyMENvbmNhdGVuYXRlZCUyMHJldmlld3MlMjBsZW5ndGglM0ElMjA5NTEn",highlighted:'<span class="hljs-string">&#x27;&gt;&gt;&gt; Concatenated reviews length: 951&#x27;</span>',wrap:!1}}),Gl=new f({props:{code:"Y2h1bmtzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwayUzQSUyMCU1QnQlNUJpJTIwJTNBJTIwaSUyMCUyQiUyMGNodW5rX3NpemUlNUQlMjBmb3IlMjBpJTIwaW4lMjByYW5nZSgwJTJDJTIwdG90YWxfbGVuZ3RoJTJDJTIwY2h1bmtfc2l6ZSklNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBrJTJDJTIwdCUyMGluJTIwY29uY2F0ZW5hdGVkX2V4YW1wbGVzLml0ZW1zKCklMEElN0QlMEElMEFmb3IlMjBjaHVuayUyMGluJTIwY2h1bmtzJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiclM0UlM0UlM0UlMjBDaHVuayUyMGxlbmd0aCUzQSUyMCU3QmxlbihjaHVuayklN0QnJTIyKQ==",highlighted:`chunks = {
    k: [t[i : i + chunk_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, chunk_size)]
    <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
}

<span class="hljs-keyword">for</span> chunk <span class="hljs-keyword">in</span> chunks[<span class="hljs-string">&quot;input_ids&quot;</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Chunk length: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(chunk)}</span>&#x27;&quot;</span>)`,wrap:!1}}),kl=new f({props:{code:"JyUzRSUzRSUzRSUyMENodW5rJTIwbGVuZ3RoJTNBJTIwMTI4JyUwQSclM0UlM0UlM0UlMjBDaHVuayUyMGxlbmd0aCUzQSUyMDEyOCclMEEnJTNFJTNFJTNFJTIwQ2h1bmslMjBsZW5ndGglM0ElMjAxMjgnJTBBJyUzRSUzRSUzRSUyMENodW5rJTIwbGVuZ3RoJTNBJTIwMTI4JyUwQSclM0UlM0UlM0UlMjBDaHVuayUyMGxlbmd0aCUzQSUyMDEyOCclMEEnJTNFJTNFJTNFJTIwQ2h1bmslMjBsZW5ndGglM0ElMjAxMjgnJTBBJyUzRSUzRSUzRSUyMENodW5rJTIwbGVuZ3RoJTNBJTIwMTI4JyUwQSclM0UlM0UlM0UlMjBDaHVuayUyMGxlbmd0aCUzQSUyMDU1Jw==",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 55&#x27;</span>`,wrap:!1}}),Xl=new f({props:{code:"ZGVmJTIwZ3JvdXBfdGV4dHMoZXhhbXBsZXMpJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwQ29uY2F0ZW5hdGUlMjBhbGwlMjB0ZXh0cyUwQSUyMCUyMCUyMCUyMGNvbmNhdGVuYXRlZF9leGFtcGxlcyUyMCUzRCUyMCU3QmslM0ElMjBzdW0oZXhhbXBsZXMlNUJrJTVEJTJDJTIwJTVCJTVEKSUyMGZvciUyMGslMjBpbiUyMGV4YW1wbGVzLmtleXMoKSU3RCUwQSUyMCUyMCUyMCUyMCUyMyUyMENvbXB1dGUlMjBsZW5ndGglMjBvZiUyMGNvbmNhdGVuYXRlZCUyMHRleHRzJTBBJTIwJTIwJTIwJTIwdG90YWxfbGVuZ3RoJTIwJTNEJTIwbGVuKGNvbmNhdGVuYXRlZF9leGFtcGxlcyU1Qmxpc3QoZXhhbXBsZXMua2V5cygpKSU1QjAlNUQlNUQpJTBBJTIwJTIwJTIwJTIwJTIzJTIwV2UlMjBkcm9wJTIwdGhlJTIwbGFzdCUyMGNodW5rJTIwaWYlMjBpdCdzJTIwc21hbGxlciUyMHRoYW4lMjBjaHVua19zaXplJTBBJTIwJTIwJTIwJTIwdG90YWxfbGVuZ3RoJTIwJTNEJTIwKHRvdGFsX2xlbmd0aCUyMCUyRiUyRiUyMGNodW5rX3NpemUpJTIwKiUyMGNodW5rX3NpemUlMEElMjAlMjAlMjAlMjAlMjMlMjBTcGxpdCUyMGJ5JTIwY2h1bmtzJTIwb2YlMjBtYXhfbGVuJTBBJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwayUzQSUyMCU1QnQlNUJpJTIwJTNBJTIwaSUyMCUyQiUyMGNodW5rX3NpemUlNUQlMjBmb3IlMjBpJTIwaW4lMjByYW5nZSgwJTJDJTIwdG90YWxfbGVuZ3RoJTJDJTIwY2h1bmtfc2l6ZSklNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBrJTJDJTIwdCUyMGluJTIwY29uY2F0ZW5hdGVkX2V4YW1wbGVzLml0ZW1zKCklMEElMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjAlMjMlMjBDcmVhdGUlMjBhJTIwbmV3JTIwbGFiZWxzJTIwY29sdW1uJTBBJTIwJTIwJTIwJTIwcmVzdWx0JTVCJTIybGFiZWxzJTIyJTVEJTIwJTNEJTIwcmVzdWx0JTVCJTIyaW5wdXRfaWRzJTIyJTVELmNvcHkoKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJlc3VsdA==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">group_texts</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-comment"># Concatenate all texts</span>
    concatenated_examples = {k: <span class="hljs-built_in">sum</span>(examples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> examples.keys()}
    <span class="hljs-comment"># Compute length of concatenated texts</span>
    total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-built_in">list</span>(examples.keys())[<span class="hljs-number">0</span>]])
    <span class="hljs-comment"># We drop the last chunk if it&#x27;s smaller than chunk_size</span>
    total_length = (total_length // chunk_size) * chunk_size
    <span class="hljs-comment"># Split by chunks of max_len</span>
    result = {
        k: [t[i : i + chunk_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, chunk_size)]
        <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
    }
    <span class="hljs-comment"># Create a new labels column</span>
    result[<span class="hljs-string">&quot;labels&quot;</span>] = result[<span class="hljs-string">&quot;input_ids&quot;</span>].copy()
    <span class="hljs-keyword">return</span> result`,wrap:!1}}),Vl=new f({props:{code:"bG1fZGF0YXNldHMlMjAlM0QlMjB0b2tlbml6ZWRfZGF0YXNldHMubWFwKGdyb3VwX3RleHRzJTJDJTIwYmF0Y2hlZCUzRFRydWUpJTBBbG1fZGF0YXNldHM=",highlighted:`lm_datasets = tokenized_datasets.<span class="hljs-built_in">map</span>(group_texts, batched=<span class="hljs-literal">True</span>)
lm_datasets`,wrap:!1}}),vl=new f({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ2F0dGVudGlvbl9tYXNrJyUyQyUyMCdpbnB1dF9pZHMnJTJDJTIwJ2xhYmVscyclMkMlMjAnd29yZF9pZHMnJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3Jvd3MlM0ElMjA2MTI4OSUwQSUyMCUyMCUyMCUyMCU3RCklMEElMjAlMjAlMjAlMjB0ZXN0JTNBJTIwRGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidhdHRlbnRpb25fbWFzayclMkMlMjAnaW5wdXRfaWRzJyUyQyUyMCdsYWJlbHMnJTJDJTIwJ3dvcmRfaWRzJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwNTk5MDUlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTIwJTIwJTIwJTIwdW5zdXBlcnZpc2VkJTNBJTIwRGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidhdHRlbnRpb25fbWFzayclMkMlMjAnaW5wdXRfaWRzJyUyQyUyMCdsYWJlbHMnJTJDJTIwJ3dvcmRfaWRzJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwMTIyOTYzJTBBJTIwJTIwJTIwJTIwJTdEKSUwQSU3RCk=",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">61289</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">59905</span>
    })
    unsupervised: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">122963</span>
    })
})`,wrap:!1}}),xl=new f({props:{code:"dG9rZW5pemVyLmRlY29kZShsbV9kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVEJTVCMSU1RCU1QiUyMmlucHV0X2lkcyUyMiU1RCk=",highlighted:'tokenizer.decode(lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;input_ids&quot;</span>])',wrap:!1}}),Nl=new f({props:{code:"JTIyLi4uLiUyMGF0Li4uLi4uLi4uLiUyMGhpZ2guJTIwYSUyMGNsYXNzaWMlMjBsaW5lJTIwJTNBJTIwaW5zcGVjdG9yJTIwJTNBJTIwaSdtJTIwaGVyZSUyMHRvJTIwc2FjayUyMG9uZSUyMG9mJTIweW91ciUyMHRlYWNoZXJzLiUyMHN0dWRlbnQlMjAlM0ElMjB3ZWxjb21lJTIwdG8lMjBicm9td2VsbCUyMGhpZ2guJTIwaSUyMGV4cGVjdCUyMHRoYXQlMjBtYW55JTIwYWR1bHRzJTIwb2YlMjBteSUyMGFnZSUyMHRoaW5rJTIwdGhhdCUyMGJyb213ZWxsJTIwaGlnaCUyMGlzJTIwZmFyJTIwZmV0Y2hlZC4lMjB3aGF0JTIwYSUyMHBpdHklMjB0aGF0JTIwaXQlMjBpc24ndCElMjAlNUJTRVAlNUQlMjAlNUJDTFMlNUQlMjBob21lbGVzc25lc3MlMjAoJTIwb3IlMjBob3VzZWxlc3NuZXNzJTIwYXMlMjBnZW9yZ2UlMjBjYXJsaW4lMjBzdGF0ZWQlMjApJTIwaGFzJTIwYmVlbiUyMGFuJTIwaXNzdWUlMjBmb3IlMjB5ZWFycyUyMGJ1dCUyMG5ldmVyJTIwYSUyMHBsYW4lMjB0byUyMGhlbHAlMjB0aG9zZSUyMG9uJTIwdGhlJTIwc3RyZWV0JTIwdGhhdCUyMHdlcmUlMjBvbmNlJTIwY29uc2lkZXJlZCUyMGh1bWFuJTIwd2hvJTIwZGlkJTIwZXZlcnl0aGluZyUyMGZyb20lMjBnb2luZyUyMHRvJTIwc2Nob29sJTJDJTIwd29yayUyQyUyMG9yJTIwdm90ZSUyMGZvciUyMHRoZSUyMG1hdHRlci4lMjBtb3N0JTIwcGVvcGxlJTIwdGhpbmslMjBvZiUyMHRoZSUyMGhvbWVsZXNzJTIy",highlighted:'<span class="hljs-string">&quot;.... at.......... high. a classic line : inspector : i&#x27;m here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn&#x27;t! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless&quot;</span>',wrap:!1}}),Ql=new f({props:{code:"dG9rZW5pemVyLmRlY29kZShsbV9kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVEJTVCMSU1RCU1QiUyMmxhYmVscyUyMiU1RCk=",highlighted:'tokenizer.decode(lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;labels&quot;</span>])',wrap:!1}}),Yl=new f({props:{code:"JTIyLi4uLiUyMGF0Li4uLi4uLi4uLiUyMGhpZ2guJTIwYSUyMGNsYXNzaWMlMjBsaW5lJTIwJTNBJTIwaW5zcGVjdG9yJTIwJTNBJTIwaSdtJTIwaGVyZSUyMHRvJTIwc2FjayUyMG9uZSUyMG9mJTIweW91ciUyMHRlYWNoZXJzLiUyMHN0dWRlbnQlMjAlM0ElMjB3ZWxjb21lJTIwdG8lMjBicm9td2VsbCUyMGhpZ2guJTIwaSUyMGV4cGVjdCUyMHRoYXQlMjBtYW55JTIwYWR1bHRzJTIwb2YlMjBteSUyMGFnZSUyMHRoaW5rJTIwdGhhdCUyMGJyb213ZWxsJTIwaGlnaCUyMGlzJTIwZmFyJTIwZmV0Y2hlZC4lMjB3aGF0JTIwYSUyMHBpdHklMjB0aGF0JTIwaXQlMjBpc24ndCElMjAlNUJTRVAlNUQlMjAlNUJDTFMlNUQlMjBob21lbGVzc25lc3MlMjAoJTIwb3IlMjBob3VzZWxlc3NuZXNzJTIwYXMlMjBnZW9yZ2UlMjBjYXJsaW4lMjBzdGF0ZWQlMjApJTIwaGFzJTIwYmVlbiUyMGFuJTIwaXNzdWUlMjBmb3IlMjB5ZWFycyUyMGJ1dCUyMG5ldmVyJTIwYSUyMHBsYW4lMjB0byUyMGhlbHAlMjB0aG9zZSUyMG9uJTIwdGhlJTIwc3RyZWV0JTIwdGhhdCUyMHdlcmUlMjBvbmNlJTIwY29uc2lkZXJlZCUyMGh1bWFuJTIwd2hvJTIwZGlkJTIwZXZlcnl0aGluZyUyMGZyb20lMjBnb2luZyUyMHRvJTIwc2Nob29sJTJDJTIwd29yayUyQyUyMG9yJTIwdm90ZSUyMGZvciUyMHRoZSUyMG1hdHRlci4lMjBtb3N0JTIwcGVvcGxlJTIwdGhpbmslMjBvZiUyMHRoZSUyMGhvbWVsZXNzJTIy",highlighted:'<span class="hljs-string">&quot;.... at.......... high. a classic line : inspector : i&#x27;m here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn&#x27;t! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless&quot;</span>',wrap:!1}}),El=new Ee({props:{title:"Fine-tuning DistilBERT with the Trainer API",local:"fine-tuning-distilbert-with-the-trainer-api",headingTag:"h2"}}),Hl=new f({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvckxhbmd1YWdlTW9kZWxpbmclMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yTGFuZ3VhZ2VNb2RlbGluZyh0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMjBtbG1fcHJvYmFiaWxpdHklM0QwLjE1KQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=<span class="hljs-number">0.15</span>)`,wrap:!1}}),ql=new f({props:{code:"c2FtcGxlcyUyMCUzRCUyMCU1QmxtX2RhdGFzZXRzJTVCJTIydHJhaW4lMjIlNUQlNUJpJTVEJTIwZm9yJTIwaSUyMGluJTIwcmFuZ2UoMiklNUQlMEFmb3IlMjBzYW1wbGUlMjBpbiUyMHNhbXBsZXMlM0ElMEElMjAlMjAlMjAlMjBfJTIwJTNEJTIwc2FtcGxlLnBvcCglMjJ3b3JkX2lkcyUyMiklMEElMEFmb3IlMjBjaHVuayUyMGluJTIwZGF0YV9jb2xsYXRvcihzYW1wbGVzKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUzQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlNUNuJyUzRSUzRSUzRSUyMCU3QnRva2VuaXplci5kZWNvZGUoY2h1bmspJTdEJyUyMik=",highlighted:`samples = [lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>)]
<span class="hljs-keyword">for</span> sample <span class="hljs-keyword">in</span> samples:
    _ = sample.pop(<span class="hljs-string">&quot;word_ids&quot;</span>)

<span class="hljs-keyword">for</span> chunk <span class="hljs-keyword">in</span> data_collator(samples)[<span class="hljs-string">&quot;input_ids&quot;</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; <span class="hljs-subst">{tokenizer.decode(chunk)}</span>&#x27;&quot;</span>)`,wrap:!1}}),Dl=new f({props:{code:"JyUzRSUzRSUzRSUyMCU1QkNMUyU1RCUyMGJyb213ZWxsJTIwJTVCTUFTSyU1RCUyMGlzJTIwYSUyMGNhcnRvb24lMjBjb21lZHkuJTIwaXQlMjByYW4lMjBhdCUyMHRoZSUyMHNhbWUlMjAlNUJNQVNLJTVEJTIwYXMlMjBzb21lJTIwb3RoZXIlMjAlNUJNQVNLJTVEJTIwYWJvdXQlMjBzY2hvb2wlMjBsaWZlJTJDJTIwJTVCTUFTSyU1RCUyMGFzJTIwJTIyJTIwdGVhY2hlcnMlMjAlMjIuJTIwJTVCTUFTSyU1RCUyMCU1Qk1BU0slNUQlMjAlNUJNQVNLJTVEJTIwaW4lMjB0aGUlMjB0ZWFjaGluZyUyMCU1Qk1BU0slNUQlMjBsZWFkJTIwJTVCTUFTSyU1RCUyMHRvJTIwYmVsaWV2ZSUyMHRoYXQlMjBicm9td2VsbCUyMGhpZ2glNUMnJTVCTUFTSyU1RCUyMHNhdGlyZSUyMGlzJTIwbXVjaCUyMGNsb3NlciUyMHRvJTIwcmVhbGl0eSUyMHRoYW4lMjBpcyUyMCUyMiUyMHRlYWNoZXJzJTIwJTIyLiUyMHRoZSUyMHNjcmFtYmxlJTIwJTVCTUFTSyU1RCUyMCU1Qk1BU0slNUQlMjBmaW5hbmNpYWxseSUyQyUyMHRoZSUyMCU1Qk1BU0slNURmdWwlMjBzdHVkZW50cyUyMHdob2duJTIwJTVCTUFTSyU1RCUyMHJpZ2h0JTIwdGhyb3VnaCUyMCU1Qk1BU0slNUQlMjBwYXRoZXRpYyUyMHRlYWNoZXJzJTVDJ3BvbXAlMkMlMjB0aGUlMjBwZXR0aW5lc3MlMjBvZiUyMHRoZSUyMHdob2xlJTIwc2l0dWF0aW9uJTJDJTIwZGlzdGluY3Rpb24lMjByZW1pbmQlMjBtZSUyMG9mJTIwdGhlJTIwc2Nob29scyUyMGklMjBrbmV3JTIwYW5kJTIwdGhlaXIlMjBzdHVkZW50cy4lMjB3aGVuJTIwaSUyMHNhdyUyMCU1Qk1BU0slNUQlMjBlcGlzb2RlJTIwaW4lMjAlNUJNQVNLJTVEJTIwYSUyMHN0dWRlbnQlMjByZXBlYXRlZGx5JTIwdHJpZWQlMjB0byUyMGJ1cm4lMjBkb3duJTIwdGhlJTIwc2Nob29sJTJDJTIwJTVCTUFTSyU1RCUyMGltbWVkaWF0ZWx5JTIwcmVjYWxsZWQuJTIwJTVCTUFTSyU1RC4uLiclMEElMEEnJTNFJTNFJTNFJTIwLi4uLiUyMGF0Li4lMjAlNUJNQVNLJTVELi4uJTIwJTVCTUFTSyU1RC4uLiUyMGhpZ2guJTIwYSUyMGNsYXNzaWMlMjBsaW5lJTIwcGx1Y2tlZCUyMGluc3BlY3RvciUyMCUzQSUyMGklNUMnJTVCTUFTSyU1RCUyMGhlcmUlMjB0byUyMCU1Qk1BU0slNUQlMjBvbmUlMjBvZiUyMHlvdXIlMjAlNUJNQVNLJTVELiUyMHN0dWRlbnQlMjAlM0ElMjB3ZWxjb21lJTIwdG8lMjBicm9td2VsbCUyMCU1Qk1BU0slNUQuJTIwaSUyMGV4cGVjdCUyMHRoYXQlMjBtYW55JTIwYWR1bHRzJTIwb2YlMjBteSUyMGFnZSUyMHRoaW5rJTIwdGhhdCUyMCU1Qk1BU0slNURtd2VsbCUyMCU1Qk1BU0slNUQlMjBpcyUyMCU1Qk1BU0slNUQlMjBmZXRjaGVkLiUyMHdoYXQlMjBhJTIwcGl0eSUyMHRoYXQlMjBpdCUyMGlzbiU1Qyd0ISUyMCU1QlNFUCU1RCUyMCU1QkNMUyU1RCUyMCU1Qk1BU0slNURuZXNzJTIwKCUyMG9yJTIwJTVCTUFTSyU1RGxlc3NuZXNzJTIwYXMlMjBnZW9yZ2UlMjAlRTUlQUUlODdpbiUyMHN0YXRlZCUyMCklRTUlODUlQUMlMjBiZWVuJTIwYW4lMjBpc3N1ZSUyMGZvciUyMHllYXJzJTIwYnV0JTIwbmV2ZXIlMjAlNUJNQVNLJTVEJTIwcGxhbiUyMHRvJTIwaGVscCUyMHRob3NlJTIwb24lMjB0aGUlMjBzdHJlZXQlMjB0aGF0JTIwd2VyZSUyMG9uY2UlMjBjb25zaWRlcmVkJTIwaHVtYW4lMjAlNUJNQVNLJTVEJTIwZGlkJTIwZXZlcnl0aGluZyUyMGZyb20lMjBnb2luZyUyMHRvJTIwc2Nob29sJTJDJTIwJTVCTUFTSyU1RCUyQyUyMCU1Qk1BU0slNUQlMjB2b3RlJTIwZm9yJTIwdGhlJTIwbWF0dGVyLiUyMG1vc3QlMjBwZW9wbGUlMjB0aGluayUyMCU1Qk1BU0slNUQlMjB0aGUlMjBob21lbGVzcyc=",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; [CLS] bromwell [MASK] is a cartoon comedy. it ran at the same [MASK] as some other [MASK] about school life, [MASK] as &quot; teachers &quot;. [MASK] [MASK] [MASK] in the teaching [MASK] lead [MASK] to believe that bromwell high\\&#x27;[MASK] satire is much closer to reality than is &quot; teachers &quot;. the scramble [MASK] [MASK] financially, the [MASK]ful students whogn [MASK] right through [MASK] pathetic teachers\\&#x27;pomp, the pettiness of the whole situation, distinction remind me of the schools i knew and their students. when i saw [MASK] episode in [MASK] a student repeatedly tried to burn down the school, [MASK] immediately recalled. [MASK]...&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; .... at.. [MASK]... [MASK]... high. a classic line plucked inspector : i\\&#x27;[MASK] here to [MASK] one of your [MASK]. student : welcome to bromwell [MASK]. i expect that many adults of my age think that [MASK]mwell [MASK] is [MASK] fetched. what a pity that it isn\\&#x27;t! [SEP] [CLS] [MASK]ness ( or [MASK]lessness as george 宇in stated )公 been an issue for years but never [MASK] plan to help those on the street that were once considered human [MASK] did everything from going to school, [MASK], [MASK] vote for the matter. most people think [MASK] the homeless&#x27;</span>`,wrap:!1}}),Ne=new Fe({props:{$$slots:{default:[ji]},$$scope:{ctx:I}}});let le=I[0]==="pt"&&On();const Wn=[bi,fi],Bt=[];function $n(e,l){return e[0]==="pt"?0:1}ce=$n(I),de=Bt[ce]=Wn[ce](I),et=new f({props:{code:"c2FtcGxlcyUyMCUzRCUyMCU1QmxtX2RhdGFzZXRzJTVCJTIydHJhaW4lMjIlNUQlNUJpJTVEJTIwZm9yJTIwaSUyMGluJTIwcmFuZ2UoMiklNUQlMEFiYXRjaCUyMCUzRCUyMHdob2xlX3dvcmRfbWFza2luZ19kYXRhX2NvbGxhdG9yKHNhbXBsZXMpJTBBJTBBZm9yJTIwY2h1bmslMjBpbiUyMGJhdGNoJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiU1Q24nJTNFJTNFJTNFJTIwJTdCdG9rZW5pemVyLmRlY29kZShjaHVuayklN0QnJTIyKQ==",highlighted:`samples = [lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>)]
batch = whole_word_masking_data_collator(samples)

<span class="hljs-keyword">for</span> chunk <span class="hljs-keyword">in</span> batch[<span class="hljs-string">&quot;input_ids&quot;</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; <span class="hljs-subst">{tokenizer.decode(chunk)}</span>&#x27;&quot;</span>)`,wrap:!1}}),lt=new f({props:{code:"JyUzRSUzRSUzRSUyMCU1QkNMUyU1RCUyMGJyb213ZWxsJTIwaGlnaCUyMGlzJTIwYSUyMGNhcnRvb24lMjBjb21lZHklMjAlNUJNQVNLJTVEJTIwaXQlMjByYW4lMjBhdCUyMHRoZSUyMHNhbWUlMjB0aW1lJTIwYXMlMjBzb21lJTIwb3RoZXIlMjBwcm9ncmFtcyUyMGFib3V0JTIwc2Nob29sJTIwbGlmZSUyQyUyMHN1Y2glMjBhcyUyMCUyMiUyMHRlYWNoZXJzJTIwJTIyLiUyMG15JTIwMzUlMjB5ZWFycyUyMGluJTIwdGhlJTIwdGVhY2hpbmclMjBwcm9mZXNzaW9uJTIwbGVhZCUyMG1lJTIwdG8lMjBiZWxpZXZlJTIwdGhhdCUyMGJyb213ZWxsJTIwaGlnaCU1QydzJTIwc2F0aXJlJTIwaXMlMjBtdWNoJTIwY2xvc2VyJTIwdG8lMjByZWFsaXR5JTIwdGhhbiUyMGlzJTIwJTIyJTIwdGVhY2hlcnMlMjAlMjIuJTIwdGhlJTIwc2NyYW1ibGUlMjB0byUyMHN1cnZpdmUlMjBmaW5hbmNpYWxseSUyQyUyMHRoZSUyMGluc2lnaHRmdWwlMjBzdHVkZW50cyUyMHdobyUyMGNhbiUyMHNlZSUyMHJpZ2h0JTIwdGhyb3VnaCUyMHRoZWlyJTIwcGF0aGV0aWMlMjB0ZWFjaGVycyU1Qydwb21wJTJDJTIwdGhlJTIwcGV0dGluZXNzJTIwb2YlMjB0aGUlMjB3aG9sZSUyMHNpdHVhdGlvbiUyQyUyMGFsbCUyMHJlbWluZCUyMG1lJTIwb2YlMjB0aGUlMjBzY2hvb2xzJTIwaSUyMGtuZXclMjBhbmQlMjB0aGVpciUyMHN0dWRlbnRzLiUyMHdoZW4lMjBpJTIwc2F3JTIwdGhlJTIwZXBpc29kZSUyMGluJTIwd2hpY2glMjBhJTIwc3R1ZGVudCUyMHJlcGVhdGVkbHklMjB0cmllZCUyMHRvJTIwYnVybiUyMGRvd24lMjB0aGUlMjBzY2hvb2wlMkMlMjBpJTIwaW1tZWRpYXRlbHklMjByZWNhbGxlZC4uLi4uJyUwQSUwQSclM0UlM0UlM0UlMjAuLi4uJTIwJTVCTUFTSyU1RCUyMCU1Qk1BU0slNUQlMjAlNUJNQVNLJTVEJTIwJTVCTUFTSyU1RC4uLi4uLi4lMjBoaWdoLiUyMGElMjBjbGFzc2ljJTIwbGluZSUyMCUzQSUyMGluc3BlY3RvciUyMCUzQSUyMGklNUMnbSUyMGhlcmUlMjB0byUyMHNhY2slMjBvbmUlMjBvZiUyMHlvdXIlMjB0ZWFjaGVycy4lMjBzdHVkZW50JTIwJTNBJTIwd2VsY29tZSUyMHRvJTIwYnJvbXdlbGwlMjBoaWdoLiUyMGklMjBleHBlY3QlMjB0aGF0JTIwbWFueSUyMGFkdWx0cyUyMG9mJTIwbXklMjBhZ2UlMjB0aGluayUyMHRoYXQlMjBicm9td2VsbCUyMGhpZ2glMjBpcyUyMGZhciUyMGZldGNoZWQuJTIwd2hhdCUyMGElMjBwaXR5JTIwdGhhdCUyMGl0JTIwaXNuJTVDJ3QhJTIwJTVCU0VQJTVEJTIwJTVCQ0xTJTVEJTIwaG9tZWxlc3NuZXNzJTIwKCUyMG9yJTIwaG91c2VsZXNzbmVzcyUyMGFzJTIwZ2VvcmdlJTIwY2FybGluJTIwc3RhdGVkJTIwKSUyMGhhcyUyMGJlZW4lMjBhbiUyMGlzc3VlJTIwZm9yJTIweWVhcnMlMjBidXQlMjBuZXZlciUyMGElMjBwbGFuJTIwdG8lMjBoZWxwJTIwdGhvc2UlMjBvbiUyMHRoZSUyMHN0cmVldCUyMHRoYXQlMjB3ZXJlJTIwb25jZSUyMGNvbnNpZGVyZWQlMjBodW1hbiUyMHdobyUyMGRpZCUyMGV2ZXJ5dGhpbmclMjBmcm9tJTIwZ29pbmclMjB0byUyMHNjaG9vbCUyQyUyMHdvcmslMkMlMjBvciUyMHZvdGUlMjBmb3IlMjB0aGUlMjBtYXR0ZXIuJTIwbW9zdCUyMHBlb3BsZSUyMHRoaW5rJTIwb2YlMjB0aGUlMjBob21lbGVzcyc=",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; [CLS] bromwell high is a cartoon comedy [MASK] it ran at the same time as some other programs about school life, such as &quot; teachers &quot;. my 35 years in the teaching profession lead me to believe that bromwell high\\&#x27;s satire is much closer to reality than is &quot; teachers &quot;. the scramble to survive financially, the insightful students who can see right through their pathetic teachers\\&#x27;pomp, the pettiness of the whole situation, all remind me of the schools i knew and their students. when i saw the episode in which a student repeatedly tried to burn down the school, i immediately recalled.....&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; .... [MASK] [MASK] [MASK] [MASK]....... high. a classic line : inspector : i\\&#x27;m here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn\\&#x27;t! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless&#x27;</span>`,wrap:!1}}),Ae=new Fe({props:{$$slots:{default:[Ii]},$$scope:{ctx:I}}}),st=new f({props:{code:"dHJhaW5fc2l6ZSUyMCUzRCUyMDEwXzAwMCUwQXRlc3Rfc2l6ZSUyMCUzRCUyMGludCgwLjElMjAqJTIwdHJhaW5fc2l6ZSklMEElMEFkb3duc2FtcGxlZF9kYXRhc2V0JTIwJTNEJTIwbG1fZGF0YXNldHMlNUIlMjJ0cmFpbiUyMiU1RC50cmFpbl90ZXN0X3NwbGl0KCUwQSUyMCUyMCUyMCUyMHRyYWluX3NpemUlM0R0cmFpbl9zaXplJTJDJTIwdGVzdF9zaXplJTNEdGVzdF9zaXplJTJDJTIwc2VlZCUzRDQyJTBBKSUwQWRvd25zYW1wbGVkX2RhdGFzZXQ=",highlighted:`train_size = <span class="hljs-number">10_000</span>
test_size = <span class="hljs-built_in">int</span>(<span class="hljs-number">0.1</span> * train_size)

downsampled_dataset = lm_datasets[<span class="hljs-string">&quot;train&quot;</span>].train_test_split(
    train_size=train_size, test_size=test_size, seed=<span class="hljs-number">42</span>
)
downsampled_dataset`,wrap:!1}}),at=new f({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ2F0dGVudGlvbl9tYXNrJyUyQyUyMCdpbnB1dF9pZHMnJTJDJTIwJ2xhYmVscyclMkMlMjAnd29yZF9pZHMnJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3Jvd3MlM0ElMjAxMDAwMCUwQSUyMCUyMCUyMCUyMCU3RCklMEElMjAlMjAlMjAlMjB0ZXN0JTNBJTIwRGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidhdHRlbnRpb25fbWFzayclMkMlMjAnaW5wdXRfaWRzJyUyQyUyMCdsYWJlbHMnJTJDJTIwJ3dvcmRfaWRzJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwMTAwMCUwQSUyMCUyMCUyMCUyMCU3RCklMEElN0Qp",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">10000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">1000</span>
    })
})`,wrap:!1}}),it=new f({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,wrap:!1}}),Mt=new f({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}});const Vn=[Ci,gi],Zt=[];function vn(e,l){return e[0]==="tf"?0:1}he=vn(I),we=Zt[he]=Vn[he](I),pt=new Ee({props:{title:"Perplexity for language models",local:"perplexity-for-language-models",headingTag:"h3"}}),yt=new Ra({props:{id:"NURcDHhYe98"}});const zn=[Gi,_i],Rt=[];function xn(e,l){return e[0]==="pt"?0:1}me=xn(I),Je=Rt[me]=zn[me](I),dt=new f({props:{code:"UGVycGxleGl0eSUzQSUyMDIxLjc1",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>Perplexity: <span class="hljs-number">21.75</span>',wrap:!1}});const Nn=[Bi,ki],Xt=[];function An(e,l){return e[0]==="pt"?0:1}ue=An(I),Ue=Xt[ue]=Nn[ue](I);const Qn=[Ri,Zi],Wt=[];function Yn(e,l){return e[0]==="pt"?0:1}Te=Yn(I),je=Wt[Te]=Qn[Te](I),mt=new f({props:{code:"UGVycGxleGl0eSUzQSUyMDExLjMy",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>Perplexity: <span class="hljs-number">11.32</span>',wrap:!1}});let N=I[0]==="pt"&&ei();Qe=new Fe({props:{$$slots:{default:[Xi]},$$scope:{ctx:I}}});let A=I[0]==="pt"&&li();return ut=new Ee({props:{title:"Using our fine-tuned model",local:"using-our-fine-tuned-model",headingTag:"h2"}}),Tt=new f({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBbWFza19maWxsZXIlMjAlM0QlMjBwaXBlbGluZSglMEElMjAlMjAlMjAlMjAlMjJmaWxsLW1hc2slMjIlMkMlMjBtb2RlbCUzRCUyMmh1Z2dpbmdmYWNlLWNvdXJzZSUyRmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkLWZpbmV0dW5lZC1pbWRiJTIyJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

mask_filler = pipeline(
    <span class="hljs-string">&quot;fill-mask&quot;</span>, model=<span class="hljs-string">&quot;huggingface-course/distilbert-base-uncased-finetuned-imdb&quot;</span>
)`,wrap:!1}}),ft=new f({props:{code:"cHJlZHMlMjAlM0QlMjBtYXNrX2ZpbGxlcih0ZXh0KSUwQSUwQWZvciUyMHByZWQlMjBpbiUyMHByZWRzJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiUzRSUzRSUzRSUyMCU3QnByZWQlNUInc2VxdWVuY2UnJTVEJTdEJTIyKQ==",highlighted:`preds = mask_filler(text)

<span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> preds:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; <span class="hljs-subst">{pred[<span class="hljs-string">&#x27;sequence&#x27;</span>]}</span>&quot;</span>)`,wrap:!1}}),bt=new f({props:{code:"JyUzRSUzRSUzRSUyMHRoaXMlMjBpcyUyMGElMjBncmVhdCUyMG1vdmllLiclMEEnJTNFJTNFJTNFJTIwdGhpcyUyMGlzJTIwYSUyMGdyZWF0JTIwZmlsbS4nJTBBJyUzRSUzRSUzRSUyMHRoaXMlMjBpcyUyMGElMjBncmVhdCUyMHN0b3J5LiclMEEnJTNFJTNFJTNFJTIwdGhpcyUyMGlzJTIwYSUyMGdyZWF0JTIwbW92aWVzLiclMEEnJTNFJTNFJTNFJTIwdGhpcyUyMGlzJTIwYSUyMGdyZWF0JTIwY2hhcmFjdGVyLic=",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great movie.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great film.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great story.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great movies.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great character.&#x27;</span>`,wrap:!1}}),gt=new Ra({props:{id:"0Oxphw4Q9fo"}}),Ye=new Fe({props:{$$slots:{default:[Wi]},$$scope:{ctx:I}}}),{c(){a=J("meta"),c=i(),n=J("p"),y=i(),d(_.$$.fragment),U=i(),d(R.$$.fragment),W=i(),X.c(),Z=i(),B=J("p"),B.textContent=j,G=i(),v=J("p"),v.textContent=Q,g=i(),$=J("p"),$.innerHTML=S,k=i(),V=J("div"),V.innerHTML=be,Ge=i(),H=J("p"),H.innerHTML=Yt,ke=i(),z=J("iframe"),se=i(),Ie=J("p"),Ie.textContent=Le,ae=i(),d(ne.$$.fragment),ie=i(),d(pe.$$.fragment),Be=i(),d(L.$$.fragment),Ze=i(),q=J("p"),q.innerHTML=Ft,Re=i(),Y=J("div"),Y.innerHTML=qe,oe=i(),ge=J("p"),ge.innerHTML=De,Me=i(),x.c(),ye=i(),D=J("p"),D.textContent=Et,Xe=i(),d(K.$$.fragment),We=i(),P=J("p"),P.innerHTML=St,$e=i(),d(O.$$.fragment),Ve=i(),ee=J("p"),ee.textContent=Ke,re=i(),E.c(),M=i(),d(b.$$.fragment),Lt=i(),Pe=J("p"),Pe.textContent=Qa,qt=i(),d(Oe.$$.fragment),Dt=i(),el=J("p"),el.innerHTML=Ya,Kt=i(),d(ll.$$.fragment),Pt=i(),d(tl.$$.fragment),Ot=i(),sl=J("p"),sl.innerHTML=Fa,es=i(),d(al.$$.fragment),ls=i(),d(nl.$$.fragment),ts=i(),il=J("p"),il.innerHTML=Ea,ss=i(),d(ve.$$.fragment),as=i(),ol=J("p"),ol.innerHTML=Sa,ns=i(),d(Ml.$$.fragment),is=i(),d(rl.$$.fragment),os=i(),pl=J("p"),pl.textContent=Ha,Ms=i(),yl=J("p"),yl.innerHTML=La,rs=i(),d(cl.$$.fragment),ps=i(),d(dl.$$.fragment),ys=i(),hl=J("p"),hl.innerHTML=qa,cs=i(),wl=J("p"),wl.innerHTML=Da,ds=i(),d(ml.$$.fragment),hs=i(),d(Jl.$$.fragment),ws=i(),ul=J("p"),ul.innerHTML=Ka,ms=i(),d(ze.$$.fragment),Js=i(),Ul=J("p"),Ul.textContent=Pa,us=i(),d(Tl.$$.fragment),Us=i(),d(xe.$$.fragment),Ts=i(),jl=J("p"),jl.textContent=Oa,js=i(),d(fl.$$.fragment),fs=i(),d(bl.$$.fragment),bs=i(),Il=J("p"),Il.textContent=en,Is=i(),d(gl.$$.fragment),gs=i(),d(Cl.$$.fragment),Cs=i(),_l=J("p"),_l.innerHTML=ln,_s=i(),d(Gl.$$.fragment),Gs=i(),d(kl.$$.fragment),ks=i(),Bl=J("p"),Bl.textContent=tn,Bs=i(),Zl=J("ul"),Zl.innerHTML=sn,Zs=i(),Rl=J("p"),Rl.textContent=an,Rs=i(),d(Xl.$$.fragment),Xs=i(),Wl=J("p"),Wl.innerHTML=nn,Ws=i(),$l=J("p"),$l.innerHTML=on,$s=i(),d(Vl.$$.fragment),Vs=i(),d(vl.$$.fragment),vs=i(),zl=J("p"),zl.innerHTML=Mn,zs=i(),d(xl.$$.fragment),xs=i(),d(Nl.$$.fragment),Ns=i(),Al=J("p"),Al.textContent=rn,As=i(),d(Ql.$$.fragment),Qs=i(),d(Yl.$$.fragment),Ys=i(),Fl=J("p"),Fl.innerHTML=pn,Fs=i(),d(El.$$.fragment),Es=i(),Sl=J("p"),Sl.innerHTML=yn,Ss=i(),d(Hl.$$.fragment),Hs=i(),Ll=J("p"),Ll.innerHTML=cn,Ls=i(),d(ql.$$.fragment),qs=i(),d(Dl.$$.fragment),Ds=i(),Kl=J("p"),Kl.innerHTML=dn,Ks=i(),d(Ne.$$.fragment),Ps=i(),le&&le.c(),$t=i(),Pl=J("p"),Pl.innerHTML=hn,Os=i(),de.c(),Vt=i(),Ol=J("p"),Ol.textContent=wn,ea=i(),d(et.$$.fragment),la=i(),d(lt.$$.fragment),ta=i(),d(Ae.$$.fragment),sa=i(),tt=J("p"),tt.innerHTML=mn,aa=i(),d(st.$$.fragment),na=i(),d(at.$$.fragment),ia=i(),nt=J("p"),nt.innerHTML=Jn,oa=i(),d(it.$$.fragment),Ma=i(),ot=J("p"),ot.textContent=un,ra=i(),d(Mt.$$.fragment),pa=i(),rt=J("p"),rt.textContent=Un,ya=i(),we.c(),vt=i(),d(pt.$$.fragment),ca=i(),d(yt.$$.fragment),da=i(),ct=J("p"),ct.textContent=Tn,ha=i(),Je.c(),zt=i(),d(dt.$$.fragment),wa=i(),ht=J("p"),ht.textContent=jn,ma=i(),Ue.c(),xt=i(),wt=J("p"),wt.textContent=fn,Ja=i(),je.c(),Nt=i(),d(mt.$$.fragment),ua=i(),Jt=J("p"),Jt.textContent=bn,Ua=i(),N&&N.c(),At=i(),d(Qe.$$.fragment),Ta=i(),A&&A.c(),Qt=i(),d(ut.$$.fragment),ja=i(),Ut=J("p"),Ut.innerHTML=In,fa=i(),d(Tt.$$.fragment),ba=i(),jt=J("p"),jt.textContent=gn,Ia=i(),d(ft.$$.fragment),ga=i(),d(bt.$$.fragment),Ca=i(),It=J("p"),It.textContent=Cn,_a=i(),d(gt.$$.fragment),Ga=i(),Ct=J("p"),Ct.innerHTML=_n,ka=i(),d(Ye.$$.fragment),Ba=i(),Ht=J("p"),this.h()},l(e){const l=Mi("svelte-u9bgzb",document.head);a=u(l,"META",{name:!0,content:!0}),l.forEach(t),c=o(e),n=u(e,"P",{}),Aa(n).forEach(t),y=o(e),h(_.$$.fragment,e),U=o(e),h(R.$$.fragment,e),W=o(e),X.l(e),Z=o(e),B=u(e,"P",{"data-svelte-h":!0}),T(B)!=="svelte-11rqc5m"&&(B.textContent=j),G=o(e),v=u(e,"P",{"data-svelte-h":!0}),T(v)!=="svelte-17pvd7s"&&(v.textContent=Q),g=o(e),$=u(e,"P",{"data-svelte-h":!0}),T($)!=="svelte-9mmj84"&&($.innerHTML=S),k=o(e),V=u(e,"DIV",{class:!0,"data-svelte-h":!0}),T(V)!=="svelte-1lvgpuv"&&(V.innerHTML=be),Ge=o(e),H=u(e,"P",{"data-svelte-h":!0}),T(H)!=="svelte-19eeiah"&&(H.innerHTML=Yt),ke=o(e),z=u(e,"IFRAME",{src:!0,frameborder:!0,height:!0,title:!0,class:!0,allow:!0,sandbox:!0}),Aa(z).forEach(t),se=o(e),Ie=u(e,"P",{"data-svelte-h":!0}),T(Ie)!=="svelte-1tyjc61"&&(Ie.textContent=Le),ae=o(e),h(ne.$$.fragment,e),ie=o(e),h(pe.$$.fragment,e),Be=o(e),h(L.$$.fragment,e),Ze=o(e),q=u(e,"P",{"data-svelte-h":!0}),T(q)!=="svelte-194w5r2"&&(q.innerHTML=Ft),Re=o(e),Y=u(e,"DIV",{class:!0,"data-svelte-h":!0}),T(Y)!=="svelte-zc90xi"&&(Y.innerHTML=qe),oe=o(e),ge=u(e,"P",{"data-svelte-h":!0}),T(ge)!=="svelte-1uiv2x4"&&(ge.innerHTML=De),Me=o(e),x.l(e),ye=o(e),D=u(e,"P",{"data-svelte-h":!0}),T(D)!=="svelte-1g3lyuf"&&(D.textContent=Et),Xe=o(e),h(K.$$.fragment,e),We=o(e),P=u(e,"P",{"data-svelte-h":!0}),T(P)!=="svelte-hat418"&&(P.innerHTML=St),$e=o(e),h(O.$$.fragment,e),Ve=o(e),ee=u(e,"P",{"data-svelte-h":!0}),T(ee)!=="svelte-1qgwydn"&&(ee.textContent=Ke),re=o(e),E.l(e),M=o(e),h(b.$$.fragment,e),Lt=o(e),Pe=u(e,"P",{"data-svelte-h":!0}),T(Pe)!=="svelte-3jwbuo"&&(Pe.textContent=Qa),qt=o(e),h(Oe.$$.fragment,e),Dt=o(e),el=u(e,"P",{"data-svelte-h":!0}),T(el)!=="svelte-e3ztt2"&&(el.innerHTML=Ya),Kt=o(e),h(ll.$$.fragment,e),Pt=o(e),h(tl.$$.fragment,e),Ot=o(e),sl=u(e,"P",{"data-svelte-h":!0}),T(sl)!=="svelte-1uxz3j4"&&(sl.innerHTML=Fa),es=o(e),h(al.$$.fragment,e),ls=o(e),h(nl.$$.fragment,e),ts=o(e),il=u(e,"P",{"data-svelte-h":!0}),T(il)!=="svelte-13i7i9s"&&(il.innerHTML=Ea),ss=o(e),h(ve.$$.fragment,e),as=o(e),ol=u(e,"P",{"data-svelte-h":!0}),T(ol)!=="svelte-1a1khye"&&(ol.innerHTML=Sa),ns=o(e),h(Ml.$$.fragment,e),is=o(e),h(rl.$$.fragment,e),os=o(e),pl=u(e,"P",{"data-svelte-h":!0}),T(pl)!=="svelte-1whaa76"&&(pl.textContent=Ha),Ms=o(e),yl=u(e,"P",{"data-svelte-h":!0}),T(yl)!=="svelte-18o7drt"&&(yl.innerHTML=La),rs=o(e),h(cl.$$.fragment,e),ps=o(e),h(dl.$$.fragment,e),ys=o(e),hl=u(e,"P",{"data-svelte-h":!0}),T(hl)!=="svelte-1btid30"&&(hl.innerHTML=qa),cs=o(e),wl=u(e,"P",{"data-svelte-h":!0}),T(wl)!=="svelte-188lv63"&&(wl.innerHTML=Da),ds=o(e),h(ml.$$.fragment,e),hs=o(e),h(Jl.$$.fragment,e),ws=o(e),ul=u(e,"P",{"data-svelte-h":!0}),T(ul)!=="svelte-x8g16c"&&(ul.innerHTML=Ka),ms=o(e),h(ze.$$.fragment,e),Js=o(e),Ul=u(e,"P",{"data-svelte-h":!0}),T(Ul)!=="svelte-xnt0eg"&&(Ul.textContent=Pa),us=o(e),h(Tl.$$.fragment,e),Us=o(e),h(xe.$$.fragment,e),Ts=o(e),jl=u(e,"P",{"data-svelte-h":!0}),T(jl)!=="svelte-b36s9w"&&(jl.textContent=Oa),js=o(e),h(fl.$$.fragment,e),fs=o(e),h(bl.$$.fragment,e),bs=o(e),Il=u(e,"P",{"data-svelte-h":!0}),T(Il)!=="svelte-mssxs4"&&(Il.textContent=en),Is=o(e),h(gl.$$.fragment,e),gs=o(e),h(Cl.$$.fragment,e),Cs=o(e),_l=u(e,"P",{"data-svelte-h":!0}),T(_l)!=="svelte-1qtwsnu"&&(_l.innerHTML=ln),_s=o(e),h(Gl.$$.fragment,e),Gs=o(e),h(kl.$$.fragment,e),ks=o(e),Bl=u(e,"P",{"data-svelte-h":!0}),T(Bl)!=="svelte-1y1gzji"&&(Bl.textContent=tn),Bs=o(e),Zl=u(e,"UL",{"data-svelte-h":!0}),T(Zl)!=="svelte-187zn68"&&(Zl.innerHTML=sn),Zs=o(e),Rl=u(e,"P",{"data-svelte-h":!0}),T(Rl)!=="svelte-jf6vcc"&&(Rl.textContent=an),Rs=o(e),h(Xl.$$.fragment,e),Xs=o(e),Wl=u(e,"P",{"data-svelte-h":!0}),T(Wl)!=="svelte-5629lr"&&(Wl.innerHTML=nn),Ws=o(e),$l=u(e,"P",{"data-svelte-h":!0}),T($l)!=="svelte-18n5xxn"&&($l.innerHTML=on),$s=o(e),h(Vl.$$.fragment,e),Vs=o(e),h(vl.$$.fragment,e),vs=o(e),zl=u(e,"P",{"data-svelte-h":!0}),T(zl)!=="svelte-sr887y"&&(zl.innerHTML=Mn),zs=o(e),h(xl.$$.fragment,e),xs=o(e),h(Nl.$$.fragment,e),Ns=o(e),Al=u(e,"P",{"data-svelte-h":!0}),T(Al)!=="svelte-kah7qo"&&(Al.textContent=rn),As=o(e),h(Ql.$$.fragment,e),Qs=o(e),h(Yl.$$.fragment,e),Ys=o(e),Fl=u(e,"P",{"data-svelte-h":!0}),T(Fl)!=="svelte-1iew6is"&&(Fl.innerHTML=pn),Fs=o(e),h(El.$$.fragment,e),Es=o(e),Sl=u(e,"P",{"data-svelte-h":!0}),T(Sl)!=="svelte-jaj1rc"&&(Sl.innerHTML=yn),Ss=o(e),h(Hl.$$.fragment,e),Hs=o(e),Ll=u(e,"P",{"data-svelte-h":!0}),T(Ll)!=="svelte-1dxdd6j"&&(Ll.innerHTML=cn),Ls=o(e),h(ql.$$.fragment,e),qs=o(e),h(Dl.$$.fragment,e),Ds=o(e),Kl=u(e,"P",{"data-svelte-h":!0}),T(Kl)!=="svelte-tf324a"&&(Kl.innerHTML=dn),Ks=o(e),h(Ne.$$.fragment,e),Ps=o(e),le&&le.l(e),$t=o(e),Pl=u(e,"P",{"data-svelte-h":!0}),T(Pl)!=="svelte-171x2x4"&&(Pl.innerHTML=hn),Os=o(e),de.l(e),Vt=o(e),Ol=u(e,"P",{"data-svelte-h":!0}),T(Ol)!=="svelte-1xi9xjp"&&(Ol.textContent=wn),ea=o(e),h(et.$$.fragment,e),la=o(e),h(lt.$$.fragment,e),ta=o(e),h(Ae.$$.fragment,e),sa=o(e),tt=u(e,"P",{"data-svelte-h":!0}),T(tt)!=="svelte-1hbi27y"&&(tt.innerHTML=mn),aa=o(e),h(st.$$.fragment,e),na=o(e),h(at.$$.fragment,e),ia=o(e),nt=u(e,"P",{"data-svelte-h":!0}),T(nt)!=="svelte-su015l"&&(nt.innerHTML=Jn),oa=o(e),h(it.$$.fragment,e),Ma=o(e),ot=u(e,"P",{"data-svelte-h":!0}),T(ot)!=="svelte-3fqg55"&&(ot.textContent=un),ra=o(e),h(Mt.$$.fragment,e),pa=o(e),rt=u(e,"P",{"data-svelte-h":!0}),T(rt)!=="svelte-1sc64uc"&&(rt.textContent=Un),ya=o(e),we.l(e),vt=o(e),h(pt.$$.fragment,e),ca=o(e),h(yt.$$.fragment,e),da=o(e),ct=u(e,"P",{"data-svelte-h":!0}),T(ct)!=="svelte-now2dv"&&(ct.textContent=Tn),ha=o(e),Je.l(e),zt=o(e),h(dt.$$.fragment,e),wa=o(e),ht=u(e,"P",{"data-svelte-h":!0}),T(ht)!=="svelte-n4gsqr"&&(ht.textContent=jn),ma=o(e),Ue.l(e),xt=o(e),wt=u(e,"P",{"data-svelte-h":!0}),T(wt)!=="svelte-e645ia"&&(wt.textContent=fn),Ja=o(e),je.l(e),Nt=o(e),h(mt.$$.fragment,e),ua=o(e),Jt=u(e,"P",{"data-svelte-h":!0}),T(Jt)!=="svelte-14b9dr5"&&(Jt.textContent=bn),Ua=o(e),N&&N.l(e),At=o(e),h(Qe.$$.fragment,e),Ta=o(e),A&&A.l(e),Qt=o(e),h(ut.$$.fragment,e),ja=o(e),Ut=u(e,"P",{"data-svelte-h":!0}),T(Ut)!=="svelte-azcu8o"&&(Ut.innerHTML=In),fa=o(e),h(Tt.$$.fragment,e),ba=o(e),jt=u(e,"P",{"data-svelte-h":!0}),T(jt)!=="svelte-12ftkgw"&&(jt.textContent=gn),Ia=o(e),h(ft.$$.fragment,e),ga=o(e),h(bt.$$.fragment,e),Ca=o(e),It=u(e,"P",{"data-svelte-h":!0}),T(It)!=="svelte-8cy1ke"&&(It.textContent=Cn),_a=o(e),h(gt.$$.fragment,e),Ga=o(e),Ct=u(e,"P",{"data-svelte-h":!0}),T(Ct)!=="svelte-w2yc62"&&(Ct.innerHTML=_n),ka=o(e),h(Ye.$$.fragment,e),Ba=o(e),Ht=u(e,"P",{}),Aa(Ht).forEach(t),this.h()},h(){fe(a,"name","hf:doc:metadata"),fe(a,"content",Vi),fe(V,"class","flex justify-center"),ai(z.src,He="https://course-demos-distilbert-base-uncased-finetuned-imdb.hf.space")||fe(z,"src",He),fe(z,"frameborder","0"),fe(z,"height","300"),fe(z,"title","Gradio app"),fe(z,"class","block dark:hidden container p-0 flex-grow space-iframe"),fe(z,"allow","accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking"),fe(z,"sandbox","allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"),fe(Y,"class","flex justify-center")},m(e,l){ri(document.head,a),s(e,c,l),s(e,n,l),s(e,y,l),w(_,e,l),s(e,U,l),w(R,e,l),s(e,W,l),_t[C].m(e,l),s(e,Z,l),s(e,B,l),s(e,G,l),s(e,v,l),s(e,g,l),s(e,$,l),s(e,k,l),s(e,V,l),s(e,Ge,l),s(e,H,l),s(e,ke,l),s(e,z,l),s(e,se,l),s(e,Ie,l),s(e,ae,l),w(ne,e,l),s(e,ie,l),w(pe,e,l),s(e,Be,l),w(L,e,l),s(e,Ze,l),s(e,q,l),s(e,Re,l),s(e,Y,l),s(e,oe,l),s(e,ge,l),s(e,Me,l),Gt[F].m(e,l),s(e,ye,l),s(e,D,l),s(e,Xe,l),w(K,e,l),s(e,We,l),s(e,P,l),s(e,$e,l),w(O,e,l),s(e,Ve,l),s(e,ee,l),s(e,re,l),kt[te].m(e,l),s(e,M,l),w(b,e,l),s(e,Lt,l),s(e,Pe,l),s(e,qt,l),w(Oe,e,l),s(e,Dt,l),s(e,el,l),s(e,Kt,l),w(ll,e,l),s(e,Pt,l),w(tl,e,l),s(e,Ot,l),s(e,sl,l),s(e,es,l),w(al,e,l),s(e,ls,l),w(nl,e,l),s(e,ts,l),s(e,il,l),s(e,ss,l),w(ve,e,l),s(e,as,l),s(e,ol,l),s(e,ns,l),w(Ml,e,l),s(e,is,l),w(rl,e,l),s(e,os,l),s(e,pl,l),s(e,Ms,l),s(e,yl,l),s(e,rs,l),w(cl,e,l),s(e,ps,l),w(dl,e,l),s(e,ys,l),s(e,hl,l),s(e,cs,l),s(e,wl,l),s(e,ds,l),w(ml,e,l),s(e,hs,l),w(Jl,e,l),s(e,ws,l),s(e,ul,l),s(e,ms,l),w(ze,e,l),s(e,Js,l),s(e,Ul,l),s(e,us,l),w(Tl,e,l),s(e,Us,l),w(xe,e,l),s(e,Ts,l),s(e,jl,l),s(e,js,l),w(fl,e,l),s(e,fs,l),w(bl,e,l),s(e,bs,l),s(e,Il,l),s(e,Is,l),w(gl,e,l),s(e,gs,l),w(Cl,e,l),s(e,Cs,l),s(e,_l,l),s(e,_s,l),w(Gl,e,l),s(e,Gs,l),w(kl,e,l),s(e,ks,l),s(e,Bl,l),s(e,Bs,l),s(e,Zl,l),s(e,Zs,l),s(e,Rl,l),s(e,Rs,l),w(Xl,e,l),s(e,Xs,l),s(e,Wl,l),s(e,Ws,l),s(e,$l,l),s(e,$s,l),w(Vl,e,l),s(e,Vs,l),w(vl,e,l),s(e,vs,l),s(e,zl,l),s(e,zs,l),w(xl,e,l),s(e,xs,l),w(Nl,e,l),s(e,Ns,l),s(e,Al,l),s(e,As,l),w(Ql,e,l),s(e,Qs,l),w(Yl,e,l),s(e,Ys,l),s(e,Fl,l),s(e,Fs,l),w(El,e,l),s(e,Es,l),s(e,Sl,l),s(e,Ss,l),w(Hl,e,l),s(e,Hs,l),s(e,Ll,l),s(e,Ls,l),w(ql,e,l),s(e,qs,l),w(Dl,e,l),s(e,Ds,l),s(e,Kl,l),s(e,Ks,l),w(Ne,e,l),s(e,Ps,l),le&&le.m(e,l),s(e,$t,l),s(e,Pl,l),s(e,Os,l),Bt[ce].m(e,l),s(e,Vt,l),s(e,Ol,l),s(e,ea,l),w(et,e,l),s(e,la,l),w(lt,e,l),s(e,ta,l),w(Ae,e,l),s(e,sa,l),s(e,tt,l),s(e,aa,l),w(st,e,l),s(e,na,l),w(at,e,l),s(e,ia,l),s(e,nt,l),s(e,oa,l),w(it,e,l),s(e,Ma,l),s(e,ot,l),s(e,ra,l),w(Mt,e,l),s(e,pa,l),s(e,rt,l),s(e,ya,l),Zt[he].m(e,l),s(e,vt,l),w(pt,e,l),s(e,ca,l),w(yt,e,l),s(e,da,l),s(e,ct,l),s(e,ha,l),Rt[me].m(e,l),s(e,zt,l),w(dt,e,l),s(e,wa,l),s(e,ht,l),s(e,ma,l),Xt[ue].m(e,l),s(e,xt,l),s(e,wt,l),s(e,Ja,l),Wt[Te].m(e,l),s(e,Nt,l),w(mt,e,l),s(e,ua,l),s(e,Jt,l),s(e,Ua,l),N&&N.m(e,l),s(e,At,l),w(Qe,e,l),s(e,Ta,l),A&&A.m(e,l),s(e,Qt,l),w(ut,e,l),s(e,ja,l),s(e,Ut,l),s(e,fa,l),w(Tt,e,l),s(e,ba,l),s(e,jt,l),s(e,Ia,l),w(ft,e,l),s(e,ga,l),w(bt,e,l),s(e,Ca,l),s(e,It,l),s(e,_a,l),w(gt,e,l),s(e,Ga,l),s(e,Ct,l),s(e,ka,l),w(Ye,e,l),s(e,Ba,l),s(e,Ht,l),Za=!0},p(e,[l]){const Fn={};l&1&&(Fn.fw=e[0]),_.$set(Fn);let Xa=C;C=kn(e),C!==Xa&&(_e(),p(_t[Xa],1,1,()=>{_t[Xa]=null}),Ce(),X=_t[C],X||(X=_t[C]=Gn[C](e),X.c()),r(X,1),X.m(Z.parentNode,Z));const En={};l&2&&(En.$$scope={dirty:l,ctx:e}),pe.$set(En);let Wa=F;F=Zn(e),F!==Wa&&(_e(),p(Gt[Wa],1,1,()=>{Gt[Wa]=null}),Ce(),x=Gt[F],x||(x=Gt[F]=Bn[F](e),x.c()),r(x,1),x.m(ye.parentNode,ye));let $a=te;te=Xn(e),te!==$a&&(_e(),p(kt[$a],1,1,()=>{kt[$a]=null}),Ce(),E=kt[te],E||(E=kt[te]=Rn[te](e),E.c()),r(E,1),E.m(M.parentNode,M));const Sn={};l&2&&(Sn.$$scope={dirty:l,ctx:e}),ve.$set(Sn);const Hn={};l&2&&(Hn.$$scope={dirty:l,ctx:e}),ze.$set(Hn);const Ln={};l&2&&(Ln.$$scope={dirty:l,ctx:e}),xe.$set(Ln);const qn={};l&2&&(qn.$$scope={dirty:l,ctx:e}),Ne.$set(qn),e[0]==="pt"?le||(le=On(),le.c(),le.m($t.parentNode,$t)):le&&(le.d(1),le=null);let Va=ce;ce=$n(e),ce!==Va&&(_e(),p(Bt[Va],1,1,()=>{Bt[Va]=null}),Ce(),de=Bt[ce],de||(de=Bt[ce]=Wn[ce](e),de.c()),r(de,1),de.m(Vt.parentNode,Vt));const Dn={};l&2&&(Dn.$$scope={dirty:l,ctx:e}),Ae.$set(Dn);let va=he;he=vn(e),he!==va&&(_e(),p(Zt[va],1,1,()=>{Zt[va]=null}),Ce(),we=Zt[he],we||(we=Zt[he]=Vn[he](e),we.c()),r(we,1),we.m(vt.parentNode,vt));let za=me;me=xn(e),me!==za&&(_e(),p(Rt[za],1,1,()=>{Rt[za]=null}),Ce(),Je=Rt[me],Je||(Je=Rt[me]=zn[me](e),Je.c()),r(Je,1),Je.m(zt.parentNode,zt));let xa=ue;ue=An(e),ue!==xa&&(_e(),p(Xt[xa],1,1,()=>{Xt[xa]=null}),Ce(),Ue=Xt[ue],Ue||(Ue=Xt[ue]=Nn[ue](e),Ue.c()),r(Ue,1),Ue.m(xt.parentNode,xt));let Na=Te;Te=Yn(e),Te!==Na&&(_e(),p(Wt[Na],1,1,()=>{Wt[Na]=null}),Ce(),je=Wt[Te],je||(je=Wt[Te]=Qn[Te](e),je.c()),r(je,1),je.m(Nt.parentNode,Nt)),e[0]==="pt"?N?l&1&&r(N,1):(N=ei(),N.c(),r(N,1),N.m(At.parentNode,At)):N&&(_e(),p(N,1,1,()=>{N=null}),Ce());const Kn={};l&2&&(Kn.$$scope={dirty:l,ctx:e}),Qe.$set(Kn),e[0]==="pt"?A?l&1&&r(A,1):(A=li(),A.c(),r(A,1),A.m(Qt.parentNode,Qt)):A&&(_e(),p(A,1,1,()=>{A=null}),Ce());const Pn={};l&2&&(Pn.$$scope={dirty:l,ctx:e}),Ye.$set(Pn)},i(e){Za||(r(_.$$.fragment,e),r(R.$$.fragment,e),r(X),r(ne.$$.fragment,e),r(pe.$$.fragment,e),r(L.$$.fragment,e),r(x),r(K.$$.fragment,e),r(O.$$.fragment,e),r(E),r(b.$$.fragment,e),r(Oe.$$.fragment,e),r(ll.$$.fragment,e),r(tl.$$.fragment,e),r(al.$$.fragment,e),r(nl.$$.fragment,e),r(ve.$$.fragment,e),r(Ml.$$.fragment,e),r(rl.$$.fragment,e),r(cl.$$.fragment,e),r(dl.$$.fragment,e),r(ml.$$.fragment,e),r(Jl.$$.fragment,e),r(ze.$$.fragment,e),r(Tl.$$.fragment,e),r(xe.$$.fragment,e),r(fl.$$.fragment,e),r(bl.$$.fragment,e),r(gl.$$.fragment,e),r(Cl.$$.fragment,e),r(Gl.$$.fragment,e),r(kl.$$.fragment,e),r(Xl.$$.fragment,e),r(Vl.$$.fragment,e),r(vl.$$.fragment,e),r(xl.$$.fragment,e),r(Nl.$$.fragment,e),r(Ql.$$.fragment,e),r(Yl.$$.fragment,e),r(El.$$.fragment,e),r(Hl.$$.fragment,e),r(ql.$$.fragment,e),r(Dl.$$.fragment,e),r(Ne.$$.fragment,e),r(de),r(et.$$.fragment,e),r(lt.$$.fragment,e),r(Ae.$$.fragment,e),r(st.$$.fragment,e),r(at.$$.fragment,e),r(it.$$.fragment,e),r(Mt.$$.fragment,e),r(we),r(pt.$$.fragment,e),r(yt.$$.fragment,e),r(Je),r(dt.$$.fragment,e),r(Ue),r(je),r(mt.$$.fragment,e),r(N),r(Qe.$$.fragment,e),r(A),r(ut.$$.fragment,e),r(Tt.$$.fragment,e),r(ft.$$.fragment,e),r(bt.$$.fragment,e),r(gt.$$.fragment,e),r(Ye.$$.fragment,e),Za=!0)},o(e){p(_.$$.fragment,e),p(R.$$.fragment,e),p(X),p(ne.$$.fragment,e),p(pe.$$.fragment,e),p(L.$$.fragment,e),p(x),p(K.$$.fragment,e),p(O.$$.fragment,e),p(E),p(b.$$.fragment,e),p(Oe.$$.fragment,e),p(ll.$$.fragment,e),p(tl.$$.fragment,e),p(al.$$.fragment,e),p(nl.$$.fragment,e),p(ve.$$.fragment,e),p(Ml.$$.fragment,e),p(rl.$$.fragment,e),p(cl.$$.fragment,e),p(dl.$$.fragment,e),p(ml.$$.fragment,e),p(Jl.$$.fragment,e),p(ze.$$.fragment,e),p(Tl.$$.fragment,e),p(xe.$$.fragment,e),p(fl.$$.fragment,e),p(bl.$$.fragment,e),p(gl.$$.fragment,e),p(Cl.$$.fragment,e),p(Gl.$$.fragment,e),p(kl.$$.fragment,e),p(Xl.$$.fragment,e),p(Vl.$$.fragment,e),p(vl.$$.fragment,e),p(xl.$$.fragment,e),p(Nl.$$.fragment,e),p(Ql.$$.fragment,e),p(Yl.$$.fragment,e),p(El.$$.fragment,e),p(Hl.$$.fragment,e),p(ql.$$.fragment,e),p(Dl.$$.fragment,e),p(Ne.$$.fragment,e),p(de),p(et.$$.fragment,e),p(lt.$$.fragment,e),p(Ae.$$.fragment,e),p(st.$$.fragment,e),p(at.$$.fragment,e),p(it.$$.fragment,e),p(Mt.$$.fragment,e),p(we),p(pt.$$.fragment,e),p(yt.$$.fragment,e),p(Je),p(dt.$$.fragment,e),p(Ue),p(je),p(mt.$$.fragment,e),p(N),p(Qe.$$.fragment,e),p(A),p(ut.$$.fragment,e),p(Tt.$$.fragment,e),p(ft.$$.fragment,e),p(bt.$$.fragment,e),p(gt.$$.fragment,e),p(Ye.$$.fragment,e),Za=!1},d(e){e&&(t(c),t(n),t(y),t(U),t(W),t(Z),t(B),t(G),t(v),t(g),t($),t(k),t(V),t(Ge),t(H),t(ke),t(z),t(se),t(Ie),t(ae),t(ie),t(Be),t(Ze),t(q),t(Re),t(Y),t(oe),t(ge),t(Me),t(ye),t(D),t(Xe),t(We),t(P),t($e),t(Ve),t(ee),t(re),t(M),t(Lt),t(Pe),t(qt),t(Dt),t(el),t(Kt),t(Pt),t(Ot),t(sl),t(es),t(ls),t(ts),t(il),t(ss),t(as),t(ol),t(ns),t(is),t(os),t(pl),t(Ms),t(yl),t(rs),t(ps),t(ys),t(hl),t(cs),t(wl),t(ds),t(hs),t(ws),t(ul),t(ms),t(Js),t(Ul),t(us),t(Us),t(Ts),t(jl),t(js),t(fs),t(bs),t(Il),t(Is),t(gs),t(Cs),t(_l),t(_s),t(Gs),t(ks),t(Bl),t(Bs),t(Zl),t(Zs),t(Rl),t(Rs),t(Xs),t(Wl),t(Ws),t($l),t($s),t(Vs),t(vs),t(zl),t(zs),t(xs),t(Ns),t(Al),t(As),t(Qs),t(Ys),t(Fl),t(Fs),t(Es),t(Sl),t(Ss),t(Hs),t(Ll),t(Ls),t(qs),t(Ds),t(Kl),t(Ks),t(Ps),t($t),t(Pl),t(Os),t(Vt),t(Ol),t(ea),t(la),t(ta),t(sa),t(tt),t(aa),t(na),t(ia),t(nt),t(oa),t(Ma),t(ot),t(ra),t(pa),t(rt),t(ya),t(vt),t(ca),t(da),t(ct),t(ha),t(zt),t(wa),t(ht),t(ma),t(xt),t(wt),t(Ja),t(Nt),t(ua),t(Jt),t(Ua),t(At),t(Ta),t(Qt),t(ja),t(Ut),t(fa),t(ba),t(jt),t(Ia),t(ga),t(Ca),t(It),t(_a),t(Ga),t(Ct),t(ka),t(Ba),t(Ht)),t(a),m(_,e),m(R,e),_t[C].d(e),m(ne,e),m(pe,e),m(L,e),Gt[F].d(e),m(K,e),m(O,e),kt[te].d(e),m(b,e),m(Oe,e),m(ll,e),m(tl,e),m(al,e),m(nl,e),m(ve,e),m(Ml,e),m(rl,e),m(cl,e),m(dl,e),m(ml,e),m(Jl,e),m(ze,e),m(Tl,e),m(xe,e),m(fl,e),m(bl,e),m(gl,e),m(Cl,e),m(Gl,e),m(kl,e),m(Xl,e),m(Vl,e),m(vl,e),m(xl,e),m(Nl,e),m(Ql,e),m(Yl,e),m(El,e),m(Hl,e),m(ql,e),m(Dl,e),m(Ne,e),le&&le.d(e),Bt[ce].d(e),m(et,e),m(lt,e),m(Ae,e),m(st,e),m(at,e),m(it,e),m(Mt,e),Zt[he].d(e),m(pt,e),m(yt,e),Rt[me].d(e),m(dt,e),Xt[ue].d(e),Wt[Te].d(e),m(mt,e),N&&N.d(e),m(Qe,e),A&&A.d(e),m(ut,e),m(Tt,e),m(ft,e),m(bt,e),m(gt,e),m(Ye,e)}}}const Vi='{"title":"Fine-tuning a masked language model","local":"fine-tuning-a-masked-language-model","sections":[{"title":"Picking a pretrained model for masked language modeling","local":"picking-a-pretrained-model-for-masked-language-modeling","sections":[],"depth":2},{"title":"The dataset","local":"the-dataset","sections":[],"depth":2},{"title":"Preprocessing the data","local":"preprocessing-the-data","sections":[],"depth":2},{"title":"Fine-tuning DistilBERT with the Trainer API","local":"fine-tuning-distilbert-with-the-trainer-api","sections":[{"title":"Perplexity for language models","local":"perplexity-for-language-models","sections":[],"depth":3}],"depth":2},{"title":"Fine-tuning DistilBERT with 🤗 Accelerate","local":"fine-tuning-distilbert-with-accelerate","sections":[],"depth":2},{"title":"Using our fine-tuned model","local":"using-our-fine-tuned-model","sections":[],"depth":2}],"depth":1}';function vi(I,a,c){let n="pt";return ni(()=>{const y=new URLSearchParams(window.location.search);c(0,n=y.get("fw")||"pt")}),[n]}class Si extends ii{constructor(a){super(),oi(this,a,vi,$i,si,{})}}export{Si as component};
