import{s as tM,f as aM,o as nM,n as Sl}from"../chunks/scheduler.37c15a92.js";import{S as MM,i as pM,g as d,s as M,r as T,A as iM,h as m,f as e,c as p,j as Qa,u as c,x as u,k as bl,y as yM,a as t,v as o,t as J,b as _l,d as y,w,p as Gl}from"../chunks/index.2bf4358c.js";import{T as Fl}from"../chunks/Tip.363c041f.js";import{Y as Ra}from"../chunks/Youtube.1e50a667.js";import{C as b}from"../chunks/CodeBlock.4f5fc1ad.js";import{D as eM}from"../chunks/DocNotebookDropdown.d9dfeb24.js";import{F as JM}from"../chunks/FrameworkSwitchCourse.8d4d4ab6.js";import{H as El}from"../chunks/Heading.8ada512a.js";function rM(I){let a,U;return a=new eM({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter7/section3_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter7/section3_tf.ipynb"}]}}),{c(){T(a.$$.fragment)},l(n){c(a.$$.fragment,n)},m(n,r){o(a,n,r),U=!0},i(n){U||(y(a.$$.fragment,n),U=!0)},o(n){J(a.$$.fragment,n),U=!1},d(n){w(a,n)}}}function UM(I){let a,U;return a=new eM({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter7/section3_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter7/section3_pt.ipynb"}]}}),{c(){T(a.$$.fragment)},l(n){c(a.$$.fragment,n)},m(n,r){o(a,n,r),U=!0},i(n){U||(y(a.$$.fragment,n),U=!0)},o(n){J(a.$$.fragment,n),U=!1},d(n){w(a,n)}}}function TM(I){let a,U='🙋 如果您对“掩码语言建模”和“预训练模型”这两个术语感到陌生, 请查看<a href="/course/chapter1">第一章</a>, 我们在其中解释了所有这些核心概念, 并附有视频!';return{c(){a=d("p"),a.innerHTML=U},l(n){a=m(n,"P",{"data-svelte-h":!0}),u(a)!=="svelte-umnnww"&&(a.innerHTML=U)},m(n,r){t(n,a,r)},p:Sl,d(n){n&&e(a)}}}function cM(I){let a,U="让我们继续, 使用 <code>AutoModelForMaskedLM</code> 类下载 DistilBERT:",n,r,G,j,R="我们可以通过调用  <code>summary()</code> 方法查看模型有多少参数:",W,_,X,k,g;return r=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yTWFza2VkTE0lMEElMEFtb2RlbF9jaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQlMjIlMEFtb2RlbCUyMCUzRCUyMFRGQXV0b01vZGVsRm9yTWFza2VkTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2NoZWNrcG9pbnQp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForMaskedLM

model_checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>
model = TFAutoModelForMaskedLM.from_pretrained(model_checkpoint)`,wrap:!1}}),_=new b({props:{code:"bW9kZWwobW9kZWwuZHVtbXlfaW5wdXRzKSUyMCUyMCUyMyUyMEJ1aWxkJTIwdGhlJTIwbW9kZWwlMEFtb2RlbC5zdW1tYXJ5KCk=",highlighted:`model(model.dummy_inputs)  <span class="hljs-comment"># Build the model</span>
model.summary()`,wrap:!1}}),k=new b({props:{code:"TW9kZWwlM0ElMjAlMjJ0Zl9kaXN0aWxfYmVydF9mb3JfbWFza2VkX2xtJTIyJTBBX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX18lMEFMYXllciUyMCh0eXBlKSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME91dHB1dCUyMFNoYXBlJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwUGFyYW0lMjAlMjMlMjAlMjAlMjAlMEElM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMEFkaXN0aWxiZXJ0JTIwKFRGRGlzdGlsQmVydE1haW4lMjBtdWx0aXBsZSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDY2MzYyODgwJTIwJTIwJTBBX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX18lMEF2b2NhYl90cmFuc2Zvcm0lMjAoRGVuc2UpJTIwJTIwJTIwJTIwJTIwJTIwbXVsdGlwbGUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjA1OTA1OTIlMjAlMjAlMjAlMjAlMEFfX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fXyUwQXZvY2FiX2xheWVyX25vcm0lMjAoTGF5ZXJOb3JtYSUyMG11bHRpcGxlJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwMTUzNiUyMCUyMCUyMCUyMCUyMCUyMCUwQV9fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fJTBBdm9jYWJfcHJvamVjdG9yJTIwKFRGRGlzdGlsQmVyJTIwbXVsdGlwbGUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAyMzg2NjE3MCUyMCUyMCUwQSUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUwQVRvdGFsJTIwcGFyYW1zJTNBJTIwNjYlMkM5ODUlMkM1MzAlMEFUcmFpbmFibGUlMjBwYXJhbXMlM0ElMjA2NiUyQzk4NSUyQzUzMCUwQU5vbi10cmFpbmFibGUlMjBwYXJhbXMlM0ElMjAwJTBBX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX18=",highlighted:`Model: <span class="hljs-string">&quot;tf_distil_bert_for_masked_lm&quot;</span>
_________________________________________________________________
Layer (<span class="hljs-built_in">type</span>)                 Output Shape              Param <span class="hljs-comment">#   </span>
=================================================================
distilbert (TFDistilBertMain multiple                  <span class="hljs-number">66362880</span>  
_________________________________________________________________
vocab_transform (Dense)      multiple                  <span class="hljs-number">590592</span>    
_________________________________________________________________
vocab_layer_norm (LayerNorma multiple                  <span class="hljs-number">1536</span>      
_________________________________________________________________
vocab_projector (TFDistilBer multiple                  <span class="hljs-number">23866170</span>  
=================================================================
Total params: <span class="hljs-number">66</span>,<span class="hljs-number">985</span>,<span class="hljs-number">530</span>
Trainable params: <span class="hljs-number">66</span>,<span class="hljs-number">985</span>,<span class="hljs-number">530</span>
Non-trainable params: <span class="hljs-number">0</span>
_________________________________________________________________`,wrap:!1}}),{c(){a=d("p"),a.innerHTML=U,n=M(),T(r.$$.fragment),G=M(),j=d("p"),j.innerHTML=R,W=M(),T(_.$$.fragment),X=M(),T(k.$$.fragment)},l(h){a=m(h,"P",{"data-svelte-h":!0}),u(a)!=="svelte-j6x8ju"&&(a.innerHTML=U),n=p(h),c(r.$$.fragment,h),G=p(h),j=m(h,"P",{"data-svelte-h":!0}),u(j)!=="svelte-sz3029"&&(j.innerHTML=R),W=p(h),c(_.$$.fragment,h),X=p(h),c(k.$$.fragment,h)},m(h,B){t(h,a,B),t(h,n,B),o(r,h,B),t(h,G,B),t(h,j,B),t(h,W,B),o(_,h,B),t(h,X,B),o(k,h,B),g=!0},i(h){g||(y(r.$$.fragment,h),y(_.$$.fragment,h),y(k.$$.fragment,h),g=!0)},o(h){J(r.$$.fragment,h),J(_.$$.fragment,h),J(k.$$.fragment,h),g=!1},d(h){h&&(e(a),e(n),e(G),e(j),e(W),e(X)),w(r,h),w(_,h),w(k,h)}}}function oM(I){let a,U="让我们继续, 使用 <code>AutoModelForMaskedLM</code> 类下载 DistilBERT:",n,r,G,j,R="我们可以通过调用 <code>num_parameters()</code> 方法查看模型有多少参数:",W,_,X,k,g;return r=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvck1hc2tlZExNJTBBJTBBbW9kZWxfY2hlY2twb2ludCUyMCUzRCUyMCUyMmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JNYXNrZWRMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfY2hlY2twb2ludCk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM

model_checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>
model = AutoModelForMaskedLM.from_pretrained(model_checkpoint)`,wrap:!1}}),_=new b({props:{code:"ZGlzdGlsYmVydF9udW1fcGFyYW1ldGVycyUyMCUzRCUyMG1vZGVsLm51bV9wYXJhbWV0ZXJzKCklMjAlMkYlMjAxXzAwMF8wMDAlMEFwcmludChmJTIyJyUzRSUzRSUzRSUyMERpc3RpbEJFUlQlMjBudW1iZXIlMjBvZiUyMHBhcmFtZXRlcnMlM0ElMjAlN0Jyb3VuZChkaXN0aWxiZXJ0X251bV9wYXJhbWV0ZXJzKSU3RE0nJTIyKSUwQXByaW50KGYlMjInJTNFJTNFJTNFJTIwQkVSVCUyMG51bWJlciUyMG9mJTIwcGFyYW1ldGVycyUzQSUyMDExME0nJTIyKQ==",highlighted:`distilbert_num_parameters = model.num_parameters() / <span class="hljs-number">1_000_000</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; DistilBERT number of parameters: <span class="hljs-subst">{<span class="hljs-built_in">round</span>(distilbert_num_parameters)}</span>M&#x27;&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; BERT number of parameters: 110M&#x27;&quot;</span>)`,wrap:!1}}),k=new b({props:{code:"JyUzRSUzRSUzRSUyMERpc3RpbEJFUlQlMjBudW1iZXIlMjBvZiUyMHBhcmFtZXRlcnMlM0ElMjA2N00nJTBBJyUzRSUzRSUzRSUyMEJFUlQlMjBudW1iZXIlMjBvZiUyMHBhcmFtZXRlcnMlM0ElMjAxMTBNJw==",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; DistilBERT number of parameters: 67M&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; BERT number of parameters: 110M&#x27;</span>`,wrap:!1}}),{c(){a=d("p"),a.innerHTML=U,n=M(),T(r.$$.fragment),G=M(),j=d("p"),j.innerHTML=R,W=M(),T(_.$$.fragment),X=M(),T(k.$$.fragment)},l(h){a=m(h,"P",{"data-svelte-h":!0}),u(a)!=="svelte-j6x8ju"&&(a.innerHTML=U),n=p(h),c(r.$$.fragment,h),G=p(h),j=m(h,"P",{"data-svelte-h":!0}),u(j)!=="svelte-1tyf55i"&&(j.innerHTML=R),W=p(h),c(_.$$.fragment,h),X=p(h),c(k.$$.fragment,h)},m(h,B){t(h,a,B),t(h,n,B),o(r,h,B),t(h,G,B),t(h,j,B),t(h,W,B),o(_,h,B),t(h,X,B),o(k,h,B),g=!0},i(h){g||(y(r.$$.fragment,h),y(_.$$.fragment,h),y(k.$$.fragment,h),g=!0)},o(h){J(r.$$.fragment,h),J(_.$$.fragment,h),J(k.$$.fragment,h),g=!1},d(h){h&&(e(a),e(n),e(G),e(j),e(W),e(X)),w(r,h),w(_,h),w(k,h)}}}function wM(I){let a,U;return a=new b({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBaW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIybnAlMjIpJTBBdG9rZW5fbG9naXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpLmxvZ2l0cyUwQSUyMyUyMEZpbmQlMjB0aGUlMjBsb2NhdGlvbiUyMG9mJTIwJTVCTUFTSyU1RCUyMGFuZCUyMGV4dHJhY3QlMjBpdHMlMjBsb2dpdHMlMEFtYXNrX3Rva2VuX2luZGV4JTIwJTNEJTIwbnAuYXJnd2hlcmUoaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTIwJTNEJTNEJTIwdG9rZW5pemVyLm1hc2tfdG9rZW5faWQpJTVCMCUyQyUyMDElNUQlMEFtYXNrX3Rva2VuX2xvZ2l0cyUyMCUzRCUyMHRva2VuX2xvZ2l0cyU1QjAlMkMlMjBtYXNrX3Rva2VuX2luZGV4JTJDJTIwJTNBJTVEJTBBJTIzJTIwUGljayUyMHRoZSUyMCU1Qk1BU0slNUQlMjBjYW5kaWRhdGVzJTIwd2l0aCUyMHRoZSUyMGhpZ2hlc3QlMjBsb2dpdHMlMEElMjMlMjBXZSUyMG5lZ2F0ZSUyMHRoZSUyMGFycmF5JTIwYmVmb3JlJTIwYXJnc29ydCUyMHRvJTIwZ2V0JTIwdGhlJTIwbGFyZ2VzdCUyQyUyMG5vdCUyMHRoZSUyMHNtYWxsZXN0JTJDJTIwbG9naXRzJTBBdG9wXzVfdG9rZW5zJTIwJTNEJTIwbnAuYXJnc29ydCgtbWFza190b2tlbl9sb2dpdHMpJTVCJTNBNSU1RC50b2xpc3QoKSUwQSUwQWZvciUyMHRva2VuJTIwaW4lMjB0b3BfNV90b2tlbnMlM0ElMEElMjAlMjAlMjAlMjBwcmludChmJTIyJTNFJTNFJTNFJTIwJTdCdGV4dC5yZXBsYWNlKHRva2VuaXplci5tYXNrX3Rva2VuJTJDJTIwdG9rZW5pemVyLmRlY29kZSglNUJ0b2tlbiU1RCkpJTdEJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)
token_logits = model(**inputs).logits
<span class="hljs-comment"># Find the location of [MASK] and extract its logits</span>
mask_token_index = np.argwhere(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>] == tokenizer.mask_token_id)[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
mask_token_logits = token_logits[<span class="hljs-number">0</span>, mask_token_index, :]
<span class="hljs-comment"># Pick the [MASK] candidates with the highest logits</span>
<span class="hljs-comment"># We negate the array before argsort to get the largest, not the smallest, logits</span>
top_5_tokens = np.argsort(-mask_token_logits)[:<span class="hljs-number">5</span>].tolist()

<span class="hljs-keyword">for</span> token <span class="hljs-keyword">in</span> top_5_tokens:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; <span class="hljs-subst">{text.replace(tokenizer.mask_token, tokenizer.decode([token]))}</span>&quot;</span>)`,wrap:!1}}),{c(){T(a.$$.fragment)},l(n){c(a.$$.fragment,n)},m(n,r){o(a,n,r),U=!0},i(n){U||(y(a.$$.fragment,n),U=!0)},o(n){J(a.$$.fragment,n),U=!1},d(n){w(a,n)}}}function dM(I){let a,U;return a=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBdG9rZW5fbG9naXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpLmxvZ2l0cyUwQSUyMyUyMEZpbmQlMjB0aGUlMjBsb2NhdGlvbiUyMG9mJTIwJTVCTUFTSyU1RCUyMGFuZCUyMGV4dHJhY3QlMjBpdHMlMjBsb2dpdHMlMEFtYXNrX3Rva2VuX2luZGV4JTIwJTNEJTIwdG9yY2gud2hlcmUoaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTIwJTNEJTNEJTIwdG9rZW5pemVyLm1hc2tfdG9rZW5faWQpJTVCMSU1RCUwQW1hc2tfdG9rZW5fbG9naXRzJTIwJTNEJTIwdG9rZW5fbG9naXRzJTVCMCUyQyUyMG1hc2tfdG9rZW5faW5kZXglMkMlMjAlM0ElNUQlMEElMjMlMjBQaWNrJTIwdGhlJTIwJTVCTUFTSyU1RCUyMGNhbmRpZGF0ZXMlMjB3aXRoJTIwdGhlJTIwaGlnaGVzdCUyMGxvZ2l0cyUwQXRvcF81X3Rva2VucyUyMCUzRCUyMHRvcmNoLnRvcGsobWFza190b2tlbl9sb2dpdHMlMkMlMjA1JTJDJTIwZGltJTNEMSkuaW5kaWNlcyU1QjAlNUQudG9saXN0KCklMEElMEFmb3IlMjB0b2tlbiUyMGluJTIwdG9wXzVfdG9rZW5zJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiclM0UlM0UlM0UlMjAlN0J0ZXh0LnJlcGxhY2UodG9rZW5pemVyLm1hc2tfdG9rZW4lMkMlMjB0b2tlbml6ZXIuZGVjb2RlKCU1QnRva2VuJTVEKSklN0QnJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> torch

inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
token_logits = model(**inputs).logits
<span class="hljs-comment"># Find the location of [MASK] and extract its logits</span>
mask_token_index = torch.where(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>] == tokenizer.mask_token_id)[<span class="hljs-number">1</span>]
mask_token_logits = token_logits[<span class="hljs-number">0</span>, mask_token_index, :]
<span class="hljs-comment"># Pick the [MASK] candidates with the highest logits</span>
top_5_tokens = torch.topk(mask_token_logits, <span class="hljs-number">5</span>, dim=<span class="hljs-number">1</span>).indices[<span class="hljs-number">0</span>].tolist()

<span class="hljs-keyword">for</span> token <span class="hljs-keyword">in</span> top_5_tokens:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; <span class="hljs-subst">{text.replace(tokenizer.mask_token, tokenizer.decode([token]))}</span>&#x27;&quot;</span>)`,wrap:!1}}),{c(){T(a.$$.fragment)},l(n){c(a.$$.fragment,n)},m(n,r){o(a,n,r),U=!0},i(n){U||(y(a.$$.fragment,n),U=!0)},o(n){J(a.$$.fragment,n),U=!1},d(n){w(a,n)}}}function mM(I){let a,U="✏️ <strong>试试看!</strong> 创建 <code>无监督</code> 拆分的随机样本, 并验证标签既不是 <code>0</code> 也不是 <code>1</code>。在此过程中, 你还可以检查 <code>train</code> 和 <code>test</code> 拆分中的标签是否确实为 <code>0</code> 或 <code>1</code> — 这是每个 NLP 从业者在新项目开始时都应该执行的有用的健全性检查!";return{c(){a=d("p"),a.innerHTML=U},l(n){a=m(n,"P",{"data-svelte-h":!0}),u(a)!=="svelte-8s7cwf"&&(a.innerHTML=U)},m(n,r){t(n,a,r)},p:Sl,d(n){n&&e(a)}}}function jM(I){let a,U='✏️ <strong>试试看!</strong> 一些 Transformer 模型, 例如 <a href="https://huggingface.co/google/bigbird-roberta-base" rel="nofollow">BigBird</a> 和 <a href="hf.co/allenai/longformer-base-4096">Longformer</a>, 它们具有比BERT和其他早期Transformer模型更长的上下文长度。为这些检查点之一实例化标记器, 并验证 <code>model_max_length</code> 是否与模型卡上引用的内容一致。';return{c(){a=d("p"),a.innerHTML=U},l(n){a=m(n,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1cswtyg"&&(a.innerHTML=U)},m(n,r){t(n,a,r)},p:Sl,d(n){n&&e(a)}}}function uM(I){let a,U="请注意, 在实际场景中使用较小的块大小可能是有害的, 因此你应该使用与将应用模型的用例相对应的大小。";return{c(){a=d("p"),a.textContent=U},l(n){a=m(n,"P",{"data-svelte-h":!0}),u(a)!=="svelte-18gfmni"&&(a.textContent=U)},m(n,r){t(n,a,r)},p:Sl,d(n){n&&e(a)}}}function hM(I){let a,U="✏️ <strong>试试看!</strong> 多次运行上面的代码片段, 看看随机屏蔽发生在你眼前! 还要将 <code>tokenizer.decode()</code> 方法替换为 <code>tokenizer.convert_ids_to_tokens()</code> 以查看有时会屏蔽给定单词中的单个标记, 而不是其他标记。";return{c(){a=d("p"),a.innerHTML=U},l(n){a=m(n,"P",{"data-svelte-h":!0}),u(a)!=="svelte-pauha9"&&(a.innerHTML=U)},m(n,r){t(n,a,r)},p:Sl,d(n){n&&e(a)}}}function On(I){let a,U="随机掩码的一个副作用是, 当使用 <code>Trainer</code> 时, 我们的评估指标将不是确定性的, 因为我们对训练集和测试集使用相同的数据整理器。稍后我们会看到, 当我们使用 🤗 Accelerate 进行微调时, 我们将如何利用自定义评估循环的灵活性来冻结随机性。";return{c(){a=d("p"),a.innerHTML=U},l(n){a=m(n,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1of0e1v"&&(a.innerHTML=U)},m(n,r){t(n,a,r)},d(n){n&&e(a)}}}function bM(I){let a,U;return a=new b({props:{code:"aW1wb3J0JTIwY29sbGVjdGlvbnMlMEFpbXBvcnQlMjBudW1weSUyMGFzJTIwbnAlMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzLmRhdGEuZGF0YV9jb2xsYXRvciUyMGltcG9ydCUyMHRmX2RlZmF1bHRfZGF0YV9jb2xsYXRvciUwQSUwQXd3bV9wcm9iYWJpbGl0eSUyMCUzRCUyMDAuMiUwQSUwQSUwQWRlZiUyMHdob2xlX3dvcmRfbWFza2luZ19kYXRhX2NvbGxhdG9yKGZlYXR1cmVzKSUzQSUwQSUyMCUyMCUyMCUyMGZvciUyMGZlYXR1cmUlMjBpbiUyMGZlYXR1cmVzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd29yZF9pZHMlMjAlM0QlMjBmZWF0dXJlLnBvcCglMjJ3b3JkX2lkcyUyMiklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBDcmVhdGUlMjBhJTIwbWFwJTIwYmV0d2VlbiUyMHdvcmRzJTIwYW5kJTIwY29ycmVzcG9uZGluZyUyMHRva2VuJTIwaW5kaWNlcyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hcHBpbmclMjAlM0QlMjBjb2xsZWN0aW9ucy5kZWZhdWx0ZGljdChsaXN0KSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGN1cnJlbnRfd29yZF9pbmRleCUyMCUzRCUyMC0xJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY3VycmVudF93b3JkJTIwJTNEJTIwTm9uZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMGlkeCUyQyUyMHdvcmRfaWQlMjBpbiUyMGVudW1lcmF0ZSh3b3JkX2lkcyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHdvcmRfaWQlMjBpcyUyMG5vdCUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHdvcmRfaWQlMjAhJTNEJTIwY3VycmVudF93b3JkJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY3VycmVudF93b3JkJTIwJTNEJTIwd29yZF9pZCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGN1cnJlbnRfd29yZF9pbmRleCUyMCUyQiUzRCUyMDElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXBwaW5nJTVCY3VycmVudF93b3JkX2luZGV4JTVELmFwcGVuZChpZHgpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwUmFuZG9tbHklMjBtYXNrJTIwd29yZHMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXNrJTIwJTNEJTIwbnAucmFuZG9tLmJpbm9taWFsKDElMkMlMjB3d21fcHJvYmFiaWxpdHklMkMlMjAobGVuKG1hcHBpbmcpJTJDKSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbnB1dF9pZHMlMjAlM0QlMjBmZWF0dXJlJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFiZWxzJTIwJTNEJTIwZmVhdHVyZSU1QiUyMmxhYmVscyUyMiU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG5ld19sYWJlbHMlMjAlM0QlMjAlNUItMTAwJTVEJTIwKiUyMGxlbihsYWJlbHMpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwd29yZF9pZCUyMGluJTIwbnAud2hlcmUobWFzayklNUIwJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd29yZF9pZCUyMCUzRCUyMHdvcmRfaWQuaXRlbSgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwaWR4JTIwaW4lMjBtYXBwaW5nJTVCd29yZF9pZCU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG5ld19sYWJlbHMlNUJpZHglNUQlMjAlM0QlMjBsYWJlbHMlNUJpZHglNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbnB1dF9pZHMlNUJpZHglNUQlMjAlM0QlMjB0b2tlbml6ZXIubWFza190b2tlbl9pZCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmUlNUIlMjJsYWJlbHMlMjIlNUQlMjAlM0QlMjBuZXdfbGFiZWxzJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwdGZfZGVmYXVsdF9kYXRhX2NvbGxhdG9yKGZlYXR1cmVzKQ==",highlighted:`<span class="hljs-keyword">import</span> collections
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-keyword">from</span> transformers.data.data_collator <span class="hljs-keyword">import</span> tf_default_data_collator

wwm_probability = <span class="hljs-number">0.2</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">whole_word_masking_data_collator</span>(<span class="hljs-params">features</span>):
    <span class="hljs-keyword">for</span> feature <span class="hljs-keyword">in</span> features:
        word_ids = feature.pop(<span class="hljs-string">&quot;word_ids&quot;</span>)

        <span class="hljs-comment"># Create a map between words and corresponding token indices</span>
        mapping = collections.defaultdict(<span class="hljs-built_in">list</span>)
        current_word_index = -<span class="hljs-number">1</span>
        current_word = <span class="hljs-literal">None</span>
        <span class="hljs-keyword">for</span> idx, word_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word_ids):
            <span class="hljs-keyword">if</span> word_id <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                <span class="hljs-keyword">if</span> word_id != current_word:
                    current_word = word_id
                    current_word_index += <span class="hljs-number">1</span>
                mapping[current_word_index].append(idx)

        <span class="hljs-comment"># Randomly mask words</span>
        mask = np.random.binomial(<span class="hljs-number">1</span>, wwm_probability, (<span class="hljs-built_in">len</span>(mapping),))
        input_ids = feature[<span class="hljs-string">&quot;input_ids&quot;</span>]
        labels = feature[<span class="hljs-string">&quot;labels&quot;</span>]
        new_labels = [-<span class="hljs-number">100</span>] * <span class="hljs-built_in">len</span>(labels)
        <span class="hljs-keyword">for</span> word_id <span class="hljs-keyword">in</span> np.where(mask)[<span class="hljs-number">0</span>]:
            word_id = word_id.item()
            <span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> mapping[word_id]:
                new_labels[idx] = labels[idx]
                input_ids[idx] = tokenizer.mask_token_id
        feature[<span class="hljs-string">&quot;labels&quot;</span>] = new_labels

    <span class="hljs-keyword">return</span> tf_default_data_collator(features)`,wrap:!1}}),{c(){T(a.$$.fragment)},l(n){c(a.$$.fragment,n)},m(n,r){o(a,n,r),U=!0},i(n){U||(y(a.$$.fragment,n),U=!0)},o(n){J(a.$$.fragment,n),U=!1},d(n){w(a,n)}}}function fM(I){let a,U;return a=new b({props:{code:"aW1wb3J0JTIwY29sbGVjdGlvbnMlMEFpbXBvcnQlMjBudW1weSUyMGFzJTIwbnAlMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwZGVmYXVsdF9kYXRhX2NvbGxhdG9yJTBBJTBBd3dtX3Byb2JhYmlsaXR5JTIwJTNEJTIwMC4yJTBBJTBBJTBBZGVmJTIwd2hvbGVfd29yZF9tYXNraW5nX2RhdGFfY29sbGF0b3IoZmVhdHVyZXMpJTNBJTBBJTIwJTIwJTIwJTIwZm9yJTIwZmVhdHVyZSUyMGluJTIwZmVhdHVyZXMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkcyUyMCUzRCUyMGZlYXR1cmUucG9wKCUyMndvcmRfaWRzJTIyKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMENyZWF0ZSUyMGElMjBtYXAlMjBiZXR3ZWVuJTIwd29yZHMlMjBhbmQlMjBjb3JyZXNwb25kaW5nJTIwdG9rZW4lMjBpbmRpY2VzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWFwcGluZyUyMCUzRCUyMGNvbGxlY3Rpb25zLmRlZmF1bHRkaWN0KGxpc3QpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY3VycmVudF93b3JkX2luZGV4JTIwJTNEJTIwLTElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjdXJyZW50X3dvcmQlMjAlM0QlMjBOb25lJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwaWR4JTJDJTIwd29yZF9pZCUyMGluJTIwZW51bWVyYXRlKHdvcmRfaWRzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZCUyMGlzJTIwbm90JTIwTm9uZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZCUyMCElM0QlMjBjdXJyZW50X3dvcmQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjdXJyZW50X3dvcmQlMjAlM0QlMjB3b3JkX2lkJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY3VycmVudF93b3JkX2luZGV4JTIwJTJCJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hcHBpbmclNUJjdXJyZW50X3dvcmRfaW5kZXglNUQuYXBwZW5kKGlkeCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBSYW5kb21seSUyMG1hc2slMjB3b3JkcyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hc2slMjAlM0QlMjBucC5yYW5kb20uYmlub21pYWwoMSUyQyUyMHd3bV9wcm9iYWJpbGl0eSUyQyUyMChsZW4obWFwcGluZyklMkMpKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0X2lkcyUyMCUzRCUyMGZlYXR1cmUlNUIlMjJpbnB1dF9pZHMlMjIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbHMlMjAlM0QlMjBmZWF0dXJlJTVCJTIybGFiZWxzJTIyJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbmV3X2xhYmVscyUyMCUzRCUyMCU1Qi0xMDAlNUQlMjAqJTIwbGVuKGxhYmVscyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjB3b3JkX2lkJTIwaW4lMjBucC53aGVyZShtYXNrKSU1QjAlNUQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkJTIwJTNEJTIwd29yZF9pZC5pdGVtKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBpZHglMjBpbiUyMG1hcHBpbmclNUJ3b3JkX2lkJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbmV3X2xhYmVscyU1QmlkeCU1RCUyMCUzRCUyMGxhYmVscyU1QmlkeCU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0X2lkcyU1QmlkeCU1RCUyMCUzRCUyMHRva2VuaXplci5tYXNrX3Rva2VuX2lkJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmVhdHVyZSU1QiUyMmxhYmVscyUyMiU1RCUyMCUzRCUyMG5ld19sYWJlbHMlMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBkZWZhdWx0X2RhdGFfY29sbGF0b3IoZmVhdHVyZXMp",highlighted:`<span class="hljs-keyword">import</span> collections
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> default_data_collator

wwm_probability = <span class="hljs-number">0.2</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">whole_word_masking_data_collator</span>(<span class="hljs-params">features</span>):
    <span class="hljs-keyword">for</span> feature <span class="hljs-keyword">in</span> features:
        word_ids = feature.pop(<span class="hljs-string">&quot;word_ids&quot;</span>)

        <span class="hljs-comment"># Create a map between words and corresponding token indices</span>
        mapping = collections.defaultdict(<span class="hljs-built_in">list</span>)
        current_word_index = -<span class="hljs-number">1</span>
        current_word = <span class="hljs-literal">None</span>
        <span class="hljs-keyword">for</span> idx, word_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word_ids):
            <span class="hljs-keyword">if</span> word_id <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                <span class="hljs-keyword">if</span> word_id != current_word:
                    current_word = word_id
                    current_word_index += <span class="hljs-number">1</span>
                mapping[current_word_index].append(idx)

        <span class="hljs-comment"># Randomly mask words</span>
        mask = np.random.binomial(<span class="hljs-number">1</span>, wwm_probability, (<span class="hljs-built_in">len</span>(mapping),))
        input_ids = feature[<span class="hljs-string">&quot;input_ids&quot;</span>]
        labels = feature[<span class="hljs-string">&quot;labels&quot;</span>]
        new_labels = [-<span class="hljs-number">100</span>] * <span class="hljs-built_in">len</span>(labels)
        <span class="hljs-keyword">for</span> word_id <span class="hljs-keyword">in</span> np.where(mask)[<span class="hljs-number">0</span>]:
            word_id = word_id.item()
            <span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> mapping[word_id]:
                new_labels[idx] = labels[idx]
                input_ids[idx] = tokenizer.mask_token_id
        feature[<span class="hljs-string">&quot;labels&quot;</span>] = new_labels

    <span class="hljs-keyword">return</span> default_data_collator(features)`,wrap:!1}}),{c(){T(a.$$.fragment)},l(n){c(a.$$.fragment,n)},m(n,r){o(a,n,r),U=!0},i(n){U||(y(a.$$.fragment,n),U=!0)},o(n){J(a.$$.fragment,n),U=!1},d(n){w(a,n)}}}function IM(I){let a,U="✏️ <strong>试试看!</strong> 多次运行上面的代码片段, 看看随机屏蔽发生在你眼前! 还要将 <code>tokenizer.decode()</code> 方法替换为 <code>tokenizer.convert_ids_to_tokens()</code> 以查看来自给定单词的标记始终被屏蔽在一起。";return{c(){a=d("p"),a.innerHTML=U},l(n){a=m(n,"P",{"data-svelte-h":!0}),u(a)!=="svelte-efynmr"&&(a.innerHTML=U)},m(n,r){t(n,a,r)},p:Sl,d(n){n&&e(a)}}}function CM(I){let a,U="登陆后, 我们可以指定 <code>Trainer</code> 参数:",n,r,G,j,R="在这里, 我们调整了一些默认选项, 包括 <code>logging_steps</code> , 以确保我们跟踪每个epoch的训练损失。我们还使用了 <code>fp16=True</code> 来实现混合精度训练, 这给我们带来了另一个速度提升。默认情况下, <code>Trainer</code> 将删除不属于模型的 <code>forward()</code> 方法的列。这意味着, 如果你使用整个单词屏蔽排序器, 你还需要设置 <code>remove_unused_columns=False</code>, 以确保我们不会在训练期间丢失 <code>word_ids</code> 列。",W,_,X='请注意, 你可以使用 <code>hub_model_id</code> 参数指定要推送到的存储库的名称((特别是你将必须使用该参数向组织推送)。例如, 当我们将模型推送到 <a href="https://huggingface.co/huggingface-course" rel="nofollow"><code>huggingface-course</code> organization</a> 时, 我们添加了 <code>hub_model_id=&quot;huggingface-course/distilbert-finetuned-imdb&quot;</code> 到 <code>TrainingArguments</code> 中。默认情况下, 使用的存储库将在你的命名空间中并以你设置的输出目录命名, 因此在我们的示例中, 它将是 <code>&quot;lewtun/distilbert-finetuned-imdb&quot;</code>。',k,g,h="我们现在拥有实例化 <code>Trainer</code> 的所有成分。这里我们只使用标准的 <code>data_collator</code>, 但你可以尝试使用整个单词掩码整理器并比较结果作为练习:",B,z,v,C,$="我们现在准备运行 <code>trainer.train()</code> — 但在此之前让我们简要地看一下 <em>perplexity</em>, 这是评估语言模型性能的常用指标。",S;return r=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTBBJTBBYmF0Y2hfc2l6ZSUyMCUzRCUyMDY0JTBBJTIzJTIwU2hvdyUyMHRoZSUyMHRyYWluaW5nJTIwbG9zcyUyMHdpdGglMjBldmVyeSUyMGVwb2NoJTBBbG9nZ2luZ19zdGVwcyUyMCUzRCUyMGxlbihkb3duc2FtcGxlZF9kYXRhc2V0JTVCJTIydHJhaW4lMjIlNUQpJTIwJTJGJTJGJTIwYmF0Y2hfc2l6ZSUwQW1vZGVsX25hbWUlMjAlM0QlMjBtb2RlbF9jaGVja3BvaW50LnNwbGl0KCUyMiUyRiUyMiklNUItMSU1RCUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEZiUyMiU3Qm1vZGVsX25hbWUlN0QtZmluZXR1bmVkLWltZGIlMjIlMkMlMEElMjAlMjAlMjAlMjBvdmVyd3JpdGVfb3V0cHV0X2RpciUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBldmFsdWF0aW9uX3N0cmF0ZWd5JTNEJTIyZXBvY2glMjIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMmUtNSUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRDAuMDElMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0RiYXRjaF9zaXplJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0RiYXRjaF9zaXplJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwZnAxNiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNEbG9nZ2luZ19zdGVwcyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

batch_size = <span class="hljs-number">64</span>
<span class="hljs-comment"># Show the training loss with every epoch</span>
logging_steps = <span class="hljs-built_in">len</span>(downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size
model_name = model_checkpoint.split(<span class="hljs-string">&quot;/&quot;</span>)[-<span class="hljs-number">1</span>]

training_args = TrainingArguments(
    output_dir=<span class="hljs-string">f&quot;<span class="hljs-subst">{model_name}</span>-finetuned-imdb&quot;</span>,
    overwrite_output_dir=<span class="hljs-literal">True</span>,
    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    push_to_hub=<span class="hljs-literal">True</span>,
    fp16=<span class="hljs-literal">True</span>,
    logging_steps=logging_steps,
)`,wrap:!1}}),z=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZG93bnNhbXBsZWRfZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEZG93bnNhbXBsZWRfZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2NvbGxhdG9yJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>],
    data_collator=data_collator,
    tokenizer=tokenizer,
)`,wrap:!1}}),{c(){a=d("p"),a.innerHTML=U,n=M(),T(r.$$.fragment),G=M(),j=d("p"),j.innerHTML=R,W=M(),_=d("p"),_.innerHTML=X,k=M(),g=d("p"),g.innerHTML=h,B=M(),T(z.$$.fragment),v=M(),C=d("p"),C.innerHTML=$},l(Z){a=m(Z,"P",{"data-svelte-h":!0}),u(a)!=="svelte-q9j5s5"&&(a.innerHTML=U),n=p(Z),c(r.$$.fragment,Z),G=p(Z),j=m(Z,"P",{"data-svelte-h":!0}),u(j)!=="svelte-6og8kn"&&(j.innerHTML=R),W=p(Z),_=m(Z,"P",{"data-svelte-h":!0}),u(_)!=="svelte-s0v9qr"&&(_.innerHTML=X),k=p(Z),g=m(Z,"P",{"data-svelte-h":!0}),u(g)!=="svelte-1r9ftt9"&&(g.innerHTML=h),B=p(Z),c(z.$$.fragment,Z),v=p(Z),C=m(Z,"P",{"data-svelte-h":!0}),u(C)!=="svelte-1ipnfce"&&(C.innerHTML=$)},m(Z,V){t(Z,a,V),t(Z,n,V),o(r,Z,V),t(Z,G,V),t(Z,j,V),t(Z,W,V),t(Z,_,V),t(Z,k,V),t(Z,g,V),t(Z,B,V),o(z,Z,V),t(Z,v,V),t(Z,C,V),S=!0},i(Z){S||(y(r.$$.fragment,Z),y(z.$$.fragment,Z),S=!0)},o(Z){J(r.$$.fragment,Z),J(z.$$.fragment,Z),S=!1},d(Z){Z&&(e(a),e(n),e(G),e(j),e(W),e(_),e(k),e(g),e(B),e(v),e(C)),w(r,Z),w(z,Z)}}}function _M(I){let a,U="登录后，我们可以创建我们的“tf.data”数据集。 为此，我们将使用 <code>prepare_tf_dataset()</code> 方法，该方法使用我们的模型自动推断哪些列应进入数据集。 如果您想准确控制要使用的列，可以改用“Dataset.to_tf_dataset()”方法。 为了简单起见，我们在这里只使用标准数据整理器，但您也可以尝试整个单词屏蔽整理器并将结果作为练习进行比较：",n,r,G,j,R="接下来, 我们设置训练超参数并编译模型。我们使用 🤗 Transformers 库中的 <code>create_optimizer()</code> 函数, 它提供了一个具有线性学习率衰减的 <code>AdamW</code> 优化器。我们还使用了模型内置的损失, 当没有损失被指定为 <code>compile()</code> 参数是, 这是默认值, 并且我们将训练精度设置为 <code>&quot;mixed_float16&quot;</code>。请注意,如果你使用的是Colab GPU或其他不支持float16加速的GPU, 你可能应该注释掉这一行。",W,_,X='此外, 我们还设置了一个 <code>PushToHubCallback</code>, 它将在每个epoch后将模型保存到Hub。你可以使用 <code>hub_model_id</code> 参数指定你想要推送到的存储库的名称 (特别是你将必须使用该参数来推送到组织)。例如, 为了将模型推送到 <a href="https://huggingface.co/huggingface-course" rel="nofollow"><code>huggingface-course</code> organization</a>, 我们添加了 <code>hub_model_id=&quot;huggingface-course/distilbert-finetuned-imdb&quot;</code>。默认情况下, 使用的存储库将位于你的命名空间中, 并以你设置的输出目录命名, 因此在我们的示例中, 它将是 <code>&quot;lewtun/distilbert-finetuned-imdb&quot;</code>。',k,g,h,B,z="我们现在已经准备好运行 <code>model.fit()</code> 了 — 但在此之前, 让我们先简单地看看 <em>perplexity</em>, 它是评估语言模型性能的常用指标。",v;return r=new b({props:{code:"dGZfdHJhaW5fZGF0YXNldCUyMCUzRCUyMG1vZGVsLnByZXBhcmVfdGZfZGF0YXNldCglMEElMjAlMjAlMjAlMjBkb3duc2FtcGxlZF9kYXRhc2V0JTVCJTIydHJhaW4lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBjb2xsYXRlX2ZuJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDMyJTJDJTBBKSUwQSUwQXRmX2V2YWxfZGF0YXNldCUyMCUzRCUyMG1vZGVsLnByZXBhcmVfdGZfZGF0YXNldCglMEElMjAlMjAlMjAlMjBkb3duc2FtcGxlZF9kYXRhc2V0JTVCJTIydGVzdCUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBJTIwJTIwJTIwJTIwc2h1ZmZsZSUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDMyJTJDJTBBKQ==",highlighted:`tf_train_dataset = model.prepare_tf_dataset(
    downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">32</span>,
)

tf_eval_dataset = model.prepare_tf_dataset(
    downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">32</span>,
)`,wrap:!1}}),g=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEFmcm9tJTIwdHJhbnNmb3JtZXJzLmtlcmFzX2NhbGxiYWNrcyUyMGltcG9ydCUyMFB1c2hUb0h1YkNhbGxiYWNrJTBBaW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFudW1fdHJhaW5fc3RlcHMlMjAlM0QlMjBsZW4odGZfdHJhaW5fZGF0YXNldCklMEFvcHRpbWl6ZXIlMkMlMjBzY2hlZHVsZSUyMCUzRCUyMGNyZWF0ZV9vcHRpbWl6ZXIoJTBBJTIwJTIwJTIwJTIwaW5pdF9sciUzRDJlLTUlMkMlMEElMjAlMjAlMjAlMjBudW1fd2FybXVwX3N0ZXBzJTNEMV8wMDAlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fc3RlcHMlM0RudW1fdHJhaW5fc3RlcHMlMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRfZGVjYXlfcmF0ZSUzRDAuMDElMkMlMEEpJTBBbW9kZWwuY29tcGlsZShvcHRpbWl6ZXIlM0RvcHRpbWl6ZXIpJTBBJTBBJTIzJTIwVHJhaW4lMjBpbiUyMG1peGVkLXByZWNpc2lvbiUyMGZsb2F0MTYlMEF0Zi5rZXJhcy5taXhlZF9wcmVjaXNpb24uc2V0X2dsb2JhbF9wb2xpY3koJTIybWl4ZWRfZmxvYXQxNiUyMiklMEElMEFtb2RlbF9uYW1lJTIwJTNEJTIwbW9kZWxfY2hlY2twb2ludC5zcGxpdCglMjIlMkYlMjIpJTVCLTElNUQlMEFjYWxsYmFjayUyMCUzRCUyMFB1c2hUb0h1YkNhbGxiYWNrKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0RmJTIyJTdCbW9kZWxfbmFtZSU3RC1maW5ldHVuZWQtaW1kYiUyMiUyQyUyMHRva2VuaXplciUzRHRva2VuaXplciUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer
<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

num_train_steps = <span class="hljs-built_in">len</span>(tf_train_dataset)
optimizer, schedule = create_optimizer(
    init_lr=<span class="hljs-number">2e-5</span>,
    num_warmup_steps=<span class="hljs-number">1_000</span>,
    num_train_steps=num_train_steps,
    weight_decay_rate=<span class="hljs-number">0.01</span>,
)
model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)

<span class="hljs-comment"># Train in mixed-precision float16</span>
tf.keras.mixed_precision.set_global_policy(<span class="hljs-string">&quot;mixed_float16&quot;</span>)

model_name = model_checkpoint.split(<span class="hljs-string">&quot;/&quot;</span>)[-<span class="hljs-number">1</span>]
callback = PushToHubCallback(
    output_dir=<span class="hljs-string">f&quot;<span class="hljs-subst">{model_name}</span>-finetuned-imdb&quot;</span>, tokenizer=tokenizer
)`,wrap:!1}}),{c(){a=d("p"),a.innerHTML=U,n=M(),T(r.$$.fragment),G=M(),j=d("p"),j.innerHTML=R,W=M(),_=d("p"),_.innerHTML=X,k=M(),T(g.$$.fragment),h=M(),B=d("p"),B.innerHTML=z},l(C){a=m(C,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1xjzifo"&&(a.innerHTML=U),n=p(C),c(r.$$.fragment,C),G=p(C),j=m(C,"P",{"data-svelte-h":!0}),u(j)!=="svelte-1fmtt95"&&(j.innerHTML=R),W=p(C),_=m(C,"P",{"data-svelte-h":!0}),u(_)!=="svelte-18yiu2l"&&(_.innerHTML=X),k=p(C),c(g.$$.fragment,C),h=p(C),B=m(C,"P",{"data-svelte-h":!0}),u(B)!=="svelte-vi1gi9"&&(B.innerHTML=z)},m(C,$){t(C,a,$),t(C,n,$),o(r,C,$),t(C,G,$),t(C,j,$),t(C,W,$),t(C,_,$),t(C,k,$),o(g,C,$),t(C,h,$),t(C,B,$),v=!0},i(C){v||(y(r.$$.fragment,C),y(g.$$.fragment,C),v=!0)},o(C){J(r.$$.fragment,C),J(g.$$.fragment,C),v=!1},d(C){C&&(e(a),e(n),e(G),e(j),e(W),e(_),e(k),e(h),e(B)),w(r,C),w(g,C)}}}function GM(I){let a,U="假设我们的测试集主要由语法正确的句子组成, 那么衡量我们的语言模型质量的一种方法是计算测试集所有句子中它分配给下一个单词的概率。高概率表明, 该模型表明该模型对未见过的示例不感到 “惊讶” 或 “困惑”, 并表明它已经学会了该语言的基本语法模式。关于perplexity度有各种不同的数学定义, 但我们要用的定义是交叉熵损失的指数。因此, 我们可以通过 <code>model.evaluate()</code> 方法计算测试集上的交叉熵损失, 然后取结果的指数来计算我们预训练模型的perplexity度:",n,r,G;return r=new b({props:{code:"aW1wb3J0JTIwbWF0aCUwQSUwQWV2YWxfbG9zcyUyMCUzRCUyMG1vZGVsLmV2YWx1YXRlKHRmX2V2YWxfZGF0YXNldCklMEFwcmludChmJTIyUGVycGxleGl0eSUzQSUyMCU3Qm1hdGguZXhwKGV2YWxfbG9zcyklM0EuMmYlN0QlMjIp",highlighted:`<span class="hljs-keyword">import</span> math

eval_loss = model.evaluate(tf_eval_dataset)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Perplexity: <span class="hljs-subst">{math.exp(eval_loss):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,wrap:!1}}),{c(){a=d("p"),a.innerHTML=U,n=M(),T(r.$$.fragment)},l(j){a=m(j,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1uqhssb"&&(a.innerHTML=U),n=p(j),c(r.$$.fragment,j)},m(j,R){t(j,a,R),t(j,n,R),o(r,j,R),G=!0},i(j){G||(y(r.$$.fragment,j),G=!0)},o(j){J(r.$$.fragment,j),G=!1},d(j){j&&(e(a),e(n)),w(r,j)}}}function BM(I){let a,U="假设我们的测试集主要由语法正确的句子组成, 那么衡量我们的语言模型质量的一种方法是计算它分配给测试集中所有句子中的下一个单词的概率。高概率表明模型对看不见的例子并不感到 “惊讶” 或 “疑惑”, 并表明它已经学习了语言中的基本语法模式。 perplexity度有多种数学定义, 但我们将使用的定义是交叉熵损失的指数。因此, 我们可以通过 <code>Trainer.evaluate()</code> 函数计算测试集上的交叉熵损失, 然后取结果的指数来计算预训练模型的perplexity度:",n,r,G;return r=new b({props:{code:"aW1wb3J0JTIwbWF0aCUwQSUwQWV2YWxfcmVzdWx0cyUyMCUzRCUyMHRyYWluZXIuZXZhbHVhdGUoKSUwQXByaW50KGYlMjIlM0UlM0UlM0UlMjBQZXJwbGV4aXR5JTNBJTIwJTdCbWF0aC5leHAoZXZhbF9yZXN1bHRzJTVCJ2V2YWxfbG9zcyclNUQpJTNBLjJmJTdEJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> math

eval_results = trainer.evaluate()
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; Perplexity: <span class="hljs-subst">{math.exp(eval_results[<span class="hljs-string">&#x27;eval_loss&#x27;</span>]):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,wrap:!1}}),{c(){a=d("p"),a.innerHTML=U,n=M(),T(r.$$.fragment)},l(j){a=m(j,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1uvxr2o"&&(a.innerHTML=U),n=p(j),c(r.$$.fragment,j)},m(j,R){t(j,a,R),t(j,n,R),o(r,j,R),G=!0},i(j){G||(y(r.$$.fragment,j),G=!0)},o(j){J(r.$$.fragment,j),G=!1},d(j){j&&(e(a),e(n)),w(r,j)}}}function ZM(I){let a,U;return a=new b({props:{code:"bW9kZWwuZml0KHRmX3RyYWluX2RhdGFzZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl9ldmFsX2RhdGFzZXQlMkMlMjBjYWxsYmFja3MlM0QlNUJjYWxsYmFjayU1RCk=",highlighted:"model.fit(tf_train_dataset, validation_data=tf_eval_dataset, callbacks=[callback])",wrap:!1}}),{c(){T(a.$$.fragment)},l(n){c(a.$$.fragment,n)},m(n,r){o(a,n,r),U=!0},i(n){U||(y(a.$$.fragment,n),U=!0)},o(n){J(a.$$.fragment,n),U=!1},d(n){w(a,n)}}}function gM(I){let a,U;return a=new b({props:{code:"dHJhaW5lci50cmFpbigp",highlighted:"trainer.train()",wrap:!1}}),{c(){T(a.$$.fragment)},l(n){c(a.$$.fragment,n)},m(n,r){o(a,n,r),U=!0},i(n){U||(y(a.$$.fragment,n),U=!0)},o(n){J(a.$$.fragment,n),U=!1},d(n){w(a,n)}}}function kM(I){let a,U;return a=new b({props:{code:"ZXZhbF9sb3NzJTIwJTNEJTIwbW9kZWwuZXZhbHVhdGUodGZfZXZhbF9kYXRhc2V0KSUwQXByaW50KGYlMjJQZXJwbGV4aXR5JTNBJTIwJTdCbWF0aC5leHAoZXZhbF9sb3NzKSUzQS4yZiU3RCUyMik=",highlighted:`eval_loss = model.evaluate(tf_eval_dataset)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Perplexity: <span class="hljs-subst">{math.exp(eval_loss):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,wrap:!1}}),{c(){T(a.$$.fragment)},l(n){c(a.$$.fragment,n)},m(n,r){o(a,n,r),U=!0},i(n){U||(y(a.$$.fragment,n),U=!0)},o(n){J(a.$$.fragment,n),U=!1},d(n){w(a,n)}}}function RM(I){let a,U;return a=new b({props:{code:"ZXZhbF9yZXN1bHRzJTIwJTNEJTIwdHJhaW5lci5ldmFsdWF0ZSgpJTBBcHJpbnQoZiUyMiUzRSUzRSUzRSUyMFBlcnBsZXhpdHklM0ElMjAlN0JtYXRoLmV4cChldmFsX3Jlc3VsdHMlNUInZXZhbF9sb3NzJyU1RCklM0EuMmYlN0QlMjIp",highlighted:`eval_results = trainer.evaluate()
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; Perplexity: <span class="hljs-subst">{math.exp(eval_results[<span class="hljs-string">&#x27;eval_loss&#x27;</span>]):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,wrap:!1}}),{c(){T(a.$$.fragment)},l(n){c(a.$$.fragment,n)},m(n,r){o(a,n,r),U=!0},i(n){U||(y(a.$$.fragment,n),U=!0)},o(n){J(a.$$.fragment,n),U=!1},d(n){w(a,n)}}}function lM(I){let a,U="训练完成后, 我们可以将带有训练信息的模型卡推送到 Hub (检查点在训练过程中自行保存):",n,r,G;return r=new b({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:"trainer.push_to_hub()",wrap:!1}}),{c(){a=d("p"),a.textContent=U,n=M(),T(r.$$.fragment)},l(j){a=m(j,"P",{"data-svelte-h":!0}),u(a)!=="svelte-vs8yx8"&&(a.textContent=U),n=p(j),c(r.$$.fragment,j)},m(j,R){t(j,a,R),t(j,n,R),o(r,j,R),G=!0},i(j){G||(y(r.$$.fragment,j),G=!0)},o(j){J(r.$$.fragment,j),G=!1},d(j){j&&(e(a),e(n)),w(r,j)}}}function XM(I){let a,U="✏️ <strong>轮到你了!</strong> 将数据整理器改为全字屏蔽整理器后运行上面的训练。你有得到更好的结果吗?";return{c(){a=d("p"),a.innerHTML=U},l(n){a=m(n,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1aioz9a"&&(a.innerHTML=U)},m(n,r){t(n,a,r)},p:Sl,d(n){n&&e(a)}}}function sM(I){let a,U="在我们的用例中, 我们不需要对训练循环做任何特别的事情, 但在某些情况下, 你可能需要实现一些自定义逻辑。对于这些应用, 你可以使用 🤗 Accelerate — 让我们来看看吧!",n,r,G,j,R='正如我们在 <code>Trainer</code> 中看到的, 对掩码语言模型的微调与 <a href="/course/chapter3">第三章</a> 中的文本分类示例非常相似。事实上, 唯一的微妙之处是使用特殊的数据整理器, 我们已经在本节的前面介绍过了!',W,_,X="然而, 我们看到 <code>DataCollatorForLanguageModeling</code> 还对每次评估应用随机掩码, 因此每次训练运行时, 我们都会看到perplexity度分数的一些波动。消除这种随机性来源的一种方法是应用掩码 <em>一次</em> 在整个测试集上, 然后使用🤗 Transformers 中的默认数据整理器在评估期间收集批次。为了看看它是如何工作的, 让我们实现一个简单的函数, 将掩码应用于批处理, 类似于我们第一次遇到的 <code>DataCollatorForLanguageModeling</code>:",k,g,h,B,z="接下来, 我们将此函数应用于我们的测试集并删除未掩码的列, 以便我们可以用掩码的列替换它们。你可以通过用适当的替换上面的 <code>data_collator</code> 来使整个单词掩码, 在这种情况下, 你应该删除此处的第一行:",v,C,$,S,Z="然后我们可以像往常一样设置数据加载器, 但我们将使用🤗 Transformers 中的 <code>default_data_collator</code> 作为评估集:",V,fl,Bl,H,Ye="从这里开始, 我们遵循🤗 Accelerate 的标准步骤。第一个任务是加载预训练模型的新版本:",Zl,N,Hl,tl,Il="然后我们需要指定优化器; 我们将使用标准的 <code>AdamW</code>:",Ll,al,nl,Ml,Jl="有了这些对象, 我们现在可以准备使用 <code>Accelerator</code> 加速器进行训练的一切:",gl,L,kl,q,Fe="现在我们的模型、优化器和数据加载器都配置好了, 我们可以指定学习率调度器如下:",Rl,Y,ql,pl,Cl="在训练之前要做的最后一件事是: 在 Hugging Face Hub 上创建一个模型库! 我们可以使用 🤗 Hub 库来首先生成我们的 repo 的全名:",Dl,il,F,A,rl,D,Ee="然后使用来自🤗 Hub 的 <code>Repository</code> 类:",Xl,K,Wl,P,Se="完成后, 只需写出完整的训练和评估循环即可:",$l,O,Vl,ll,Kl,yl,el="很棒, 我们已经能够评估每个 epoch 的perplexity度, 并确保多次训练运行是可重复的!",E;return r=new El({props:{title:"使用 🤗 Accelerate 微调 DistilBERT",local:"使用 🤗 Accelerate 微调 DistilBERT",headingTag:"h2"}}),g=new b({props:{code:"ZGVmJTIwaW5zZXJ0X3JhbmRvbV9tYXNrKGJhdGNoKSUzQSUwQSUyMCUyMCUyMCUyMGZlYXR1cmVzJTIwJTNEJTIwJTVCZGljdCh6aXAoYmF0Y2glMkMlMjB0KSklMjBmb3IlMjB0JTIwaW4lMjB6aXAoKmJhdGNoLnZhbHVlcygpKSU1RCUwQSUyMCUyMCUyMCUyMG1hc2tlZF9pbnB1dHMlMjAlM0QlMjBkYXRhX2NvbGxhdG9yKGZlYXR1cmVzKSUwQSUyMCUyMCUyMCUyMCUyMyUyMENyZWF0ZSUyMGElMjBuZXclMjAlMjJtYXNrZWQlMjIlMjBjb2x1bW4lMjBmb3IlMjBlYWNoJTIwY29sdW1uJTIwaW4lMjB0aGUlMjBkYXRhc2V0JTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTdCJTIybWFza2VkXyUyMiUyMCUyQiUyMGslM0ElMjB2Lm51bXB5KCklMjBmb3IlMjBrJTJDJTIwdiUyMGluJTIwbWFza2VkX2lucHV0cy5pdGVtcygpJTdE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">insert_random_mask</span>(<span class="hljs-params">batch</span>):
    features = [<span class="hljs-built_in">dict</span>(<span class="hljs-built_in">zip</span>(batch, t)) <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(*batch.values())]
    masked_inputs = data_collator(features)
    <span class="hljs-comment"># Create a new &quot;masked&quot; column for each column in the dataset</span>
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;masked_&quot;</span> + k: v.numpy() <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> masked_inputs.items()}`,wrap:!1}}),C=new b({props:{code:"ZG93bnNhbXBsZWRfZGF0YXNldCUyMCUzRCUyMGRvd25zYW1wbGVkX2RhdGFzZXQucmVtb3ZlX2NvbHVtbnMoJTVCJTIyd29yZF9pZHMlMjIlNUQpJTBBZXZhbF9kYXRhc2V0JTIwJTNEJTIwZG93bnNhbXBsZWRfZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQubWFwKCUwQSUyMCUyMCUyMCUyMGluc2VydF9yYW5kb21fbWFzayUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoZWQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcmVtb3ZlX2NvbHVtbnMlM0Rkb3duc2FtcGxlZF9kYXRhc2V0JTVCJTIydGVzdCUyMiU1RC5jb2x1bW5fbmFtZXMlMkMlMEEpJTBBZXZhbF9kYXRhc2V0JTIwJTNEJTIwZXZhbF9kYXRhc2V0LnJlbmFtZV9jb2x1bW5zKCUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1hc2tlZF9pbnB1dF9pZHMlMjIlM0ElMjAlMjJpbnB1dF9pZHMlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJtYXNrZWRfYXR0ZW50aW9uX21hc2slMjIlM0ElMjAlMjJhdHRlbnRpb25fbWFzayUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1hc2tlZF9sYWJlbHMlMjIlM0ElMjAlMjJsYWJlbHMlMjIlMkMlMEElMjAlMjAlMjAlMjAlN0QlMEEp",highlighted:`downsampled_dataset = downsampled_dataset.remove_columns([<span class="hljs-string">&quot;word_ids&quot;</span>])
eval_dataset = downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>].<span class="hljs-built_in">map</span>(
    insert_random_mask,
    batched=<span class="hljs-literal">True</span>,
    remove_columns=downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>].column_names,
)
eval_dataset = eval_dataset.rename_columns(
    {
        <span class="hljs-string">&quot;masked_input_ids&quot;</span>: <span class="hljs-string">&quot;input_ids&quot;</span>,
        <span class="hljs-string">&quot;masked_attention_mask&quot;</span>: <span class="hljs-string">&quot;attention_mask&quot;</span>,
        <span class="hljs-string">&quot;masked_labels&quot;</span>: <span class="hljs-string">&quot;labels&quot;</span>,
    }
)`,wrap:!1}}),fl=new b({props:{code:"ZnJvbSUyMHRvcmNoLnV0aWxzLmRhdGElMjBpbXBvcnQlMjBEYXRhTG9hZGVyJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGRlZmF1bHRfZGF0YV9jb2xsYXRvciUwQSUwQWJhdGNoX3NpemUlMjAlM0QlMjA2NCUwQXRyYWluX2RhdGFsb2FkZXIlMjAlM0QlMjBEYXRhTG9hZGVyKCUwQSUyMCUyMCUyMCUyMGRvd25zYW1wbGVkX2RhdGFzZXQlNUIlMjJ0cmFpbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRGJhdGNoX3NpemUlMkMlMEElMjAlMjAlMjAlMjBjb2xsYXRlX2ZuJTNEZGF0YV9jb2xsYXRvciUyQyUwQSklMEFldmFsX2RhdGFsb2FkZXIlMjAlM0QlMjBEYXRhTG9hZGVyKCUwQSUyMCUyMCUyMCUyMGV2YWxfZGF0YXNldCUyQyUyMGJhdGNoX3NpemUlM0RiYXRjaF9zaXplJTJDJTIwY29sbGF0ZV9mbiUzRGRlZmF1bHRfZGF0YV9jb2xsYXRvciUwQSk=",highlighted:`<span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> default_data_collator

batch_size = <span class="hljs-number">64</span>
train_dataloader = DataLoader(
    downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=batch_size,
    collate_fn=data_collator,
)
eval_dataloader = DataLoader(
    eval_dataset, batch_size=batch_size, collate_fn=default_data_collator
)`,wrap:!1}}),N=new b({props:{code:"bW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JNYXNrZWRMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfY2hlY2twb2ludCk=",highlighted:'model = <span class="hljs-module-access"><span class="hljs-module"><span class="hljs-identifier">AutoModelForMaskedLM</span>.</span></span>from<span class="hljs-constructor">_pretrained(<span class="hljs-params">model_checkpoint</span>)</span>',wrap:!1}}),al=new b({props:{code:"ZnJvbSUyMHRvcmNoLm9wdGltJTIwaW1wb3J0JTIwQWRhbVclMEElMEFvcHRpbWl6ZXIlMjAlM0QlMjBBZGFtVyhtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMjBsciUzRDVlLTUp",highlighted:`<span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW

optimizer = AdamW(model.parameters(), lr=<span class="hljs-number">5e-5</span>)`,wrap:!1}}),L=new b({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoKSUwQW1vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwdHJhaW5fZGF0YWxvYWRlciUyQyUyMGV2YWxfZGF0YWxvYWRlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUoJTBBJTIwJTIwJTIwJTIwbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjB0cmFpbl9kYXRhbG9hZGVyJTJDJTIwZXZhbF9kYXRhbG9hZGVyJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`,wrap:!1}}),Y=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGdldF9zY2hlZHVsZXIlMEElMEFudW1fdHJhaW5fZXBvY2hzJTIwJTNEJTIwMyUwQW51bV91cGRhdGVfc3RlcHNfcGVyX2Vwb2NoJTIwJTNEJTIwbGVuKHRyYWluX2RhdGFsb2FkZXIpJTBBbnVtX3RyYWluaW5nX3N0ZXBzJTIwJTNEJTIwbnVtX3RyYWluX2Vwb2NocyUyMColMjBudW1fdXBkYXRlX3N0ZXBzX3Blcl9lcG9jaCUwQSUwQWxyX3NjaGVkdWxlciUyMCUzRCUyMGdldF9zY2hlZHVsZXIoJTBBJTIwJTIwJTIwJTIwJTIybGluZWFyJTIyJTJDJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyJTNEb3B0aW1pemVyJTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5pbmdfc3RlcHMlM0RudW1fdHJhaW5pbmdfc3RlcHMlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

num_train_epochs = <span class="hljs-number">3</span>
num_update_steps_per_epoch = <span class="hljs-built_in">len</span>(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    <span class="hljs-string">&quot;linear&quot;</span>,
    optimizer=optimizer,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_training_steps=num_training_steps,
)`,wrap:!1}}),il=new b({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGdldF9mdWxsX3JlcG9fbmFtZSUwQSUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZC1maW5ldHVuZWQtaW1kYi1hY2NlbGVyYXRlJTIyJTBBcmVwb19uYW1lJTIwJTNEJTIwZ2V0X2Z1bGxfcmVwb19uYW1lKG1vZGVsX25hbWUpJTBBcmVwb19uYW1l",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> get_full_repo_name

model_name = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-imdb-accelerate&quot;</span>
repo_name = get_full_repo_name(model_name)
repo_name`,wrap:!1}}),A=new b({props:{code:"J2xld3R1biUyRmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkLWZpbmV0dW5lZC1pbWRiLWFjY2VsZXJhdGUn",highlighted:'<span class="hljs-string">&#x27;lewtun/distilbert-base-uncased-finetuned-imdb-accelerate&#x27;</span>',wrap:!1}}),K=new b({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMFJlcG9zaXRvcnklMEElMEFvdXRwdXRfZGlyJTIwJTNEJTIwbW9kZWxfbmFtZSUwQXJlcG8lMjAlM0QlMjBSZXBvc2l0b3J5KG91dHB1dF9kaXIlMkMlMjBjbG9uZV9mcm9tJTNEcmVwb19uYW1lKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository

output_dir = model_name
repo = Repository(output_dir, clone_from=repo_name)`,wrap:!1}}),O=new b({props:{code:"ZnJvbSUyMHRxZG0uYXV0byUyMGltcG9ydCUyMHRxZG0lMEFpbXBvcnQlMjB0b3JjaCUwQWltcG9ydCUyMG1hdGglMEElMEFwcm9ncmVzc19iYXIlMjAlM0QlMjB0cWRtKHJhbmdlKG51bV90cmFpbmluZ19zdGVwcykpJTBBJTBBZm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKG51bV90cmFpbl9lcG9jaHMpJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwVHJhaW5pbmclMEElMjAlMjAlMjAlMjBtb2RlbC50cmFpbigpJTBBJTIwJTIwJTIwJTIwZm9yJTIwYmF0Y2glMjBpbiUyMHRyYWluX2RhdGFsb2FkZXIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwb3V0cHV0cy5sb3NzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IuYmFja3dhcmQobG9zcyklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplci56ZXJvX2dyYWQoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByb2dyZXNzX2Jhci51cGRhdGUoMSklMEElMEElMjAlMjAlMjAlMjAlMjMlMjBFdmFsdWF0aW9uJTBBJTIwJTIwJTIwJTIwbW9kZWwuZXZhbCgpJTBBJTIwJTIwJTIwJTIwbG9zc2VzJTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwc3RlcCUyQyUyMGJhdGNoJTIwaW4lMjBlbnVtZXJhdGUoZXZhbF9kYXRhbG9hZGVyKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKipiYXRjaCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwb3V0cHV0cy5sb3NzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbG9zc2VzLmFwcGVuZChhY2NlbGVyYXRvci5nYXRoZXIobG9zcy5yZXBlYXQoYmF0Y2hfc2l6ZSkpKSUwQSUwQSUyMCUyMCUyMCUyMGxvc3NlcyUyMCUzRCUyMHRvcmNoLmNhdChsb3NzZXMpJTBBJTIwJTIwJTIwJTIwbG9zc2VzJTIwJTNEJTIwbG9zc2VzJTVCJTNBJTIwbGVuKGV2YWxfZGF0YXNldCklNUQlMEElMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwZXJwbGV4aXR5JTIwJTNEJTIwbWF0aC5leHAodG9yY2gubWVhbihsb3NzZXMpKSUwQSUyMCUyMCUyMCUyMGV4Y2VwdCUyME92ZXJmbG93RXJyb3IlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwZXJwbGV4aXR5JTIwJTNEJTIwZmxvYXQoJTIyaW5mJTIyKSUwQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlM0UlM0UlM0UlMjBFcG9jaCUyMCU3QmVwb2NoJTdEJTNBJTIwUGVycGxleGl0eSUzQSUyMCU3QnBlcnBsZXhpdHklN0QlMjIpJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwU2F2ZSUyMGFuZCUyMHVwbG9hZCUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLndhaXRfZm9yX2V2ZXJ5b25lKCklMEElMjAlMjAlMjAlMjB1bndyYXBwZWRfbW9kZWwlMjAlM0QlMjBhY2NlbGVyYXRvci51bndyYXBfbW9kZWwobW9kZWwpJTBBJTIwJTIwJTIwJTIwdW53cmFwcGVkX21vZGVsLnNhdmVfcHJldHJhaW5lZChvdXRwdXRfZGlyJTJDJTIwc2F2ZV9mdW5jdGlvbiUzRGFjY2VsZXJhdG9yLnNhdmUpJTBBJTIwJTIwJTIwJTIwaWYlMjBhY2NlbGVyYXRvci5pc19tYWluX3Byb2Nlc3MlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b2tlbml6ZXIuc2F2ZV9wcmV0cmFpbmVkKG91dHB1dF9kaXIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVwby5wdXNoX3RvX2h1YiglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb21taXRfbWVzc2FnZSUzRGYlMjJUcmFpbmluZyUyMGluJTIwcHJvZ3Jlc3MlMjBlcG9jaCUyMCU3QmVwb2NoJTdEJTIyJTJDJTIwYmxvY2tpbmclM0RGYWxzZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCk=",highlighted:`<span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> math

progress_bar = tqdm(<span class="hljs-built_in">range</span>(num_training_steps))

<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_train_epochs):
    <span class="hljs-comment"># Training</span>
    model.train()
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(<span class="hljs-number">1</span>)

    <span class="hljs-comment"># Evaluation</span>
    model.<span class="hljs-built_in">eval</span>()
    losses = []
    <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(eval_dataloader):
        <span class="hljs-keyword">with</span> torch.no_grad():
            outputs = model(**batch)

        loss = outputs.loss
        losses.append(accelerator.gather(loss.repeat(batch_size)))

    losses = torch.cat(losses)
    losses = losses[: <span class="hljs-built_in">len</span>(eval_dataset)]
    <span class="hljs-keyword">try</span>:
        perplexity = math.exp(torch.mean(losses))
    <span class="hljs-keyword">except</span> OverflowError:
        perplexity = <span class="hljs-built_in">float</span>(<span class="hljs-string">&quot;inf&quot;</span>)

    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; Epoch <span class="hljs-subst">{epoch}</span>: Perplexity: <span class="hljs-subst">{perplexity}</span>&quot;</span>)

    <span class="hljs-comment"># Save and upload</span>
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    <span class="hljs-keyword">if</span> accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=<span class="hljs-string">f&quot;Training in progress epoch <span class="hljs-subst">{epoch}</span>&quot;</span>, blocking=<span class="hljs-literal">False</span>
        )`,wrap:!1}}),ll=new b({props:{code:"RXBvY2glMjAwJTNBJTIwUGVycGxleGl0eSUzQSUyMDExLjM5NzU0NTMwNzkwMDQ3MiUwQUVwb2NoJTIwMSUzQSUyMFBlcnBsZXhpdHklM0ElMjAxMC45MDQ5MDkzMzA5ODMwOTIlMEFFcG9jaCUyMDIlM0ElMjBQZXJwbGV4aXR5JTNBJTIwMTAuNzI5NTAzNTA1MzQwNDA5",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>Epoch <span class="hljs-number">0</span>: Perplexity: <span class="hljs-number">11.397545307900472</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>Epoch <span class="hljs-number">1</span>: Perplexity: <span class="hljs-number">10.904909330983092</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>Epoch <span class="hljs-number">2</span>: Perplexity: <span class="hljs-number">10.729503505340409</span>`,wrap:!1}}),{c(){a=d("p"),a.textContent=U,n=M(),T(r.$$.fragment),G=M(),j=d("p"),j.innerHTML=R,W=M(),_=d("p"),_.innerHTML=X,k=M(),T(g.$$.fragment),h=M(),B=d("p"),B.innerHTML=z,v=M(),T(C.$$.fragment),$=M(),S=d("p"),S.innerHTML=Z,V=M(),T(fl.$$.fragment),Bl=M(),H=d("p"),H.textContent=Ye,Zl=M(),T(N.$$.fragment),Hl=M(),tl=d("p"),tl.innerHTML=Il,Ll=M(),T(al.$$.fragment),nl=M(),Ml=d("p"),Ml.innerHTML=Jl,gl=M(),T(L.$$.fragment),kl=M(),q=d("p"),q.textContent=Fe,Rl=M(),T(Y.$$.fragment),ql=M(),pl=d("p"),pl.textContent=Cl,Dl=M(),T(il.$$.fragment),F=M(),T(A.$$.fragment),rl=M(),D=d("p"),D.innerHTML=Ee,Xl=M(),T(K.$$.fragment),Wl=M(),P=d("p"),P.textContent=Se,$l=M(),T(O.$$.fragment),Vl=M(),T(ll.$$.fragment),Kl=M(),yl=d("p"),yl.textContent=el},l(i){a=m(i,"P",{"data-svelte-h":!0}),u(a)!=="svelte-tr2o92"&&(a.textContent=U),n=p(i),c(r.$$.fragment,i),G=p(i),j=m(i,"P",{"data-svelte-h":!0}),u(j)!=="svelte-8qnj9i"&&(j.innerHTML=R),W=p(i),_=m(i,"P",{"data-svelte-h":!0}),u(_)!=="svelte-7lskgm"&&(_.innerHTML=X),k=p(i),c(g.$$.fragment,i),h=p(i),B=m(i,"P",{"data-svelte-h":!0}),u(B)!=="svelte-fl551i"&&(B.innerHTML=z),v=p(i),c(C.$$.fragment,i),$=p(i),S=m(i,"P",{"data-svelte-h":!0}),u(S)!=="svelte-1vtnpzq"&&(S.innerHTML=Z),V=p(i),c(fl.$$.fragment,i),Bl=p(i),H=m(i,"P",{"data-svelte-h":!0}),u(H)!=="svelte-98lmdg"&&(H.textContent=Ye),Zl=p(i),c(N.$$.fragment,i),Hl=p(i),tl=m(i,"P",{"data-svelte-h":!0}),u(tl)!=="svelte-t6efoy"&&(tl.innerHTML=Il),Ll=p(i),c(al.$$.fragment,i),nl=p(i),Ml=m(i,"P",{"data-svelte-h":!0}),u(Ml)!=="svelte-1iqxsqn"&&(Ml.innerHTML=Jl),gl=p(i),c(L.$$.fragment,i),kl=p(i),q=m(i,"P",{"data-svelte-h":!0}),u(q)!=="svelte-lmfbd3"&&(q.textContent=Fe),Rl=p(i),c(Y.$$.fragment,i),ql=p(i),pl=m(i,"P",{"data-svelte-h":!0}),u(pl)!=="svelte-1oke1jm"&&(pl.textContent=Cl),Dl=p(i),c(il.$$.fragment,i),F=p(i),c(A.$$.fragment,i),rl=p(i),D=m(i,"P",{"data-svelte-h":!0}),u(D)!=="svelte-wma9bm"&&(D.innerHTML=Ee),Xl=p(i),c(K.$$.fragment,i),Wl=p(i),P=m(i,"P",{"data-svelte-h":!0}),u(P)!=="svelte-1f306ko"&&(P.textContent=Se),$l=p(i),c(O.$$.fragment,i),Vl=p(i),c(ll.$$.fragment,i),Kl=p(i),yl=m(i,"P",{"data-svelte-h":!0}),u(yl)!=="svelte-upd9zn"&&(yl.textContent=el)},m(i,f){t(i,a,f),t(i,n,f),o(r,i,f),t(i,G,f),t(i,j,f),t(i,W,f),t(i,_,f),t(i,k,f),o(g,i,f),t(i,h,f),t(i,B,f),t(i,v,f),o(C,i,f),t(i,$,f),t(i,S,f),t(i,V,f),o(fl,i,f),t(i,Bl,f),t(i,H,f),t(i,Zl,f),o(N,i,f),t(i,Hl,f),t(i,tl,f),t(i,Ll,f),o(al,i,f),t(i,nl,f),t(i,Ml,f),t(i,gl,f),o(L,i,f),t(i,kl,f),t(i,q,f),t(i,Rl,f),o(Y,i,f),t(i,ql,f),t(i,pl,f),t(i,Dl,f),o(il,i,f),t(i,F,f),o(A,i,f),t(i,rl,f),t(i,D,f),t(i,Xl,f),o(K,i,f),t(i,Wl,f),t(i,P,f),t(i,$l,f),o(O,i,f),t(i,Vl,f),o(ll,i,f),t(i,Kl,f),t(i,yl,f),E=!0},i(i){E||(y(r.$$.fragment,i),y(g.$$.fragment,i),y(C.$$.fragment,i),y(fl.$$.fragment,i),y(N.$$.fragment,i),y(al.$$.fragment,i),y(L.$$.fragment,i),y(Y.$$.fragment,i),y(il.$$.fragment,i),y(A.$$.fragment,i),y(K.$$.fragment,i),y(O.$$.fragment,i),y(ll.$$.fragment,i),E=!0)},o(i){J(r.$$.fragment,i),J(g.$$.fragment,i),J(C.$$.fragment,i),J(fl.$$.fragment,i),J(N.$$.fragment,i),J(al.$$.fragment,i),J(L.$$.fragment,i),J(Y.$$.fragment,i),J(il.$$.fragment,i),J(A.$$.fragment,i),J(K.$$.fragment,i),J(O.$$.fragment,i),J(ll.$$.fragment,i),E=!1},d(i){i&&(e(a),e(n),e(G),e(j),e(W),e(_),e(k),e(h),e(B),e(v),e($),e(S),e(V),e(Bl),e(H),e(Zl),e(Hl),e(tl),e(Ll),e(nl),e(Ml),e(gl),e(kl),e(q),e(Rl),e(ql),e(pl),e(Dl),e(F),e(rl),e(D),e(Xl),e(Wl),e(P),e($l),e(Vl),e(Kl),e(yl)),w(r,i),w(g,i),w(C,i),w(fl,i),w(N,i),w(al,i),w(L,i),w(Y,i),w(il,i),w(A,i),w(K,i),w(O,i),w(ll,i)}}}function WM(I){let a,U='✏️ <strong>试试看!</strong> 为了量化域适应的好处, 微调 IMDb 标签上的分类器和预先训练和微调的Distil BERT检查点。如果你需要复习文本分类, 请查看 <a href="/course/chapter3">第三章</a>。';return{c(){a=d("p"),a.innerHTML=U},l(n){a=m(n,"P",{"data-svelte-h":!0}),u(a)!=="svelte-1inh7i4"&&(a.innerHTML=U)},m(n,r){t(n,a,r)},p:Sl,d(n){n&&e(a)}}}function $M(I){let a,U,n,r,G,j,R,W,_,X,k,g,h="对于许多涉及 Transformer 模型的 NLP 程序, 你可以简单地从 Hugging Face Hub 中获取一个预训练的模型, 然后直接在你的数据上对其进行微调, 以完成手头的任务。只要用于预训练的语料库与用于微调的语料库没有太大区别, 迁移学习通常会产生很好的结果。",B,z,v="但是, 在某些情况下, 你需要先微调数据上的语言模型, 然后再训练特定于任务的head。例如, 如果您的数据集包含法律合同或科学文章, 像 BERT 这样的普通 Transformer 模型通常会将您语料库中的特定领域词视为稀有标记, 结果性能可能不尽如人意。通过在域内数据上微调语言模型, 你可以提高许多下游任务的性能, 这意味着您通常只需执行一次此步骤!",C,$,S='这种在域内数据上微调预训练语言模型的过程通常称为 <em>领域适应</em>。 它于 2018 年由 <a href="https://arxiv.org/abs/1801.06146" rel="nofollow">ULMFiT</a>推广, 这是使迁移学习真正适用于 NLP 的首批神经架构之一 (基于 LSTM)。 下图显示了使用 ULMFiT 进行域自适应的示例; 在本节中, 我们将做类似的事情, 但使用的是 Transformer 而不是 LSTM!',Z,V,fl='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/ulmfit.svg" alt="ULMFiT."/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/ulmfit-dark.svg" alt="ULMFiT."/>',Bl,H,Ye='在本节结束时, 你将在Hub上拥有一个<a href="https://huggingface.co/huggingface-course/distilbert-base-uncased-finetuned-imdb?text=This+is+a+great+%5BMASK%5D." rel="nofollow">掩码语言模型(masked language model)</a>, 该模型可以自动完成句子, 如下所示:',Zl,N,Hl,tl,Il,Ll="让我们开始吧!",al,nl,Ml,Jl,gl,L,kl,q,Fe='首先, 让我们为掩码语言建模选择一个合适的预训练模型。如以下屏幕截图所示, 你可以通过在<a href="https://huggingface.co/models?pipeline_tag=fill-mask&amp;sort=downloads" rel="nofollow">Hugging Face Hub</a>上应用”Fill-Mask”过滤器找到:',Rl,Y,ql='<img src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/mlm-models.png" alt="Hub models." width="80%"/>',pl,Cl,Dl=`尽管 BERT 和 RoBERTa 系列模型的下载量最大, 但我们将使用名为 <a href="https://huggingface.co/distilbert-base-uncased" rel="nofollow">DistilBERT</a>的模型。
可以更快地训练, 而下游性能几乎没有损失。这个模型使用一种称为<a href="https://en.wikipedia.org/wiki/Knowledge_distillation" rel="nofollow"><em>知识蒸馏</em></a>的特殊技术进行训练, 其中使用像 BERT 这样的大型“教师模型”来指导参数少得多的“学生模型”的训练。在本节中对知识蒸馏细节的解释会使我们离题太远, 但如果你有兴趣, 可以阅读所有相关内容 <a href="https://www.oreilly.com/library/view/natural-language-processing/9781098136789/" rel="nofollow"><em>Natural Language Processing with Transformers</em></a> (俗称Transformers教科书)。`,il,F,A,rl,D,Ee="DistilBERT 大约有 6700 万个参数, 大约比 BERT 基本模型小两倍, 这大致意味着训练的速度提高了两倍 — 非常棒! 现在让我们看看这个模型预测什么样的标记最有可能完成一小部分文本:",Xl,K,Wl,P,Se='作为人类, 我们可以想象 <code>[MASK]</code> 标记有很多可能性, 例如 “day”、 “ride” 或者 “painting”。对于预训练模型, 预测取决于模型所训练的语料库, 因为它会学习获取数据中存在的统计模式。与 BERT 一样, DistilBERT 在<a href="https://huggingface.co/datasets/wikipedia" rel="nofollow">English Wikipedia</a> 和 <a href="https://huggingface.co/datasets/bookcorpus" rel="nofollow">BookCorpus</a> 数据集上进行预训练, 所以我们期望对 <code>[MASK]</code> 的预测能够反映这些领域。为了预测掩码, 我们需要 DistilBERT 的标记器来生成模型的输入, 所以让我们也从 Hub 下载它:',$l,O,Vl,ll,Kl="使用标记器和模型, 我们现在可以将我们的文本示例传递给模型, 提取 logits, 并打印出前 5 个候选:",yl,el,E,i,f,Le,Pl,va="我们可以从输出中看到模型的预测是指日常用语, 鉴于英语维基百科的基础, 这也许并不奇怪。让我们看看我们如何将这个领域改变为更小众的东西 — 高度两极分化的电影评论!",qe,Ol,De,ls,Ya='为了展示域适配, 我们将使用著名的<a href="https://huggingface.co/datasets/imdb" rel="nofollow">大型电影评论数据集(Large Movie Review Dataset)</a> (或者简称为IMDb), 这是一个电影评论语料库, 通常用于对情感分析模型进行基准测试。通过在这个语料库上对 DistilBERT 进行微调, 我们预计语言模型将根据维基百科的事实数据调整其词汇表, 这些数据已经预先训练到电影评论中更主观的元素。我们可以使用🤗 Datasets中的<code>load_dataset()</code>函数从Hugging Face 中获取数据:',Ke,ss,Pe,es,Oe,ts,Fa="我们可以看到 <code>train</code> 和 <code>test</code> 每个拆分包含 25,000 条评论, 而有一个未标记的拆分称为 <code>unsupervised</code> 包含 50,000 条评论。让我们看一些示例, 以了解我们正在处理的文本类型。正如我们在本课程的前几章中所做的那样, 我们将链接 <code>Dataset.shuffle()</code> 和 <code>Dataset.select()</code> 函数创建随机样本:",lt,as,st,ns,et,Ms,Ea="是的, 这些肯定是电影评论, 如果你年龄足够,你甚至可能会理解上次评论中关于拥有 VHS 版本的评论😜! 虽然我们不需要语言建模的标签, 但我们已经可以看到 <code>0</code> 表示负面评论, 而 <code>1</code> 对应正面。",tt,zl,at,ps,Sa='现在我们已经快速浏览了数据, 让我们深入研究为掩码语言建模做准备。正如我们将看到的, 与我们在<a href="/course/chapter3">第三章</a>中看到的序列分类任务相比, 还需要采取一些额外的步骤。让我们继续!',nt,is,Mt,ys,pt,Js,Ha="对于自回归和掩码语言建模, 一个常见的预处理步骤是连接所有示例, 然后将整个语料库拆分为相同大小的块。 这与我们通常的方法完全不同, 我们只是简单地标记单个示例。为什么要将所有内容连接在一起? 原因是单个示例如果太长可能会被截断, 这将导致丢失可能对语言建模任务有用的信息!",it,rs,La='因此, 我们将像往常一样首先标记我们的语料库, 但是 <em>没有</em> 在我们的标记器中设置 <code>truncation=True</code> 选项。 我们还将获取可用的单词 ID ((如果我们使用快速标记器, 它们是可用的, 如 <a href="/course/chapter6/3">第六章</a>中所述), 因为我们稍后将需要它们来进行全字屏蔽。我们将把它包装在一个简单的函数中, 当我们在做的时候, 我们将删除 <code>text</code> 和 <code>label</code> 列, 因为我们不再需要它们:',yt,Us,Jt,Ts,rt,cs,qa="由于 DistilBERT 是一个类似 BERT 的模型, 我们可以看到编码文本由我们在其他章节中看到的 <code>input_ids</code> 和 <code>attention_mask</code> 组成, 以及我们添加的 <code>word_ids</code>。",Ut,os,Da="现在我们已经标记了我们的电影评论, 下一步是将它们组合在一起并将结果分成块。 但是这些块应该有多大? 这最终将取决于你可用的 GPU 内存量, 但一个好的起点是查看模型的最大上下文大小是多少。这可以通过检查标记器的 <code>model_max_length</code> 属性来判断:",Tt,ws,ct,ds,ot,ms,Ka="该值来自于与检查点相关联的 <em>tokenizer_config.json</em> 文件; 在这种情况下, 我们可以看到上下文大小是 512 个标记, 就像 BERT 一样。",wt,Nl,dt,js,Pa="因此, 以便在像Google Colab 那样的 GPU 上运行我们的实验, 我们将选择可以放入内存的更小一些的东西:",mt,us,jt,Al,ut,hs,Oa="有趣的来了。为了展示串联是如何工作的, 让我们从我们的标记化训练集中取一些评论并打印出每个评论的标记数量:",ht,bs,bt,fs,ft,Is,ln="然后我们可以用一个简单的字典理解来连接所有例子, 如下所示:",It,Cs,Ct,_s,_t,Gs,sn="很棒, 总长度检查出来了 — 现在, 让我们将连接的评论拆分为大小为 <code>block_size</code> 的块。为此, 我们迭代了 <code>concatenated_examples</code> 中的特征, 并使用列表理解来创建每个特征的切片。结果是每个特征的块字典:",Gt,Bs,Bt,Zs,Zt,gs,en="正如你在这个例子中看到的, 最后一个块通常会小于最大块大小。有两种主要的策略来处理这个问题:",gt,ks,tn="<li>如果最后一个块小于 <code>chunk_size</code>, 请删除它。</li> <li>填充最后一个块, 直到其长度等于 <code>chunk_size</code>。</li>",kt,Rs,an="我们将在这里采用第一种方法, 因此让我们将上述所有逻辑包装在一个函数中, 我们可以将其应用于我们的标记化数据集:",Rt,Xs,Xt,Ws,nn="注意, 在 <code>group_texts()</code> 的最后一步中, 我们创建了一个新的 <code>labels</code> 列, 它是 <code>input_ids</code> 列的副本。我们很快就会看到, 这是因为在掩码语言建模中, 目标是预测输入批次中随机掩码的标记, 并通过创建一个 <code>labels</code> 列, 们为我们的语言模型提供了基础事实以供学习。",Wt,$s,Mn="现在, 让我们使用我们可信赖的 <code>Dataset.map()</code> 函数将 <code>group_texts()</code> 应用到我们的标记化数据集:",$t,Vs,Vt,zs,zt,Ns,pn="你可以看到, 对文本进行分组, 然后对文本进行分块, 产生的示例比我们最初的 25,000 用于 <code>train</code>和 <code>test</code> 拆分的示例多得多。那是因为我们现在有了涉及 <em>连续标记</em> 的示例, 这些示例跨越了原始语料库中的多个示例。你可以通过在其中一个块中查找特殊的 <code>[SEP]</code> 和 <code>[CLS]</code> 标记来明确的看到这一点:",Nt,As,At,xs,xt,Qs,yn="在此示例中, 你可以看到两篇重叠的电影评论, 一篇关于高中电影, 另一篇关于无家可归。 让我们也看看掩码语言建模的标签是什么样的:",Qt,vs,vt,Ys,Yt,Fs,Jn="正如前面的 <code>group_texts()</code> 函数所期望的那样, 这看起来与解码后的 <code>input_ids</code> 相同 — 但是我们的模型怎么可能学到任何东西呢? 我们错过了一个关键步骤: 在输入中的随机位置插入 <code>[MASK]</code> 标记! 让我们看看如何使用特殊的数据整理器在微调期间即时执行此操作。",Ft,Es,Et,Ss,rn='微调屏蔽语言模型几乎与微调序列分类模型相同, 就像我们在 <a href="/course/chapter3">第三章</a>所作的那样。 唯一的区别是我们需要一个特殊的数据整理器, 它可以随机屏蔽每批文本中的一些标记。幸运的是, 🤗 Transformers 为这项任务准备了专用的 <code>DataCollatorForLanguageModeling</code> 。我们只需要将它转递给标记器和一个 <code>mlm_probability</code> 参数, 该参数指定要屏蔽的标记的分数。我们将选择 15%, 这是 BERT 使用的数量也是文献中的常见选择:',St,Hs,Ht,Ls,Un="要了解随机掩码的工作原理, 让我们向数据整理器提供一些示例。由于它需要一个 <code>dict</code> 的列表, 其中每个 <code>dict</code> 表示单个连续文本块, 我们首先迭代数据集, 然后再将批次提供给整理器。我们删除了这个数据整理器的 <code>&quot;word_ids&quot;</code> 键, 因为它不需要它:",Lt,qs,qt,Ds,Dt,Ks,Tn="很棒, 成功了! 我们可以看到, <code>[MASK]</code> 标记已随机插入我们文本中的不同位置。 这些将是我们的模型在训练期间必须预测的标记 — 数据整理器的美妙之处在于, 它将随机化每个批次的 <code>[MASK]</code> 插入!",Kt,xl,Pt,$e,Ps,cn="在为掩码语言建模训练模型时, 可以使用的一种技术是将整个单词一起屏蔽, 而不仅仅是单个标记。这种方法称为 <em>全词屏蔽</em>。 如果我们想使用全词屏蔽, 我们需要自己构建一个数据整理器。数据整理器只是一个函数, 它接受一个样本列表并将它们转换为一个批次, 所以现在让我们这样做吧! 我们将使用之前计算的单词 ID 在单词索引和相应标记之间进行映射, 然后随机决定要屏蔽哪些单词并将该屏蔽应用于输入。请注意, 除了与掩码对应的标签外, 所有的标签均为 <code>-100</code>。",Ot,Ul,Tl,Ve,Os,on="Next, we can try it on the same samples as before:",la,le,sa,se,ea,Ql,ta,ee,wn='现在我们有两个数据整理器, 其余的微调步骤是标准的。如果您没有足够幸运地获得神话般的 P100 GPU 😭, 在 Google Colab 上进行训练可能需要一段时间, 因此我们将首先将训练集的大小缩减为几千个示例。别担心, 我们仍然会得到一个相当不错的语言模型! 在 🤗 Datasets 中快速下采样数据集的方法是通过我们在 <a href="/course/chapter5">第五章</a> 中看到的 <code>Dataset.train_test_split()</code> 函数:',aa,te,na,ae,Ma,ne,dn="这会自动创建新的 <code>train</code> 和 <code>test</code> 拆分, 训练集大小设置为 10,000 个示例, 验证设置为其中的 10% — 如果你有一个强大的 GPU, 可以随意增加它! 我们需要做的下一件事是登录 Hugging Face Hub。如果你在笔记本中运行此代码, 则可以使用以下实用程序函数执行此操作:",pa,Me,ia,pe,mn="这将显示一个小部件, 你可以在其中输入你的凭据。或者, 你可以运行:",ya,ie,Ja,ye,jn="在你最喜欢的终端中登录。",ra,cl,ol,ze,Je,Ua,re,Ta,Ue,un="与文本分类或问答等其他任务不同, 在这些任务中, 我们会得到一个带标签的语料库进行训练, 而语言建模则没有任何明确的标签。那么我们如何确定什么是好的语言模型呢? 就像手机中的自动更正功能一样, 一个好的语言模型是为语法正确的句子分配高概率, 为无意义的句子分配低概率。为了让你更好地了解这是什么样子, 您可以在网上找到一整套 “autocorrect fails”, 其中一个人手机中的模型产生了一些相当有趣 (而且通常不合适) 的结果!",ca,wl,dl,Ne,Te,oa,ce,hn="较低的perplexity分数意味着更好的语言模型, 我们可以在这里看到我们的起始模型有一个较大的值。看看我们能不能通过微调来降低它! 为此, 我们首先运行训练循环:",wa,ml,jl,Ae,oe,bn="然后像以前一样计算测试集上的perplexity度:",da,ul,hl,xe,we,ma,de,fn="很好 — 这大大减少了困惑, 这告诉我们模型已经了解了一些关于电影评论领域的知识!",ja,Qe,vl,ua,ve,me,ha,je,In="你可以通过在Hub上使用其他小部件或在本地使用🤗 Transformers 的<code>管道</code>于微调模型进行交互。让我们使用后者来下载我们的模型, 使用 <code>fill-mask</code> 管道:",ba,ue,fa,he,Cn="然后, 我们可以向管道提供 “This is a great [MASK]” 示例文本, 并查看前 5 个预测是什么:",Ia,be,Ca,fe,_a,Ie,_n="好的 — 我们的模型显然已经调整了它的权重来预测与电影更密切相关的词!",Ga,Ce,Ba,_e,Gn='这结束了我们训练语言模型的第一个实验。在 <a href="/course/chapter7/section6">第六节</a>中你将学习如何从头开始训练像 GPT-2 这样的自回归模型; 如果你想了解如何预训练您自己的 Transformer 模型, 请前往那里!',Za,Yl,ga,He,ka;G=new JM({props:{fw:I[0]}}),R=new El({props:{title:"微调掩码语言模型",local:"微调掩码语言模型",headingTag:"h1"}});const Bn=[UM,rM],Ge=[];function Zn(l,s){return l[0]==="pt"?0:1}_=Zn(I),X=Ge[_]=Bn[_](I),nl=new Ra({props:{id:"mqElG5QJWUg"}}),Jl=new Fl({props:{$$slots:{default:[TM]},$$scope:{ctx:I}}}),L=new El({props:{title:"选择用于掩码语言建模的预训练模型",local:"选择用于掩码语言建模的预训练模型",headingTag:"h2"}});const gn=[oM,cM],Be=[];function kn(l,s){return l[0]==="pt"?0:1}F=kn(I),A=Be[F]=gn[F](I),K=new b({props:{code:"dGV4dCUyMCUzRCUyMCUyMlRoaXMlMjBpcyUyMGElMjBncmVhdCUyMCU1Qk1BU0slNUQuJTIy",highlighted:'text = <span class="hljs-string">&quot;This is a great [MASK].&quot;</span>',wrap:!1}}),O=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9jaGVja3BvaW50KQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)`,wrap:!1}});const Rn=[dM,wM],Ze=[];function Xn(l,s){return l[0]==="pt"?0:1}el=Xn(I),E=Ze[el]=Rn[el](I),f=new b({props:{code:"JyUzRSUzRSUzRSUyMFRoaXMlMjBpcyUyMGElMjBncmVhdCUyMGRlYWwuJyUwQSclM0UlM0UlM0UlMjBUaGlzJTIwaXMlMjBhJTIwZ3JlYXQlMjBzdWNjZXNzLiclMEEnJTNFJTNFJTNFJTIwVGhpcyUyMGlzJTIwYSUyMGdyZWF0JTIwYWR2ZW50dXJlLiclMEEnJTNFJTNFJTNFJTIwVGhpcyUyMGlzJTIwYSUyMGdyZWF0JTIwaWRlYS4nJTBBJyUzRSUzRSUzRSUyMFRoaXMlMjBpcyUyMGElMjBncmVhdCUyMGZlYXQuJw==",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great deal.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great success.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great adventure.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great idea.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great feat.&#x27;</span>`,wrap:!1}}),Ol=new El({props:{title:"数据集",local:"数据集",headingTag:"h2"}}),ss=new b({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBaW1kYl9kYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmltZGIlMjIpJTBBaW1kYl9kYXRhc2V0",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

imdb_dataset = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>)
imdb_dataset`,wrap:!1}}),es=new b({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ3RleHQnJTJDJTIwJ2xhYmVsJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwMjUwMDAlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTIwJTIwJTIwJTIwdGVzdCUzQSUyMERhdGFzZXQoJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmVhdHVyZXMlM0ElMjAlNUIndGV4dCclMkMlMjAnbGFiZWwnJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3Jvd3MlM0ElMjAyNTAwMCUwQSUyMCUyMCUyMCUyMCU3RCklMEElMjAlMjAlMjAlMjB1bnN1cGVydmlzZWQlM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ3RleHQnJTJDJTIwJ2xhYmVsJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwNTAwMDAlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTdEKQ==",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;text&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;text&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    unsupervised: Dataset({
        features: [<span class="hljs-string">&#x27;text&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>],
        num_rows: <span class="hljs-number">50000</span>
    })
})`,wrap:!1}}),as=new b({props:{code:"c2FtcGxlJTIwJTNEJTIwaW1kYl9kYXRhc2V0JTVCJTIydHJhaW4lMjIlNUQuc2h1ZmZsZShzZWVkJTNENDIpLnNlbGVjdChyYW5nZSgzKSklMEElMEFmb3IlMjByb3clMjBpbiUyMHNhbXBsZSUzQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlNUNuJyUzRSUzRSUzRSUyMFJldmlldyUzQSUyMCU3QnJvdyU1Qid0ZXh0JyU1RCU3RCclMjIpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiclM0UlM0UlM0UlMjBMYWJlbCUzQSUyMCU3QnJvdyU1QidsYWJlbCclNUQlN0QnJTIyKQ==",highlighted:`sample = imdb_dataset[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>))

<span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> sample:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; Review: <span class="hljs-subst">{row[<span class="hljs-string">&#x27;text&#x27;</span>]}</span>&#x27;&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Label: <span class="hljs-subst">{row[<span class="hljs-string">&#x27;label&#x27;</span>]}</span>&#x27;&quot;</span>)`,wrap:!1}}),ns=new b({props:{code:" ",highlighted:`
<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: This is your typical Priyadarshan movie--a bunch of loony characters out on some silly mission. His signature climax has the entire cast of the film coming together and fighting each other in some crazy moshpit over hidden money. Whether it is a winning lottery ticket in Malamaal Weekly, black money in Hera Pheri, &quot;kodokoo&quot; in Phir Hera Pheri, etc., etc., the director is becoming ridiculously predictable. Don\\&#x27;t get me wrong; as clichéd and preposterous his movies may be, I usually end up enjoying the comedy. However, in most his previous movies there has actually been some good humor, (Hungama and Hera Pheri being noteworthy ones). Now, the hilarity of his films is fading as he is using the same formula over and over again.&lt;br /&gt;&lt;br /&gt;Songs are good. Tanushree Datta looks awesome. Rajpal Yadav is irritating, and Tusshar is not a whole lot better. Kunal Khemu is OK, and Sharman Joshi is the best.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Label: 0&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: Okay, the story makes no sense, the characters lack any dimensionally, the best dialogue is ad-libs about the low quality of movie, the cinematography is dismal, and only editing saves a bit of the muddle, but Sam&quot; Peckinpah directed the film. Somehow, his direction is not enough. For those who appreciate Peckinpah and his great work, this movie is a disappointment. Even a great cast cannot redeem the time the viewer wastes with this minimal effort.&lt;br /&gt;&lt;br /&gt;The proper response to the movie is the contempt that the director San Peckinpah, James Caan, Robert Duvall, Burt Young, Bo Hopkins, Arthur Hill, and even Gig Young bring to their work. Watch the great Peckinpah films. Skip this mess.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Label: 0&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: I saw this movie at the theaters when I was about 6 or 7 years old. I loved it then, and have recently come to own a VHS version. &lt;br /&gt;&lt;br /&gt;My 4 and 6 year old children love this movie and have been asking again and again to watch it. &lt;br /&gt;&lt;br /&gt;I have enjoyed watching it again too. Though I have to admit it is not as good on a little TV.&lt;br /&gt;&lt;br /&gt;I do not have older children so I do not know what they would think of it. &lt;br /&gt;&lt;br /&gt;The songs are very cute. My daughter keeps singing them over and over.&lt;br /&gt;&lt;br /&gt;Hope this helps.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Label: 1&#x27;</span>`,wrap:!1}}),zl=new Fl({props:{$$slots:{default:[mM]},$$scope:{ctx:I}}}),is=new El({props:{title:"预处理数据",local:"预处理数据",headingTag:"h2"}}),ys=new Ra({props:{id:"8PmhEIXhBvI"}}),Us=new b({props:{code:"ZGVmJTIwdG9rZW5pemVfZnVuY3Rpb24oZXhhbXBsZXMpJTNBJTBBJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydGV4dCUyMiU1RCklMEElMjAlMjAlMjAlMjBpZiUyMHRva2VuaXplci5pc19mYXN0JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTVCJTIyd29yZF9pZHMlMjIlNUQlMjAlM0QlMjAlNUJyZXN1bHQud29yZF9pZHMoaSklMjBmb3IlMjBpJTIwaW4lMjByYW5nZShsZW4ocmVzdWx0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKSklNUQlMEElMjAlMjAlMjAlMjByZXR1cm4lMjByZXN1bHQlMEElMEElMEElMjMlMjBVc2UlMjBiYXRjaGVkJTNEVHJ1ZSUyMHRvJTIwYWN0aXZhdGUlMjBmYXN0JTIwbXVsdGl0aHJlYWRpbmchJTBBdG9rZW5pemVkX2RhdGFzZXRzJTIwJTNEJTIwaW1kYl9kYXRhc2V0Lm1hcCglMEElMjAlMjAlMjAlMjB0b2tlbml6ZV9mdW5jdGlvbiUyQyUyMGJhdGNoZWQlM0RUcnVlJTJDJTIwcmVtb3ZlX2NvbHVtbnMlM0QlNUIlMjJ0ZXh0JTIyJTJDJTIwJTIybGFiZWwlMjIlNUQlMEEpJTBBdG9rZW5pemVkX2RhdGFzZXRz",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_function</span>(<span class="hljs-params">examples</span>):
    result = tokenizer(examples[<span class="hljs-string">&quot;text&quot;</span>])
    <span class="hljs-keyword">if</span> tokenizer.is_fast:
        result[<span class="hljs-string">&quot;word_ids&quot;</span>] = [result.word_ids(i) <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(result[<span class="hljs-string">&quot;input_ids&quot;</span>]))]
    <span class="hljs-keyword">return</span> result


<span class="hljs-comment"># Use batched=True to activate fast multithreading!</span>
tokenized_datasets = imdb_dataset.<span class="hljs-built_in">map</span>(
    tokenize_function, batched=<span class="hljs-literal">True</span>, remove_columns=[<span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;label&quot;</span>]
)
tokenized_datasets`,wrap:!1}}),Ts=new b({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ2F0dGVudGlvbl9tYXNrJyUyQyUyMCdpbnB1dF9pZHMnJTJDJTIwJ3dvcmRfaWRzJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwMjUwMDAlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTIwJTIwJTIwJTIwdGVzdCUzQSUyMERhdGFzZXQoJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmVhdHVyZXMlM0ElMjAlNUInYXR0ZW50aW9uX21hc2snJTJDJTIwJ2lucHV0X2lkcyclMkMlMjAnd29yZF9pZHMnJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3Jvd3MlM0ElMjAyNTAwMCUwQSUyMCUyMCUyMCUyMCU3RCklMEElMjAlMjAlMjAlMjB1bnN1cGVydmlzZWQlM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ2F0dGVudGlvbl9tYXNrJyUyQyUyMCdpbnB1dF9pZHMnJTJDJTIwJ3dvcmRfaWRzJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwNTAwMDAlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTdEKQ==",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    unsupervised: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">50000</span>
    })
})`,wrap:!1}}),ws=new b({props:{code:"dG9rZW5pemVyLm1vZGVsX21heF9sZW5ndGg=",highlighted:"tokenizer.model_max_length",wrap:!1}}),ds=new b({props:{code:"NTEy",highlighted:'<span class="hljs-number">512</span>',wrap:!1}}),Nl=new Fl({props:{$$slots:{default:[jM]},$$scope:{ctx:I}}}),us=new b({props:{code:"Y2h1bmtfc2l6ZSUyMCUzRCUyMDEyOA==",highlighted:'chunk_size = <span class="hljs-number">128</span>',wrap:!1}}),Al=new Fl({props:{warning:!0,$$slots:{default:[uM]},$$scope:{ctx:I}}}),bs=new b({props:{code:"JTIzJTIwU2xpY2luZyUyMHByb2R1Y2VzJTIwYSUyMGxpc3QlMjBvZiUyMGxpc3RzJTIwZm9yJTIwZWFjaCUyMGZlYXR1cmUlMEF0b2tlbml6ZWRfc2FtcGxlcyUyMCUzRCUyMHRva2VuaXplZF9kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVEJTVCJTNBMyU1RCUwQSUwQWZvciUyMGlkeCUyQyUyMHNhbXBsZSUyMGluJTIwZW51bWVyYXRlKHRva2VuaXplZF9zYW1wbGVzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUzQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjInJTNFJTNFJTNFJTIwUmV2aWV3JTIwJTdCaWR4JTdEJTIwbGVuZ3RoJTNBJTIwJTdCbGVuKHNhbXBsZSklN0QnJTIyKQ==",highlighted:`<span class="hljs-comment"># Slicing produces a list of lists for each feature</span>
tokenized_samples = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][:<span class="hljs-number">3</span>]

<span class="hljs-keyword">for</span> idx, sample <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(tokenized_samples[<span class="hljs-string">&quot;input_ids&quot;</span>]):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Review <span class="hljs-subst">{idx}</span> length: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(sample)}</span>&#x27;&quot;</span>)`,wrap:!1}}),fs=new b({props:{code:"JyUzRSUzRSUzRSUyMFJldmlldyUyMDAlMjBsZW5ndGglM0ElMjAyMDAnJTBBJyUzRSUzRSUzRSUyMFJldmlldyUyMDElMjBsZW5ndGglM0ElMjA1NTknJTBBJyUzRSUzRSUzRSUyMFJldmlldyUyMDIlMjBsZW5ndGglM0ElMjAxOTIn",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; Review 0 length: 200&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Review 1 length: 559&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Review 2 length: 192&#x27;</span>`,wrap:!1}}),Cs=new b({props:{code:"Y29uY2F0ZW5hdGVkX2V4YW1wbGVzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwayUzQSUyMHN1bSh0b2tlbml6ZWRfc2FtcGxlcyU1QmslNUQlMkMlMjAlNUIlNUQpJTIwZm9yJTIwayUyMGluJTIwdG9rZW5pemVkX3NhbXBsZXMua2V5cygpJTBBJTdEJTBBdG90YWxfbGVuZ3RoJTIwJTNEJTIwbGVuKGNvbmNhdGVuYXRlZF9leGFtcGxlcyU1QiUyMmlucHV0X2lkcyUyMiU1RCklMEFwcmludChmJTIyJyUzRSUzRSUzRSUyMENvbmNhdGVuYXRlZCUyMHJldmlld3MlMjBsZW5ndGglM0ElMjAlN0J0b3RhbF9sZW5ndGglN0QnJTIyKQ==",highlighted:`concatenated_examples = {
    k: <span class="hljs-built_in">sum</span>(tokenized_samples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> tokenized_samples.keys()
}
total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Concatenated reviews length: <span class="hljs-subst">{total_length}</span>&#x27;&quot;</span>)`,wrap:!1}}),_s=new b({props:{code:"JyUzRSUzRSUzRSUyMENvbmNhdGVuYXRlZCUyMHJldmlld3MlMjBsZW5ndGglM0ElMjA5NTEn",highlighted:'<span class="hljs-string">&#x27;&gt;&gt;&gt; Concatenated reviews length: 951&#x27;</span>',wrap:!1}}),Bs=new b({props:{code:"Y2h1bmtzJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwayUzQSUyMCU1QnQlNUJpJTIwJTNBJTIwaSUyMCUyQiUyMGNodW5rX3NpemUlNUQlMjBmb3IlMjBpJTIwaW4lMjByYW5nZSgwJTJDJTIwdG90YWxfbGVuZ3RoJTJDJTIwY2h1bmtfc2l6ZSklNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBrJTJDJTIwdCUyMGluJTIwY29uY2F0ZW5hdGVkX2V4YW1wbGVzLml0ZW1zKCklMEElN0QlMEElMEFmb3IlMjBjaHVuayUyMGluJTIwY2h1bmtzJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiclM0UlM0UlM0UlMjBDaHVuayUyMGxlbmd0aCUzQSUyMCU3QmxlbihjaHVuayklN0QnJTIyKQ==",highlighted:`chunks = {
    k: [t[i : i + chunk_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, chunk_size)]
    <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
}

<span class="hljs-keyword">for</span> chunk <span class="hljs-keyword">in</span> chunks[<span class="hljs-string">&quot;input_ids&quot;</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Chunk length: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(chunk)}</span>&#x27;&quot;</span>)`,wrap:!1}}),Zs=new b({props:{code:"JyUzRSUzRSUzRSUyMENodW5rJTIwbGVuZ3RoJTNBJTIwMTI4JyUwQSclM0UlM0UlM0UlMjBDaHVuayUyMGxlbmd0aCUzQSUyMDEyOCclMEEnJTNFJTNFJTNFJTIwQ2h1bmslMjBsZW5ndGglM0ElMjAxMjgnJTBBJyUzRSUzRSUzRSUyMENodW5rJTIwbGVuZ3RoJTNBJTIwMTI4JyUwQSclM0UlM0UlM0UlMjBDaHVuayUyMGxlbmd0aCUzQSUyMDEyOCclMEEnJTNFJTNFJTNFJTIwQ2h1bmslMjBsZW5ndGglM0ElMjAxMjgnJTBBJyUzRSUzRSUzRSUyMENodW5rJTIwbGVuZ3RoJTNBJTIwMTI4JyUwQSclM0UlM0UlM0UlMjBDaHVuayUyMGxlbmd0aCUzQSUyMDU1Jw==",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 55&#x27;</span>`,wrap:!1}}),Xs=new b({props:{code:"ZGVmJTIwZ3JvdXBfdGV4dHMoZXhhbXBsZXMpJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwQ29uY2F0ZW5hdGUlMjBhbGwlMjB0ZXh0cyUwQSUyMCUyMCUyMCUyMGNvbmNhdGVuYXRlZF9leGFtcGxlcyUyMCUzRCUyMCU3QmslM0ElMjBzdW0oZXhhbXBsZXMlNUJrJTVEJTJDJTIwJTVCJTVEKSUyMGZvciUyMGslMjBpbiUyMGV4YW1wbGVzLmtleXMoKSU3RCUwQSUyMCUyMCUyMCUyMCUyMyUyMENvbXB1dGUlMjBsZW5ndGglMjBvZiUyMGNvbmNhdGVuYXRlZCUyMHRleHRzJTBBJTIwJTIwJTIwJTIwdG90YWxfbGVuZ3RoJTIwJTNEJTIwbGVuKGNvbmNhdGVuYXRlZF9leGFtcGxlcyU1Qmxpc3QoZXhhbXBsZXMua2V5cygpKSU1QjAlNUQlNUQpJTBBJTIwJTIwJTIwJTIwJTIzJTIwV2UlMjBkcm9wJTIwdGhlJTIwbGFzdCUyMGNodW5rJTIwaWYlMjBpdCdzJTIwc21hbGxlciUyMHRoYW4lMjBjaHVua19zaXplJTBBJTIwJTIwJTIwJTIwdG90YWxfbGVuZ3RoJTIwJTNEJTIwKHRvdGFsX2xlbmd0aCUyMCUyRiUyRiUyMGNodW5rX3NpemUpJTIwKiUyMGNodW5rX3NpemUlMEElMjAlMjAlMjAlMjAlMjMlMjBTcGxpdCUyMGJ5JTIwY2h1bmtzJTIwb2YlMjBtYXhfbGVuJTBBJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwayUzQSUyMCU1QnQlNUJpJTIwJTNBJTIwaSUyMCUyQiUyMGNodW5rX3NpemUlNUQlMjBmb3IlMjBpJTIwaW4lMjByYW5nZSgwJTJDJTIwdG90YWxfbGVuZ3RoJTJDJTIwY2h1bmtfc2l6ZSklNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBrJTJDJTIwdCUyMGluJTIwY29uY2F0ZW5hdGVkX2V4YW1wbGVzLml0ZW1zKCklMEElMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjAlMjMlMjBDcmVhdGUlMjBhJTIwbmV3JTIwbGFiZWxzJTIwY29sdW1uJTBBJTIwJTIwJTIwJTIwcmVzdWx0JTVCJTIybGFiZWxzJTIyJTVEJTIwJTNEJTIwcmVzdWx0JTVCJTIyaW5wdXRfaWRzJTIyJTVELmNvcHkoKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJlc3VsdA==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">group_texts</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-comment"># Concatenate all texts</span>
    concatenated_examples = {k: <span class="hljs-built_in">sum</span>(examples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> examples.keys()}
    <span class="hljs-comment"># Compute length of concatenated texts</span>
    total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-built_in">list</span>(examples.keys())[<span class="hljs-number">0</span>]])
    <span class="hljs-comment"># We drop the last chunk if it&#x27;s smaller than chunk_size</span>
    total_length = (total_length // chunk_size) * chunk_size
    <span class="hljs-comment"># Split by chunks of max_len</span>
    result = {
        k: [t[i : i + chunk_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, chunk_size)]
        <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
    }
    <span class="hljs-comment"># Create a new labels column</span>
    result[<span class="hljs-string">&quot;labels&quot;</span>] = result[<span class="hljs-string">&quot;input_ids&quot;</span>].copy()
    <span class="hljs-keyword">return</span> result`,wrap:!1}}),Vs=new b({props:{code:"bG1fZGF0YXNldHMlMjAlM0QlMjB0b2tlbml6ZWRfZGF0YXNldHMubWFwKGdyb3VwX3RleHRzJTJDJTIwYmF0Y2hlZCUzRFRydWUpJTBBbG1fZGF0YXNldHM=",highlighted:`lm_datasets = tokenized_datasets.<span class="hljs-built_in">map</span>(group_texts, batched=<span class="hljs-literal">True</span>)
lm_datasets`,wrap:!1}}),zs=new b({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ2F0dGVudGlvbl9tYXNrJyUyQyUyMCdpbnB1dF9pZHMnJTJDJTIwJ2xhYmVscyclMkMlMjAnd29yZF9pZHMnJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3Jvd3MlM0ElMjA2MTI4OSUwQSUyMCUyMCUyMCUyMCU3RCklMEElMjAlMjAlMjAlMjB0ZXN0JTNBJTIwRGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidhdHRlbnRpb25fbWFzayclMkMlMjAnaW5wdXRfaWRzJyUyQyUyMCdsYWJlbHMnJTJDJTIwJ3dvcmRfaWRzJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwNTk5MDUlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTIwJTIwJTIwJTIwdW5zdXBlcnZpc2VkJTNBJTIwRGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidhdHRlbnRpb25fbWFzayclMkMlMjAnaW5wdXRfaWRzJyUyQyUyMCdsYWJlbHMnJTJDJTIwJ3dvcmRfaWRzJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwMTIyOTYzJTBBJTIwJTIwJTIwJTIwJTdEKSUwQSU3RCk=",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">61289</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">59905</span>
    })
    unsupervised: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">122963</span>
    })
})`,wrap:!1}}),As=new b({props:{code:"dG9rZW5pemVyLmRlY29kZShsbV9kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVEJTVCMSU1RCU1QiUyMmlucHV0X2lkcyUyMiU1RCk=",highlighted:'tokenizer.decode(lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;input_ids&quot;</span>])',wrap:!1}}),xs=new b({props:{code:"JTIyLi4uLiUyMGF0Li4uLi4uLi4uLiUyMGhpZ2guJTIwYSUyMGNsYXNzaWMlMjBsaW5lJTIwJTNBJTIwaW5zcGVjdG9yJTIwJTNBJTIwaSdtJTIwaGVyZSUyMHRvJTIwc2FjayUyMG9uZSUyMG9mJTIweW91ciUyMHRlYWNoZXJzLiUyMHN0dWRlbnQlMjAlM0ElMjB3ZWxjb21lJTIwdG8lMjBicm9td2VsbCUyMGhpZ2guJTIwaSUyMGV4cGVjdCUyMHRoYXQlMjBtYW55JTIwYWR1bHRzJTIwb2YlMjBteSUyMGFnZSUyMHRoaW5rJTIwdGhhdCUyMGJyb213ZWxsJTIwaGlnaCUyMGlzJTIwZmFyJTIwZmV0Y2hlZC4lMjB3aGF0JTIwYSUyMHBpdHklMjB0aGF0JTIwaXQlMjBpc24ndCElMjAlNUJTRVAlNUQlMjAlNUJDTFMlNUQlMjBob21lbGVzc25lc3MlMjAoJTIwb3IlMjBob3VzZWxlc3NuZXNzJTIwYXMlMjBnZW9yZ2UlMjBjYXJsaW4lMjBzdGF0ZWQlMjApJTIwaGFzJTIwYmVlbiUyMGFuJTIwaXNzdWUlMjBmb3IlMjB5ZWFycyUyMGJ1dCUyMG5ldmVyJTIwYSUyMHBsYW4lMjB0byUyMGhlbHAlMjB0aG9zZSUyMG9uJTIwdGhlJTIwc3RyZWV0JTIwdGhhdCUyMHdlcmUlMjBvbmNlJTIwY29uc2lkZXJlZCUyMGh1bWFuJTIwd2hvJTIwZGlkJTIwZXZlcnl0aGluZyUyMGZyb20lMjBnb2luZyUyMHRvJTIwc2Nob29sJTJDJTIwd29yayUyQyUyMG9yJTIwdm90ZSUyMGZvciUyMHRoZSUyMG1hdHRlci4lMjBtb3N0JTIwcGVvcGxlJTIwdGhpbmslMjBvZiUyMHRoZSUyMGhvbWVsZXNzJTIy",highlighted:'<span class="hljs-string">&quot;.... at.......... high. a classic line : inspector : i&#x27;m here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn&#x27;t! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless&quot;</span>',wrap:!1}}),vs=new b({props:{code:"dG9rZW5pemVyLmRlY29kZShsbV9kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVEJTVCMSU1RCU1QiUyMmxhYmVscyUyMiU1RCk=",highlighted:'tokenizer.decode(lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;labels&quot;</span>])',wrap:!1}}),Ys=new b({props:{code:"JTIyLi4uLiUyMGF0Li4uLi4uLi4uLiUyMGhpZ2guJTIwYSUyMGNsYXNzaWMlMjBsaW5lJTIwJTNBJTIwaW5zcGVjdG9yJTIwJTNBJTIwaSdtJTIwaGVyZSUyMHRvJTIwc2FjayUyMG9uZSUyMG9mJTIweW91ciUyMHRlYWNoZXJzLiUyMHN0dWRlbnQlMjAlM0ElMjB3ZWxjb21lJTIwdG8lMjBicm9td2VsbCUyMGhpZ2guJTIwaSUyMGV4cGVjdCUyMHRoYXQlMjBtYW55JTIwYWR1bHRzJTIwb2YlMjBteSUyMGFnZSUyMHRoaW5rJTIwdGhhdCUyMGJyb213ZWxsJTIwaGlnaCUyMGlzJTIwZmFyJTIwZmV0Y2hlZC4lMjB3aGF0JTIwYSUyMHBpdHklMjB0aGF0JTIwaXQlMjBpc24ndCElMjAlNUJTRVAlNUQlMjAlNUJDTFMlNUQlMjBob21lbGVzc25lc3MlMjAoJTIwb3IlMjBob3VzZWxlc3NuZXNzJTIwYXMlMjBnZW9yZ2UlMjBjYXJsaW4lMjBzdGF0ZWQlMjApJTIwaGFzJTIwYmVlbiUyMGFuJTIwaXNzdWUlMjBmb3IlMjB5ZWFycyUyMGJ1dCUyMG5ldmVyJTIwYSUyMHBsYW4lMjB0byUyMGhlbHAlMjB0aG9zZSUyMG9uJTIwdGhlJTIwc3RyZWV0JTIwdGhhdCUyMHdlcmUlMjBvbmNlJTIwY29uc2lkZXJlZCUyMGh1bWFuJTIwd2hvJTIwZGlkJTIwZXZlcnl0aGluZyUyMGZyb20lMjBnb2luZyUyMHRvJTIwc2Nob29sJTJDJTIwd29yayUyQyUyMG9yJTIwdm90ZSUyMGZvciUyMHRoZSUyMG1hdHRlci4lMjBtb3N0JTIwcGVvcGxlJTIwdGhpbmslMjBvZiUyMHRoZSUyMGhvbWVsZXNzJTIy",highlighted:'<span class="hljs-string">&quot;.... at.......... high. a classic line : inspector : i&#x27;m here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn&#x27;t! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless&quot;</span>',wrap:!1}}),Es=new El({props:{title:"使用 Trainer API 微调DistilBERT",local:"使用 Trainer API 微调DistilBERT",headingTag:"h2"}}),Hs=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvckxhbmd1YWdlTW9kZWxpbmclMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yTGFuZ3VhZ2VNb2RlbGluZyh0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMjBtbG1fcHJvYmFiaWxpdHklM0QwLjE1KQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=<span class="hljs-number">0.15</span>)`,wrap:!1}}),qs=new b({props:{code:"c2FtcGxlcyUyMCUzRCUyMCU1QmxtX2RhdGFzZXRzJTVCJTIydHJhaW4lMjIlNUQlNUJpJTVEJTIwZm9yJTIwaSUyMGluJTIwcmFuZ2UoMiklNUQlMEFmb3IlMjBzYW1wbGUlMjBpbiUyMHNhbXBsZXMlM0ElMEElMjAlMjAlMjAlMjBfJTIwJTNEJTIwc2FtcGxlLnBvcCglMjJ3b3JkX2lkcyUyMiklMEElMEFmb3IlMjBjaHVuayUyMGluJTIwZGF0YV9jb2xsYXRvcihzYW1wbGVzKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUzQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlNUNuJyUzRSUzRSUzRSUyMCU3QnRva2VuaXplci5kZWNvZGUoY2h1bmspJTdEJyUyMik=",highlighted:`samples = [lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>)]
<span class="hljs-keyword">for</span> sample <span class="hljs-keyword">in</span> samples:
    _ = sample.pop(<span class="hljs-string">&quot;word_ids&quot;</span>)

<span class="hljs-keyword">for</span> chunk <span class="hljs-keyword">in</span> data_collator(samples)[<span class="hljs-string">&quot;input_ids&quot;</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; <span class="hljs-subst">{tokenizer.decode(chunk)}</span>&#x27;&quot;</span>)`,wrap:!1}}),Ds=new b({props:{code:"JyUzRSUzRSUzRSUyMCU1QkNMUyU1RCUyMGJyb213ZWxsJTIwJTVCTUFTSyU1RCUyMGlzJTIwYSUyMGNhcnRvb24lMjBjb21lZHkuJTIwaXQlMjByYW4lMjBhdCUyMHRoZSUyMHNhbWUlMjAlNUJNQVNLJTVEJTIwYXMlMjBzb21lJTIwb3RoZXIlMjAlNUJNQVNLJTVEJTIwYWJvdXQlMjBzY2hvb2wlMjBsaWZlJTJDJTIwJTVCTUFTSyU1RCUyMGFzJTIwJTIyJTIwdGVhY2hlcnMlMjAlMjIuJTIwJTVCTUFTSyU1RCUyMCU1Qk1BU0slNUQlMjAlNUJNQVNLJTVEJTIwaW4lMjB0aGUlMjB0ZWFjaGluZyUyMCU1Qk1BU0slNUQlMjBsZWFkJTIwJTVCTUFTSyU1RCUyMHRvJTIwYmVsaWV2ZSUyMHRoYXQlMjBicm9td2VsbCUyMGhpZ2glNUMnJTVCTUFTSyU1RCUyMHNhdGlyZSUyMGlzJTIwbXVjaCUyMGNsb3NlciUyMHRvJTIwcmVhbGl0eSUyMHRoYW4lMjBpcyUyMCUyMiUyMHRlYWNoZXJzJTIwJTIyLiUyMHRoZSUyMHNjcmFtYmxlJTIwJTVCTUFTSyU1RCUyMCU1Qk1BU0slNUQlMjBmaW5hbmNpYWxseSUyQyUyMHRoZSUyMCU1Qk1BU0slNURmdWwlMjBzdHVkZW50cyUyMHdob2duJTIwJTVCTUFTSyU1RCUyMHJpZ2h0JTIwdGhyb3VnaCUyMCU1Qk1BU0slNUQlMjBwYXRoZXRpYyUyMHRlYWNoZXJzJTVDJ3BvbXAlMkMlMjB0aGUlMjBwZXR0aW5lc3MlMjBvZiUyMHRoZSUyMHdob2xlJTIwc2l0dWF0aW9uJTJDJTIwZGlzdGluY3Rpb24lMjByZW1pbmQlMjBtZSUyMG9mJTIwdGhlJTIwc2Nob29scyUyMGklMjBrbmV3JTIwYW5kJTIwdGhlaXIlMjBzdHVkZW50cy4lMjB3aGVuJTIwaSUyMHNhdyUyMCU1Qk1BU0slNUQlMjBlcGlzb2RlJTIwaW4lMjAlNUJNQVNLJTVEJTIwYSUyMHN0dWRlbnQlMjByZXBlYXRlZGx5JTIwdHJpZWQlMjB0byUyMGJ1cm4lMjBkb3duJTIwdGhlJTIwc2Nob29sJTJDJTIwJTVCTUFTSyU1RCUyMGltbWVkaWF0ZWx5JTIwcmVjYWxsZWQuJTIwJTVCTUFTSyU1RC4uLiclMEElMEEnJTNFJTNFJTNFJTIwLi4uLiUyMGF0Li4lMjAlNUJNQVNLJTVELi4uJTIwJTVCTUFTSyU1RC4uLiUyMGhpZ2guJTIwYSUyMGNsYXNzaWMlMjBsaW5lJTIwcGx1Y2tlZCUyMGluc3BlY3RvciUyMCUzQSUyMGklNUMnJTVCTUFTSyU1RCUyMGhlcmUlMjB0byUyMCU1Qk1BU0slNUQlMjBvbmUlMjBvZiUyMHlvdXIlMjAlNUJNQVNLJTVELiUyMHN0dWRlbnQlMjAlM0ElMjB3ZWxjb21lJTIwdG8lMjBicm9td2VsbCUyMCU1Qk1BU0slNUQuJTIwaSUyMGV4cGVjdCUyMHRoYXQlMjBtYW55JTIwYWR1bHRzJTIwb2YlMjBteSUyMGFnZSUyMHRoaW5rJTIwdGhhdCUyMCU1Qk1BU0slNURtd2VsbCUyMCU1Qk1BU0slNUQlMjBpcyUyMCU1Qk1BU0slNUQlMjBmZXRjaGVkLiUyMHdoYXQlMjBhJTIwcGl0eSUyMHRoYXQlMjBpdCUyMGlzbiU1Qyd0ISUyMCU1QlNFUCU1RCUyMCU1QkNMUyU1RCUyMCU1Qk1BU0slNURuZXNzJTIwKCUyMG9yJTIwJTVCTUFTSyU1RGxlc3NuZXNzJTIwYXMlMjBnZW9yZ2UlMjAlRTUlQUUlODdpbiUyMHN0YXRlZCUyMCklRTUlODUlQUMlMjBiZWVuJTIwYW4lMjBpc3N1ZSUyMGZvciUyMHllYXJzJTIwYnV0JTIwbmV2ZXIlMjAlNUJNQVNLJTVEJTIwcGxhbiUyMHRvJTIwaGVscCUyMHRob3NlJTIwb24lMjB0aGUlMjBzdHJlZXQlMjB0aGF0JTIwd2VyZSUyMG9uY2UlMjBjb25zaWRlcmVkJTIwaHVtYW4lMjAlNUJNQVNLJTVEJTIwZGlkJTIwZXZlcnl0aGluZyUyMGZyb20lMjBnb2luZyUyMHRvJTIwc2Nob29sJTJDJTIwJTVCTUFTSyU1RCUyQyUyMCU1Qk1BU0slNUQlMjB2b3RlJTIwZm9yJTIwdGhlJTIwbWF0dGVyLiUyMG1vc3QlMjBwZW9wbGUlMjB0aGluayUyMCU1Qk1BU0slNUQlMjB0aGUlMjBob21lbGVzcyc=",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; [CLS] bromwell [MASK] is a cartoon comedy. it ran at the same [MASK] as some other [MASK] about school life, [MASK] as &quot; teachers &quot;. [MASK] [MASK] [MASK] in the teaching [MASK] lead [MASK] to believe that bromwell high\\&#x27;[MASK] satire is much closer to reality than is &quot; teachers &quot;. the scramble [MASK] [MASK] financially, the [MASK]ful students whogn [MASK] right through [MASK] pathetic teachers\\&#x27;pomp, the pettiness of the whole situation, distinction remind me of the schools i knew and their students. when i saw [MASK] episode in [MASK] a student repeatedly tried to burn down the school, [MASK] immediately recalled. [MASK]...&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; .... at.. [MASK]... [MASK]... high. a classic line plucked inspector : i\\&#x27;[MASK] here to [MASK] one of your [MASK]. student : welcome to bromwell [MASK]. i expect that many adults of my age think that [MASK]mwell [MASK] is [MASK] fetched. what a pity that it isn\\&#x27;t! [SEP] [CLS] [MASK]ness ( or [MASK]lessness as george 宇in stated )公 been an issue for years but never [MASK] plan to help those on the street that were once considered human [MASK] did everything from going to school, [MASK], [MASK] vote for the matter. most people think [MASK] the homeless&#x27;</span>`,wrap:!1}}),xl=new Fl({props:{$$slots:{default:[hM]},$$scope:{ctx:I}}});let sl=I[0]==="pt"&&On();const Wn=[fM,bM],ge=[];function $n(l,s){return l[0]==="pt"?0:1}Ul=$n(I),Tl=ge[Ul]=Wn[Ul](I),le=new b({props:{code:"c2FtcGxlcyUyMCUzRCUyMCU1QmxtX2RhdGFzZXRzJTVCJTIydHJhaW4lMjIlNUQlNUJpJTVEJTIwZm9yJTIwaSUyMGluJTIwcmFuZ2UoMiklNUQlMEFiYXRjaCUyMCUzRCUyMHdob2xlX3dvcmRfbWFza2luZ19kYXRhX2NvbGxhdG9yKHNhbXBsZXMpJTBBJTBBZm9yJTIwY2h1bmslMjBpbiUyMGJhdGNoJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiU1Q24nJTNFJTNFJTNFJTIwJTdCdG9rZW5pemVyLmRlY29kZShjaHVuayklN0QnJTIyKQ==",highlighted:`samples = [lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>)]
batch = whole_word_masking_data_collator(samples)

<span class="hljs-keyword">for</span> chunk <span class="hljs-keyword">in</span> batch[<span class="hljs-string">&quot;input_ids&quot;</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; <span class="hljs-subst">{tokenizer.decode(chunk)}</span>&#x27;&quot;</span>)`,wrap:!1}}),se=new b({props:{code:"JyUzRSUzRSUzRSUyMCU1QkNMUyU1RCUyMGJyb213ZWxsJTIwaGlnaCUyMGlzJTIwYSUyMGNhcnRvb24lMjBjb21lZHklMjAlNUJNQVNLJTVEJTIwaXQlMjByYW4lMjBhdCUyMHRoZSUyMHNhbWUlMjB0aW1lJTIwYXMlMjBzb21lJTIwb3RoZXIlMjBwcm9ncmFtcyUyMGFib3V0JTIwc2Nob29sJTIwbGlmZSUyQyUyMHN1Y2glMjBhcyUyMCUyMiUyMHRlYWNoZXJzJTIwJTIyLiUyMG15JTIwMzUlMjB5ZWFycyUyMGluJTIwdGhlJTIwdGVhY2hpbmclMjBwcm9mZXNzaW9uJTIwbGVhZCUyMG1lJTIwdG8lMjBiZWxpZXZlJTIwdGhhdCUyMGJyb213ZWxsJTIwaGlnaCU1QydzJTIwc2F0aXJlJTIwaXMlMjBtdWNoJTIwY2xvc2VyJTIwdG8lMjByZWFsaXR5JTIwdGhhbiUyMGlzJTIwJTIyJTIwdGVhY2hlcnMlMjAlMjIuJTIwdGhlJTIwc2NyYW1ibGUlMjB0byUyMHN1cnZpdmUlMjBmaW5hbmNpYWxseSUyQyUyMHRoZSUyMGluc2lnaHRmdWwlMjBzdHVkZW50cyUyMHdobyUyMGNhbiUyMHNlZSUyMHJpZ2h0JTIwdGhyb3VnaCUyMHRoZWlyJTIwcGF0aGV0aWMlMjB0ZWFjaGVycyU1Qydwb21wJTJDJTIwdGhlJTIwcGV0dGluZXNzJTIwb2YlMjB0aGUlMjB3aG9sZSUyMHNpdHVhdGlvbiUyQyUyMGFsbCUyMHJlbWluZCUyMG1lJTIwb2YlMjB0aGUlMjBzY2hvb2xzJTIwaSUyMGtuZXclMjBhbmQlMjB0aGVpciUyMHN0dWRlbnRzLiUyMHdoZW4lMjBpJTIwc2F3JTIwdGhlJTIwZXBpc29kZSUyMGluJTIwd2hpY2glMjBhJTIwc3R1ZGVudCUyMHJlcGVhdGVkbHklMjB0cmllZCUyMHRvJTIwYnVybiUyMGRvd24lMjB0aGUlMjBzY2hvb2wlMkMlMjBpJTIwaW1tZWRpYXRlbHklMjByZWNhbGxlZC4uLi4uJyUwQSUwQSclM0UlM0UlM0UlMjAuLi4uJTIwJTVCTUFTSyU1RCUyMCU1Qk1BU0slNUQlMjAlNUJNQVNLJTVEJTIwJTVCTUFTSyU1RC4uLi4uLi4lMjBoaWdoLiUyMGElMjBjbGFzc2ljJTIwbGluZSUyMCUzQSUyMGluc3BlY3RvciUyMCUzQSUyMGklNUMnbSUyMGhlcmUlMjB0byUyMHNhY2slMjBvbmUlMjBvZiUyMHlvdXIlMjB0ZWFjaGVycy4lMjBzdHVkZW50JTIwJTNBJTIwd2VsY29tZSUyMHRvJTIwYnJvbXdlbGwlMjBoaWdoLiUyMGklMjBleHBlY3QlMjB0aGF0JTIwbWFueSUyMGFkdWx0cyUyMG9mJTIwbXklMjBhZ2UlMjB0aGluayUyMHRoYXQlMjBicm9td2VsbCUyMGhpZ2glMjBpcyUyMGZhciUyMGZldGNoZWQuJTIwd2hhdCUyMGElMjBwaXR5JTIwdGhhdCUyMGl0JTIwaXNuJTVDJ3QhJTIwJTVCU0VQJTVEJTIwJTVCQ0xTJTVEJTIwaG9tZWxlc3NuZXNzJTIwKCUyMG9yJTIwaG91c2VsZXNzbmVzcyUyMGFzJTIwZ2VvcmdlJTIwY2FybGluJTIwc3RhdGVkJTIwKSUyMGhhcyUyMGJlZW4lMjBhbiUyMGlzc3VlJTIwZm9yJTIweWVhcnMlMjBidXQlMjBuZXZlciUyMGElMjBwbGFuJTIwdG8lMjBoZWxwJTIwdGhvc2UlMjBvbiUyMHRoZSUyMHN0cmVldCUyMHRoYXQlMjB3ZXJlJTIwb25jZSUyMGNvbnNpZGVyZWQlMjBodW1hbiUyMHdobyUyMGRpZCUyMGV2ZXJ5dGhpbmclMjBmcm9tJTIwZ29pbmclMjB0byUyMHNjaG9vbCUyQyUyMHdvcmslMkMlMjBvciUyMHZvdGUlMjBmb3IlMjB0aGUlMjBtYXR0ZXIuJTIwbW9zdCUyMHBlb3BsZSUyMHRoaW5rJTIwb2YlMjB0aGUlMjBob21lbGVzcyc=",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; [CLS] bromwell high is a cartoon comedy [MASK] it ran at the same time as some other programs about school life, such as &quot; teachers &quot;. my 35 years in the teaching profession lead me to believe that bromwell high\\&#x27;s satire is much closer to reality than is &quot; teachers &quot;. the scramble to survive financially, the insightful students who can see right through their pathetic teachers\\&#x27;pomp, the pettiness of the whole situation, all remind me of the schools i knew and their students. when i saw the episode in which a student repeatedly tried to burn down the school, i immediately recalled.....&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; .... [MASK] [MASK] [MASK] [MASK]....... high. a classic line : inspector : i\\&#x27;m here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn\\&#x27;t! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless&#x27;</span>`,wrap:!1}}),Ql=new Fl({props:{$$slots:{default:[IM]},$$scope:{ctx:I}}}),te=new b({props:{code:"dHJhaW5fc2l6ZSUyMCUzRCUyMDEwXzAwMCUwQXRlc3Rfc2l6ZSUyMCUzRCUyMGludCgwLjElMjAqJTIwdHJhaW5fc2l6ZSklMEElMEFkb3duc2FtcGxlZF9kYXRhc2V0JTIwJTNEJTIwbG1fZGF0YXNldHMlNUIlMjJ0cmFpbiUyMiU1RC50cmFpbl90ZXN0X3NwbGl0KCUwQSUyMCUyMCUyMCUyMHRyYWluX3NpemUlM0R0cmFpbl9zaXplJTJDJTIwdGVzdF9zaXplJTNEdGVzdF9zaXplJTJDJTIwc2VlZCUzRDQyJTBBKSUwQWRvd25zYW1wbGVkX2RhdGFzZXQ=",highlighted:`train_size = <span class="hljs-number">10_000</span>
test_size = <span class="hljs-built_in">int</span>(<span class="hljs-number">0.1</span> * train_size)

downsampled_dataset = lm_datasets[<span class="hljs-string">&quot;train&quot;</span>].train_test_split(
    train_size=train_size, test_size=test_size, seed=<span class="hljs-number">42</span>
)
downsampled_dataset`,wrap:!1}}),ae=new b({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ2F0dGVudGlvbl9tYXNrJyUyQyUyMCdpbnB1dF9pZHMnJTJDJTIwJ2xhYmVscyclMkMlMjAnd29yZF9pZHMnJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3Jvd3MlM0ElMjAxMDAwMCUwQSUyMCUyMCUyMCUyMCU3RCklMEElMjAlMjAlMjAlMjB0ZXN0JTNBJTIwRGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidhdHRlbnRpb25fbWFzayclMkMlMjAnaW5wdXRfaWRzJyUyQyUyMCdsYWJlbHMnJTJDJTIwJ3dvcmRfaWRzJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwMTAwMCUwQSUyMCUyMCUyMCUyMCU3RCklMEElN0Qp",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">10000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">1000</span>
    })
})`,wrap:!1}}),Me=new b({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,wrap:!1}}),ie=new b({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}});const Vn=[_M,CM],ke=[];function zn(l,s){return l[0]==="tf"?0:1}cl=zn(I),ol=ke[cl]=Vn[cl](I),Je=new El({props:{title:"语言模型的perplexity",local:"语言模型的perplexity",headingTag:"h3"}}),re=new Ra({props:{id:"NURcDHhYe98"}});const Nn=[BM,GM],Re=[];function An(l,s){return l[0]==="pt"?0:1}wl=An(I),dl=Re[wl]=Nn[wl](I),Te=new b({props:{code:"UGVycGxleGl0eSUzQSUyMDIxLjc1",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>Perplexity: <span class="hljs-number">21.75</span>',wrap:!1}});const xn=[gM,ZM],Xe=[];function Qn(l,s){return l[0]==="pt"?0:1}ml=Qn(I),jl=Xe[ml]=xn[ml](I);const vn=[RM,kM],We=[];function Yn(l,s){return l[0]==="pt"?0:1}ul=Yn(I),hl=We[ul]=vn[ul](I),we=new b({props:{code:"UGVycGxleGl0eSUzQSUyMDExLjMy",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>Perplexity: <span class="hljs-number">11.32</span>',wrap:!1}});let x=I[0]==="pt"&&lM();vl=new Fl({props:{$$slots:{default:[XM]},$$scope:{ctx:I}}});let Q=I[0]==="pt"&&sM();return me=new El({props:{title:"使用我们微调的模型",local:"使用我们微调的模型",headingTag:"h2"}}),ue=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBbWFza19maWxsZXIlMjAlM0QlMjBwaXBlbGluZSglMEElMjAlMjAlMjAlMjAlMjJmaWxsLW1hc2slMjIlMkMlMjBtb2RlbCUzRCUyMmh1Z2dpbmdmYWNlLWNvdXJzZSUyRmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkLWZpbmV0dW5lZC1pbWRiJTIyJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

mask_filler = pipeline(
    <span class="hljs-string">&quot;fill-mask&quot;</span>, model=<span class="hljs-string">&quot;huggingface-course/distilbert-base-uncased-finetuned-imdb&quot;</span>
)`,wrap:!1}}),be=new b({props:{code:"cHJlZHMlMjAlM0QlMjBtYXNrX2ZpbGxlcih0ZXh0KSUwQSUwQWZvciUyMHByZWQlMjBpbiUyMHByZWRzJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiUzRSUzRSUzRSUyMCU3QnByZWQlNUInc2VxdWVuY2UnJTVEJTdEJTIyKQ==",highlighted:`preds = mask_filler(text)

<span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> preds:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; <span class="hljs-subst">{pred[<span class="hljs-string">&#x27;sequence&#x27;</span>]}</span>&quot;</span>)`,wrap:!1}}),fe=new b({props:{code:"JyUzRSUzRSUzRSUyMHRoaXMlMjBpcyUyMGElMjBncmVhdCUyMG1vdmllLiclMEEnJTNFJTNFJTNFJTIwdGhpcyUyMGlzJTIwYSUyMGdyZWF0JTIwZmlsbS4nJTBBJyUzRSUzRSUzRSUyMHRoaXMlMjBpcyUyMGElMjBncmVhdCUyMHN0b3J5LiclMEEnJTNFJTNFJTNFJTIwdGhpcyUyMGlzJTIwYSUyMGdyZWF0JTIwbW92aWVzLiclMEEnJTNFJTNFJTNFJTIwdGhpcyUyMGlzJTIwYSUyMGdyZWF0JTIwY2hhcmFjdGVyLic=",highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great movie.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great film.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great story.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great movies.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great character.&#x27;</span>`,wrap:!1}}),Ce=new Ra({props:{id:"0Oxphw4Q9fo"}}),Yl=new Fl({props:{$$slots:{default:[WM]},$$scope:{ctx:I}}}),{c(){a=d("meta"),U=M(),n=d("p"),r=M(),T(G.$$.fragment),j=M(),T(R.$$.fragment),W=M(),X.c(),k=M(),g=d("p"),g.textContent=h,B=M(),z=d("p"),z.textContent=v,C=M(),$=d("p"),$.innerHTML=S,Z=M(),V=d("div"),V.innerHTML=fl,Bl=M(),H=d("p"),H.innerHTML=Ye,Zl=M(),N=d("iframe"),tl=M(),Il=d("p"),Il.textContent=Ll,al=M(),T(nl.$$.fragment),Ml=M(),T(Jl.$$.fragment),gl=M(),T(L.$$.fragment),kl=M(),q=d("p"),q.innerHTML=Fe,Rl=M(),Y=d("div"),Y.innerHTML=ql,pl=M(),Cl=d("p"),Cl.innerHTML=Dl,il=M(),A.c(),rl=M(),D=d("p"),D.textContent=Ee,Xl=M(),T(K.$$.fragment),Wl=M(),P=d("p"),P.innerHTML=Se,$l=M(),T(O.$$.fragment),Vl=M(),ll=d("p"),ll.textContent=Kl,yl=M(),E.c(),i=M(),T(f.$$.fragment),Le=M(),Pl=d("p"),Pl.textContent=va,qe=M(),T(Ol.$$.fragment),De=M(),ls=d("p"),ls.innerHTML=Ya,Ke=M(),T(ss.$$.fragment),Pe=M(),T(es.$$.fragment),Oe=M(),ts=d("p"),ts.innerHTML=Fa,lt=M(),T(as.$$.fragment),st=M(),T(ns.$$.fragment),et=M(),Ms=d("p"),Ms.innerHTML=Ea,tt=M(),T(zl.$$.fragment),at=M(),ps=d("p"),ps.innerHTML=Sa,nt=M(),T(is.$$.fragment),Mt=M(),T(ys.$$.fragment),pt=M(),Js=d("p"),Js.textContent=Ha,it=M(),rs=d("p"),rs.innerHTML=La,yt=M(),T(Us.$$.fragment),Jt=M(),T(Ts.$$.fragment),rt=M(),cs=d("p"),cs.innerHTML=qa,Ut=M(),os=d("p"),os.innerHTML=Da,Tt=M(),T(ws.$$.fragment),ct=M(),T(ds.$$.fragment),ot=M(),ms=d("p"),ms.innerHTML=Ka,wt=M(),T(Nl.$$.fragment),dt=M(),js=d("p"),js.textContent=Pa,mt=M(),T(us.$$.fragment),jt=M(),T(Al.$$.fragment),ut=M(),hs=d("p"),hs.textContent=Oa,ht=M(),T(bs.$$.fragment),bt=M(),T(fs.$$.fragment),ft=M(),Is=d("p"),Is.textContent=ln,It=M(),T(Cs.$$.fragment),Ct=M(),T(_s.$$.fragment),_t=M(),Gs=d("p"),Gs.innerHTML=sn,Gt=M(),T(Bs.$$.fragment),Bt=M(),T(Zs.$$.fragment),Zt=M(),gs=d("p"),gs.textContent=en,gt=M(),ks=d("ul"),ks.innerHTML=tn,kt=M(),Rs=d("p"),Rs.textContent=an,Rt=M(),T(Xs.$$.fragment),Xt=M(),Ws=d("p"),Ws.innerHTML=nn,Wt=M(),$s=d("p"),$s.innerHTML=Mn,$t=M(),T(Vs.$$.fragment),Vt=M(),T(zs.$$.fragment),zt=M(),Ns=d("p"),Ns.innerHTML=pn,Nt=M(),T(As.$$.fragment),At=M(),T(xs.$$.fragment),xt=M(),Qs=d("p"),Qs.textContent=yn,Qt=M(),T(vs.$$.fragment),vt=M(),T(Ys.$$.fragment),Yt=M(),Fs=d("p"),Fs.innerHTML=Jn,Ft=M(),T(Es.$$.fragment),Et=M(),Ss=d("p"),Ss.innerHTML=rn,St=M(),T(Hs.$$.fragment),Ht=M(),Ls=d("p"),Ls.innerHTML=Un,Lt=M(),T(qs.$$.fragment),qt=M(),T(Ds.$$.fragment),Dt=M(),Ks=d("p"),Ks.innerHTML=Tn,Kt=M(),T(xl.$$.fragment),Pt=M(),sl&&sl.c(),$e=M(),Ps=d("p"),Ps.innerHTML=cn,Ot=M(),Tl.c(),Ve=M(),Os=d("p"),Os.textContent=on,la=M(),T(le.$$.fragment),sa=M(),T(se.$$.fragment),ea=M(),T(Ql.$$.fragment),ta=M(),ee=d("p"),ee.innerHTML=wn,aa=M(),T(te.$$.fragment),na=M(),T(ae.$$.fragment),Ma=M(),ne=d("p"),ne.innerHTML=dn,pa=M(),T(Me.$$.fragment),ia=M(),pe=d("p"),pe.textContent=mn,ya=M(),T(ie.$$.fragment),Ja=M(),ye=d("p"),ye.textContent=jn,ra=M(),ol.c(),ze=M(),T(Je.$$.fragment),Ua=M(),T(re.$$.fragment),Ta=M(),Ue=d("p"),Ue.textContent=un,ca=M(),dl.c(),Ne=M(),T(Te.$$.fragment),oa=M(),ce=d("p"),ce.textContent=hn,wa=M(),jl.c(),Ae=M(),oe=d("p"),oe.textContent=bn,da=M(),hl.c(),xe=M(),T(we.$$.fragment),ma=M(),de=d("p"),de.textContent=fn,ja=M(),x&&x.c(),Qe=M(),T(vl.$$.fragment),ua=M(),Q&&Q.c(),ve=M(),T(me.$$.fragment),ha=M(),je=d("p"),je.innerHTML=In,ba=M(),T(ue.$$.fragment),fa=M(),he=d("p"),he.textContent=Cn,Ia=M(),T(be.$$.fragment),Ca=M(),T(fe.$$.fragment),_a=M(),Ie=d("p"),Ie.textContent=_n,Ga=M(),T(Ce.$$.fragment),Ba=M(),_e=d("p"),_e.innerHTML=Gn,Za=M(),T(Yl.$$.fragment),ga=M(),He=d("p"),this.h()},l(l){const s=iM("svelte-u9bgzb",document.head);a=m(s,"META",{name:!0,content:!0}),s.forEach(e),U=p(l),n=m(l,"P",{}),Qa(n).forEach(e),r=p(l),c(G.$$.fragment,l),j=p(l),c(R.$$.fragment,l),W=p(l),X.l(l),k=p(l),g=m(l,"P",{"data-svelte-h":!0}),u(g)!=="svelte-1xh6yjl"&&(g.textContent=h),B=p(l),z=m(l,"P",{"data-svelte-h":!0}),u(z)!=="svelte-14ufjjm"&&(z.textContent=v),C=p(l),$=m(l,"P",{"data-svelte-h":!0}),u($)!=="svelte-1ocjhax"&&($.innerHTML=S),Z=p(l),V=m(l,"DIV",{class:!0,"data-svelte-h":!0}),u(V)!=="svelte-1lvgpuv"&&(V.innerHTML=fl),Bl=p(l),H=m(l,"P",{"data-svelte-h":!0}),u(H)!=="svelte-p5b197"&&(H.innerHTML=Ye),Zl=p(l),N=m(l,"IFRAME",{src:!0,frameborder:!0,height:!0,title:!0,class:!0,allow:!0,sandbox:!0}),Qa(N).forEach(e),tl=p(l),Il=m(l,"P",{"data-svelte-h":!0}),u(Il)!=="svelte-1jj86ur"&&(Il.textContent=Ll),al=p(l),c(nl.$$.fragment,l),Ml=p(l),c(Jl.$$.fragment,l),gl=p(l),c(L.$$.fragment,l),kl=p(l),q=m(l,"P",{"data-svelte-h":!0}),u(q)!=="svelte-1d2fdlt"&&(q.innerHTML=Fe),Rl=p(l),Y=m(l,"DIV",{class:!0,"data-svelte-h":!0}),u(Y)!=="svelte-zc90xi"&&(Y.innerHTML=ql),pl=p(l),Cl=m(l,"P",{"data-svelte-h":!0}),u(Cl)!=="svelte-vqat7s"&&(Cl.innerHTML=Dl),il=p(l),A.l(l),rl=p(l),D=m(l,"P",{"data-svelte-h":!0}),u(D)!=="svelte-2wdc9e"&&(D.textContent=Ee),Xl=p(l),c(K.$$.fragment,l),Wl=p(l),P=m(l,"P",{"data-svelte-h":!0}),u(P)!=="svelte-1ytko7"&&(P.innerHTML=Se),$l=p(l),c(O.$$.fragment,l),Vl=p(l),ll=m(l,"P",{"data-svelte-h":!0}),u(ll)!=="svelte-zjynbe"&&(ll.textContent=Kl),yl=p(l),E.l(l),i=p(l),c(f.$$.fragment,l),Le=p(l),Pl=m(l,"P",{"data-svelte-h":!0}),u(Pl)!=="svelte-36932k"&&(Pl.textContent=va),qe=p(l),c(Ol.$$.fragment,l),De=p(l),ls=m(l,"P",{"data-svelte-h":!0}),u(ls)!=="svelte-hfddrp"&&(ls.innerHTML=Ya),Ke=p(l),c(ss.$$.fragment,l),Pe=p(l),c(es.$$.fragment,l),Oe=p(l),ts=m(l,"P",{"data-svelte-h":!0}),u(ts)!=="svelte-uwo9y3"&&(ts.innerHTML=Fa),lt=p(l),c(as.$$.fragment,l),st=p(l),c(ns.$$.fragment,l),et=p(l),Ms=m(l,"P",{"data-svelte-h":!0}),u(Ms)!=="svelte-e0lxy5"&&(Ms.innerHTML=Ea),tt=p(l),c(zl.$$.fragment,l),at=p(l),ps=m(l,"P",{"data-svelte-h":!0}),u(ps)!=="svelte-1mm7saq"&&(ps.innerHTML=Sa),nt=p(l),c(is.$$.fragment,l),Mt=p(l),c(ys.$$.fragment,l),pt=p(l),Js=m(l,"P",{"data-svelte-h":!0}),u(Js)!=="svelte-1a25z57"&&(Js.textContent=Ha),it=p(l),rs=m(l,"P",{"data-svelte-h":!0}),u(rs)!=="svelte-flvp59"&&(rs.innerHTML=La),yt=p(l),c(Us.$$.fragment,l),Jt=p(l),c(Ts.$$.fragment,l),rt=p(l),cs=m(l,"P",{"data-svelte-h":!0}),u(cs)!=="svelte-1tdwwel"&&(cs.innerHTML=qa),Ut=p(l),os=m(l,"P",{"data-svelte-h":!0}),u(os)!=="svelte-yalypi"&&(os.innerHTML=Da),Tt=p(l),c(ws.$$.fragment,l),ct=p(l),c(ds.$$.fragment,l),ot=p(l),ms=m(l,"P",{"data-svelte-h":!0}),u(ms)!=="svelte-j3gz2l"&&(ms.innerHTML=Ka),wt=p(l),c(Nl.$$.fragment,l),dt=p(l),js=m(l,"P",{"data-svelte-h":!0}),u(js)!=="svelte-100twfi"&&(js.textContent=Pa),mt=p(l),c(us.$$.fragment,l),jt=p(l),c(Al.$$.fragment,l),ut=p(l),hs=m(l,"P",{"data-svelte-h":!0}),u(hs)!=="svelte-111po0h"&&(hs.textContent=Oa),ht=p(l),c(bs.$$.fragment,l),bt=p(l),c(fs.$$.fragment,l),ft=p(l),Is=m(l,"P",{"data-svelte-h":!0}),u(Is)!=="svelte-bs7cfj"&&(Is.textContent=ln),It=p(l),c(Cs.$$.fragment,l),Ct=p(l),c(_s.$$.fragment,l),_t=p(l),Gs=m(l,"P",{"data-svelte-h":!0}),u(Gs)!=="svelte-td66cd"&&(Gs.innerHTML=sn),Gt=p(l),c(Bs.$$.fragment,l),Bt=p(l),c(Zs.$$.fragment,l),Zt=p(l),gs=m(l,"P",{"data-svelte-h":!0}),u(gs)!=="svelte-1h39xei"&&(gs.textContent=en),gt=p(l),ks=m(l,"UL",{"data-svelte-h":!0}),u(ks)!=="svelte-1ysct4r"&&(ks.innerHTML=tn),kt=p(l),Rs=m(l,"P",{"data-svelte-h":!0}),u(Rs)!=="svelte-1m4hwow"&&(Rs.textContent=an),Rt=p(l),c(Xs.$$.fragment,l),Xt=p(l),Ws=m(l,"P",{"data-svelte-h":!0}),u(Ws)!=="svelte-1jbc338"&&(Ws.innerHTML=nn),Wt=p(l),$s=m(l,"P",{"data-svelte-h":!0}),u($s)!=="svelte-1l8tppt"&&($s.innerHTML=Mn),$t=p(l),c(Vs.$$.fragment,l),Vt=p(l),c(zs.$$.fragment,l),zt=p(l),Ns=m(l,"P",{"data-svelte-h":!0}),u(Ns)!=="svelte-g5tu54"&&(Ns.innerHTML=pn),Nt=p(l),c(As.$$.fragment,l),At=p(l),c(xs.$$.fragment,l),xt=p(l),Qs=m(l,"P",{"data-svelte-h":!0}),u(Qs)!=="svelte-1jre16r"&&(Qs.textContent=yn),Qt=p(l),c(vs.$$.fragment,l),vt=p(l),c(Ys.$$.fragment,l),Yt=p(l),Fs=m(l,"P",{"data-svelte-h":!0}),u(Fs)!=="svelte-ciyvx7"&&(Fs.innerHTML=Jn),Ft=p(l),c(Es.$$.fragment,l),Et=p(l),Ss=m(l,"P",{"data-svelte-h":!0}),u(Ss)!=="svelte-perv4b"&&(Ss.innerHTML=rn),St=p(l),c(Hs.$$.fragment,l),Ht=p(l),Ls=m(l,"P",{"data-svelte-h":!0}),u(Ls)!=="svelte-wm10bz"&&(Ls.innerHTML=Un),Lt=p(l),c(qs.$$.fragment,l),qt=p(l),c(Ds.$$.fragment,l),Dt=p(l),Ks=m(l,"P",{"data-svelte-h":!0}),u(Ks)!=="svelte-czyoa"&&(Ks.innerHTML=Tn),Kt=p(l),c(xl.$$.fragment,l),Pt=p(l),sl&&sl.l(l),$e=p(l),Ps=m(l,"P",{"data-svelte-h":!0}),u(Ps)!=="svelte-1ew0o7r"&&(Ps.innerHTML=cn),Ot=p(l),Tl.l(l),Ve=p(l),Os=m(l,"P",{"data-svelte-h":!0}),u(Os)!=="svelte-1xi9xjp"&&(Os.textContent=on),la=p(l),c(le.$$.fragment,l),sa=p(l),c(se.$$.fragment,l),ea=p(l),c(Ql.$$.fragment,l),ta=p(l),ee=m(l,"P",{"data-svelte-h":!0}),u(ee)!=="svelte-1dutaj6"&&(ee.innerHTML=wn),aa=p(l),c(te.$$.fragment,l),na=p(l),c(ae.$$.fragment,l),Ma=p(l),ne=m(l,"P",{"data-svelte-h":!0}),u(ne)!=="svelte-18x4q3a"&&(ne.innerHTML=dn),pa=p(l),c(Me.$$.fragment,l),ia=p(l),pe=m(l,"P",{"data-svelte-h":!0}),u(pe)!=="svelte-du03v9"&&(pe.textContent=mn),ya=p(l),c(ie.$$.fragment,l),Ja=p(l),ye=m(l,"P",{"data-svelte-h":!0}),u(ye)!=="svelte-rkzfqo"&&(ye.textContent=jn),ra=p(l),ol.l(l),ze=p(l),c(Je.$$.fragment,l),Ua=p(l),c(re.$$.fragment,l),Ta=p(l),Ue=m(l,"P",{"data-svelte-h":!0}),u(Ue)!=="svelte-1er2qp1"&&(Ue.textContent=un),ca=p(l),dl.l(l),Ne=p(l),c(Te.$$.fragment,l),oa=p(l),ce=m(l,"P",{"data-svelte-h":!0}),u(ce)!=="svelte-uyxso0"&&(ce.textContent=hn),wa=p(l),jl.l(l),Ae=p(l),oe=m(l,"P",{"data-svelte-h":!0}),u(oe)!=="svelte-fprpc8"&&(oe.textContent=bn),da=p(l),hl.l(l),xe=p(l),c(we.$$.fragment,l),ma=p(l),de=m(l,"P",{"data-svelte-h":!0}),u(de)!=="svelte-8d1fm1"&&(de.textContent=fn),ja=p(l),x&&x.l(l),Qe=p(l),c(vl.$$.fragment,l),ua=p(l),Q&&Q.l(l),ve=p(l),c(me.$$.fragment,l),ha=p(l),je=m(l,"P",{"data-svelte-h":!0}),u(je)!=="svelte-1kphjgn"&&(je.innerHTML=In),ba=p(l),c(ue.$$.fragment,l),fa=p(l),he=m(l,"P",{"data-svelte-h":!0}),u(he)!=="svelte-1oo3dg2"&&(he.textContent=Cn),Ia=p(l),c(be.$$.fragment,l),Ca=p(l),c(fe.$$.fragment,l),_a=p(l),Ie=m(l,"P",{"data-svelte-h":!0}),u(Ie)!=="svelte-r8dsox"&&(Ie.textContent=_n),Ga=p(l),c(Ce.$$.fragment,l),Ba=p(l),_e=m(l,"P",{"data-svelte-h":!0}),u(_e)!=="svelte-1utiwse"&&(_e.innerHTML=Gn),Za=p(l),c(Yl.$$.fragment,l),ga=p(l),He=m(l,"P",{}),Qa(He).forEach(e),this.h()},h(){bl(a,"name","hf:doc:metadata"),bl(a,"content",VM),bl(V,"class","flex justify-center"),aM(N.src,Hl="https://course-demos-distilbert-base-uncased-finetune-7400b54.hf.space")||bl(N,"src",Hl),bl(N,"frameborder","0"),bl(N,"height","300"),bl(N,"title","Gradio app"),bl(N,"class","block dark:hidden container p-0 flex-grow space-iframe"),bl(N,"allow","accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking"),bl(N,"sandbox","allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"),bl(Y,"class","flex justify-center")},m(l,s){yM(document.head,a),t(l,U,s),t(l,n,s),t(l,r,s),o(G,l,s),t(l,j,s),o(R,l,s),t(l,W,s),Ge[_].m(l,s),t(l,k,s),t(l,g,s),t(l,B,s),t(l,z,s),t(l,C,s),t(l,$,s),t(l,Z,s),t(l,V,s),t(l,Bl,s),t(l,H,s),t(l,Zl,s),t(l,N,s),t(l,tl,s),t(l,Il,s),t(l,al,s),o(nl,l,s),t(l,Ml,s),o(Jl,l,s),t(l,gl,s),o(L,l,s),t(l,kl,s),t(l,q,s),t(l,Rl,s),t(l,Y,s),t(l,pl,s),t(l,Cl,s),t(l,il,s),Be[F].m(l,s),t(l,rl,s),t(l,D,s),t(l,Xl,s),o(K,l,s),t(l,Wl,s),t(l,P,s),t(l,$l,s),o(O,l,s),t(l,Vl,s),t(l,ll,s),t(l,yl,s),Ze[el].m(l,s),t(l,i,s),o(f,l,s),t(l,Le,s),t(l,Pl,s),t(l,qe,s),o(Ol,l,s),t(l,De,s),t(l,ls,s),t(l,Ke,s),o(ss,l,s),t(l,Pe,s),o(es,l,s),t(l,Oe,s),t(l,ts,s),t(l,lt,s),o(as,l,s),t(l,st,s),o(ns,l,s),t(l,et,s),t(l,Ms,s),t(l,tt,s),o(zl,l,s),t(l,at,s),t(l,ps,s),t(l,nt,s),o(is,l,s),t(l,Mt,s),o(ys,l,s),t(l,pt,s),t(l,Js,s),t(l,it,s),t(l,rs,s),t(l,yt,s),o(Us,l,s),t(l,Jt,s),o(Ts,l,s),t(l,rt,s),t(l,cs,s),t(l,Ut,s),t(l,os,s),t(l,Tt,s),o(ws,l,s),t(l,ct,s),o(ds,l,s),t(l,ot,s),t(l,ms,s),t(l,wt,s),o(Nl,l,s),t(l,dt,s),t(l,js,s),t(l,mt,s),o(us,l,s),t(l,jt,s),o(Al,l,s),t(l,ut,s),t(l,hs,s),t(l,ht,s),o(bs,l,s),t(l,bt,s),o(fs,l,s),t(l,ft,s),t(l,Is,s),t(l,It,s),o(Cs,l,s),t(l,Ct,s),o(_s,l,s),t(l,_t,s),t(l,Gs,s),t(l,Gt,s),o(Bs,l,s),t(l,Bt,s),o(Zs,l,s),t(l,Zt,s),t(l,gs,s),t(l,gt,s),t(l,ks,s),t(l,kt,s),t(l,Rs,s),t(l,Rt,s),o(Xs,l,s),t(l,Xt,s),t(l,Ws,s),t(l,Wt,s),t(l,$s,s),t(l,$t,s),o(Vs,l,s),t(l,Vt,s),o(zs,l,s),t(l,zt,s),t(l,Ns,s),t(l,Nt,s),o(As,l,s),t(l,At,s),o(xs,l,s),t(l,xt,s),t(l,Qs,s),t(l,Qt,s),o(vs,l,s),t(l,vt,s),o(Ys,l,s),t(l,Yt,s),t(l,Fs,s),t(l,Ft,s),o(Es,l,s),t(l,Et,s),t(l,Ss,s),t(l,St,s),o(Hs,l,s),t(l,Ht,s),t(l,Ls,s),t(l,Lt,s),o(qs,l,s),t(l,qt,s),o(Ds,l,s),t(l,Dt,s),t(l,Ks,s),t(l,Kt,s),o(xl,l,s),t(l,Pt,s),sl&&sl.m(l,s),t(l,$e,s),t(l,Ps,s),t(l,Ot,s),ge[Ul].m(l,s),t(l,Ve,s),t(l,Os,s),t(l,la,s),o(le,l,s),t(l,sa,s),o(se,l,s),t(l,ea,s),o(Ql,l,s),t(l,ta,s),t(l,ee,s),t(l,aa,s),o(te,l,s),t(l,na,s),o(ae,l,s),t(l,Ma,s),t(l,ne,s),t(l,pa,s),o(Me,l,s),t(l,ia,s),t(l,pe,s),t(l,ya,s),o(ie,l,s),t(l,Ja,s),t(l,ye,s),t(l,ra,s),ke[cl].m(l,s),t(l,ze,s),o(Je,l,s),t(l,Ua,s),o(re,l,s),t(l,Ta,s),t(l,Ue,s),t(l,ca,s),Re[wl].m(l,s),t(l,Ne,s),o(Te,l,s),t(l,oa,s),t(l,ce,s),t(l,wa,s),Xe[ml].m(l,s),t(l,Ae,s),t(l,oe,s),t(l,da,s),We[ul].m(l,s),t(l,xe,s),o(we,l,s),t(l,ma,s),t(l,de,s),t(l,ja,s),x&&x.m(l,s),t(l,Qe,s),o(vl,l,s),t(l,ua,s),Q&&Q.m(l,s),t(l,ve,s),o(me,l,s),t(l,ha,s),t(l,je,s),t(l,ba,s),o(ue,l,s),t(l,fa,s),t(l,he,s),t(l,Ia,s),o(be,l,s),t(l,Ca,s),o(fe,l,s),t(l,_a,s),t(l,Ie,s),t(l,Ga,s),o(Ce,l,s),t(l,Ba,s),t(l,_e,s),t(l,Za,s),o(Yl,l,s),t(l,ga,s),t(l,He,s),ka=!0},p(l,[s]){const Fn={};s&1&&(Fn.fw=l[0]),G.$set(Fn);let Xa=_;_=Zn(l),_!==Xa&&(Gl(),J(Ge[Xa],1,1,()=>{Ge[Xa]=null}),_l(),X=Ge[_],X||(X=Ge[_]=Bn[_](l),X.c()),y(X,1),X.m(k.parentNode,k));const En={};s&2&&(En.$$scope={dirty:s,ctx:l}),Jl.$set(En);let Wa=F;F=kn(l),F!==Wa&&(Gl(),J(Be[Wa],1,1,()=>{Be[Wa]=null}),_l(),A=Be[F],A||(A=Be[F]=gn[F](l),A.c()),y(A,1),A.m(rl.parentNode,rl));let $a=el;el=Xn(l),el!==$a&&(Gl(),J(Ze[$a],1,1,()=>{Ze[$a]=null}),_l(),E=Ze[el],E||(E=Ze[el]=Rn[el](l),E.c()),y(E,1),E.m(i.parentNode,i));const Sn={};s&2&&(Sn.$$scope={dirty:s,ctx:l}),zl.$set(Sn);const Hn={};s&2&&(Hn.$$scope={dirty:s,ctx:l}),Nl.$set(Hn);const Ln={};s&2&&(Ln.$$scope={dirty:s,ctx:l}),Al.$set(Ln);const qn={};s&2&&(qn.$$scope={dirty:s,ctx:l}),xl.$set(qn),l[0]==="pt"?sl||(sl=On(),sl.c(),sl.m($e.parentNode,$e)):sl&&(sl.d(1),sl=null);let Va=Ul;Ul=$n(l),Ul!==Va&&(Gl(),J(ge[Va],1,1,()=>{ge[Va]=null}),_l(),Tl=ge[Ul],Tl||(Tl=ge[Ul]=Wn[Ul](l),Tl.c()),y(Tl,1),Tl.m(Ve.parentNode,Ve));const Dn={};s&2&&(Dn.$$scope={dirty:s,ctx:l}),Ql.$set(Dn);let za=cl;cl=zn(l),cl!==za&&(Gl(),J(ke[za],1,1,()=>{ke[za]=null}),_l(),ol=ke[cl],ol||(ol=ke[cl]=Vn[cl](l),ol.c()),y(ol,1),ol.m(ze.parentNode,ze));let Na=wl;wl=An(l),wl!==Na&&(Gl(),J(Re[Na],1,1,()=>{Re[Na]=null}),_l(),dl=Re[wl],dl||(dl=Re[wl]=Nn[wl](l),dl.c()),y(dl,1),dl.m(Ne.parentNode,Ne));let Aa=ml;ml=Qn(l),ml!==Aa&&(Gl(),J(Xe[Aa],1,1,()=>{Xe[Aa]=null}),_l(),jl=Xe[ml],jl||(jl=Xe[ml]=xn[ml](l),jl.c()),y(jl,1),jl.m(Ae.parentNode,Ae));let xa=ul;ul=Yn(l),ul!==xa&&(Gl(),J(We[xa],1,1,()=>{We[xa]=null}),_l(),hl=We[ul],hl||(hl=We[ul]=vn[ul](l),hl.c()),y(hl,1),hl.m(xe.parentNode,xe)),l[0]==="pt"?x?s&1&&y(x,1):(x=lM(),x.c(),y(x,1),x.m(Qe.parentNode,Qe)):x&&(Gl(),J(x,1,1,()=>{x=null}),_l());const Kn={};s&2&&(Kn.$$scope={dirty:s,ctx:l}),vl.$set(Kn),l[0]==="pt"?Q?s&1&&y(Q,1):(Q=sM(),Q.c(),y(Q,1),Q.m(ve.parentNode,ve)):Q&&(Gl(),J(Q,1,1,()=>{Q=null}),_l());const Pn={};s&2&&(Pn.$$scope={dirty:s,ctx:l}),Yl.$set(Pn)},i(l){ka||(y(G.$$.fragment,l),y(R.$$.fragment,l),y(X),y(nl.$$.fragment,l),y(Jl.$$.fragment,l),y(L.$$.fragment,l),y(A),y(K.$$.fragment,l),y(O.$$.fragment,l),y(E),y(f.$$.fragment,l),y(Ol.$$.fragment,l),y(ss.$$.fragment,l),y(es.$$.fragment,l),y(as.$$.fragment,l),y(ns.$$.fragment,l),y(zl.$$.fragment,l),y(is.$$.fragment,l),y(ys.$$.fragment,l),y(Us.$$.fragment,l),y(Ts.$$.fragment,l),y(ws.$$.fragment,l),y(ds.$$.fragment,l),y(Nl.$$.fragment,l),y(us.$$.fragment,l),y(Al.$$.fragment,l),y(bs.$$.fragment,l),y(fs.$$.fragment,l),y(Cs.$$.fragment,l),y(_s.$$.fragment,l),y(Bs.$$.fragment,l),y(Zs.$$.fragment,l),y(Xs.$$.fragment,l),y(Vs.$$.fragment,l),y(zs.$$.fragment,l),y(As.$$.fragment,l),y(xs.$$.fragment,l),y(vs.$$.fragment,l),y(Ys.$$.fragment,l),y(Es.$$.fragment,l),y(Hs.$$.fragment,l),y(qs.$$.fragment,l),y(Ds.$$.fragment,l),y(xl.$$.fragment,l),y(Tl),y(le.$$.fragment,l),y(se.$$.fragment,l),y(Ql.$$.fragment,l),y(te.$$.fragment,l),y(ae.$$.fragment,l),y(Me.$$.fragment,l),y(ie.$$.fragment,l),y(ol),y(Je.$$.fragment,l),y(re.$$.fragment,l),y(dl),y(Te.$$.fragment,l),y(jl),y(hl),y(we.$$.fragment,l),y(x),y(vl.$$.fragment,l),y(Q),y(me.$$.fragment,l),y(ue.$$.fragment,l),y(be.$$.fragment,l),y(fe.$$.fragment,l),y(Ce.$$.fragment,l),y(Yl.$$.fragment,l),ka=!0)},o(l){J(G.$$.fragment,l),J(R.$$.fragment,l),J(X),J(nl.$$.fragment,l),J(Jl.$$.fragment,l),J(L.$$.fragment,l),J(A),J(K.$$.fragment,l),J(O.$$.fragment,l),J(E),J(f.$$.fragment,l),J(Ol.$$.fragment,l),J(ss.$$.fragment,l),J(es.$$.fragment,l),J(as.$$.fragment,l),J(ns.$$.fragment,l),J(zl.$$.fragment,l),J(is.$$.fragment,l),J(ys.$$.fragment,l),J(Us.$$.fragment,l),J(Ts.$$.fragment,l),J(ws.$$.fragment,l),J(ds.$$.fragment,l),J(Nl.$$.fragment,l),J(us.$$.fragment,l),J(Al.$$.fragment,l),J(bs.$$.fragment,l),J(fs.$$.fragment,l),J(Cs.$$.fragment,l),J(_s.$$.fragment,l),J(Bs.$$.fragment,l),J(Zs.$$.fragment,l),J(Xs.$$.fragment,l),J(Vs.$$.fragment,l),J(zs.$$.fragment,l),J(As.$$.fragment,l),J(xs.$$.fragment,l),J(vs.$$.fragment,l),J(Ys.$$.fragment,l),J(Es.$$.fragment,l),J(Hs.$$.fragment,l),J(qs.$$.fragment,l),J(Ds.$$.fragment,l),J(xl.$$.fragment,l),J(Tl),J(le.$$.fragment,l),J(se.$$.fragment,l),J(Ql.$$.fragment,l),J(te.$$.fragment,l),J(ae.$$.fragment,l),J(Me.$$.fragment,l),J(ie.$$.fragment,l),J(ol),J(Je.$$.fragment,l),J(re.$$.fragment,l),J(dl),J(Te.$$.fragment,l),J(jl),J(hl),J(we.$$.fragment,l),J(x),J(vl.$$.fragment,l),J(Q),J(me.$$.fragment,l),J(ue.$$.fragment,l),J(be.$$.fragment,l),J(fe.$$.fragment,l),J(Ce.$$.fragment,l),J(Yl.$$.fragment,l),ka=!1},d(l){l&&(e(U),e(n),e(r),e(j),e(W),e(k),e(g),e(B),e(z),e(C),e($),e(Z),e(V),e(Bl),e(H),e(Zl),e(N),e(tl),e(Il),e(al),e(Ml),e(gl),e(kl),e(q),e(Rl),e(Y),e(pl),e(Cl),e(il),e(rl),e(D),e(Xl),e(Wl),e(P),e($l),e(Vl),e(ll),e(yl),e(i),e(Le),e(Pl),e(qe),e(De),e(ls),e(Ke),e(Pe),e(Oe),e(ts),e(lt),e(st),e(et),e(Ms),e(tt),e(at),e(ps),e(nt),e(Mt),e(pt),e(Js),e(it),e(rs),e(yt),e(Jt),e(rt),e(cs),e(Ut),e(os),e(Tt),e(ct),e(ot),e(ms),e(wt),e(dt),e(js),e(mt),e(jt),e(ut),e(hs),e(ht),e(bt),e(ft),e(Is),e(It),e(Ct),e(_t),e(Gs),e(Gt),e(Bt),e(Zt),e(gs),e(gt),e(ks),e(kt),e(Rs),e(Rt),e(Xt),e(Ws),e(Wt),e($s),e($t),e(Vt),e(zt),e(Ns),e(Nt),e(At),e(xt),e(Qs),e(Qt),e(vt),e(Yt),e(Fs),e(Ft),e(Et),e(Ss),e(St),e(Ht),e(Ls),e(Lt),e(qt),e(Dt),e(Ks),e(Kt),e(Pt),e($e),e(Ps),e(Ot),e(Ve),e(Os),e(la),e(sa),e(ea),e(ta),e(ee),e(aa),e(na),e(Ma),e(ne),e(pa),e(ia),e(pe),e(ya),e(Ja),e(ye),e(ra),e(ze),e(Ua),e(Ta),e(Ue),e(ca),e(Ne),e(oa),e(ce),e(wa),e(Ae),e(oe),e(da),e(xe),e(ma),e(de),e(ja),e(Qe),e(ua),e(ve),e(ha),e(je),e(ba),e(fa),e(he),e(Ia),e(Ca),e(_a),e(Ie),e(Ga),e(Ba),e(_e),e(Za),e(ga),e(He)),e(a),w(G,l),w(R,l),Ge[_].d(l),w(nl,l),w(Jl,l),w(L,l),Be[F].d(l),w(K,l),w(O,l),Ze[el].d(l),w(f,l),w(Ol,l),w(ss,l),w(es,l),w(as,l),w(ns,l),w(zl,l),w(is,l),w(ys,l),w(Us,l),w(Ts,l),w(ws,l),w(ds,l),w(Nl,l),w(us,l),w(Al,l),w(bs,l),w(fs,l),w(Cs,l),w(_s,l),w(Bs,l),w(Zs,l),w(Xs,l),w(Vs,l),w(zs,l),w(As,l),w(xs,l),w(vs,l),w(Ys,l),w(Es,l),w(Hs,l),w(qs,l),w(Ds,l),w(xl,l),sl&&sl.d(l),ge[Ul].d(l),w(le,l),w(se,l),w(Ql,l),w(te,l),w(ae,l),w(Me,l),w(ie,l),ke[cl].d(l),w(Je,l),w(re,l),Re[wl].d(l),w(Te,l),Xe[ml].d(l),We[ul].d(l),w(we,l),x&&x.d(l),w(vl,l),Q&&Q.d(l),w(me,l),w(ue,l),w(be,l),w(fe,l),w(Ce,l),w(Yl,l)}}}const VM='{"title":"微调掩码语言模型","local":"微调掩码语言模型","sections":[{"title":"选择用于掩码语言建模的预训练模型","local":"选择用于掩码语言建模的预训练模型","sections":[],"depth":2},{"title":"数据集","local":"数据集","sections":[],"depth":2},{"title":"预处理数据","local":"预处理数据","sections":[],"depth":2},{"title":"使用 Trainer API 微调DistilBERT","local":"使用 Trainer API 微调DistilBERT","sections":[{"title":"语言模型的perplexity","local":"语言模型的perplexity","sections":[],"depth":3}],"depth":2},{"title":"使用 🤗 Accelerate 微调 DistilBERT","local":"使用 🤗 Accelerate 微调 DistilBERT","sections":[],"depth":2},{"title":"使用我们微调的模型","local":"使用我们微调的模型","sections":[],"depth":2}],"depth":1}';function zM(I,a,U){let n="pt";return nM(()=>{const r=new URLSearchParams(window.location.search);U(0,n=r.get("fw")||"pt")}),[n]}class SM extends MM{constructor(a){super(),pM(this,a,zM,$M,tM,{})}}export{SM as component};
