import{s as ll,o as el,n as ws}from"../chunks/scheduler.9bc65507.js";import{S as nl,i as pl,g as h,s as r,u as d,A as rl,h as M,f as a,c as o,j as V,v as j,y as w,k as R,o as E,a as t,w as y,d as f,t as g,x as b,m as al,n as tl}from"../chunks/index.78c82d43.js";import{T as jt}from"../chunks/Tip.87d55b76.js";import{Y as sl}from"../chunks/Youtube.b19762b6.js";import{I as ma}from"../chunks/IconCopyLink.bedaa44d.js";import{C as k}from"../chunks/CodeBlock.48618ec2.js";import{D as ol}from"../chunks/DocNotebookDropdown.40f7599b.js";import{F as wt,M as ha}from"../chunks/Markdown.993b48ef.js";function il(C){let e,c,l='<a href="../model_doc/albert">ALBERT</a>, <a href="../model_doc/bert">BERT</a>, <a href="../model_doc/big_bird">BigBird</a>, <a href="../model_doc/biogpt">BioGpt</a>, <a href="../model_doc/bloom">BLOOM</a>, <a href="../model_doc/camembert">CamemBERT</a>, <a href="../model_doc/canine">CANINE</a>, <a href="../model_doc/convbert">ConvBERT</a>, <a href="../model_doc/data2vec-text">Data2VecText</a>, <a href="../model_doc/deberta">DeBERTa</a>, <a href="../model_doc/deberta-v2">DeBERTa-v2</a>, <a href="../model_doc/distilbert">DistilBERT</a>, <a href="../model_doc/electra">ELECTRA</a>, <a href="../model_doc/ernie">ERNIE</a>, <a href="../model_doc/ernie_m">ErnieM</a>, <a href="../model_doc/esm">ESM</a>, <a href="../model_doc/falcon">Falcon</a>, <a href="../model_doc/flaubert">FlauBERT</a>, <a href="../model_doc/fnet">FNet</a>, <a href="../model_doc/funnel">Funnel Transformer</a>, <a href="../model_doc/gpt-sw3">GPT-Sw3</a>, <a href="../model_doc/gpt2">OpenAI GPT-2</a>, <a href="../model_doc/gpt_bigcode">GPTBigCode</a>, <a href="../model_doc/gpt_neo">GPT Neo</a>, <a href="../model_doc/gpt_neox">GPT NeoX</a>, <a href="../model_doc/ibert">I-BERT</a>, <a href="../model_doc/layoutlm">LayoutLM</a>, <a href="../model_doc/layoutlmv2">LayoutLMv2</a>, <a href="../model_doc/layoutlmv3">LayoutLMv3</a>, <a href="../model_doc/lilt">LiLT</a>, <a href="../model_doc/longformer">Longformer</a>, <a href="../model_doc/luke">LUKE</a>, <a href="../model_doc/markuplm">MarkupLM</a>, <a href="../model_doc/mega">MEGA</a>, <a href="../model_doc/megatron-bert">Megatron-BERT</a>, <a href="../model_doc/mobilebert">MobileBERT</a>, <a href="../model_doc/mpnet">MPNet</a>, <a href="../model_doc/mpt">MPT</a>, <a href="../model_doc/mra">MRA</a>, <a href="../model_doc/nezha">Nezha</a>, <a href="../model_doc/nystromformer">Nyströmformer</a>, <a href="../model_doc/qdqbert">QDQBert</a>, <a href="../model_doc/rembert">RemBERT</a>, <a href="../model_doc/roberta">RoBERTa</a>, <a href="../model_doc/roberta-prelayernorm">RoBERTa-PreLayerNorm</a>, <a href="../model_doc/roc_bert">RoCBert</a>, <a href="../model_doc/roformer">RoFormer</a>, <a href="../model_doc/squeezebert">SqueezeBERT</a>, <a href="../model_doc/xlm">XLM</a>, <a href="../model_doc/xlm-roberta">XLM-RoBERTa</a>, <a href="../model_doc/xlm-roberta-xl">XLM-RoBERTa-XL</a>, <a href="../model_doc/xlnet">XLNet</a>, <a href="../model_doc/xmod">X-MOD</a>, <a href="../model_doc/yoso">YOSO</a>';return{c(){e=al(`The task illustrated in this tutorial is supported by the following model architectures:

`),c=h("p"),c.innerHTML=l},l(m){e=tl(m,`The task illustrated in this tutorial is supported by the following model architectures:

`),c=M(m,"P",{"data-svelte-h":!0}),w(c)!=="svelte-n7grks"&&(c.innerHTML=l)},m(m,T){t(m,e,T),t(m,c,T)},p:ws,d(m){m&&(a(e),a(c))}}}function cl(C){let e,c;return e=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`}}),{c(){d(e.$$.fragment)},l(l){j(e.$$.fragment,l)},m(l,m){y(e,l,m),c=!0},p:ws,i(l){c||(f(e.$$.fragment,l),c=!0)},o(l){g(e.$$.fragment,l),c=!1},d(l){b(e,l)}}}function ml(C){let e,c;return e=new ha({props:{$$slots:{default:[cl]},$$scope:{ctx:C}}}),{c(){d(e.$$.fragment)},l(l){j(e.$$.fragment,l)},m(l,m){y(e,l,m),c=!0},p(l,m){const T={};m&2&&(T.$$scope={dirty:m,ctx:l}),e.$set(T)},i(l){c||(f(e.$$.fragment,l),c=!0)},o(l){g(e.$$.fragment,l),c=!1},d(l){b(e,l)}}}function hl(C){let e,c;return e=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){d(e.$$.fragment)},l(l){j(e.$$.fragment,l)},m(l,m){y(e,l,m),c=!0},p:ws,i(l){c||(f(e.$$.fragment,l),c=!0)},o(l){g(e.$$.fragment,l),c=!1},d(l){b(e,l)}}}function Ml(C){let e,c;return e=new ha({props:{$$slots:{default:[hl]},$$scope:{ctx:C}}}),{c(){d(e.$$.fragment)},l(l){j(e.$$.fragment,l)},m(l,m){y(e,l,m),c=!0},p(l,m){const T={};m&2&&(T.$$scope={dirty:m,ctx:l}),e.$set(T)},i(l){c||(f(e.$$.fragment,l),c=!0)},o(l){g(e.$$.fragment,l),c=!1},d(l){b(e,l)}}}function ul(C){let e,c='If you aren’t familiar with finetuning a model with the <a href="/docs/transformers/v4.33.3/en/main_classes/trainer#transformers.Trainer">Trainer</a>, take a look at the basic tutorial <a href="../training#train-with-pytorch-trainer">here</a>!';return{c(){e=h("p"),e.innerHTML=c},l(l){e=M(l,"P",{"data-svelte-h":!0}),w(e)!=="svelte-1kww4sq"&&(e.innerHTML=c)},m(l,m){t(l,e,m)},p:ws,d(l){l&&a(e)}}}function dl(C){let e,c,l,m='You’re ready to start training your model now! Load DistilBERT with <a href="/docs/transformers/v4.33.3/en/model_doc/auto#transformers.AutoModelForTokenClassification">AutoModelForTokenClassification</a> along with the number of expected labels, and the label mappings:',T,U,W,$,G="At this point, only three steps remain:",Z,B,N='<li>Define your training hyperparameters in <a href="/docs/transformers/v4.33.3/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>. The only required parameter is <code>output_dir</code> which specifies where to save your model. You’ll push this model to the Hub by setting <code>push_to_hub=True</code> (you need to be signed in to Hugging Face to upload your model). At the end of each epoch, the <a href="/docs/transformers/v4.33.3/en/main_classes/trainer#transformers.Trainer">Trainer</a> will evaluate the seqeval scores and save the training checkpoint.</li> <li>Pass the training arguments to <a href="/docs/transformers/v4.33.3/en/main_classes/trainer#transformers.Trainer">Trainer</a> along with the model, dataset, tokenizer, data collator, and <code>compute_metrics</code> function.</li> <li>Call <a href="/docs/transformers/v4.33.3/en/main_classes/trainer#transformers.Trainer.train">train()</a> to finetune your model.</li>',v,_,I,i,x='Once training is completed, share your model to the Hub with the <a href="/docs/transformers/v4.33.3/en/main_classes/trainer#transformers.Trainer.push_to_hub">push_to_hub()</a> method so everyone can use your model:',q,H,X;return e=new jt({props:{$$slots:{default:[ul]},$$scope:{ctx:C}}}),U=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMkMlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMFRyYWluZXIlMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkJTIyJTJDJTIwbnVtX2xhYmVscyUzRDEzJTJDJTIwaWQybGFiZWwlM0RpZDJsYWJlbCUyQyUyMGxhYmVsMmlkJTNEbGFiZWwyaWQlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">13</span>, id2label=id2label, label2id=label2id
<span class="hljs-meta">... </span>)`}}),_=new k({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJteV9hd2Vzb21lX3dudXRfbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMmUtNSUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QyJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5JTNEMC4wMSUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMGxvYWRfYmVzdF9tb2RlbF9hdF9lbmQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0R0b2tlbml6ZWRfd251dCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX3dudXQlNUIlMjJ0ZXN0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVyJTNEdG9rZW5pemVyJTJDJTBBJTIwJTIwJTIwJTIwZGF0YV9jb2xsYXRvciUzRGRhdGFfY29sbGF0b3IlMkMlMEElMjAlMjAlMjAlMjBjb21wdXRlX21ldHJpY3MlM0Rjb21wdXRlX21ldHJpY3MlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_wnut_model&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    save_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    load_best_model_at_end=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>    compute_metrics=compute_metrics,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),H=new k({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()'}}),{c(){d(e.$$.fragment),c=r(),l=h("p"),l.innerHTML=m,T=r(),d(U.$$.fragment),W=r(),$=h("p"),$.textContent=G,Z=r(),B=h("ol"),B.innerHTML=N,v=r(),d(_.$$.fragment),I=r(),i=h("p"),i.innerHTML=x,q=r(),d(H.$$.fragment)},l(u){j(e.$$.fragment,u),c=o(u),l=M(u,"P",{"data-svelte-h":!0}),w(l)!=="svelte-5trzna"&&(l.innerHTML=m),T=o(u),j(U.$$.fragment,u),W=o(u),$=M(u,"P",{"data-svelte-h":!0}),w($)!=="svelte-l42k0i"&&($.textContent=G),Z=o(u),B=M(u,"OL",{"data-svelte-h":!0}),w(B)!=="svelte-14bk8cy"&&(B.innerHTML=N),v=o(u),j(_.$$.fragment,u),I=o(u),i=M(u,"P",{"data-svelte-h":!0}),w(i)!=="svelte-158pnaw"&&(i.innerHTML=x),q=o(u),j(H.$$.fragment,u)},m(u,A){y(e,u,A),t(u,c,A),t(u,l,A),t(u,T,A),y(U,u,A),t(u,W,A),t(u,$,A),t(u,Z,A),t(u,B,A),t(u,v,A),y(_,u,A),t(u,I,A),t(u,i,A),t(u,q,A),y(H,u,A),X=!0},p(u,A){const F={};A&2&&(F.$$scope={dirty:A,ctx:u}),e.$set(F)},i(u){X||(f(e.$$.fragment,u),f(U.$$.fragment,u),f(_.$$.fragment,u),f(H.$$.fragment,u),X=!0)},o(u){g(e.$$.fragment,u),g(U.$$.fragment,u),g(_.$$.fragment,u),g(H.$$.fragment,u),X=!1},d(u){u&&(a(c),a(l),a(T),a(W),a($),a(Z),a(B),a(v),a(I),a(i),a(q)),b(e,u),b(U,u),b(_,u),b(H,u)}}}function jl(C){let e,c;return e=new ha({props:{$$slots:{default:[dl]},$$scope:{ctx:C}}}),{c(){d(e.$$.fragment)},l(l){j(e.$$.fragment,l)},m(l,m){y(e,l,m),c=!0},p(l,m){const T={};m&2&&(T.$$scope={dirty:m,ctx:l}),e.$set(T)},i(l){c||(f(e.$$.fragment,l),c=!0)},o(l){g(e.$$.fragment,l),c=!1},d(l){b(e,l)}}}function yl(C){let e,c='If you aren’t familiar with finetuning a model with Keras, take a look at the basic tutorial <a href="../training#train-a-tensorflow-model-with-keras">here</a>!';return{c(){e=h("p"),e.innerHTML=c},l(l){e=M(l,"P",{"data-svelte-h":!0}),w(e)!=="svelte-1rd4nl8"&&(e.innerHTML=c)},m(l,m){t(l,e,m)},p:ws,d(l){l&&a(e)}}}function fl(C){let e,c,l,m,T,U='Then you can load DistilBERT with <a href="/docs/transformers/v4.33.3/en/model_doc/auto#transformers.TFAutoModelForTokenClassification">TFAutoModelForTokenClassification</a> along with the number of expected labels, and the label mappings:',W,$,G,Z,B='Convert your datasets to the <code>tf.data.Dataset</code> format with <a href="/docs/transformers/v4.33.3/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset">prepare_tf_dataset()</a>:',N,v,_,I,i='Configure the model for training with <a href="https://keras.io/api/models/model_training_apis/#compile-method" rel="nofollow"><code>compile</code></a>. Note that Transformers models all have a default task-relevant loss function, so you don’t need to specify one unless you want to:',x,q,H,X,u='The last two things to setup before you start training is to compute the seqeval scores from the predictions, and provide a way to push your model to the Hub. Both are done by using <a href="../main_classes/keras_callbacks">Keras callbacks</a>.',A,F,Ma='Pass your <code>compute_metrics</code> function to <a href="/docs/transformers/v4.33.3/en/main_classes/keras_callbacks#transformers.KerasMetricCallback">KerasMetricCallback</a>:',as,Q,ts,Y,ua='Specify where to push your model and tokenizer in the <a href="/docs/transformers/v4.33.3/en/main_classes/keras_callbacks#transformers.PushToHubCallback">PushToHubCallback</a>:',ls,S,es,L,da="Then bundle your callbacks together:",ns,P,ps,z,O='Finally, you’re ready to start training your model! Call <a href="https://keras.io/api/models/model_training_apis/#fit-method" rel="nofollow"><code>fit</code></a> with your training and validation datasets, the number of epochs, and your callbacks to finetune the model:',ss,D,hs,K,ja="Once training is completed, your model is automatically uploaded to the Hub so everyone can use it!",rs;return e=new jt({props:{$$slots:{default:[yl]},$$scope:{ctx:C}}}),l=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEElMEFiYXRjaF9zaXplJTIwJTNEJTIwMTYlMEFudW1fdHJhaW5fZXBvY2hzJTIwJTNEJTIwMyUwQW51bV90cmFpbl9zdGVwcyUyMCUzRCUyMChsZW4odG9rZW5pemVkX3dudXQlNUIlMjJ0cmFpbiUyMiU1RCklMjAlMkYlMkYlMjBiYXRjaF9zaXplKSUyMColMjBudW1fdHJhaW5fZXBvY2hzJTBBb3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGUlMjAlM0QlMjBjcmVhdGVfb3B0aW1pemVyKCUwQSUyMCUyMCUyMCUyMGluaXRfbHIlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX3N0ZXBzJTNEbnVtX3RyYWluX3N0ZXBzJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5X3JhdGUlM0QwLjAxJTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, lr_schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_steps=num_train_steps,
<span class="hljs-meta">... </span>    weight_decay_rate=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>)`}}),$=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMiUyQyUyMG51bV9sYWJlbHMlM0QxMyUyQyUyMGlkMmxhYmVsJTNEaWQybGFiZWwlMkMlMjBsYWJlbDJpZCUzRGxhYmVsMmlkJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">13</span>, id2label=id2label, label2id=label2id
<span class="hljs-meta">... </span>)`}}),v=new k({props:{code:"dGZfdHJhaW5fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF93bnV0JTVCJTIydHJhaW4lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBzaHVmZmxlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBKSUwQSUwQXRmX3ZhbGlkYXRpb25fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF93bnV0JTVCJTIydmFsaWRhdGlvbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),q=new k({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplciklMjAlMjAlMjMlMjBObyUyMGxvc3MlMjBhcmd1bWVudCE=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)  <span class="hljs-comment"># No loss argument!</span>`}}),Q=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBLZXJhc01ldHJpY0NhbGxiYWNrJTBBJTBBbWV0cmljX2NhbGxiYWNrJTIwJTNEJTIwS2VyYXNNZXRyaWNDYWxsYmFjayhtZXRyaWNfZm4lM0Rjb21wdXRlX21ldHJpY3MlMkMlMjBldmFsX2RhdGFzZXQlM0R0Zl92YWxpZGF0aW9uX3NldCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> KerasMetricCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_validation_set)`}}),S=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBQdXNoVG9IdWJDYWxsYmFjayUwQSUwQXB1c2hfdG9faHViX2NhbGxiYWNrJTIwJTNEJTIwUHVzaFRvSHViQ2FsbGJhY2soJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>push_to_hub_callback = PushToHubCallback(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_wnut_model&quot;</span>,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>)`}}),P=new k({props:{code:"Y2FsbGJhY2tzJTIwJTNEJTIwJTVCbWV0cmljX2NhbGxiYWNrJTJDJTIwcHVzaF90b19odWJfY2FsbGJhY2slNUQ=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>callbacks = [metric_callback, push_to_hub_callback]'}}),D=new k({props:{code:"bW9kZWwuZml0KHglM0R0Zl90cmFpbl9zZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl92YWxpZGF0aW9uX3NldCUyQyUyMGVwb2NocyUzRDMlMkMlMjBjYWxsYmFja3MlM0RjYWxsYmFja3Mp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>, callbacks=callbacks)'}}),{c(){d(e.$$.fragment),c=al(`
To finetune a model in TensorFlow, start by setting up an optimizer function, learning rate schedule, and some training hyperparameters:

	`),d(l.$$.fragment),m=r(),T=h("p"),T.innerHTML=U,W=r(),d($.$$.fragment),G=r(),Z=h("p"),Z.innerHTML=B,N=r(),d(v.$$.fragment),_=r(),I=h("p"),I.innerHTML=i,x=r(),d(q.$$.fragment),H=r(),X=h("p"),X.innerHTML=u,A=r(),F=h("p"),F.innerHTML=Ma,as=r(),d(Q.$$.fragment),ts=r(),Y=h("p"),Y.innerHTML=ua,ls=r(),d(S.$$.fragment),es=r(),L=h("p"),L.textContent=da,ns=r(),d(P.$$.fragment),ps=r(),z=h("p"),z.innerHTML=O,ss=r(),d(D.$$.fragment),hs=r(),K=h("p"),K.textContent=ja},l(p){j(e.$$.fragment,p),c=tl(p,`
To finetune a model in TensorFlow, start by setting up an optimizer function, learning rate schedule, and some training hyperparameters:

	`),j(l.$$.fragment,p),m=o(p),T=M(p,"P",{"data-svelte-h":!0}),w(T)!=="svelte-z56e6s"&&(T.innerHTML=U),W=o(p),j($.$$.fragment,p),G=o(p),Z=M(p,"P",{"data-svelte-h":!0}),w(Z)!=="svelte-1hx2ar1"&&(Z.innerHTML=B),N=o(p),j(v.$$.fragment,p),_=o(p),I=M(p,"P",{"data-svelte-h":!0}),w(I)!=="svelte-17cxx5e"&&(I.innerHTML=i),x=o(p),j(q.$$.fragment,p),H=o(p),X=M(p,"P",{"data-svelte-h":!0}),w(X)!=="svelte-nf8aa0"&&(X.innerHTML=u),A=o(p),F=M(p,"P",{"data-svelte-h":!0}),w(F)!=="svelte-1gikp1p"&&(F.innerHTML=Ma),as=o(p),j(Q.$$.fragment,p),ts=o(p),Y=M(p,"P",{"data-svelte-h":!0}),w(Y)!=="svelte-173qpz1"&&(Y.innerHTML=ua),ls=o(p),j(S.$$.fragment,p),es=o(p),L=M(p,"P",{"data-svelte-h":!0}),w(L)!=="svelte-1lw9xm8"&&(L.textContent=da),ns=o(p),j(P.$$.fragment,p),ps=o(p),z=M(p,"P",{"data-svelte-h":!0}),w(z)!=="svelte-1hrpv1v"&&(z.innerHTML=O),ss=o(p),j(D.$$.fragment,p),hs=o(p),K=M(p,"P",{"data-svelte-h":!0}),w(K)!=="svelte-2s71om"&&(K.textContent=ja)},m(p,J){y(e,p,J),t(p,c,J),y(l,p,J),t(p,m,J),t(p,T,J),t(p,W,J),y($,p,J),t(p,G,J),t(p,Z,J),t(p,N,J),y(v,p,J),t(p,_,J),t(p,I,J),t(p,x,J),y(q,p,J),t(p,H,J),t(p,X,J),t(p,A,J),t(p,F,J),t(p,as,J),y(Q,p,J),t(p,ts,J),t(p,Y,J),t(p,ls,J),y(S,p,J),t(p,es,J),t(p,L,J),t(p,ns,J),y(P,p,J),t(p,ps,J),t(p,z,J),t(p,ss,J),y(D,p,J),t(p,hs,J),t(p,K,J),rs=!0},p(p,J){const Ts={};J&2&&(Ts.$$scope={dirty:J,ctx:p}),e.$set(Ts)},i(p){rs||(f(e.$$.fragment,p),f(l.$$.fragment,p),f($.$$.fragment,p),f(v.$$.fragment,p),f(q.$$.fragment,p),f(Q.$$.fragment,p),f(S.$$.fragment,p),f(P.$$.fragment,p),f(D.$$.fragment,p),rs=!0)},o(p){g(e.$$.fragment,p),g(l.$$.fragment,p),g($.$$.fragment,p),g(v.$$.fragment,p),g(q.$$.fragment,p),g(Q.$$.fragment,p),g(S.$$.fragment,p),g(P.$$.fragment,p),g(D.$$.fragment,p),rs=!1},d(p){p&&(a(c),a(m),a(T),a(W),a(G),a(Z),a(N),a(_),a(I),a(x),a(H),a(X),a(A),a(F),a(as),a(ts),a(Y),a(ls),a(es),a(L),a(ns),a(ps),a(z),a(ss),a(hs),a(K)),b(e,p),b(l,p),b($,p),b(v,p),b(q,p),b(Q,p),b(S,p),b(P,p),b(D,p)}}}function gl(C){let e,c;return e=new ha({props:{$$slots:{default:[fl]},$$scope:{ctx:C}}}),{c(){d(e.$$.fragment)},l(l){j(e.$$.fragment,l)},m(l,m){y(e,l,m),c=!0},p(l,m){const T={};m&2&&(T.$$scope={dirty:m,ctx:l}),e.$set(T)},i(l){c||(f(e.$$.fragment,l),c=!0)},o(l){g(e.$$.fragment,l),c=!1},d(l){b(e,l)}}}function bl(C){let e,c=`For a more in-depth example of how to finetune a model for token classification, take a look at the corresponding
<a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb" rel="nofollow">PyTorch notebook</a>
or <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb" rel="nofollow">TensorFlow notebook</a>.`;return{c(){e=h("p"),e.innerHTML=c},l(l){e=M(l,"P",{"data-svelte-h":!0}),w(e)!=="svelte-j7j3qo"&&(e.innerHTML=c)},m(l,m){t(l,e,m)},p:ws,d(l){l&&a(e)}}}function wl(C){let e,c="Tokenize the text and return PyTorch tensors:",l,m,T,U,W="Pass your inputs to the model and return the <code>logits</code>:",$,G,Z,B,N="Get the class with the highest probability, and use the model’s <code>id2label</code> mapping to convert it to a text label:",v,_,I;return m=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)`}}),G=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMnN0ZXZobGl1JTJGbXlfYXdlc29tZV93bnV0X21vZGVsJTIyKSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    logits = model(**inputs).logits`}}),_=new k({props:{code:"cHJlZGljdGlvbnMlMjAlM0QlMjB0b3JjaC5hcmdtYXgobG9naXRzJTJDJTIwZGltJTNEMiklMEFwcmVkaWN0ZWRfdG9rZW5fY2xhc3MlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0Lml0ZW0oKSU1RCUyMGZvciUyMHQlMjBpbiUyMHByZWRpY3Rpb25zJTVCMCU1RCU1RCUwQXByZWRpY3RlZF90b2tlbl9jbGFzcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predictions = torch.argmax(logits, dim=<span class="hljs-number">2</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class = [model.config.id2label[t.item()] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predictions[<span class="hljs-number">0</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class
[<span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;I-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-group&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>]`}}),{c(){e=h("p"),e.textContent=c,l=r(),d(m.$$.fragment),T=r(),U=h("p"),U.innerHTML=W,$=r(),d(G.$$.fragment),Z=r(),B=h("p"),B.innerHTML=N,v=r(),d(_.$$.fragment)},l(i){e=M(i,"P",{"data-svelte-h":!0}),w(e)!=="svelte-1qcz1wr"&&(e.textContent=c),l=o(i),j(m.$$.fragment,i),T=o(i),U=M(i,"P",{"data-svelte-h":!0}),w(U)!=="svelte-f3g043"&&(U.innerHTML=W),$=o(i),j(G.$$.fragment,i),Z=o(i),B=M(i,"P",{"data-svelte-h":!0}),w(B)!=="svelte-6mgrol"&&(B.innerHTML=N),v=o(i),j(_.$$.fragment,i)},m(i,x){t(i,e,x),t(i,l,x),y(m,i,x),t(i,T,x),t(i,U,x),t(i,$,x),y(G,i,x),t(i,Z,x),t(i,B,x),t(i,v,x),y(_,i,x),I=!0},p:ws,i(i){I||(f(m.$$.fragment,i),f(G.$$.fragment,i),f(_.$$.fragment,i),I=!0)},o(i){g(m.$$.fragment,i),g(G.$$.fragment,i),g(_.$$.fragment,i),I=!1},d(i){i&&(a(e),a(l),a(T),a(U),a($),a(Z),a(B),a(v)),b(m,i),b(G,i),b(_,i)}}}function Tl(C){let e,c;return e=new ha({props:{$$slots:{default:[wl]},$$scope:{ctx:C}}}),{c(){d(e.$$.fragment)},l(l){j(e.$$.fragment,l)},m(l,m){y(e,l,m),c=!0},p(l,m){const T={};m&2&&(T.$$scope={dirty:m,ctx:l}),e.$set(T)},i(l){c||(f(e.$$.fragment,l),c=!0)},o(l){g(e.$$.fragment,l),c=!1},d(l){b(e,l)}}}function Jl(C){let e,c="Tokenize the text and return TensorFlow tensors:",l,m,T,U,W="Pass your inputs to the model and return the <code>logits</code>:",$,G,Z,B,N="Get the class with the highest probability, and use the model’s <code>id2label</code> mapping to convert it to a text label:",v,_,I;return m=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),G=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>logits = model(**inputs).logits`}}),_=new k({props:{code:"cHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyUyMCUzRCUyMHRmLm1hdGguYXJnbWF4KGxvZ2l0cyUyQyUyMGF4aXMlM0QtMSklMEFwcmVkaWN0ZWRfdG9rZW5fY2xhc3MlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0JTVEJTIwZm9yJTIwdCUyMGluJTIwcHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyU1QjAlNUQubnVtcHkoKS50b2xpc3QoKSU1RCUwQXByZWRpY3RlZF90b2tlbl9jbGFzcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class_ids = tf.math.argmax(logits, axis=-<span class="hljs-number">1</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class = [model.config.id2label[t] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predicted_token_class_ids[<span class="hljs-number">0</span>].numpy().tolist()]
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class
[<span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;I-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-group&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>]`}}),{c(){e=h("p"),e.textContent=c,l=r(),d(m.$$.fragment),T=r(),U=h("p"),U.innerHTML=W,$=r(),d(G.$$.fragment),Z=r(),B=h("p"),B.innerHTML=N,v=r(),d(_.$$.fragment)},l(i){e=M(i,"P",{"data-svelte-h":!0}),w(e)!=="svelte-s1qr7b"&&(e.textContent=c),l=o(i),j(m.$$.fragment,i),T=o(i),U=M(i,"P",{"data-svelte-h":!0}),w(U)!=="svelte-f3g043"&&(U.innerHTML=W),$=o(i),j(G.$$.fragment,i),Z=o(i),B=M(i,"P",{"data-svelte-h":!0}),w(B)!=="svelte-6mgrol"&&(B.innerHTML=N),v=o(i),j(_.$$.fragment,i)},m(i,x){t(i,e,x),t(i,l,x),y(m,i,x),t(i,T,x),t(i,U,x),t(i,$,x),y(G,i,x),t(i,Z,x),t(i,B,x),t(i,v,x),y(_,i,x),I=!0},p:ws,i(i){I||(f(m.$$.fragment,i),f(G.$$.fragment,i),f(_.$$.fragment,i),I=!0)},o(i){g(m.$$.fragment,i),g(G.$$.fragment,i),g(_.$$.fragment,i),I=!1},d(i){i&&(a(e),a(l),a(T),a(U),a($),a(Z),a(B),a(v)),b(m,i),b(G,i),b(_,i)}}}function xl(C){let e,c;return e=new ha({props:{$$slots:{default:[Jl]},$$scope:{ctx:C}}}),{c(){d(e.$$.fragment)},l(l){j(e.$$.fragment,l)},m(l,m){y(e,l,m),c=!0},p(l,m){const T={};m&2&&(T.$$scope={dirty:m,ctx:l}),e.$set(T)},i(l){c||(f(e.$$.fragment,l),c=!0)},o(l){g(e.$$.fragment,l),c=!1},d(l){b(e,l)}}}function Ul(C){let e,c,l,m,T,U,W,$,G,Z,B="Token classification",N,v,_,I,i,x,q="Token classification assigns a label to individual tokens in a sentence. One of the most common token classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence, such as a person, location, or organization.",H,X,u="This guide will show you how to:",A,F,Ma='<li>Finetune <a href="https://huggingface.co/distilbert-base-uncased" rel="nofollow">DistilBERT</a> on the <a href="https://huggingface.co/datasets/wnut_17" rel="nofollow">WNUT 17</a> dataset to detect new entities.</li> <li>Use your finetuned model for inference.</li>',as,Q,ts,Y,ua="Before you begin, make sure you have all the necessary libraries installed:",ls,S,es,L,da="We encourage you to login to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to login:",ns,P,ps,z,O,ss,D,hs,K,ja="Load WNUT 17 dataset",rs,p,J="Start by loading the WNUT 17 dataset from the 🤗 Datasets library:",Ts,Js,Ua,xs,Tt="Then take a look at an example:",$a,Us,_a,$s,Jt="Each number in <code>ner_tags</code> represents an entity. Convert the numbers to their label names to find out what the entities are:",ka,_s,Ca,ks,xt="The letter that prefixes each <code>ner_tag</code> indicates the token position of the entity:",va,Cs,Ut=`<li><code>B-</code> indicates the beginning of an entity.</li> <li><code>I-</code> indicates a token is contained inside the same entity (for example, the <code>State</code> token is a part of an entity like
<code>Empire State Building</code>).</li> <li><code>0</code> indicates the token doesn’t correspond to any entity.</li>`,Ia,os,Ms,ya,vs,yt,ra,$t="Preprocess",Za,Is,Ba,Zs,_t="The next step is to load a DistilBERT tokenizer to preprocess the <code>tokens</code> field:",Aa,Bs,Ga,As,kt="As you saw in the example <code>tokens</code> field above, it looks like the input has already been tokenized. But the input actually hasn’t been tokenized yet and you’ll need to set <code>is_split_into_words=True</code> to tokenize the words into subwords. For example:",Ra,Gs,Ea,Rs,Ct="However, this adds some special tokens <code>[CLS]</code> and <code>[SEP]</code> and the subword tokenization creates a mismatch between the input and labels. A single word corresponding to a single label may now be split into two subwords. You’ll need to realign the tokens and labels by:",Wa,Es,vt='<li>Mapping all tokens to their corresponding word with the <a href="https://huggingface.co/docs/transformers/main_classes/tokenizer#transformers.BatchEncoding.word_ids" rel="nofollow"><code>word_ids</code></a> method.</li> <li>Assigning the label <code>-100</code> to the special tokens <code>[CLS]</code> and <code>[SEP]</code> so they’re ignored by the PyTorch loss function (see <a href="https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html" rel="nofollow">CrossEntropyLoss</a>).</li> <li>Only labeling the first token of a given word. Assign <code>-100</code> to other subtokens from the same word.</li>',Xa,Ws,It="Here is how you can create a function to realign the tokens and labels, and truncate sequences to be no longer than DistilBERT’s maximum input length:",Va,Xs,Na,Vs,Zt='To apply the preprocessing function over the entire dataset, use 🤗 Datasets <a href="https://huggingface.co/docs/datasets/v2.14.5/en/package_reference/main_classes#datasets.Dataset.map" rel="nofollow">map</a> function. You can speed up the <code>map</code> function by setting <code>batched=True</code> to process multiple elements of the dataset at once:',Fa,Ns,Ha,Fs,Bt='Now create a batch of examples using <a href="/docs/transformers/v4.33.3/en/main_classes/data_collator#transformers.DataCollatorWithPadding">DataCollatorWithPadding</a>. It’s more efficient to <em>dynamically pad</em> the sentences to the longest length in a batch during collation, instead of padding the whole dataset to the maximum length.',za,us,Qa,is,ds,fa,Hs,ft,oa,At="Evaluate",qa,zs,Gt='Including a metric during training is often helpful for evaluating your model’s performance. You can quickly load a evaluation method with the 🤗 <a href="https://huggingface.co/docs/evaluate/index" rel="nofollow">Evaluate</a> library. For this task, load the <a href="https://huggingface.co/spaces/evaluate-metric/seqeval" rel="nofollow">seqeval</a> framework (see the 🤗 Evaluate <a href="https://huggingface.co/docs/evaluate/a_quick_tour" rel="nofollow">quick tour</a> to learn more about how to load and compute a metric). Seqeval actually produces several scores: precision, recall, F1, and accuracy.',Ya,Qs,Sa,qs,Rt='Get the NER labels first, and then create a function that passes your true predictions and true labels to <a href="https://huggingface.co/docs/evaluate/v0.4.0/en/package_reference/main_classes#evaluate.EvaluationModule.compute" rel="nofollow">compute</a> to calculate the scores:',La,Ys,Pa,Ss,Et="Your <code>compute_metrics</code> function is ready to go now, and you’ll return to it when you setup your training.",Da,cs,js,ga,Ls,gt,ia,Wt="Train",Ka,Ps,Xt="Before you start training your model, create a map of the expected ids to their labels with <code>id2label</code> and <code>label2id</code>:",Oa,Ds,st,ys,at,fs,tt,ms,gs,ba,Ks,bt,ca,Vt="Inference",lt,Os,Nt="Great, now that you’ve finetuned a model, you can use it for inference!",et,sa,Ft="Grab some text you’d like to run inference on:",nt,aa,pt,ta,Ht='The simplest way to try out your finetuned model for inference is to use it in a <a href="/docs/transformers/v4.33.3/en/main_classes/pipelines#transformers.pipeline">pipeline()</a>. Instantiate a <code>pipeline</code> for NER with your model, and pass your text to it:',rt,la,ot,ea,zt="You can also manually replicate the results of the <code>pipeline</code> if you’d like:",it,bs,ct,wa,mt;return $=new ma({}),v=new ol({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/token_classification.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/token_classification.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/token_classification.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/token_classification.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/token_classification.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/token_classification.ipynb"}]}}),I=new sl({props:{id:"wVHdVlPScxA"}}),Q=new jt({props:{$$slots:{default:[il]},$$scope:{ctx:C}}}),S=new k({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwZXZhbHVhdGUlMjBzZXFldmFs",highlighted:"pip install transformers datasets evaluate seqeval"}}),P=new k({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_login()`}}),D=new ma({}),Js=new k({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBd251dCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ3bnV0XzE3JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`}}),Us=new k({props:{code:"d251dCU1QiUyMnRyYWluJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`}}),_s=new k({props:{code:"bGFiZWxfbGlzdCUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RC5mZWF0dXJlcyU1QmYlMjJuZXJfdGFncyUyMiU1RC5mZWF0dXJlLm5hbWVzJTBBbGFiZWxfbGlzdA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`}}),vs=new ma({}),Is=new sl({props:{id:"iY2AZYdZAr0"}}),Bs=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),Gs=new k({props:{code:"ZXhhbXBsZSUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RCU1QjAlNUQlMEF0b2tlbml6ZWRfaW5wdXQlMjAlM0QlMjB0b2tlbml6ZXIoZXhhbXBsZSU1QiUyMnRva2VucyUyMiU1RCUyQyUyMGlzX3NwbGl0X2ludG9fd29yZHMlM0RUcnVlKSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci5jb252ZXJ0X2lkc190b190b2tlbnModG9rZW5pemVkX2lucHV0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQXRva2Vucw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>example = wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),Xs=new k({props:{code:"ZGVmJTIwdG9rZW5pemVfYW5kX2FsaWduX2xhYmVscyhleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydG9rZW5zJTIyJTVEJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMjBpc19zcGxpdF9pbnRvX3dvcmRzJTNEVHJ1ZSklMEElMEElMjAlMjAlMjAlMjBsYWJlbHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBpJTJDJTIwbGFiZWwlMjBpbiUyMGVudW1lcmF0ZShleGFtcGxlcyU1QmYlMjJuZXJfdGFncyUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkcyUyMCUzRCUyMHRva2VuaXplZF9pbnB1dHMud29yZF9pZHMoYmF0Y2hfaW5kZXglM0RpKSUyMCUyMCUyMyUyME1hcCUyMHRva2VucyUyMHRvJTIwdGhlaXIlMjByZXNwZWN0aXZlJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyME5vbmUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjB3b3JkX2lkeCUyMGluJTIwd29yZF9pZHMlM0ElMjAlMjAlMjMlMjBTZXQlMjB0aGUlMjBzcGVjaWFsJTIwdG9rZW5zJTIwdG8lMjAtMTAwLiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZHglMjBpcyUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKC0xMDApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxpZiUyMHdvcmRfaWR4JTIwISUzRCUyMHByZXZpb3VzX3dvcmRfaWR4JTNBJTIwJTIwJTIzJTIwT25seSUyMGxhYmVsJTIwdGhlJTIwZmlyc3QlMjB0b2tlbiUyMG9mJTIwYSUyMGdpdmVuJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKGxhYmVsJTVCd29yZF9pZHglNUQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVsX2lkcy5hcHBlbmQoLTEwMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyMHdvcmRfaWR4JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFiZWxzLmFwcGVuZChsYWJlbF9pZHMpJTBBJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVkX2lucHV0cyU1QiUyMmxhYmVscyUyMiU1RCUyMCUzRCUyMGxhYmVscyUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHRva2VuaXplZF9pbnB1dHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
<span class="hljs-meta">... </span>        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
<span class="hljs-meta">... </span>        previous_word_idx = <span class="hljs-literal">None</span>
<span class="hljs-meta">... </span>        label_ids = []
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
<span class="hljs-meta">... </span>                label_ids.append(label[word_idx])
<span class="hljs-meta">... </span>            <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            previous_word_idx = word_idx
<span class="hljs-meta">... </span>        labels.append(label_ids)

<span class="hljs-meta">... </span>    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenized_inputs`}}),Ns=new k({props:{code:"dG9rZW5pemVkX3dudXQlMjAlM0QlMjB3bnV0Lm1hcCh0b2tlbml6ZV9hbmRfYWxpZ25fbGFiZWxzJTJDJTIwYmF0Y2hlZCUzRFRydWUp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)'}}),us=new wt({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Ml],pytorch:[ml]},$$scope:{ctx:C}}}),Hs=new ma({}),Qs=new k({props:{code:"aW1wb3J0JTIwZXZhbHVhdGUlMEElMEFzZXFldmFsJTIwJTNEJTIwZXZhbHVhdGUubG9hZCglMjJzZXFldmFsJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> evaluate

<span class="hljs-meta">&gt;&gt;&gt; </span>seqeval = evaluate.load(<span class="hljs-string">&quot;seqeval&quot;</span>)`}}),Ys=new k({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBJTBBbGFiZWxzJTIwJTNEJTIwJTVCbGFiZWxfbGlzdCU1QmklNUQlMjBmb3IlMjBpJTIwaW4lMjBleGFtcGxlJTVCZiUyMm5lcl90YWdzJTIyJTVEJTVEJTBBJTBBJTBBZGVmJTIwY29tcHV0ZV9tZXRyaWNzKHApJTNBJTBBJTIwJTIwJTIwJTIwcHJlZGljdGlvbnMlMkMlMjBsYWJlbHMlMjAlM0QlMjBwJTBBJTIwJTIwJTIwJTIwcHJlZGljdGlvbnMlMjAlM0QlMjBucC5hcmdtYXgocHJlZGljdGlvbnMlMkMlMjBheGlzJTNEMiklMEElMEElMjAlMjAlMjAlMjB0cnVlX3ByZWRpY3Rpb25zJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCbGFiZWxfbGlzdCU1QnAlNUQlMjBmb3IlMjAocCUyQyUyMGwpJTIwaW4lMjB6aXAocHJlZGljdGlvbiUyQyUyMGxhYmVsKSUyMGlmJTIwbCUyMCElM0QlMjAtMTAwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwcHJlZGljdGlvbiUyQyUyMGxhYmVsJTIwaW4lMjB6aXAocHJlZGljdGlvbnMlMkMlMjBsYWJlbHMpJTBBJTIwJTIwJTIwJTIwJTVEJTBBJTIwJTIwJTIwJTIwdHJ1ZV9sYWJlbHMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUJsYWJlbF9saXN0JTVCbCU1RCUyMGZvciUyMChwJTJDJTIwbCklMjBpbiUyMHppcChwcmVkaWN0aW9uJTJDJTIwbGFiZWwpJTIwaWYlMjBsJTIwISUzRCUyMC0xMDAlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBwcmVkaWN0aW9uJTJDJTIwbGFiZWwlMjBpbiUyMHppcChwcmVkaWN0aW9ucyUyQyUyMGxhYmVscyklMEElMjAlMjAlMjAlMjAlNUQlMEElMEElMjAlMjAlMjAlMjByZXN1bHRzJTIwJTNEJTIwc2VxZXZhbC5jb21wdXRlKHByZWRpY3Rpb25zJTNEdHJ1ZV9wcmVkaWN0aW9ucyUyQyUyMHJlZmVyZW5jZXMlM0R0cnVlX2xhYmVscyklMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwcmVjaXNpb24lMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9wcmVjaXNpb24lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZWNhbGwlMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9yZWNhbGwlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJmMSUyMiUzQSUyMHJlc3VsdHMlNUIlMjJvdmVyYWxsX2YxJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWNjdXJhY3klMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9hY2N1cmFjeSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-meta">&gt;&gt;&gt; </span>labels = [label_list[i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> example[<span class="hljs-string">f&quot;ner_tags&quot;</span>]]


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">p</span>):
<span class="hljs-meta">... </span>    predictions, labels = p
<span class="hljs-meta">... </span>    predictions = np.argmax(predictions, axis=<span class="hljs-number">2</span>)

<span class="hljs-meta">... </span>    true_predictions = [
<span class="hljs-meta">... </span>        [label_list[p] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
<span class="hljs-meta">... </span>    ]
<span class="hljs-meta">... </span>    true_labels = [
<span class="hljs-meta">... </span>        [label_list[l] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
<span class="hljs-meta">... </span>    ]

<span class="hljs-meta">... </span>    results = seqeval.compute(predictions=true_predictions, references=true_labels)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;precision&quot;</span>: results[<span class="hljs-string">&quot;overall_precision&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;recall&quot;</span>: results[<span class="hljs-string">&quot;overall_recall&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;f1&quot;</span>: results[<span class="hljs-string">&quot;overall_f1&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;accuracy&quot;</span>: results[<span class="hljs-string">&quot;overall_accuracy&quot;</span>],
<span class="hljs-meta">... </span>    }`}}),Ls=new ma({}),Ds=new k({props:{code:"aWQybGFiZWwlMjAlM0QlMjAlN0IlMEElMjAlMjAlMjAlMjAwJTNBJTIwJTIyTyUyMiUyQyUwQSUyMCUyMCUyMCUyMDElM0ElMjAlMjJCLWNvcnBvcmF0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwMiUzQSUyMCUyMkktY29ycG9yYXRpb24lMjIlMkMlMEElMjAlMjAlMjAlMjAzJTNBJTIwJTIyQi1jcmVhdGl2ZS13b3JrJTIyJTJDJTBBJTIwJTIwJTIwJTIwNCUzQSUyMCUyMkktY3JlYXRpdmUtd29yayUyMiUyQyUwQSUyMCUyMCUyMCUyMDUlM0ElMjAlMjJCLWdyb3VwJTIyJTJDJTBBJTIwJTIwJTIwJTIwNiUzQSUyMCUyMkktZ3JvdXAlMjIlMkMlMEElMjAlMjAlMjAlMjA3JTNBJTIwJTIyQi1sb2NhdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMDglM0ElMjAlMjJJLWxvY2F0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwOSUzQSUyMCUyMkItcGVyc29uJTIyJTJDJTBBJTIwJTIwJTIwJTIwMTAlM0ElMjAlMjJJLXBlcnNvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMDExJTNBJTIwJTIyQi1wcm9kdWN0JTIyJTJDJTBBJTIwJTIwJTIwJTIwMTIlM0ElMjAlMjJJLXByb2R1Y3QlMjIlMkMlMEElN0QlMEFsYWJlbDJpZCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMk8lMjIlM0ElMjAwJTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1jb3Jwb3JhdGlvbiUyMiUzQSUyMDElMkMlMEElMjAlMjAlMjAlMjAlMjJJLWNvcnBvcmF0aW9uJTIyJTNBJTIwMiUyQyUwQSUyMCUyMCUyMCUyMCUyMkItY3JlYXRpdmUtd29yayUyMiUzQSUyMDMlMkMlMEElMjAlMjAlMjAlMjAlMjJJLWNyZWF0aXZlLXdvcmslMjIlM0ElMjA0JTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1ncm91cCUyMiUzQSUyMDUlMkMlMEElMjAlMjAlMjAlMjAlMjJJLWdyb3VwJTIyJTNBJTIwNiUyQyUwQSUyMCUyMCUyMCUyMCUyMkItbG9jYXRpb24lMjIlM0ElMjA3JTJDJTBBJTIwJTIwJTIwJTIwJTIySS1sb2NhdGlvbiUyMiUzQSUyMDglMkMlMEElMjAlMjAlMjAlMjAlMjJCLXBlcnNvbiUyMiUzQSUyMDklMkMlMEElMjAlMjAlMjAlMjAlMjJJLXBlcnNvbiUyMiUzQSUyMDEwJTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1wcm9kdWN0JTIyJTNBJTIwMTElMkMlMEElMjAlMjAlMjAlMjAlMjJJLXByb2R1Y3QlMjIlM0ElMjAxMiUyQyUwQSU3RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>id2label = {
<span class="hljs-meta">... </span>    <span class="hljs-number">0</span>: <span class="hljs-string">&quot;O&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">1</span>: <span class="hljs-string">&quot;B-corporation&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">2</span>: <span class="hljs-string">&quot;I-corporation&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">3</span>: <span class="hljs-string">&quot;B-creative-work&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">4</span>: <span class="hljs-string">&quot;I-creative-work&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">5</span>: <span class="hljs-string">&quot;B-group&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">6</span>: <span class="hljs-string">&quot;I-group&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">7</span>: <span class="hljs-string">&quot;B-location&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">8</span>: <span class="hljs-string">&quot;I-location&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">9</span>: <span class="hljs-string">&quot;B-person&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">10</span>: <span class="hljs-string">&quot;I-person&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">11</span>: <span class="hljs-string">&quot;B-product&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">12</span>: <span class="hljs-string">&quot;I-product&quot;</span>,
<span class="hljs-meta">... </span>}
<span class="hljs-meta">&gt;&gt;&gt; </span>label2id = {
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;O&quot;</span>: <span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-corporation&quot;</span>: <span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-corporation&quot;</span>: <span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-creative-work&quot;</span>: <span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-creative-work&quot;</span>: <span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-group&quot;</span>: <span class="hljs-number">5</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-group&quot;</span>: <span class="hljs-number">6</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-location&quot;</span>: <span class="hljs-number">7</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-location&quot;</span>: <span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-person&quot;</span>: <span class="hljs-number">9</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-person&quot;</span>: <span class="hljs-number">10</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-product&quot;</span>: <span class="hljs-number">11</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-product&quot;</span>: <span class="hljs-number">12</span>,
<span class="hljs-meta">... </span>}`}}),ys=new wt({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[gl],pytorch:[jl]},$$scope:{ctx:C}}}),fs=new jt({props:{$$slots:{default:[bl]},$$scope:{ctx:C}}}),Ks=new ma({}),aa=new k({props:{code:"dGV4dCUyMCUzRCUyMCUyMlRoZSUyMEdvbGRlbiUyMFN0YXRlJTIwV2FycmlvcnMlMjBhcmUlMjBhbiUyMEFtZXJpY2FuJTIwcHJvZmVzc2lvbmFsJTIwYmFza2V0YmFsbCUyMHRlYW0lMjBiYXNlZCUyMGluJTIwU2FuJTIwRnJhbmNpc2NvLiUyMg==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>text = <span class="hljs-string">&quot;The Golden State Warriors are an American professional basketball team based in San Francisco.&quot;</span>'}}),la=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBY2xhc3NpZmllciUyMCUzRCUyMHBpcGVsaW5lKCUyMm5lciUyMiUyQyUyMG1vZGVsJTNEJTIyc3RldmhsaXUlMkZteV9hd2Vzb21lX3dudXRfbW9kZWwlMjIpJTBBY2xhc3NpZmllcih0ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>classifier = pipeline(<span class="hljs-string">&quot;ner&quot;</span>, model=<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>classifier(text)
[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.42658573</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">2</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;golden&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">4</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">10</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.35856336</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">3</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;state&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-group&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.3064001</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;warriors&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">17</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">25</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.65523505</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;san&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">80</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">83</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.4668663</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;francisco&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">84</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">93</span>}]`}}),bs=new wt({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[xl],pytorch:[Tl]},$$scope:{ctx:C}}}),{c(){e=h("meta"),c=r(),l=h("p"),m=r(),T=h("h1"),U=h("a"),W=h("span"),d($.$$.fragment),G=r(),Z=h("span"),Z.textContent=B,N=r(),d(v.$$.fragment),_=r(),d(I.$$.fragment),i=r(),x=h("p"),x.textContent=q,H=r(),X=h("p"),X.textContent=u,A=r(),F=h("ol"),F.innerHTML=Ma,as=r(),d(Q.$$.fragment),ts=r(),Y=h("p"),Y.textContent=ua,ls=r(),d(S.$$.fragment),es=r(),L=h("p"),L.textContent=da,ns=r(),d(P.$$.fragment),ps=r(),z=h("h2"),O=h("a"),ss=h("span"),d(D.$$.fragment),hs=r(),K=h("span"),K.textContent=ja,rs=r(),p=h("p"),p.textContent=J,Ts=r(),d(Js.$$.fragment),Ua=r(),xs=h("p"),xs.textContent=Tt,$a=r(),d(Us.$$.fragment),_a=r(),$s=h("p"),$s.innerHTML=Jt,ka=r(),d(_s.$$.fragment),Ca=r(),ks=h("p"),ks.innerHTML=xt,va=r(),Cs=h("ul"),Cs.innerHTML=Ut,Ia=r(),os=h("h2"),Ms=h("a"),ya=h("span"),d(vs.$$.fragment),yt=r(),ra=h("span"),ra.textContent=$t,Za=r(),d(Is.$$.fragment),Ba=r(),Zs=h("p"),Zs.innerHTML=_t,Aa=r(),d(Bs.$$.fragment),Ga=r(),As=h("p"),As.innerHTML=kt,Ra=r(),d(Gs.$$.fragment),Ea=r(),Rs=h("p"),Rs.innerHTML=Ct,Wa=r(),Es=h("ol"),Es.innerHTML=vt,Xa=r(),Ws=h("p"),Ws.textContent=It,Va=r(),d(Xs.$$.fragment),Na=r(),Vs=h("p"),Vs.innerHTML=Zt,Fa=r(),d(Ns.$$.fragment),Ha=r(),Fs=h("p"),Fs.innerHTML=Bt,za=r(),d(us.$$.fragment),Qa=r(),is=h("h2"),ds=h("a"),fa=h("span"),d(Hs.$$.fragment),ft=r(),oa=h("span"),oa.textContent=At,qa=r(),zs=h("p"),zs.innerHTML=Gt,Ya=r(),d(Qs.$$.fragment),Sa=r(),qs=h("p"),qs.innerHTML=Rt,La=r(),d(Ys.$$.fragment),Pa=r(),Ss=h("p"),Ss.innerHTML=Et,Da=r(),cs=h("h2"),js=h("a"),ga=h("span"),d(Ls.$$.fragment),gt=r(),ia=h("span"),ia.textContent=Wt,Ka=r(),Ps=h("p"),Ps.innerHTML=Xt,Oa=r(),d(Ds.$$.fragment),st=r(),d(ys.$$.fragment),at=r(),d(fs.$$.fragment),tt=r(),ms=h("h2"),gs=h("a"),ba=h("span"),d(Ks.$$.fragment),bt=r(),ca=h("span"),ca.textContent=Vt,lt=r(),Os=h("p"),Os.textContent=Nt,et=r(),sa=h("p"),sa.textContent=Ft,nt=r(),d(aa.$$.fragment),pt=r(),ta=h("p"),ta.innerHTML=Ht,rt=r(),d(la.$$.fragment),ot=r(),ea=h("p"),ea.innerHTML=zt,it=r(),d(bs.$$.fragment),ct=r(),wa=h("p"),this.h()},l(s){const n=rl("svelte-1phssyn",document.head);e=M(n,"META",{name:!0,content:!0}),n.forEach(a),c=o(s),l=M(s,"P",{}),V(l).forEach(a),m=o(s),T=M(s,"H1",{class:!0});var na=V(T);U=M(na,"A",{id:!0,class:!0,href:!0});var Ta=V(U);W=M(Ta,"SPAN",{});var Ja=V(W);j($.$$.fragment,Ja),Ja.forEach(a),Ta.forEach(a),G=o(na),Z=M(na,"SPAN",{"data-svelte-h":!0}),w(Z)!=="svelte-18ca1ds"&&(Z.textContent=B),na.forEach(a),N=o(s),j(v.$$.fragment,s),_=o(s),j(I.$$.fragment,s),i=o(s),x=M(s,"P",{"data-svelte-h":!0}),w(x)!=="svelte-o0annf"&&(x.textContent=q),H=o(s),X=M(s,"P",{"data-svelte-h":!0}),w(X)!=="svelte-1aff4p7"&&(X.textContent=u),A=o(s),F=M(s,"OL",{"data-svelte-h":!0}),w(F)!=="svelte-hnqwcf"&&(F.innerHTML=Ma),as=o(s),j(Q.$$.fragment,s),ts=o(s),Y=M(s,"P",{"data-svelte-h":!0}),w(Y)!=="svelte-1c9nexd"&&(Y.textContent=ua),ls=o(s),j(S.$$.fragment,s),es=o(s),L=M(s,"P",{"data-svelte-h":!0}),w(L)!=="svelte-k76o1m"&&(L.textContent=da),ns=o(s),j(P.$$.fragment,s),ps=o(s),z=M(s,"H2",{class:!0});var pa=V(z);O=M(pa,"A",{id:!0,class:!0,href:!0});var xa=V(O);ss=M(xa,"SPAN",{});var Qt=V(ss);j(D.$$.fragment,Qt),Qt.forEach(a),xa.forEach(a),hs=o(pa),K=M(pa,"SPAN",{"data-svelte-h":!0}),w(K)!=="svelte-zacbk9"&&(K.textContent=ja),pa.forEach(a),rs=o(s),p=M(s,"P",{"data-svelte-h":!0}),w(p)!=="svelte-tluco0"&&(p.textContent=J),Ts=o(s),j(Js.$$.fragment,s),Ua=o(s),xs=M(s,"P",{"data-svelte-h":!0}),w(xs)!=="svelte-1m91ua0"&&(xs.textContent=Tt),$a=o(s),j(Us.$$.fragment,s),_a=o(s),$s=M(s,"P",{"data-svelte-h":!0}),w($s)!=="svelte-1684qxr"&&($s.innerHTML=Jt),ka=o(s),j(_s.$$.fragment,s),Ca=o(s),ks=M(s,"P",{"data-svelte-h":!0}),w(ks)!=="svelte-b2md1a"&&(ks.innerHTML=xt),va=o(s),Cs=M(s,"UL",{"data-svelte-h":!0}),w(Cs)!=="svelte-13punrg"&&(Cs.innerHTML=Ut),Ia=o(s),os=M(s,"H2",{class:!0});var ht=V(os);Ms=M(ht,"A",{id:!0,class:!0,href:!0});var qt=V(Ms);ya=M(qt,"SPAN",{});var Yt=V(ya);j(vs.$$.fragment,Yt),Yt.forEach(a),qt.forEach(a),yt=o(ht),ra=M(ht,"SPAN",{"data-svelte-h":!0}),w(ra)!=="svelte-1cg9qj"&&(ra.textContent=$t),ht.forEach(a),Za=o(s),j(Is.$$.fragment,s),Ba=o(s),Zs=M(s,"P",{"data-svelte-h":!0}),w(Zs)!=="svelte-1pk56gi"&&(Zs.innerHTML=_t),Aa=o(s),j(Bs.$$.fragment,s),Ga=o(s),As=M(s,"P",{"data-svelte-h":!0}),w(As)!=="svelte-el75ld"&&(As.innerHTML=kt),Ra=o(s),j(Gs.$$.fragment,s),Ea=o(s),Rs=M(s,"P",{"data-svelte-h":!0}),w(Rs)!=="svelte-1lf9iv0"&&(Rs.innerHTML=Ct),Wa=o(s),Es=M(s,"OL",{"data-svelte-h":!0}),w(Es)!=="svelte-1heekds"&&(Es.innerHTML=vt),Xa=o(s),Ws=M(s,"P",{"data-svelte-h":!0}),w(Ws)!=="svelte-yddais"&&(Ws.textContent=It),Va=o(s),j(Xs.$$.fragment,s),Na=o(s),Vs=M(s,"P",{"data-svelte-h":!0}),w(Vs)!=="svelte-147cojc"&&(Vs.innerHTML=Zt),Fa=o(s),j(Ns.$$.fragment,s),Ha=o(s),Fs=M(s,"P",{"data-svelte-h":!0}),w(Fs)!=="svelte-lsqz9r"&&(Fs.innerHTML=Bt),za=o(s),j(us.$$.fragment,s),Qa=o(s),is=M(s,"H2",{class:!0});var Mt=V(is);ds=M(Mt,"A",{id:!0,class:!0,href:!0});var St=V(ds);fa=M(St,"SPAN",{});var Lt=V(fa);j(Hs.$$.fragment,Lt),Lt.forEach(a),St.forEach(a),ft=o(Mt),oa=M(Mt,"SPAN",{"data-svelte-h":!0}),w(oa)!=="svelte-sh8s6s"&&(oa.textContent=At),Mt.forEach(a),qa=o(s),zs=M(s,"P",{"data-svelte-h":!0}),w(zs)!=="svelte-434hvn"&&(zs.innerHTML=Gt),Ya=o(s),j(Qs.$$.fragment,s),Sa=o(s),qs=M(s,"P",{"data-svelte-h":!0}),w(qs)!=="svelte-1nd9y0m"&&(qs.innerHTML=Rt),La=o(s),j(Ys.$$.fragment,s),Pa=o(s),Ss=M(s,"P",{"data-svelte-h":!0}),w(Ss)!=="svelte-183aynn"&&(Ss.innerHTML=Et),Da=o(s),cs=M(s,"H2",{class:!0});var ut=V(cs);js=M(ut,"A",{id:!0,class:!0,href:!0});var Pt=V(js);ga=M(Pt,"SPAN",{});var Dt=V(ga);j(Ls.$$.fragment,Dt),Dt.forEach(a),Pt.forEach(a),gt=o(ut),ia=M(ut,"SPAN",{"data-svelte-h":!0}),w(ia)!=="svelte-5arm0l"&&(ia.textContent=Wt),ut.forEach(a),Ka=o(s),Ps=M(s,"P",{"data-svelte-h":!0}),w(Ps)!=="svelte-18c6io4"&&(Ps.innerHTML=Xt),Oa=o(s),j(Ds.$$.fragment,s),st=o(s),j(ys.$$.fragment,s),at=o(s),j(fs.$$.fragment,s),tt=o(s),ms=M(s,"H2",{class:!0});var dt=V(ms);gs=M(dt,"A",{id:!0,class:!0,href:!0});var Kt=V(gs);ba=M(Kt,"SPAN",{});var Ot=V(ba);j(Ks.$$.fragment,Ot),Ot.forEach(a),Kt.forEach(a),bt=o(dt),ca=M(dt,"SPAN",{"data-svelte-h":!0}),w(ca)!=="svelte-199uz7g"&&(ca.textContent=Vt),dt.forEach(a),lt=o(s),Os=M(s,"P",{"data-svelte-h":!0}),w(Os)!=="svelte-633ppb"&&(Os.textContent=Nt),et=o(s),sa=M(s,"P",{"data-svelte-h":!0}),w(sa)!=="svelte-o1jbfg"&&(sa.textContent=Ft),nt=o(s),j(aa.$$.fragment,s),pt=o(s),ta=M(s,"P",{"data-svelte-h":!0}),w(ta)!=="svelte-g5h1kj"&&(ta.innerHTML=Ht),rt=o(s),j(la.$$.fragment,s),ot=o(s),ea=M(s,"P",{"data-svelte-h":!0}),w(ea)!=="svelte-1njl8vm"&&(ea.innerHTML=zt),it=o(s),j(bs.$$.fragment,s),ct=o(s),wa=M(s,"P",{}),V(wa).forEach(a),this.h()},h(){R(e,"name","hf:doc:metadata"),R(e,"content",JSON.stringify($l)),R(U,"id","token-classification"),R(U,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(U,"href","#token-classification"),R(T,"class","relative group"),R(O,"id","load-wnut-17-dataset"),R(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(O,"href","#load-wnut-17-dataset"),R(z,"class","relative group"),R(Ms,"id","preprocess"),R(Ms,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(Ms,"href","#preprocess"),R(os,"class","relative group"),R(ds,"id","evaluate"),R(ds,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(ds,"href","#evaluate"),R(is,"class","relative group"),R(js,"id","train"),R(js,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(js,"href","#train"),R(cs,"class","relative group"),R(gs,"id","inference"),R(gs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(gs,"href","#inference"),R(ms,"class","relative group")},m(s,n){E(document.head,e),t(s,c,n),t(s,l,n),t(s,m,n),t(s,T,n),E(T,U),E(U,W),y($,W,null),E(T,G),E(T,Z),t(s,N,n),y(v,s,n),t(s,_,n),y(I,s,n),t(s,i,n),t(s,x,n),t(s,H,n),t(s,X,n),t(s,A,n),t(s,F,n),t(s,as,n),y(Q,s,n),t(s,ts,n),t(s,Y,n),t(s,ls,n),y(S,s,n),t(s,es,n),t(s,L,n),t(s,ns,n),y(P,s,n),t(s,ps,n),t(s,z,n),E(z,O),E(O,ss),y(D,ss,null),E(z,hs),E(z,K),t(s,rs,n),t(s,p,n),t(s,Ts,n),y(Js,s,n),t(s,Ua,n),t(s,xs,n),t(s,$a,n),y(Us,s,n),t(s,_a,n),t(s,$s,n),t(s,ka,n),y(_s,s,n),t(s,Ca,n),t(s,ks,n),t(s,va,n),t(s,Cs,n),t(s,Ia,n),t(s,os,n),E(os,Ms),E(Ms,ya),y(vs,ya,null),E(os,yt),E(os,ra),t(s,Za,n),y(Is,s,n),t(s,Ba,n),t(s,Zs,n),t(s,Aa,n),y(Bs,s,n),t(s,Ga,n),t(s,As,n),t(s,Ra,n),y(Gs,s,n),t(s,Ea,n),t(s,Rs,n),t(s,Wa,n),t(s,Es,n),t(s,Xa,n),t(s,Ws,n),t(s,Va,n),y(Xs,s,n),t(s,Na,n),t(s,Vs,n),t(s,Fa,n),y(Ns,s,n),t(s,Ha,n),t(s,Fs,n),t(s,za,n),y(us,s,n),t(s,Qa,n),t(s,is,n),E(is,ds),E(ds,fa),y(Hs,fa,null),E(is,ft),E(is,oa),t(s,qa,n),t(s,zs,n),t(s,Ya,n),y(Qs,s,n),t(s,Sa,n),t(s,qs,n),t(s,La,n),y(Ys,s,n),t(s,Pa,n),t(s,Ss,n),t(s,Da,n),t(s,cs,n),E(cs,js),E(js,ga),y(Ls,ga,null),E(cs,gt),E(cs,ia),t(s,Ka,n),t(s,Ps,n),t(s,Oa,n),y(Ds,s,n),t(s,st,n),y(ys,s,n),t(s,at,n),y(fs,s,n),t(s,tt,n),t(s,ms,n),E(ms,gs),E(gs,ba),y(Ks,ba,null),E(ms,bt),E(ms,ca),t(s,lt,n),t(s,Os,n),t(s,et,n),t(s,sa,n),t(s,nt,n),y(aa,s,n),t(s,pt,n),t(s,ta,n),t(s,rt,n),y(la,s,n),t(s,ot,n),t(s,ea,n),t(s,it,n),y(bs,s,n),t(s,ct,n),t(s,wa,n),mt=!0},p(s,[n]){const na={};n&2&&(na.$$scope={dirty:n,ctx:s}),Q.$set(na);const Ta={};n&2&&(Ta.$$scope={dirty:n,ctx:s}),us.$set(Ta);const Ja={};n&2&&(Ja.$$scope={dirty:n,ctx:s}),ys.$set(Ja);const pa={};n&2&&(pa.$$scope={dirty:n,ctx:s}),fs.$set(pa);const xa={};n&2&&(xa.$$scope={dirty:n,ctx:s}),bs.$set(xa)},i(s){mt||(f($.$$.fragment,s),f(v.$$.fragment,s),f(I.$$.fragment,s),f(Q.$$.fragment,s),f(S.$$.fragment,s),f(P.$$.fragment,s),f(D.$$.fragment,s),f(Js.$$.fragment,s),f(Us.$$.fragment,s),f(_s.$$.fragment,s),f(vs.$$.fragment,s),f(Is.$$.fragment,s),f(Bs.$$.fragment,s),f(Gs.$$.fragment,s),f(Xs.$$.fragment,s),f(Ns.$$.fragment,s),f(us.$$.fragment,s),f(Hs.$$.fragment,s),f(Qs.$$.fragment,s),f(Ys.$$.fragment,s),f(Ls.$$.fragment,s),f(Ds.$$.fragment,s),f(ys.$$.fragment,s),f(fs.$$.fragment,s),f(Ks.$$.fragment,s),f(aa.$$.fragment,s),f(la.$$.fragment,s),f(bs.$$.fragment,s),mt=!0)},o(s){g($.$$.fragment,s),g(v.$$.fragment,s),g(I.$$.fragment,s),g(Q.$$.fragment,s),g(S.$$.fragment,s),g(P.$$.fragment,s),g(D.$$.fragment,s),g(Js.$$.fragment,s),g(Us.$$.fragment,s),g(_s.$$.fragment,s),g(vs.$$.fragment,s),g(Is.$$.fragment,s),g(Bs.$$.fragment,s),g(Gs.$$.fragment,s),g(Xs.$$.fragment,s),g(Ns.$$.fragment,s),g(us.$$.fragment,s),g(Hs.$$.fragment,s),g(Qs.$$.fragment,s),g(Ys.$$.fragment,s),g(Ls.$$.fragment,s),g(Ds.$$.fragment,s),g(ys.$$.fragment,s),g(fs.$$.fragment,s),g(Ks.$$.fragment,s),g(aa.$$.fragment,s),g(la.$$.fragment,s),g(bs.$$.fragment,s),mt=!1},d(s){s&&(a(c),a(l),a(m),a(T),a(N),a(_),a(i),a(x),a(H),a(X),a(A),a(F),a(as),a(ts),a(Y),a(ls),a(es),a(L),a(ns),a(ps),a(z),a(rs),a(p),a(Ts),a(Ua),a(xs),a($a),a(_a),a($s),a(ka),a(Ca),a(ks),a(va),a(Cs),a(Ia),a(os),a(Za),a(Ba),a(Zs),a(Aa),a(Ga),a(As),a(Ra),a(Ea),a(Rs),a(Wa),a(Es),a(Xa),a(Ws),a(Va),a(Na),a(Vs),a(Fa),a(Ha),a(Fs),a(za),a(Qa),a(is),a(qa),a(zs),a(Ya),a(Sa),a(qs),a(La),a(Pa),a(Ss),a(Da),a(cs),a(Ka),a(Ps),a(Oa),a(st),a(at),a(tt),a(ms),a(lt),a(Os),a(et),a(sa),a(nt),a(pt),a(ta),a(rt),a(ot),a(ea),a(it),a(ct),a(wa)),a(e),b($),b(v,s),b(I,s),b(Q,s),b(S,s),b(P,s),b(D),b(Js,s),b(Us,s),b(_s,s),b(vs),b(Is,s),b(Bs,s),b(Gs,s),b(Xs,s),b(Ns,s),b(us,s),b(Hs),b(Qs,s),b(Ys,s),b(Ls),b(Ds,s),b(ys,s),b(fs,s),b(Ks),b(aa,s),b(la,s),b(bs,s)}}}const $l={local:"token-classification",sections:[{local:"load-wnut-17-dataset",title:"Load WNUT 17 dataset"},{local:"preprocess",title:"Preprocess"},{local:"evaluate",title:"Evaluate"},{local:"train",title:"Train"},{local:"inference",title:"Inference"}],title:"Token classification"};function _l(C){return el(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Rl extends nl{constructor(e){super(),pl(this,e,_l,Ul,ll,{})}}export{Rl as component};
