import{s as Ea,o as Wa,n as ks}from"../chunks/scheduler.d586627e.js";import{S as za,i as Xa,g as d,s as i,u as h,A as Ba,h as u,f as t,c,j as B,v as M,y as b,k as v,o as A,a as l,w as j,d as f,t as g,x as y}from"../chunks/index.b757c891.js";import{T as ca}from"../chunks/Tip.4380e997.js";import{Y as Ra}from"../chunks/Youtube.cab44801.js";import{I as ia}from"../chunks/IconCopyLink.c074b82b.js";import{C as E}from"../chunks/CodeBlock.36019a2e.js";import{F as Ga,M as ma}from"../chunks/Markdown.1f113748.js";function Va(J){let a,p='Consulte a <a href="https://huggingface.co/tasks/token-classification" rel="nofollow">página de tarefas de classificação de tokens</a> para obter mais informações sobre outras formas de classificação de tokens e seus modelos, conjuntos de dados e métricas associadas.';return{c(){a=d("p"),a.innerHTML=p},l(e){a=u(e,"P",{"data-svelte-h":!0}),b(a)!=="svelte-dbkroz"&&(a.innerHTML=p)},m(e,r){l(e,a,r)},p:ks,d(e){e&&t(a)}}}function Fa(J){let a,p;return a=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`}}),{c(){h(a.$$.fragment)},l(e){M(a.$$.fragment,e)},m(e,r){j(a,e,r),p=!0},p:ks,i(e){p||(f(a.$$.fragment,e),p=!0)},o(e){g(a.$$.fragment,e),p=!1},d(e){y(a,e)}}}function Ha(J){let a,p;return a=new ma({props:{$$slots:{default:[Fa]},$$scope:{ctx:J}}}),{c(){h(a.$$.fragment)},l(e){M(a.$$.fragment,e)},m(e,r){j(a,e,r),p=!0},p(e,r){const w={};r&2&&(w.$$scope={dirty:r,ctx:e}),a.$set(w)},i(e){p||(f(a.$$.fragment,e),p=!0)},o(e){g(a.$$.fragment,e),p=!1},d(e){y(a,e)}}}function Ya(J){let a,p;return a=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){h(a.$$.fragment)},l(e){M(a.$$.fragment,e)},m(e,r){j(a,e,r),p=!0},p:ks,i(e){p||(f(a.$$.fragment,e),p=!0)},o(e){g(a.$$.fragment,e),p=!1},d(e){y(a,e)}}}function qa(J){let a,p;return a=new ma({props:{$$slots:{default:[Ya]},$$scope:{ctx:J}}}),{c(){h(a.$$.fragment)},l(e){M(a.$$.fragment,e)},m(e,r){j(a,e,r),p=!0},p(e,r){const w={};r&2&&(w.$$scope={dirty:r,ctx:e}),a.$set(w)},i(e){p||(f(a.$$.fragment,e),p=!0)},o(e){g(a.$$.fragment,e),p=!1},d(e){y(a,e)}}}function Na(J){let a,p='Se você não estiver familiarizado com o fine-tuning de um modelo com o <code>Trainer</code>, dê uma olhada no tutorial básico <a href="../training#finetune-with-trainer">aqui</a>!';return{c(){a=d("p"),a.innerHTML=p},l(e){a=u(e,"P",{"data-svelte-h":!0}),b(a)!=="svelte-796e4s"&&(a.innerHTML=p)},m(e,r){l(e,a,r)},p:ks,d(e){e&&t(a)}}}function Qa(J){let a,p="Carregue o DistilBERT com o <code>AutoModelForTokenClassification</code> junto com o número de rótulos esperados:",e,r,w,T,W,U,Y="Nesse ponto, restam apenas três passos:",R,k,V="<li>Definir seus hiperparâmetros de treinamento em <code>TrainingArguments</code>.</li> <li>Passar os argumentos de treinamento para o <code>Trainer</code> junto com o modelo, conjunto de dados, tokenizador e o data collator.</li> <li>Chamar a função <code>train()</code> para executar o fine-tuning do seu modelo.</li>",_,z,Z;return r=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMkMlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMFRyYWluZXIlMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkJTIyJTJDJTIwbnVtX2xhYmVscyUzRDE0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">14</span>)`}}),T=new ca({props:{$$slots:{default:[Na]},$$scope:{ctx:J}}}),z=new E({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjIuJTJGcmVzdWx0cyUyMiUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX2V2YWxfYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX2Vwb2NocyUzRDMlMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRfZGVjYXklM0QwLjAxJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0R0b2tlbml6ZWRfd251dCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX3dudXQlNUIlMjJ0ZXN0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVyJTNEdG9rZW5pemVyJTJDJTBBJTIwJTIwJTIwJTIwZGF0YV9jb2xsYXRvciUzRGRhdGFfY29sbGF0b3IlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),{c(){a=d("p"),a.innerHTML=p,e=i(),h(r.$$.fragment),w=i(),h(T.$$.fragment),W=i(),U=d("p"),U.textContent=Y,R=i(),k=d("ol"),k.innerHTML=V,_=i(),h(z.$$.fragment)},l(m){a=u(m,"P",{"data-svelte-h":!0}),b(a)!=="svelte-1vwp4j2"&&(a.innerHTML=p),e=c(m),M(r.$$.fragment,m),w=c(m),M(T.$$.fragment,m),W=c(m),U=u(m,"P",{"data-svelte-h":!0}),b(U)!=="svelte-1dd35iq"&&(U.textContent=Y),R=c(m),k=u(m,"OL",{"data-svelte-h":!0}),b(k)!=="svelte-1aiypan"&&(k.innerHTML=V),_=c(m),M(z.$$.fragment,m)},m(m,x){l(m,a,x),l(m,e,x),j(r,m,x),l(m,w,x),j(T,m,x),l(m,W,x),l(m,U,x),l(m,R,x),l(m,k,x),l(m,_,x),j(z,m,x),Z=!0},p(m,x){const C={};x&2&&(C.$$scope={dirty:x,ctx:m}),T.$set(C)},i(m){Z||(f(r.$$.fragment,m),f(T.$$.fragment,m),f(z.$$.fragment,m),Z=!0)},o(m){g(r.$$.fragment,m),g(T.$$.fragment,m),g(z.$$.fragment,m),Z=!1},d(m){m&&(t(a),t(e),t(w),t(W),t(U),t(R),t(k),t(_)),y(r,m),y(T,m),y(z,m)}}}function Sa(J){let a,p;return a=new ma({props:{$$slots:{default:[Qa]},$$scope:{ctx:J}}}),{c(){h(a.$$.fragment)},l(e){M(a.$$.fragment,e)},m(e,r){j(a,e,r),p=!0},p(e,r){const w={};r&2&&(w.$$scope={dirty:r,ctx:e}),a.$set(w)},i(e){p||(f(a.$$.fragment,e),p=!0)},o(e){g(a.$$.fragment,e),p=!1},d(e){y(a,e)}}}function La(J){let a,p='Se você não estiver familiarizado com o fine-tuning de um modelo com o Keras, dê uma olhada no tutorial básico <a href="training#finetune-with-keras">aqui</a>!';return{c(){a=d("p"),a.innerHTML=p},l(e){a=u(e,"P",{"data-svelte-h":!0}),b(a)!=="svelte-1vtmrxe"&&(a.innerHTML=p)},m(e,r){l(e,a,r)},p:ks,d(e){e&&t(a)}}}function Pa(J){let a,p='Para executar o fine-tuning de um modelo no TensorFlow, comece convertendo seu conjunto de dados para o formato <code>tf.data.Dataset</code> com <a href="https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset" rel="nofollow"><code>to_tf_dataset</code></a>. Nessa execução você deverá especificar as entradas e rótulos (no parâmetro <code>columns</code>), se deseja embaralhar o conjunto de dados, o tamanho do batch e o data collator:',e,r,w,T,W,U,Y="Configure o otimizador e alguns hiperparâmetros de treinamento:",R,k,V,_,z="Carregue o DistilBERT com o <code>TFAutoModelForTokenClassification</code> junto com o número de rótulos esperados:",Z,m,x,C,Cs='Configure o modelo para treinamento com o método <a href="https://keras.io/api/models/model_training_apis/#compile-method" rel="nofollow"><code>compile</code></a>:',q,G,N,I,F='Chame o método <a href="https://keras.io/api/models/model_training_apis/#fit-method" rel="nofollow"><code>fit</code></a> para executar o fine-tuning do modelo:',H,X,L;return r=new E({props:{code:"dGZfdHJhaW5fc2V0JTIwJTNEJTIwdG9rZW5pemVkX3dudXQlNUIlMjJ0cmFpbiUyMiU1RC50b190Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMGNvbHVtbnMlM0QlNUIlMjJhdHRlbnRpb25fbWFzayUyMiUyQyUyMCUyMmlucHV0X2lkcyUyMiUyQyUyMCUyMmxhYmVscyUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwY29sbGF0ZV9mbiUzRGRhdGFfY29sbGF0b3IlMkMlMEEpJTBBJTBBdGZfdmFsaWRhdGlvbl9zZXQlMjAlM0QlMjB0b2tlbml6ZWRfd251dCU1QiUyMnZhbGlkYXRpb24lMjIlNUQudG9fdGZfZGF0YXNldCglMEElMjAlMjAlMjAlMjBjb2x1bW5zJTNEJTVCJTIyYXR0ZW50aW9uX21hc2slMjIlMkMlMjAlMjJpbnB1dF9pZHMlMjIlMkMlMjAlMjJsYWJlbHMlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBzaHVmZmxlJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBjb2xsYXRlX2ZuJTNEZGF0YV9jb2xsYXRvciUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),T=new ca({props:{$$slots:{default:[La]},$$scope:{ctx:J}}}),k=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEElMEFiYXRjaF9zaXplJTIwJTNEJTIwMTYlMEFudW1fdHJhaW5fZXBvY2hzJTIwJTNEJTIwMyUwQW51bV90cmFpbl9zdGVwcyUyMCUzRCUyMChsZW4odG9rZW5pemVkX3dudXQlNUIlMjJ0cmFpbiUyMiU1RCklMjAlMkYlMkYlMjBiYXRjaF9zaXplKSUyMColMjBudW1fdHJhaW5fZXBvY2hzJTBBb3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGUlMjAlM0QlMjBjcmVhdGVfb3B0aW1pemVyKCUwQSUyMCUyMCUyMCUyMGluaXRfbHIlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX3N0ZXBzJTNEbnVtX3RyYWluX3N0ZXBzJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5X3JhdGUlM0QwLjAxJTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, lr_schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_steps=num_train_steps,
<span class="hljs-meta">... </span>    weight_decay_rate=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>)`}}),m=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMiUyQyUyMG51bV9sYWJlbHMlM0QyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">2</span>)`}}),G=new E({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplcik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),X=new E({props:{code:"bW9kZWwuZml0KHglM0R0Zl90cmFpbl9zZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl92YWxpZGF0aW9uX3NldCUyQyUyMGVwb2NocyUzRDMp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>)'}}),{c(){a=d("p"),a.innerHTML=p,e=i(),h(r.$$.fragment),w=i(),h(T.$$.fragment),W=i(),U=d("p"),U.textContent=Y,R=i(),h(k.$$.fragment),V=i(),_=d("p"),_.innerHTML=z,Z=i(),h(m.$$.fragment),x=i(),C=d("p"),C.innerHTML=Cs,q=i(),h(G.$$.fragment),N=i(),I=d("p"),I.innerHTML=F,H=i(),h(X.$$.fragment)},l(o){a=u(o,"P",{"data-svelte-h":!0}),b(a)!=="svelte-1n0wtrp"&&(a.innerHTML=p),e=c(o),M(r.$$.fragment,o),w=c(o),M(T.$$.fragment,o),W=c(o),U=u(o,"P",{"data-svelte-h":!0}),b(U)!=="svelte-xdplem"&&(U.textContent=Y),R=c(o),M(k.$$.fragment,o),V=c(o),_=u(o,"P",{"data-svelte-h":!0}),b(_)!=="svelte-1cdpi18"&&(_.innerHTML=z),Z=c(o),M(m.$$.fragment,o),x=c(o),C=u(o,"P",{"data-svelte-h":!0}),b(C)!=="svelte-espbvd"&&(C.innerHTML=Cs),q=c(o),M(G.$$.fragment,o),N=c(o),I=u(o,"P",{"data-svelte-h":!0}),b(I)!=="svelte-t6x1cw"&&(I.innerHTML=F),H=c(o),M(X.$$.fragment,o)},m(o,$){l(o,a,$),l(o,e,$),j(r,o,$),l(o,w,$),j(T,o,$),l(o,W,$),l(o,U,$),l(o,R,$),j(k,o,$),l(o,V,$),l(o,_,$),l(o,Z,$),j(m,o,$),l(o,x,$),l(o,C,$),l(o,q,$),j(G,o,$),l(o,N,$),l(o,I,$),l(o,H,$),j(X,o,$),L=!0},p(o,$){const as={};$&2&&(as.$$scope={dirty:$,ctx:o}),T.$set(as)},i(o){L||(f(r.$$.fragment,o),f(T.$$.fragment,o),f(k.$$.fragment,o),f(m.$$.fragment,o),f(G.$$.fragment,o),f(X.$$.fragment,o),L=!0)},o(o){g(r.$$.fragment,o),g(T.$$.fragment,o),g(k.$$.fragment,o),g(m.$$.fragment,o),g(G.$$.fragment,o),g(X.$$.fragment,o),L=!1},d(o){o&&(t(a),t(e),t(w),t(W),t(U),t(R),t(V),t(_),t(Z),t(x),t(C),t(q),t(N),t(I),t(H)),y(r,o),y(T,o),y(k,o),y(m,o),y(G,o),y(X,o)}}}function Da(J){let a,p;return a=new ma({props:{$$slots:{default:[Pa]},$$scope:{ctx:J}}}),{c(){h(a.$$.fragment)},l(e){M(a.$$.fragment,e)},m(e,r){j(a,e,r),p=!0},p(e,r){const w={};r&2&&(w.$$scope={dirty:r,ctx:e}),a.$set(w)},i(e){p||(f(a.$$.fragment,e),p=!0)},o(e){g(a.$$.fragment,e),p=!1},d(e){y(a,e)}}}function Ka(J){let a,p='Para obter um exemplo mais aprofundado de como executar o fine-tuning de um modelo para classificação de tokens, dê uma olhada nesse <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb" rel="nofollow">notebook utilizando PyTorch</a> ou nesse <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb" rel="nofollow">notebook utilizando TensorFlow</a>.';return{c(){a=d("p"),a.innerHTML=p},l(e){a=u(e,"P",{"data-svelte-h":!0}),b(a)!=="svelte-1evwr8u"&&(a.innerHTML=p)},m(e,r){l(e,a,r)},p:ks,d(e){e&&t(a)}}}function Oa(J){let a,p,e,r,w,T,W,U,Y,R,k="Classificação de tokens",V,_,z,Z,m="A classificação de tokens atribui um rótulo a tokens individuais em uma frase. Uma das tarefas de classificação de tokens mais comuns é o Reconhecimento de Entidade Nomeada, também chamada de NER (sigla em inglês para Named Entity Recognition). O NER tenta encontrar um rótulo para cada entidade em uma frase, como uma pessoa, local ou organização.",x,C,Cs='Este guia mostrará como realizar o fine-tuning do <a href="https://huggingface.co/distilbert-base-uncased" rel="nofollow">DistilBERT</a> no conjunto de dados <a href="https://huggingface.co/datasets/wnut_17" rel="nofollow">WNUT 17</a> para detectar novas entidades.',q,G,N,I,F,H,X,L,o,$="Carregando o conjunto de dados WNUT 17",as,es,ha="Carregue o conjunto de dados WNUT 17 da biblioteca 🤗 Datasets:",Gs,ts,Es,ls,Ma="E dê uma olhada em um exemplo:",Ws,ns,zs,os,ja="Cada número em <code>ner_tags</code> representa uma entidade. Converta o número em um rótulo para obter mais informações:",Xs,ps,Bs,rs,fa="O <code>ner_tag</code> descreve uma entidade, como uma organização, local ou pessoa. A letra que prefixa cada <code>ner_tag</code> indica a posição do token da entidade:",Vs,is,ga="<li><code>B-</code> indica o início de uma entidade.</li> <li><code>I-</code> indica que um token está contido dentro da mesma entidade (por exemplo, o token <code>State</code> pode fazer parte de uma entidade como <code>Empire State Building</code>).</li> <li><code>0</code> indica que o token não corresponde a nenhuma entidade.</li>",Fs,Q,P,vs,cs,da,Us,ya="Pré-processamento",Hs,ms,Ys,ds,wa="Carregue o tokenizer do DistilBERT para processar os <code>tokens</code>:",qs,us,Ns,hs,ba="Como a entrada já foi dividida em palavras, defina <code>is_split_into_words=True</code> para tokenizar as palavras em subpalavras:",Qs,Ms,Ss,js,$a="Ao adicionar os tokens especiais <code>[CLS]</code> e <code>[SEP]</code> e a tokenização de subpalavras uma incompatibilidade é gerada entre a entrada e os rótulos. Uma única palavra correspondente a um único rótulo pode ser dividida em duas subpalavras. Você precisará realinhar os tokens e os rótulos da seguinte forma:",Ls,fs,Ta='<li>Mapeie todos os tokens para a palavra correspondente com o método <a href="https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#tokenizers.Encoding.word_ids" rel="nofollow"><code>word_ids</code></a>.</li> <li>Atribuindo o rótulo <code>-100</code> aos tokens especiais <code>[CLS]</code> e <code>[SEP]</code> para que a função de loss do PyTorch ignore eles.</li> <li>Rotular apenas o primeiro token de uma determinada palavra. Atribuindo <code>-100</code> a outros subtokens da mesma palavra.</li>',Ps,gs,Ja="Aqui está como você pode criar uma função para realinhar os tokens e rótulos e truncar sequências para não serem maiores que o comprimento máximo de entrada do DistilBERT:",Ds,ys,Ks,ws,xa='Use a função <a href="https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map" rel="nofollow"><code>map</code></a> do 🤗 Datasets para tokenizar e alinhar os rótulos em todo o conjunto de dados. Você pode acelerar a função <code>map</code> configurando <code>batched=True</code> para processar vários elementos do conjunto de dados de uma só vez:',Os,bs,sa,$s,Ua="Use o <code>DataCollatorForTokenClassification</code> para criar um batch de exemplos. Ele também <em>preencherá dinamicamente</em> seu texto e rótulos para o comprimento do elemento mais longo em seu batch, para que tenham um comprimento uniforme. Embora seja possível preencher seu texto na função <code>tokenizer</code> configurando <code>padding=True</code>, o preenchimento dinâmico é mais eficiente.",aa,D,ea,S,K,Zs,Ts,ua,_s,_a="Treinamento",ta,O,la,ss,na,Is,oa;return U=new ia({}),_=new Ra({props:{id:"wVHdVlPScxA"}}),G=new ca({props:{$$slots:{default:[Va]},$$scope:{ctx:J}}}),X=new ia({}),ts=new E({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBd251dCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ3bnV0XzE3JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`}}),ns=new E({props:{code:"d251dCU1QiUyMnRyYWluJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`}}),ps=new E({props:{code:"bGFiZWxfbGlzdCUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RC5mZWF0dXJlcyU1QmYlMjJuZXJfdGFncyUyMiU1RC5mZWF0dXJlLm5hbWVzJTBBbGFiZWxfbGlzdA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`}}),cs=new ia({}),ms=new Ra({props:{id:"iY2AZYdZAr0"}}),us=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),Ms=new E({props:{code:"dG9rZW5pemVkX2lucHV0JTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGUlNUIlMjJ0b2tlbnMlMjIlNUQlMkMlMjBpc19zcGxpdF9pbnRvX3dvcmRzJTNEVHJ1ZSklMEF0b2tlbnMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF9pZHNfdG9fdG9rZW5zKHRva2VuaXplZF9pbnB1dCU1QiUyMmlucHV0X2lkcyUyMiU1RCklMEF0b2tlbnM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),ys=new E({props:{code:"ZGVmJTIwdG9rZW5pemVfYW5kX2FsaWduX2xhYmVscyhleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydG9rZW5zJTIyJTVEJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMjBpc19zcGxpdF9pbnRvX3dvcmRzJTNEVHJ1ZSklMEElMEElMjAlMjAlMjAlMjBsYWJlbHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBpJTJDJTIwbGFiZWwlMjBpbiUyMGVudW1lcmF0ZShleGFtcGxlcyU1QmYlMjJuZXJfdGFncyUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkcyUyMCUzRCUyMHRva2VuaXplZF9pbnB1dHMud29yZF9pZHMoYmF0Y2hfaW5kZXglM0RpKSUyMCUyMCUyMyUyME1hcCUyMHRva2VucyUyMHRvJTIwdGhlaXIlMjByZXNwZWN0aXZlJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyME5vbmUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjB3b3JkX2lkeCUyMGluJTIwd29yZF9pZHMlM0ElMjAlMjAlMjMlMjBTZXQlMjB0aGUlMjBzcGVjaWFsJTIwdG9rZW5zJTIwdG8lMjAtMTAwLiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZHglMjBpcyUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKC0xMDApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxpZiUyMHdvcmRfaWR4JTIwISUzRCUyMHByZXZpb3VzX3dvcmRfaWR4JTNBJTIwJTIwJTIzJTIwT25seSUyMGxhYmVsJTIwdGhlJTIwZmlyc3QlMjB0b2tlbiUyMG9mJTIwYSUyMGdpdmVuJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKGxhYmVsJTVCd29yZF9pZHglNUQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVsX2lkcy5hcHBlbmQoLTEwMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyMHdvcmRfaWR4JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFiZWxzLmFwcGVuZChsYWJlbF9pZHMpJTBBJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVkX2lucHV0cyU1QiUyMmxhYmVscyUyMiU1RCUyMCUzRCUyMGxhYmVscyUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHRva2VuaXplZF9pbnB1dHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
<span class="hljs-meta">... </span>        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
<span class="hljs-meta">... </span>        previous_word_idx = <span class="hljs-literal">None</span>
<span class="hljs-meta">... </span>        label_ids = []
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
<span class="hljs-meta">... </span>                label_ids.append(label[word_idx])
<span class="hljs-meta">... </span>            <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            previous_word_idx = word_idx
<span class="hljs-meta">... </span>        labels.append(label_ids)

<span class="hljs-meta">... </span>    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenized_inputs`}}),bs=new E({props:{code:"dG9rZW5pemVkX3dudXQlMjAlM0QlMjB3bnV0Lm1hcCh0b2tlbml6ZV9hbmRfYWxpZ25fbGFiZWxzJTJDJTIwYmF0Y2hlZCUzRFRydWUp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)'}}),D=new Ga({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[qa],pytorch:[Ha]},$$scope:{ctx:J}}}),Ts=new ia({}),O=new Ga({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Da],pytorch:[Sa]},$$scope:{ctx:J}}}),ss=new ca({props:{$$slots:{default:[Ka]},$$scope:{ctx:J}}}),{c(){a=d("meta"),p=i(),e=d("p"),r=i(),w=d("h1"),T=d("a"),W=d("span"),h(U.$$.fragment),Y=i(),R=d("span"),R.textContent=k,V=i(),h(_.$$.fragment),z=i(),Z=d("p"),Z.textContent=m,x=i(),C=d("p"),C.innerHTML=Cs,q=i(),h(G.$$.fragment),N=i(),I=d("h2"),F=d("a"),H=d("span"),h(X.$$.fragment),L=i(),o=d("span"),o.textContent=$,as=i(),es=d("p"),es.textContent=ha,Gs=i(),h(ts.$$.fragment),Es=i(),ls=d("p"),ls.textContent=Ma,Ws=i(),h(ns.$$.fragment),zs=i(),os=d("p"),os.innerHTML=ja,Xs=i(),h(ps.$$.fragment),Bs=i(),rs=d("p"),rs.innerHTML=fa,Vs=i(),is=d("ul"),is.innerHTML=ga,Fs=i(),Q=d("h2"),P=d("a"),vs=d("span"),h(cs.$$.fragment),da=i(),Us=d("span"),Us.textContent=ya,Hs=i(),h(ms.$$.fragment),Ys=i(),ds=d("p"),ds.innerHTML=wa,qs=i(),h(us.$$.fragment),Ns=i(),hs=d("p"),hs.innerHTML=ba,Qs=i(),h(Ms.$$.fragment),Ss=i(),js=d("p"),js.innerHTML=$a,Ls=i(),fs=d("ol"),fs.innerHTML=Ta,Ps=i(),gs=d("p"),gs.textContent=Ja,Ds=i(),h(ys.$$.fragment),Ks=i(),ws=d("p"),ws.innerHTML=xa,Os=i(),h(bs.$$.fragment),sa=i(),$s=d("p"),$s.innerHTML=Ua,aa=i(),h(D.$$.fragment),ea=i(),S=d("h2"),K=d("a"),Zs=d("span"),h(Ts.$$.fragment),ua=i(),_s=d("span"),_s.textContent=_a,ta=i(),h(O.$$.fragment),la=i(),h(ss.$$.fragment),na=i(),Is=d("p"),this.h()},l(s){const n=Ba("svelte-1phssyn",document.head);a=u(n,"META",{name:!0,content:!0}),n.forEach(t),p=c(s),e=u(s,"P",{}),B(e).forEach(t),r=c(s),w=u(s,"H1",{class:!0});var Js=B(w);T=u(Js,"A",{id:!0,class:!0,href:!0});var As=B(T);W=u(As,"SPAN",{});var Rs=B(W);M(U.$$.fragment,Rs),Rs.forEach(t),As.forEach(t),Y=c(Js),R=u(Js,"SPAN",{"data-svelte-h":!0}),b(R)!=="svelte-167lb59"&&(R.textContent=k),Js.forEach(t),V=c(s),M(_.$$.fragment,s),z=c(s),Z=u(s,"P",{"data-svelte-h":!0}),b(Z)!=="svelte-2rm2e9"&&(Z.textContent=m),x=c(s),C=u(s,"P",{"data-svelte-h":!0}),b(C)!=="svelte-173m660"&&(C.innerHTML=Cs),q=c(s),M(G.$$.fragment,s),N=c(s),I=u(s,"H2",{class:!0});var xs=B(I);F=u(xs,"A",{id:!0,class:!0,href:!0});var ka=B(F);H=u(ka,"SPAN",{});var Ca=B(H);M(X.$$.fragment,Ca),Ca.forEach(t),ka.forEach(t),L=c(xs),o=u(xs,"SPAN",{"data-svelte-h":!0}),b(o)!=="svelte-scs1ho"&&(o.textContent=$),xs.forEach(t),as=c(s),es=u(s,"P",{"data-svelte-h":!0}),b(es)!=="svelte-3wpxsw"&&(es.textContent=ha),Gs=c(s),M(ts.$$.fragment,s),Es=c(s),ls=u(s,"P",{"data-svelte-h":!0}),b(ls)!=="svelte-1vbpr8z"&&(ls.textContent=Ma),Ws=c(s),M(ns.$$.fragment,s),zs=c(s),os=u(s,"P",{"data-svelte-h":!0}),b(os)!=="svelte-1cz4103"&&(os.innerHTML=ja),Xs=c(s),M(ps.$$.fragment,s),Bs=c(s),rs=u(s,"P",{"data-svelte-h":!0}),b(rs)!=="svelte-cqfv4y"&&(rs.innerHTML=fa),Vs=c(s),is=u(s,"UL",{"data-svelte-h":!0}),b(is)!=="svelte-13k6qsm"&&(is.innerHTML=ga),Fs=c(s),Q=u(s,"H2",{class:!0});var pa=B(Q);P=u(pa,"A",{id:!0,class:!0,href:!0});var va=B(P);vs=u(va,"SPAN",{});var Za=B(vs);M(cs.$$.fragment,Za),Za.forEach(t),va.forEach(t),da=c(pa),Us=u(pa,"SPAN",{"data-svelte-h":!0}),b(Us)!=="svelte-f29ifu"&&(Us.textContent=ya),pa.forEach(t),Hs=c(s),M(ms.$$.fragment,s),Ys=c(s),ds=u(s,"P",{"data-svelte-h":!0}),b(ds)!=="svelte-1e5ex1w"&&(ds.innerHTML=wa),qs=c(s),M(us.$$.fragment,s),Ns=c(s),hs=u(s,"P",{"data-svelte-h":!0}),b(hs)!=="svelte-g8rrel"&&(hs.innerHTML=ba),Qs=c(s),M(Ms.$$.fragment,s),Ss=c(s),js=u(s,"P",{"data-svelte-h":!0}),b(js)!=="svelte-17o23dm"&&(js.innerHTML=$a),Ls=c(s),fs=u(s,"OL",{"data-svelte-h":!0}),b(fs)!=="svelte-9yvtqn"&&(fs.innerHTML=Ta),Ps=c(s),gs=u(s,"P",{"data-svelte-h":!0}),b(gs)!=="svelte-576loj"&&(gs.textContent=Ja),Ds=c(s),M(ys.$$.fragment,s),Ks=c(s),ws=u(s,"P",{"data-svelte-h":!0}),b(ws)!=="svelte-3dfyhm"&&(ws.innerHTML=xa),Os=c(s),M(bs.$$.fragment,s),sa=c(s),$s=u(s,"P",{"data-svelte-h":!0}),b($s)!=="svelte-yw3ehb"&&($s.innerHTML=Ua),aa=c(s),M(D.$$.fragment,s),ea=c(s),S=u(s,"H2",{class:!0});var ra=B(S);K=u(ra,"A",{id:!0,class:!0,href:!0});var Ia=B(K);Zs=u(Ia,"SPAN",{});var Aa=B(Zs);M(Ts.$$.fragment,Aa),Aa.forEach(t),Ia.forEach(t),ua=c(ra),_s=u(ra,"SPAN",{"data-svelte-h":!0}),b(_s)!=="svelte-1rp0x95"&&(_s.textContent=_a),ra.forEach(t),ta=c(s),M(O.$$.fragment,s),la=c(s),M(ss.$$.fragment,s),na=c(s),Is=u(s,"P",{}),B(Is).forEach(t),this.h()},h(){v(a,"name","hf:doc:metadata"),v(a,"content",JSON.stringify(se)),v(T,"id","classificao-de-tokens"),v(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),v(T,"href","#classificao-de-tokens"),v(w,"class","relative group"),v(F,"id","carregando-o-conjunto-de-dados-wnut-17"),v(F,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),v(F,"href","#carregando-o-conjunto-de-dados-wnut-17"),v(I,"class","relative group"),v(P,"id","prprocessamento"),v(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),v(P,"href","#prprocessamento"),v(Q,"class","relative group"),v(K,"id","treinamento"),v(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),v(K,"href","#treinamento"),v(S,"class","relative group")},m(s,n){A(document.head,a),l(s,p,n),l(s,e,n),l(s,r,n),l(s,w,n),A(w,T),A(T,W),j(U,W,null),A(w,Y),A(w,R),l(s,V,n),j(_,s,n),l(s,z,n),l(s,Z,n),l(s,x,n),l(s,C,n),l(s,q,n),j(G,s,n),l(s,N,n),l(s,I,n),A(I,F),A(F,H),j(X,H,null),A(I,L),A(I,o),l(s,as,n),l(s,es,n),l(s,Gs,n),j(ts,s,n),l(s,Es,n),l(s,ls,n),l(s,Ws,n),j(ns,s,n),l(s,zs,n),l(s,os,n),l(s,Xs,n),j(ps,s,n),l(s,Bs,n),l(s,rs,n),l(s,Vs,n),l(s,is,n),l(s,Fs,n),l(s,Q,n),A(Q,P),A(P,vs),j(cs,vs,null),A(Q,da),A(Q,Us),l(s,Hs,n),j(ms,s,n),l(s,Ys,n),l(s,ds,n),l(s,qs,n),j(us,s,n),l(s,Ns,n),l(s,hs,n),l(s,Qs,n),j(Ms,s,n),l(s,Ss,n),l(s,js,n),l(s,Ls,n),l(s,fs,n),l(s,Ps,n),l(s,gs,n),l(s,Ds,n),j(ys,s,n),l(s,Ks,n),l(s,ws,n),l(s,Os,n),j(bs,s,n),l(s,sa,n),l(s,$s,n),l(s,aa,n),j(D,s,n),l(s,ea,n),l(s,S,n),A(S,K),A(K,Zs),j(Ts,Zs,null),A(S,ua),A(S,_s),l(s,ta,n),j(O,s,n),l(s,la,n),j(ss,s,n),l(s,na,n),l(s,Is,n),oa=!0},p(s,[n]){const Js={};n&2&&(Js.$$scope={dirty:n,ctx:s}),G.$set(Js);const As={};n&2&&(As.$$scope={dirty:n,ctx:s}),D.$set(As);const Rs={};n&2&&(Rs.$$scope={dirty:n,ctx:s}),O.$set(Rs);const xs={};n&2&&(xs.$$scope={dirty:n,ctx:s}),ss.$set(xs)},i(s){oa||(f(U.$$.fragment,s),f(_.$$.fragment,s),f(G.$$.fragment,s),f(X.$$.fragment,s),f(ts.$$.fragment,s),f(ns.$$.fragment,s),f(ps.$$.fragment,s),f(cs.$$.fragment,s),f(ms.$$.fragment,s),f(us.$$.fragment,s),f(Ms.$$.fragment,s),f(ys.$$.fragment,s),f(bs.$$.fragment,s),f(D.$$.fragment,s),f(Ts.$$.fragment,s),f(O.$$.fragment,s),f(ss.$$.fragment,s),oa=!0)},o(s){g(U.$$.fragment,s),g(_.$$.fragment,s),g(G.$$.fragment,s),g(X.$$.fragment,s),g(ts.$$.fragment,s),g(ns.$$.fragment,s),g(ps.$$.fragment,s),g(cs.$$.fragment,s),g(ms.$$.fragment,s),g(us.$$.fragment,s),g(Ms.$$.fragment,s),g(ys.$$.fragment,s),g(bs.$$.fragment,s),g(D.$$.fragment,s),g(Ts.$$.fragment,s),g(O.$$.fragment,s),g(ss.$$.fragment,s),oa=!1},d(s){s&&(t(p),t(e),t(r),t(w),t(V),t(z),t(Z),t(x),t(C),t(q),t(N),t(I),t(as),t(es),t(Gs),t(Es),t(ls),t(Ws),t(zs),t(os),t(Xs),t(Bs),t(rs),t(Vs),t(is),t(Fs),t(Q),t(Hs),t(Ys),t(ds),t(qs),t(Ns),t(hs),t(Qs),t(Ss),t(js),t(Ls),t(fs),t(Ps),t(gs),t(Ds),t(Ks),t(ws),t(Os),t(sa),t($s),t(aa),t(ea),t(S),t(ta),t(la),t(na),t(Is)),t(a),y(U),y(_,s),y(G,s),y(X),y(ts,s),y(ns,s),y(ps,s),y(cs),y(ms,s),y(us,s),y(Ms,s),y(ys,s),y(bs,s),y(D,s),y(Ts),y(O,s),y(ss,s)}}}const se={local:"classificao-de-tokens",sections:[{local:"carregando-o-conjunto-de-dados-wnut-17",title:"Carregando o conjunto de dados WNUT 17"},{local:"prprocessamento",title:"Pré-processamento"},{local:"treinamento",title:"Treinamento"}],title:"Classificação de tokens"};function ae(J){return Wa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ie extends za{constructor(a){super(),Xa(this,a,ae,Oa,Ea,{})}}export{ie as component};
