import{s as tt,o as et,n as ws}from"../chunks/scheduler.889ebb13.js";import{S as nt,i as pt,g as h,s as r,u,A as rt,h as M,f as l,c as o,j as V,v as d,y as w,k as R,o as E,a,w as y,d as f,t as b,x as g,m as lt,n as at}from"../chunks/index.8980e686.js";import{T as da}from"../chunks/Tip.7191df11.js";import{Y as st}from"../chunks/Youtube.532e53e0.js";import{I as ml}from"../chunks/IconCopyLink.357761cd.js";import{C as k}from"../chunks/CodeBlock.25fc3430.js";import{D as ot}from"../chunks/DocNotebookDropdown.f8a50280.js";import{F as wa,M as hl}from"../chunks/Markdown.31057263.js";function it(C){let e,c,t='<a href="../model_doc/albert">ALBERT</a>, <a href="../model_doc/bert">BERT</a>, <a href="../model_doc/big_bird">BigBird</a>, <a href="../model_doc/biogpt">BioGpt</a>, <a href="../model_doc/bloom">BLOOM</a>, <a href="../model_doc/camembert">CamemBERT</a>, <a href="../model_doc/canine">CANINE</a>, <a href="../model_doc/convbert">ConvBERT</a>, <a href="../model_doc/data2vec-text">Data2VecText</a>, <a href="../model_doc/deberta">DeBERTa</a>, <a href="../model_doc/deberta-v2">DeBERTa-v2</a>, <a href="../model_doc/distilbert">DistilBERT</a>, <a href="../model_doc/electra">ELECTRA</a>, <a href="../model_doc/ernie">ERNIE</a>, <a href="../model_doc/ernie_m">ErnieM</a>, <a href="../model_doc/esm">ESM</a>, <a href="../model_doc/flaubert">FlauBERT</a>, <a href="../model_doc/fnet">FNet</a>, <a href="../model_doc/funnel">Funnel Transformer</a>, <a href="../model_doc/gpt-sw3">GPT-Sw3</a>, <a href="../model_doc/gpt2">OpenAI GPT-2</a>, <a href="../model_doc/gpt_bigcode">GPTBigCode</a>, <a href="../model_doc/ibert">I-BERT</a>, <a href="../model_doc/layoutlm">LayoutLM</a>, <a href="../model_doc/layoutlmv2">LayoutLMv2</a>, <a href="../model_doc/layoutlmv3">LayoutLMv3</a>, <a href="../model_doc/lilt">LiLT</a>, <a href="../model_doc/longformer">Longformer</a>, <a href="../model_doc/luke">LUKE</a>, <a href="../model_doc/markuplm">MarkupLM</a>, <a href="../model_doc/mega">MEGA</a>, <a href="../model_doc/megatron-bert">Megatron-BERT</a>, <a href="../model_doc/mobilebert">MobileBERT</a>, <a href="../model_doc/mpnet">MPNet</a>, <a href="../model_doc/nezha">Nezha</a>, <a href="../model_doc/nystromformer">Nyströmformer</a>, <a href="../model_doc/qdqbert">QDQBert</a>, <a href="../model_doc/rembert">RemBERT</a>, <a href="../model_doc/roberta">RoBERTa</a>, <a href="../model_doc/roberta-prelayernorm">RoBERTa-PreLayerNorm</a>, <a href="../model_doc/roc_bert">RoCBert</a>, <a href="../model_doc/roformer">RoFormer</a>, <a href="../model_doc/squeezebert">SqueezeBERT</a>, <a href="../model_doc/xlm">XLM</a>, <a href="../model_doc/xlm-roberta">XLM-RoBERTa</a>, <a href="../model_doc/xlm-roberta-xl">XLM-RoBERTa-XL</a>, <a href="../model_doc/xlnet">XLNet</a>, <a href="../model_doc/xmod">X-MOD</a>, <a href="../model_doc/yoso">YOSO</a>';return{c(){e=lt(`이 튜토리얼에서 설명하는 작업은 다음 모델 아키텍처에 의해 지원됩니다:

`),c=h("p"),c.innerHTML=t},l(m){e=at(m,`이 튜토리얼에서 설명하는 작업은 다음 모델 아키텍처에 의해 지원됩니다:

`),c=M(m,"P",{"data-svelte-h":!0}),w(c)!=="svelte-17zgief"&&(c.innerHTML=t)},m(m,J){a(m,e,J),a(m,c,J)},p:ws,d(m){m&&(l(e),l(c))}}}function ct(C){let e,c;return e=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`}}),{c(){u(e.$$.fragment)},l(t){d(e.$$.fragment,t)},m(t,m){y(e,t,m),c=!0},p:ws,i(t){c||(f(e.$$.fragment,t),c=!0)},o(t){b(e.$$.fragment,t),c=!1},d(t){g(e,t)}}}function mt(C){let e,c;return e=new hl({props:{$$slots:{default:[ct]},$$scope:{ctx:C}}}),{c(){u(e.$$.fragment)},l(t){d(e.$$.fragment,t)},m(t,m){y(e,t,m),c=!0},p(t,m){const J={};m&2&&(J.$$scope={dirty:m,ctx:t}),e.$set(J)},i(t){c||(f(e.$$.fragment,t),c=!0)},o(t){b(e.$$.fragment,t),c=!1},d(t){g(e,t)}}}function ht(C){let e,c;return e=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){u(e.$$.fragment)},l(t){d(e.$$.fragment,t)},m(t,m){y(e,t,m),c=!0},p:ws,i(t){c||(f(e.$$.fragment,t),c=!0)},o(t){b(e.$$.fragment,t),c=!1},d(t){g(e,t)}}}function Mt(C){let e,c;return e=new hl({props:{$$slots:{default:[ht]},$$scope:{ctx:C}}}),{c(){u(e.$$.fragment)},l(t){d(e.$$.fragment,t)},m(t,m){y(e,t,m),c=!0},p(t,m){const J={};m&2&&(J.$$scope={dirty:m,ctx:t}),e.$set(J)},i(t){c||(f(e.$$.fragment,t),c=!0)},o(t){b(e.$$.fragment,t),c=!1},d(t){g(e,t)}}}function jt(C){let e,c='<code>Trainer</code>를 사용하여 모델을 파인 튜닝하는 방법에 익숙하지 않은 경우, <a href="../training#train-with-pytorch-trainer">여기</a>에서 기본 튜토리얼을 확인하세요!';return{c(){e=h("p"),e.innerHTML=c},l(t){e=M(t,"P",{"data-svelte-h":!0}),w(e)!=="svelte-j16k5v"&&(e.innerHTML=c)},m(t,m){a(t,e,m)},p:ws,d(t){t&&l(e)}}}function ut(C){let e,c,t,m="이제 모델을 훈련시킬 준비가 되었습니다! <code>AutoModelForSequenceClassification</code>로 DistilBERT를 가져오고 예상되는 레이블 수와 레이블 매핑을 지정하세요:",J,$,W,x,G="이제 세 단계만 거치면 끝입니다:",Z,B,N="<li><code>TrainingArguments</code>에서 하이퍼파라미터를 정의하세요. <code>output_dir</code>는 모델을 저장할 위치를 지정하는 유일한 매개변수입니다. 이 모델을 허브에 업로드하기 위해 <code>push_to_hub=True</code>를 설정합니다(모델을 업로드하기 위해 Hugging Face에 로그인해야합니다.) 각 에폭이 끝날 때마다, <code>Trainer</code>는 seqeval 점수를 평가하고 훈련 체크포인트를 저장합니다.</li> <li><code>Trainer</code>에 훈련 인수와 모델, 데이터 세트, 토크나이저, 데이터 콜레이터 및 <code>compute_metrics</code> 함수를 전달하세요.</li> <li><code>train()</code>를 호출하여 모델을 파인 튜닝하세요.</li>",v,_,I,i,U="훈련이 완료되면, <code>push_to_hub()</code> 메소드를 사용하여 모델을 허브에 공유할 수 있습니다.",q,F,X;return e=new da({props:{$$slots:{default:[jt]},$$scope:{ctx:C}}}),$=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMkMlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMFRyYWluZXIlMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkJTIyJTJDJTIwbnVtX2xhYmVscyUzRDEzJTJDJTIwaWQybGFiZWwlM0RpZDJsYWJlbCUyQyUyMGxhYmVsMmlkJTNEbGFiZWwyaWQlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">13</span>, id2label=id2label, label2id=label2id
<span class="hljs-meta">... </span>)`}}),_=new k({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJteV9hd2Vzb21lX3dudXRfbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMmUtNSUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QyJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5JTNEMC4wMSUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMGxvYWRfYmVzdF9tb2RlbF9hdF9lbmQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0R0b2tlbml6ZWRfd251dCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX3dudXQlNUIlMjJ0ZXN0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVyJTNEdG9rZW5pemVyJTJDJTBBJTIwJTIwJTIwJTIwZGF0YV9jb2xsYXRvciUzRGRhdGFfY29sbGF0b3IlMkMlMEElMjAlMjAlMjAlMjBjb21wdXRlX21ldHJpY3MlM0Rjb21wdXRlX21ldHJpY3MlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_wnut_model&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    save_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    load_best_model_at_end=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>    compute_metrics=compute_metrics,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),F=new k({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()'}}),{c(){u(e.$$.fragment),c=r(),t=h("p"),t.innerHTML=m,J=r(),u($.$$.fragment),W=r(),x=h("p"),x.textContent=G,Z=r(),B=h("ol"),B.innerHTML=N,v=r(),u(_.$$.fragment),I=r(),i=h("p"),i.innerHTML=U,q=r(),u(F.$$.fragment)},l(j){d(e.$$.fragment,j),c=o(j),t=M(j,"P",{"data-svelte-h":!0}),w(t)!=="svelte-155uoos"&&(t.innerHTML=m),J=o(j),d($.$$.fragment,j),W=o(j),x=M(j,"P",{"data-svelte-h":!0}),w(x)!=="svelte-14zzcxs"&&(x.textContent=G),Z=o(j),B=M(j,"OL",{"data-svelte-h":!0}),w(B)!=="svelte-yqkkyp"&&(B.innerHTML=N),v=o(j),d(_.$$.fragment,j),I=o(j),i=M(j,"P",{"data-svelte-h":!0}),w(i)!=="svelte-ydcgmn"&&(i.innerHTML=U),q=o(j),d(F.$$.fragment,j)},m(j,A){y(e,j,A),a(j,c,A),a(j,t,A),a(j,J,A),y($,j,A),a(j,W,A),a(j,x,A),a(j,Z,A),a(j,B,A),a(j,v,A),y(_,j,A),a(j,I,A),a(j,i,A),a(j,q,A),y(F,j,A),X=!0},p(j,A){const H={};A&2&&(H.$$scope={dirty:A,ctx:j}),e.$set(H)},i(j){X||(f(e.$$.fragment,j),f($.$$.fragment,j),f(_.$$.fragment,j),f(F.$$.fragment,j),X=!0)},o(j){b(e.$$.fragment,j),b($.$$.fragment,j),b(_.$$.fragment,j),b(F.$$.fragment,j),X=!1},d(j){j&&(l(c),l(t),l(J),l(W),l(x),l(Z),l(B),l(v),l(I),l(i),l(q)),g(e,j),g($,j),g(_,j),g(F,j)}}}function dt(C){let e,c;return e=new hl({props:{$$slots:{default:[ut]},$$scope:{ctx:C}}}),{c(){u(e.$$.fragment)},l(t){d(e.$$.fragment,t)},m(t,m){y(e,t,m),c=!0},p(t,m){const J={};m&2&&(J.$$scope={dirty:m,ctx:t}),e.$set(J)},i(t){c||(f(e.$$.fragment,t),c=!0)},o(t){b(e.$$.fragment,t),c=!1},d(t){g(e,t)}}}function yt(C){let e,c='Keras를 사용하여 모델을 파인 튜닝하는 방법에 익숙하지 않은 경우, <a href="../training#train-a-tensorflow-model-with-keras">여기</a>의 기본 튜토리얼을 확인하세요!';return{c(){e=h("p"),e.innerHTML=c},l(t){e=M(t,"P",{"data-svelte-h":!0}),w(e)!=="svelte-nkj0lu"&&(e.innerHTML=c)},m(t,m){a(t,e,m)},p:ws,d(t){t&&l(e)}}}function ft(C){let e,c,t,m,J,$="그런 다음 <code>TFAutoModelForSequenceClassification</code>을 사용하여 DistilBERT를 가져오고, 예상되는 레이블 수와 레이블 매핑을 지정합니다:",W,x,G,Z,B="<code>prepare_tf_dataset()</code>을 사용하여 데이터 세트를 <code>tf.data.Dataset</code> 형식으로 변환합니다:",N,v,_,I,i='<a href="https://keras.io/api/models/model_training_apis/#compile-method" rel="nofollow"><code>compile</code></a>를 사용하여 훈련할 모델을 구성합니다:',U,q,F,X,j='훈련을 시작하기 전에 설정해야할 마지막 두 가지는 예측에서 seqeval 점수를 계산하고, 모델을 허브에 업로드할 방법을 제공하는 것입니다. 모두 <a href="../main_classes/keras_callbacks">Keras callbacks</a>를 사용하여 수행됩니다.',A,H,Ml="<code>KerasMetricCallback</code>에 <code>compute_metrics</code> 함수를 전달하세요:",ls,Q,as,Y,jl="<code>PushToHubCallback</code>에서 모델과 토크나이저를 업로드할 위치를 지정합니다:",ts,S,es,L,ul="그런 다음 콜백을 함께 묶습니다:",ns,D,ps,z,O='드디어, 모델 훈련을 시작할 준비가 되었습니다! <a href="https://keras.io/api/models/model_training_apis/#fit-method" rel="nofollow"><code>fit</code></a>에 훈련 데이터 세트, 검증 데이터 세트, 에폭의 수 및 콜백을 전달하여 파인 튜닝합니다:',ss,P,hs,K,dl="훈련이 완료되면, 모델이 자동으로 허브에 업로드되어 누구나 사용할 수 있습니다!",rs;return e=new da({props:{$$slots:{default:[yt]},$$scope:{ctx:C}}}),t=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEElMEFiYXRjaF9zaXplJTIwJTNEJTIwMTYlMEFudW1fdHJhaW5fZXBvY2hzJTIwJTNEJTIwMyUwQW51bV90cmFpbl9zdGVwcyUyMCUzRCUyMChsZW4odG9rZW5pemVkX3dudXQlNUIlMjJ0cmFpbiUyMiU1RCklMjAlMkYlMkYlMjBiYXRjaF9zaXplKSUyMColMjBudW1fdHJhaW5fZXBvY2hzJTBBb3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGUlMjAlM0QlMjBjcmVhdGVfb3B0aW1pemVyKCUwQSUyMCUyMCUyMCUyMGluaXRfbHIlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX3N0ZXBzJTNEbnVtX3RyYWluX3N0ZXBzJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5X3JhdGUlM0QwLjAxJTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, lr_schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_steps=num_train_steps,
<span class="hljs-meta">... </span>    weight_decay_rate=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>)`}}),x=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMiUyQyUyMG51bV9sYWJlbHMlM0QxMyUyQyUyMGlkMmxhYmVsJTNEaWQybGFiZWwlMkMlMjBsYWJlbDJpZCUzRGxhYmVsMmlkJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">13</span>, id2label=id2label, label2id=label2id
<span class="hljs-meta">... </span>)`}}),v=new k({props:{code:"dGZfdHJhaW5fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF93bnV0JTVCJTIydHJhaW4lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBzaHVmZmxlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBKSUwQSUwQXRmX3ZhbGlkYXRpb25fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF93bnV0JTVCJTIydmFsaWRhdGlvbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),q=new k({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplcik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),Q=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBLZXJhc01ldHJpY0NhbGxiYWNrJTBBJTBBbWV0cmljX2NhbGxiYWNrJTIwJTNEJTIwS2VyYXNNZXRyaWNDYWxsYmFjayhtZXRyaWNfZm4lM0Rjb21wdXRlX21ldHJpY3MlMkMlMjBldmFsX2RhdGFzZXQlM0R0Zl92YWxpZGF0aW9uX3NldCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> KerasMetricCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_validation_set)`}}),S=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBQdXNoVG9IdWJDYWxsYmFjayUwQSUwQXB1c2hfdG9faHViX2NhbGxiYWNrJTIwJTNEJTIwUHVzaFRvSHViQ2FsbGJhY2soJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>push_to_hub_callback = PushToHubCallback(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_wnut_model&quot;</span>,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>)`}}),D=new k({props:{code:"Y2FsbGJhY2tzJTIwJTNEJTIwJTVCbWV0cmljX2NhbGxiYWNrJTJDJTIwcHVzaF90b19odWJfY2FsbGJhY2slNUQ=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>callbacks = [metric_callback, push_to_hub_callback]'}}),P=new k({props:{code:"bW9kZWwuZml0KHglM0R0Zl90cmFpbl9zZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl92YWxpZGF0aW9uX3NldCUyQyUyMGVwb2NocyUzRDMlMkMlMjBjYWxsYmFja3MlM0RjYWxsYmFja3Mp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>, callbacks=callbacks)'}}),{c(){u(e.$$.fragment),c=lt(`
TensorFlow에서 모델을 파인 튜닝하려면, 먼저 옵티마이저 함수와 학습률 스케쥴, 그리고 일부 훈련 하이퍼파라미터를 설정해야 합니다:

	`),u(t.$$.fragment),m=r(),J=h("p"),J.innerHTML=$,W=r(),u(x.$$.fragment),G=r(),Z=h("p"),Z.innerHTML=B,N=r(),u(v.$$.fragment),_=r(),I=h("p"),I.innerHTML=i,U=r(),u(q.$$.fragment),F=r(),X=h("p"),X.innerHTML=j,A=r(),H=h("p"),H.innerHTML=Ml,ls=r(),u(Q.$$.fragment),as=r(),Y=h("p"),Y.innerHTML=jl,ts=r(),u(S.$$.fragment),es=r(),L=h("p"),L.textContent=ul,ns=r(),u(D.$$.fragment),ps=r(),z=h("p"),z.innerHTML=O,ss=r(),u(P.$$.fragment),hs=r(),K=h("p"),K.textContent=dl},l(p){d(e.$$.fragment,p),c=at(p,`
TensorFlow에서 모델을 파인 튜닝하려면, 먼저 옵티마이저 함수와 학습률 스케쥴, 그리고 일부 훈련 하이퍼파라미터를 설정해야 합니다:

	`),d(t.$$.fragment,p),m=o(p),J=M(p,"P",{"data-svelte-h":!0}),w(J)!=="svelte-1opo5t0"&&(J.innerHTML=$),W=o(p),d(x.$$.fragment,p),G=o(p),Z=M(p,"P",{"data-svelte-h":!0}),w(Z)!=="svelte-txv5rs"&&(Z.innerHTML=B),N=o(p),d(v.$$.fragment,p),_=o(p),I=M(p,"P",{"data-svelte-h":!0}),w(I)!=="svelte-qrlpiv"&&(I.innerHTML=i),U=o(p),d(q.$$.fragment,p),F=o(p),X=M(p,"P",{"data-svelte-h":!0}),w(X)!=="svelte-10vlwlw"&&(X.innerHTML=j),A=o(p),H=M(p,"P",{"data-svelte-h":!0}),w(H)!=="svelte-1fwgit4"&&(H.innerHTML=Ml),ls=o(p),d(Q.$$.fragment,p),as=o(p),Y=M(p,"P",{"data-svelte-h":!0}),w(Y)!=="svelte-1ilfuc9"&&(Y.innerHTML=jl),ts=o(p),d(S.$$.fragment,p),es=o(p),L=M(p,"P",{"data-svelte-h":!0}),w(L)!=="svelte-90s2we"&&(L.textContent=ul),ns=o(p),d(D.$$.fragment,p),ps=o(p),z=M(p,"P",{"data-svelte-h":!0}),w(z)!=="svelte-1kbr19n"&&(z.innerHTML=O),ss=o(p),d(P.$$.fragment,p),hs=o(p),K=M(p,"P",{"data-svelte-h":!0}),w(K)!=="svelte-w14up1"&&(K.textContent=dl)},m(p,T){y(e,p,T),a(p,c,T),y(t,p,T),a(p,m,T),a(p,J,T),a(p,W,T),y(x,p,T),a(p,G,T),a(p,Z,T),a(p,N,T),y(v,p,T),a(p,_,T),a(p,I,T),a(p,U,T),y(q,p,T),a(p,F,T),a(p,X,T),a(p,A,T),a(p,H,T),a(p,ls,T),y(Q,p,T),a(p,as,T),a(p,Y,T),a(p,ts,T),y(S,p,T),a(p,es,T),a(p,L,T),a(p,ns,T),y(D,p,T),a(p,ps,T),a(p,z,T),a(p,ss,T),y(P,p,T),a(p,hs,T),a(p,K,T),rs=!0},p(p,T){const Js={};T&2&&(Js.$$scope={dirty:T,ctx:p}),e.$set(Js)},i(p){rs||(f(e.$$.fragment,p),f(t.$$.fragment,p),f(x.$$.fragment,p),f(v.$$.fragment,p),f(q.$$.fragment,p),f(Q.$$.fragment,p),f(S.$$.fragment,p),f(D.$$.fragment,p),f(P.$$.fragment,p),rs=!0)},o(p){b(e.$$.fragment,p),b(t.$$.fragment,p),b(x.$$.fragment,p),b(v.$$.fragment,p),b(q.$$.fragment,p),b(Q.$$.fragment,p),b(S.$$.fragment,p),b(D.$$.fragment,p),b(P.$$.fragment,p),rs=!1},d(p){p&&(l(c),l(m),l(J),l(W),l(G),l(Z),l(N),l(_),l(I),l(U),l(F),l(X),l(A),l(H),l(ls),l(as),l(Y),l(ts),l(es),l(L),l(ns),l(ps),l(z),l(ss),l(hs),l(K)),g(e,p),g(t,p),g(x,p),g(v,p),g(q,p),g(Q,p),g(S,p),g(D,p),g(P,p)}}}function bt(C){let e,c;return e=new hl({props:{$$slots:{default:[ft]},$$scope:{ctx:C}}}),{c(){u(e.$$.fragment)},l(t){d(e.$$.fragment,t)},m(t,m){y(e,t,m),c=!0},p(t,m){const J={};m&2&&(J.$$scope={dirty:m,ctx:t}),e.$set(J)},i(t){c||(f(e.$$.fragment,t),c=!0)},o(t){b(e.$$.fragment,t),c=!1},d(t){g(e,t)}}}function gt(C){let e,c=`토큰 분류를 위한 모델을 파인 튜닝하는 자세한 예제는 다음
<a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb" rel="nofollow">PyTorch notebook</a>
또는 <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb" rel="nofollow">TensorFlow notebook</a>를 참조하세요.`;return{c(){e=h("p"),e.innerHTML=c},l(t){e=M(t,"P",{"data-svelte-h":!0}),w(e)!=="svelte-139pooy"&&(e.innerHTML=c)},m(t,m){a(t,e,m)},p:ws,d(t){t&&l(e)}}}function wt(C){let e,c="텍스트를 토큰화하고 PyTorch 텐서를 반환합니다:",t,m,J,$,W="입력을 모델에 전달하고 <code>logits</code>을 반환합니다:",x,G,Z,B,N="가장 높은 확률을 가진 클래스를 모델의 <code>id2label</code> 매핑을 사용하여 텍스트 레이블로 변환합니다:",v,_,I;return m=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)`}}),G=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMnN0ZXZobGl1JTJGbXlfYXdlc29tZV93bnV0X21vZGVsJTIyKSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    logits = model(**inputs).logits`}}),_=new k({props:{code:"cHJlZGljdGlvbnMlMjAlM0QlMjB0b3JjaC5hcmdtYXgobG9naXRzJTJDJTIwZGltJTNEMiklMEFwcmVkaWN0ZWRfdG9rZW5fY2xhc3MlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0Lml0ZW0oKSU1RCUyMGZvciUyMHQlMjBpbiUyMHByZWRpY3Rpb25zJTVCMCU1RCU1RCUwQXByZWRpY3RlZF90b2tlbl9jbGFzcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predictions = torch.argmax(logits, dim=<span class="hljs-number">2</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class = [model.config.id2label[t.item()] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predictions[<span class="hljs-number">0</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class
[<span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;I-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-group&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>]`}}),{c(){e=h("p"),e.textContent=c,t=r(),u(m.$$.fragment),J=r(),$=h("p"),$.innerHTML=W,x=r(),u(G.$$.fragment),Z=r(),B=h("p"),B.innerHTML=N,v=r(),u(_.$$.fragment)},l(i){e=M(i,"P",{"data-svelte-h":!0}),w(e)!=="svelte-ctuaol"&&(e.textContent=c),t=o(i),d(m.$$.fragment,i),J=o(i),$=M(i,"P",{"data-svelte-h":!0}),w($)!=="svelte-1hjuppo"&&($.innerHTML=W),x=o(i),d(G.$$.fragment,i),Z=o(i),B=M(i,"P",{"data-svelte-h":!0}),w(B)!=="svelte-1jbp04u"&&(B.innerHTML=N),v=o(i),d(_.$$.fragment,i)},m(i,U){a(i,e,U),a(i,t,U),y(m,i,U),a(i,J,U),a(i,$,U),a(i,x,U),y(G,i,U),a(i,Z,U),a(i,B,U),a(i,v,U),y(_,i,U),I=!0},p:ws,i(i){I||(f(m.$$.fragment,i),f(G.$$.fragment,i),f(_.$$.fragment,i),I=!0)},o(i){b(m.$$.fragment,i),b(G.$$.fragment,i),b(_.$$.fragment,i),I=!1},d(i){i&&(l(e),l(t),l(J),l($),l(x),l(Z),l(B),l(v)),g(m,i),g(G,i),g(_,i)}}}function Jt(C){let e,c;return e=new hl({props:{$$slots:{default:[wt]},$$scope:{ctx:C}}}),{c(){u(e.$$.fragment)},l(t){d(e.$$.fragment,t)},m(t,m){y(e,t,m),c=!0},p(t,m){const J={};m&2&&(J.$$scope={dirty:m,ctx:t}),e.$set(J)},i(t){c||(f(e.$$.fragment,t),c=!0)},o(t){b(e.$$.fragment,t),c=!1},d(t){g(e,t)}}}function Tt(C){let e,c="텍스트를 토큰화하고 TensorFlow 텐서를 반환합니다:",t,m,J,$,W="입력값을 모델에 전달하고 <code>logits</code>을 반환합니다:",x,G,Z,B,N="가장 높은 확률을 가진 클래스를 모델의 <code>id2label</code> 매핑을 사용하여 텍스트 레이블로 변환합니다:",v,_,I;return m=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),G=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>logits = model(**inputs).logits`}}),_=new k({props:{code:"cHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyUyMCUzRCUyMHRmLm1hdGguYXJnbWF4KGxvZ2l0cyUyQyUyMGF4aXMlM0QtMSklMEFwcmVkaWN0ZWRfdG9rZW5fY2xhc3MlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0JTVEJTIwZm9yJTIwdCUyMGluJTIwcHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyU1QjAlNUQubnVtcHkoKS50b2xpc3QoKSU1RCUwQXByZWRpY3RlZF90b2tlbl9jbGFzcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class_ids = tf.math.argmax(logits, axis=-<span class="hljs-number">1</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class = [model.config.id2label[t] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predicted_token_class_ids[<span class="hljs-number">0</span>].numpy().tolist()]
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class
[<span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;I-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-group&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>]`}}),{c(){e=h("p"),e.textContent=c,t=r(),u(m.$$.fragment),J=r(),$=h("p"),$.innerHTML=W,x=r(),u(G.$$.fragment),Z=r(),B=h("p"),B.innerHTML=N,v=r(),u(_.$$.fragment)},l(i){e=M(i,"P",{"data-svelte-h":!0}),w(e)!=="svelte-1m12ipr"&&(e.textContent=c),t=o(i),d(m.$$.fragment,i),J=o(i),$=M(i,"P",{"data-svelte-h":!0}),w($)!=="svelte-tcnp5y"&&($.innerHTML=W),x=o(i),d(G.$$.fragment,i),Z=o(i),B=M(i,"P",{"data-svelte-h":!0}),w(B)!=="svelte-1jbp04u"&&(B.innerHTML=N),v=o(i),d(_.$$.fragment,i)},m(i,U){a(i,e,U),a(i,t,U),y(m,i,U),a(i,J,U),a(i,$,U),a(i,x,U),y(G,i,U),a(i,Z,U),a(i,B,U),a(i,v,U),y(_,i,U),I=!0},p:ws,i(i){I||(f(m.$$.fragment,i),f(G.$$.fragment,i),f(_.$$.fragment,i),I=!0)},o(i){b(m.$$.fragment,i),b(G.$$.fragment,i),b(_.$$.fragment,i),I=!1},d(i){i&&(l(e),l(t),l(J),l($),l(x),l(Z),l(B),l(v)),g(m,i),g(G,i),g(_,i)}}}function Ut(C){let e,c;return e=new hl({props:{$$slots:{default:[Tt]},$$scope:{ctx:C}}}),{c(){u(e.$$.fragment)},l(t){d(e.$$.fragment,t)},m(t,m){y(e,t,m),c=!0},p(t,m){const J={};m&2&&(J.$$scope={dirty:m,ctx:t}),e.$set(J)},i(t){c||(f(e.$$.fragment,t),c=!0)},o(t){b(e.$$.fragment,t),c=!1},d(t){g(e,t)}}}function $t(C){let e,c,t,m,J,$,W,x,G,Z,B="토큰 분류",N,v,_,I,i,U,q="토큰 분류는 문장의 개별 토큰에 레이블을 할당합니다. 가장 일반적인 토큰 분류 작업 중 하나는 개체명 인식(Named Entity Recognition, NER)입니다. 개체명 인식은 문장에서 사람, 위치 또는 조직과 같은 각 개체의 레이블을 찾으려고 시도합니다.",F,X,j="이 가이드에서 학습할 내용은:",A,H,Ml='<li><a href="https://huggingface.co/datasets/wnut_17" rel="nofollow">WNUT 17</a> 데이터 세트에서 <a href="https://huggingface.co/distilbert-base-uncased" rel="nofollow">DistilBERT</a>를 파인 튜닝하여 새로운 개체를 탐지합니다.</li> <li>추론을 위해 파인 튜닝 모델을 사용합니다.</li>',ls,Q,as,Y,jl="시작하기 전에, 필요한 모든 라이브러리가 설치되어 있는지 확인하세요:",ts,S,es,L,ul="Hugging Face 계정에 로그인하여 모델을 업로드하고 커뮤니티에 공유하는 것을 권장합니다. 메시지가 표시되면, 토큰을 입력하여 로그인하세요:",ns,D,ps,z,O,ss,P,hs,K,dl="WNUT 17 데이터 세트 가져오기",rs,p,T="먼저 🤗 Datasets 라이브러리에서 WNUT 17 데이터 세트를 가져옵니다:",Js,Ts,$l,Us,Ja="다음 예제를 살펴보세요:",xl,$s,_l,xs,Ta="<code>ner_tags</code>의 각 숫자는 개체를 나타냅니다. 숫자를 레이블 이름으로 변환하여 개체가 무엇인지 확인합니다:",kl,_s,Cl,ks,Ua="각 <code>ner_tag</code>의 앞에 붙은 문자는 개체의 토큰 위치를 나타냅니다:",vl,Cs,$a="<li><code>B-</code>는 개체의 시작을 나타냅니다.</li> <li><code>I-</code>는 토큰이 동일한 개체 내부에 포함되어 있음을 나타냅니다(예를 들어 <code>State</code> 토큰은 <code>Empire State Building</code>와 같은 개체의 일부입니다).</li> <li><code>0</code>는 토큰이 어떤 개체에도 해당하지 않음을 나타냅니다.</li>",Il,os,Ms,yl,vs,ya,rl,xa="전처리",Zl,Is,Bl,Zs,_a="다음으로 <code>tokens</code> 필드를 전처리하기 위해 DistilBERT 토크나이저를 가져옵니다:",Al,Bs,Gl,As,ka="위의 예제 <code>tokens</code> 필드를 보면 입력이 이미 토큰화된 것처럼 보입니다. 그러나 실제로 입력은 아직 토큰화되지 않았으므로 단어를 하위 단어로 토큰화하기 위해 <code>is_split_into_words=True</code>를 설정해야 합니다. 예제로 확인합니다:",Rl,Gs,El,Rs,Ca="그러나 이로 인해 <code>[CLS]</code>과 <code>[SEP]</code>라는 특수 토큰이 추가되고, 하위 단어 토큰화로 인해 입력과 레이블 간에 불일치가 발생합니다. 하나의 레이블에 해당하는 단일 단어는 이제 두 개의 하위 단어로 분할될 수 있습니다. 토큰과 레이블을 다음과 같이 재정렬해야 합니다:",Wl,Es,va='<li><a href="https://huggingface.co/docs/transformers/main_classes/tokenizer#transformers.BatchEncoding.word_ids" rel="nofollow"><code>word_ids</code></a> 메소드로 모든 토큰을 해당 단어에 매핑합니다.</li> <li>특수 토큰 <code>[CLS]</code>와 <code>[SEP]</code>에 <code>-100</code> 레이블을 할당하여, PyTorch 손실 함수가 해당 토큰을 무시하도록 합니다.</li> <li>주어진 단어의 첫 번째 토큰에만 레이블을 지정합니다. 같은 단어의 다른 하위 토큰에 <code>-100</code>을 할당합니다.</li>',Xl,Ws,Ia="다음은 토큰과 레이블을 재정렬하고 DistilBERT의 최대 입력 길이보다 길지 않도록 시퀀스를 잘라내는 함수를 만드는 방법입니다:",Vl,Xs,Nl,Vs,Za="전체 데이터 세트에 전처리 함수를 적용하려면, 🤗 Datasets <code>map</code> 함수를 사용하세요. <code>batched=True</code>로 설정하여 데이터 세트의 여러 요소를 한 번에 처리하면 <code>map</code> 함수의 속도를 높일 수 있습니다:",Hl,Ns,Fl,Hs,Ba="이제 <code>DataCollatorWithPadding</code>를 사용하여 예제 배치를 만들어봅시다. 데이터 세트 전체를 최대 길이로 패딩하는 대신, <em>동적 패딩</em>을 사용하여 배치에서 가장 긴 길이에 맞게 문장을 패딩하는 것이 효율적입니다.",zl,js,Ql,is,us,fl,Fs,fa,ol,Aa="평가",ql,zs,Ga='훈련 중 모델의 성능을 평가하기 위해 평가 지표를 포함하는 것이 유용합니다. 🤗 <a href="https://huggingface.co/docs/evaluate/index" rel="nofollow">Evaluate</a> 라이브러리를 사용하여 빠르게 평가 방법을 가져올 수 있습니다. 이 작업에서는 <a href="https://huggingface.co/spaces/evaluate-metric/seqeval" rel="nofollow">seqeval</a> 평가 지표를 가져옵니다. (평가 지표를 가져오고 계산하는 방법에 대해서는 🤗 Evaluate <a href="https://huggingface.co/docs/evaluate/a_quick_tour" rel="nofollow">빠른 둘러보기</a>를 참조하세요). Seqeval은 실제로 정밀도, 재현률, F1 및 정확도와 같은 여러 점수를 산출합니다.',Yl,Qs,Sl,qs,Ra="먼저 NER 레이블을 가져온 다음, <code>compute</code>에 실제 예측과 실제 레이블을 전달하여 점수를 계산하는 함수를 만듭니다:",Ll,Ys,Dl,Ss,Ea="이제 <code>compute_metrics</code> 함수를 사용할 준비가 되었으며, 훈련을 설정하면 이 함수로 되돌아올 것입니다.",Pl,cs,ds,bl,Ls,ba,il,Wa="훈련",Kl,Ds,Xa="모델을 훈련하기 전에, <code>id2label</code>와 <code>label2id</code>를 사용하여 예상되는 id와 레이블의 맵을 생성하세요:",Ol,Ps,sa,ys,la,fs,aa,ms,bs,gl,Ks,ga,cl,Va="추론",ta,Os,Na="좋아요, 이제 모델을 파인 튜닝했으니 추론에 사용할 수 있습니다!",ea,sl,Ha="추론을 수행하고자 하는 텍스트를 가져와봅시다:",na,ll,pa,al,Fa="파인 튜닝된 모델로 추론을 시도하는 가장 간단한 방법은 <code>pipeline()</code>를 사용하는 것입니다. 모델로 NER의 <code>pipeline</code>을 인스턴스화하고, 텍스트를 전달해보세요:",ra,tl,oa,el,za="원한다면, <code>pipeline</code>의 결과를 수동으로 복제할 수도 있습니다:",ia,gs,ca,wl,ma;return x=new ml({}),v=new ot({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/token_classification.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/pytorch/token_classification.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/tensorflow/token_classification.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/token_classification.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/pytorch/token_classification.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/tensorflow/token_classification.ipynb"}]}}),I=new st({props:{id:"wVHdVlPScxA"}}),Q=new da({props:{$$slots:{default:[it]},$$scope:{ctx:C}}}),S=new k({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwZXZhbHVhdGUlMjBzZXFldmFs",highlighted:"pip install transformers datasets evaluate seqeval"}}),D=new k({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_login()`}}),P=new ml({}),Ts=new k({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBd251dCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ3bnV0XzE3JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`}}),$s=new k({props:{code:"d251dCU1QiUyMnRyYWluJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`}}),_s=new k({props:{code:"bGFiZWxfbGlzdCUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RC5mZWF0dXJlcyU1QmYlMjJuZXJfdGFncyUyMiU1RC5mZWF0dXJlLm5hbWVzJTBBbGFiZWxfbGlzdA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`}}),vs=new ml({}),Is=new st({props:{id:"iY2AZYdZAr0"}}),Bs=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),Gs=new k({props:{code:"ZXhhbXBsZSUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RCU1QjAlNUQlMEF0b2tlbml6ZWRfaW5wdXQlMjAlM0QlMjB0b2tlbml6ZXIoZXhhbXBsZSU1QiUyMnRva2VucyUyMiU1RCUyQyUyMGlzX3NwbGl0X2ludG9fd29yZHMlM0RUcnVlKSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci5jb252ZXJ0X2lkc190b190b2tlbnModG9rZW5pemVkX2lucHV0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQXRva2Vucw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>example = wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),Xs=new k({props:{code:"ZGVmJTIwdG9rZW5pemVfYW5kX2FsaWduX2xhYmVscyhleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydG9rZW5zJTIyJTVEJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMjBpc19zcGxpdF9pbnRvX3dvcmRzJTNEVHJ1ZSklMEElMEElMjAlMjAlMjAlMjBsYWJlbHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBpJTJDJTIwbGFiZWwlMjBpbiUyMGVudW1lcmF0ZShleGFtcGxlcyU1QmYlMjJuZXJfdGFncyUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkcyUyMCUzRCUyMHRva2VuaXplZF9pbnB1dHMud29yZF9pZHMoYmF0Y2hfaW5kZXglM0RpKSUyMCUyMCUyMyUyME1hcCUyMHRva2VucyUyMHRvJTIwdGhlaXIlMjByZXNwZWN0aXZlJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyME5vbmUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjB3b3JkX2lkeCUyMGluJTIwd29yZF9pZHMlM0ElMjAlMjAlMjMlMjBTZXQlMjB0aGUlMjBzcGVjaWFsJTIwdG9rZW5zJTIwdG8lMjAtMTAwLiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZHglMjBpcyUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKC0xMDApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxpZiUyMHdvcmRfaWR4JTIwISUzRCUyMHByZXZpb3VzX3dvcmRfaWR4JTNBJTIwJTIwJTIzJTIwT25seSUyMGxhYmVsJTIwdGhlJTIwZmlyc3QlMjB0b2tlbiUyMG9mJTIwYSUyMGdpdmVuJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKGxhYmVsJTVCd29yZF9pZHglNUQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVsX2lkcy5hcHBlbmQoLTEwMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyMHdvcmRfaWR4JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFiZWxzLmFwcGVuZChsYWJlbF9pZHMpJTBBJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVkX2lucHV0cyU1QiUyMmxhYmVscyUyMiU1RCUyMCUzRCUyMGxhYmVscyUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHRva2VuaXplZF9pbnB1dHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
<span class="hljs-meta">... </span>        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
<span class="hljs-meta">... </span>        previous_word_idx = <span class="hljs-literal">None</span>
<span class="hljs-meta">... </span>        label_ids = []
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
<span class="hljs-meta">... </span>                label_ids.append(label[word_idx])
<span class="hljs-meta">... </span>            <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            previous_word_idx = word_idx
<span class="hljs-meta">... </span>        labels.append(label_ids)

<span class="hljs-meta">... </span>    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenized_inputs`}}),Ns=new k({props:{code:"dG9rZW5pemVkX3dudXQlMjAlM0QlMjB3bnV0Lm1hcCh0b2tlbml6ZV9hbmRfYWxpZ25fbGFiZWxzJTJDJTIwYmF0Y2hlZCUzRFRydWUp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)'}}),js=new wa({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Mt],pytorch:[mt]},$$scope:{ctx:C}}}),Fs=new ml({}),Qs=new k({props:{code:"aW1wb3J0JTIwZXZhbHVhdGUlMEElMEFzZXFldmFsJTIwJTNEJTIwZXZhbHVhdGUubG9hZCglMjJzZXFldmFsJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> evaluate

<span class="hljs-meta">&gt;&gt;&gt; </span>seqeval = evaluate.load(<span class="hljs-string">&quot;seqeval&quot;</span>)`}}),Ys=new k({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBJTBBbGFiZWxzJTIwJTNEJTIwJTVCbGFiZWxfbGlzdCU1QmklNUQlMjBmb3IlMjBpJTIwaW4lMjBleGFtcGxlJTVCZiUyMm5lcl90YWdzJTIyJTVEJTVEJTBBJTBBJTBBZGVmJTIwY29tcHV0ZV9tZXRyaWNzKHApJTNBJTBBJTIwJTIwJTIwJTIwcHJlZGljdGlvbnMlMkMlMjBsYWJlbHMlMjAlM0QlMjBwJTBBJTIwJTIwJTIwJTIwcHJlZGljdGlvbnMlMjAlM0QlMjBucC5hcmdtYXgocHJlZGljdGlvbnMlMkMlMjBheGlzJTNEMiklMEElMEElMjAlMjAlMjAlMjB0cnVlX3ByZWRpY3Rpb25zJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCbGFiZWxfbGlzdCU1QnAlNUQlMjBmb3IlMjAocCUyQyUyMGwpJTIwaW4lMjB6aXAocHJlZGljdGlvbiUyQyUyMGxhYmVsKSUyMGlmJTIwbCUyMCElM0QlMjAtMTAwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwcHJlZGljdGlvbiUyQyUyMGxhYmVsJTIwaW4lMjB6aXAocHJlZGljdGlvbnMlMkMlMjBsYWJlbHMpJTBBJTIwJTIwJTIwJTIwJTVEJTBBJTIwJTIwJTIwJTIwdHJ1ZV9sYWJlbHMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUJsYWJlbF9saXN0JTVCbCU1RCUyMGZvciUyMChwJTJDJTIwbCklMjBpbiUyMHppcChwcmVkaWN0aW9uJTJDJTIwbGFiZWwpJTIwaWYlMjBsJTIwISUzRCUyMC0xMDAlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBwcmVkaWN0aW9uJTJDJTIwbGFiZWwlMjBpbiUyMHppcChwcmVkaWN0aW9ucyUyQyUyMGxhYmVscyklMEElMjAlMjAlMjAlMjAlNUQlMEElMEElMjAlMjAlMjAlMjByZXN1bHRzJTIwJTNEJTIwc2VxZXZhbC5jb21wdXRlKHByZWRpY3Rpb25zJTNEdHJ1ZV9wcmVkaWN0aW9ucyUyQyUyMHJlZmVyZW5jZXMlM0R0cnVlX2xhYmVscyklMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwcmVjaXNpb24lMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9wcmVjaXNpb24lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZWNhbGwlMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9yZWNhbGwlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJmMSUyMiUzQSUyMHJlc3VsdHMlNUIlMjJvdmVyYWxsX2YxJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWNjdXJhY3klMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9hY2N1cmFjeSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-meta">&gt;&gt;&gt; </span>labels = [label_list[i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> example[<span class="hljs-string">f&quot;ner_tags&quot;</span>]]


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">p</span>):
<span class="hljs-meta">... </span>    predictions, labels = p
<span class="hljs-meta">... </span>    predictions = np.argmax(predictions, axis=<span class="hljs-number">2</span>)

<span class="hljs-meta">... </span>    true_predictions = [
<span class="hljs-meta">... </span>        [label_list[p] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
<span class="hljs-meta">... </span>    ]
<span class="hljs-meta">... </span>    true_labels = [
<span class="hljs-meta">... </span>        [label_list[l] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
<span class="hljs-meta">... </span>    ]

<span class="hljs-meta">... </span>    results = seqeval.compute(predictions=true_predictions, references=true_labels)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;precision&quot;</span>: results[<span class="hljs-string">&quot;overall_precision&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;recall&quot;</span>: results[<span class="hljs-string">&quot;overall_recall&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;f1&quot;</span>: results[<span class="hljs-string">&quot;overall_f1&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;accuracy&quot;</span>: results[<span class="hljs-string">&quot;overall_accuracy&quot;</span>],
<span class="hljs-meta">... </span>    }`}}),Ls=new ml({}),Ps=new k({props:{code:"aWQybGFiZWwlMjAlM0QlMjAlN0IlMEElMjAlMjAlMjAlMjAwJTNBJTIwJTIyTyUyMiUyQyUwQSUyMCUyMCUyMCUyMDElM0ElMjAlMjJCLWNvcnBvcmF0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwMiUzQSUyMCUyMkktY29ycG9yYXRpb24lMjIlMkMlMEElMjAlMjAlMjAlMjAzJTNBJTIwJTIyQi1jcmVhdGl2ZS13b3JrJTIyJTJDJTBBJTIwJTIwJTIwJTIwNCUzQSUyMCUyMkktY3JlYXRpdmUtd29yayUyMiUyQyUwQSUyMCUyMCUyMCUyMDUlM0ElMjAlMjJCLWdyb3VwJTIyJTJDJTBBJTIwJTIwJTIwJTIwNiUzQSUyMCUyMkktZ3JvdXAlMjIlMkMlMEElMjAlMjAlMjAlMjA3JTNBJTIwJTIyQi1sb2NhdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMDglM0ElMjAlMjJJLWxvY2F0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwOSUzQSUyMCUyMkItcGVyc29uJTIyJTJDJTBBJTIwJTIwJTIwJTIwMTAlM0ElMjAlMjJJLXBlcnNvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMDExJTNBJTIwJTIyQi1wcm9kdWN0JTIyJTJDJTBBJTIwJTIwJTIwJTIwMTIlM0ElMjAlMjJJLXByb2R1Y3QlMjIlMkMlMEElN0QlMEFsYWJlbDJpZCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMk8lMjIlM0ElMjAwJTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1jb3Jwb3JhdGlvbiUyMiUzQSUyMDElMkMlMEElMjAlMjAlMjAlMjAlMjJJLWNvcnBvcmF0aW9uJTIyJTNBJTIwMiUyQyUwQSUyMCUyMCUyMCUyMCUyMkItY3JlYXRpdmUtd29yayUyMiUzQSUyMDMlMkMlMEElMjAlMjAlMjAlMjAlMjJJLWNyZWF0aXZlLXdvcmslMjIlM0ElMjA0JTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1ncm91cCUyMiUzQSUyMDUlMkMlMEElMjAlMjAlMjAlMjAlMjJJLWdyb3VwJTIyJTNBJTIwNiUyQyUwQSUyMCUyMCUyMCUyMCUyMkItbG9jYXRpb24lMjIlM0ElMjA3JTJDJTBBJTIwJTIwJTIwJTIwJTIySS1sb2NhdGlvbiUyMiUzQSUyMDglMkMlMEElMjAlMjAlMjAlMjAlMjJCLXBlcnNvbiUyMiUzQSUyMDklMkMlMEElMjAlMjAlMjAlMjAlMjJJLXBlcnNvbiUyMiUzQSUyMDEwJTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1wcm9kdWN0JTIyJTNBJTIwMTElMkMlMEElMjAlMjAlMjAlMjAlMjJJLXByb2R1Y3QlMjIlM0ElMjAxMiUyQyUwQSU3RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>id2label = {
<span class="hljs-meta">... </span>    <span class="hljs-number">0</span>: <span class="hljs-string">&quot;O&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">1</span>: <span class="hljs-string">&quot;B-corporation&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">2</span>: <span class="hljs-string">&quot;I-corporation&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">3</span>: <span class="hljs-string">&quot;B-creative-work&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">4</span>: <span class="hljs-string">&quot;I-creative-work&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">5</span>: <span class="hljs-string">&quot;B-group&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">6</span>: <span class="hljs-string">&quot;I-group&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">7</span>: <span class="hljs-string">&quot;B-location&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">8</span>: <span class="hljs-string">&quot;I-location&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">9</span>: <span class="hljs-string">&quot;B-person&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">10</span>: <span class="hljs-string">&quot;I-person&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">11</span>: <span class="hljs-string">&quot;B-product&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">12</span>: <span class="hljs-string">&quot;I-product&quot;</span>,
<span class="hljs-meta">... </span>}
<span class="hljs-meta">&gt;&gt;&gt; </span>label2id = {
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;O&quot;</span>: <span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-corporation&quot;</span>: <span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-corporation&quot;</span>: <span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-creative-work&quot;</span>: <span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-creative-work&quot;</span>: <span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-group&quot;</span>: <span class="hljs-number">5</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-group&quot;</span>: <span class="hljs-number">6</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-location&quot;</span>: <span class="hljs-number">7</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-location&quot;</span>: <span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-person&quot;</span>: <span class="hljs-number">9</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-person&quot;</span>: <span class="hljs-number">10</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-product&quot;</span>: <span class="hljs-number">11</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-product&quot;</span>: <span class="hljs-number">12</span>,
<span class="hljs-meta">... </span>}`}}),ys=new wa({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[bt],pytorch:[dt]},$$scope:{ctx:C}}}),fs=new da({props:{$$slots:{default:[gt]},$$scope:{ctx:C}}}),Ks=new ml({}),ll=new k({props:{code:"dGV4dCUyMCUzRCUyMCUyMlRoZSUyMEdvbGRlbiUyMFN0YXRlJTIwV2FycmlvcnMlMjBhcmUlMjBhbiUyMEFtZXJpY2FuJTIwcHJvZmVzc2lvbmFsJTIwYmFza2V0YmFsbCUyMHRlYW0lMjBiYXNlZCUyMGluJTIwU2FuJTIwRnJhbmNpc2NvLiUyMg==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>text = <span class="hljs-string">&quot;The Golden State Warriors are an American professional basketball team based in San Francisco.&quot;</span>'}}),tl=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBY2xhc3NpZmllciUyMCUzRCUyMHBpcGVsaW5lKCUyMm5lciUyMiUyQyUyMG1vZGVsJTNEJTIyc3RldmhsaXUlMkZteV9hd2Vzb21lX3dudXRfbW9kZWwlMjIpJTBBY2xhc3NpZmllcih0ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>classifier = pipeline(<span class="hljs-string">&quot;ner&quot;</span>, model=<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>classifier(text)
[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.42658573</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">2</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;golden&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">4</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">10</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.35856336</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">3</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;state&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-group&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.3064001</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;warriors&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">17</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">25</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.65523505</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;san&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">80</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">83</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.4668663</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;francisco&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">84</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">93</span>}]`}}),gs=new wa({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Ut],pytorch:[Jt]},$$scope:{ctx:C}}}),{c(){e=h("meta"),c=r(),t=h("p"),m=r(),J=h("h1"),$=h("a"),W=h("span"),u(x.$$.fragment),G=r(),Z=h("span"),Z.textContent=B,N=r(),u(v.$$.fragment),_=r(),u(I.$$.fragment),i=r(),U=h("p"),U.textContent=q,F=r(),X=h("p"),X.textContent=j,A=r(),H=h("ol"),H.innerHTML=Ml,ls=r(),u(Q.$$.fragment),as=r(),Y=h("p"),Y.textContent=jl,ts=r(),u(S.$$.fragment),es=r(),L=h("p"),L.textContent=ul,ns=r(),u(D.$$.fragment),ps=r(),z=h("h2"),O=h("a"),ss=h("span"),u(P.$$.fragment),hs=r(),K=h("span"),K.textContent=dl,rs=r(),p=h("p"),p.textContent=T,Js=r(),u(Ts.$$.fragment),$l=r(),Us=h("p"),Us.textContent=Ja,xl=r(),u($s.$$.fragment),_l=r(),xs=h("p"),xs.innerHTML=Ta,kl=r(),u(_s.$$.fragment),Cl=r(),ks=h("p"),ks.innerHTML=Ua,vl=r(),Cs=h("ul"),Cs.innerHTML=$a,Il=r(),os=h("h2"),Ms=h("a"),yl=h("span"),u(vs.$$.fragment),ya=r(),rl=h("span"),rl.textContent=xa,Zl=r(),u(Is.$$.fragment),Bl=r(),Zs=h("p"),Zs.innerHTML=_a,Al=r(),u(Bs.$$.fragment),Gl=r(),As=h("p"),As.innerHTML=ka,Rl=r(),u(Gs.$$.fragment),El=r(),Rs=h("p"),Rs.innerHTML=Ca,Wl=r(),Es=h("ol"),Es.innerHTML=va,Xl=r(),Ws=h("p"),Ws.textContent=Ia,Vl=r(),u(Xs.$$.fragment),Nl=r(),Vs=h("p"),Vs.innerHTML=Za,Hl=r(),u(Ns.$$.fragment),Fl=r(),Hs=h("p"),Hs.innerHTML=Ba,zl=r(),u(js.$$.fragment),Ql=r(),is=h("h2"),us=h("a"),fl=h("span"),u(Fs.$$.fragment),fa=r(),ol=h("span"),ol.textContent=Aa,ql=r(),zs=h("p"),zs.innerHTML=Ga,Yl=r(),u(Qs.$$.fragment),Sl=r(),qs=h("p"),qs.innerHTML=Ra,Ll=r(),u(Ys.$$.fragment),Dl=r(),Ss=h("p"),Ss.innerHTML=Ea,Pl=r(),cs=h("h2"),ds=h("a"),bl=h("span"),u(Ls.$$.fragment),ba=r(),il=h("span"),il.textContent=Wa,Kl=r(),Ds=h("p"),Ds.innerHTML=Xa,Ol=r(),u(Ps.$$.fragment),sa=r(),u(ys.$$.fragment),la=r(),u(fs.$$.fragment),aa=r(),ms=h("h2"),bs=h("a"),gl=h("span"),u(Ks.$$.fragment),ga=r(),cl=h("span"),cl.textContent=Va,ta=r(),Os=h("p"),Os.textContent=Na,ea=r(),sl=h("p"),sl.textContent=Ha,na=r(),u(ll.$$.fragment),pa=r(),al=h("p"),al.innerHTML=Fa,ra=r(),u(tl.$$.fragment),oa=r(),el=h("p"),el.innerHTML=za,ia=r(),u(gs.$$.fragment),ca=r(),wl=h("p"),this.h()},l(s){const n=rt("svelte-1phssyn",document.head);e=M(n,"META",{name:!0,content:!0}),n.forEach(l),c=o(s),t=M(s,"P",{}),V(t).forEach(l),m=o(s),J=M(s,"H1",{class:!0});var nl=V(J);$=M(nl,"A",{id:!0,class:!0,href:!0});var Jl=V($);W=M(Jl,"SPAN",{});var Tl=V(W);d(x.$$.fragment,Tl),Tl.forEach(l),Jl.forEach(l),G=o(nl),Z=M(nl,"SPAN",{"data-svelte-h":!0}),w(Z)!=="svelte-m2p5mn"&&(Z.textContent=B),nl.forEach(l),N=o(s),d(v.$$.fragment,s),_=o(s),d(I.$$.fragment,s),i=o(s),U=M(s,"P",{"data-svelte-h":!0}),w(U)!=="svelte-h22iel"&&(U.textContent=q),F=o(s),X=M(s,"P",{"data-svelte-h":!0}),w(X)!=="svelte-14hiaa1"&&(X.textContent=j),A=o(s),H=M(s,"OL",{"data-svelte-h":!0}),w(H)!=="svelte-1hfy8fu"&&(H.innerHTML=Ml),ls=o(s),d(Q.$$.fragment,s),as=o(s),Y=M(s,"P",{"data-svelte-h":!0}),w(Y)!=="svelte-1bc8bfk"&&(Y.textContent=jl),ts=o(s),d(S.$$.fragment,s),es=o(s),L=M(s,"P",{"data-svelte-h":!0}),w(L)!=="svelte-1hhyu5y"&&(L.textContent=ul),ns=o(s),d(D.$$.fragment,s),ps=o(s),z=M(s,"H2",{class:!0});var pl=V(z);O=M(pl,"A",{id:!0,class:!0,href:!0});var Ul=V(O);ss=M(Ul,"SPAN",{});var Qa=V(ss);d(P.$$.fragment,Qa),Qa.forEach(l),Ul.forEach(l),hs=o(pl),K=M(pl,"SPAN",{"data-svelte-h":!0}),w(K)!=="svelte-hs73v3"&&(K.textContent=dl),pl.forEach(l),rs=o(s),p=M(s,"P",{"data-svelte-h":!0}),w(p)!=="svelte-1v008tf"&&(p.textContent=T),Js=o(s),d(Ts.$$.fragment,s),$l=o(s),Us=M(s,"P",{"data-svelte-h":!0}),w(Us)!=="svelte-7y00sm"&&(Us.textContent=Ja),xl=o(s),d($s.$$.fragment,s),_l=o(s),xs=M(s,"P",{"data-svelte-h":!0}),w(xs)!=="svelte-19kj2kd"&&(xs.innerHTML=Ta),kl=o(s),d(_s.$$.fragment,s),Cl=o(s),ks=M(s,"P",{"data-svelte-h":!0}),w(ks)!=="svelte-1jetaks"&&(ks.innerHTML=Ua),vl=o(s),Cs=M(s,"UL",{"data-svelte-h":!0}),w(Cs)!=="svelte-wzon1c"&&(Cs.innerHTML=$a),Il=o(s),os=M(s,"H2",{class:!0});var ha=V(os);Ms=M(ha,"A",{id:!0,class:!0,href:!0});var qa=V(Ms);yl=M(qa,"SPAN",{});var Ya=V(yl);d(vs.$$.fragment,Ya),Ya.forEach(l),qa.forEach(l),ya=o(ha),rl=M(ha,"SPAN",{"data-svelte-h":!0}),w(rl)!=="svelte-1cte2wb"&&(rl.textContent=xa),ha.forEach(l),Zl=o(s),d(Is.$$.fragment,s),Bl=o(s),Zs=M(s,"P",{"data-svelte-h":!0}),w(Zs)!=="svelte-xprzxq"&&(Zs.innerHTML=_a),Al=o(s),d(Bs.$$.fragment,s),Gl=o(s),As=M(s,"P",{"data-svelte-h":!0}),w(As)!=="svelte-n9m45o"&&(As.innerHTML=ka),Rl=o(s),d(Gs.$$.fragment,s),El=o(s),Rs=M(s,"P",{"data-svelte-h":!0}),w(Rs)!=="svelte-1slve6o"&&(Rs.innerHTML=Ca),Wl=o(s),Es=M(s,"OL",{"data-svelte-h":!0}),w(Es)!=="svelte-1ynfql3"&&(Es.innerHTML=va),Xl=o(s),Ws=M(s,"P",{"data-svelte-h":!0}),w(Ws)!=="svelte-1uatru5"&&(Ws.textContent=Ia),Vl=o(s),d(Xs.$$.fragment,s),Nl=o(s),Vs=M(s,"P",{"data-svelte-h":!0}),w(Vs)!=="svelte-o2pm0z"&&(Vs.innerHTML=Za),Hl=o(s),d(Ns.$$.fragment,s),Fl=o(s),Hs=M(s,"P",{"data-svelte-h":!0}),w(Hs)!=="svelte-11dkppi"&&(Hs.innerHTML=Ba),zl=o(s),d(js.$$.fragment,s),Ql=o(s),is=M(s,"H2",{class:!0});var Ma=V(is);us=M(Ma,"A",{id:!0,class:!0,href:!0});var Sa=V(us);fl=M(Sa,"SPAN",{});var La=V(fl);d(Fs.$$.fragment,La),La.forEach(l),Sa.forEach(l),fa=o(Ma),ol=M(Ma,"SPAN",{"data-svelte-h":!0}),w(ol)!=="svelte-fd7xxy"&&(ol.textContent=Aa),Ma.forEach(l),ql=o(s),zs=M(s,"P",{"data-svelte-h":!0}),w(zs)!=="svelte-1xei3jh"&&(zs.innerHTML=Ga),Yl=o(s),d(Qs.$$.fragment,s),Sl=o(s),qs=M(s,"P",{"data-svelte-h":!0}),w(qs)!=="svelte-no1q5"&&(qs.innerHTML=Ra),Ll=o(s),d(Ys.$$.fragment,s),Dl=o(s),Ss=M(s,"P",{"data-svelte-h":!0}),w(Ss)!=="svelte-1tfmmd0"&&(Ss.innerHTML=Ea),Pl=o(s),cs=M(s,"H2",{class:!0});var ja=V(cs);ds=M(ja,"A",{id:!0,class:!0,href:!0});var Da=V(ds);bl=M(Da,"SPAN",{});var Pa=V(bl);d(Ls.$$.fragment,Pa),Pa.forEach(l),Da.forEach(l),ba=o(ja),il=M(ja,"SPAN",{"data-svelte-h":!0}),w(il)!=="svelte-tdy10p"&&(il.textContent=Wa),ja.forEach(l),Kl=o(s),Ds=M(s,"P",{"data-svelte-h":!0}),w(Ds)!=="svelte-84b3vk"&&(Ds.innerHTML=Xa),Ol=o(s),d(Ps.$$.fragment,s),sa=o(s),d(ys.$$.fragment,s),la=o(s),d(fs.$$.fragment,s),aa=o(s),ms=M(s,"H2",{class:!0});var ua=V(ms);bs=M(ua,"A",{id:!0,class:!0,href:!0});var Ka=V(bs);gl=M(Ka,"SPAN",{});var Oa=V(gl);d(Ks.$$.fragment,Oa),Oa.forEach(l),Ka.forEach(l),ga=o(ua),cl=M(ua,"SPAN",{"data-svelte-h":!0}),w(cl)!=="svelte-8litrx"&&(cl.textContent=Va),ua.forEach(l),ta=o(s),Os=M(s,"P",{"data-svelte-h":!0}),w(Os)!=="svelte-1r6hgyn"&&(Os.textContent=Na),ea=o(s),sl=M(s,"P",{"data-svelte-h":!0}),w(sl)!=="svelte-ej1eir"&&(sl.textContent=Ha),na=o(s),d(ll.$$.fragment,s),pa=o(s),al=M(s,"P",{"data-svelte-h":!0}),w(al)!=="svelte-bavprk"&&(al.innerHTML=Fa),ra=o(s),d(tl.$$.fragment,s),oa=o(s),el=M(s,"P",{"data-svelte-h":!0}),w(el)!=="svelte-1smgeha"&&(el.innerHTML=za),ia=o(s),d(gs.$$.fragment,s),ca=o(s),wl=M(s,"P",{}),V(wl).forEach(l),this.h()},h(){R(e,"name","hf:doc:metadata"),R(e,"content",JSON.stringify(xt)),R($,"id","token-classification"),R($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R($,"href","#token-classification"),R(J,"class","relative group"),R(O,"id","load-wnut-17-dataset"),R(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(O,"href","#load-wnut-17-dataset"),R(z,"class","relative group"),R(Ms,"id","preprocess"),R(Ms,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(Ms,"href","#preprocess"),R(os,"class","relative group"),R(us,"id","evaluation"),R(us,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(us,"href","#evaluation"),R(is,"class","relative group"),R(ds,"id","train"),R(ds,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(ds,"href","#train"),R(cs,"class","relative group"),R(bs,"id","inference"),R(bs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),R(bs,"href","#inference"),R(ms,"class","relative group")},m(s,n){E(document.head,e),a(s,c,n),a(s,t,n),a(s,m,n),a(s,J,n),E(J,$),E($,W),y(x,W,null),E(J,G),E(J,Z),a(s,N,n),y(v,s,n),a(s,_,n),y(I,s,n),a(s,i,n),a(s,U,n),a(s,F,n),a(s,X,n),a(s,A,n),a(s,H,n),a(s,ls,n),y(Q,s,n),a(s,as,n),a(s,Y,n),a(s,ts,n),y(S,s,n),a(s,es,n),a(s,L,n),a(s,ns,n),y(D,s,n),a(s,ps,n),a(s,z,n),E(z,O),E(O,ss),y(P,ss,null),E(z,hs),E(z,K),a(s,rs,n),a(s,p,n),a(s,Js,n),y(Ts,s,n),a(s,$l,n),a(s,Us,n),a(s,xl,n),y($s,s,n),a(s,_l,n),a(s,xs,n),a(s,kl,n),y(_s,s,n),a(s,Cl,n),a(s,ks,n),a(s,vl,n),a(s,Cs,n),a(s,Il,n),a(s,os,n),E(os,Ms),E(Ms,yl),y(vs,yl,null),E(os,ya),E(os,rl),a(s,Zl,n),y(Is,s,n),a(s,Bl,n),a(s,Zs,n),a(s,Al,n),y(Bs,s,n),a(s,Gl,n),a(s,As,n),a(s,Rl,n),y(Gs,s,n),a(s,El,n),a(s,Rs,n),a(s,Wl,n),a(s,Es,n),a(s,Xl,n),a(s,Ws,n),a(s,Vl,n),y(Xs,s,n),a(s,Nl,n),a(s,Vs,n),a(s,Hl,n),y(Ns,s,n),a(s,Fl,n),a(s,Hs,n),a(s,zl,n),y(js,s,n),a(s,Ql,n),a(s,is,n),E(is,us),E(us,fl),y(Fs,fl,null),E(is,fa),E(is,ol),a(s,ql,n),a(s,zs,n),a(s,Yl,n),y(Qs,s,n),a(s,Sl,n),a(s,qs,n),a(s,Ll,n),y(Ys,s,n),a(s,Dl,n),a(s,Ss,n),a(s,Pl,n),a(s,cs,n),E(cs,ds),E(ds,bl),y(Ls,bl,null),E(cs,ba),E(cs,il),a(s,Kl,n),a(s,Ds,n),a(s,Ol,n),y(Ps,s,n),a(s,sa,n),y(ys,s,n),a(s,la,n),y(fs,s,n),a(s,aa,n),a(s,ms,n),E(ms,bs),E(bs,gl),y(Ks,gl,null),E(ms,ga),E(ms,cl),a(s,ta,n),a(s,Os,n),a(s,ea,n),a(s,sl,n),a(s,na,n),y(ll,s,n),a(s,pa,n),a(s,al,n),a(s,ra,n),y(tl,s,n),a(s,oa,n),a(s,el,n),a(s,ia,n),y(gs,s,n),a(s,ca,n),a(s,wl,n),ma=!0},p(s,[n]){const nl={};n&2&&(nl.$$scope={dirty:n,ctx:s}),Q.$set(nl);const Jl={};n&2&&(Jl.$$scope={dirty:n,ctx:s}),js.$set(Jl);const Tl={};n&2&&(Tl.$$scope={dirty:n,ctx:s}),ys.$set(Tl);const pl={};n&2&&(pl.$$scope={dirty:n,ctx:s}),fs.$set(pl);const Ul={};n&2&&(Ul.$$scope={dirty:n,ctx:s}),gs.$set(Ul)},i(s){ma||(f(x.$$.fragment,s),f(v.$$.fragment,s),f(I.$$.fragment,s),f(Q.$$.fragment,s),f(S.$$.fragment,s),f(D.$$.fragment,s),f(P.$$.fragment,s),f(Ts.$$.fragment,s),f($s.$$.fragment,s),f(_s.$$.fragment,s),f(vs.$$.fragment,s),f(Is.$$.fragment,s),f(Bs.$$.fragment,s),f(Gs.$$.fragment,s),f(Xs.$$.fragment,s),f(Ns.$$.fragment,s),f(js.$$.fragment,s),f(Fs.$$.fragment,s),f(Qs.$$.fragment,s),f(Ys.$$.fragment,s),f(Ls.$$.fragment,s),f(Ps.$$.fragment,s),f(ys.$$.fragment,s),f(fs.$$.fragment,s),f(Ks.$$.fragment,s),f(ll.$$.fragment,s),f(tl.$$.fragment,s),f(gs.$$.fragment,s),ma=!0)},o(s){b(x.$$.fragment,s),b(v.$$.fragment,s),b(I.$$.fragment,s),b(Q.$$.fragment,s),b(S.$$.fragment,s),b(D.$$.fragment,s),b(P.$$.fragment,s),b(Ts.$$.fragment,s),b($s.$$.fragment,s),b(_s.$$.fragment,s),b(vs.$$.fragment,s),b(Is.$$.fragment,s),b(Bs.$$.fragment,s),b(Gs.$$.fragment,s),b(Xs.$$.fragment,s),b(Ns.$$.fragment,s),b(js.$$.fragment,s),b(Fs.$$.fragment,s),b(Qs.$$.fragment,s),b(Ys.$$.fragment,s),b(Ls.$$.fragment,s),b(Ps.$$.fragment,s),b(ys.$$.fragment,s),b(fs.$$.fragment,s),b(Ks.$$.fragment,s),b(ll.$$.fragment,s),b(tl.$$.fragment,s),b(gs.$$.fragment,s),ma=!1},d(s){s&&(l(c),l(t),l(m),l(J),l(N),l(_),l(i),l(U),l(F),l(X),l(A),l(H),l(ls),l(as),l(Y),l(ts),l(es),l(L),l(ns),l(ps),l(z),l(rs),l(p),l(Js),l($l),l(Us),l(xl),l(_l),l(xs),l(kl),l(Cl),l(ks),l(vl),l(Cs),l(Il),l(os),l(Zl),l(Bl),l(Zs),l(Al),l(Gl),l(As),l(Rl),l(El),l(Rs),l(Wl),l(Es),l(Xl),l(Ws),l(Vl),l(Nl),l(Vs),l(Hl),l(Fl),l(Hs),l(zl),l(Ql),l(is),l(ql),l(zs),l(Yl),l(Sl),l(qs),l(Ll),l(Dl),l(Ss),l(Pl),l(cs),l(Kl),l(Ds),l(Ol),l(sa),l(la),l(aa),l(ms),l(ta),l(Os),l(ea),l(sl),l(na),l(pa),l(al),l(ra),l(oa),l(el),l(ia),l(ca),l(wl)),l(e),g(x),g(v,s),g(I,s),g(Q,s),g(S,s),g(D,s),g(P),g(Ts,s),g($s,s),g(_s,s),g(vs),g(Is,s),g(Bs,s),g(Gs,s),g(Xs,s),g(Ns,s),g(js,s),g(Fs),g(Qs,s),g(Ys,s),g(Ls),g(Ps,s),g(ys,s),g(fs,s),g(Ks),g(ll,s),g(tl,s),g(gs,s)}}}const xt={local:"token-classification",sections:[{local:"load-wnut-17-dataset",title:"WNUT 17 데이터 세트 가져오기"},{local:"preprocess",title:"전처리"},{local:"evaluation",title:"평가"},{local:"train",title:"훈련"},{local:"inference",title:"추론"}],title:"토큰 분류"};function _t(C){return et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Rt extends nt{constructor(e){super(),pt(this,e,_t,$t,tt,{})}}export{Rt as component};
