import{s as Fs,o as Hs,n as me}from"../chunks/scheduler.56730f09.js";import{S as Qs,i as qs,g as M,s as r,u as w,A as Ys,h,f as t,c as o,j as z,v as j,y as f,k as G,o as E,a as n,w as y,d as g,t as J,x as T,m as Ns,n as zs}from"../chunks/index.ba464511.js";import{T as ts}from"../chunks/Tip.70442610.js";import{Y as Vs}from"../chunks/Youtube.7b20230c.js";import{I as st}from"../chunks/IconCopyLink.89bde88e.js";import{C as B}from"../chunks/CodeBlock.2302fd0d.js";import{D as Ss}from"../chunks/DocNotebookDropdown.68fd8653.js";import{F as rs,M as lt}from"../chunks/Markdown.6303afae.js";function Ls(I){let a,c,l='<a href="../model_doc/albert">ALBERT</a>, <a href="../model_doc/bart">BART</a>, <a href="../model_doc/bert">BERT</a>, <a href="../model_doc/big_bird">BigBird</a>, <a href="../model_doc/bigbird_pegasus">BigBird-Pegasus</a>, <a href="../model_doc/bloom">BLOOM</a>, <a href="../model_doc/camembert">CamemBERT</a>, <a href="../model_doc/canine">CANINE</a>, <a href="../model_doc/convbert">ConvBERT</a>, <a href="../model_doc/data2vec-text">Data2VecText</a>, <a href="../model_doc/deberta">DeBERTa</a>, <a href="../model_doc/deberta-v2">DeBERTa-v2</a>, <a href="../model_doc/distilbert">DistilBERT</a>, <a href="../model_doc/electra">ELECTRA</a>, <a href="../model_doc/ernie">ERNIE</a>, <a href="../model_doc/ernie_m">ErnieM</a>, <a href="../model_doc/flaubert">FlauBERT</a>, <a href="../model_doc/fnet">FNet</a>, <a href="../model_doc/funnel">Funnel Transformer</a>, <a href="../model_doc/gptj">GPT-J</a>, <a href="../model_doc/ibert">I-BERT</a>, <a href="../model_doc/layoutlmv2">LayoutLMv2</a>, <a href="../model_doc/layoutlmv3">LayoutLMv3</a>, <a href="../model_doc/led">LED</a>, <a href="../model_doc/lilt">LiLT</a>, <a href="../model_doc/longformer">Longformer</a>, <a href="../model_doc/luke">LUKE</a>, <a href="../model_doc/lxmert">LXMERT</a>, <a href="../model_doc/markuplm">MarkupLM</a>, <a href="../model_doc/mbart">mBART</a>, <a href="../model_doc/mega">MEGA</a>, <a href="../model_doc/megatron-bert">Megatron-BERT</a>, <a href="../model_doc/mobilebert">MobileBERT</a>, <a href="../model_doc/mpnet">MPNet</a>, <a href="../model_doc/mvp">MVP</a>, <a href="../model_doc/nezha">Nezha</a>, <a href="../model_doc/nystromformer">Nyströmformer</a>, <a href="../model_doc/opt">OPT</a>, <a href="../model_doc/qdqbert">QDQBert</a>, <a href="../model_doc/reformer">Reformer</a>, <a href="../model_doc/rembert">RemBERT</a>, <a href="../model_doc/roberta">RoBERTa</a>, <a href="../model_doc/roberta-prelayernorm">RoBERTa-PreLayerNorm</a>, <a href="../model_doc/roc_bert">RoCBert</a>, <a href="../model_doc/roformer">RoFormer</a>, <a href="../model_doc/splinter">Splinter</a>, <a href="../model_doc/squeezebert">SqueezeBERT</a>, <a href="../model_doc/xlm">XLM</a>, <a href="../model_doc/xlm-roberta">XLM-RoBERTa</a>, <a href="../model_doc/xlm-roberta-xl">XLM-RoBERTa-XL</a>, <a href="../model_doc/xlnet">XLNet</a>, <a href="../model_doc/xmod">X-MOD</a>, <a href="../model_doc/yoso">YOSO</a>';return{c(){a=Ns(`이 튜토리얼에서 설명하는 태스크는 다음과 같은 모델 아키텍처에서 지원됩니다.

`),c=M("p"),c.innerHTML=l},l(m){a=zs(m,`이 튜토리얼에서 설명하는 태스크는 다음과 같은 모델 아키텍처에서 지원됩니다.

`),c=h(m,"P",{"data-svelte-h":!0}),f(c)!=="svelte-pkvvxe"&&(c.innerHTML=l)},m(m,d){n(m,a,d),n(m,c,d)},p:me,d(m){m&&(t(a),t(c))}}}function Ps(I){let a,c;return a=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERlZmF1bHREYXRhQ29sbGF0b3IlMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGVmYXVsdERhdGFDb2xsYXRvcigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DefaultDataCollator

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DefaultDataCollator()`,wrap:!1}}),{c(){w(a.$$.fragment)},l(l){j(a.$$.fragment,l)},m(l,m){y(a,l,m),c=!0},p:me,i(l){c||(g(a.$$.fragment,l),c=!0)},o(l){J(a.$$.fragment,l),c=!1},d(l){T(a,l)}}}function Ds(I){let a,c;return a=new lt({props:{$$slots:{default:[Ps]},$$scope:{ctx:I}}}),{c(){w(a.$$.fragment)},l(l){j(a.$$.fragment,l)},m(l,m){y(a,l,m),c=!0},p(l,m){const d={};m&2&&(d.$$scope={dirty:m,ctx:l}),a.$set(d)},i(l){c||(g(a.$$.fragment,l),c=!0)},o(l){J(a.$$.fragment,l),c=!1},d(l){T(a,l)}}}function Ks(I){let a,c;return a=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERlZmF1bHREYXRhQ29sbGF0b3IlMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGVmYXVsdERhdGFDb2xsYXRvcihyZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DefaultDataCollator

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DefaultDataCollator(return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),{c(){w(a.$$.fragment)},l(l){j(a.$$.fragment,l)},m(l,m){y(a,l,m),c=!0},p:me,i(l){c||(g(a.$$.fragment,l),c=!0)},o(l){J(a.$$.fragment,l),c=!1},d(l){T(a,l)}}}function Os(I){let a,c;return a=new lt({props:{$$slots:{default:[Ks]},$$scope:{ctx:I}}}),{c(){w(a.$$.fragment)},l(l){j(a.$$.fragment,l)},m(l,m){y(a,l,m),c=!0},p(l,m){const d={};m&2&&(d.$$scope={dirty:m,ctx:l}),a.$set(d)},i(l){c||(g(a.$$.fragment,l),c=!0)},o(l){J(a.$$.fragment,l),c=!1},d(l){T(a,l)}}}function el(I){let a,c='<code>Trainer</code>를 이용해 모델을 미세 조정하는 것에 익숙하지 않다면, <a href="../training#train-with-pytorch-trainer">여기</a>에서 기초 튜토리얼을 살펴보세요!';return{c(){a=M("p"),a.innerHTML=c},l(l){a=h(l,"P",{"data-svelte-h":!0}),f(a)!=="svelte-1mkjug2"&&(a.innerHTML=c)},m(l,m){n(l,a,m)},p:me,d(l){l&&t(a)}}}function tl(I){let a,c,l,m="이제 모델 훈련을 시작할 준비가 되었습니다! <code>AutoModelForQuestionAnswering</code>으로 DistilBERT를 가져옵니다:",d,$,V,U,A="이제 세 단계만 남았습니다:",R,X,F="<li><code>TrainingArguments</code>에서 훈련 하이퍼파라미터를 정합니다. 꼭 필요한 매개변수는 모델을 저장할 위치를 지정하는 <code>output_dir</code> 입니다. <code>push_to_hub=True</code>로 설정해서 이 모델을 Hub로 푸시합니다 (모델을 업로드하려면 Hugging Face에 로그인해야 합니다).</li> <li>모델, 데이터 세트, 토크나이저, 데이터 콜레이터와 함께 <code>Trainer</code>에 훈련 인수들을 전달합니다.</li> <li><code>train()</code>을 호출해서 모델을 미세 조정합니다.</li>",v,_,k,Z,N="훈련이 완료되면, <code>push_to_hub()</code> 매소드를 사용해 모델을 Hub에 공유해서 모든 사람들이 사용할 수 있게 공유해주세요:",W,C,x;return a=new ts({props:{$$slots:{default:[el]},$$scope:{ctx:I}}}),$=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclF1ZXN0aW9uQW5zd2VyaW5nJTJDJTIwVHJhaW5pbmdBcmd1bWVudHMlMkMlMjBUcmFpbmVyJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JRdWVzdGlvbkFuc3dlcmluZy5mcm9tX3ByZXRyYWluZWQoJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForQuestionAnswering, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForQuestionAnswering.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`,wrap:!1}}),_=new B({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJteV9hd2Vzb21lX3FhX21vZGVsJTIyJTJDJTBBJTIwJTIwJTIwJTIwZXZhbHVhdGlvbl9zdHJhdGVneSUzRCUyMmVwb2NoJTIyJTJDJTBBJTIwJTIwJTIwJTIwbGVhcm5pbmdfcmF0ZSUzRDJlLTUlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfZXZhbF9iYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMyUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRDAuMDElMkMlMEElMjAlMjAlMjAlMjBwdXNoX3RvX2h1YiUzRFRydWUlMkMlMEEpJTBBJTBBdHJhaW5lciUyMCUzRCUyMFRyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRHRva2VuaXplZF9zcXVhZCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX3NxdWFkJTVCJTIydGVzdCUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMGRhdGFfY29sbGF0b3IlM0RkYXRhX2NvbGxhdG9yJTJDJTBBKSUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_squad[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_squad[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`,wrap:!1}}),C=new B({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()',wrap:!1}}),{c(){w(a.$$.fragment),c=r(),l=M("p"),l.innerHTML=m,d=r(),w($.$$.fragment),V=r(),U=M("p"),U.textContent=A,R=r(),X=M("ol"),X.innerHTML=F,v=r(),w(_.$$.fragment),k=r(),Z=M("p"),Z.innerHTML=N,W=r(),w(C.$$.fragment)},l(s){j(a.$$.fragment,s),c=o(s),l=h(s,"P",{"data-svelte-h":!0}),f(l)!=="svelte-1l92urq"&&(l.innerHTML=m),d=o(s),j($.$$.fragment,s),V=o(s),U=h(s,"P",{"data-svelte-h":!0}),f(U)!=="svelte-1vwg7jz"&&(U.textContent=A),R=o(s),X=h(s,"OL",{"data-svelte-h":!0}),f(X)!=="svelte-11eqy5p"&&(X.innerHTML=F),v=o(s),j(_.$$.fragment,s),k=o(s),Z=h(s,"P",{"data-svelte-h":!0}),f(Z)!=="svelte-1btg14f"&&(Z.innerHTML=N),W=o(s),j(C.$$.fragment,s)},m(s,u){y(a,s,u),n(s,c,u),n(s,l,u),n(s,d,u),y($,s,u),n(s,V,u),n(s,U,u),n(s,R,u),n(s,X,u),n(s,v,u),y(_,s,u),n(s,k,u),n(s,Z,u),n(s,W,u),y(C,s,u),x=!0},p(s,u){const H={};u&2&&(H.$$scope={dirty:u,ctx:s}),a.$set(H)},i(s){x||(g(a.$$.fragment,s),g($.$$.fragment,s),g(_.$$.fragment,s),g(C.$$.fragment,s),x=!0)},o(s){J(a.$$.fragment,s),J($.$$.fragment,s),J(_.$$.fragment,s),J(C.$$.fragment,s),x=!1},d(s){s&&(t(c),t(l),t(d),t(V),t(U),t(R),t(X),t(v),t(k),t(Z),t(W)),T(a,s),T($,s),T(_,s),T(C,s)}}}function sl(I){let a,c;return a=new lt({props:{$$slots:{default:[tl]},$$scope:{ctx:I}}}),{c(){w(a.$$.fragment)},l(l){j(a.$$.fragment,l)},m(l,m){y(a,l,m),c=!0},p(l,m){const d={};m&2&&(d.$$scope={dirty:m,ctx:l}),a.$set(d)},i(l){c||(g(a.$$.fragment,l),c=!0)},o(l){J(a.$$.fragment,l),c=!1},d(l){T(a,l)}}}function ll(I){let a,c='Keras로 모델을 미세 조정하는 것에 익숙하지 않다면, <a href="../training#train-a-tensorflow-model-with-keras">여기</a>에서 기초 튜토리얼을 살펴보세요!';return{c(){a=M("p"),a.innerHTML=c},l(l){a=h(l,"P",{"data-svelte-h":!0}),f(a)!=="svelte-1313y60"&&(a.innerHTML=c)},m(l,m){n(l,a,m)},p:me,d(l){l&&t(a)}}}function al(I){let a,c,l,m,d,$="그 다음 <code>TFAutoModelForQuestionAnswering</code>으로 DistilBERT를 가져옵니다:",V,U,A,R,X="<code>prepare_tf_dataset()</code>을 사용해서 데이터 세트를 <code>tf.data.Dataset</code> 형식으로 변환합니다:",F,v,_,k,Z='<a href="https://keras.io/api/models/model_training_apis/#compile-method" rel="nofollow"><code>compile</code></a>로 훈련할 모델을 설정합니다:',N,W,C,x,s="마지막으로 모델을 Hub로 푸시할 방법을 설정합니다. <code>PushToHubCallback</code>에서 모델과 토크나이저를 푸시할 경로를 설정합니다:",u,H,Me,q,L='드디어 모델 훈련을 시작할 준비가 되었습니다! 훈련 데이터 세트와 평가 데이터 세트, 에폭 수, 콜백을 설정한 후 <a href="https://keras.io/api/models/model_training_apis/#fit-method" rel="nofollow"><code>fit</code></a>을 이용해 모델을 미세 조정합니다:',he,Y,Q,S,P="훈련이 완료되면 모델이 자동으로 Hub에 업로드되어 누구나 사용할 수 있습니다!",ue;return a=new ts({props:{$$slots:{default:[ll]},$$scope:{ctx:I}}}),l=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEElMEFiYXRjaF9zaXplJTIwJTNEJTIwMTYlMEFudW1fZXBvY2hzJTIwJTNEJTIwMiUwQXRvdGFsX3RyYWluX3N0ZXBzJTIwJTNEJTIwKGxlbih0b2tlbml6ZWRfc3F1YWQlNUIlMjJ0cmFpbiUyMiU1RCklMjAlMkYlMkYlMjBiYXRjaF9zaXplKSUyMColMjBudW1fZXBvY2hzJTBBb3B0aW1pemVyJTJDJTIwc2NoZWR1bGUlMjAlM0QlMjBjcmVhdGVfb3B0aW1pemVyKCUwQSUyMCUyMCUyMCUyMGluaXRfbHIlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fc3RlcHMlM0R0b3RhbF90cmFpbl9zdGVwcyUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_epochs = <span class="hljs-number">2</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>total_train_steps = (<span class="hljs-built_in">len</span>(tokenized_squad[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>    num_train_steps=total_train_steps,
<span class="hljs-meta">... </span>)`,wrap:!1}}),U=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yUXVlc3Rpb25BbnN3ZXJpbmclMEElMEFtb2RlbCUyMCUzRCUyMFRGQXV0b01vZGVsRm9yUXVlc3Rpb25BbnN3ZXJpbmcoJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForQuestionAnswering

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForQuestionAnswering(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`,wrap:!1}}),v=new B({props:{code:"dGZfdHJhaW5fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF9zcXVhZCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwc2h1ZmZsZSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBjb2xsYXRlX2ZuJTNEZGF0YV9jb2xsYXRvciUyQyUwQSklMEElMEF0Zl92YWxpZGF0aW9uX3NldCUyMCUzRCUyMG1vZGVsLnByZXBhcmVfdGZfZGF0YXNldCglMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfc3F1YWQlNUIlMjJ0ZXN0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwc2h1ZmZsZSUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwY29sbGF0ZV9mbiUzRGRhdGFfY29sbGF0b3IlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_squad[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_squad[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`,wrap:!1}}),W=new B({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplcik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`,wrap:!1}}),H=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBQdXNoVG9IdWJDYWxsYmFjayUwQSUwQWNhbGxiYWNrJTIwJTNEJTIwUHVzaFRvSHViQ2FsbGJhY2soJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm15X2F3ZXNvbWVfcWFfbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>callback = PushToHubCallback(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>)`,wrap:!1}}),Y=new B({props:{code:"bW9kZWwuZml0KHglM0R0Zl90cmFpbl9zZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl92YWxpZGF0aW9uX3NldCUyQyUyMGVwb2NocyUzRDMlMkMlMjBjYWxsYmFja3MlM0QlNUJjYWxsYmFjayU1RCk=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>, callbacks=[callback])',wrap:!1}}),{c(){w(a.$$.fragment),c=Ns(`
TensorFlow를 이용한 모델을 미세 조정하려면 옵티마이저 함수, 학습률 스케쥴 및 몇 가지 훈련 하이퍼파라미터를 설정하는 것부터 시작해야합니다:

	`),w(l.$$.fragment),m=r(),d=M("p"),d.innerHTML=$,V=r(),w(U.$$.fragment),A=r(),R=M("p"),R.innerHTML=X,F=r(),w(v.$$.fragment),_=r(),k=M("p"),k.innerHTML=Z,N=r(),w(W.$$.fragment),C=r(),x=M("p"),x.innerHTML=s,u=r(),w(H.$$.fragment),Me=r(),q=M("p"),q.innerHTML=L,he=r(),w(Y.$$.fragment),Q=r(),S=M("p"),S.textContent=P},l(i){j(a.$$.fragment,i),c=zs(i,`
TensorFlow를 이용한 모델을 미세 조정하려면 옵티마이저 함수, 학습률 스케쥴 및 몇 가지 훈련 하이퍼파라미터를 설정하는 것부터 시작해야합니다:

	`),j(l.$$.fragment,i),m=o(i),d=h(i,"P",{"data-svelte-h":!0}),f(d)!=="svelte-1bkfga3"&&(d.innerHTML=$),V=o(i),j(U.$$.fragment,i),A=o(i),R=h(i,"P",{"data-svelte-h":!0}),f(R)!=="svelte-1nof74s"&&(R.innerHTML=X),F=o(i),j(v.$$.fragment,i),_=o(i),k=h(i,"P",{"data-svelte-h":!0}),f(k)!=="svelte-1ieukoa"&&(k.innerHTML=Z),N=o(i),j(W.$$.fragment,i),C=o(i),x=h(i,"P",{"data-svelte-h":!0}),f(x)!=="svelte-ivpgxr"&&(x.innerHTML=s),u=o(i),j(H.$$.fragment,i),Me=o(i),q=h(i,"P",{"data-svelte-h":!0}),f(q)!=="svelte-1fld86y"&&(q.innerHTML=L),he=o(i),j(Y.$$.fragment,i),Q=o(i),S=h(i,"P",{"data-svelte-h":!0}),f(S)!=="svelte-ga84hg"&&(S.textContent=P)},m(i,b){y(a,i,b),n(i,c,b),y(l,i,b),n(i,m,b),n(i,d,b),n(i,V,b),y(U,i,b),n(i,A,b),n(i,R,b),n(i,F,b),y(v,i,b),n(i,_,b),n(i,k,b),n(i,N,b),y(W,i,b),n(i,C,b),n(i,x,b),n(i,u,b),y(H,i,b),n(i,Me,b),n(i,q,b),n(i,he,b),y(Y,i,b),n(i,Q,b),n(i,S,b),ue=!0},p(i,b){const de={};b&2&&(de.$$scope={dirty:b,ctx:i}),a.$set(de)},i(i){ue||(g(a.$$.fragment,i),g(l.$$.fragment,i),g(U.$$.fragment,i),g(v.$$.fragment,i),g(W.$$.fragment,i),g(H.$$.fragment,i),g(Y.$$.fragment,i),ue=!0)},o(i){J(a.$$.fragment,i),J(l.$$.fragment,i),J(U.$$.fragment,i),J(v.$$.fragment,i),J(W.$$.fragment,i),J(H.$$.fragment,i),J(Y.$$.fragment,i),ue=!1},d(i){i&&(t(c),t(m),t(d),t(V),t(A),t(R),t(F),t(_),t(k),t(N),t(C),t(x),t(u),t(Me),t(q),t(he),t(Q),t(S)),T(a,i),T(l,i),T(U,i),T(v,i),T(W,i),T(H,i),T(Y,i)}}}function nl(I){let a,c;return a=new lt({props:{$$slots:{default:[al]},$$scope:{ctx:I}}}),{c(){w(a.$$.fragment)},l(l){j(a.$$.fragment,l)},m(l,m){y(a,l,m),c=!0},p(l,m){const d={};m&2&&(d.$$scope={dirty:m,ctx:l}),a.$set(d)},i(l){c||(g(a.$$.fragment,l),c=!0)},o(l){J(a.$$.fragment,l),c=!1},d(l){T(a,l)}}}function pl(I){let a,c='질의 응답을 위해 모델을 미세 조정하는 방법에 대한 더 자세한 예시는 <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering.ipynb" rel="nofollow">PyTorch notebook</a> 또는 <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering-tf.ipynb" rel="nofollow">TensorFlow notebook</a>을 참조하세요.';return{c(){a=M("p"),a.innerHTML=c},l(l){a=h(l,"P",{"data-svelte-h":!0}),f(a)!=="svelte-1ikgkjc"&&(a.innerHTML=c)},m(l,m){n(l,a,m)},p:me,d(l){l&&t(a)}}}function rl(I){let a,c="텍스트를 토큰화해서 PyTorch 텐서를 반환합니다:",l,m,d,$,V="모델에 입력을 전달하고 <code>logits</code>을 반환합니다:",U,A,R,X,F="모델의 출력에서 시작 및 종료 위치가 어딘지 가장 높은 확률을 얻습니다:",v,_,k,Z,N="예측된 토큰을 해독해서 답을 얻습니다:",W,C,x;return m=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJteV9hd2Vzb21lX3FhX21vZGVsJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihxdWVzdGlvbiUyQyUyMGNvbnRleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(question, context, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)`,wrap:!1}}),A=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclF1ZXN0aW9uQW5zd2VyaW5nJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JRdWVzdGlvbkFuc3dlcmluZy5mcm9tX3ByZXRyYWluZWQoJTIybXlfYXdlc29tZV9xYV9tb2RlbCUyMiklMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForQuestionAnswering

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForQuestionAnswering.from_pretrained(<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    outputs = model(**inputs)`,wrap:!1}}),_=new B({props:{code:"YW5zd2VyX3N0YXJ0X2luZGV4JTIwJTNEJTIwb3V0cHV0cy5zdGFydF9sb2dpdHMuYXJnbWF4KCklMEFhbnN3ZXJfZW5kX2luZGV4JTIwJTNEJTIwb3V0cHV0cy5lbmRfbG9naXRzLmFyZ21heCgp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>answer_start_index = outputs.start_logits.argmax()
<span class="hljs-meta">&gt;&gt;&gt; </span>answer_end_index = outputs.end_logits.argmax()`,wrap:!1}}),C=new B({props:{code:"cHJlZGljdF9hbnN3ZXJfdG9rZW5zJTIwJTNEJTIwaW5wdXRzLmlucHV0X2lkcyU1QjAlMkMlMjBhbnN3ZXJfc3RhcnRfaW5kZXglMjAlM0ElMjBhbnN3ZXJfZW5kX2luZGV4JTIwJTJCJTIwMSU1RCUwQXRva2VuaXplci5kZWNvZGUocHJlZGljdF9hbnN3ZXJfdG9rZW5zKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predict_answer_tokens = inputs.input_ids[<span class="hljs-number">0</span>, answer_start_index : answer_end_index + <span class="hljs-number">1</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(predict_answer_tokens)
<span class="hljs-string">&#x27;176 billion parameters and can generate text in 46 languages natural languages and 13&#x27;</span>`,wrap:!1}}),{c(){a=M("p"),a.textContent=c,l=r(),w(m.$$.fragment),d=r(),$=M("p"),$.innerHTML=V,U=r(),w(A.$$.fragment),R=r(),X=M("p"),X.textContent=F,v=r(),w(_.$$.fragment),k=r(),Z=M("p"),Z.textContent=N,W=r(),w(C.$$.fragment)},l(s){a=h(s,"P",{"data-svelte-h":!0}),f(a)!=="svelte-hm1s3x"&&(a.textContent=c),l=o(s),j(m.$$.fragment,s),d=o(s),$=h(s,"P",{"data-svelte-h":!0}),f($)!=="svelte-kli2cc"&&($.innerHTML=V),U=o(s),j(A.$$.fragment,s),R=o(s),X=h(s,"P",{"data-svelte-h":!0}),f(X)!=="svelte-1cy5gzu"&&(X.textContent=F),v=o(s),j(_.$$.fragment,s),k=o(s),Z=h(s,"P",{"data-svelte-h":!0}),f(Z)!=="svelte-1hhfukj"&&(Z.textContent=N),W=o(s),j(C.$$.fragment,s)},m(s,u){n(s,a,u),n(s,l,u),y(m,s,u),n(s,d,u),n(s,$,u),n(s,U,u),y(A,s,u),n(s,R,u),n(s,X,u),n(s,v,u),y(_,s,u),n(s,k,u),n(s,Z,u),n(s,W,u),y(C,s,u),x=!0},p:me,i(s){x||(g(m.$$.fragment,s),g(A.$$.fragment,s),g(_.$$.fragment,s),g(C.$$.fragment,s),x=!0)},o(s){J(m.$$.fragment,s),J(A.$$.fragment,s),J(_.$$.fragment,s),J(C.$$.fragment,s),x=!1},d(s){s&&(t(a),t(l),t(d),t($),t(U),t(R),t(X),t(v),t(k),t(Z),t(W)),T(m,s),T(A,s),T(_,s),T(C,s)}}}function ol(I){let a,c;return a=new lt({props:{$$slots:{default:[rl]},$$scope:{ctx:I}}}),{c(){w(a.$$.fragment)},l(l){j(a.$$.fragment,l)},m(l,m){y(a,l,m),c=!0},p(l,m){const d={};m&2&&(d.$$scope={dirty:m,ctx:l}),a.$set(d)},i(l){c||(g(a.$$.fragment,l),c=!0)},o(l){J(a.$$.fragment,l),c=!1},d(l){T(a,l)}}}function il(I){let a,c="텍스트를 토큰화해서 TensorFlow 텐서를 반환합니다:",l,m,d,$,V="모델에 입력을 전달하고 <code>logits</code>을 반환합니다:",U,A,R,X,F="모델의 출력에서 시작 및 종료 위치가 어딘지 가장 높은 확률을 얻습니다:",v,_,k,Z,N="예측된 토큰을 해독해서 답을 얻습니다:",W,C,x;return m=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJteV9hd2Vzb21lX3FhX21vZGVsJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihxdWVzdGlvbiUyQyUyMHRleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(question, text, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),A=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yUXVlc3Rpb25BbnN3ZXJpbmclMEElMEFtb2RlbCUyMCUzRCUyMFRGQXV0b01vZGVsRm9yUXVlc3Rpb25BbnN3ZXJpbmcuZnJvbV9wcmV0cmFpbmVkKCUyMm15X2F3ZXNvbWVfcWFfbW9kZWwlMjIpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForQuestionAnswering

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForQuestionAnswering.from_pretrained(<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(**inputs)`,wrap:!1}}),_=new B({props:{code:"YW5zd2VyX3N0YXJ0X2luZGV4JTIwJTNEJTIwaW50KHRmLm1hdGguYXJnbWF4KG91dHB1dHMuc3RhcnRfbG9naXRzJTJDJTIwYXhpcyUzRC0xKSU1QjAlNUQpJTBBYW5zd2VyX2VuZF9pbmRleCUyMCUzRCUyMGludCh0Zi5tYXRoLmFyZ21heChvdXRwdXRzLmVuZF9sb2dpdHMlMkMlMjBheGlzJTNELTEpJTVCMCU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>answer_start_index = <span class="hljs-built_in">int</span>(tf.math.argmax(outputs.start_logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>answer_end_index = <span class="hljs-built_in">int</span>(tf.math.argmax(outputs.end_logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>])`,wrap:!1}}),C=new B({props:{code:"cHJlZGljdF9hbnN3ZXJfdG9rZW5zJTIwJTNEJTIwaW5wdXRzLmlucHV0X2lkcyU1QjAlMkMlMjBhbnN3ZXJfc3RhcnRfaW5kZXglMjAlM0ElMjBhbnN3ZXJfZW5kX2luZGV4JTIwJTJCJTIwMSU1RCUwQXRva2VuaXplci5kZWNvZGUocHJlZGljdF9hbnN3ZXJfdG9rZW5zKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predict_answer_tokens = inputs.input_ids[<span class="hljs-number">0</span>, answer_start_index : answer_end_index + <span class="hljs-number">1</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(predict_answer_tokens)
<span class="hljs-string">&#x27;176 billion parameters and can generate text in 46 languages natural languages and 13&#x27;</span>`,wrap:!1}}),{c(){a=M("p"),a.textContent=c,l=r(),w(m.$$.fragment),d=r(),$=M("p"),$.innerHTML=V,U=r(),w(A.$$.fragment),R=r(),X=M("p"),X.textContent=F,v=r(),w(_.$$.fragment),k=r(),Z=M("p"),Z.textContent=N,W=r(),w(C.$$.fragment)},l(s){a=h(s,"P",{"data-svelte-h":!0}),f(a)!=="svelte-e85p7z"&&(a.textContent=c),l=o(s),j(m.$$.fragment,s),d=o(s),$=h(s,"P",{"data-svelte-h":!0}),f($)!=="svelte-kli2cc"&&($.innerHTML=V),U=o(s),j(A.$$.fragment,s),R=o(s),X=h(s,"P",{"data-svelte-h":!0}),f(X)!=="svelte-1cy5gzu"&&(X.textContent=F),v=o(s),j(_.$$.fragment,s),k=o(s),Z=h(s,"P",{"data-svelte-h":!0}),f(Z)!=="svelte-1hhfukj"&&(Z.textContent=N),W=o(s),j(C.$$.fragment,s)},m(s,u){n(s,a,u),n(s,l,u),y(m,s,u),n(s,d,u),n(s,$,u),n(s,U,u),y(A,s,u),n(s,R,u),n(s,X,u),n(s,v,u),y(_,s,u),n(s,k,u),n(s,Z,u),n(s,W,u),y(C,s,u),x=!0},p:me,i(s){x||(g(m.$$.fragment,s),g(A.$$.fragment,s),g(_.$$.fragment,s),g(C.$$.fragment,s),x=!0)},o(s){J(m.$$.fragment,s),J(A.$$.fragment,s),J(_.$$.fragment,s),J(C.$$.fragment,s),x=!1},d(s){s&&(t(a),t(l),t(d),t($),t(U),t(R),t(X),t(v),t(k),t(Z),t(W)),T(m,s),T(A,s),T(_,s),T(C,s)}}}function cl(I){let a,c;return a=new lt({props:{$$slots:{default:[il]},$$scope:{ctx:I}}}),{c(){w(a.$$.fragment)},l(l){j(a.$$.fragment,l)},m(l,m){y(a,l,m),c=!0},p(l,m){const d={};m&2&&(d.$$scope={dirty:m,ctx:l}),a.$set(d)},i(l){c||(g(a.$$.fragment,l),c=!0)},o(l){J(a.$$.fragment,l),c=!1},d(l){T(a,l)}}}function ml(I){let a,c,l,m,d,$,V,U,A,R,X="질의 응답(Question Answering)",F,v,_,k,Z,N,W="질의 응답 태스크는 주어진 질문에 대한 답변을 제공합니다. Alexa, Siri 또는 Google과 같은 가상 비서에게 날씨가 어떤지 물어본 적이 있다면 질의 응답 모델을 사용해본 적이 있을 것입니다. 질의 응답 태스크에는 일반적으로 두 가지 유형이 있습니다.",C,x,s="<li>추출적(Extractive) 질의 응답: 주어진 문맥에서 답변을 추출합니다.</li> <li>생성적(Abstractive) 질의 응답: 문맥에서 질문에 올바르게 답하는 답변을 생성합니다.</li>",u,H,Me="이 가이드는 다음과 같은 방법들을 보여줍니다.",q,L,he='<li>추출적 질의 응답을 하기 위해 <a href="https://huggingface.co/datasets/squad" rel="nofollow">SQuAD</a> 데이터 세트에서 <a href="https://huggingface.co/distilbert-base-uncased" rel="nofollow">DistilBERT</a> 미세 조정하기</li> <li>추론에 미세 조정된 모델 사용하기</li>',Y,Q,S,P,ue="시작하기 전에, 필요한 라이브러리가 모두 설치되어 있는지 확인하세요:",i,b,de,fe,os="여러분의 모델을 업로드하고 커뮤니티에 공유할 수 있도록 Hugging Face 계정에 로그인하는 것이 좋습니다. 메시지가 표시되면 토큰을 입력해서 로그인합니다:",ht,we,ut,D,se,at,je,ss,De,is="SQuAD 데이터 세트 가져오기",dt,ye,cs="먼저 🤗 Datasets 라이브러리에서 SQuAD 데이터 세트의 일부를 가져옵니다. 이렇게 하면 전체 데이터 세트로 훈련하며 더 많은 시간을 할애하기 전에 모든 것이 잘 작동하는지 실험하고 확인할 수 있습니다.",ft,ge,wt,Je,ms="데이터 세트의 분할된 <code>train</code>을 <code>train_test_split</code> 메소드를 사용해 훈련 데이터 세트와 테스트 데이터 세트로 나누어줍니다:",jt,Te,yt,be,Ms="그리고나서 예시로 데이터를 하나 살펴봅니다:",gt,$e,Jt,Ue,hs="이 중에서 몇 가지 중요한 항목이 있습니다:",Tt,_e,us="<li><code>answers</code>: 답안 토큰의 시작 위치와 답안 텍스트</li> <li><code>context</code>: 모델이 답을 추출하는데 필요한 배경 지식</li> <li><code>question</code>: 모델이 답해야 하는 질문</li>",bt,K,le,nt,Ce,ls,Ke,ds="전처리",$t,Ie,Ut,Ze,fs="다음 단계에서는 <code>question</code> 및 <code>context</code> 항목을 처리하기 위해 DistilBERT 토크나이저를 가져옵니다:",_t,ve,Ct,ke,ws="질의 응답 태스크와 관련해서 특히 유의해야할 몇 가지 전처리 단계가 있습니다:",It,xe,js="<li>데이터 세트의 일부 예제에는 모델의 최대 입력 길이를 초과하는 매우 긴 <code>context</code>가 있을 수 있습니다. 긴 시퀀스를 다루기 위해서는, <code>truncation=&quot;only_second&quot;</code>로 설정해 <code>context</code>만 잘라내면 됩니다.</li> <li>그 다음, <code>return_offset_mapping=True</code>로 설정해 답변의 시작과 종료 위치를 원래의 <code>context</code>에 매핑합니다.</li> <li>매핑을 완료하면, 이제 답변에서 시작 토큰과 종료 토큰을 찾을 수 있습니다. 오프셋의 어느 부분이 <code>question</code>과 <code>context</code>에 해당하는지 찾을 수 있도록 <code>sequence_ids</code> 메소드를 사용하세요.</li>",Zt,Re,ys="다음은 <code>answer</code>의 시작 토큰과 종료 토큰을 잘라내서 <code>context</code>에 매핑하는 함수를 만드는 방법입니다:",vt,Xe,kt,Be,gs="모든 데이터 세트에 전처리를 적용하려면, 🤗 Datasets <code>map</code> 함수를 사용하세요. <code>batched=True</code>로 설정해 데이터 세트의 여러 요소들을 한 번에 처리하면 <code>map</code> 함수의 속도를 빠르게 할 수 있습니다. 필요하지 않은 열은 모두 제거합니다:",xt,Ae,Rt,Ge,Js="이제 <code>DefaultDataCollator</code>를 이용해 예시 배치를 생성합니다. 🤗 Transformers의 다른 데이터 콜레이터(data collator)와 달리, <code>DefaultDataCollator</code>는 패딩과 같은 추가 전처리를 적용하지 않습니다:",Xt,ae,Bt,O,ne,pt,We,as,Oe,Ts="훈련",At,pe,Gt,re,Wt,ee,oe,rt,Ee,ns,et,bs="평가",Et,Ve,$s="질의 응답을 평가하려면 상당한 양의 후처리가 필요합니다. 시간이 너무 많이 걸리지 않도록 이 가이드에서는 평가 단계를 생략합니다. <code>Trainer</code>는 훈련 과정에서 평가 손실(evaluation loss)을 계속 계산하기 때문에 모델의 성능을 대략적으로 알 수 있습니다.",Vt,Ne,Us='시간에 여유가 있고 질의 응답 모델을 평가하는 방법에 관심이 있다면 🤗 Hugging Face Course의 <a href="https://huggingface.co/course/chapter7/7?fw=pt#postprocessing" rel="nofollow">Question answering</a> 챕터를 살펴보세요!',Nt,te,ie,ot,ze,ps,tt,_s="추론",zt,Fe,Cs="이제 모델을 미세 조정했으니 추론에 사용할 수 있습니다!",Ft,He,Is="질문과 모델이 예측하기 원하는 문맥(context)를 생각해보세요:",Ht,Qe,Qt,qe,Zs="추론을 위해 미세 조정한 모델을 테스트하는 가장 쉬운 방법은 <code>pipeline()</code>을 사용하는 것 입니다. 모델을 사용해 질의 응답을 하기 위해서 <code>pipeline</code>을 인스턴스화하고 텍스트를 입력합니다:",qt,Ye,Yt,Se,vs="원한다면 <code>pipeline</code>의 결과를 직접 복제할 수도 있습니다:",St,ce,Lt,it,Pt;return U=new st({}),v=new Ss({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/question_answering.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/pytorch/question_answering.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/tensorflow/question_answering.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/question_answering.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/pytorch/question_answering.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/tensorflow/question_answering.ipynb"}]}}),k=new Vs({props:{id:"ajPx5LwJD-I"}}),Q=new ts({props:{$$slots:{default:[Ls]},$$scope:{ctx:I}}}),b=new B({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwZXZhbHVhdGU=",highlighted:"pip install transformers datasets evaluate",wrap:!1}}),we=new B({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_login()`,wrap:!1}}),je=new st({}),ge=new B({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBc3F1YWQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyc3F1YWQlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTVCJTNBNTAwMCU1RCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>squad = load_dataset(<span class="hljs-string">&quot;squad&quot;</span>, split=<span class="hljs-string">&quot;train[:5000]&quot;</span>)`,wrap:!1}}),Te=new B({props:{code:"c3F1YWQlMjAlM0QlMjBzcXVhZC50cmFpbl90ZXN0X3NwbGl0KHRlc3Rfc2l6ZSUzRDAuMik=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>squad = squad.train_test_split(test_size=<span class="hljs-number">0.2</span>)',wrap:!1}}),$e=new B({props:{code:"c3F1YWQlNUIlMjJ0cmFpbiUyMiU1RCU1QjAlNUQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>squad[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers&#x27;</span>: {<span class="hljs-string">&#x27;answer_start&#x27;</span>: [<span class="hljs-number">515</span>], <span class="hljs-string">&#x27;text&#x27;</span>: [<span class="hljs-string">&#x27;Saint Bernadette Soubirous&#x27;</span>]},
 <span class="hljs-string">&#x27;context&#x27;</span>: <span class="hljs-string">&#x27;Architecturally, the school has a Catholic character. Atop the Main Building\\&#x27;s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend &quot;Venite Ad Me Omnes&quot;. Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.&#x27;</span>,
 <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;5733be284776f41900661182&#x27;</span>,
 <span class="hljs-string">&#x27;question&#x27;</span>: <span class="hljs-string">&#x27;To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;University_of_Notre_Dame&#x27;</span>
}`,wrap:!1}}),Ce=new st({}),Ie=new Vs({props:{id:"qgaM0weJHpA"}}),ve=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`,wrap:!1}}),Xe=new B({props:{code:"ZGVmJTIwcHJlcHJvY2Vzc19mdW5jdGlvbihleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjBxdWVzdGlvbnMlMjAlM0QlMjAlNUJxLnN0cmlwKCklMjBmb3IlMjBxJTIwaW4lMjBleGFtcGxlcyU1QiUyMnF1ZXN0aW9uJTIyJTVEJTVEJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXN0aW9ucyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV4YW1wbGVzJTVCJTIyY29udGV4dCUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1heF9sZW5ndGglM0QzODQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cnVuY2F0aW9uJTNEJTIyb25seV9zZWNvbmQlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm5fb2Zmc2V0c19tYXBwaW5nJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBhZGRpbmclM0QlMjJtYXhfbGVuZ3RoJTIyJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMG9mZnNldF9tYXBwaW5nJTIwJTNEJTIwaW5wdXRzLnBvcCglMjJvZmZzZXRfbWFwcGluZyUyMiklMEElMjAlMjAlMjAlMjBhbnN3ZXJzJTIwJTNEJTIwZXhhbXBsZXMlNUIlMjJhbnN3ZXJzJTIyJTVEJTBBJTIwJTIwJTIwJTIwc3RhcnRfcG9zaXRpb25zJTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwZW5kX3Bvc2l0aW9ucyUyMCUzRCUyMCU1QiU1RCUwQSUwQSUyMCUyMCUyMCUyMGZvciUyMGklMkMlMjBvZmZzZXQlMjBpbiUyMGVudW1lcmF0ZShvZmZzZXRfbWFwcGluZyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXIlMjAlM0QlMjBhbnN3ZXJzJTVCaSU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0YXJ0X2NoYXIlMjAlM0QlMjBhbnN3ZXIlNUIlMjJhbnN3ZXJfc3RhcnQlMjIlNUQlNUIwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5kX2NoYXIlMjAlM0QlMjBhbnN3ZXIlNUIlMjJhbnN3ZXJfc3RhcnQlMjIlNUQlNUIwJTVEJTIwJTJCJTIwbGVuKGFuc3dlciU1QiUyMnRleHQlMjIlNUQlNUIwJTVEKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlcXVlbmNlX2lkcyUyMCUzRCUyMGlucHV0cy5zZXF1ZW5jZV9pZHMoaSklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBGaW5kJTIwdGhlJTIwc3RhcnQlMjBhbmQlMjBlbmQlMjBvZiUyMHRoZSUyMGNvbnRleHQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZHglMjAlM0QlMjAwJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2hpbGUlMjBzZXF1ZW5jZV9pZHMlNUJpZHglNUQlMjAhJTNEJTIwMSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlkeCUyMCUyQiUzRCUyMDElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb250ZXh0X3N0YXJ0JTIwJTNEJTIwaWR4JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2hpbGUlMjBzZXF1ZW5jZV9pZHMlNUJpZHglNUQlMjAlM0QlM0QlMjAxJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWR4JTIwJTJCJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbnRleHRfZW5kJTIwJTNEJTIwaWR4JTIwLSUyMDElMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBJZiUyMHRoZSUyMGFuc3dlciUyMGlzJTIwbm90JTIwZnVsbHklMjBpbnNpZGUlMjB0aGUlMjBjb250ZXh0JTJDJTIwbGFiZWwlMjBpdCUyMCgwJTJDJTIwMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMG9mZnNldCU1QmNvbnRleHRfc3RhcnQlNUQlNUIwJTVEJTIwJTNFJTIwZW5kX2NoYXIlMjBvciUyMG9mZnNldCU1QmNvbnRleHRfZW5kJTVEJTVCMSU1RCUyMCUzQyUyMHN0YXJ0X2NoYXIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdGFydF9wb3NpdGlvbnMuYXBwZW5kKDApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5kX3Bvc2l0aW9ucy5hcHBlbmQoMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbHNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwT3RoZXJ3aXNlJTIwaXQncyUyMHRoZSUyMHN0YXJ0JTIwYW5kJTIwZW5kJTIwdG9rZW4lMjBwb3NpdGlvbnMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZHglMjAlM0QlMjBjb250ZXh0X3N0YXJ0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2hpbGUlMjBpZHglMjAlM0MlM0QlMjBjb250ZXh0X2VuZCUyMGFuZCUyMG9mZnNldCU1QmlkeCU1RCU1QjAlNUQlMjAlM0MlM0QlMjBzdGFydF9jaGFyJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWR4JTIwJTJCJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0YXJ0X3Bvc2l0aW9ucy5hcHBlbmQoaWR4JTIwLSUyMDEpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWR4JTIwJTNEJTIwY29udGV4dF9lbmQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3aGlsZSUyMGlkeCUyMCUzRSUzRCUyMGNvbnRleHRfc3RhcnQlMjBhbmQlMjBvZmZzZXQlNUJpZHglNUQlNUIxJTVEJTIwJTNFJTNEJTIwZW5kX2NoYXIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZHglMjAtJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVuZF9wb3NpdGlvbnMuYXBwZW5kKGlkeCUyMCUyQiUyMDEpJTBBJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTVCJTIyc3RhcnRfcG9zaXRpb25zJTIyJTVEJTIwJTNEJTIwc3RhcnRfcG9zaXRpb25zJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTVCJTIyZW5kX3Bvc2l0aW9ucyUyMiU1RCUyMCUzRCUyMGVuZF9wb3NpdGlvbnMlMEElMjAlMjAlMjAlMjByZXR1cm4lMjBpbnB1dHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    questions = [q.strip() <span class="hljs-keyword">for</span> q <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;question&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = tokenizer(
<span class="hljs-meta">... </span>        questions,
<span class="hljs-meta">... </span>        examples[<span class="hljs-string">&quot;context&quot;</span>],
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">384</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-string">&quot;only_second&quot;</span>,
<span class="hljs-meta">... </span>        return_offsets_mapping=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-string">&quot;max_length&quot;</span>,
<span class="hljs-meta">... </span>    )

<span class="hljs-meta">... </span>    offset_mapping = inputs.pop(<span class="hljs-string">&quot;offset_mapping&quot;</span>)
<span class="hljs-meta">... </span>    answers = examples[<span class="hljs-string">&quot;answers&quot;</span>]
<span class="hljs-meta">... </span>    start_positions = []
<span class="hljs-meta">... </span>    end_positions = []

<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, offset <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(offset_mapping):
<span class="hljs-meta">... </span>        answer = answers[i]
<span class="hljs-meta">... </span>        start_char = answer[<span class="hljs-string">&quot;answer_start&quot;</span>][<span class="hljs-number">0</span>]
<span class="hljs-meta">... </span>        end_char = answer[<span class="hljs-string">&quot;answer_start&quot;</span>][<span class="hljs-number">0</span>] + <span class="hljs-built_in">len</span>(answer[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
<span class="hljs-meta">... </span>        sequence_ids = inputs.sequence_ids(i)

<span class="hljs-meta">... </span>        <span class="hljs-comment"># Find the start and end of the context</span>
<span class="hljs-meta">... </span>        idx = <span class="hljs-number">0</span>
<span class="hljs-meta">... </span>        <span class="hljs-keyword">while</span> sequence_ids[idx] != <span class="hljs-number">1</span>:
<span class="hljs-meta">... </span>            idx += <span class="hljs-number">1</span>
<span class="hljs-meta">... </span>        context_start = idx
<span class="hljs-meta">... </span>        <span class="hljs-keyword">while</span> sequence_ids[idx] == <span class="hljs-number">1</span>:
<span class="hljs-meta">... </span>            idx += <span class="hljs-number">1</span>
<span class="hljs-meta">... </span>        context_end = idx - <span class="hljs-number">1</span>

<span class="hljs-meta">... </span>        <span class="hljs-comment"># If the answer is not fully inside the context, label it (0, 0)</span>
<span class="hljs-meta">... </span>        <span class="hljs-keyword">if</span> offset[context_start][<span class="hljs-number">0</span>] &gt; end_char <span class="hljs-keyword">or</span> offset[context_end][<span class="hljs-number">1</span>] &lt; start_char:
<span class="hljs-meta">... </span>            start_positions.append(<span class="hljs-number">0</span>)
<span class="hljs-meta">... </span>            end_positions.append(<span class="hljs-number">0</span>)
<span class="hljs-meta">... </span>        <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>            <span class="hljs-comment"># Otherwise it&#x27;s the start and end token positions</span>
<span class="hljs-meta">... </span>            idx = context_start
<span class="hljs-meta">... </span>            <span class="hljs-keyword">while</span> idx &lt;= context_end <span class="hljs-keyword">and</span> offset[idx][<span class="hljs-number">0</span>] &lt;= start_char:
<span class="hljs-meta">... </span>                idx += <span class="hljs-number">1</span>
<span class="hljs-meta">... </span>            start_positions.append(idx - <span class="hljs-number">1</span>)

<span class="hljs-meta">... </span>            idx = context_end
<span class="hljs-meta">... </span>            <span class="hljs-keyword">while</span> idx &gt;= context_start <span class="hljs-keyword">and</span> offset[idx][<span class="hljs-number">1</span>] &gt;= end_char:
<span class="hljs-meta">... </span>                idx -= <span class="hljs-number">1</span>
<span class="hljs-meta">... </span>            end_positions.append(idx + <span class="hljs-number">1</span>)

<span class="hljs-meta">... </span>    inputs[<span class="hljs-string">&quot;start_positions&quot;</span>] = start_positions
<span class="hljs-meta">... </span>    inputs[<span class="hljs-string">&quot;end_positions&quot;</span>] = end_positions
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`,wrap:!1}}),Ae=new B({props:{code:"dG9rZW5pemVkX3NxdWFkJTIwJTNEJTIwc3F1YWQubWFwKHByZXByb2Nlc3NfZnVuY3Rpb24lMkMlMjBiYXRjaGVkJTNEVHJ1ZSUyQyUyMHJlbW92ZV9jb2x1bW5zJTNEc3F1YWQlNUIlMjJ0cmFpbiUyMiU1RC5jb2x1bW5fbmFtZXMp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_squad = squad.<span class="hljs-built_in">map</span>(preprocess_function, batched=<span class="hljs-literal">True</span>, remove_columns=squad[<span class="hljs-string">&quot;train&quot;</span>].column_names)',wrap:!1}}),ae=new rs({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Os],pytorch:[Ds]},$$scope:{ctx:I}}}),We=new st({}),pe=new rs({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[nl],pytorch:[sl]},$$scope:{ctx:I}}}),re=new ts({props:{$$slots:{default:[pl]},$$scope:{ctx:I}}}),Ee=new st({}),ze=new st({}),Qe=new B({props:{code:"cXVlc3Rpb24lMjAlM0QlMjAlMjJIb3clMjBtYW55JTIwcHJvZ3JhbW1pbmclMjBsYW5ndWFnZXMlMjBkb2VzJTIwQkxPT00lMjBzdXBwb3J0JTNGJTIyJTBBY29udGV4dCUyMCUzRCUyMCUyMkJMT09NJTIwaGFzJTIwMTc2JTIwYmlsbGlvbiUyMHBhcmFtZXRlcnMlMjBhbmQlMjBjYW4lMjBnZW5lcmF0ZSUyMHRleHQlMjBpbiUyMDQ2JTIwbGFuZ3VhZ2VzJTIwbmF0dXJhbCUyMGxhbmd1YWdlcyUyMGFuZCUyMDEzJTIwcHJvZ3JhbW1pbmclMjBsYW5ndWFnZXMuJTIy",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>question = <span class="hljs-string">&quot;How many programming languages does BLOOM support?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>context = <span class="hljs-string">&quot;BLOOM has 176 billion parameters and can generate text in 46 languages natural languages and 13 programming languages.&quot;</span>`,wrap:!1}}),Ye=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBcXVlc3Rpb25fYW5zd2VyZXIlMjAlM0QlMjBwaXBlbGluZSglMjJxdWVzdGlvbi1hbnN3ZXJpbmclMjIlMkMlMjBtb2RlbCUzRCUyMm15X2F3ZXNvbWVfcWFfbW9kZWwlMjIpJTBBcXVlc3Rpb25fYW5zd2VyZXIocXVlc3Rpb24lM0RxdWVzdGlvbiUyQyUyMGNvbnRleHQlM0Rjb250ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>question_answerer = pipeline(<span class="hljs-string">&quot;question-answering&quot;</span>, model=<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>question_answerer(question=question, context=context)
{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.2058267742395401</span>,
 <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">10</span>,
 <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">95</span>,
 <span class="hljs-string">&#x27;answer&#x27;</span>: <span class="hljs-string">&#x27;176 billion parameters and can generate text in 46 languages natural languages and 13&#x27;</span>}`,wrap:!1}}),ce=new rs({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[cl],pytorch:[ol]},$$scope:{ctx:I}}}),{c(){a=M("meta"),c=r(),l=M("p"),m=r(),d=M("h1"),$=M("a"),V=M("span"),w(U.$$.fragment),A=r(),R=M("span"),R.textContent=X,F=r(),w(v.$$.fragment),_=r(),w(k.$$.fragment),Z=r(),N=M("p"),N.textContent=W,C=r(),x=M("ul"),x.innerHTML=s,u=r(),H=M("p"),H.textContent=Me,q=r(),L=M("ol"),L.innerHTML=he,Y=r(),w(Q.$$.fragment),S=r(),P=M("p"),P.textContent=ue,i=r(),w(b.$$.fragment),de=r(),fe=M("p"),fe.textContent=os,ht=r(),w(we.$$.fragment),ut=r(),D=M("h2"),se=M("a"),at=M("span"),w(je.$$.fragment),ss=r(),De=M("span"),De.textContent=is,dt=r(),ye=M("p"),ye.textContent=cs,ft=r(),w(ge.$$.fragment),wt=r(),Je=M("p"),Je.innerHTML=ms,jt=r(),w(Te.$$.fragment),yt=r(),be=M("p"),be.textContent=Ms,gt=r(),w($e.$$.fragment),Jt=r(),Ue=M("p"),Ue.textContent=hs,Tt=r(),_e=M("ul"),_e.innerHTML=us,bt=r(),K=M("h2"),le=M("a"),nt=M("span"),w(Ce.$$.fragment),ls=r(),Ke=M("span"),Ke.textContent=ds,$t=r(),w(Ie.$$.fragment),Ut=r(),Ze=M("p"),Ze.innerHTML=fs,_t=r(),w(ve.$$.fragment),Ct=r(),ke=M("p"),ke.textContent=ws,It=r(),xe=M("ol"),xe.innerHTML=js,Zt=r(),Re=M("p"),Re.innerHTML=ys,vt=r(),w(Xe.$$.fragment),kt=r(),Be=M("p"),Be.innerHTML=gs,xt=r(),w(Ae.$$.fragment),Rt=r(),Ge=M("p"),Ge.innerHTML=Js,Xt=r(),w(ae.$$.fragment),Bt=r(),O=M("h2"),ne=M("a"),pt=M("span"),w(We.$$.fragment),as=r(),Oe=M("span"),Oe.textContent=Ts,At=r(),w(pe.$$.fragment),Gt=r(),w(re.$$.fragment),Wt=r(),ee=M("h2"),oe=M("a"),rt=M("span"),w(Ee.$$.fragment),ns=r(),et=M("span"),et.textContent=bs,Et=r(),Ve=M("p"),Ve.innerHTML=$s,Vt=r(),Ne=M("p"),Ne.innerHTML=Us,Nt=r(),te=M("h2"),ie=M("a"),ot=M("span"),w(ze.$$.fragment),ps=r(),tt=M("span"),tt.textContent=_s,zt=r(),Fe=M("p"),Fe.textContent=Cs,Ft=r(),He=M("p"),He.textContent=Is,Ht=r(),w(Qe.$$.fragment),Qt=r(),qe=M("p"),qe.innerHTML=Zs,qt=r(),w(Ye.$$.fragment),Yt=r(),Se=M("p"),Se.innerHTML=vs,St=r(),w(ce.$$.fragment),Lt=r(),it=M("p"),this.h()},l(e){const p=Ys("svelte-1phssyn",document.head);a=h(p,"META",{name:!0,content:!0}),p.forEach(t),c=o(e),l=h(e,"P",{}),z(l).forEach(t),m=o(e),d=h(e,"H1",{class:!0});var Le=z(d);$=h(Le,"A",{id:!0,class:!0,href:!0});var ct=z($);V=h(ct,"SPAN",{});var mt=z(V);j(U.$$.fragment,mt),mt.forEach(t),ct.forEach(t),A=o(Le),R=h(Le,"SPAN",{"data-svelte-h":!0}),f(R)!=="svelte-12p8stw"&&(R.textContent=X),Le.forEach(t),F=o(e),j(v.$$.fragment,e),_=o(e),j(k.$$.fragment,e),Z=o(e),N=h(e,"P",{"data-svelte-h":!0}),f(N)!=="svelte-egp0yq"&&(N.textContent=W),C=o(e),x=h(e,"UL",{"data-svelte-h":!0}),f(x)!=="svelte-1l55cmw"&&(x.innerHTML=s),u=o(e),H=h(e,"P",{"data-svelte-h":!0}),f(H)!=="svelte-1a7n7l8"&&(H.textContent=Me),q=o(e),L=h(e,"OL",{"data-svelte-h":!0}),f(L)!=="svelte-cv9fgi"&&(L.innerHTML=he),Y=o(e),j(Q.$$.fragment,e),S=o(e),P=h(e,"P",{"data-svelte-h":!0}),f(P)!=="svelte-19ozuj4"&&(P.textContent=ue),i=o(e),j(b.$$.fragment,e),de=o(e),fe=h(e,"P",{"data-svelte-h":!0}),f(fe)!=="svelte-1tz3h56"&&(fe.textContent=os),ht=o(e),j(we.$$.fragment,e),ut=o(e),D=h(e,"H2",{class:!0});var Pe=z(D);se=h(Pe,"A",{id:!0,class:!0,href:!0});var Mt=z(se);at=h(Mt,"SPAN",{});var ks=z(at);j(je.$$.fragment,ks),ks.forEach(t),Mt.forEach(t),ss=o(Pe),De=h(Pe,"SPAN",{"data-svelte-h":!0}),f(De)!=="svelte-14rqwwp"&&(De.textContent=is),Pe.forEach(t),dt=o(e),ye=h(e,"P",{"data-svelte-h":!0}),f(ye)!=="svelte-ufwwq5"&&(ye.textContent=cs),ft=o(e),j(ge.$$.fragment,e),wt=o(e),Je=h(e,"P",{"data-svelte-h":!0}),f(Je)!=="svelte-1mkzjhu"&&(Je.innerHTML=ms),jt=o(e),j(Te.$$.fragment,e),yt=o(e),be=h(e,"P",{"data-svelte-h":!0}),f(be)!=="svelte-iguerl"&&(be.textContent=Ms),gt=o(e),j($e.$$.fragment,e),Jt=o(e),Ue=h(e,"P",{"data-svelte-h":!0}),f(Ue)!=="svelte-xbz2jo"&&(Ue.textContent=hs),Tt=o(e),_e=h(e,"UL",{"data-svelte-h":!0}),f(_e)!=="svelte-xcrfgi"&&(_e.innerHTML=us),bt=o(e),K=h(e,"H2",{class:!0});var Dt=z(K);le=h(Dt,"A",{id:!0,class:!0,href:!0});var xs=z(le);nt=h(xs,"SPAN",{});var Rs=z(nt);j(Ce.$$.fragment,Rs),Rs.forEach(t),xs.forEach(t),ls=o(Dt),Ke=h(Dt,"SPAN",{"data-svelte-h":!0}),f(Ke)!=="svelte-1cte2wb"&&(Ke.textContent=ds),Dt.forEach(t),$t=o(e),j(Ie.$$.fragment,e),Ut=o(e),Ze=h(e,"P",{"data-svelte-h":!0}),f(Ze)!=="svelte-1s1xwu7"&&(Ze.innerHTML=fs),_t=o(e),j(ve.$$.fragment,e),Ct=o(e),ke=h(e,"P",{"data-svelte-h":!0}),f(ke)!=="svelte-gwemrx"&&(ke.textContent=ws),It=o(e),xe=h(e,"OL",{"data-svelte-h":!0}),f(xe)!=="svelte-18o3dcu"&&(xe.innerHTML=js),Zt=o(e),Re=h(e,"P",{"data-svelte-h":!0}),f(Re)!=="svelte-16yk931"&&(Re.innerHTML=ys),vt=o(e),j(Xe.$$.fragment,e),kt=o(e),Be=h(e,"P",{"data-svelte-h":!0}),f(Be)!=="svelte-10qnkwc"&&(Be.innerHTML=gs),xt=o(e),j(Ae.$$.fragment,e),Rt=o(e),Ge=h(e,"P",{"data-svelte-h":!0}),f(Ge)!=="svelte-1i4emod"&&(Ge.innerHTML=Js),Xt=o(e),j(ae.$$.fragment,e),Bt=o(e),O=h(e,"H2",{class:!0});var Kt=z(O);ne=h(Kt,"A",{id:!0,class:!0,href:!0});var Xs=z(ne);pt=h(Xs,"SPAN",{});var Bs=z(pt);j(We.$$.fragment,Bs),Bs.forEach(t),Xs.forEach(t),as=o(Kt),Oe=h(Kt,"SPAN",{"data-svelte-h":!0}),f(Oe)!=="svelte-tdy10p"&&(Oe.textContent=Ts),Kt.forEach(t),At=o(e),j(pe.$$.fragment,e),Gt=o(e),j(re.$$.fragment,e),Wt=o(e),ee=h(e,"H2",{class:!0});var Ot=z(ee);oe=h(Ot,"A",{id:!0,class:!0,href:!0});var As=z(oe);rt=h(As,"SPAN",{});var Gs=z(rt);j(Ee.$$.fragment,Gs),Gs.forEach(t),As.forEach(t),ns=o(Ot),et=h(Ot,"SPAN",{"data-svelte-h":!0}),f(et)!=="svelte-fd7xxy"&&(et.textContent=bs),Ot.forEach(t),Et=o(e),Ve=h(e,"P",{"data-svelte-h":!0}),f(Ve)!=="svelte-h42vxk"&&(Ve.innerHTML=$s),Vt=o(e),Ne=h(e,"P",{"data-svelte-h":!0}),f(Ne)!=="svelte-179emkt"&&(Ne.innerHTML=Us),Nt=o(e),te=h(e,"H2",{class:!0});var es=z(te);ie=h(es,"A",{id:!0,class:!0,href:!0});var Ws=z(ie);ot=h(Ws,"SPAN",{});var Es=z(ot);j(ze.$$.fragment,Es),Es.forEach(t),Ws.forEach(t),ps=o(es),tt=h(es,"SPAN",{"data-svelte-h":!0}),f(tt)!=="svelte-8litrx"&&(tt.textContent=_s),es.forEach(t),zt=o(e),Fe=h(e,"P",{"data-svelte-h":!0}),f(Fe)!=="svelte-174wmmm"&&(Fe.textContent=Cs),Ft=o(e),He=h(e,"P",{"data-svelte-h":!0}),f(He)!=="svelte-19ecll8"&&(He.textContent=Is),Ht=o(e),j(Qe.$$.fragment,e),Qt=o(e),qe=h(e,"P",{"data-svelte-h":!0}),f(qe)!=="svelte-18d8pbh"&&(qe.innerHTML=Zs),qt=o(e),j(Ye.$$.fragment,e),Yt=o(e),Se=h(e,"P",{"data-svelte-h":!0}),f(Se)!=="svelte-ckakkh"&&(Se.innerHTML=vs),St=o(e),j(ce.$$.fragment,e),Lt=o(e),it=h(e,"P",{}),z(it).forEach(t),this.h()},h(){G(a,"name","hf:doc:metadata"),G(a,"content",JSON.stringify(Ml)),G($,"id","question-answering"),G($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),G($,"href","#question-answering"),G(d,"class","relative group"),G(se,"id","load-squad-dataset"),G(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),G(se,"href","#load-squad-dataset"),G(D,"class","relative group"),G(le,"id","preprocess"),G(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),G(le,"href","#preprocess"),G(K,"class","relative group"),G(ne,"id","train"),G(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),G(ne,"href","#train"),G(O,"class","relative group"),G(oe,"id","evaluate"),G(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),G(oe,"href","#evaluate"),G(ee,"class","relative group"),G(ie,"id","inference"),G(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),G(ie,"href","#inference"),G(te,"class","relative group")},m(e,p){E(document.head,a),n(e,c,p),n(e,l,p),n(e,m,p),n(e,d,p),E(d,$),E($,V),y(U,V,null),E(d,A),E(d,R),n(e,F,p),y(v,e,p),n(e,_,p),y(k,e,p),n(e,Z,p),n(e,N,p),n(e,C,p),n(e,x,p),n(e,u,p),n(e,H,p),n(e,q,p),n(e,L,p),n(e,Y,p),y(Q,e,p),n(e,S,p),n(e,P,p),n(e,i,p),y(b,e,p),n(e,de,p),n(e,fe,p),n(e,ht,p),y(we,e,p),n(e,ut,p),n(e,D,p),E(D,se),E(se,at),y(je,at,null),E(D,ss),E(D,De),n(e,dt,p),n(e,ye,p),n(e,ft,p),y(ge,e,p),n(e,wt,p),n(e,Je,p),n(e,jt,p),y(Te,e,p),n(e,yt,p),n(e,be,p),n(e,gt,p),y($e,e,p),n(e,Jt,p),n(e,Ue,p),n(e,Tt,p),n(e,_e,p),n(e,bt,p),n(e,K,p),E(K,le),E(le,nt),y(Ce,nt,null),E(K,ls),E(K,Ke),n(e,$t,p),y(Ie,e,p),n(e,Ut,p),n(e,Ze,p),n(e,_t,p),y(ve,e,p),n(e,Ct,p),n(e,ke,p),n(e,It,p),n(e,xe,p),n(e,Zt,p),n(e,Re,p),n(e,vt,p),y(Xe,e,p),n(e,kt,p),n(e,Be,p),n(e,xt,p),y(Ae,e,p),n(e,Rt,p),n(e,Ge,p),n(e,Xt,p),y(ae,e,p),n(e,Bt,p),n(e,O,p),E(O,ne),E(ne,pt),y(We,pt,null),E(O,as),E(O,Oe),n(e,At,p),y(pe,e,p),n(e,Gt,p),y(re,e,p),n(e,Wt,p),n(e,ee,p),E(ee,oe),E(oe,rt),y(Ee,rt,null),E(ee,ns),E(ee,et),n(e,Et,p),n(e,Ve,p),n(e,Vt,p),n(e,Ne,p),n(e,Nt,p),n(e,te,p),E(te,ie),E(ie,ot),y(ze,ot,null),E(te,ps),E(te,tt),n(e,zt,p),n(e,Fe,p),n(e,Ft,p),n(e,He,p),n(e,Ht,p),y(Qe,e,p),n(e,Qt,p),n(e,qe,p),n(e,qt,p),y(Ye,e,p),n(e,Yt,p),n(e,Se,p),n(e,St,p),y(ce,e,p),n(e,Lt,p),n(e,it,p),Pt=!0},p(e,[p]){const Le={};p&2&&(Le.$$scope={dirty:p,ctx:e}),Q.$set(Le);const ct={};p&2&&(ct.$$scope={dirty:p,ctx:e}),ae.$set(ct);const mt={};p&2&&(mt.$$scope={dirty:p,ctx:e}),pe.$set(mt);const Pe={};p&2&&(Pe.$$scope={dirty:p,ctx:e}),re.$set(Pe);const Mt={};p&2&&(Mt.$$scope={dirty:p,ctx:e}),ce.$set(Mt)},i(e){Pt||(g(U.$$.fragment,e),g(v.$$.fragment,e),g(k.$$.fragment,e),g(Q.$$.fragment,e),g(b.$$.fragment,e),g(we.$$.fragment,e),g(je.$$.fragment,e),g(ge.$$.fragment,e),g(Te.$$.fragment,e),g($e.$$.fragment,e),g(Ce.$$.fragment,e),g(Ie.$$.fragment,e),g(ve.$$.fragment,e),g(Xe.$$.fragment,e),g(Ae.$$.fragment,e),g(ae.$$.fragment,e),g(We.$$.fragment,e),g(pe.$$.fragment,e),g(re.$$.fragment,e),g(Ee.$$.fragment,e),g(ze.$$.fragment,e),g(Qe.$$.fragment,e),g(Ye.$$.fragment,e),g(ce.$$.fragment,e),Pt=!0)},o(e){J(U.$$.fragment,e),J(v.$$.fragment,e),J(k.$$.fragment,e),J(Q.$$.fragment,e),J(b.$$.fragment,e),J(we.$$.fragment,e),J(je.$$.fragment,e),J(ge.$$.fragment,e),J(Te.$$.fragment,e),J($e.$$.fragment,e),J(Ce.$$.fragment,e),J(Ie.$$.fragment,e),J(ve.$$.fragment,e),J(Xe.$$.fragment,e),J(Ae.$$.fragment,e),J(ae.$$.fragment,e),J(We.$$.fragment,e),J(pe.$$.fragment,e),J(re.$$.fragment,e),J(Ee.$$.fragment,e),J(ze.$$.fragment,e),J(Qe.$$.fragment,e),J(Ye.$$.fragment,e),J(ce.$$.fragment,e),Pt=!1},d(e){e&&(t(c),t(l),t(m),t(d),t(F),t(_),t(Z),t(N),t(C),t(x),t(u),t(H),t(q),t(L),t(Y),t(S),t(P),t(i),t(de),t(fe),t(ht),t(ut),t(D),t(dt),t(ye),t(ft),t(wt),t(Je),t(jt),t(yt),t(be),t(gt),t(Jt),t(Ue),t(Tt),t(_e),t(bt),t(K),t($t),t(Ut),t(Ze),t(_t),t(Ct),t(ke),t(It),t(xe),t(Zt),t(Re),t(vt),t(kt),t(Be),t(xt),t(Rt),t(Ge),t(Xt),t(Bt),t(O),t(At),t(Gt),t(Wt),t(ee),t(Et),t(Ve),t(Vt),t(Ne),t(Nt),t(te),t(zt),t(Fe),t(Ft),t(He),t(Ht),t(Qt),t(qe),t(qt),t(Yt),t(Se),t(St),t(Lt),t(it)),t(a),T(U),T(v,e),T(k,e),T(Q,e),T(b,e),T(we,e),T(je),T(ge,e),T(Te,e),T($e,e),T(Ce),T(Ie,e),T(ve,e),T(Xe,e),T(Ae,e),T(ae,e),T(We),T(pe,e),T(re,e),T(Ee),T(ze),T(Qe,e),T(Ye,e),T(ce,e)}}}const Ml={local:"question-answering",sections:[{local:"load-squad-dataset",title:"SQuAD 데이터 세트 가져오기"},{local:"preprocess",title:"전처리"},{local:"train",title:"훈련"},{local:"evaluate",title:"평가"},{local:"inference",title:"추론"}],title:"질의 응답(Question Answering)"};function hl(I){return Hs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Tl extends Qs{constructor(a){super(),qs(this,a,hl,ml,Fs,{})}}export{Tl as component};
