import{s as Qe,o as qe,n as es}from"../chunks/scheduler.9bc65507.js";import{S as Ye,i as Le,g as T,s as o,r as f,A as Se,h as y,f as a,c as m,j as Fe,u as d,x as g,k as Ne,y as Pe,a as n,v as u,d as h,t as j,w,m as De,n as Ke}from"../chunks/index.707bf1b6.js";import{T as Te}from"../chunks/Tip.c2ecdbf4.js";import{Y as He}from"../chunks/Youtube.e1129c6f.js";import{C as k}from"../chunks/CodeBlock.54a9f38d.js";import{D as Oe}from"../chunks/DocNotebookDropdown.3e6b3817.js";import{F as ye,M as vs}from"../chunks/Markdown.8ab98a13.js";import{H as Bs}from"../chunks/Heading.342b1fa6.js";function st(I){let l,i,t='<a href="../model_doc/albert">ALBERT</a>, <a href="../model_doc/bart">BART</a>, <a href="../model_doc/bert">BERT</a>, <a href="../model_doc/big_bird">BigBird</a>, <a href="../model_doc/bigbird_pegasus">BigBird-Pegasus</a>, <a href="../model_doc/bloom">BLOOM</a>, <a href="../model_doc/camembert">CamemBERT</a>, <a href="../model_doc/canine">CANINE</a>, <a href="../model_doc/convbert">ConvBERT</a>, <a href="../model_doc/data2vec-text">Data2VecText</a>, <a href="../model_doc/deberta">DeBERTa</a>, <a href="../model_doc/deberta-v2">DeBERTa-v2</a>, <a href="../model_doc/distilbert">DistilBERT</a>, <a href="../model_doc/electra">ELECTRA</a>, <a href="../model_doc/ernie">ERNIE</a>, <a href="../model_doc/ernie_m">ErnieM</a>, <a href="../model_doc/falcon">Falcon</a>, <a href="../model_doc/flaubert">FlauBERT</a>, <a href="../model_doc/fnet">FNet</a>, <a href="../model_doc/funnel">Funnel Transformer</a>, <a href="../model_doc/gpt2">OpenAI GPT-2</a>, <a href="../model_doc/gpt_neo">GPT Neo</a>, <a href="../model_doc/gpt_neox">GPT NeoX</a>, <a href="../model_doc/gptj">GPT-J</a>, <a href="../model_doc/ibert">I-BERT</a>, <a href="../model_doc/layoutlmv2">LayoutLMv2</a>, <a href="../model_doc/layoutlmv3">LayoutLMv3</a>, <a href="../model_doc/led">LED</a>, <a href="../model_doc/lilt">LiLT</a>, <a href="../model_doc/longformer">Longformer</a>, <a href="../model_doc/luke">LUKE</a>, <a href="../model_doc/lxmert">LXMERT</a>, <a href="../model_doc/markuplm">MarkupLM</a>, <a href="../model_doc/mbart">mBART</a>, <a href="../model_doc/mega">MEGA</a>, <a href="../model_doc/megatron-bert">Megatron-BERT</a>, <a href="../model_doc/mobilebert">MobileBERT</a>, <a href="../model_doc/mpnet">MPNet</a>, <a href="../model_doc/mpt">MPT</a>, <a href="../model_doc/mra">MRA</a>, <a href="../model_doc/mt5">MT5</a>, <a href="../model_doc/mvp">MVP</a>, <a href="../model_doc/nezha">Nezha</a>, <a href="../model_doc/nystromformer">Nyströmformer</a>, <a href="../model_doc/opt">OPT</a>, <a href="../model_doc/qdqbert">QDQBert</a>, <a href="../model_doc/reformer">Reformer</a>, <a href="../model_doc/rembert">RemBERT</a>, <a href="../model_doc/roberta">RoBERTa</a>, <a href="../model_doc/roberta-prelayernorm">RoBERTa-PreLayerNorm</a>, <a href="../model_doc/roc_bert">RoCBert</a>, <a href="../model_doc/roformer">RoFormer</a>, <a href="../model_doc/splinter">Splinter</a>, <a href="../model_doc/squeezebert">SqueezeBERT</a>, <a href="../model_doc/t5">T5</a>, <a href="../model_doc/umt5">UMT5</a>, <a href="../model_doc/xlm">XLM</a>, <a href="../model_doc/xlm-roberta">XLM-RoBERTa</a>, <a href="../model_doc/xlm-roberta-xl">XLM-RoBERTa-XL</a>, <a href="../model_doc/xlnet">XLNet</a>, <a href="../model_doc/xmod">X-MOD</a>, <a href="../model_doc/yoso">YOSO</a>';return{c(){l=De(`このチュートリアルで説明するタスクは、次のモデル アーキテクチャでサポートされています。

`),i=T("p"),i.innerHTML=t},l(c){l=Ke(c,`このチュートリアルで説明するタスクは、次のモデル アーキテクチャでサポートされています。

`),i=y(c,"P",{"data-svelte-h":!0}),g(i)!=="svelte-1grwhx8"&&(i.innerHTML=t)},m(c,J){n(c,l,J),n(c,i,J)},p:es,d(c){c&&(a(l),a(i))}}}function et(I){let l,i;return l=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERlZmF1bHREYXRhQ29sbGF0b3IlMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGVmYXVsdERhdGFDb2xsYXRvcigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DefaultDataCollator

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DefaultDataCollator()`,wrap:!1}}),{c(){f(l.$$.fragment)},l(t){d(l.$$.fragment,t)},m(t,c){u(l,t,c),i=!0},p:es,i(t){i||(h(l.$$.fragment,t),i=!0)},o(t){j(l.$$.fragment,t),i=!1},d(t){w(l,t)}}}function tt(I){let l,i;return l=new vs({props:{$$slots:{default:[et]},$$scope:{ctx:I}}}),{c(){f(l.$$.fragment)},l(t){d(l.$$.fragment,t)},m(t,c){u(l,t,c),i=!0},p(t,c){const J={};c&2&&(J.$$scope={dirty:c,ctx:t}),l.$set(J)},i(t){i||(h(l.$$.fragment,t),i=!0)},o(t){j(l.$$.fragment,t),i=!1},d(t){w(l,t)}}}function lt(I){let l,i;return l=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERlZmF1bHREYXRhQ29sbGF0b3IlMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGVmYXVsdERhdGFDb2xsYXRvcihyZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DefaultDataCollator

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DefaultDataCollator(return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),{c(){f(l.$$.fragment)},l(t){d(l.$$.fragment,t)},m(t,c){u(l,t,c),i=!0},p:es,i(t){i||(h(l.$$.fragment,t),i=!0)},o(t){j(l.$$.fragment,t),i=!1},d(t){w(l,t)}}}function at(I){let l,i;return l=new vs({props:{$$slots:{default:[lt]},$$scope:{ctx:I}}}),{c(){f(l.$$.fragment)},l(t){d(l.$$.fragment,t)},m(t,c){u(l,t,c),i=!0},p(t,c){const J={};c&2&&(J.$$scope={dirty:c,ctx:t}),l.$set(J)},i(t){i||(h(l.$$.fragment,t),i=!0)},o(t){j(l.$$.fragment,t),i=!1},d(t){w(l,t)}}}function nt(I){let l,i='<a href="/docs/transformers/v4.37.0/ja/main_classes/trainer#transformers.Trainer">Trainer</a> を使用したモデルの微調整に慣れていない場合は、<a href="../training#train-with-pytorch-trainer">ここ</a> の基本的なチュートリアルをご覧ください。';return{c(){l=T("p"),l.innerHTML=i},l(t){l=y(t,"P",{"data-svelte-h":!0}),g(l)!=="svelte-1moeiqp"&&(l.innerHTML=i)},m(t,c){n(t,l,c)},p:es,d(t){t&&a(l)}}}function pt(I){let l,i,t,c='これでモデルのトレーニングを開始する準備が整いました。 <a href="/docs/transformers/v4.37.0/ja/model_doc/auto#transformers.AutoModelForQuestionAnswering">AutoModelForQuestionAnswering</a> を使用して DitilBERT をロードします。',J,U,G,R,X="この時点で残っている手順は次の 3 つだけです。",A,b,z='<li><a href="/docs/transformers/v4.37.0/ja/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a> でトレーニング ハイパーパラメータを定義します。唯一の必須パラメータは、モデルの保存場所を指定する <code>output_dir</code> です。 <code>push_to_hub=True</code>を設定して、このモデルをハブにプッシュします (モデルをアップロードするには、Hugging Face にサインインする必要があります)。</li> <li>トレーニング引数をモデル、データセット、トークナイザー、データ照合器とともに <a href="/docs/transformers/v4.37.0/ja/main_classes/trainer#transformers.Trainer">Trainer</a> に渡します。</li> <li><a href="/docs/transformers/v4.37.0/ja/main_classes/trainer#transformers.Trainer.train">train()</a> を呼び出してモデルを微調整します。</li>',x,C,W,_,E='トレーニングが完了したら、 <a href="/docs/transformers/v4.37.0/ja/main_classes/trainer#transformers.Trainer.push_to_hub">push_to_hub()</a> メソッドを使用してモデルをハブに共有し、誰もがモデルを使用できるようにします。',v,Z,B;return l=new Te({props:{$$slots:{default:[nt]},$$scope:{ctx:I}}}),U=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclF1ZXN0aW9uQW5zd2VyaW5nJTJDJTIwVHJhaW5pbmdBcmd1bWVudHMlMkMlMjBUcmFpbmVyJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JRdWVzdGlvbkFuc3dlcmluZy5mcm9tX3ByZXRyYWluZWQoJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForQuestionAnswering, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForQuestionAnswering.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`,wrap:!1}}),C=new k({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJteV9hd2Vzb21lX3FhX21vZGVsJTIyJTJDJTBBJTIwJTIwJTIwJTIwZXZhbHVhdGlvbl9zdHJhdGVneSUzRCUyMmVwb2NoJTIyJTJDJTBBJTIwJTIwJTIwJTIwbGVhcm5pbmdfcmF0ZSUzRDJlLTUlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfZXZhbF9iYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMyUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRDAuMDElMkMlMEElMjAlMjAlMjAlMjBwdXNoX3RvX2h1YiUzRFRydWUlMkMlMEEpJTBBJTBBdHJhaW5lciUyMCUzRCUyMFRyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRHRva2VuaXplZF9zcXVhZCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX3NxdWFkJTVCJTIydGVzdCUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMGRhdGFfY29sbGF0b3IlM0RkYXRhX2NvbGxhdG9yJTJDJTBBKSUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_squad[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_squad[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`,wrap:!1}}),Z=new k({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()',wrap:!1}}),{c(){f(l.$$.fragment),i=o(),t=T("p"),t.innerHTML=c,J=o(),f(U.$$.fragment),G=o(),R=T("p"),R.textContent=X,A=o(),b=T("ol"),b.innerHTML=z,x=o(),f(C.$$.fragment),W=o(),_=T("p"),_.innerHTML=E,v=o(),f(Z.$$.fragment)},l(e){d(l.$$.fragment,e),i=m(e),t=y(e,"P",{"data-svelte-h":!0}),g(t)!=="svelte-1o9jl6f"&&(t.innerHTML=c),J=m(e),d(U.$$.fragment,e),G=m(e),R=y(e,"P",{"data-svelte-h":!0}),g(R)!=="svelte-1j8bgyv"&&(R.textContent=X),A=m(e),b=y(e,"OL",{"data-svelte-h":!0}),g(b)!=="svelte-1qnjz7l"&&(b.innerHTML=z),x=m(e),d(C.$$.fragment,e),W=m(e),_=y(e,"P",{"data-svelte-h":!0}),g(_)!=="svelte-1yyme3m"&&(_.innerHTML=E),v=m(e),d(Z.$$.fragment,e)},m(e,M){u(l,e,M),n(e,i,M),n(e,t,M),n(e,J,M),u(U,e,M),n(e,G,M),n(e,R,M),n(e,A,M),n(e,b,M),n(e,x,M),u(C,e,M),n(e,W,M),n(e,_,M),n(e,v,M),u(Z,e,M),B=!0},p(e,M){const V={};M&2&&(V.$$scope={dirty:M,ctx:e}),l.$set(V)},i(e){B||(h(l.$$.fragment,e),h(U.$$.fragment,e),h(C.$$.fragment,e),h(Z.$$.fragment,e),B=!0)},o(e){j(l.$$.fragment,e),j(U.$$.fragment,e),j(C.$$.fragment,e),j(Z.$$.fragment,e),B=!1},d(e){e&&(a(i),a(t),a(J),a(G),a(R),a(A),a(b),a(x),a(W),a(_),a(v)),w(l,e),w(U,e),w(C,e),w(Z,e)}}}function rt(I){let l,i;return l=new vs({props:{$$slots:{default:[pt]},$$scope:{ctx:I}}}),{c(){f(l.$$.fragment)},l(t){d(l.$$.fragment,t)},m(t,c){u(l,t,c),i=!0},p(t,c){const J={};c&2&&(J.$$scope={dirty:c,ctx:t}),l.$set(J)},i(t){i||(h(l.$$.fragment,t),i=!0)},o(t){j(l.$$.fragment,t),i=!1},d(t){w(l,t)}}}function ot(I){let l,i='Keras を使用したモデルの微調整に慣れていない場合は、<a href="../training#train-a-tensorflow-model-with-keras">こちら</a> の基本的なチュートリアルをご覧ください。';return{c(){l=T("p"),l.innerHTML=i},l(t){l=y(t,"P",{"data-svelte-h":!0}),g(l)!=="svelte-1jwo7q8"&&(l.innerHTML=i)},m(t,c){n(t,l,c)},p:es,d(t){t&&a(l)}}}function mt(I){let l,i,t,c=`</ヒント>
TensorFlow でモデルを微調整するには、オプティマイザー関数、学習率スケジュール、およびいくつかのトレーニング ハイパーパラメーターをセットアップすることから始めます。`,J,U,G,R,X='次に、<a href="/docs/transformers/v4.37.0/ja/model_doc/auto#transformers.TFAutoModelForQuestionAnswering">TFAutoModelForQuestionAnswering</a> を使用して DistilBERT をロードできます。',A,b,z,x,C='<a href="/docs/transformers/v4.37.0/ja/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset">prepare_tf_dataset()</a> を使用して、データセットを <code>tf.data.Dataset</code> 形式に変換します。',W,_,E,v,Z='<a href="https://keras.io/api/models/model_training_apis/#compile-method" rel="nofollow"><code>compile</code></a> を使用してトレーニング用のモデルを設定します。',B,e,M,V,ts='トレーニングを開始する前に最後にセットアップすることは、モデルをハブにプッシュする方法を提供することです。これは、モデルとトークナイザーを <a href="/docs/transformers/v4.37.0/ja/main_classes/keras_callbacks#transformers.PushToHubCallback">PushToHubCallback</a> でプッシュする場所を指定することで実行できます。',H,Y,S,F,ls='ついに、モデルのトレーニングを開始する準備が整いました。トレーニングおよび検証データセット、エポック数、コールバックを指定して <a href="https://keras.io/api/models/model_training_apis/#fit-method" rel="nofollow"><code>fit</code></a> を呼び出し、モデルを微調整します。',Q,L,P,N,as="トレーニングが完了すると、モデルは自動的にハブにアップロードされ、誰でも使用できるようになります。",q;return l=new Te({props:{$$slots:{default:[ot]},$$scope:{ctx:I}}}),U=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEElMEFiYXRjaF9zaXplJTIwJTNEJTIwMTYlMEFudW1fZXBvY2hzJTIwJTNEJTIwMiUwQXRvdGFsX3RyYWluX3N0ZXBzJTIwJTNEJTIwKGxlbih0b2tlbml6ZWRfc3F1YWQlNUIlMjJ0cmFpbiUyMiU1RCklMjAlMkYlMkYlMjBiYXRjaF9zaXplKSUyMColMjBudW1fZXBvY2hzJTBBb3B0aW1pemVyJTJDJTIwc2NoZWR1bGUlMjAlM0QlMjBjcmVhdGVfb3B0aW1pemVyKCUwQSUyMCUyMCUyMCUyMGluaXRfbHIlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fc3RlcHMlM0R0b3RhbF90cmFpbl9zdGVwcyUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_epochs = <span class="hljs-number">2</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>total_train_steps = (<span class="hljs-built_in">len</span>(tokenized_squad[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>    num_train_steps=total_train_steps,
<span class="hljs-meta">... </span>)`,wrap:!1}}),b=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yUXVlc3Rpb25BbnN3ZXJpbmclMEElMEFtb2RlbCUyMCUzRCUyMFRGQXV0b01vZGVsRm9yUXVlc3Rpb25BbnN3ZXJpbmcoJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForQuestionAnswering

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForQuestionAnswering(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`,wrap:!1}}),_=new k({props:{code:"dGZfdHJhaW5fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF9zcXVhZCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwc2h1ZmZsZSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBjb2xsYXRlX2ZuJTNEZGF0YV9jb2xsYXRvciUyQyUwQSklMEElMEF0Zl92YWxpZGF0aW9uX3NldCUyMCUzRCUyMG1vZGVsLnByZXBhcmVfdGZfZGF0YXNldCglMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfc3F1YWQlNUIlMjJ0ZXN0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwc2h1ZmZsZSUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwY29sbGF0ZV9mbiUzRGRhdGFfY29sbGF0b3IlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_squad[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_squad[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`,wrap:!1}}),e=new k({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplcik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`,wrap:!1}}),Y=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBQdXNoVG9IdWJDYWxsYmFjayUwQSUwQWNhbGxiYWNrJTIwJTNEJTIwUHVzaFRvSHViQ2FsbGJhY2soJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm15X2F3ZXNvbWVfcWFfbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>callback = PushToHubCallback(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>)`,wrap:!1}}),L=new k({props:{code:"bW9kZWwuZml0KHglM0R0Zl90cmFpbl9zZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl92YWxpZGF0aW9uX3NldCUyQyUyMGVwb2NocyUzRDMlMkMlMjBjYWxsYmFja3MlM0QlNUJjYWxsYmFjayU1RCk=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>, callbacks=[callback])',wrap:!1}}),{c(){f(l.$$.fragment),i=o(),t=T("p"),t.textContent=c,J=o(),f(U.$$.fragment),G=o(),R=T("p"),R.innerHTML=X,A=o(),f(b.$$.fragment),z=o(),x=T("p"),x.innerHTML=C,W=o(),f(_.$$.fragment),E=o(),v=T("p"),v.innerHTML=Z,B=o(),f(e.$$.fragment),M=o(),V=T("p"),V.innerHTML=ts,H=o(),f(Y.$$.fragment),S=o(),F=T("p"),F.innerHTML=ls,Q=o(),f(L.$$.fragment),P=o(),N=T("p"),N.textContent=as},l(r){d(l.$$.fragment,r),i=m(r),t=y(r,"P",{"data-svelte-h":!0}),g(t)!=="svelte-1v9esfa"&&(t.textContent=c),J=m(r),d(U.$$.fragment,r),G=m(r),R=y(r,"P",{"data-svelte-h":!0}),g(R)!=="svelte-1ta6zr1"&&(R.innerHTML=X),A=m(r),d(b.$$.fragment,r),z=m(r),x=y(r,"P",{"data-svelte-h":!0}),g(x)!=="svelte-ekds79"&&(x.innerHTML=C),W=m(r),d(_.$$.fragment,r),E=m(r),v=y(r,"P",{"data-svelte-h":!0}),g(v)!=="svelte-kzu0b7"&&(v.innerHTML=Z),B=m(r),d(e.$$.fragment,r),M=m(r),V=y(r,"P",{"data-svelte-h":!0}),g(V)!=="svelte-xznoed"&&(V.innerHTML=ts),H=m(r),d(Y.$$.fragment,r),S=m(r),F=y(r,"P",{"data-svelte-h":!0}),g(F)!=="svelte-ffgub5"&&(F.innerHTML=ls),Q=m(r),d(L.$$.fragment,r),P=m(r),N=y(r,"P",{"data-svelte-h":!0}),g(N)!=="svelte-vh7z0v"&&(N.textContent=as)},m(r,$){u(l,r,$),n(r,i,$),n(r,t,$),n(r,J,$),u(U,r,$),n(r,G,$),n(r,R,$),n(r,A,$),u(b,r,$),n(r,z,$),n(r,x,$),n(r,W,$),u(_,r,$),n(r,E,$),n(r,v,$),n(r,B,$),u(e,r,$),n(r,M,$),n(r,V,$),n(r,H,$),u(Y,r,$),n(r,S,$),n(r,F,$),n(r,Q,$),u(L,r,$),n(r,P,$),n(r,N,$),q=!0},p(r,$){const As={};$&2&&(As.$$scope={dirty:$,ctx:r}),l.$set(As)},i(r){q||(h(l.$$.fragment,r),h(U.$$.fragment,r),h(b.$$.fragment,r),h(_.$$.fragment,r),h(e.$$.fragment,r),h(Y.$$.fragment,r),h(L.$$.fragment,r),q=!0)},o(r){j(l.$$.fragment,r),j(U.$$.fragment,r),j(b.$$.fragment,r),j(_.$$.fragment,r),j(e.$$.fragment,r),j(Y.$$.fragment,r),j(L.$$.fragment,r),q=!1},d(r){r&&(a(i),a(t),a(J),a(G),a(R),a(A),a(z),a(x),a(W),a(E),a(v),a(B),a(M),a(V),a(H),a(S),a(F),a(Q),a(P),a(N)),w(l,r),w(U,r),w(b,r),w(_,r),w(e,r),w(Y,r),w(L,r)}}}function it(I){let l,i;return l=new vs({props:{$$slots:{default:[mt]},$$scope:{ctx:I}}}),{c(){f(l.$$.fragment)},l(t){d(l.$$.fragment,t)},m(t,c){u(l,t,c),i=!0},p(t,c){const J={};c&2&&(J.$$scope={dirty:c,ctx:t}),l.$set(J)},i(t){i||(h(l.$$.fragment,t),i=!0)},o(t){j(l.$$.fragment,t),i=!1},d(t){w(l,t)}}}function ct(I){let l,i=`質問応答用のモデルを微調整する方法の詳細な例については、対応するドキュメントを参照してください。
<a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering.ipynb" rel="nofollow">PyTorch ノートブック</a>
または <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering-tf.ipynb" rel="nofollow">TensorFlow ノートブック</a>。`;return{c(){l=T("p"),l.innerHTML=i},l(t){l=y(t,"P",{"data-svelte-h":!0}),g(l)!=="svelte-1n4kqih"&&(l.innerHTML=i)},m(t,c){n(t,l,c)},p:es,d(t){t&&a(l)}}}function Mt(I){let l,i="テキストをトークン化して PyTorch テンソルを返します。",t,c,J,U,G="入力をモデルに渡し、<code>logits</code>を返します。",R,X,A,b,z="モデル出力から開始位置と終了位置の最も高い確率を取得します。",x,C,W,_,E="予測されたトークンをデコードして答えを取得します。",v,Z,B;return c=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJteV9hd2Vzb21lX3FhX21vZGVsJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihxdWVzdGlvbiUyQyUyMGNvbnRleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(question, context, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)`,wrap:!1}}),X=new k({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yUXVlc3Rpb25BbnN3ZXJpbmclMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclF1ZXN0aW9uQW5zd2VyaW5nLmZyb21fcHJldHJhaW5lZCglMjJteV9hd2Vzb21lX3FhX21vZGVsJTIyKSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForQuestionAnswering

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForQuestionAnswering.from_pretrained(<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    outputs = model(**inputs)`,wrap:!1}}),C=new k({props:{code:"YW5zd2VyX3N0YXJ0X2luZGV4JTIwJTNEJTIwb3V0cHV0cy5zdGFydF9sb2dpdHMuYXJnbWF4KCklMEFhbnN3ZXJfZW5kX2luZGV4JTIwJTNEJTIwb3V0cHV0cy5lbmRfbG9naXRzLmFyZ21heCgp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>answer_start_index = outputs.start_logits.argmax()
<span class="hljs-meta">&gt;&gt;&gt; </span>answer_end_index = outputs.end_logits.argmax()`,wrap:!1}}),Z=new k({props:{code:"cHJlZGljdF9hbnN3ZXJfdG9rZW5zJTIwJTNEJTIwaW5wdXRzLmlucHV0X2lkcyU1QjAlMkMlMjBhbnN3ZXJfc3RhcnRfaW5kZXglMjAlM0ElMjBhbnN3ZXJfZW5kX2luZGV4JTIwJTJCJTIwMSU1RCUwQXRva2VuaXplci5kZWNvZGUocHJlZGljdF9hbnN3ZXJfdG9rZW5zKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predict_answer_tokens = inputs.input_ids[<span class="hljs-number">0</span>, answer_start_index : answer_end_index + <span class="hljs-number">1</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(predict_answer_tokens)
<span class="hljs-string">&#x27;176 billion parameters and can generate text in 46 languages natural languages and 13&#x27;</span>`,wrap:!1}}),{c(){l=T("p"),l.textContent=i,t=o(),f(c.$$.fragment),J=o(),U=T("p"),U.innerHTML=G,R=o(),f(X.$$.fragment),A=o(),b=T("p"),b.textContent=z,x=o(),f(C.$$.fragment),W=o(),_=T("p"),_.textContent=E,v=o(),f(Z.$$.fragment)},l(e){l=y(e,"P",{"data-svelte-h":!0}),g(l)!=="svelte-1n0k6or"&&(l.textContent=i),t=m(e),d(c.$$.fragment,e),J=m(e),U=y(e,"P",{"data-svelte-h":!0}),g(U)!=="svelte-1rvunpz"&&(U.innerHTML=G),R=m(e),d(X.$$.fragment,e),A=m(e),b=y(e,"P",{"data-svelte-h":!0}),g(b)!=="svelte-bytl0h"&&(b.textContent=z),x=m(e),d(C.$$.fragment,e),W=m(e),_=y(e,"P",{"data-svelte-h":!0}),g(_)!=="svelte-1yq6861"&&(_.textContent=E),v=m(e),d(Z.$$.fragment,e)},m(e,M){n(e,l,M),n(e,t,M),u(c,e,M),n(e,J,M),n(e,U,M),n(e,R,M),u(X,e,M),n(e,A,M),n(e,b,M),n(e,x,M),u(C,e,M),n(e,W,M),n(e,_,M),n(e,v,M),u(Z,e,M),B=!0},p:es,i(e){B||(h(c.$$.fragment,e),h(X.$$.fragment,e),h(C.$$.fragment,e),h(Z.$$.fragment,e),B=!0)},o(e){j(c.$$.fragment,e),j(X.$$.fragment,e),j(C.$$.fragment,e),j(Z.$$.fragment,e),B=!1},d(e){e&&(a(l),a(t),a(J),a(U),a(R),a(A),a(b),a(x),a(W),a(_),a(v)),w(c,e),w(X,e),w(C,e),w(Z,e)}}}function ft(I){let l,i;return l=new vs({props:{$$slots:{default:[Mt]},$$scope:{ctx:I}}}),{c(){f(l.$$.fragment)},l(t){d(l.$$.fragment,t)},m(t,c){u(l,t,c),i=!0},p(t,c){const J={};c&2&&(J.$$scope={dirty:c,ctx:t}),l.$set(J)},i(t){i||(h(l.$$.fragment,t),i=!0)},o(t){j(l.$$.fragment,t),i=!1},d(t){w(l,t)}}}function dt(I){let l,i="テキストをトークン化し、TensorFlow テンソルを返します。",t,c,J,U,G="入力をモデルに渡し、<code>logits</code>を返します。",R,X,A,b,z="モデル出力から開始位置と終了位置の最も高い確率を取得します。",x,C,W,_,E="予測されたトークンをデコードして答えを取得します。",v,Z,B;return c=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJteV9hd2Vzb21lX3FhX21vZGVsJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihxdWVzdGlvbiUyQyUyMHRleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(question, text, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),X=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yUXVlc3Rpb25BbnN3ZXJpbmclMEElMEFtb2RlbCUyMCUzRCUyMFRGQXV0b01vZGVsRm9yUXVlc3Rpb25BbnN3ZXJpbmcuZnJvbV9wcmV0cmFpbmVkKCUyMm15X2F3ZXNvbWVfcWFfbW9kZWwlMjIpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForQuestionAnswering

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForQuestionAnswering.from_pretrained(<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(**inputs)`,wrap:!1}}),C=new k({props:{code:"YW5zd2VyX3N0YXJ0X2luZGV4JTIwJTNEJTIwaW50KHRmLm1hdGguYXJnbWF4KG91dHB1dHMuc3RhcnRfbG9naXRzJTJDJTIwYXhpcyUzRC0xKSU1QjAlNUQpJTBBYW5zd2VyX2VuZF9pbmRleCUyMCUzRCUyMGludCh0Zi5tYXRoLmFyZ21heChvdXRwdXRzLmVuZF9sb2dpdHMlMkMlMjBheGlzJTNELTEpJTVCMCU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>answer_start_index = <span class="hljs-built_in">int</span>(tf.math.argmax(outputs.start_logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>answer_end_index = <span class="hljs-built_in">int</span>(tf.math.argmax(outputs.end_logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>])`,wrap:!1}}),Z=new k({props:{code:"cHJlZGljdF9hbnN3ZXJfdG9rZW5zJTIwJTNEJTIwaW5wdXRzLmlucHV0X2lkcyU1QjAlMkMlMjBhbnN3ZXJfc3RhcnRfaW5kZXglMjAlM0ElMjBhbnN3ZXJfZW5kX2luZGV4JTIwJTJCJTIwMSU1RCUwQXRva2VuaXplci5kZWNvZGUocHJlZGljdF9hbnN3ZXJfdG9rZW5zKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predict_answer_tokens = inputs.input_ids[<span class="hljs-number">0</span>, answer_start_index : answer_end_index + <span class="hljs-number">1</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(predict_answer_tokens)
<span class="hljs-string">&#x27;176 billion parameters and can generate text in 46 languages natural languages and 13&#x27;</span>`,wrap:!1}}),{c(){l=T("p"),l.textContent=i,t=o(),f(c.$$.fragment),J=o(),U=T("p"),U.innerHTML=G,R=o(),f(X.$$.fragment),A=o(),b=T("p"),b.textContent=z,x=o(),f(C.$$.fragment),W=o(),_=T("p"),_.textContent=E,v=o(),f(Z.$$.fragment)},l(e){l=y(e,"P",{"data-svelte-h":!0}),g(l)!=="svelte-1bnmfqq"&&(l.textContent=i),t=m(e),d(c.$$.fragment,e),J=m(e),U=y(e,"P",{"data-svelte-h":!0}),g(U)!=="svelte-1rvunpz"&&(U.innerHTML=G),R=m(e),d(X.$$.fragment,e),A=m(e),b=y(e,"P",{"data-svelte-h":!0}),g(b)!=="svelte-bytl0h"&&(b.textContent=z),x=m(e),d(C.$$.fragment,e),W=m(e),_=y(e,"P",{"data-svelte-h":!0}),g(_)!=="svelte-1yq6861"&&(_.textContent=E),v=m(e),d(Z.$$.fragment,e)},m(e,M){n(e,l,M),n(e,t,M),u(c,e,M),n(e,J,M),n(e,U,M),n(e,R,M),u(X,e,M),n(e,A,M),n(e,b,M),n(e,x,M),u(C,e,M),n(e,W,M),n(e,_,M),n(e,v,M),u(Z,e,M),B=!0},p:es,i(e){B||(h(c.$$.fragment,e),h(X.$$.fragment,e),h(C.$$.fragment,e),h(Z.$$.fragment,e),B=!0)},o(e){j(c.$$.fragment,e),j(X.$$.fragment,e),j(C.$$.fragment,e),j(Z.$$.fragment,e),B=!1},d(e){e&&(a(l),a(t),a(J),a(U),a(R),a(A),a(b),a(x),a(W),a(_),a(v)),w(c,e),w(X,e),w(C,e),w(Z,e)}}}function ut(I){let l,i;return l=new vs({props:{$$slots:{default:[dt]},$$scope:{ctx:I}}}),{c(){f(l.$$.fragment)},l(t){d(l.$$.fragment,t)},m(t,c){u(l,t,c),i=!0},p(t,c){const J={};c&2&&(J.$$scope={dirty:c,ctx:t}),l.$set(J)},i(t){i||(h(l.$$.fragment,t),i=!0)},o(t){j(l.$$.fragment,t),i=!1},d(t){w(l,t)}}}function ht(I){let l,i,t,c,J,U,G,R,X,A,b,z="質問応答タスクは、質問に対して回答を返します。 Alexa、Siri、Google などの仮想アシスタントに天気を尋ねたことがあるなら、質問応答モデルを使用したことがあるはずです。質問応答タスクには一般的に 2 つのタイプがあります。",x,C,W="<li>抽出: 与えられたコンテキストから回答を抽出します。</li> <li>抽象的: 質問に正しく答えるコンテキストから回答を生成します。</li>",_,E,v="このガイドでは、次の方法を説明します。",Z,B,e='<li>抽出的質問応答用に <a href="https://huggingface.co/datasets/squad" rel="nofollow">SQuAD</a> データセット上の <a href="https://huggingface.co/distilbert-base-uncased" rel="nofollow">DistilBERT</a> を微調整します。</li> <li>微調整したモデルを推論に使用します。</li>',M,V,ts,H,Y="始める前に、必要なライブラリがすべてインストールされていることを確認してください。",S,F,ls,Q,L="モデルをアップロードしてコミュニティと共有できるように、Hugging Face アカウントにログインすることをお勧めします。プロンプトが表示されたら、トークンを入力してログインします。",P,N,as,q,r,$,As="まず、🤗 データセット ライブラリから SQuAD データセットの小さいサブセットを読み込みます。これにより、完全なデータセットのトレーニングにさらに時間を費やす前に、実験してすべてが機能することを確認する機会が得られます。",Ws,ns,Es,ps,Je="<code>train_test_split</code> メソッドを使用して、データセットの <code>train</code> 分割をトレイン セットとテスト セットに分割します。",Vs,rs,zs,os,ge="次に、例を見てみましょう。",Fs,ms,Ns,is,be="ここにはいくつかの重要なフィールドがあります。",Hs,cs,$e="<li><code>answers</code>: 回答トークンと回答テキストの開始位置。</li> <li><code>context</code>: モデルが答えを抽出するために必要な背景情報。</li> <li><code>question</code>: モデルが答える必要がある質問。</li>",Qs,Ms,qs,fs,Ys,ds,Ue="次のステップでは、DistilBERT トークナイザーをロードして<code>question</code>フィールドと<code>context</code>フィールドを処理します。",Ls,us,Ss,hs,_e="質問応答タスクに特有の、注意すべき前処理手順がいくつかあります。",Ps,js,Ce=`<li>データセット内の一部の例には、モデルの最大入力長を超える非常に長い「コンテキスト」が含まれる場合があります。より長いシーケンスを処理するには、<code>truncation=&quot;only_second&quot;</code> を設定して <code>context</code> のみを切り捨てます。</li> <li>次に、設定によって、回答の開始位置と終了位置を元の <code>context</code>にマッピングします。
「<code>return_offset_mapping=True</code>」。</li> <li>マッピングが手元にあるので、答えの開始トークンと終了トークンを見つけることができます。 <code>sequence_ids</code> メソッドを使用して、
オフセットのどの部分が<code>question</code>に対応し、どの部分が<code>context</code>に対応するかを見つけます。</li>`,Ds,ws,Ie="以下に、<code>answer</code>の開始トークンと終了トークンを切り詰めて<code>context</code>にマッピングする関数を作成する方法を示します。",Ks,Ts,Os,ys,Ze="データセット全体に前処理関数を適用するには、🤗 Datasets <code>map</code> 関数を使用します。 <code>batched=True</code> を設定してデータセットの複数の要素を一度に処理することで、<code>map</code> 関数を高速化できます。不要な列を削除します。",se,Js,ee,gs,Re="次に、<code>DefaultDataCollat​​or</code> を使用してサンプルのバッチを作成します。 🤗 Transformers の他のデータ照合器とは異なり、<code>DefaultDataCollat​​or</code> はパディングなどの追加の前処理を適用しません。",te,D,le,bs,ae,K,ne,O,pe,$s,re,Us,Xe='質問応答の評価には、大量の後処理が必要です。時間がかかりすぎないように、このガイドでは評価ステップを省略しています。 <a href="/docs/transformers/v4.37.0/ja/main_classes/trainer#transformers.Trainer">Trainer</a> はトレーニング中に評価損失を計算するため、モデルのパフォーマンスについて完全に分からないわけではありません。',oe,_s,ke='もっと時間があり、質問応答用のモデルを評価する方法に興味がある場合は、<a href="https://huggingface.co/course/chapter7/7?fw=pt#postprocessing" rel="nofollow">質問応答</a> の章を参照してください。 🤗ハグフェイスコースから！',me,Cs,ie,Is,xe="モデルを微調整したので、それを推論に使用できるようになりました。",ce,Zs,Be="質問と、モデルに予測させたいコンテキストを考え出します。",Me,Rs,fe,Xs,ve='推論用に微調整されたモデルを試す最も簡単な方法は、それを <a href="/docs/transformers/v4.37.0/ja/main_classes/pipelines#transformers.pipeline">pipeline()</a> で使用することです。モデルを使用して質問応答用の<code>pipeline</code>をインスタンス化し、それにテキストを渡します。',de,ks,ue,xs,Ae="必要に応じて、<code>pipeline</code>の結果を手動で複製することもできます。",he,ss,je,Gs,we;return J=new Bs({props:{title:"Question answering",local:"question-answering",headingTag:"h1"}}),G=new Oe({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ja/question_answering.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ja/pytorch/question_answering.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ja/tensorflow/question_answering.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ja/question_answering.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ja/pytorch/question_answering.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ja/tensorflow/question_answering.ipynb"}]}}),X=new He({props:{id:"ajPx5LwJD-I"}}),V=new Te({props:{$$slots:{default:[st]},$$scope:{ctx:I}}}),F=new k({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwZXZhbHVhdGU=",highlighted:"pip install transformers datasets evaluate",wrap:!1}}),N=new k({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_login()`,wrap:!1}}),q=new Bs({props:{title:"Load SQuAD dataset",local:"load-squad-dataset",headingTag:"h2"}}),ns=new k({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBc3F1YWQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyc3F1YWQlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTVCJTNBNTAwMCU1RCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>squad = load_dataset(<span class="hljs-string">&quot;squad&quot;</span>, split=<span class="hljs-string">&quot;train[:5000]&quot;</span>)`,wrap:!1}}),rs=new k({props:{code:"c3F1YWQlMjAlM0QlMjBzcXVhZC50cmFpbl90ZXN0X3NwbGl0KHRlc3Rfc2l6ZSUzRDAuMik=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>squad = squad.train_test_split(test_size=<span class="hljs-number">0.2</span>)',wrap:!1}}),ms=new k({props:{code:"c3F1YWQlNUIlMjJ0cmFpbiUyMiU1RCU1QjAlNUQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>squad[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers&#x27;</span>: {<span class="hljs-string">&#x27;answer_start&#x27;</span>: [<span class="hljs-number">515</span>], <span class="hljs-string">&#x27;text&#x27;</span>: [<span class="hljs-string">&#x27;Saint Bernadette Soubirous&#x27;</span>]},
 <span class="hljs-string">&#x27;context&#x27;</span>: <span class="hljs-string">&#x27;Architecturally, the school has a Catholic character. Atop the Main Building\\&#x27;s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend &quot;Venite Ad Me Omnes&quot;. Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.&#x27;</span>,
 <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;5733be284776f41900661182&#x27;</span>,
 <span class="hljs-string">&#x27;question&#x27;</span>: <span class="hljs-string">&#x27;To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;University_of_Notre_Dame&#x27;</span>
}`,wrap:!1}}),Ms=new Bs({props:{title:"Preprocess",local:"preprocess",headingTag:"h2"}}),fs=new He({props:{id:"qgaM0weJHpA"}}),us=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`,wrap:!1}}),Ts=new k({props:{code:"ZGVmJTIwcHJlcHJvY2Vzc19mdW5jdGlvbihleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjBxdWVzdGlvbnMlMjAlM0QlMjAlNUJxLnN0cmlwKCklMjBmb3IlMjBxJTIwaW4lMjBleGFtcGxlcyU1QiUyMnF1ZXN0aW9uJTIyJTVEJTVEJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXN0aW9ucyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV4YW1wbGVzJTVCJTIyY29udGV4dCUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1heF9sZW5ndGglM0QzODQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cnVuY2F0aW9uJTNEJTIyb25seV9zZWNvbmQlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm5fb2Zmc2V0c19tYXBwaW5nJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBhZGRpbmclM0QlMjJtYXhfbGVuZ3RoJTIyJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMG9mZnNldF9tYXBwaW5nJTIwJTNEJTIwaW5wdXRzLnBvcCglMjJvZmZzZXRfbWFwcGluZyUyMiklMEElMjAlMjAlMjAlMjBhbnN3ZXJzJTIwJTNEJTIwZXhhbXBsZXMlNUIlMjJhbnN3ZXJzJTIyJTVEJTBBJTIwJTIwJTIwJTIwc3RhcnRfcG9zaXRpb25zJTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwZW5kX3Bvc2l0aW9ucyUyMCUzRCUyMCU1QiU1RCUwQSUwQSUyMCUyMCUyMCUyMGZvciUyMGklMkMlMjBvZmZzZXQlMjBpbiUyMGVudW1lcmF0ZShvZmZzZXRfbWFwcGluZyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXIlMjAlM0QlMjBhbnN3ZXJzJTVCaSU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0YXJ0X2NoYXIlMjAlM0QlMjBhbnN3ZXIlNUIlMjJhbnN3ZXJfc3RhcnQlMjIlNUQlNUIwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5kX2NoYXIlMjAlM0QlMjBhbnN3ZXIlNUIlMjJhbnN3ZXJfc3RhcnQlMjIlNUQlNUIwJTVEJTIwJTJCJTIwbGVuKGFuc3dlciU1QiUyMnRleHQlMjIlNUQlNUIwJTVEKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlcXVlbmNlX2lkcyUyMCUzRCUyMGlucHV0cy5zZXF1ZW5jZV9pZHMoaSklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBGaW5kJTIwdGhlJTIwc3RhcnQlMjBhbmQlMjBlbmQlMjBvZiUyMHRoZSUyMGNvbnRleHQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZHglMjAlM0QlMjAwJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2hpbGUlMjBzZXF1ZW5jZV9pZHMlNUJpZHglNUQlMjAhJTNEJTIwMSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlkeCUyMCUyQiUzRCUyMDElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb250ZXh0X3N0YXJ0JTIwJTNEJTIwaWR4JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2hpbGUlMjBzZXF1ZW5jZV9pZHMlNUJpZHglNUQlMjAlM0QlM0QlMjAxJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWR4JTIwJTJCJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbnRleHRfZW5kJTIwJTNEJTIwaWR4JTIwLSUyMDElMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBJZiUyMHRoZSUyMGFuc3dlciUyMGlzJTIwbm90JTIwZnVsbHklMjBpbnNpZGUlMjB0aGUlMjBjb250ZXh0JTJDJTIwbGFiZWwlMjBpdCUyMCgwJTJDJTIwMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMG9mZnNldCU1QmNvbnRleHRfc3RhcnQlNUQlNUIwJTVEJTIwJTNFJTIwZW5kX2NoYXIlMjBvciUyMG9mZnNldCU1QmNvbnRleHRfZW5kJTVEJTVCMSU1RCUyMCUzQyUyMHN0YXJ0X2NoYXIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdGFydF9wb3NpdGlvbnMuYXBwZW5kKDApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5kX3Bvc2l0aW9ucy5hcHBlbmQoMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbHNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwT3RoZXJ3aXNlJTIwaXQncyUyMHRoZSUyMHN0YXJ0JTIwYW5kJTIwZW5kJTIwdG9rZW4lMjBwb3NpdGlvbnMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZHglMjAlM0QlMjBjb250ZXh0X3N0YXJ0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2hpbGUlMjBpZHglMjAlM0MlM0QlMjBjb250ZXh0X2VuZCUyMGFuZCUyMG9mZnNldCU1QmlkeCU1RCU1QjAlNUQlMjAlM0MlM0QlMjBzdGFydF9jaGFyJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWR4JTIwJTJCJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0YXJ0X3Bvc2l0aW9ucy5hcHBlbmQoaWR4JTIwLSUyMDEpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWR4JTIwJTNEJTIwY29udGV4dF9lbmQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3aGlsZSUyMGlkeCUyMCUzRSUzRCUyMGNvbnRleHRfc3RhcnQlMjBhbmQlMjBvZmZzZXQlNUJpZHglNUQlNUIxJTVEJTIwJTNFJTNEJTIwZW5kX2NoYXIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZHglMjAtJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVuZF9wb3NpdGlvbnMuYXBwZW5kKGlkeCUyMCUyQiUyMDEpJTBBJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTVCJTIyc3RhcnRfcG9zaXRpb25zJTIyJTVEJTIwJTNEJTIwc3RhcnRfcG9zaXRpb25zJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTVCJTIyZW5kX3Bvc2l0aW9ucyUyMiU1RCUyMCUzRCUyMGVuZF9wb3NpdGlvbnMlMEElMjAlMjAlMjAlMjByZXR1cm4lMjBpbnB1dHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    questions = [q.strip() <span class="hljs-keyword">for</span> q <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;question&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = tokenizer(
<span class="hljs-meta">... </span>        questions,
<span class="hljs-meta">... </span>        examples[<span class="hljs-string">&quot;context&quot;</span>],
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">384</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-string">&quot;only_second&quot;</span>,
<span class="hljs-meta">... </span>        return_offsets_mapping=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-string">&quot;max_length&quot;</span>,
<span class="hljs-meta">... </span>    )

<span class="hljs-meta">... </span>    offset_mapping = inputs.pop(<span class="hljs-string">&quot;offset_mapping&quot;</span>)
<span class="hljs-meta">... </span>    answers = examples[<span class="hljs-string">&quot;answers&quot;</span>]
<span class="hljs-meta">... </span>    start_positions = []
<span class="hljs-meta">... </span>    end_positions = []

<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, offset <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(offset_mapping):
<span class="hljs-meta">... </span>        answer = answers[i]
<span class="hljs-meta">... </span>        start_char = answer[<span class="hljs-string">&quot;answer_start&quot;</span>][<span class="hljs-number">0</span>]
<span class="hljs-meta">... </span>        end_char = answer[<span class="hljs-string">&quot;answer_start&quot;</span>][<span class="hljs-number">0</span>] + <span class="hljs-built_in">len</span>(answer[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
<span class="hljs-meta">... </span>        sequence_ids = inputs.sequence_ids(i)

<span class="hljs-meta">... </span>        <span class="hljs-comment"># Find the start and end of the context</span>
<span class="hljs-meta">... </span>        idx = <span class="hljs-number">0</span>
<span class="hljs-meta">... </span>        <span class="hljs-keyword">while</span> sequence_ids[idx] != <span class="hljs-number">1</span>:
<span class="hljs-meta">... </span>            idx += <span class="hljs-number">1</span>
<span class="hljs-meta">... </span>        context_start = idx
<span class="hljs-meta">... </span>        <span class="hljs-keyword">while</span> sequence_ids[idx] == <span class="hljs-number">1</span>:
<span class="hljs-meta">... </span>            idx += <span class="hljs-number">1</span>
<span class="hljs-meta">... </span>        context_end = idx - <span class="hljs-number">1</span>

<span class="hljs-meta">... </span>        <span class="hljs-comment"># If the answer is not fully inside the context, label it (0, 0)</span>
<span class="hljs-meta">... </span>        <span class="hljs-keyword">if</span> offset[context_start][<span class="hljs-number">0</span>] &gt; end_char <span class="hljs-keyword">or</span> offset[context_end][<span class="hljs-number">1</span>] &lt; start_char:
<span class="hljs-meta">... </span>            start_positions.append(<span class="hljs-number">0</span>)
<span class="hljs-meta">... </span>            end_positions.append(<span class="hljs-number">0</span>)
<span class="hljs-meta">... </span>        <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>            <span class="hljs-comment"># Otherwise it&#x27;s the start and end token positions</span>
<span class="hljs-meta">... </span>            idx = context_start
<span class="hljs-meta">... </span>            <span class="hljs-keyword">while</span> idx &lt;= context_end <span class="hljs-keyword">and</span> offset[idx][<span class="hljs-number">0</span>] &lt;= start_char:
<span class="hljs-meta">... </span>                idx += <span class="hljs-number">1</span>
<span class="hljs-meta">... </span>            start_positions.append(idx - <span class="hljs-number">1</span>)

<span class="hljs-meta">... </span>            idx = context_end
<span class="hljs-meta">... </span>            <span class="hljs-keyword">while</span> idx &gt;= context_start <span class="hljs-keyword">and</span> offset[idx][<span class="hljs-number">1</span>] &gt;= end_char:
<span class="hljs-meta">... </span>                idx -= <span class="hljs-number">1</span>
<span class="hljs-meta">... </span>            end_positions.append(idx + <span class="hljs-number">1</span>)

<span class="hljs-meta">... </span>    inputs[<span class="hljs-string">&quot;start_positions&quot;</span>] = start_positions
<span class="hljs-meta">... </span>    inputs[<span class="hljs-string">&quot;end_positions&quot;</span>] = end_positions
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`,wrap:!1}}),Js=new k({props:{code:"dG9rZW5pemVkX3NxdWFkJTIwJTNEJTIwc3F1YWQubWFwKHByZXByb2Nlc3NfZnVuY3Rpb24lMkMlMjBiYXRjaGVkJTNEVHJ1ZSUyQyUyMHJlbW92ZV9jb2x1bW5zJTNEc3F1YWQlNUIlMjJ0cmFpbiUyMiU1RC5jb2x1bW5fbmFtZXMp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_squad = squad.<span class="hljs-built_in">map</span>(preprocess_function, batched=<span class="hljs-literal">True</span>, remove_columns=squad[<span class="hljs-string">&quot;train&quot;</span>].column_names)',wrap:!1}}),D=new ye({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[at],pytorch:[tt]},$$scope:{ctx:I}}}),bs=new Bs({props:{title:"Train",local:"train",headingTag:"h2"}}),K=new ye({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[it],pytorch:[rt]},$$scope:{ctx:I}}}),O=new Te({props:{$$slots:{default:[ct]},$$scope:{ctx:I}}}),$s=new Bs({props:{title:"Evaluate",local:"evaluate",headingTag:"h2"}}),Cs=new Bs({props:{title:"Inference",local:"inference",headingTag:"h2"}}),Rs=new k({props:{code:"cXVlc3Rpb24lMjAlM0QlMjAlMjJIb3clMjBtYW55JTIwcHJvZ3JhbW1pbmclMjBsYW5ndWFnZXMlMjBkb2VzJTIwQkxPT00lMjBzdXBwb3J0JTNGJTIyJTBBY29udGV4dCUyMCUzRCUyMCUyMkJMT09NJTIwaGFzJTIwMTc2JTIwYmlsbGlvbiUyMHBhcmFtZXRlcnMlMjBhbmQlMjBjYW4lMjBnZW5lcmF0ZSUyMHRleHQlMjBpbiUyMDQ2JTIwbGFuZ3VhZ2VzJTIwbmF0dXJhbCUyMGxhbmd1YWdlcyUyMGFuZCUyMDEzJTIwcHJvZ3JhbW1pbmclMjBsYW5ndWFnZXMuJTIy",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>question = <span class="hljs-string">&quot;How many programming languages does BLOOM support?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>context = <span class="hljs-string">&quot;BLOOM has 176 billion parameters and can generate text in 46 languages natural languages and 13 programming languages.&quot;</span>`,wrap:!1}}),ks=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBcXVlc3Rpb25fYW5zd2VyZXIlMjAlM0QlMjBwaXBlbGluZSglMjJxdWVzdGlvbi1hbnN3ZXJpbmclMjIlMkMlMjBtb2RlbCUzRCUyMm15X2F3ZXNvbWVfcWFfbW9kZWwlMjIpJTBBcXVlc3Rpb25fYW5zd2VyZXIocXVlc3Rpb24lM0RxdWVzdGlvbiUyQyUyMGNvbnRleHQlM0Rjb250ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>question_answerer = pipeline(<span class="hljs-string">&quot;question-answering&quot;</span>, model=<span class="hljs-string">&quot;my_awesome_qa_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>question_answerer(question=question, context=context)
{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.2058267742395401</span>,
 <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">10</span>,
 <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">95</span>,
 <span class="hljs-string">&#x27;answer&#x27;</span>: <span class="hljs-string">&#x27;176 billion parameters and can generate text in 46 languages natural languages and 13&#x27;</span>}`,wrap:!1}}),ss=new ye({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[ut],pytorch:[ft]},$$scope:{ctx:I}}}),{c(){l=T("meta"),i=o(),t=T("p"),c=o(),f(J.$$.fragment),U=o(),f(G.$$.fragment),R=o(),f(X.$$.fragment),A=o(),b=T("p"),b.textContent=z,x=o(),C=T("ul"),C.innerHTML=W,_=o(),E=T("p"),E.textContent=v,Z=o(),B=T("ol"),B.innerHTML=e,M=o(),f(V.$$.fragment),ts=o(),H=T("p"),H.textContent=Y,S=o(),f(F.$$.fragment),ls=o(),Q=T("p"),Q.textContent=L,P=o(),f(N.$$.fragment),as=o(),f(q.$$.fragment),r=o(),$=T("p"),$.textContent=As,Ws=o(),f(ns.$$.fragment),Es=o(),ps=T("p"),ps.innerHTML=Je,Vs=o(),f(rs.$$.fragment),zs=o(),os=T("p"),os.textContent=ge,Fs=o(),f(ms.$$.fragment),Ns=o(),is=T("p"),is.textContent=be,Hs=o(),cs=T("ul"),cs.innerHTML=$e,Qs=o(),f(Ms.$$.fragment),qs=o(),f(fs.$$.fragment),Ys=o(),ds=T("p"),ds.innerHTML=Ue,Ls=o(),f(us.$$.fragment),Ss=o(),hs=T("p"),hs.textContent=_e,Ps=o(),js=T("ol"),js.innerHTML=Ce,Ds=o(),ws=T("p"),ws.innerHTML=Ie,Ks=o(),f(Ts.$$.fragment),Os=o(),ys=T("p"),ys.innerHTML=Ze,se=o(),f(Js.$$.fragment),ee=o(),gs=T("p"),gs.innerHTML=Re,te=o(),f(D.$$.fragment),le=o(),f(bs.$$.fragment),ae=o(),f(K.$$.fragment),ne=o(),f(O.$$.fragment),pe=o(),f($s.$$.fragment),re=o(),Us=T("p"),Us.innerHTML=Xe,oe=o(),_s=T("p"),_s.innerHTML=ke,me=o(),f(Cs.$$.fragment),ie=o(),Is=T("p"),Is.textContent=xe,ce=o(),Zs=T("p"),Zs.textContent=Be,Me=o(),f(Rs.$$.fragment),fe=o(),Xs=T("p"),Xs.innerHTML=ve,de=o(),f(ks.$$.fragment),ue=o(),xs=T("p"),xs.innerHTML=Ae,he=o(),f(ss.$$.fragment),je=o(),Gs=T("p"),this.h()},l(s){const p=Se("svelte-u9bgzb",document.head);l=y(p,"META",{name:!0,content:!0}),p.forEach(a),i=m(s),t=y(s,"P",{}),Fe(t).forEach(a),c=m(s),d(J.$$.fragment,s),U=m(s),d(G.$$.fragment,s),R=m(s),d(X.$$.fragment,s),A=m(s),b=y(s,"P",{"data-svelte-h":!0}),g(b)!=="svelte-1tgwu4t"&&(b.textContent=z),x=m(s),C=y(s,"UL",{"data-svelte-h":!0}),g(C)!=="svelte-ku9lav"&&(C.innerHTML=W),_=m(s),E=y(s,"P",{"data-svelte-h":!0}),g(E)!=="svelte-w5jzhi"&&(E.textContent=v),Z=m(s),B=y(s,"OL",{"data-svelte-h":!0}),g(B)!=="svelte-10ircjf"&&(B.innerHTML=e),M=m(s),d(V.$$.fragment,s),ts=m(s),H=y(s,"P",{"data-svelte-h":!0}),g(H)!=="svelte-1lya3k8"&&(H.textContent=Y),S=m(s),d(F.$$.fragment,s),ls=m(s),Q=y(s,"P",{"data-svelte-h":!0}),g(Q)!=="svelte-193zy02"&&(Q.textContent=L),P=m(s),d(N.$$.fragment,s),as=m(s),d(q.$$.fragment,s),r=m(s),$=y(s,"P",{"data-svelte-h":!0}),g($)!=="svelte-12zmynt"&&($.textContent=As),Ws=m(s),d(ns.$$.fragment,s),Es=m(s),ps=y(s,"P",{"data-svelte-h":!0}),g(ps)!=="svelte-m4w3yq"&&(ps.innerHTML=Je),Vs=m(s),d(rs.$$.fragment,s),zs=m(s),os=y(s,"P",{"data-svelte-h":!0}),g(os)!=="svelte-1r6oj5w"&&(os.textContent=ge),Fs=m(s),d(ms.$$.fragment,s),Ns=m(s),is=y(s,"P",{"data-svelte-h":!0}),g(is)!=="svelte-y9dab"&&(is.textContent=be),Hs=m(s),cs=y(s,"UL",{"data-svelte-h":!0}),g(cs)!=="svelte-1kxnjgl"&&(cs.innerHTML=$e),Qs=m(s),d(Ms.$$.fragment,s),qs=m(s),d(fs.$$.fragment,s),Ys=m(s),ds=y(s,"P",{"data-svelte-h":!0}),g(ds)!=="svelte-1bamyze"&&(ds.innerHTML=Ue),Ls=m(s),d(us.$$.fragment,s),Ss=m(s),hs=y(s,"P",{"data-svelte-h":!0}),g(hs)!=="svelte-lhw9mt"&&(hs.textContent=_e),Ps=m(s),js=y(s,"OL",{"data-svelte-h":!0}),g(js)!=="svelte-1syxwtd"&&(js.innerHTML=Ce),Ds=m(s),ws=y(s,"P",{"data-svelte-h":!0}),g(ws)!=="svelte-1ryfrne"&&(ws.innerHTML=Ie),Ks=m(s),d(Ts.$$.fragment,s),Os=m(s),ys=y(s,"P",{"data-svelte-h":!0}),g(ys)!=="svelte-t1eqy4"&&(ys.innerHTML=Ze),se=m(s),d(Js.$$.fragment,s),ee=m(s),gs=y(s,"P",{"data-svelte-h":!0}),g(gs)!=="svelte-me6jz3"&&(gs.innerHTML=Re),te=m(s),d(D.$$.fragment,s),le=m(s),d(bs.$$.fragment,s),ae=m(s),d(K.$$.fragment,s),ne=m(s),d(O.$$.fragment,s),pe=m(s),d($s.$$.fragment,s),re=m(s),Us=y(s,"P",{"data-svelte-h":!0}),g(Us)!=="svelte-yhd700"&&(Us.innerHTML=Xe),oe=m(s),_s=y(s,"P",{"data-svelte-h":!0}),g(_s)!=="svelte-ihy4ok"&&(_s.innerHTML=ke),me=m(s),d(Cs.$$.fragment,s),ie=m(s),Is=y(s,"P",{"data-svelte-h":!0}),g(Is)!=="svelte-cyrfc8"&&(Is.textContent=xe),ce=m(s),Zs=y(s,"P",{"data-svelte-h":!0}),g(Zs)!=="svelte-151wogl"&&(Zs.textContent=Be),Me=m(s),d(Rs.$$.fragment,s),fe=m(s),Xs=y(s,"P",{"data-svelte-h":!0}),g(Xs)!=="svelte-1x03gx8"&&(Xs.innerHTML=ve),de=m(s),d(ks.$$.fragment,s),ue=m(s),xs=y(s,"P",{"data-svelte-h":!0}),g(xs)!=="svelte-p649vi"&&(xs.innerHTML=Ae),he=m(s),d(ss.$$.fragment,s),je=m(s),Gs=y(s,"P",{}),Fe(Gs).forEach(a),this.h()},h(){Ne(l,"name","hf:doc:metadata"),Ne(l,"content",jt)},m(s,p){Pe(document.head,l),n(s,i,p),n(s,t,p),n(s,c,p),u(J,s,p),n(s,U,p),u(G,s,p),n(s,R,p),u(X,s,p),n(s,A,p),n(s,b,p),n(s,x,p),n(s,C,p),n(s,_,p),n(s,E,p),n(s,Z,p),n(s,B,p),n(s,M,p),u(V,s,p),n(s,ts,p),n(s,H,p),n(s,S,p),u(F,s,p),n(s,ls,p),n(s,Q,p),n(s,P,p),u(N,s,p),n(s,as,p),u(q,s,p),n(s,r,p),n(s,$,p),n(s,Ws,p),u(ns,s,p),n(s,Es,p),n(s,ps,p),n(s,Vs,p),u(rs,s,p),n(s,zs,p),n(s,os,p),n(s,Fs,p),u(ms,s,p),n(s,Ns,p),n(s,is,p),n(s,Hs,p),n(s,cs,p),n(s,Qs,p),u(Ms,s,p),n(s,qs,p),u(fs,s,p),n(s,Ys,p),n(s,ds,p),n(s,Ls,p),u(us,s,p),n(s,Ss,p),n(s,hs,p),n(s,Ps,p),n(s,js,p),n(s,Ds,p),n(s,ws,p),n(s,Ks,p),u(Ts,s,p),n(s,Os,p),n(s,ys,p),n(s,se,p),u(Js,s,p),n(s,ee,p),n(s,gs,p),n(s,te,p),u(D,s,p),n(s,le,p),u(bs,s,p),n(s,ae,p),u(K,s,p),n(s,ne,p),u(O,s,p),n(s,pe,p),u($s,s,p),n(s,re,p),n(s,Us,p),n(s,oe,p),n(s,_s,p),n(s,me,p),u(Cs,s,p),n(s,ie,p),n(s,Is,p),n(s,ce,p),n(s,Zs,p),n(s,Me,p),u(Rs,s,p),n(s,fe,p),n(s,Xs,p),n(s,de,p),u(ks,s,p),n(s,ue,p),n(s,xs,p),n(s,he,p),u(ss,s,p),n(s,je,p),n(s,Gs,p),we=!0},p(s,[p]){const Ge={};p&2&&(Ge.$$scope={dirty:p,ctx:s}),V.$set(Ge);const We={};p&2&&(We.$$scope={dirty:p,ctx:s}),D.$set(We);const Ee={};p&2&&(Ee.$$scope={dirty:p,ctx:s}),K.$set(Ee);const Ve={};p&2&&(Ve.$$scope={dirty:p,ctx:s}),O.$set(Ve);const ze={};p&2&&(ze.$$scope={dirty:p,ctx:s}),ss.$set(ze)},i(s){we||(h(J.$$.fragment,s),h(G.$$.fragment,s),h(X.$$.fragment,s),h(V.$$.fragment,s),h(F.$$.fragment,s),h(N.$$.fragment,s),h(q.$$.fragment,s),h(ns.$$.fragment,s),h(rs.$$.fragment,s),h(ms.$$.fragment,s),h(Ms.$$.fragment,s),h(fs.$$.fragment,s),h(us.$$.fragment,s),h(Ts.$$.fragment,s),h(Js.$$.fragment,s),h(D.$$.fragment,s),h(bs.$$.fragment,s),h(K.$$.fragment,s),h(O.$$.fragment,s),h($s.$$.fragment,s),h(Cs.$$.fragment,s),h(Rs.$$.fragment,s),h(ks.$$.fragment,s),h(ss.$$.fragment,s),we=!0)},o(s){j(J.$$.fragment,s),j(G.$$.fragment,s),j(X.$$.fragment,s),j(V.$$.fragment,s),j(F.$$.fragment,s),j(N.$$.fragment,s),j(q.$$.fragment,s),j(ns.$$.fragment,s),j(rs.$$.fragment,s),j(ms.$$.fragment,s),j(Ms.$$.fragment,s),j(fs.$$.fragment,s),j(us.$$.fragment,s),j(Ts.$$.fragment,s),j(Js.$$.fragment,s),j(D.$$.fragment,s),j(bs.$$.fragment,s),j(K.$$.fragment,s),j(O.$$.fragment,s),j($s.$$.fragment,s),j(Cs.$$.fragment,s),j(Rs.$$.fragment,s),j(ks.$$.fragment,s),j(ss.$$.fragment,s),we=!1},d(s){s&&(a(i),a(t),a(c),a(U),a(R),a(A),a(b),a(x),a(C),a(_),a(E),a(Z),a(B),a(M),a(ts),a(H),a(S),a(ls),a(Q),a(P),a(as),a(r),a($),a(Ws),a(Es),a(ps),a(Vs),a(zs),a(os),a(Fs),a(Ns),a(is),a(Hs),a(cs),a(Qs),a(qs),a(Ys),a(ds),a(Ls),a(Ss),a(hs),a(Ps),a(js),a(Ds),a(ws),a(Ks),a(Os),a(ys),a(se),a(ee),a(gs),a(te),a(le),a(ae),a(ne),a(pe),a(re),a(Us),a(oe),a(_s),a(me),a(ie),a(Is),a(ce),a(Zs),a(Me),a(fe),a(Xs),a(de),a(ue),a(xs),a(he),a(je),a(Gs)),a(l),w(J,s),w(G,s),w(X,s),w(V,s),w(F,s),w(N,s),w(q,s),w(ns,s),w(rs,s),w(ms,s),w(Ms,s),w(fs,s),w(us,s),w(Ts,s),w(Js,s),w(D,s),w(bs,s),w(K,s),w(O,s),w($s,s),w(Cs,s),w(Rs,s),w(ks,s),w(ss,s)}}}const jt='{"title":"Question answering","local":"question-answering","sections":[{"title":"Load SQuAD dataset","local":"load-squad-dataset","sections":[],"depth":2},{"title":"Preprocess","local":"preprocess","sections":[],"depth":2},{"title":"Train","local":"train","sections":[],"depth":2},{"title":"Evaluate","local":"evaluate","sections":[],"depth":2},{"title":"Inference","local":"inference","sections":[],"depth":2}],"depth":1}';function wt(I){return qe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ct extends Ye{constructor(l){super(),Le(this,l,wt,ht,Qe,{})}}export{Ct as component};
