import{s as ne,o as ae}from"../chunks/scheduler.cc52f4b9.js";import{S as pe,i as oe,e as M,s as p,c as u,h as ie,a as U,d as s,b as o,f as Kl,g as c,j as y,k as le,l as re,m as n,n as m,o as r,u as ee,t as J,p as T,v as se}from"../chunks/index.9480bb2b.js";import{C as Je,H as Bl,E as ue}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.009dc98d.js";import{C as w}from"../chunks/CodeBlock.184aadb6.js";import{C as te}from"../chunks/CourseFloatingBanner.6488bde3.js";import{F as ce}from"../chunks/FrameworkSwitchCourse.42d12053.js";function me(j){let a,i;return a=new te({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_tf.ipynb"}]}}),{c(){u(a.$$.fragment)},l(t){c(a.$$.fragment,t)},m(t,b){m(a,t,b),i=!0},i(t){i||(J(a.$$.fragment,t),i=!0)},o(t){r(a.$$.fragment,t),i=!1},d(t){T(a,t)}}}function Te(j){let a,i;return a=new te({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_pt.ipynb"}]}}),{c(){u(a.$$.fragment)},l(t){c(a.$$.fragment,t)},m(t,b){m(a,t,b),i=!0},i(t){i||(J(a.$$.fragment,t),i=!0)},o(t){r(a.$$.fragment,t),i=!1},d(t){T(a,t)}}}function Me(j){let a,i;return a=new w({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMFRGQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbiUwQSUwQWNoZWNrcG9pbnQlMjAlM0QlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZC1maW5ldHVuZWQtc3N0LTItZW5nbGlzaCUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBbW9kZWwlMjAlM0QlMjBURkF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBc2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEF0b2tlbnMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwoKip0b2tlbnMp",highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){u(a.$$.fragment)},l(t){c(a.$$.fragment,t)},m(t,b){m(a,t,b),i=!0},i(t){i||(J(a.$$.fragment,t),i=!0)},o(t){r(a.$$.fragment,t),i=!1},d(t){T(a,t)}}}function Ue(j){let a,i;return a=new w({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFzZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMkMlMjAlMjJTbyUyMGhhdmUlMjBJISUyMiU1RCUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbCgqKnRva2Vucyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){u(a.$$.fragment)},l(t){c(a.$$.fragment,t)},m(t,b){m(a,t,b),i=!0},i(t){i||(J(a.$$.fragment,t),i=!0)},o(t){r(a.$$.fragment,t),i=!1},d(t){T(a,t)}}}function be(j){let a,i,t,b,k,nl,g,al,V,pl,f,d,el,Z,Sl="在过去的几个章节中，我们已经尝试尽可能手动完成大部分工作。我们探索了 tokenizer 的运行机制，并且了解了分词、转换为 inputs ID、填充、截断以及注意力掩码的处理方式。",ol,$,zl="然而，正如我们在第二节中看到的那样，🤗 Transformers API 能够通过一个高级函数为我们处理所有这些工作，接下来我们就要深入研究这个函数。当你直接在句子上调用你的 <code>tokenizer</code> 时，就可以得到转换后的可以直接放入模型的数据了：",il,N,rl,E,Cl="这里， <code>model_inputs</code> 变量包含模型运行所需的所有数据。在 DistilBERT 中，它包括 inputs ID 和注意力掩码（attention mask）。其他接受额外输入的模型也会有对应的 tokenizer 可以将输入转化为模型所需要的输入。",Jl,Q,_l="正如我们将在下面的一些例子中看到的，这个函数非常强大。首先，它可以对单个句子进行处理：",ul,R,cl,W,Gl="它还一次处理多个多个，并且 API 的用法完全一致：",ml,B,Tl,S,Yl="它可以使用多种不同的方法对目标进行填充：",Ml,z,Ul,C,Fl="它还可以对序列进行截断：",bl,_,yl,G,Xl="<code>tokenizer</code> 对象可以处理指定框架张量的转换，然后可以直接发送到模型。例如，在下面的代码示例中，我们告诉 tokenizer 返回不同框架的张量 —— <code>&quot;pt&quot;</code> 返回 PyTorch 张量， <code>&quot;tf&quot;</code> 返回 TensorFlow 张量，而 <code>&quot;np&quot;</code> 则返回 NumPy 数组：",wl,Y,jl,F,fl,X,xl="如果我们看一下 tokenizer 返回的 inputs ID，我们会发现它们与之前的略有不同：",dl,x,hl,v,Il,q,vl="句子的开始和结束分别增加了一个 inputs ID。我们来解码上述的两个 ID 序列，看看是怎么回事：",kl,H,gl,A,Vl,D,ql="tokenizer 在开头添加了特殊单词 <code>[CLS]</code> ，在结尾添加了特殊单词 <code>[SEP]</code> 。这是因为模型在预训练时使用了这些字词，所以为了得到相同的推断结果，我们也需要添加它们。请注意，有些模型不添加特殊单词，或者添加不同的特殊单词；模型也可能只在开头或结尾添加这些特殊单词。无论如何，tokenizer 知道哪些是必需的，并会为你处理这些问题。",Zl,O,$l,L,Hl="现在我们已经看到 <code>tokenizer</code> 对象在处理文本时的所有步骤，让我们最后再看一次它如何通过 tokenizer  API 处理多个序列（填充！），非常长的序列（截断！）以及多种类型的张量：",Nl,h,I,sl,P,El,tl,Ql;k=new ce({props:{fw:j[0]}}),g=new Je({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),V=new Bl({props:{title:"综合应用",local:"综合应用",headingTag:"h1"}});const Al=[Te,me],K=[];function Dl(l,e){return l[0]==="pt"?0:1}f=Dl(j),d=K[f]=Al[f](j),N=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQXNlcXVlbmNlJTIwJTNEJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),R=new w({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2Up",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),B=new w({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`,wrap:!1}}),z=new w({props:{code:"JTIzJTIwJUU1JUIwJTg2JUU1JThGJUE1JUU1JUFEJTkwJUU1JUJBJThGJUU1JTg4JTk3JUU1JUExJUFCJUU1JTg1JTg1JUU1JTg4JUIwJUU2JTlDJTgwJUU5JTk1JUJGJUU1JThGJUE1JUU1JUFEJTkwJUU3JTlBJTg0JUU5JTk1JUJGJUU1JUJBJUE2JTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0QlMjJsb25nZXN0JTIyKSUwQSUwQSUyMyUyMCVFNSVCMCU4NiVFNSU4RiVBNSVFNSVBRCU5MCVFNSVCQSU4RiVFNSU4OCU5NyVFNSVBMSVBQiVFNSU4NSU4NSVFNSU4OCVCMCVFNiVBOCVBMSVFNSU5RSU4QiVFNyU5QSU4NCVFNiU5QyU4MCVFNSVBNCVBNyVFOSU5NSVCRiVFNSVCQSVBNiUwQSUyMyUyMCg1MTIlMjBmb3IlMjBCRVJUJTIwb3IlMjBEaXN0aWxCRVJUKSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiklMEElMEElMjMlMjAlRTUlQjAlODYlRTUlOEYlQTUlRTUlQUQlOTAlRTUlQkElOEYlRTUlODglOTclRTUlQTElQUIlRTUlODUlODUlRTUlODglQjAlRTYlOEMlODclRTUlQUUlOUElRTclOUElODQlRTYlOUMlODAlRTUlQTQlQTclRTklOTUlQkYlRTUlQkElQTYlMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMm1heF9sZW5ndGglMjIlMkMlMjBtYXhfbGVuZ3RoJTNEOCk=",highlighted:`<span class="hljs-comment"># 将句子序列填充到最长句子的长度</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># 将句子序列填充到模型的最大长度</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># 将句子序列填充到指定的最大长度</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`,wrap:!1}}),_=new w({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjAlRTUlQjAlODYlRTYlODglQUElRTYlOTYlQUQlRTYlQUYlOTQlRTYlQTglQTElRTUlOUUlOEIlRTYlOUMlODAlRTUlQTQlQTclRTklOTUlQkYlRTUlQkElQTYlRTklOTUlQkYlRTclOUElODQlRTUlOEYlQTUlRTUlQUQlOTAlRTUlQkElOEYlRTUlODglOTclMEElMjMlMjAoNTEyJTIwZm9yJTIwQkVSVCUyMG9yJTIwRGlzdGlsQkVSVCklMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUpJTBBJTBBJTIzJTIwJUU1JUIwJTg2JUU2JTg4JUFBJUU2JTk2JUFEJUU5JTk1JUJGJUU0JUJBJThFJUU2JThDJTg3JUU1JUFFJTlBJUU2JTlDJTgwJUU1JUE0JUE3JUU5JTk1JUJGJUU1JUJBJUE2JUU3JTlBJTg0JUU1JThGJUE1JUU1JUFEJTkwJUU1JUJBJThGJUU1JTg4JTk3JTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMG1heF9sZW5ndGglM0Q4JTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUp",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># 将截断比模型最大长度长的句子序列</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># 将截断长于指定最大长度的句子序列</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`,wrap:!1}}),Y=new w({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjAlRTglQkYlOTQlRTUlOUIlOUUlMjBQeVRvcmNoJTIwdGVuc29ycyUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBJTIzJTIwJUU4JUJGJTk0JUU1JTlCJTlFJTIwVGVuc29yRmxvdyUyMHRlbnNvcnMlMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKSUwQSUwQSUyMyUyMCVFOCVCRiU5NCVFNSU5QiU5RSUyME51bVB5JTIwYXJyYXlzJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJucCUyMik=",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># 返回 PyTorch tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># 返回 TensorFlow tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)

<span class="hljs-comment"># 返回 NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`,wrap:!1}}),F=new Bl({props:{title:"特殊的 tokens",local:"特殊的 tokens",headingTag:"h2"}}),x=new w({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2UpJTBBcHJpbnQobW9kZWxfaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci50b2tlbml6ZShzZXF1ZW5jZSklMEFpZHMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF90b2tlbnNfdG9faWRzKHRva2VucyklMEFwcmludChpZHMp",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`,wrap:!1}}),v=new w({props:{code:"JTVCMTAxJTJDJTIwMTA0NSUyQyUyMDEwMDUlMkMlMjAyMzEwJTJDJTIwMjA0MiUyQyUyMDM0MDMlMkMlMjAyMDA1JTJDJTIwMTAzNyUyQyUyMDE3NjYyJTJDJTIwMTIxNzIlMkMlMjAyNjA3JTJDJTIwMjAyNiUyQyUyMDI4NzglMkMlMjAyMTY2JTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUwQSU1QjEwNDUlMkMlMjAxMDA1JTJDJTIwMjMxMCUyQyUyMDIwNDIlMkMlMjAzNDAzJTJDJTIwMjAwNSUyQyUyMDEwMzclMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMjYwNyUyQyUyMDIwMjYlMkMlMjAyODc4JTJDJTIwMjE2NiUyQyUyMDEwMTIlNUQ=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`,wrap:!1}}),H=new w({props:{code:"cHJpbnQodG9rZW5pemVyLmRlY29kZShtb2RlbF9pbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUoaWRzKSk=",highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`,wrap:!1}}),A=new w({props:{code:"JTIyJTVCQ0xTJTVEJTIwaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIwJTVCU0VQJTVEJTIyJTBBJTIyaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIy",highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`,wrap:!1}}),O=new Bl({props:{title:"小结：从 tokenizer 到模型",local:"小结：从 tokenizer 到模型",headingTag:"h2"}});const Ol=[Ue,Me],ll=[];function Ll(l,e){return l[0]==="pt"?0:1}return h=Ll(j),I=ll[h]=Ol[h](j),P=new ue({props:{source:"https://github.com/huggingface/course/blob/main/chapters/zh-CN/chapter2/6.mdx"}}),{c(){a=M("meta"),i=p(),t=M("p"),b=p(),u(k.$$.fragment),nl=p(),u(g.$$.fragment),al=p(),u(V.$$.fragment),pl=p(),d.c(),el=p(),Z=M("p"),Z.textContent=Sl,ol=p(),$=M("p"),$.innerHTML=zl,il=p(),u(N.$$.fragment),rl=p(),E=M("p"),E.innerHTML=Cl,Jl=p(),Q=M("p"),Q.textContent=_l,ul=p(),u(R.$$.fragment),cl=p(),W=M("p"),W.textContent=Gl,ml=p(),u(B.$$.fragment),Tl=p(),S=M("p"),S.textContent=Yl,Ml=p(),u(z.$$.fragment),Ul=p(),C=M("p"),C.textContent=Fl,bl=p(),u(_.$$.fragment),yl=p(),G=M("p"),G.innerHTML=Xl,wl=p(),u(Y.$$.fragment),jl=p(),u(F.$$.fragment),fl=p(),X=M("p"),X.textContent=xl,dl=p(),u(x.$$.fragment),hl=p(),u(v.$$.fragment),Il=p(),q=M("p"),q.textContent=vl,kl=p(),u(H.$$.fragment),gl=p(),u(A.$$.fragment),Vl=p(),D=M("p"),D.innerHTML=ql,Zl=p(),u(O.$$.fragment),$l=p(),L=M("p"),L.innerHTML=Hl,Nl=p(),I.c(),sl=p(),u(P.$$.fragment),El=p(),tl=M("p"),this.h()},l(l){const e=ie("svelte-u9bgzb",document.head);a=U(e,"META",{name:!0,content:!0}),e.forEach(s),i=o(l),t=U(l,"P",{}),Kl(t).forEach(s),b=o(l),c(k.$$.fragment,l),nl=o(l),c(g.$$.fragment,l),al=o(l),c(V.$$.fragment,l),pl=o(l),d.l(l),el=o(l),Z=U(l,"P",{"data-svelte-h":!0}),y(Z)!=="svelte-khgo0k"&&(Z.textContent=Sl),ol=o(l),$=U(l,"P",{"data-svelte-h":!0}),y($)!=="svelte-10bo7tr"&&($.innerHTML=zl),il=o(l),c(N.$$.fragment,l),rl=o(l),E=U(l,"P",{"data-svelte-h":!0}),y(E)!=="svelte-1yxjh7p"&&(E.innerHTML=Cl),Jl=o(l),Q=U(l,"P",{"data-svelte-h":!0}),y(Q)!=="svelte-1yhxykx"&&(Q.textContent=_l),ul=o(l),c(R.$$.fragment,l),cl=o(l),W=U(l,"P",{"data-svelte-h":!0}),y(W)!=="svelte-171v091"&&(W.textContent=Gl),ml=o(l),c(B.$$.fragment,l),Tl=o(l),S=U(l,"P",{"data-svelte-h":!0}),y(S)!=="svelte-bj8bs1"&&(S.textContent=Yl),Ml=o(l),c(z.$$.fragment,l),Ul=o(l),C=U(l,"P",{"data-svelte-h":!0}),y(C)!=="svelte-u3ow8k"&&(C.textContent=Fl),bl=o(l),c(_.$$.fragment,l),yl=o(l),G=U(l,"P",{"data-svelte-h":!0}),y(G)!=="svelte-1nlo48x"&&(G.innerHTML=Xl),wl=o(l),c(Y.$$.fragment,l),jl=o(l),c(F.$$.fragment,l),fl=o(l),X=U(l,"P",{"data-svelte-h":!0}),y(X)!=="svelte-isnfts"&&(X.textContent=xl),dl=o(l),c(x.$$.fragment,l),hl=o(l),c(v.$$.fragment,l),Il=o(l),q=U(l,"P",{"data-svelte-h":!0}),y(q)!=="svelte-1z0h8tc"&&(q.textContent=vl),kl=o(l),c(H.$$.fragment,l),gl=o(l),c(A.$$.fragment,l),Vl=o(l),D=U(l,"P",{"data-svelte-h":!0}),y(D)!=="svelte-vskbu9"&&(D.innerHTML=ql),Zl=o(l),c(O.$$.fragment,l),$l=o(l),L=U(l,"P",{"data-svelte-h":!0}),y(L)!=="svelte-zcw8dw"&&(L.innerHTML=Hl),Nl=o(l),I.l(l),sl=o(l),c(P.$$.fragment,l),El=o(l),tl=U(l,"P",{}),Kl(tl).forEach(s),this.h()},h(){le(a,"name","hf:doc:metadata"),le(a,"content",ye)},m(l,e){re(document.head,a),n(l,i,e),n(l,t,e),n(l,b,e),m(k,l,e),n(l,nl,e),m(g,l,e),n(l,al,e),m(V,l,e),n(l,pl,e),K[f].m(l,e),n(l,el,e),n(l,Z,e),n(l,ol,e),n(l,$,e),n(l,il,e),m(N,l,e),n(l,rl,e),n(l,E,e),n(l,Jl,e),n(l,Q,e),n(l,ul,e),m(R,l,e),n(l,cl,e),n(l,W,e),n(l,ml,e),m(B,l,e),n(l,Tl,e),n(l,S,e),n(l,Ml,e),m(z,l,e),n(l,Ul,e),n(l,C,e),n(l,bl,e),m(_,l,e),n(l,yl,e),n(l,G,e),n(l,wl,e),m(Y,l,e),n(l,jl,e),m(F,l,e),n(l,fl,e),n(l,X,e),n(l,dl,e),m(x,l,e),n(l,hl,e),m(v,l,e),n(l,Il,e),n(l,q,e),n(l,kl,e),m(H,l,e),n(l,gl,e),m(A,l,e),n(l,Vl,e),n(l,D,e),n(l,Zl,e),m(O,l,e),n(l,$l,e),n(l,L,e),n(l,Nl,e),ll[h].m(l,e),n(l,sl,e),m(P,l,e),n(l,El,e),n(l,tl,e),Ql=!0},p(l,[e]){const Pl={};e&1&&(Pl.fw=l[0]),k.$set(Pl);let Rl=f;f=Dl(l),f!==Rl&&(se(),r(K[Rl],1,1,()=>{K[Rl]=null}),ee(),d=K[f],d||(d=K[f]=Al[f](l),d.c()),J(d,1),d.m(el.parentNode,el));let Wl=h;h=Ll(l),h!==Wl&&(se(),r(ll[Wl],1,1,()=>{ll[Wl]=null}),ee(),I=ll[h],I||(I=ll[h]=Ol[h](l),I.c()),J(I,1),I.m(sl.parentNode,sl))},i(l){Ql||(J(k.$$.fragment,l),J(g.$$.fragment,l),J(V.$$.fragment,l),J(d),J(N.$$.fragment,l),J(R.$$.fragment,l),J(B.$$.fragment,l),J(z.$$.fragment,l),J(_.$$.fragment,l),J(Y.$$.fragment,l),J(F.$$.fragment,l),J(x.$$.fragment,l),J(v.$$.fragment,l),J(H.$$.fragment,l),J(A.$$.fragment,l),J(O.$$.fragment,l),J(I),J(P.$$.fragment,l),Ql=!0)},o(l){r(k.$$.fragment,l),r(g.$$.fragment,l),r(V.$$.fragment,l),r(d),r(N.$$.fragment,l),r(R.$$.fragment,l),r(B.$$.fragment,l),r(z.$$.fragment,l),r(_.$$.fragment,l),r(Y.$$.fragment,l),r(F.$$.fragment,l),r(x.$$.fragment,l),r(v.$$.fragment,l),r(H.$$.fragment,l),r(A.$$.fragment,l),r(O.$$.fragment,l),r(I),r(P.$$.fragment,l),Ql=!1},d(l){l&&(s(i),s(t),s(b),s(nl),s(al),s(pl),s(el),s(Z),s(ol),s($),s(il),s(rl),s(E),s(Jl),s(Q),s(ul),s(cl),s(W),s(ml),s(Tl),s(S),s(Ml),s(Ul),s(C),s(bl),s(yl),s(G),s(wl),s(jl),s(fl),s(X),s(dl),s(hl),s(Il),s(q),s(kl),s(gl),s(Vl),s(D),s(Zl),s($l),s(L),s(Nl),s(sl),s(El),s(tl)),s(a),T(k,l),T(g,l),T(V,l),K[f].d(l),T(N,l),T(R,l),T(B,l),T(z,l),T(_,l),T(Y,l),T(F,l),T(x,l),T(v,l),T(H,l),T(A,l),T(O,l),ll[h].d(l),T(P,l)}}}const ye='{"title":"综合应用","local":"综合应用","sections":[{"title":"特殊的 tokens","local":"特殊的 tokens","sections":[],"depth":2},{"title":"小结：从 tokenizer 到模型","local":"小结：从 tokenizer 到模型","sections":[],"depth":2}],"depth":1}';function we(j,a,i){let t="pt";return ae(()=>{const b=new URLSearchParams(window.location.search);i(0,t=b.get("fw")||"pt")}),[t]}class ge extends pe{constructor(a){super(),oe(this,a,we,be,ne,{})}}export{ge as component};
