import{s as Qe,o as Fe}from"../chunks/scheduler.cc52f4b9.js";import{S as _e,i as Ae,e as M,s as n,c as p,h as De,a as c,d as l,b as a,f as He,g as i,j as h,k as Ye,l as Pe,m as s,n as o,t as r,o as m,p as u}from"../chunks/index.5b747fde.js";import{C as Le,H as Ve,E as Ke}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.d38ac5a9.js";import{C as y}from"../chunks/CodeBlock.665bfc37.js";import{C as Oe}from"../chunks/CourseFloatingBanner.b33fcd79.js";import{F as et}from"../chunks/FrameworkSwitchCourse.b3b657b3.js";function tt(ke){let d,A,w,D,b,L,J,K,T,O,j,ee,f,We="In the last few sections, we’ve been trying our best to do most of the work by hand. We’ve explored how tokenizers work and looked at tokenization, conversion to input IDs, padding, truncation, and attention masks.",te,I,Be="However, as we saw in section 2, the 🤗 Transformers API can handle all of this for us with a high-level function that we’ll dive into here. When you call your <code>tokenizer</code> directly on the sentence, you get back inputs that are ready to pass through your model:",le,U,se,g,xe="Here, the <code>model_inputs</code> variable contains everything that’s necessary for a model to operate well. For DistilBERT, that includes the input IDs as well as the attention mask. Other models that accept additional inputs will also have those output by the <code>tokenizer</code> object.",ne,Z,ve="As we’ll see in some examples below, this method is very powerful. First, it can tokenize a single sequence:",ae,$,pe,k,Xe="It also handles multiple sequences at a time, with no change in the API:",ie,V,oe,W,ze="It can pad according to several objectives:",re,B,me,x,Se="It can also truncate sequences:",ue,v,Me,X,Ge="The <code>tokenizer</code> object can handle the conversion to specific framework tensors, which can then be directly sent to the model. For example, in the following code sample we are prompting the tokenizer to return tensors from the different frameworks — <code>&quot;pt&quot;</code> returns PyTorch tensors and <code>&quot;np&quot;</code> returns NumPy arrays:",ce,z,he,S,ye,G,Ee="If we take a look at the input IDs returned by the tokenizer, we will see they are a tiny bit different from what we had earlier:",de,E,we,q,be,C,qe="One token ID was added at the beginning, and one at the end. Let’s decode the two sequences of IDs above to see what this is about:",Je,R,Te,N,je,H,Ce="The tokenizer added the special word <code>[CLS]</code> at the beginning and the special word <code>[SEP]</code> at the end. This is because the model was pretrained with those, so to get the same results for inference we need to add them as well. Note that some models don’t add special words, or add different ones; models may also add these special words only at the beginning, or only at the end. In any case, the tokenizer knows which ones are expected and will deal with this for you.",fe,Y,Ie,Q,Re="Now that we’ve seen all the individual steps the <code>tokenizer</code> object uses when applied on texts, let’s see one final time how it can handle multiple sequences (padding!), very long sequences (truncation!), and multiple types of tensors with its main API:",Ue,F,ge,_,Ze,P,$e;return b=new et({props:{fw:ke[0]}}),J=new Le({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),T=new Ve({props:{title:"Putting it all together",local:"putting-it-all-together",headingTag:"h1"}}),j=new Oe({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"}]}}),U=new y({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQXNlcXVlbmNlJTIwJTNEJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),$=new y({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2Up",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),V=new y({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`,wrap:!1}}),B=new y({props:{code:"JTIzJTIwV2lsbCUyMHBhZCUyMHRoZSUyMHNlcXVlbmNlcyUyMHVwJTIwdG8lMjB0aGUlMjBtYXhpbXVtJTIwc2VxdWVuY2UlMjBsZW5ndGglMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMmxvbmdlc3QlMjIpJTBBJTBBJTIzJTIwV2lsbCUyMHBhZCUyMHRoZSUyMHNlcXVlbmNlcyUyMHVwJTIwdG8lMjB0aGUlMjBtb2RlbCUyMG1heCUyMGxlbmd0aCUwQSUyMyUyMCg1MTIlMjBmb3IlMjBCRVJUJTIwb3IlMjBEaXN0aWxCRVJUKSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiklMEElMEElMjMlMjBXaWxsJTIwcGFkJTIwdGhlJTIwc2VxdWVuY2VzJTIwdXAlMjB0byUyMHRoZSUyMHNwZWNpZmllZCUyMG1heCUyMGxlbmd0aCUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiUyQyUyMG1heF9sZW5ndGglM0Q4KQ==",highlighted:`<span class="hljs-comment"># Will pad the sequences up to the maximum sequence length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Will pad the sequences up to the specified max length</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`,wrap:!1}}),v=new y({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBXaWxsJTIwdHJ1bmNhdGUlMjB0aGUlMjBzZXF1ZW5jZXMlMjB0aGF0JTIwYXJlJTIwbG9uZ2VyJTIwdGhhbiUyMHRoZSUyMG1vZGVsJTIwbWF4JTIwbGVuZ3RoJTBBJTIzJTIwKDUxMiUyMGZvciUyMEJFUlQlMjBvciUyMERpc3RpbEJFUlQpJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHRydW5jYXRpb24lM0RUcnVlKSUwQSUwQSUyMyUyMFdpbGwlMjB0cnVuY2F0ZSUyMHRoZSUyMHNlcXVlbmNlcyUyMHRoYXQlMjBhcmUlMjBsb25nZXIlMjB0aGFuJTIwdGhlJTIwc3BlY2lmaWVkJTIwbWF4JTIwbGVuZ3RoJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMG1heF9sZW5ndGglM0Q4JTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUp",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Will truncate the sequences that are longer than the model max length</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Will truncate the sequences that are longer than the specified max length</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`,wrap:!1}}),z=new y({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBSZXR1cm5zJTIwUHlUb3JjaCUyMHRlbnNvcnMlMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQSUyMyUyMFJldHVybnMlMjBOdW1QeSUyMGFycmF5cyUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIybnAlMjIp",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Returns PyTorch tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Returns NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`,wrap:!1}}),S=new Ve({props:{title:"Special tokens",local:"special-tokens",headingTag:"h2"}}),E=new y({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2UpJTBBcHJpbnQobW9kZWxfaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci50b2tlbml6ZShzZXF1ZW5jZSklMEFpZHMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF90b2tlbnNfdG9faWRzKHRva2VucyklMEFwcmludChpZHMp",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`,wrap:!1}}),q=new y({props:{code:"JTVCMTAxJTJDJTIwMTA0NSUyQyUyMDEwMDUlMkMlMjAyMzEwJTJDJTIwMjA0MiUyQyUyMDM0MDMlMkMlMjAyMDA1JTJDJTIwMTAzNyUyQyUyMDE3NjYyJTJDJTIwMTIxNzIlMkMlMjAyNjA3JTJDJTIwMjAyNiUyQyUyMDI4NzglMkMlMjAyMTY2JTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUwQSU1QjEwNDUlMkMlMjAxMDA1JTJDJTIwMjMxMCUyQyUyMDIwNDIlMkMlMjAzNDAzJTJDJTIwMjAwNSUyQyUyMDEwMzclMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMjYwNyUyQyUyMDIwMjYlMkMlMjAyODc4JTJDJTIwMjE2NiUyQyUyMDEwMTIlNUQ=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`,wrap:!1}}),R=new y({props:{code:"cHJpbnQodG9rZW5pemVyLmRlY29kZShtb2RlbF9pbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUoaWRzKSk=",highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`,wrap:!1}}),N=new y({props:{code:"JTIyJTVCQ0xTJTVEJTIwaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIwJTVCU0VQJTVEJTIyJTBBJTIyaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIy",highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`,wrap:!1}}),Y=new Ve({props:{title:"Wrapping up: From tokenizer to model",local:"wrapping-up-from-tokenizer-to-model",headingTag:"h2"}}),F=new y({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFzZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMkMlMjAlMjJTbyUyMGhhdmUlMjBJISUyMiU1RCUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbCgqKnRva2Vucyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`,wrap:!1}}),_=new Ke({props:{source:"https://github.com/huggingface/course/blob/main/chapters/en/chapter2/6.mdx"}}),{c(){d=M("meta"),A=n(),w=M("p"),D=n(),p(b.$$.fragment),L=n(),p(J.$$.fragment),K=n(),p(T.$$.fragment),O=n(),p(j.$$.fragment),ee=n(),f=M("p"),f.textContent=We,te=n(),I=M("p"),I.innerHTML=Be,le=n(),p(U.$$.fragment),se=n(),g=M("p"),g.innerHTML=xe,ne=n(),Z=M("p"),Z.textContent=ve,ae=n(),p($.$$.fragment),pe=n(),k=M("p"),k.textContent=Xe,ie=n(),p(V.$$.fragment),oe=n(),W=M("p"),W.textContent=ze,re=n(),p(B.$$.fragment),me=n(),x=M("p"),x.textContent=Se,ue=n(),p(v.$$.fragment),Me=n(),X=M("p"),X.innerHTML=Ge,ce=n(),p(z.$$.fragment),he=n(),p(S.$$.fragment),ye=n(),G=M("p"),G.textContent=Ee,de=n(),p(E.$$.fragment),we=n(),p(q.$$.fragment),be=n(),C=M("p"),C.textContent=qe,Je=n(),p(R.$$.fragment),Te=n(),p(N.$$.fragment),je=n(),H=M("p"),H.innerHTML=Ce,fe=n(),p(Y.$$.fragment),Ie=n(),Q=M("p"),Q.innerHTML=Re,Ue=n(),p(F.$$.fragment),ge=n(),p(_.$$.fragment),Ze=n(),P=M("p"),this.h()},l(e){const t=De("svelte-u9bgzb",document.head);d=c(t,"META",{name:!0,content:!0}),t.forEach(l),A=a(e),w=c(e,"P",{}),He(w).forEach(l),D=a(e),i(b.$$.fragment,e),L=a(e),i(J.$$.fragment,e),K=a(e),i(T.$$.fragment,e),O=a(e),i(j.$$.fragment,e),ee=a(e),f=c(e,"P",{"data-svelte-h":!0}),h(f)!=="svelte-8x3t43"&&(f.textContent=We),te=a(e),I=c(e,"P",{"data-svelte-h":!0}),h(I)!=="svelte-1m0etnl"&&(I.innerHTML=Be),le=a(e),i(U.$$.fragment,e),se=a(e),g=c(e,"P",{"data-svelte-h":!0}),h(g)!=="svelte-1ppe5qh"&&(g.innerHTML=xe),ne=a(e),Z=c(e,"P",{"data-svelte-h":!0}),h(Z)!=="svelte-y34ype"&&(Z.textContent=ve),ae=a(e),i($.$$.fragment,e),pe=a(e),k=c(e,"P",{"data-svelte-h":!0}),h(k)!=="svelte-164017d"&&(k.textContent=Xe),ie=a(e),i(V.$$.fragment,e),oe=a(e),W=c(e,"P",{"data-svelte-h":!0}),h(W)!=="svelte-lvhia5"&&(W.textContent=ze),re=a(e),i(B.$$.fragment,e),me=a(e),x=c(e,"P",{"data-svelte-h":!0}),h(x)!=="svelte-rx5s8"&&(x.textContent=Se),ue=a(e),i(v.$$.fragment,e),Me=a(e),X=c(e,"P",{"data-svelte-h":!0}),h(X)!=="svelte-17mlhci"&&(X.innerHTML=Ge),ce=a(e),i(z.$$.fragment,e),he=a(e),i(S.$$.fragment,e),ye=a(e),G=c(e,"P",{"data-svelte-h":!0}),h(G)!=="svelte-1tipyqt"&&(G.textContent=Ee),de=a(e),i(E.$$.fragment,e),we=a(e),i(q.$$.fragment,e),be=a(e),C=c(e,"P",{"data-svelte-h":!0}),h(C)!=="svelte-s6ie2x"&&(C.textContent=qe),Je=a(e),i(R.$$.fragment,e),Te=a(e),i(N.$$.fragment,e),je=a(e),H=c(e,"P",{"data-svelte-h":!0}),h(H)!=="svelte-yfjwtr"&&(H.innerHTML=Ce),fe=a(e),i(Y.$$.fragment,e),Ie=a(e),Q=c(e,"P",{"data-svelte-h":!0}),h(Q)!=="svelte-mudtzg"&&(Q.innerHTML=Re),Ue=a(e),i(F.$$.fragment,e),ge=a(e),i(_.$$.fragment,e),Ze=a(e),P=c(e,"P",{}),He(P).forEach(l),this.h()},h(){Ye(d,"name","hf:doc:metadata"),Ye(d,"content",lt)},m(e,t){Pe(document.head,d),s(e,A,t),s(e,w,t),s(e,D,t),o(b,e,t),s(e,L,t),o(J,e,t),s(e,K,t),o(T,e,t),s(e,O,t),o(j,e,t),s(e,ee,t),s(e,f,t),s(e,te,t),s(e,I,t),s(e,le,t),o(U,e,t),s(e,se,t),s(e,g,t),s(e,ne,t),s(e,Z,t),s(e,ae,t),o($,e,t),s(e,pe,t),s(e,k,t),s(e,ie,t),o(V,e,t),s(e,oe,t),s(e,W,t),s(e,re,t),o(B,e,t),s(e,me,t),s(e,x,t),s(e,ue,t),o(v,e,t),s(e,Me,t),s(e,X,t),s(e,ce,t),o(z,e,t),s(e,he,t),o(S,e,t),s(e,ye,t),s(e,G,t),s(e,de,t),o(E,e,t),s(e,we,t),o(q,e,t),s(e,be,t),s(e,C,t),s(e,Je,t),o(R,e,t),s(e,Te,t),o(N,e,t),s(e,je,t),s(e,H,t),s(e,fe,t),o(Y,e,t),s(e,Ie,t),s(e,Q,t),s(e,Ue,t),o(F,e,t),s(e,ge,t),o(_,e,t),s(e,Ze,t),s(e,P,t),$e=!0},p(e,[t]){const Ne={};t&1&&(Ne.fw=e[0]),b.$set(Ne)},i(e){$e||(r(b.$$.fragment,e),r(J.$$.fragment,e),r(T.$$.fragment,e),r(j.$$.fragment,e),r(U.$$.fragment,e),r($.$$.fragment,e),r(V.$$.fragment,e),r(B.$$.fragment,e),r(v.$$.fragment,e),r(z.$$.fragment,e),r(S.$$.fragment,e),r(E.$$.fragment,e),r(q.$$.fragment,e),r(R.$$.fragment,e),r(N.$$.fragment,e),r(Y.$$.fragment,e),r(F.$$.fragment,e),r(_.$$.fragment,e),$e=!0)},o(e){m(b.$$.fragment,e),m(J.$$.fragment,e),m(T.$$.fragment,e),m(j.$$.fragment,e),m(U.$$.fragment,e),m($.$$.fragment,e),m(V.$$.fragment,e),m(B.$$.fragment,e),m(v.$$.fragment,e),m(z.$$.fragment,e),m(S.$$.fragment,e),m(E.$$.fragment,e),m(q.$$.fragment,e),m(R.$$.fragment,e),m(N.$$.fragment,e),m(Y.$$.fragment,e),m(F.$$.fragment,e),m(_.$$.fragment,e),$e=!1},d(e){e&&(l(A),l(w),l(D),l(L),l(K),l(O),l(ee),l(f),l(te),l(I),l(le),l(se),l(g),l(ne),l(Z),l(ae),l(pe),l(k),l(ie),l(oe),l(W),l(re),l(me),l(x),l(ue),l(Me),l(X),l(ce),l(he),l(ye),l(G),l(de),l(we),l(be),l(C),l(Je),l(Te),l(je),l(H),l(fe),l(Ie),l(Q),l(Ue),l(ge),l(Ze),l(P)),l(d),u(b,e),u(J,e),u(T,e),u(j,e),u(U,e),u($,e),u(V,e),u(B,e),u(v,e),u(z,e),u(S,e),u(E,e),u(q,e),u(R,e),u(N,e),u(Y,e),u(F,e),u(_,e)}}}const lt='{"title":"Putting it all together","local":"putting-it-all-together","sections":[{"title":"Special tokens","local":"special-tokens","sections":[],"depth":2},{"title":"Wrapping up: From tokenizer to model","local":"wrapping-up-from-tokenizer-to-model","sections":[],"depth":2}],"depth":1}';function st(ke,d,A){let w="pt";return Fe(()=>{const D=new URLSearchParams(window.location.search);A(0,w=D.get("fw")||"pt")}),[w]}class mt extends _e{constructor(d){super(),Ae(this,d,st,tt,Qe,{})}}export{mt as component};
