import{s as el,o as ll}from"../chunks/scheduler.37c15a92.js";import{S as sl,i as tl,g as b,s as i,r as u,A as nl,h as J,f as t,c as o,j as De,u as m,x as T,k as Pe,y as al,a as n,v as M,t as r,b as Le,d as c,w as d,p as Ke}from"../chunks/index.2bf4358c.js";import{C as w}from"../chunks/CodeBlock.4f5fc1ad.js";import{C as Oe}from"../chunks/CourseFloatingBanner.06521e8b.js";import{F as il}from"../chunks/FrameworkSwitchCourse.8d4d4ab6.js";import{H as $e}from"../chunks/Heading.8ada512a.js";function ol(j){let a,p;return a=new Oe({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_tf.ipynb"}]}}),{c(){u(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,y){M(a,s,y),p=!0},i(s){p||(c(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){d(a,s)}}}function pl(j){let a,p;return a=new Oe({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"}]}}),{c(){u(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,y){M(a,s,y),p=!0},i(s){p||(c(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){d(a,s)}}}function rl(j){let a,p;return a=new w({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMFRGQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbiUwQSUwQWNoZWNrcG9pbnQlMjAlM0QlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZC1maW5ldHVuZWQtc3N0LTItZW5nbGlzaCUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBbW9kZWwlMjAlM0QlMjBURkF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBc2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEF0b2tlbnMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwoKip0b2tlbnMp",highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){u(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,y){M(a,s,y),p=!0},i(s){p||(c(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){d(a,s)}}}function cl(j){let a,p;return a=new w({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFzZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMkMlMjAlMjJTbyUyMGhhdmUlMjBJISUyMiU1RCUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbCgqKnRva2Vucyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){u(a.$$.fragment)},l(s){m(a.$$.fragment,s)},m(s,y){M(a,s,y),p=!0},i(s){p||(c(a.$$.fragment,s),p=!0)},o(s){r(a.$$.fragment,s),p=!1},d(s){d(a,s)}}}function ul(j){let a,p,s,y,U,se,Z,te,h,f,O,k,Be="Nelle ultime sezioni abbiamo fatto del nostro meglio per fare la maggior parte del lavoro a mano. Abbiamo esplorato il funzionamento dei tokenizer e abbiamo esaminato la tokenizzazione, la conversione in ID di input, il padding, il troncamento e le maschere di attenzione.",ne,z,ve="Tuttavia, come abbiamo visto nella sezione 2, l’API 🤗 Transformers può gestire tutto questo con una funzione di alto livello che approfondiremo qui. Quando si chiama il <code>tokenizer</code> direttamente sulla frase, si ottengono input pronti per passare attraverso il modello:",ae,V,ie,W,Ge="Qui, la variabile <code>model_inputs</code> contiene tutto ciò che è necessario per il buon funzionamento del modello. Per DistilBERT, questo include gli ID degli ingressi e la maschera di attenzione. Altri modelli che accettano input aggiuntivi avranno anche questi output dall’oggetto <code>tokenizer</code>.",oe,$,Se="Come vedremo in alcuni esempi, questo metodo è molto potente. Innanzitutto, può tokenizzare una singola sequenza:",pe,B,re,v,qe="Gestisce anche più sequenze alla volta, senza alcuna modifica dell’API:",ce,G,ue,S,Xe="Possiamo implementare il padding in diversi modi",me,q,Me,X,Re="Può anche troncare le sequenze:",de,R,be,_,_e="L’oggetto <code>tokenizer</code> può gestire la conversione in tensori di framework specifici, che possono successivamente essere inviati direttamente al modello. Per esempio, nel seguente esempio di codice si chiede al tokenizer di restituire i tensori dei diversi framework: <code>&quot;pt&quot;</code> restituisce i tensori di PyTorch, <code>&quot;tf&quot;</code> restituisce i tensori di TensorFlow e <code>&quot;np&quot;</code> restituisce gli array di NumPy:",Je,E,ye,x,Te,N,Ee="Se diamo un’occhiata agli ID di input restituiti dal tokenizer, noteremo che sono leggermente diversi da quelli che avevamo prima:",we,Y,je,Q,he,C,xe="Un ID token è stato aggiunto all’inizio e uno alla fine. Decodifichiamo le due sequenze di ID qui sopra per capire di cosa si tratta:",fe,F,ge,H,Ie,A,Ne="Il tokenizer ha aggiunto la parola speciale <code>[CLS]</code> all’inizio e la parola speciale <code>[SEP]</code> alla fine. Questo perché il modello è stato preaddestrato con queste parole, quindi per ottenere gli stessi risultati per l’inferenza dobbiamo aggiungerle anche noi. Si noti che alcuni modelli non aggiungono parole speciali, o ne aggiungono di diverse; i modelli possono anche aggiungere queste parole speciali solo all’inizio o solo alla fine. In ogni caso, il tokenizer sa quali sono previste e se ne occuperà per voi.",Ue,D,Ze,P,Ye="Ora che abbiamo visto tutti i singoli passaggi che l’oggetto <code>tokenizer</code> utilizza quando viene applicato ai testi, vediamo un’ultima volta come può gestire sequenze multiple (padding!), sequenze molto lunghe (troncamento!) e diversi tipi di tensori con la sua API principale:",ke,g,I,ee,le,ze;U=new il({props:{fw:j[0]}}),Z=new $e({props:{title:"Mettiamo insieme i pezzi",local:"mettiamo-insieme-i-pezzi",headingTag:"h1"}});const Qe=[pl,ol],L=[];function Ce(e,l){return e[0]==="pt"?0:1}h=Ce(j),f=L[h]=Qe[h](j),V=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQXNlcXVlbmNlJTIwJTNEJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),B=new w({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2Up",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),G=new w({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`,wrap:!1}}),q=new w({props:{code:"JTIzJTIwRWZmZXR0dWElMjBpbCUyMHBhZGRpbmclMjBkZWxsYSUyMHNlcXVlbnphJTIwZmlubyUyMGFsbGxhJTIwbWFzc2ltYSUyMGx1bmdoZXp6YSUyMGRlbGxhJTIwc2VxdWVuemElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMmxvbmdlc3QlMjIpJTBBJTBBJTIzJTIwRWZmZXR0dWElMjBpbCUyMHBhZGRpbmclMjBmaW5vJTIwYWxsYSUyMGx1bmdoZXp6YSUyMG1hc3NpbWElMjBkZWwlMjBtb2RlbGxvJTBBJTIzJTIwKDUxMiUyMHBlciUyMEJFUlQlMjBvJTIwRGlzdGlsQkVSVCklMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMm1heF9sZW5ndGglMjIpJTBBJTBBJTIzJTIwRWZmZXR0dWElMjBpbCUyMHBhZGRpbmclMjBmaW5vJTIwYWxsYSUyMGx1bmdoZXp6YSUyMG1hc3NpbWElMjBzcGVjaWZpY2F0YSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiUyQyUyMG1heF9sZW5ndGglM0Q4KQ==",highlighted:`<span class="hljs-comment"># Effettua il padding della sequenza fino allla massima lunghezza della sequenza</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Effettua il padding fino alla lunghezza massima del modello</span>
<span class="hljs-comment"># (512 per BERT o DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Effettua il padding fino alla lunghezza massima specificata</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`,wrap:!1}}),R=new w({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBUcm9uY2ElMjBsZSUyMHNlcXVlbnplJTIwcGklQzMlQjklMjBsdW5naGUlMjBkZWxsYSUyMGx1bmdoZXp6YSUyMG1hc3NpbWElMjBkZWwlMjBtb2RlbGxvLiUwQSUyMyUyMCg1MTIlMjBwZXIlMjBCRVJUJTIwbyUyMERpc3RpbEJFUlQpJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHRydW5jYXRpb24lM0RUcnVlKSUwQSUwQSUyMyUyMFRyb25jYSUyMGxlJTIwc2VxdWVuemUlMjBwaSVDMyVCOSUyMGx1bmdoZSUyMGRlbGxhJTIwbHVuZ2hlenphJTIwbWFzc2ltYSUyMHNwZWNpZmljYXRhLiUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBtYXhfbGVuZ3RoJTNEOCUyQyUyMHRydW5jYXRpb24lM0RUcnVlKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Tronca le sequenze più lunghe della lunghezza massima del modello.</span>
<span class="hljs-comment"># (512 per BERT o DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Tronca le sequenze più lunghe della lunghezza massima specificata.</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`,wrap:!1}}),E=new w({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBSaXRvcm5hJTIwdGVuc29yaSUyMFB5VG9yY2glMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQSUyMyUyMFJpdG9ybmElMjB0ZW5zb3JpJTIwVGVuc29yRmxvdyUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBJTBBJTIzJTIwUml0b3JuYSUyME51bVB5JTIwYXJyYXlzJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJucCUyMik=",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Ritorna tensori PyTorch</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Ritorna tensori TensorFlow</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)

<span class="hljs-comment"># Ritorna NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`,wrap:!1}}),x=new $e({props:{title:"Token speciali",local:"token-speciali",headingTag:"h2"}}),Y=new w({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2UpJTBBcHJpbnQobW9kZWxfaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci50b2tlbml6ZShzZXF1ZW5jZSklMEFpZHMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF90b2tlbnNfdG9faWRzKHRva2VucyklMEFwcmludChpZHMp",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`,wrap:!1}}),Q=new w({props:{code:"JTVCMTAxJTJDJTIwMTA0NSUyQyUyMDEwMDUlMkMlMjAyMzEwJTJDJTIwMjA0MiUyQyUyMDM0MDMlMkMlMjAyMDA1JTJDJTIwMTAzNyUyQyUyMDE3NjYyJTJDJTIwMTIxNzIlMkMlMjAyNjA3JTJDJTIwMjAyNiUyQyUyMDI4NzglMkMlMjAyMTY2JTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUwQSU1QjEwNDUlMkMlMjAxMDA1JTJDJTIwMjMxMCUyQyUyMDIwNDIlMkMlMjAzNDAzJTJDJTIwMjAwNSUyQyUyMDEwMzclMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMjYwNyUyQyUyMDIwMjYlMkMlMjAyODc4JTJDJTIwMjE2NiUyQyUyMDEwMTIlNUQ=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`,wrap:!1}}),F=new w({props:{code:"cHJpbnQodG9rZW5pemVyLmRlY29kZShtb2RlbF9pbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUoaWRzKSk=",highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`,wrap:!1}}),H=new w({props:{code:"JTIyJTVCQ0xTJTVEJTIwaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIwJTVCU0VQJTVEJTIyJTBBJTIyaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIy",highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`,wrap:!1}}),D=new $e({props:{title:"Conclusione: Dal tokenizer al modello",local:"conclusione-dal-tokenizer-al-modello",headingTag:"h2"}});const Fe=[cl,rl],K=[];function He(e,l){return e[0]==="pt"?0:1}return g=He(j),I=K[g]=Fe[g](j),{c(){a=b("meta"),p=i(),s=b("p"),y=i(),u(U.$$.fragment),se=i(),u(Z.$$.fragment),te=i(),f.c(),O=i(),k=b("p"),k.textContent=Be,ne=i(),z=b("p"),z.innerHTML=ve,ae=i(),u(V.$$.fragment),ie=i(),W=b("p"),W.innerHTML=Ge,oe=i(),$=b("p"),$.textContent=Se,pe=i(),u(B.$$.fragment),re=i(),v=b("p"),v.textContent=qe,ce=i(),u(G.$$.fragment),ue=i(),S=b("p"),S.textContent=Xe,me=i(),u(q.$$.fragment),Me=i(),X=b("p"),X.textContent=Re,de=i(),u(R.$$.fragment),be=i(),_=b("p"),_.innerHTML=_e,Je=i(),u(E.$$.fragment),ye=i(),u(x.$$.fragment),Te=i(),N=b("p"),N.textContent=Ee,we=i(),u(Y.$$.fragment),je=i(),u(Q.$$.fragment),he=i(),C=b("p"),C.textContent=xe,fe=i(),u(F.$$.fragment),ge=i(),u(H.$$.fragment),Ie=i(),A=b("p"),A.innerHTML=Ne,Ue=i(),u(D.$$.fragment),Ze=i(),P=b("p"),P.innerHTML=Ye,ke=i(),I.c(),ee=i(),le=b("p"),this.h()},l(e){const l=nl("svelte-u9bgzb",document.head);a=J(l,"META",{name:!0,content:!0}),l.forEach(t),p=o(e),s=J(e,"P",{}),De(s).forEach(t),y=o(e),m(U.$$.fragment,e),se=o(e),m(Z.$$.fragment,e),te=o(e),f.l(e),O=o(e),k=J(e,"P",{"data-svelte-h":!0}),T(k)!=="svelte-1gq1u3k"&&(k.textContent=Be),ne=o(e),z=J(e,"P",{"data-svelte-h":!0}),T(z)!=="svelte-1xosnnr"&&(z.innerHTML=ve),ae=o(e),m(V.$$.fragment,e),ie=o(e),W=J(e,"P",{"data-svelte-h":!0}),T(W)!=="svelte-1aee5q6"&&(W.innerHTML=Ge),oe=o(e),$=J(e,"P",{"data-svelte-h":!0}),T($)!=="svelte-1518yby"&&($.textContent=Se),pe=o(e),m(B.$$.fragment,e),re=o(e),v=J(e,"P",{"data-svelte-h":!0}),T(v)!=="svelte-t4crif"&&(v.textContent=qe),ce=o(e),m(G.$$.fragment,e),ue=o(e),S=J(e,"P",{"data-svelte-h":!0}),T(S)!=="svelte-110r6wo"&&(S.textContent=Xe),me=o(e),m(q.$$.fragment,e),Me=o(e),X=J(e,"P",{"data-svelte-h":!0}),T(X)!=="svelte-c5od7r"&&(X.textContent=Re),de=o(e),m(R.$$.fragment,e),be=o(e),_=J(e,"P",{"data-svelte-h":!0}),T(_)!=="svelte-avke6z"&&(_.innerHTML=_e),Je=o(e),m(E.$$.fragment,e),ye=o(e),m(x.$$.fragment,e),Te=o(e),N=J(e,"P",{"data-svelte-h":!0}),T(N)!=="svelte-1r8bndt"&&(N.textContent=Ee),we=o(e),m(Y.$$.fragment,e),je=o(e),m(Q.$$.fragment,e),he=o(e),C=J(e,"P",{"data-svelte-h":!0}),T(C)!=="svelte-di8hcs"&&(C.textContent=xe),fe=o(e),m(F.$$.fragment,e),ge=o(e),m(H.$$.fragment,e),Ie=o(e),A=J(e,"P",{"data-svelte-h":!0}),T(A)!=="svelte-1m0dtz8"&&(A.innerHTML=Ne),Ue=o(e),m(D.$$.fragment,e),Ze=o(e),P=J(e,"P",{"data-svelte-h":!0}),T(P)!=="svelte-8r23w7"&&(P.innerHTML=Ye),ke=o(e),I.l(e),ee=o(e),le=J(e,"P",{}),De(le).forEach(t),this.h()},h(){Pe(a,"name","hf:doc:metadata"),Pe(a,"content",ml)},m(e,l){al(document.head,a),n(e,p,l),n(e,s,l),n(e,y,l),M(U,e,l),n(e,se,l),M(Z,e,l),n(e,te,l),L[h].m(e,l),n(e,O,l),n(e,k,l),n(e,ne,l),n(e,z,l),n(e,ae,l),M(V,e,l),n(e,ie,l),n(e,W,l),n(e,oe,l),n(e,$,l),n(e,pe,l),M(B,e,l),n(e,re,l),n(e,v,l),n(e,ce,l),M(G,e,l),n(e,ue,l),n(e,S,l),n(e,me,l),M(q,e,l),n(e,Me,l),n(e,X,l),n(e,de,l),M(R,e,l),n(e,be,l),n(e,_,l),n(e,Je,l),M(E,e,l),n(e,ye,l),M(x,e,l),n(e,Te,l),n(e,N,l),n(e,we,l),M(Y,e,l),n(e,je,l),M(Q,e,l),n(e,he,l),n(e,C,l),n(e,fe,l),M(F,e,l),n(e,ge,l),M(H,e,l),n(e,Ie,l),n(e,A,l),n(e,Ue,l),M(D,e,l),n(e,Ze,l),n(e,P,l),n(e,ke,l),K[g].m(e,l),n(e,ee,l),n(e,le,l),ze=!0},p(e,[l]){const Ae={};l&1&&(Ae.fw=e[0]),U.$set(Ae);let Ve=h;h=Ce(e),h!==Ve&&(Ke(),r(L[Ve],1,1,()=>{L[Ve]=null}),Le(),f=L[h],f||(f=L[h]=Qe[h](e),f.c()),c(f,1),f.m(O.parentNode,O));let We=g;g=He(e),g!==We&&(Ke(),r(K[We],1,1,()=>{K[We]=null}),Le(),I=K[g],I||(I=K[g]=Fe[g](e),I.c()),c(I,1),I.m(ee.parentNode,ee))},i(e){ze||(c(U.$$.fragment,e),c(Z.$$.fragment,e),c(f),c(V.$$.fragment,e),c(B.$$.fragment,e),c(G.$$.fragment,e),c(q.$$.fragment,e),c(R.$$.fragment,e),c(E.$$.fragment,e),c(x.$$.fragment,e),c(Y.$$.fragment,e),c(Q.$$.fragment,e),c(F.$$.fragment,e),c(H.$$.fragment,e),c(D.$$.fragment,e),c(I),ze=!0)},o(e){r(U.$$.fragment,e),r(Z.$$.fragment,e),r(f),r(V.$$.fragment,e),r(B.$$.fragment,e),r(G.$$.fragment,e),r(q.$$.fragment,e),r(R.$$.fragment,e),r(E.$$.fragment,e),r(x.$$.fragment,e),r(Y.$$.fragment,e),r(Q.$$.fragment,e),r(F.$$.fragment,e),r(H.$$.fragment,e),r(D.$$.fragment,e),r(I),ze=!1},d(e){e&&(t(p),t(s),t(y),t(se),t(te),t(O),t(k),t(ne),t(z),t(ae),t(ie),t(W),t(oe),t($),t(pe),t(re),t(v),t(ce),t(ue),t(S),t(me),t(Me),t(X),t(de),t(be),t(_),t(Je),t(ye),t(Te),t(N),t(we),t(je),t(he),t(C),t(fe),t(ge),t(Ie),t(A),t(Ue),t(Ze),t(P),t(ke),t(ee),t(le)),t(a),d(U,e),d(Z,e),L[h].d(e),d(V,e),d(B,e),d(G,e),d(q,e),d(R,e),d(E,e),d(x,e),d(Y,e),d(Q,e),d(F,e),d(H,e),d(D,e),K[g].d(e)}}}const ml='{"title":"Mettiamo insieme i pezzi","local":"mettiamo-insieme-i-pezzi","sections":[{"title":"Token speciali","local":"token-speciali","sections":[],"depth":2},{"title":"Conclusione: Dal tokenizer al modello","local":"conclusione-dal-tokenizer-al-modello","sections":[],"depth":2}],"depth":1}';function Ml(j,a,p){let s="pt";return ll(()=>{const y=new URLSearchParams(window.location.search);p(0,s=y.get("fw")||"pt")}),[s]}class jl extends sl{constructor(a){super(),tl(this,a,Ml,ul,el,{})}}export{jl as component};
