import{s as An,o as Nn,n as Qn}from"../chunks/scheduler.37c15a92.js";import{S as Sn,i as Wn,g as y,s as t,r as u,A as En,h as T,f as e,c as p,j as $n,u as J,x as d,k as Ee,y as vn,a as n,v as o,t as c,b as Be,d as M,w as j,p as Qe}from"../chunks/index.2bf4358c.js";import{T as Bn}from"../chunks/Tip.363c041f.js";import{Y as Ze}from"../chunks/Youtube.1e50a667.js";import{C as w}from"../chunks/CodeBlock.4e987730.js";import{C as Zn}from"../chunks/CourseFloatingBanner.6add7356.js";import{F as Vn}from"../chunks/FrameworkSwitchCourse.8d4d4ab6.js";import{H as cl,E as Gn}from"../chunks/getInferenceSnippets.97e75ee2.js";function Rn(x){let i,m;return i=new Zn({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter6/section3_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter6/section3_tf.ipynb"}]}}),{c(){u(i.$$.fragment)},l(a){J(i.$$.fragment,a)},m(a,U){o(i,a,U),m=!0},i(a){m||(M(i.$$.fragment,a),m=!0)},o(a){c(i.$$.fragment,a),m=!1},d(a){j(i,a)}}}function Hn(x){let i,m;return i=new Zn({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter6/section3_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter6/section3_pt.ipynb"}]}}),{c(){u(i.$$.fragment)},l(a){J(i.$$.fragment,a)},m(a,U){o(i,a,U),m=!0},i(a){m||(M(i.$$.fragment,a),m=!0)},o(a){c(i.$$.fragment,a),m=!1},d(a){j(i,a)}}}function Xn(x){let i,m="⚠️ Atunci când tokenizați o singură propoziție, nu veți vedea întotdeauna o diferență de viteză între versiunea lentă și rapidă ale aceluiași tokenizer. De fapt, versiunea rapidă poate fi chiar mai lentă! Abia atunci când tokenizați multe texte în paralel, în același timp, veți putea observa clar diferența.";return{c(){i=y("p"),i.textContent=m},l(a){i=T(a,"P",{"data-svelte-h":!0}),d(i)!=="svelte-1v2hpym"&&(i.textContent=m)},m(a,U){n(a,i,U)},p:Qn,d(a){a&&e(i)}}}function Dn(x){let i,m="✏️ <strong>Încercați!</strong> Creați propriul exemplu de text și încercați să înțelegeți care tokenuri sunt asociate cu ID-ul cuvântului și, de asemenea, cum să extrageți spanurile pentru singur cuvânt. Pentru puncte bonus, încercați să utilizați două propoziții ca inputuri și să vedeți dacă ID-urile propozițiilor au sens pentru voi.";return{c(){i=y("p"),i.innerHTML=m},l(a){i=T(a,"P",{"data-svelte-h":!0}),d(i)!=="svelte-156m91r"&&(i.innerHTML=m)},m(a,U){n(a,i,U)},p:Qn,d(a){a&&e(i)}}}function _n(x){let i,m;return i=new Ze({props:{id:"PrX4CjrVnNc"}}),{c(){u(i.$$.fragment)},l(a){J(i.$$.fragment,a)},m(a,U){o(i,a,U),m=!0},i(a){m||(M(i.$$.fragment,a),m=!0)},o(a){c(i.$$.fragment,a),m=!1},d(a){j(i,a)}}}function Yn(x){let i,m;return i=new Ze({props:{id:"0E7ltQB7fM8"}}),{c(){u(i.$$.fragment)},l(a){J(i.$$.fragment,a)},m(a,U){o(i,a,U),m=!0},i(a){m||(M(i.$$.fragment,a),m=!0)},o(a){c(i.$$.fragment,a),m=!1},d(a){j(i,a)}}}function Fn(x){let i,m='În primul rând trebuie să tokenizăm inputurile și să le trecem prin model. Acest lucru este făcut excat ca în <a href="/course/chapter2">Capitolul 2</a>; noi am inițializat tokenizerul și modelul folosind clasa <code>TFAutoXxx</code> și apoi am folosit-o pe exemplul nostru:',a,U,k,g,$="Deoarece aici folosim <code>TFAutoModelForTokenClassification</code>, noi primim un set de logits pentru fiecare token în input sequence:",z,b,f,I,C;return U=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBURkF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFtb2RlbF9jaGVja3BvaW50JTIwJTNEJTIwJTIyZGJtZHolMkZiZXJ0LWxhcmdlLWNhc2VkLWZpbmV0dW5lZC1jb25sbDAzLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9jaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZChtb2RlbF9jaGVja3BvaW50KSUwQSUwQWV4YW1wbGUlMjAlM0QlMjAlMjJNeSUyMG5hbWUlMjBpcyUyMFN5bHZhaW4lMjBhbmQlMjBJJTIwd29yayUyMGF0JTIwSHVnZ2luZyUyMEZhY2UlMjBpbiUyMEJyb29rbHluLiUyMiUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihleGFtcGxlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJ0ZiUyMiklMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForTokenClassification

model_checkpoint = <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = TFAutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
inputs = tokenizer(example, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
outputs = model(**inputs)`,wrap:!1}}),b=new w({props:{code:"cHJpbnQoaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVELnNoYXBlKSUwQXByaW50KG91dHB1dHMubG9naXRzLnNoYXBlKQ==",highlighted:`<span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape)
<span class="hljs-built_in">print</span>(outputs.logits.shape)`,wrap:!1}}),I=new w({props:{code:"KDElMkMlMjAxOSklMEEoMSUyQyUyMDE5JTJDJTIwOSk=",highlighted:`(<span class="hljs-number">1</span>, <span class="hljs-number">19</span>)
(<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>)`,wrap:!1}}),{c(){i=y("p"),i.innerHTML=m,a=t(),u(U.$$.fragment),k=t(),g=y("p"),g.innerHTML=$,z=t(),u(b.$$.fragment),f=t(),u(I.$$.fragment)},l(r){i=T(r,"P",{"data-svelte-h":!0}),d(i)!=="svelte-xeer8m"&&(i.innerHTML=m),a=p(r),J(U.$$.fragment,r),k=p(r),g=T(r,"P",{"data-svelte-h":!0}),d(g)!=="svelte-xwcf2h"&&(g.innerHTML=$),z=p(r),J(b.$$.fragment,r),f=p(r),J(I.$$.fragment,r)},m(r,h){n(r,i,h),n(r,a,h),o(U,r,h),n(r,k,h),n(r,g,h),n(r,z,h),o(b,r,h),n(r,f,h),o(I,r,h),C=!0},i(r){C||(M(U.$$.fragment,r),M(b.$$.fragment,r),M(I.$$.fragment,r),C=!0)},o(r){c(U.$$.fragment,r),c(b.$$.fragment,r),c(I.$$.fragment,r),C=!1},d(r){r&&(e(i),e(a),e(k),e(g),e(z),e(f)),j(U,r),j(b,r),j(I,r)}}}function Ln(x){let i,m='În primul rând trebuie să tokenizăm inputurile și sp le trecem prin model. Acest lucru este făcut excat ca în <a href="/course/chapter2">Capitolul 2</a>; noi am inițializat tokenizerul și modelul folosind clasa <code>AutoXxx</code> și apoi am folosit-o pe exemplul nostru:',a,U,k,g,$="Deoarece aici folosim <code>AutoModelForTokenClassification</code> , noi primim un set de logits pentru fiecare token în input sequence:",z,b,f,I,C;return U=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uJTBBJTBBbW9kZWxfY2hlY2twb2ludCUyMCUzRCUyMCUyMmRibWR6JTJGYmVydC1sYXJnZS1jYXNlZC1maW5ldHVuZWQtY29ubGwwMy1lbmdsaXNoJTIyJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfY2hlY2twb2ludCklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2NoZWNrcG9pbnQpJTBBJTBBZXhhbXBsZSUyMCUzRCUyMCUyMk15JTIwbmFtZSUyMGlzJTIwU3lsdmFpbiUyMGFuZCUyMEklMjB3b3JrJTIwYXQlMjBIdWdnaW5nJTIwRmFjZSUyMGluJTIwQnJvb2tseW4uJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForTokenClassification

model_checkpoint = <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
inputs = tokenizer(example, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
outputs = model(**inputs)`,wrap:!1}}),b=new w({props:{code:"cHJpbnQoaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVELnNoYXBlKSUwQXByaW50KG91dHB1dHMubG9naXRzLnNoYXBlKQ==",highlighted:`<span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape)
<span class="hljs-built_in">print</span>(outputs.logits.shape)`,wrap:!1}}),I=new w({props:{code:"dG9yY2guU2l6ZSglNUIxJTJDJTIwMTklNUQpJTBBdG9yY2guU2l6ZSglNUIxJTJDJTIwMTklMkMlMjA5JTVEKQ==",highlighted:`torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>])
torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>])`,wrap:!1}}),{c(){i=y("p"),i.innerHTML=m,a=t(),u(U.$$.fragment),k=t(),g=y("p"),g.innerHTML=$,z=t(),u(b.$$.fragment),f=t(),u(I.$$.fragment)},l(r){i=T(r,"P",{"data-svelte-h":!0}),d(i)!=="svelte-51gmlr"&&(i.innerHTML=m),a=p(r),J(U.$$.fragment,r),k=p(r),g=T(r,"P",{"data-svelte-h":!0}),d(g)!=="svelte-ro5kdh"&&(g.innerHTML=$),z=p(r),J(b.$$.fragment,r),f=p(r),J(I.$$.fragment,r)},m(r,h){n(r,i,h),n(r,a,h),o(U,r,h),n(r,k,h),n(r,g,h),n(r,z,h),o(b,r,h),n(r,f,h),o(I,r,h),C=!0},i(r){C||(M(U.$$.fragment,r),M(b.$$.fragment,r),M(I.$$.fragment,r),C=!0)},o(r){c(U.$$.fragment,r),c(b.$$.fragment,r),c(I.$$.fragment,r),C=!1},d(r){r&&(e(i),e(a),e(k),e(g),e(z),e(f)),j(U,r),j(b,r),j(I,r)}}}function On(x){let i,m;return i=new w({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFwcm9iYWJpbGl0aWVzJTIwJTNEJTIwdGYubWF0aC5zb2Z0bWF4KG91dHB1dHMubG9naXRzJTJDJTIwYXhpcyUzRC0xKSU1QjAlNUQlMEFwcm9iYWJpbGl0aWVzJTIwJTNEJTIwcHJvYmFiaWxpdGllcy5udW1weSgpLnRvbGlzdCgpJTBBcHJlZGljdGlvbnMlMjAlM0QlMjB0Zi5tYXRoLmFyZ21heChvdXRwdXRzLmxvZ2l0cyUyQyUyMGF4aXMlM0QtMSklNUIwJTVEJTBBcHJlZGljdGlvbnMlMjAlM0QlMjBwcmVkaWN0aW9ucy5udW1weSgpLnRvbGlzdCgpJTBBcHJpbnQocHJlZGljdGlvbnMp",highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

probabilities = tf.math.softmax(outputs.logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]
probabilities = probabilities.numpy().tolist()
predictions = tf.math.argmax(outputs.logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]
predictions = predictions.numpy().tolist()
<span class="hljs-built_in">print</span>(predictions)`,wrap:!1}}),{c(){u(i.$$.fragment)},l(a){J(i.$$.fragment,a)},m(a,U){o(i,a,U),m=!0},i(a){m||(M(i.$$.fragment,a),m=!0)},o(a){c(i.$$.fragment,a),m=!1},d(a){j(i,a)}}}function qn(x){let i,m;return i=new w({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFwcm9iYWJpbGl0aWVzJTIwJTNEJTIwdG9yY2gubm4uZnVuY3Rpb25hbC5zb2Z0bWF4KG91dHB1dHMubG9naXRzJTJDJTIwZGltJTNELTEpJTVCMCU1RC50b2xpc3QoKSUwQXByZWRpY3Rpb25zJTIwJTNEJTIwb3V0cHV0cy5sb2dpdHMuYXJnbWF4KGRpbSUzRC0xKSU1QjAlNUQudG9saXN0KCklMEFwcmludChwcmVkaWN0aW9ucyk=",highlighted:`<span class="hljs-keyword">import</span> torch

probabilities = torch.nn.functional.softmax(outputs.logits, dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist()
predictions = outputs.logits.argmax(dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist()
<span class="hljs-built_in">print</span>(predictions)`,wrap:!1}}),{c(){u(i.$$.fragment)},l(a){J(i.$$.fragment,a)},m(a,U){o(i,a,U),m=!0},i(a){m||(M(i.$$.fragment,a),m=!0)},o(a){c(i.$$.fragment,a),m=!1},d(a){j(i,a)}}}function Pn(x){let i,m,a,U,k,g,$,z,b,f,I,C,r='În această secțiune, vom analiza mai atent capacitățile tokenizerilor din 🤗 Transformers. Până acum, i-am folosit doar pentru tokenizarea inputurilor sau decodificarea ID-urilor înapoi în text, dar tokenizerii - în special cei susținuți de biblioteca 🤗 Tokenizers - pot face mult mai multe. Pentru a ilustra aceste funcții suplimentare, vom explora modul de reproducere a rezultatelor pipeline-urilor <code>token-classification</code> (pe care le-am numit <code>ner</code>) și <code>question-answering</code> pe care le-am întâlnit pentru prima dată în <a href="/course/chapter1">Capitolul 1</a>.',h,V,ul,G,ve='În discuția următoare, vom face adesea distincția între tokenizatori “lenți” și “rapizi”. Tokenizerii lenți sunt cei scriși în Python în interiorul bibliotecii 🤗 Transformers, în timp ce versiunile rapide sunt cele furnizate de 🤗 Tokenizers, care sunt scrise în Rust. Dacă vă amintiți de tabelul din <a href="/course/chapter5/3">Capitolul 5</a> care a raportat cât timp a durat un tokenizer rapid și unul lent pentru a tokeniza datasetul Drug Review, ar trebui să aveți o idee despre de ce îi numim lenți și rapizi:',Jl,R,Ve='<thead><tr><th align="center"></th> <th align="center">Fast tokenizer</th> <th align="center">Slow tokenizer</th></tr></thead> <tbody><tr><td align="center"><code>batched=True</code></td> <td align="center">10,8s</td> <td align="center">4min41s</td></tr> <tr><td align="center"><code>batched=False</code></td> <td align="center">59,2s</td> <td align="center">5min3s</td></tr></tbody>',ol,W,jl,H,ml,X,yl,D,Ge="Rezultatul unui tokenizer nu este un simplu dicționar Python; ceea ce obținem este de fapt un obiect special <code>BatchEncoding</code>. Este o subclasă a unui dicționar (de aceea am putut indexa acel rezultat fără nici o problemă mai devreme), dar cu metode suplimentare care sunt utilizate în principal de către tokenizerii rapizi.",Tl,_,Re="Pe lângă capacitățile lor de paralelizare, funcționalitatea principală a tokenizerilor rapizi este aceea că aceștia țin întotdeauna evidența intervalului original de texte din care provin tokenii finali - o funcționalitate pe care o numim <em>offset mapping</em>. Acest lucru deblochează funcții precum mappingul fiecărui cuvânt la tokens pe care îi generează sau mappingul fiecărui caracter al textului original la tokenul în care se află și invers.",dl,Y,He="Acum hai să aruncăm o privire la un exemplu:",Ul,F,wl,L,Xe="Așa cum s-a menționat anterior, în outputul tokenizerului obținem un obiect <code>BatchEncoding</code>:",bl,O,hl,q,De="Deoarece clasa <code>AutoTokenizer</code> selectează un tokenizer rapid în mod implicit, putem utiliza metodele suplimentare pe care acest obiect <code>BatchEncoding</code> le oferă. Avem două metode de verificare dacă tokenizerul nostru este unul lent sau rapid. Putem verifica fie atributul <code>is_fast</code> al tokenizer-ului:",xl,P,Il,K,fl,ss,_e="sau același atribut al <code>encoding</code>:",gl,ls,kl,es,Cl,ns,Ye="Hai să vedem ce ne permite un tokenizer rapid să facem. În primul rând, putem accesa tokenul fără a fi nevoie să facem convert ID-urile înapoi în tokens:",zl,as,$l,ts,Bl,ps,Fe="În acest caz tokenul la indexul 5 este <code>##yl</code>, ceea ce este o parte a cuvântului “Sylvain” în propoziția originală. În același timp putem folosi metoda <code>word_ids()</code> pentru a obține indexul cuvântului din care provine fiecare token:",Ql,is,Zl,rs,Al,cs,Nl,Ms,Le="Așa cum am menționat anterior, toate acestea sunt posibile datorită faptului că tokenizerul rapid ține evidența spanului de text de la care provine fiecare token într-o listă de <em>offseturi</em>. Pentru a ilustra modul în care se utilizează acestea, în continuare vă vom arăta cum să replicați rezultatele pipelineului <code>token-classification</code> manual.",Sl,E,Wl,us,El,Js,Oe='În <a href="/course/chapter1">Capitolul 1</a> am avut primul nostru contact aplicând NER - unde sarcina constă în identificarea părților textului care corespund entităților precum persoane, locații sau organizații - cu funcția <code>pipeline()</code> din 🤗 Transformers. Apoi, în <a href="/course/chapter2">Capitolul 2</a>, am văzut cum un pipeline grupează cele trei etape necesare pentru a obține predicțiile de la un text “raw”: tokenizare, trecerea inputurilor prin model și post-procesare. Primele două etape din pipelineul <code>token-classification</code> sunt la fel ca în orice alt pipeline, dar post-procesarea este puțin mai complexă - hai să vedem cum!',vl,B,Q,pl,os,Vl,js,qe='În primul rând, trebuie să luăm un token classification pipeline pentru a obține câteva rezultate ca să le putem compara manual. Modelul utilizat în mod implicit este <a href="https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english" rel="nofollow"><code>dbmdz/bert-large-cased-finetuned-conll03-english</code></a>; acesta aplică NER pe propoziții:',Gl,ms,Rl,ys,Hl,Ts,Pe="Modelul a identificat fiecare token generdat de “Sylvain” ca o persoană, fiecare token generat de “Hugging Face” ca o organizație, și fiecare token “Brooklin” ca o locație”. În același timp putem întreba pipelineul să grupeze împreună tokenurile care corespund cu aceeași entitate.",Xl,ds,Dl,Us,_l,ws,Ke="<code>Aggregation_strategy</code> aleasă va modifica scorurile calculate pentru fiecare entitate grupată. Cu <code>&quot;simple&quot;</code>, scorul este doar media scorurilor pentru fiecare token din entitatea dată: de exemplu, scorul pentru “Sylvain” este media scorurilor pe care le-am văzut în exemplele anterioare pentru token-urile <code>S</code>, <code>##yl</code>, <code>##va</code> și <code>##in</code>. Alte strategii disponibile sunt:",Yl,bs,sn="<li><code>&quot;first&quot;</code>, unde scorul fiecărei entități este scorul primului token al acelei entități (astfel, pentru “Sylvain” ar fi 0.993828, scorul token-ului <code>S</code>)</li> <li><code>&quot;max&quot;</code>, unde scorul fiecărei entități este scorul maxim al tokenilor din acea entitate (astfel, pentru “Hugging Face” ar fi 0.98879766, scorul “Face”)</li> <li><code>&quot;average&quot;</code>, unde scorul fiecărei entități este media scorurilor cuvintelor care compun acea entitate (astfel, pentru “Sylvain” nu ar exista nicio diferență față de strategia <code>&quot;simple&quot;</code>, dar “Hugging Face” ar avea un scor de 0.9819, media scorurilor pentru “Hugging”, 0.975 și “Face”, 0.98879)</li>",Fl,hs,ln="Acum să vedem cum putem obține aceste rezultate fără a folosi funcția <code>pipeline()</code>!",Ll,xs,Ol,Z,A,il,Is,en="Noi avem un batch cu 1 secvență din 19 tokenuri și modelul are 9 labeluri diferite, deci outputul modelului are un shape de 1 x 19 x 9. Ca și pentru text classification pipeline, noi folosim o funcție softmax pentru a face convert logiturilor în probabilități și luăm argmax pentru a obține predicții(atrage atenția asupra fatpului că putem lua argmax pe logituri deoarece softmax nu schimbă ordinea):",ql,N,S,rl,fs,Pl,gs,nn="Atributul <code>model.config.id2label</code> con;ine mappingul indexilor la labeluri pe care le putem folosi pentru a înțelege predicțiile:",Kl,ks,se,Cs,le,zs,an="Cum am văzut mai devreme, există 9 labeluri: <code>O</code> este labelul pentru tokenurile care nu se află în nicio entitate numită(aceasta reprezintă “exteriorul”), și avem apoi două labeluri pentru fiecare tip de entitate (divers, persoană, organizație și locație). Eticheta <code>B-XXX</code> indică faptul că tokenul se află la începutul entității <code>XXX</code> și eticheta <code>I-XXX</code> indică faptul că tokenul se află în interiorul entității <code>XXX</code>. De exemplu, în exemplul curent ne-am aștepta ca modelul nostru să clasifice tokenul <code>S</code> ca <code>B-PER</code> (începutul unei entități de-tip persoană) și tokenurile <code>##yl</code>, <code>##va</code> și <code>##in</code> ca <code>I-PER</code> (în interiorul unei entități de tip persoană).",ee,$s,tn="S-ar putea să credeți că modelul a greșit în acest caz, deoarece a atribuit eticheta <code>I-PER</code> tuturor acestor patru tokeni, dar acesta nu este în întregime adevărat. Există, de fapt, două formate pentru labelurile <code>B-</code> și <code>I-</code>: <em>IOB1</em> și <em>IOB2</em>. Formatul IOB2 (în roz mai jos), este cel pe care l-am introdus, în timp ce formatul IOB1 (în albastru), utilizează labelurile care încep cu <code>B-</code> doar pentru a separa două entități adiacente de același tip. Modelul pe care îl utilizăm a fost fine-tuned pe un dataset care utilizează acel format, ceea ce explică de ce atribuie labelul <code>I-PER</code> tokenului <code>S</code>.",ne,v,pn='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions.svg" alt="IOB1 vs IOB2 format"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions-dark.svg" alt="IOB1 vs IOB2 format"/>',ae,Bs,rn="Cu maparea aceasta, suntem gat a să reproducem(aproape în total) rezultat primului pipeline — noi putem lua scorul și labelul fiecărui token care nu a fost clasificat ca <code>O</code>:",te,Qs,pe,Zs,ie,As,cn="Acest lucru este foarte similar cu ce am avut mai devreme, cu o excepție: pipelineul de asemenea ne-a oferit informație despre <code>start</code> și <code>end</code> al fiecărei entități în propoziția originală. Acum e momentul când offset mappingul nostru ne va ajuta. Pentru a obține offseturile, noi trebuie să setăm <code>return_offsets_mapping=True</code> când aplicăm tokenizerul pe inputurile noastre:",re,Ns,ce,Ss,Me,Ws,Mn="Fiecare tuple este spanul de text care corespunde fiecărui token, unde <code>(0, 0)</code> este rezervat pentru tokenii speciali. Noi am văzut înainte că tokenul la indexul 5 este <code>##yl</code>, care are aici <code>(12, 14)</code> ca offsets. Dacă luăm sliceul corespunzător în exemplul nostru:",ue,Es,Je,vs,un="noi obținem spanul propriu de text fără <code>##</code>:",oe,Vs,je,Gs,Jn="Folosind aceasta, putem acum completa rezultatele anterioare:",me,Rs,ye,Hs,Te,Xs,on="Acest răspuns e același răspuns pe care l-am primit de la primul pipeline:",de,Ds,Ue,_s,jn="Utilizarea offseturilor pentru a determina cheile de start și de sfârșit pentru fiecare entitate este util, dar această informație nu este strict necesară. Când dorim să grupăm entitățile împreună, totuși, offseturile ne vor salva o mulțime de messy code. De exemplu, dacă am dori să grupăm împreună tokenii <code>Hu</code>, <code>##gging</code> și <code>Face</code>, am putea crea reguli speciale care să spună că primele două ar trebui să fie atașate și să înlăturăm <code>##</code>, iar <code>Face</code> ar trebui adăugat cu un spațiu, deoarece nu începe cu <code>##</code> — dar acest lucru ar funcționa doar pentru acest tip particular de tokenizer. Ar trebui să scriem un alt set de reguli pentru un tokenizer SentencePiece sau unul Byte-Pair-Encoding (discutat mai târziu în acest capitol).",we,Ys,mn="Cu offseturile, tot acel cod custom dispare: pur și simplu putem lua spanul din textul original care începe cu primul token și se termină cu ultimul token. Deci, în cazul tokenurilor <code>Hu</code>, <code>##gging</code> și <code>Face</code>, ar trebui să începem la caracterul 33 (începutul lui <code>Hu</code>) și să ne oprim înainte de caracterul 45 (sfârșitul lui <code>Face</code>):",be,Fs,he,Ls,xe,Os,yn="Pentru a scrie codul care post-procesează predicțiile în timp ce grupăm entitățile, vom grupa entitățile care sunt consecutive și labeled cu <code>I-XXX</code>, cu excepția primeia, care poate fi labeled ca <code>B-XXX</code> sau <code>I-XXX</code> (decidem să oprim gruparea unei entități atunci când întâlnim un <code>O</code>, un nou tip de entitate, sau un <code>B-XXX</code> care ne spune că o entitate de același tip începe):",Ie,qs,fe,Ps,Tn="Și obținem aceleași răspuns ca de la pipelineul secundar!",ge,Ks,ke,sl,dn="Alt exemplu de sarcină unde offseturile sunt extrem de useful pentru răspunderea la întrebări. Scufundându-ne în pipelineuri, un lucru pe care îl vom face în următoarea secțiune, ne vom premite să ne uităm peste o caracteristică a tokenizerului în librăria 🤗 Transformers: vom avea de-a face cu overflowing tokens când truncăm un input de o anumită lungime.",Ce,ll,ze,Ml,$e;k=new Vn({props:{fw:x[0]}}),$=new cl({props:{title:"Superputerile tokenizerilor rapizi",local:"fast-tokenizers-special-powers",headingTag:"h1"}});const Un=[Hn,Rn],el=[];function wn(s,l){return s[0]==="pt"?0:1}b=wn(x),f=el[b]=Un[b](x),V=new Ze({props:{id:"g8quOxoqhHQ"}}),W=new Bn({props:{warning:!0,$$slots:{default:[Xn]},$$scope:{ctx:x}}}),H=new cl({props:{title:"Batch encoding",local:"batch-encoding",headingTag:"h2"}}),X=new Ze({props:{id:"3umI3tm27Vw"}}),F=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIpJTBBZXhhbXBsZSUyMCUzRCUyMCUyMk15JTIwbmFtZSUyMGlzJTIwU3lsdmFpbiUyMGFuZCUyMEklMjB3b3JrJTIwYXQlMjBIdWdnaW5nJTIwRmFjZSUyMGluJTIwQnJvb2tseW4uJTIyJTBBZW5jb2RpbmclMjAlM0QlMjB0b2tlbml6ZXIoZXhhbXBsZSklMEFwcmludCh0eXBlKGVuY29kaW5nKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
encoding = tokenizer(example)
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(encoding))`,wrap:!1}}),O=new w({props:{code:"JTNDY2xhc3MlMjAndHJhbnNmb3JtZXJzLnRva2VuaXphdGlvbl91dGlsc19iYXNlLkJhdGNoRW5jb2RpbmcnJTNF",highlighted:'&lt;<span class="hljs-keyword">class</span> <span class="hljs-string">&#x27;transformers.tokenization_utils_base.BatchEncoding&#x27;</span>&gt;',wrap:!1}}),P=new w({props:{code:"dG9rZW5pemVyLmlzX2Zhc3Q=",highlighted:"tokenizer.is_fast",wrap:!1}}),K=new w({props:{code:"VHJ1ZQ==",highlighted:'<span class="hljs-literal">True</span>',wrap:!1}}),ls=new w({props:{code:"ZW5jb2RpbmcuaXNfZmFzdA==",highlighted:"encoding.is_fast",wrap:!1}}),es=new w({props:{code:"VHJ1ZQ==",highlighted:'<span class="hljs-literal">True</span>',wrap:!1}}),as=new w({props:{code:"ZW5jb2RpbmcudG9rZW5zKCk=",highlighted:"encoding.tokens()",wrap:!1}}),ts=new w({props:{code:"JTVCJyU1QkNMUyU1RCclMkMlMjAnTXknJTJDJTIwJ25hbWUnJTJDJTIwJ2lzJyUyQyUyMCdTJyUyQyUyMCclMjMlMjN5bCclMkMlMjAnJTIzJTIzdmEnJTJDJTIwJyUyMyUyM2luJyUyQyUyMCdhbmQnJTJDJTIwJ0knJTJDJTIwJ3dvcmsnJTJDJTIwJ2F0JyUyQyUyMCdIdSclMkMlMjAnJTIzJTIzZ2dpbmcnJTJDJTIwJ0ZhY2UnJTJDJTIwJ2luJyUyQyUwQSUyMCdCcm9va2x5biclMkMlMjAnLiclMkMlMjAnJTVCU0VQJTVEJyU1RA==",highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;My&#x27;</span>, <span class="hljs-string">&#x27;name&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&#x27;work&#x27;</span>, <span class="hljs-string">&#x27;at&#x27;</span>, <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>,
 <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`,wrap:!1}}),is=new w({props:{code:"ZW5jb2Rpbmcud29yZF9pZHMoKQ==",highlighted:"encoding.word_ids()",wrap:!1}}),rs=new w({props:{code:" ",highlighted:'[<span class="hljs-literal">None</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>, <span class="hljs-number">10</span>, <span class="hljs-number">11</span>, <span class="hljs-number">12</span>, <span class="hljs-literal">None</span>]\n\nPutem vedea că tokenizerul are tokeni speciali `[CLS]` și `[SEP]` care sunt mapped la `<span class="hljs-literal">None</span>`, iar apoi fiecare token este mapped la cuvântul din care provine. Acest lucru este deosebit de util pentru a determina dacă un token este la începutul unui cuvânt sau dacă două tokenuri sunt în același cuvânt. Ne-am putea baza pe prefixul `<span class="hljs-comment">##` pentru aceasta, dar funcționează doar pentru tokenizeri de tip BERT; această metodă funcționează pentru orice tip de tokenizator, atâta timp cât este unul rapid. În capitolul următor, vom vedea cum putem utiliza această capabilitate pentru a aplica labeluri pe care le avem pentru fiecare cuvânt în mod corespunzător tokenurilor în sarcini precum named entity recognition (NER) și part-of-speech (POS). De asemenea, îl putem utiliza pentru a face mask tuturor tokenurilor care provin din același cuvânt în masked language modeling(o tehnică numită _whole word masking_).</span>\n\n&lt;Tip&gt;\n\nNoțiunea de ceea ce este un cuvânt este complicată. De exemplu, <span class="hljs-string">&quot;I&#x27;ll&quot;</span> (o prescurtare a <span class="hljs-string">&quot;I will&quot;</span>) contează ca unul sau două cuvinte? Acest lucru depinde de tokenizer și de operațiunea de pre-tokenizare pe care o aplică. Unii tokenizeri se divid doar pe spații, așa că vor considera acest lucru ca un singur cuvânt. Alții folosesc punctuația pe lângă spații, deci vor considera două cuvinte.\n\n✏️ **Încercați!** Creați un tokenizer din checkpointurile `bert-base-cased` și `roberta-base` și tokenizați <span class="hljs-string">&quot;81s&quot;</span> cu ele. Ce observați? Care sunt ID-urile cuvintelor?\n\n&lt;/Tip&gt;\n\nÎn mod similar, există o metodă `sentence_ids()` pe care o putem utiliza pentru a face <span class="hljs-built_in">map</span> unui token la propoziția din care provine (deși, în acest caz, `token_type_ids` returnate de tokenizer ne pot oferi aceeași informație).\n\nÎn cele din urmă, putem face <span class="hljs-built_in">map</span> oricărui cuvânt sau token la caracterele din textul original și invers, prin intermediul metodelor `word_to_chars()` sau `token_to_chars()` și `char_to_word()` sau `char_to_token()`. De exemplu, metoda `word_ids()` ne-a spus că `<span class="hljs-comment">##yl` face parte din cuvântul cu indicele 3, dar ce cuvânt este în propoziție? Putem afla astfel:</span>\n\n```py\nstart, end = encoding.word_to_chars(<span class="hljs-number">3</span>)\nexample[start:end]',wrap:!1}}),cs=new w({props:{code:"U3lsdmFpbg==",highlighted:"Sylvain",wrap:!1}}),E=new Bn({props:{$$slots:{default:[Dn]},$$scope:{ctx:x}}}),us=new cl({props:{title:"În interiorul pipelineului token-classification",local:"inside-the-token-classification-pipeline",headingTag:"h2"}});const bn=[Yn,_n],nl=[];function hn(s,l){return s[0]==="pt"?0:1}B=hn(x),Q=nl[B]=bn[B](x),os=new cl({props:{title:"Obținerea rezultatelor de bază cu pipelineul",local:"getting-the-base-results-with-the-pipeline",headingTag:"h3"}}),ms=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBdG9rZW5fY2xhc3NpZmllciUyMCUzRCUyMHBpcGVsaW5lKCUyMnRva2VuLWNsYXNzaWZpY2F0aW9uJTIyKSUwQXRva2VuX2NsYXNzaWZpZXIoJTIyTXklMjBuYW1lJTIwaXMlMjBTeWx2YWluJTIwYW5kJTIwSSUyMHdvcmslMjBhdCUyMEh1Z2dpbmclMjBGYWNlJTIwaW4lMjBCcm9va2x5bi4lMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

token_classifier = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`,wrap:!1}}),ys=new w({props:{code:"JTVCJTdCJ2VudGl0eSclM0ElMjAnSS1QRVInJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTk5MzgyOCUyQyUyMCdpbmRleCclM0ElMjA0JTJDJTIwJ3dvcmQnJTNBJTIwJ1MnJTJDJTIwJ3N0YXJ0JyUzQSUyMDExJTJDJTIwJ2VuZCclM0ElMjAxMiU3RCUyQyUwQSUyMCU3QidlbnRpdHknJTNBJTIwJ0ktUEVSJyUyQyUyMCdzY29yZSclM0ElMjAwLjk5ODE1NDc2JTJDJTIwJ2luZGV4JyUzQSUyMDUlMkMlMjAnd29yZCclM0ElMjAnJTIzJTIzeWwnJTJDJTIwJ3N0YXJ0JyUzQSUyMDEyJTJDJTIwJ2VuZCclM0ElMjAxNCU3RCUyQyUwQSUyMCU3QidlbnRpdHknJTNBJTIwJ0ktUEVSJyUyQyUyMCdzY29yZSclM0ElMjAwLjk5NTkwNzI1JTJDJTIwJ2luZGV4JyUzQSUyMDYlMkMlMjAnd29yZCclM0ElMjAnJTIzJTIzdmEnJTJDJTIwJ3N0YXJ0JyUzQSUyMDE0JTJDJTIwJ2VuZCclM0ElMjAxNiU3RCUyQyUwQSUyMCU3QidlbnRpdHknJTNBJTIwJ0ktUEVSJyUyQyUyMCdzY29yZSclM0ElMjAwLjk5OTIzMjclMkMlMjAnaW5kZXgnJTNBJTIwNyUyQyUyMCd3b3JkJyUzQSUyMCclMjMlMjNpbiclMkMlMjAnc3RhcnQnJTNBJTIwMTYlMkMlMjAnZW5kJyUzQSUyMDE4JTdEJTJDJTBBJTIwJTdCJ2VudGl0eSclM0ElMjAnSS1PUkcnJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTczODkzMzQlMkMlMjAnaW5kZXgnJTNBJTIwMTIlMkMlMjAnd29yZCclM0ElMjAnSHUnJTJDJTIwJ3N0YXJ0JyUzQSUyMDMzJTJDJTIwJ2VuZCclM0ElMjAzNSU3RCUyQyUwQSUyMCU3QidlbnRpdHknJTNBJTIwJ0ktT1JHJyUyQyUyMCdzY29yZSclM0ElMjAwLjk3NjExNSUyQyUyMCdpbmRleCclM0ElMjAxMyUyQyUyMCd3b3JkJyUzQSUyMCclMjMlMjNnZ2luZyclMkMlMjAnc3RhcnQnJTNBJTIwMzUlMkMlMjAnZW5kJyUzQSUyMDQwJTdEJTJDJTBBJTIwJTdCJ2VudGl0eSclM0ElMjAnSS1PUkcnJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTg4Nzk3NjYlMkMlMjAnaW5kZXgnJTNBJTIwMTQlMkMlMjAnd29yZCclM0ElMjAnRmFjZSclMkMlMjAnc3RhcnQnJTNBJTIwNDElMkMlMjAnZW5kJyUzQSUyMDQ1JTdEJTJDJTBBJTIwJTdCJ2VudGl0eSclM0ElMjAnSS1MT0MnJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTkzMjEwNTUlMkMlMjAnaW5kZXgnJTNBJTIwMTYlMkMlMjAnd29yZCclM0ElMjAnQnJvb2tseW4nJTJDJTIwJ3N0YXJ0JyUzQSUyMDQ5JTJDJTIwJ2VuZCclM0ElMjA1NyU3RCU1RA==",highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">12</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">14</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">35</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">35</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">40</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">41</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`,wrap:!1}}),ds=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBdG9rZW5fY2xhc3NpZmllciUyMCUzRCUyMHBpcGVsaW5lKCUyMnRva2VuLWNsYXNzaWZpY2F0aW9uJTIyJTJDJTIwYWdncmVnYXRpb25fc3RyYXRlZ3klM0QlMjJzaW1wbGUlMjIpJTBBdG9rZW5fY2xhc3NpZmllciglMjJNeSUyMG5hbWUlMjBpcyUyMFN5bHZhaW4lMjBhbmQlMjBJJTIwd29yayUyMGF0JTIwSHVnZ2luZyUyMEZhY2UlMjBpbiUyMEJyb29rbHluLiUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

token_classifier = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>, aggregation_strategy=<span class="hljs-string">&quot;simple&quot;</span>)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`,wrap:!1}}),Us=new w({props:{code:"JTVCJTdCJ2VudGl0eV9ncm91cCclM0ElMjAnUEVSJyUyQyUyMCdzY29yZSclM0ElMjAwLjk5ODE2OTQlMkMlMjAnd29yZCclM0ElMjAnU3lsdmFpbiclMkMlMjAnc3RhcnQnJTNBJTIwMTElMkMlMjAnZW5kJyUzQSUyMDE4JTdEJTJDJTBBJTIwJTdCJ2VudGl0eV9ncm91cCclM0ElMjAnT1JHJyUyQyUyMCdzY29yZSclM0ElMjAwLjk3OTYwMjA0JTJDJTIwJ3dvcmQnJTNBJTIwJ0h1Z2dpbmclMjBGYWNlJyUyQyUyMCdzdGFydCclM0ElMjAzMyUyQyUyMCdlbmQnJTNBJTIwNDUlN0QlMkMlMEElMjAlN0InZW50aXR5X2dyb3VwJyUzQSUyMCdMT0MnJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTkzMjEwNTUlMkMlMjAnd29yZCclM0ElMjAnQnJvb2tseW4nJTJDJTIwJ3N0YXJ0JyUzQSUyMDQ5JTJDJTIwJ2VuZCclM0ElMjA1NyU3RCU1RA==",highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`,wrap:!1}}),xs=new cl({props:{title:"De la inputuri la predicții",local:"from-inputs-to-predictions",headingTag:"h3"}});const xn=[Ln,Fn],al=[];function In(s,l){return s[0]==="pt"?0:1}Z=In(x),A=al[Z]=xn[Z](x);const fn=[qn,On],tl=[];function gn(s,l){return s[0]==="pt"?0:1}return N=gn(x),S=tl[N]=fn[N](x),fs=new w({props:{code:"JTVCMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDQlMkMlMjA0JTJDJTIwNCUyQyUyMDQlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwNiUyQyUyMDYlMkMlMjA2JTJDJTIwMCUyQyUyMDglMkMlMjAwJTJDJTIwMCU1RA==",highlighted:'[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">0</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]',wrap:!1}}),ks=new w({props:{code:"bW9kZWwuY29uZmlnLmlkMmxhYmVs",highlighted:"model.config.id2label",wrap:!1}}),Cs=new w({props:{code:"JTdCMCUzQSUyMCdPJyUyQyUwQSUyMDElM0ElMjAnQi1NSVNDJyUyQyUwQSUyMDIlM0ElMjAnSS1NSVNDJyUyQyUwQSUyMDMlM0ElMjAnQi1QRVInJTJDJTBBJTIwNCUzQSUyMCdJLVBFUiclMkMlMEElMjA1JTNBJTIwJ0ItT1JHJyUyQyUwQSUyMDYlM0ElMjAnSS1PUkcnJTJDJTBBJTIwNyUzQSUyMCdCLUxPQyclMkMlMEElMjA4JTNBJTIwJ0ktTE9DJyU3RA==",highlighted:`{<span class="hljs-number">0</span>: <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-number">1</span>: <span class="hljs-string">&#x27;B-MISC&#x27;</span>,
 <span class="hljs-number">2</span>: <span class="hljs-string">&#x27;I-MISC&#x27;</span>,
 <span class="hljs-number">3</span>: <span class="hljs-string">&#x27;B-PER&#x27;</span>,
 <span class="hljs-number">4</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>,
 <span class="hljs-number">5</span>: <span class="hljs-string">&#x27;B-ORG&#x27;</span>,
 <span class="hljs-number">6</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>,
 <span class="hljs-number">7</span>: <span class="hljs-string">&#x27;B-LOC&#x27;</span>,
 <span class="hljs-number">8</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>}`,wrap:!1}}),Qs=new w({props:{code:"cmVzdWx0cyUyMCUzRCUyMCU1QiU1RCUwQXRva2VucyUyMCUzRCUyMGlucHV0cy50b2tlbnMoKSUwQSUwQWZvciUyMGlkeCUyQyUyMHByZWQlMjBpbiUyMGVudW1lcmF0ZShwcmVkaWN0aW9ucyklM0ElMEElMjAlMjAlMjAlMjBsYWJlbCUyMCUzRCUyMG1vZGVsLmNvbmZpZy5pZDJsYWJlbCU1QnByZWQlNUQlMEElMjAlMjAlMjAlMjBpZiUyMGxhYmVsJTIwISUzRCUyMCUyMk8lMjIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXN1bHRzLmFwcGVuZCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJlbnRpdHklMjIlM0ElMjBsYWJlbCUyQyUyMCUyMnNjb3JlJTIyJTNBJTIwcHJvYmFiaWxpdGllcyU1QmlkeCU1RCU1QnByZWQlNUQlMkMlMjAlMjJ3b3JkJTIyJTNBJTIwdG9rZW5zJTVCaWR4JTVEJTdEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUwQXByaW50KHJlc3VsdHMp",highlighted:`results = []
tokens = inputs.tokens()

<span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions):
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        results.append(
            {<span class="hljs-string">&quot;entity&quot;</span>: label, <span class="hljs-string">&quot;score&quot;</span>: probabilities[idx][pred], <span class="hljs-string">&quot;word&quot;</span>: tokens[idx]}
        )

<span class="hljs-built_in">print</span>(results)`,wrap:!1}}),Zs=new w({props:{code:"JTVCJTdCJ2VudGl0eSclM0ElMjAnSS1QRVInJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTk5MzgyOCUyQyUyMCdpbmRleCclM0ElMjA0JTJDJTIwJ3dvcmQnJTNBJTIwJ1MnJTdEJTJDJTBBJTIwJTdCJ2VudGl0eSclM0ElMjAnSS1QRVInJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTk4MTU0NzYlMkMlMjAnaW5kZXgnJTNBJTIwNSUyQyUyMCd3b3JkJyUzQSUyMCclMjMlMjN5bCclN0QlMkMlMEElMjAlN0InZW50aXR5JyUzQSUyMCdJLVBFUiclMkMlMjAnc2NvcmUnJTNBJTIwMC45OTU5MDcyNSUyQyUyMCdpbmRleCclM0ElMjA2JTJDJTIwJ3dvcmQnJTNBJTIwJyUyMyUyM3ZhJyU3RCUyQyUwQSUyMCU3QidlbnRpdHknJTNBJTIwJ0ktUEVSJyUyQyUyMCdzY29yZSclM0ElMjAwLjk5OTIzMjclMkMlMjAnaW5kZXgnJTNBJTIwNyUyQyUyMCd3b3JkJyUzQSUyMCclMjMlMjNpbiclN0QlMkMlMEElMjAlN0InZW50aXR5JyUzQSUyMCdJLU9SRyclMkMlMjAnc2NvcmUnJTNBJTIwMC45NzM4OTMzNCUyQyUyMCdpbmRleCclM0ElMjAxMiUyQyUyMCd3b3JkJyUzQSUyMCdIdSclN0QlMkMlMEElMjAlN0InZW50aXR5JyUzQSUyMCdJLU9SRyclMkMlMjAnc2NvcmUnJTNBJTIwMC45NzYxMTUlMkMlMjAnaW5kZXgnJTNBJTIwMTMlMkMlMjAnd29yZCclM0ElMjAnJTIzJTIzZ2dpbmcnJTdEJTJDJTBBJTIwJTdCJ2VudGl0eSclM0ElMjAnSS1PUkcnJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTg4Nzk3NjYlMkMlMjAnaW5kZXgnJTNBJTIwMTQlMkMlMjAnd29yZCclM0ElMjAnRmFjZSclN0QlMkMlMEElMjAlN0InZW50aXR5JyUzQSUyMCdJLUxPQyclMkMlMjAnc2NvcmUnJTNBJTIwMC45OTMyMTA1NSUyQyUyMCdpbmRleCclM0ElMjAxNiUyQyUyMCd3b3JkJyUzQSUyMCdCcm9va2x5biclN0QlNUQ=",highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>}]`,wrap:!1}}),Ns=new w({props:{code:"aW5wdXRzX3dpdGhfb2Zmc2V0cyUyMCUzRCUyMHRva2VuaXplcihleGFtcGxlJTJDJTIwcmV0dXJuX29mZnNldHNfbWFwcGluZyUzRFRydWUpJTBBaW5wdXRzX3dpdGhfb2Zmc2V0cyU1QiUyMm9mZnNldF9tYXBwaW5nJTIyJTVE",highlighted:`inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]`,wrap:!1}}),Ss=new w({props:{code:"JTVCKDAlMkMlMjAwKSUyQyUyMCgwJTJDJTIwMiklMkMlMjAoMyUyQyUyMDcpJTJDJTIwKDglMkMlMjAxMCklMkMlMjAoMTElMkMlMjAxMiklMkMlMjAoMTIlMkMlMjAxNCklMkMlMjAoMTQlMkMlMjAxNiklMkMlMjAoMTYlMkMlMjAxOCklMkMlMjAoMTklMkMlMjAyMiklMkMlMjAoMjMlMkMlMjAyNCklMkMlMjAoMjUlMkMlMjAyOSklMkMlMjAoMzAlMkMlMjAzMiklMkMlMEElMjAoMzMlMkMlMjAzNSklMkMlMjAoMzUlMkMlMjA0MCklMkMlMjAoNDElMkMlMjA0NSklMkMlMjAoNDYlMkMlMjA0OCklMkMlMjAoNDklMkMlMjA1NyklMkMlMjAoNTclMkMlMjA1OCklMkMlMjAoMCUyQyUyMDApJTVE",highlighted:`[(<span class="hljs-number">0</span>, <span class="hljs-number">0</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">2</span>), (<span class="hljs-number">3</span>, <span class="hljs-number">7</span>), (<span class="hljs-number">8</span>, <span class="hljs-number">10</span>), (<span class="hljs-number">11</span>, <span class="hljs-number">12</span>), (<span class="hljs-number">12</span>, <span class="hljs-number">14</span>), (<span class="hljs-number">14</span>, <span class="hljs-number">16</span>), (<span class="hljs-number">16</span>, <span class="hljs-number">18</span>), (<span class="hljs-number">19</span>, <span class="hljs-number">22</span>), (<span class="hljs-number">23</span>, <span class="hljs-number">24</span>), (<span class="hljs-number">25</span>, <span class="hljs-number">29</span>), (<span class="hljs-number">30</span>, <span class="hljs-number">32</span>),
 (<span class="hljs-number">33</span>, <span class="hljs-number">35</span>), (<span class="hljs-number">35</span>, <span class="hljs-number">40</span>), (<span class="hljs-number">41</span>, <span class="hljs-number">45</span>), (<span class="hljs-number">46</span>, <span class="hljs-number">48</span>), (<span class="hljs-number">49</span>, <span class="hljs-number">57</span>), (<span class="hljs-number">57</span>, <span class="hljs-number">58</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">0</span>)]`,wrap:!1}}),Es=new w({props:{code:"ZXhhbXBsZSU1QjEyJTNBMTQlNUQ=",highlighted:'example[<span class="hljs-number">12</span>:<span class="hljs-number">14</span>]',wrap:!1}}),Vs=new w({props:{code:"eWw=",highlighted:"yl",wrap:!1}}),Rs=new w({props:{code:"cmVzdWx0cyUyMCUzRCUyMCU1QiU1RCUwQWlucHV0c193aXRoX29mZnNldHMlMjAlM0QlMjB0b2tlbml6ZXIoZXhhbXBsZSUyQyUyMHJldHVybl9vZmZzZXRzX21hcHBpbmclM0RUcnVlKSUwQXRva2VucyUyMCUzRCUyMGlucHV0c193aXRoX29mZnNldHMudG9rZW5zKCklMEFvZmZzZXRzJTIwJTNEJTIwaW5wdXRzX3dpdGhfb2Zmc2V0cyU1QiUyMm9mZnNldF9tYXBwaW5nJTIyJTVEJTBBJTBBZm9yJTIwaWR4JTJDJTIwcHJlZCUyMGluJTIwZW51bWVyYXRlKHByZWRpY3Rpb25zKSUzQSUwQSUyMCUyMCUyMCUyMGxhYmVsJTIwJTNEJTIwbW9kZWwuY29uZmlnLmlkMmxhYmVsJTVCcHJlZCU1RCUwQSUyMCUyMCUyMCUyMGlmJTIwbGFiZWwlMjAhJTNEJTIwJTIyTyUyMiUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0YXJ0JTJDJTIwZW5kJTIwJTNEJTIwb2Zmc2V0cyU1QmlkeCU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlc3VsdHMuYXBwZW5kKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVudGl0eSUyMiUzQSUyMGxhYmVsJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyc2NvcmUlMjIlM0ElMjBwcm9iYWJpbGl0aWVzJTVCaWR4JTVEJTVCcHJlZCU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMndvcmQlMjIlM0ElMjB0b2tlbnMlNUJpZHglNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFydCUyMiUzQSUyMHN0YXJ0JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZW5kJTIyJTNBJTIwZW5kJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUwQXByaW50KHJlc3VsdHMp",highlighted:`results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]

<span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions):
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        start, end = offsets[idx]
        results.append(
            {
                <span class="hljs-string">&quot;entity&quot;</span>: label,
                <span class="hljs-string">&quot;score&quot;</span>: probabilities[idx][pred],
                <span class="hljs-string">&quot;word&quot;</span>: tokens[idx],
                <span class="hljs-string">&quot;start&quot;</span>: start,
                <span class="hljs-string">&quot;end&quot;</span>: end,
            }
        )

<span class="hljs-built_in">print</span>(results)`,wrap:!1}}),Hs=new w({props:{code:"JTVCJTdCJ2VudGl0eSclM0ElMjAnSS1QRVInJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTk5MzgyOCUyQyUyMCdpbmRleCclM0ElMjA0JTJDJTIwJ3dvcmQnJTNBJTIwJ1MnJTJDJTIwJ3N0YXJ0JyUzQSUyMDExJTJDJTIwJ2VuZCclM0ElMjAxMiU3RCUyQyUwQSUyMCU3QidlbnRpdHknJTNBJTIwJ0ktUEVSJyUyQyUyMCdzY29yZSclM0ElMjAwLjk5ODE1NDc2JTJDJTIwJ2luZGV4JyUzQSUyMDUlMkMlMjAnd29yZCclM0ElMjAnJTIzJTIzeWwnJTJDJTIwJ3N0YXJ0JyUzQSUyMDEyJTJDJTIwJ2VuZCclM0ElMjAxNCU3RCUyQyUwQSUyMCU3QidlbnRpdHknJTNBJTIwJ0ktUEVSJyUyQyUyMCdzY29yZSclM0ElMjAwLjk5NTkwNzI1JTJDJTIwJ2luZGV4JyUzQSUyMDYlMkMlMjAnd29yZCclM0ElMjAnJTIzJTIzdmEnJTJDJTIwJ3N0YXJ0JyUzQSUyMDE0JTJDJTIwJ2VuZCclM0ElMjAxNiU3RCUyQyUwQSUyMCU3QidlbnRpdHknJTNBJTIwJ0ktUEVSJyUyQyUyMCdzY29yZSclM0ElMjAwLjk5OTIzMjclMkMlMjAnaW5kZXgnJTNBJTIwNyUyQyUyMCd3b3JkJyUzQSUyMCclMjMlMjNpbiclMkMlMjAnc3RhcnQnJTNBJTIwMTYlMkMlMjAnZW5kJyUzQSUyMDE4JTdEJTJDJTBBJTIwJTdCJ2VudGl0eSclM0ElMjAnSS1PUkcnJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTczODkzMzQlMkMlMjAnaW5kZXgnJTNBJTIwMTIlMkMlMjAnd29yZCclM0ElMjAnSHUnJTJDJTIwJ3N0YXJ0JyUzQSUyMDMzJTJDJTIwJ2VuZCclM0ElMjAzNSU3RCUyQyUwQSUyMCU3QidlbnRpdHknJTNBJTIwJ0ktT1JHJyUyQyUyMCdzY29yZSclM0ElMjAwLjk3NjExNSUyQyUyMCdpbmRleCclM0ElMjAxMyUyQyUyMCd3b3JkJyUzQSUyMCclMjMlMjNnZ2luZyclMkMlMjAnc3RhcnQnJTNBJTIwMzUlMkMlMjAnZW5kJyUzQSUyMDQwJTdEJTJDJTBBJTIwJTdCJ2VudGl0eSclM0ElMjAnSS1PUkcnJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTg4Nzk3NjYlMkMlMjAnaW5kZXgnJTNBJTIwMTQlMkMlMjAnd29yZCclM0ElMjAnRmFjZSclMkMlMjAnc3RhcnQnJTNBJTIwNDElMkMlMjAnZW5kJyUzQSUyMDQ1JTdEJTJDJTBBJTIwJTdCJ2VudGl0eSclM0ElMjAnSS1MT0MnJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTkzMjEwNTUlMkMlMjAnaW5kZXgnJTNBJTIwMTYlMkMlMjAnd29yZCclM0ElMjAnQnJvb2tseW4nJTJDJTIwJ3N0YXJ0JyUzQSUyMDQ5JTJDJTIwJ2VuZCclM0ElMjA1NyU3RCU1RA==",highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">12</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">14</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">35</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">35</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">40</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">41</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`,wrap:!1}}),Ds=new cl({props:{title:"Gruparea entităților",local:"grouping-entities",headingTag:"h3"}}),Fs=new w({props:{code:"ZXhhbXBsZSU1QjMzJTNBNDUlNUQ=",highlighted:'example[<span class="hljs-number">33</span>:<span class="hljs-number">45</span>]',wrap:!1}}),Ls=new w({props:{code:"SHVnZ2luZyUyMEZhY2U=",highlighted:"Hugging Face",wrap:!1}}),qs=new w({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBJTBBcmVzdWx0cyUyMCUzRCUyMCU1QiU1RCUwQWlucHV0c193aXRoX29mZnNldHMlMjAlM0QlMjB0b2tlbml6ZXIoZXhhbXBsZSUyQyUyMHJldHVybl9vZmZzZXRzX21hcHBpbmclM0RUcnVlKSUwQXRva2VucyUyMCUzRCUyMGlucHV0c193aXRoX29mZnNldHMudG9rZW5zKCklMEFvZmZzZXRzJTIwJTNEJTIwaW5wdXRzX3dpdGhfb2Zmc2V0cyU1QiUyMm9mZnNldF9tYXBwaW5nJTIyJTVEJTBBJTBBaWR4JTIwJTNEJTIwMCUwQXdoaWxlJTIwaWR4JTIwJTNDJTIwbGVuKHByZWRpY3Rpb25zKSUzQSUwQSUyMCUyMCUyMCUyMHByZWQlMjAlM0QlMjBwcmVkaWN0aW9ucyU1QmlkeCU1RCUwQSUyMCUyMCUyMCUyMGxhYmVsJTIwJTNEJTIwbW9kZWwuY29uZmlnLmlkMmxhYmVsJTVCcHJlZCU1RCUwQSUyMCUyMCUyMCUyMGlmJTIwbGFiZWwlMjAhJTNEJTIwJTIyTyUyMiUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFJlbW92ZSUyMHRoZSUyMEItJTIwb3IlMjBJLSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVsJTIwJTNEJTIwbGFiZWwlNUIyJTNBJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RhcnQlMkMlMjBfJTIwJTNEJTIwb2Zmc2V0cyU1QmlkeCU1RCUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMEdyYWIlMjBhbGwlMjB0aGUlMjB0b2tlbnMlMjBsYWJlbGVkJTIwd2l0aCUyMEktbGFiZWwlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbGxfc2NvcmVzJTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2hpbGUlMjAoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWR4JTIwJTNDJTIwbGVuKHByZWRpY3Rpb25zKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFuZCUyMG1vZGVsLmNvbmZpZy5pZDJsYWJlbCU1QnByZWRpY3Rpb25zJTVCaWR4JTVEJTVEJTIwJTNEJTNEJTIwZiUyMkktJTdCbGFiZWwlN0QlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWxsX3Njb3Jlcy5hcHBlbmQocHJvYmFiaWxpdGllcyU1QmlkeCU1RCU1QnByZWQlNUQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwXyUyQyUyMGVuZCUyMCUzRCUyMG9mZnNldHMlNUJpZHglNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZHglMjAlMkIlM0QlMjAxJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwVGhlJTIwc2NvcmUlMjBpcyUyMHRoZSUyMG1lYW4lMjBvZiUyMGFsbCUyMHRoZSUyMHNjb3JlcyUyMG9mJTIwdGhlJTIwdG9rZW5zJTIwaW4lMjB0aGF0JTIwZ3JvdXBlZCUyMGVudGl0eSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNjb3JlJTIwJTNEJTIwbnAubWVhbihhbGxfc2NvcmVzKS5pdGVtKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkJTIwJTNEJTIwZXhhbXBsZSU1QnN0YXJ0JTNBZW5kJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0cy5hcHBlbmQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZW50aXR5X2dyb3VwJTIyJTNBJTIwbGFiZWwlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzY29yZSUyMiUzQSUyMHNjb3JlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyd29yZCUyMiUzQSUyMHdvcmQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFydCUyMiUzQSUyMHN0YXJ0JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZW5kJTIyJTNBJTIwZW5kJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMGlkeCUyMCUyQiUzRCUyMDElMEElMEFwcmludChyZXN1bHRzKQ==",highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]

idx = <span class="hljs-number">0</span>
<span class="hljs-keyword">while</span> idx &lt; <span class="hljs-built_in">len</span>(predictions):
    pred = predictions[idx]
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        <span class="hljs-comment"># Remove the B- or I-</span>
        label = label[<span class="hljs-number">2</span>:]
        start, _ = offsets[idx]

        <span class="hljs-comment"># Grab all the tokens labeled with I-label</span>
        all_scores = []
        <span class="hljs-keyword">while</span> (
            idx &lt; <span class="hljs-built_in">len</span>(predictions)
            <span class="hljs-keyword">and</span> model.config.id2label[predictions[idx]] == <span class="hljs-string">f&quot;I-<span class="hljs-subst">{label}</span>&quot;</span>
        ):
            all_scores.append(probabilities[idx][pred])
            _, end = offsets[idx]
            idx += <span class="hljs-number">1</span>

        <span class="hljs-comment"># The score is the mean of all the scores of the tokens in that grouped entity</span>
        score = np.mean(all_scores).item()
        word = example[start:end]
        results.append(
            {
                <span class="hljs-string">&quot;entity_group&quot;</span>: label,
                <span class="hljs-string">&quot;score&quot;</span>: score,
                <span class="hljs-string">&quot;word&quot;</span>: word,
                <span class="hljs-string">&quot;start&quot;</span>: start,
                <span class="hljs-string">&quot;end&quot;</span>: end,
            }
        )
    idx += <span class="hljs-number">1</span>

<span class="hljs-built_in">print</span>(results)`,wrap:!1}}),Ks=new w({props:{code:"JTVCJTdCJ2VudGl0eV9ncm91cCclM0ElMjAnUEVSJyUyQyUyMCdzY29yZSclM0ElMjAwLjk5ODE2OTQlMkMlMjAnd29yZCclM0ElMjAnU3lsdmFpbiclMkMlMjAnc3RhcnQnJTNBJTIwMTElMkMlMjAnZW5kJyUzQSUyMDE4JTdEJTJDJTBBJTIwJTdCJ2VudGl0eV9ncm91cCclM0ElMjAnT1JHJyUyQyUyMCdzY29yZSclM0ElMjAwLjk3OTYwMjA0JTJDJTIwJ3dvcmQnJTNBJTIwJ0h1Z2dpbmclMjBGYWNlJyUyQyUyMCdzdGFydCclM0ElMjAzMyUyQyUyMCdlbmQnJTNBJTIwNDUlN0QlMkMlMEElMjAlN0InZW50aXR5X2dyb3VwJyUzQSUyMCdMT0MnJTJDJTIwJ3Njb3JlJyUzQSUyMDAuOTkzMjEwNTUlMkMlMjAnd29yZCclM0ElMjAnQnJvb2tseW4nJTJDJTIwJ3N0YXJ0JyUzQSUyMDQ5JTJDJTIwJ2VuZCclM0ElMjA1NyU3RCU1RA==",highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`,wrap:!1}}),ll=new Gn({props:{source:"https://github.com/huggingface/course/blob/main/chapters/rum/chapter6/3.mdx"}}),{c(){i=y("meta"),m=t(),a=y("p"),U=t(),u(k.$$.fragment),g=t(),u($.$$.fragment),z=t(),f.c(),I=t(),C=y("p"),C.innerHTML=r,h=t(),u(V.$$.fragment),ul=t(),G=y("p"),G.innerHTML=ve,Jl=t(),R=y("table"),R.innerHTML=Ve,ol=t(),u(W.$$.fragment),jl=t(),u(H.$$.fragment),ml=t(),u(X.$$.fragment),yl=t(),D=y("p"),D.innerHTML=Ge,Tl=t(),_=y("p"),_.innerHTML=Re,dl=t(),Y=y("p"),Y.textContent=He,Ul=t(),u(F.$$.fragment),wl=t(),L=y("p"),L.innerHTML=Xe,bl=t(),u(O.$$.fragment),hl=t(),q=y("p"),q.innerHTML=De,xl=t(),u(P.$$.fragment),Il=t(),u(K.$$.fragment),fl=t(),ss=y("p"),ss.innerHTML=_e,gl=t(),u(ls.$$.fragment),kl=t(),u(es.$$.fragment),Cl=t(),ns=y("p"),ns.textContent=Ye,zl=t(),u(as.$$.fragment),$l=t(),u(ts.$$.fragment),Bl=t(),ps=y("p"),ps.innerHTML=Fe,Ql=t(),u(is.$$.fragment),Zl=t(),u(rs.$$.fragment),Al=t(),u(cs.$$.fragment),Nl=t(),Ms=y("p"),Ms.innerHTML=Le,Sl=t(),u(E.$$.fragment),Wl=t(),u(us.$$.fragment),El=t(),Js=y("p"),Js.innerHTML=Oe,vl=t(),Q.c(),pl=t(),u(os.$$.fragment),Vl=t(),js=y("p"),js.innerHTML=qe,Gl=t(),u(ms.$$.fragment),Rl=t(),u(ys.$$.fragment),Hl=t(),Ts=y("p"),Ts.textContent=Pe,Xl=t(),u(ds.$$.fragment),Dl=t(),u(Us.$$.fragment),_l=t(),ws=y("p"),ws.innerHTML=Ke,Yl=t(),bs=y("ul"),bs.innerHTML=sn,Fl=t(),hs=y("p"),hs.innerHTML=ln,Ll=t(),u(xs.$$.fragment),Ol=t(),A.c(),il=t(),Is=y("p"),Is.textContent=en,ql=t(),S.c(),rl=t(),u(fs.$$.fragment),Pl=t(),gs=y("p"),gs.innerHTML=nn,Kl=t(),u(ks.$$.fragment),se=t(),u(Cs.$$.fragment),le=t(),zs=y("p"),zs.innerHTML=an,ee=t(),$s=y("p"),$s.innerHTML=tn,ne=t(),v=y("div"),v.innerHTML=pn,ae=t(),Bs=y("p"),Bs.innerHTML=rn,te=t(),u(Qs.$$.fragment),pe=t(),u(Zs.$$.fragment),ie=t(),As=y("p"),As.innerHTML=cn,re=t(),u(Ns.$$.fragment),ce=t(),u(Ss.$$.fragment),Me=t(),Ws=y("p"),Ws.innerHTML=Mn,ue=t(),u(Es.$$.fragment),Je=t(),vs=y("p"),vs.innerHTML=un,oe=t(),u(Vs.$$.fragment),je=t(),Gs=y("p"),Gs.textContent=Jn,me=t(),u(Rs.$$.fragment),ye=t(),u(Hs.$$.fragment),Te=t(),Xs=y("p"),Xs.textContent=on,de=t(),u(Ds.$$.fragment),Ue=t(),_s=y("p"),_s.innerHTML=jn,we=t(),Ys=y("p"),Ys.innerHTML=mn,be=t(),u(Fs.$$.fragment),he=t(),u(Ls.$$.fragment),xe=t(),Os=y("p"),Os.innerHTML=yn,Ie=t(),u(qs.$$.fragment),fe=t(),Ps=y("p"),Ps.textContent=Tn,ge=t(),u(Ks.$$.fragment),ke=t(),sl=y("p"),sl.textContent=dn,Ce=t(),u(ll.$$.fragment),ze=t(),Ml=y("p"),this.h()},l(s){const l=En("svelte-u9bgzb",document.head);i=T(l,"META",{name:!0,content:!0}),l.forEach(e),m=p(s),a=T(s,"P",{}),$n(a).forEach(e),U=p(s),J(k.$$.fragment,s),g=p(s),J($.$$.fragment,s),z=p(s),f.l(s),I=p(s),C=T(s,"P",{"data-svelte-h":!0}),d(C)!=="svelte-8rg866"&&(C.innerHTML=r),h=p(s),J(V.$$.fragment,s),ul=p(s),G=T(s,"P",{"data-svelte-h":!0}),d(G)!=="svelte-1nrltkq"&&(G.innerHTML=ve),Jl=p(s),R=T(s,"TABLE",{"data-svelte-h":!0}),d(R)!=="svelte-abdajt"&&(R.innerHTML=Ve),ol=p(s),J(W.$$.fragment,s),jl=p(s),J(H.$$.fragment,s),ml=p(s),J(X.$$.fragment,s),yl=p(s),D=T(s,"P",{"data-svelte-h":!0}),d(D)!=="svelte-2l0va5"&&(D.innerHTML=Ge),Tl=p(s),_=T(s,"P",{"data-svelte-h":!0}),d(_)!=="svelte-1pkab8t"&&(_.innerHTML=Re),dl=p(s),Y=T(s,"P",{"data-svelte-h":!0}),d(Y)!=="svelte-1t7zxyh"&&(Y.textContent=He),Ul=p(s),J(F.$$.fragment,s),wl=p(s),L=T(s,"P",{"data-svelte-h":!0}),d(L)!=="svelte-ikrrxj"&&(L.innerHTML=Xe),bl=p(s),J(O.$$.fragment,s),hl=p(s),q=T(s,"P",{"data-svelte-h":!0}),d(q)!=="svelte-xwn6gy"&&(q.innerHTML=De),xl=p(s),J(P.$$.fragment,s),Il=p(s),J(K.$$.fragment,s),fl=p(s),ss=T(s,"P",{"data-svelte-h":!0}),d(ss)!=="svelte-kwefih"&&(ss.innerHTML=_e),gl=p(s),J(ls.$$.fragment,s),kl=p(s),J(es.$$.fragment,s),Cl=p(s),ns=T(s,"P",{"data-svelte-h":!0}),d(ns)!=="svelte-14wtt33"&&(ns.textContent=Ye),zl=p(s),J(as.$$.fragment,s),$l=p(s),J(ts.$$.fragment,s),Bl=p(s),ps=T(s,"P",{"data-svelte-h":!0}),d(ps)!=="svelte-1n63hf7"&&(ps.innerHTML=Fe),Ql=p(s),J(is.$$.fragment,s),Zl=p(s),J(rs.$$.fragment,s),Al=p(s),J(cs.$$.fragment,s),Nl=p(s),Ms=T(s,"P",{"data-svelte-h":!0}),d(Ms)!=="svelte-162s1ta"&&(Ms.innerHTML=Le),Sl=p(s),J(E.$$.fragment,s),Wl=p(s),J(us.$$.fragment,s),El=p(s),Js=T(s,"P",{"data-svelte-h":!0}),d(Js)!=="svelte-ou4i8v"&&(Js.innerHTML=Oe),vl=p(s),Q.l(s),pl=p(s),J(os.$$.fragment,s),Vl=p(s),js=T(s,"P",{"data-svelte-h":!0}),d(js)!=="svelte-1il1qfv"&&(js.innerHTML=qe),Gl=p(s),J(ms.$$.fragment,s),Rl=p(s),J(ys.$$.fragment,s),Hl=p(s),Ts=T(s,"P",{"data-svelte-h":!0}),d(Ts)!=="svelte-e9zjj8"&&(Ts.textContent=Pe),Xl=p(s),J(ds.$$.fragment,s),Dl=p(s),J(Us.$$.fragment,s),_l=p(s),ws=T(s,"P",{"data-svelte-h":!0}),d(ws)!=="svelte-1y4icev"&&(ws.innerHTML=Ke),Yl=p(s),bs=T(s,"UL",{"data-svelte-h":!0}),d(bs)!=="svelte-101w73y"&&(bs.innerHTML=sn),Fl=p(s),hs=T(s,"P",{"data-svelte-h":!0}),d(hs)!=="svelte-1po40j3"&&(hs.innerHTML=ln),Ll=p(s),J(xs.$$.fragment,s),Ol=p(s),A.l(s),il=p(s),Is=T(s,"P",{"data-svelte-h":!0}),d(Is)!=="svelte-1jjvzlr"&&(Is.textContent=en),ql=p(s),S.l(s),rl=p(s),J(fs.$$.fragment,s),Pl=p(s),gs=T(s,"P",{"data-svelte-h":!0}),d(gs)!=="svelte-buv0zd"&&(gs.innerHTML=nn),Kl=p(s),J(ks.$$.fragment,s),se=p(s),J(Cs.$$.fragment,s),le=p(s),zs=T(s,"P",{"data-svelte-h":!0}),d(zs)!=="svelte-14c0e2l"&&(zs.innerHTML=an),ee=p(s),$s=T(s,"P",{"data-svelte-h":!0}),d($s)!=="svelte-19dgt1c"&&($s.innerHTML=tn),ne=p(s),v=T(s,"DIV",{class:!0,"data-svelte-h":!0}),d(v)!=="svelte-qf2tof"&&(v.innerHTML=pn),ae=p(s),Bs=T(s,"P",{"data-svelte-h":!0}),d(Bs)!=="svelte-1hl7uvg"&&(Bs.innerHTML=rn),te=p(s),J(Qs.$$.fragment,s),pe=p(s),J(Zs.$$.fragment,s),ie=p(s),As=T(s,"P",{"data-svelte-h":!0}),d(As)!=="svelte-161dpss"&&(As.innerHTML=cn),re=p(s),J(Ns.$$.fragment,s),ce=p(s),J(Ss.$$.fragment,s),Me=p(s),Ws=T(s,"P",{"data-svelte-h":!0}),d(Ws)!=="svelte-d6hya3"&&(Ws.innerHTML=Mn),ue=p(s),J(Es.$$.fragment,s),Je=p(s),vs=T(s,"P",{"data-svelte-h":!0}),d(vs)!=="svelte-11u2lrs"&&(vs.innerHTML=un),oe=p(s),J(Vs.$$.fragment,s),je=p(s),Gs=T(s,"P",{"data-svelte-h":!0}),d(Gs)!=="svelte-19h3phn"&&(Gs.textContent=Jn),me=p(s),J(Rs.$$.fragment,s),ye=p(s),J(Hs.$$.fragment,s),Te=p(s),Xs=T(s,"P",{"data-svelte-h":!0}),d(Xs)!=="svelte-1c61k0i"&&(Xs.textContent=on),de=p(s),J(Ds.$$.fragment,s),Ue=p(s),_s=T(s,"P",{"data-svelte-h":!0}),d(_s)!=="svelte-28ykn1"&&(_s.innerHTML=jn),we=p(s),Ys=T(s,"P",{"data-svelte-h":!0}),d(Ys)!=="svelte-f7mp48"&&(Ys.innerHTML=mn),be=p(s),J(Fs.$$.fragment,s),he=p(s),J(Ls.$$.fragment,s),xe=p(s),Os=T(s,"P",{"data-svelte-h":!0}),d(Os)!=="svelte-jzfa8h"&&(Os.innerHTML=yn),Ie=p(s),J(qs.$$.fragment,s),fe=p(s),Ps=T(s,"P",{"data-svelte-h":!0}),d(Ps)!=="svelte-1qh60rk"&&(Ps.textContent=Tn),ge=p(s),J(Ks.$$.fragment,s),ke=p(s),sl=T(s,"P",{"data-svelte-h":!0}),d(sl)!=="svelte-5h0r78"&&(sl.textContent=dn),Ce=p(s),J(ll.$$.fragment,s),ze=p(s),Ml=T(s,"P",{}),$n(Ml).forEach(e),this.h()},h(){Ee(i,"name","hf:doc:metadata"),Ee(i,"content",Kn),Ee(v,"class","flex justify-center")},m(s,l){vn(document.head,i),n(s,m,l),n(s,a,l),n(s,U,l),o(k,s,l),n(s,g,l),o($,s,l),n(s,z,l),el[b].m(s,l),n(s,I,l),n(s,C,l),n(s,h,l),o(V,s,l),n(s,ul,l),n(s,G,l),n(s,Jl,l),n(s,R,l),n(s,ol,l),o(W,s,l),n(s,jl,l),o(H,s,l),n(s,ml,l),o(X,s,l),n(s,yl,l),n(s,D,l),n(s,Tl,l),n(s,_,l),n(s,dl,l),n(s,Y,l),n(s,Ul,l),o(F,s,l),n(s,wl,l),n(s,L,l),n(s,bl,l),o(O,s,l),n(s,hl,l),n(s,q,l),n(s,xl,l),o(P,s,l),n(s,Il,l),o(K,s,l),n(s,fl,l),n(s,ss,l),n(s,gl,l),o(ls,s,l),n(s,kl,l),o(es,s,l),n(s,Cl,l),n(s,ns,l),n(s,zl,l),o(as,s,l),n(s,$l,l),o(ts,s,l),n(s,Bl,l),n(s,ps,l),n(s,Ql,l),o(is,s,l),n(s,Zl,l),o(rs,s,l),n(s,Al,l),o(cs,s,l),n(s,Nl,l),n(s,Ms,l),n(s,Sl,l),o(E,s,l),n(s,Wl,l),o(us,s,l),n(s,El,l),n(s,Js,l),n(s,vl,l),nl[B].m(s,l),n(s,pl,l),o(os,s,l),n(s,Vl,l),n(s,js,l),n(s,Gl,l),o(ms,s,l),n(s,Rl,l),o(ys,s,l),n(s,Hl,l),n(s,Ts,l),n(s,Xl,l),o(ds,s,l),n(s,Dl,l),o(Us,s,l),n(s,_l,l),n(s,ws,l),n(s,Yl,l),n(s,bs,l),n(s,Fl,l),n(s,hs,l),n(s,Ll,l),o(xs,s,l),n(s,Ol,l),al[Z].m(s,l),n(s,il,l),n(s,Is,l),n(s,ql,l),tl[N].m(s,l),n(s,rl,l),o(fs,s,l),n(s,Pl,l),n(s,gs,l),n(s,Kl,l),o(ks,s,l),n(s,se,l),o(Cs,s,l),n(s,le,l),n(s,zs,l),n(s,ee,l),n(s,$s,l),n(s,ne,l),n(s,v,l),n(s,ae,l),n(s,Bs,l),n(s,te,l),o(Qs,s,l),n(s,pe,l),o(Zs,s,l),n(s,ie,l),n(s,As,l),n(s,re,l),o(Ns,s,l),n(s,ce,l),o(Ss,s,l),n(s,Me,l),n(s,Ws,l),n(s,ue,l),o(Es,s,l),n(s,Je,l),n(s,vs,l),n(s,oe,l),o(Vs,s,l),n(s,je,l),n(s,Gs,l),n(s,me,l),o(Rs,s,l),n(s,ye,l),o(Hs,s,l),n(s,Te,l),n(s,Xs,l),n(s,de,l),o(Ds,s,l),n(s,Ue,l),n(s,_s,l),n(s,we,l),n(s,Ys,l),n(s,be,l),o(Fs,s,l),n(s,he,l),o(Ls,s,l),n(s,xe,l),n(s,Os,l),n(s,Ie,l),o(qs,s,l),n(s,fe,l),n(s,Ps,l),n(s,ge,l),o(Ks,s,l),n(s,ke,l),n(s,sl,l),n(s,Ce,l),o(ll,s,l),n(s,ze,l),n(s,Ml,l),$e=!0},p(s,[l]){const kn={};l&1&&(kn.fw=s[0]),k.$set(kn);let Ae=b;b=wn(s),b!==Ae&&(Qe(),c(el[Ae],1,1,()=>{el[Ae]=null}),Be(),f=el[b],f||(f=el[b]=Un[b](s),f.c()),M(f,1),f.m(I.parentNode,I));const Cn={};l&2&&(Cn.$$scope={dirty:l,ctx:s}),W.$set(Cn);const zn={};l&2&&(zn.$$scope={dirty:l,ctx:s}),E.$set(zn);let Ne=B;B=hn(s),B!==Ne&&(Qe(),c(nl[Ne],1,1,()=>{nl[Ne]=null}),Be(),Q=nl[B],Q||(Q=nl[B]=bn[B](s),Q.c()),M(Q,1),Q.m(pl.parentNode,pl));let Se=Z;Z=In(s),Z!==Se&&(Qe(),c(al[Se],1,1,()=>{al[Se]=null}),Be(),A=al[Z],A||(A=al[Z]=xn[Z](s),A.c()),M(A,1),A.m(il.parentNode,il));let We=N;N=gn(s),N!==We&&(Qe(),c(tl[We],1,1,()=>{tl[We]=null}),Be(),S=tl[N],S||(S=tl[N]=fn[N](s),S.c()),M(S,1),S.m(rl.parentNode,rl))},i(s){$e||(M(k.$$.fragment,s),M($.$$.fragment,s),M(f),M(V.$$.fragment,s),M(W.$$.fragment,s),M(H.$$.fragment,s),M(X.$$.fragment,s),M(F.$$.fragment,s),M(O.$$.fragment,s),M(P.$$.fragment,s),M(K.$$.fragment,s),M(ls.$$.fragment,s),M(es.$$.fragment,s),M(as.$$.fragment,s),M(ts.$$.fragment,s),M(is.$$.fragment,s),M(rs.$$.fragment,s),M(cs.$$.fragment,s),M(E.$$.fragment,s),M(us.$$.fragment,s),M(Q),M(os.$$.fragment,s),M(ms.$$.fragment,s),M(ys.$$.fragment,s),M(ds.$$.fragment,s),M(Us.$$.fragment,s),M(xs.$$.fragment,s),M(A),M(S),M(fs.$$.fragment,s),M(ks.$$.fragment,s),M(Cs.$$.fragment,s),M(Qs.$$.fragment,s),M(Zs.$$.fragment,s),M(Ns.$$.fragment,s),M(Ss.$$.fragment,s),M(Es.$$.fragment,s),M(Vs.$$.fragment,s),M(Rs.$$.fragment,s),M(Hs.$$.fragment,s),M(Ds.$$.fragment,s),M(Fs.$$.fragment,s),M(Ls.$$.fragment,s),M(qs.$$.fragment,s),M(Ks.$$.fragment,s),M(ll.$$.fragment,s),$e=!0)},o(s){c(k.$$.fragment,s),c($.$$.fragment,s),c(f),c(V.$$.fragment,s),c(W.$$.fragment,s),c(H.$$.fragment,s),c(X.$$.fragment,s),c(F.$$.fragment,s),c(O.$$.fragment,s),c(P.$$.fragment,s),c(K.$$.fragment,s),c(ls.$$.fragment,s),c(es.$$.fragment,s),c(as.$$.fragment,s),c(ts.$$.fragment,s),c(is.$$.fragment,s),c(rs.$$.fragment,s),c(cs.$$.fragment,s),c(E.$$.fragment,s),c(us.$$.fragment,s),c(Q),c(os.$$.fragment,s),c(ms.$$.fragment,s),c(ys.$$.fragment,s),c(ds.$$.fragment,s),c(Us.$$.fragment,s),c(xs.$$.fragment,s),c(A),c(S),c(fs.$$.fragment,s),c(ks.$$.fragment,s),c(Cs.$$.fragment,s),c(Qs.$$.fragment,s),c(Zs.$$.fragment,s),c(Ns.$$.fragment,s),c(Ss.$$.fragment,s),c(Es.$$.fragment,s),c(Vs.$$.fragment,s),c(Rs.$$.fragment,s),c(Hs.$$.fragment,s),c(Ds.$$.fragment,s),c(Fs.$$.fragment,s),c(Ls.$$.fragment,s),c(qs.$$.fragment,s),c(Ks.$$.fragment,s),c(ll.$$.fragment,s),$e=!1},d(s){s&&(e(m),e(a),e(U),e(g),e(z),e(I),e(C),e(h),e(ul),e(G),e(Jl),e(R),e(ol),e(jl),e(ml),e(yl),e(D),e(Tl),e(_),e(dl),e(Y),e(Ul),e(wl),e(L),e(bl),e(hl),e(q),e(xl),e(Il),e(fl),e(ss),e(gl),e(kl),e(Cl),e(ns),e(zl),e($l),e(Bl),e(ps),e(Ql),e(Zl),e(Al),e(Nl),e(Ms),e(Sl),e(Wl),e(El),e(Js),e(vl),e(pl),e(Vl),e(js),e(Gl),e(Rl),e(Hl),e(Ts),e(Xl),e(Dl),e(_l),e(ws),e(Yl),e(bs),e(Fl),e(hs),e(Ll),e(Ol),e(il),e(Is),e(ql),e(rl),e(Pl),e(gs),e(Kl),e(se),e(le),e(zs),e(ee),e($s),e(ne),e(v),e(ae),e(Bs),e(te),e(pe),e(ie),e(As),e(re),e(ce),e(Me),e(Ws),e(ue),e(Je),e(vs),e(oe),e(je),e(Gs),e(me),e(ye),e(Te),e(Xs),e(de),e(Ue),e(_s),e(we),e(Ys),e(be),e(he),e(xe),e(Os),e(Ie),e(fe),e(Ps),e(ge),e(ke),e(sl),e(Ce),e(ze),e(Ml)),e(i),j(k,s),j($,s),el[b].d(s),j(V,s),j(W,s),j(H,s),j(X,s),j(F,s),j(O,s),j(P,s),j(K,s),j(ls,s),j(es,s),j(as,s),j(ts,s),j(is,s),j(rs,s),j(cs,s),j(E,s),j(us,s),nl[B].d(s),j(os,s),j(ms,s),j(ys,s),j(ds,s),j(Us,s),j(xs,s),al[Z].d(s),tl[N].d(s),j(fs,s),j(ks,s),j(Cs,s),j(Qs,s),j(Zs,s),j(Ns,s),j(Ss,s),j(Es,s),j(Vs,s),j(Rs,s),j(Hs,s),j(Ds,s),j(Fs,s),j(Ls,s),j(qs,s),j(Ks,s),j(ll,s)}}}const Kn='{"title":"Superputerile tokenizerilor rapizi","local":"fast-tokenizers-special-powers","sections":[{"title":"Batch encoding","local":"batch-encoding","sections":[],"depth":2},{"title":"În interiorul pipelineului token-classification","local":"inside-the-token-classification-pipeline","sections":[{"title":"Obținerea rezultatelor de bază cu pipelineul","local":"getting-the-base-results-with-the-pipeline","sections":[],"depth":3},{"title":"De la inputuri la predicții","local":"from-inputs-to-predictions","sections":[],"depth":3},{"title":"Gruparea entităților","local":"grouping-entities","sections":[],"depth":3}],"depth":2}],"depth":1}';function sa(x,i,m){let a="pt";return Nn(()=>{const U=new URLSearchParams(window.location.search);m(0,a=U.get("fw")||"pt")}),[a]}class ca extends Sn{constructor(i){super(),Wn(this,i,sa,Pn,An,{})}}export{ca as component};
