import{s as xe,n as Ie,o as Fe}from"../chunks/scheduler.d586627e.js";import{S as Ae,i as Qe,g as n,s as l,u as B,A as Se,h as o,f as t,c as r,j as m,v as E,y as c,k as i,o as p,a,w as P,d as q,t as x,x as I}from"../chunks/index.b757c891.js";import{I as ye}from"../chunks/IconCopyLink.c074b82b.js";import{C as fe}from"../chunks/CodeBlock.36019a2e.js";function We(Me){let d,H,S,L,u,k,W,y,ke,F,je="Usando os Tokenizers do 🤗 Tokenizers",Y,M,ze='O <code>PreTrainedTokenizerFast</code> depende da biblioteca <a href="https://huggingface.co/docs/tokenizers" rel="nofollow">🤗 Tokenizers</a>. O Tokenizer obtido da biblioteca 🤗 Tokenizers pode ser carregado facilmente pelo 🤗 Transformers.',K,j,be="Antes de entrar nos detalhes, vamos começar criando um tokenizer fictício em algumas linhas:",O,z,D,b,Te="Agora temos um tokenizer treinado nos arquivos que foram definidos. Nós podemos continuar usando nessa execução ou salvar em um arquivo JSON para re-utilizar no futuro.",ee,h,g,X,T,ge,A,$e="Carregando diretamente de um objeto tokenizer",te,$,we="Vamos ver como aproveitar esse objeto tokenizer na biblioteca 🤗 Transformers. A classe <code>PreTrainedTokenizerFast</code> permite uma instanciação fácil, aceitando o objeto <em>tokenizer</em> instanciado como um argumento:",se,w,ae,U,Ue='Esse objeto pode ser utilizado com todos os métodos compartilhados pelos tokenizers dos 🤗 Transformers! Vá para <a href="main_classes/tokenizer">a página do tokenizer</a> para mais informações.',ne,f,v,G,_,ve,Q,_e="Carregando de um arquivo JSON",oe,C,Ce="Para carregar um tokenizer de um arquivo JSON vamos primeiro começar salvando nosso tokenizer:",le,Z,re,V,Ze="A pasta para qual salvamos esse arquivo pode ser passada para o método de inicialização do <code>PreTrainedTokenizerFast</code> usando o <code>tokenizer_file</code> parâmetro:",ie,J,pe,N,Ve='Esse objeto pode ser utilizado com todos os métodos compartilhados pelos tokenizers dos 🤗 Transformers! Vá para <a href="main_classes/tokenizer">a página do tokenizer</a> para mais informações.',me,R,ce;return y=new ye({}),z=new fe({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`}}),T=new ye({}),w=new fe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`}}),_=new ye({}),Z=new fe({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),J=new fe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`}}),{c(){d=n("meta"),H=l(),S=n("p"),L=l(),u=n("h1"),k=n("a"),W=n("span"),B(y.$$.fragment),ke=l(),F=n("span"),F.textContent=je,Y=l(),M=n("p"),M.innerHTML=ze,K=l(),j=n("p"),j.textContent=be,O=l(),B(z.$$.fragment),D=l(),b=n("p"),b.textContent=Te,ee=l(),h=n("h2"),g=n("a"),X=n("span"),B(T.$$.fragment),ge=l(),A=n("span"),A.textContent=$e,te=l(),$=n("p"),$.innerHTML=we,se=l(),B(w.$$.fragment),ae=l(),U=n("p"),U.innerHTML=Ue,ne=l(),f=n("h2"),v=n("a"),G=n("span"),B(_.$$.fragment),ve=l(),Q=n("span"),Q.textContent=_e,oe=l(),C=n("p"),C.textContent=Ce,le=l(),B(Z.$$.fragment),re=l(),V=n("p"),V.innerHTML=Ze,ie=l(),B(J.$$.fragment),pe=l(),N=n("p"),N.innerHTML=Ve,me=l(),R=n("p"),this.h()},l(e){const s=Se("svelte-1phssyn",document.head);d=o(s,"META",{name:!0,content:!0}),s.forEach(t),H=r(e),S=o(e,"P",{}),m(S).forEach(t),L=r(e),u=o(e,"H1",{class:!0});var de=m(u);k=o(de,"A",{id:!0,class:!0,href:!0});var Je=m(k);W=o(Je,"SPAN",{});var Ne=m(W);E(y.$$.fragment,Ne),Ne.forEach(t),Je.forEach(t),ke=r(de),F=o(de,"SPAN",{"data-svelte-h":!0}),c(F)!=="svelte-17z60lz"&&(F.textContent=je),de.forEach(t),Y=r(e),M=o(e,"P",{"data-svelte-h":!0}),c(M)!=="svelte-borl69"&&(M.innerHTML=ze),K=r(e),j=o(e,"P",{"data-svelte-h":!0}),c(j)!=="svelte-1q50rcm"&&(j.textContent=be),O=r(e),E(z.$$.fragment,e),D=r(e),b=o(e,"P",{"data-svelte-h":!0}),c(b)!=="svelte-17y7lov"&&(b.textContent=Te),ee=r(e),h=o(e,"H2",{class:!0});var ue=m(h);g=o(ue,"A",{id:!0,class:!0,href:!0});var Be=m(g);X=o(Be,"SPAN",{});var Ee=m(X);E(T.$$.fragment,Ee),Ee.forEach(t),Be.forEach(t),ge=r(ue),A=o(ue,"SPAN",{"data-svelte-h":!0}),c(A)!=="svelte-l92g5s"&&(A.textContent=$e),ue.forEach(t),te=r(e),$=o(e,"P",{"data-svelte-h":!0}),c($)!=="svelte-11rm4tu"&&($.innerHTML=we),se=r(e),E(w.$$.fragment,e),ae=r(e),U=o(e,"P",{"data-svelte-h":!0}),c(U)!=="svelte-bzpmh"&&(U.innerHTML=Ue),ne=r(e),f=o(e,"H2",{class:!0});var he=m(f);v=o(he,"A",{id:!0,class:!0,href:!0});var Pe=m(v);G=o(Pe,"SPAN",{});var qe=m(G);E(_.$$.fragment,qe),qe.forEach(t),Pe.forEach(t),ve=r(he),Q=o(he,"SPAN",{"data-svelte-h":!0}),c(Q)!=="svelte-1gndkez"&&(Q.textContent=_e),he.forEach(t),oe=r(e),C=o(e,"P",{"data-svelte-h":!0}),c(C)!=="svelte-vycu7g"&&(C.textContent=Ce),le=r(e),E(Z.$$.fragment,e),re=r(e),V=o(e,"P",{"data-svelte-h":!0}),c(V)!=="svelte-15so4os"&&(V.innerHTML=Ze),ie=r(e),E(J.$$.fragment,e),pe=r(e),N=o(e,"P",{"data-svelte-h":!0}),c(N)!=="svelte-bzpmh"&&(N.innerHTML=Ve),me=r(e),R=o(e,"P",{}),m(R).forEach(t),this.h()},h(){i(d,"name","hf:doc:metadata"),i(d,"content",JSON.stringify(Xe)),i(k,"id","usando-os-tokenizers-do-tokenizers"),i(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(k,"href","#usando-os-tokenizers-do-tokenizers"),i(u,"class","relative group"),i(g,"id","carregando-diretamente-de-um-objeto-tokenizer"),i(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(g,"href","#carregando-diretamente-de-um-objeto-tokenizer"),i(h,"class","relative group"),i(v,"id","carregando-de-um-arquivo-json"),i(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(v,"href","#carregando-de-um-arquivo-json"),i(f,"class","relative group")},m(e,s){p(document.head,d),a(e,H,s),a(e,S,s),a(e,L,s),a(e,u,s),p(u,k),p(k,W),P(y,W,null),p(u,ke),p(u,F),a(e,Y,s),a(e,M,s),a(e,K,s),a(e,j,s),a(e,O,s),P(z,e,s),a(e,D,s),a(e,b,s),a(e,ee,s),a(e,h,s),p(h,g),p(g,X),P(T,X,null),p(h,ge),p(h,A),a(e,te,s),a(e,$,s),a(e,se,s),P(w,e,s),a(e,ae,s),a(e,U,s),a(e,ne,s),a(e,f,s),p(f,v),p(v,G),P(_,G,null),p(f,ve),p(f,Q),a(e,oe,s),a(e,C,s),a(e,le,s),P(Z,e,s),a(e,re,s),a(e,V,s),a(e,ie,s),P(J,e,s),a(e,pe,s),a(e,N,s),a(e,me,s),a(e,R,s),ce=!0},p:Ie,i(e){ce||(q(y.$$.fragment,e),q(z.$$.fragment,e),q(T.$$.fragment,e),q(w.$$.fragment,e),q(_.$$.fragment,e),q(Z.$$.fragment,e),q(J.$$.fragment,e),ce=!0)},o(e){x(y.$$.fragment,e),x(z.$$.fragment,e),x(T.$$.fragment,e),x(w.$$.fragment,e),x(_.$$.fragment,e),x(Z.$$.fragment,e),x(J.$$.fragment,e),ce=!1},d(e){e&&(t(H),t(S),t(L),t(u),t(Y),t(M),t(K),t(j),t(O),t(D),t(b),t(ee),t(h),t(te),t($),t(se),t(ae),t(U),t(ne),t(f),t(oe),t(C),t(le),t(re),t(V),t(ie),t(pe),t(N),t(me),t(R)),t(d),I(y),I(z,e),I(T),I(w,e),I(_),I(Z,e),I(J,e)}}}const Xe={local:"usando-os-tokenizers-do-tokenizers",sections:[{local:"carregando-diretamente-de-um-objeto-tokenizer",title:"Carregando diretamente de um objeto tokenizer"},{local:"carregando-de-um-arquivo-json",title:"Carregando de um arquivo JSON"}],title:"Usando os Tokenizers do 🤗 Tokenizers"};function Ge(Me){return Fe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ke extends Ae{constructor(d){super(),Qe(this,d,Ge,We,xe,{})}}export{Ke as component};
