import{s as Ie,n as Fe,o as Ae}from"../chunks/scheduler.36a0863c.js";import{S as Qe,i as Se,g as n,s as o,u as x,A as We,h as l,f as t,c as r,j as d,v as B,y as c,k as i,o as p,a,w as E,d as P,t as I,x as F}from"../chunks/index.d0f7a4c4.js";import{I as Me}from"../chunks/IconCopyLink.bc5fad79.js";import{C as fe}from"../chunks/CodeBlock.69433f87.js";function Xe(je){let m,R,W,H,h,k,X,M,ke,A,ve="Usa los tokenizadores de 🤗 Tokenizers",Y,j,ze=`<code>PreTrainedTokenizerFast</code> depende de la biblioteca <a href="https://huggingface.co/docs/tokenizers" rel="nofollow">🤗 Tokenizers</a>. Los tokenizadores obtenidos desde la biblioteca 🤗 Tokenizers pueden ser
cargados de forma muy sencilla en los 🤗 Transformers.`,K,v,be="Antes de entrar en detalles, comencemos creando un tokenizador dummy en unas cuantas líneas:",O,z,D,b,Te=`Ahora tenemos un tokenizador entrenado en los archivos que definimos. Lo podemos seguir utilizando en ese entorno de ejecución (runtime en inglés), o puedes guardarlo
en un archivo JSON para reutilizarlo en un futuro.`,ee,u,g,q,T,ge,Q,$e="Cargando directamente desde el objeto tokenizador",te,$,we=`Veamos cómo utilizar este objeto tokenizador en la biblioteca 🤗 Transformers. La clase
<code>PreTrainedTokenizerFast</code> permite una instanciación fácil, al aceptar el objeto
<em>tokenizer</em> instanciado como argumento:`,se,w,ae,U,Ue='Este objeto ya puede ser utilizado con todos los métodos compartidos por los tokenizadores de 🤗 Transformers! Visita la <a href="main_classes/tokenizer">página sobre tokenizadores</a> para más información.',ne,f,y,G,_,ye,S,_e="Cargando desde un archivo JSON",le,C,Ce="Para cargar un tokenizador desde un archivo JSON, comencemos por guardar nuestro tokenizador:",oe,Z,re,V,Ze=`La localización (path en inglés) donde este archivo es guardado puede ser incluida en el método de inicialización de <code>PreTrainedTokenizerFast</code>
utilizando el parámetro <code>tokenizer_file</code>:`,ie,J,pe,N,Ve='Este objeto ya puede ser utilizado con todos los métodos compartidos por los tokenizadores de 🤗 Transformers! Visita la <a href="main_classes/tokenizer">página sobre tokenizadores</a> para más información.',de,L,ce;return M=new Me({}),z=new fe({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`}}),T=new Me({}),w=new fe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`}}),_=new Me({}),Z=new fe({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),J=new fe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`}}),{c(){m=n("meta"),R=o(),W=n("p"),H=o(),h=n("h1"),k=n("a"),X=n("span"),x(M.$$.fragment),ke=o(),A=n("span"),A.textContent=ve,Y=o(),j=n("p"),j.innerHTML=ze,K=o(),v=n("p"),v.textContent=be,O=o(),x(z.$$.fragment),D=o(),b=n("p"),b.textContent=Te,ee=o(),u=n("h2"),g=n("a"),q=n("span"),x(T.$$.fragment),ge=o(),Q=n("span"),Q.textContent=$e,te=o(),$=n("p"),$.innerHTML=we,se=o(),x(w.$$.fragment),ae=o(),U=n("p"),U.innerHTML=Ue,ne=o(),f=n("h2"),y=n("a"),G=n("span"),x(_.$$.fragment),ye=o(),S=n("span"),S.textContent=_e,le=o(),C=n("p"),C.textContent=Ce,oe=o(),x(Z.$$.fragment),re=o(),V=n("p"),V.innerHTML=Ze,ie=o(),x(J.$$.fragment),pe=o(),N=n("p"),N.innerHTML=Ve,de=o(),L=n("p"),this.h()},l(e){const s=We("svelte-1phssyn",document.head);m=l(s,"META",{name:!0,content:!0}),s.forEach(t),R=r(e),W=l(e,"P",{}),d(W).forEach(t),H=r(e),h=l(e,"H1",{class:!0});var me=d(h);k=l(me,"A",{id:!0,class:!0,href:!0});var Je=d(k);X=l(Je,"SPAN",{});var Ne=d(X);B(M.$$.fragment,Ne),Ne.forEach(t),Je.forEach(t),ke=r(me),A=l(me,"SPAN",{"data-svelte-h":!0}),c(A)!=="svelte-gy9qi2"&&(A.textContent=ve),me.forEach(t),Y=r(e),j=l(e,"P",{"data-svelte-h":!0}),c(j)!=="svelte-1jbck1w"&&(j.innerHTML=ze),K=r(e),v=l(e,"P",{"data-svelte-h":!0}),c(v)!=="svelte-161ig80"&&(v.textContent=be),O=r(e),B(z.$$.fragment,e),D=r(e),b=l(e,"P",{"data-svelte-h":!0}),c(b)!=="svelte-1d3xwh8"&&(b.textContent=Te),ee=r(e),u=l(e,"H2",{class:!0});var he=d(u);g=l(he,"A",{id:!0,class:!0,href:!0});var xe=d(g);q=l(xe,"SPAN",{});var Be=d(q);B(T.$$.fragment,Be),Be.forEach(t),xe.forEach(t),ge=r(he),Q=l(he,"SPAN",{"data-svelte-h":!0}),c(Q)!=="svelte-12x0lh2"&&(Q.textContent=$e),he.forEach(t),te=r(e),$=l(e,"P",{"data-svelte-h":!0}),c($)!=="svelte-v80ac3"&&($.innerHTML=we),se=r(e),B(w.$$.fragment,e),ae=r(e),U=l(e,"P",{"data-svelte-h":!0}),c(U)!=="svelte-o2trt2"&&(U.innerHTML=Ue),ne=r(e),f=l(e,"H2",{class:!0});var ue=d(f);y=l(ue,"A",{id:!0,class:!0,href:!0});var Ee=d(y);G=l(Ee,"SPAN",{});var Pe=d(G);B(_.$$.fragment,Pe),Pe.forEach(t),Ee.forEach(t),ye=r(ue),S=l(ue,"SPAN",{"data-svelte-h":!0}),c(S)!=="svelte-hcgs0q"&&(S.textContent=_e),ue.forEach(t),le=r(e),C=l(e,"P",{"data-svelte-h":!0}),c(C)!=="svelte-19no5wd"&&(C.textContent=Ce),oe=r(e),B(Z.$$.fragment,e),re=r(e),V=l(e,"P",{"data-svelte-h":!0}),c(V)!=="svelte-9rgu8a"&&(V.innerHTML=Ze),ie=r(e),B(J.$$.fragment,e),pe=r(e),N=l(e,"P",{"data-svelte-h":!0}),c(N)!=="svelte-o2trt2"&&(N.innerHTML=Ve),de=r(e),L=l(e,"P",{}),d(L).forEach(t),this.h()},h(){i(m,"name","hf:doc:metadata"),i(m,"content",JSON.stringify(qe)),i(k,"id","usa-los-tokenizadores-de-tokenizers"),i(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(k,"href","#usa-los-tokenizadores-de-tokenizers"),i(h,"class","relative group"),i(g,"id","cargando-directamente-desde-el-objeto-tokenizador"),i(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(g,"href","#cargando-directamente-desde-el-objeto-tokenizador"),i(u,"class","relative group"),i(y,"id","cargando-desde-un-archivo-json"),i(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(y,"href","#cargando-desde-un-archivo-json"),i(f,"class","relative group")},m(e,s){p(document.head,m),a(e,R,s),a(e,W,s),a(e,H,s),a(e,h,s),p(h,k),p(k,X),E(M,X,null),p(h,ke),p(h,A),a(e,Y,s),a(e,j,s),a(e,K,s),a(e,v,s),a(e,O,s),E(z,e,s),a(e,D,s),a(e,b,s),a(e,ee,s),a(e,u,s),p(u,g),p(g,q),E(T,q,null),p(u,ge),p(u,Q),a(e,te,s),a(e,$,s),a(e,se,s),E(w,e,s),a(e,ae,s),a(e,U,s),a(e,ne,s),a(e,f,s),p(f,y),p(y,G),E(_,G,null),p(f,ye),p(f,S),a(e,le,s),a(e,C,s),a(e,oe,s),E(Z,e,s),a(e,re,s),a(e,V,s),a(e,ie,s),E(J,e,s),a(e,pe,s),a(e,N,s),a(e,de,s),a(e,L,s),ce=!0},p:Fe,i(e){ce||(P(M.$$.fragment,e),P(z.$$.fragment,e),P(T.$$.fragment,e),P(w.$$.fragment,e),P(_.$$.fragment,e),P(Z.$$.fragment,e),P(J.$$.fragment,e),ce=!0)},o(e){I(M.$$.fragment,e),I(z.$$.fragment,e),I(T.$$.fragment,e),I(w.$$.fragment,e),I(_.$$.fragment,e),I(Z.$$.fragment,e),I(J.$$.fragment,e),ce=!1},d(e){e&&(t(R),t(W),t(H),t(h),t(Y),t(j),t(K),t(v),t(O),t(D),t(b),t(ee),t(u),t(te),t($),t(se),t(ae),t(U),t(ne),t(f),t(le),t(C),t(oe),t(re),t(V),t(ie),t(pe),t(N),t(de),t(L)),t(m),F(M),F(z,e),F(T),F(w,e),F(_),F(Z,e),F(J,e)}}}const qe={local:"usa-los-tokenizadores-de-tokenizers",sections:[{local:"cargando-directamente-desde-el-objeto-tokenizador",title:"Cargando directamente desde el objeto tokenizador "},{local:"cargando-desde-un-archivo-json",title:"Cargando desde un archivo JSON"}],title:"Usa los tokenizadores de 🤗 Tokenizers"};function Ge(je){return Ae(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ke extends Qe{constructor(m){super(),Se(this,m,Ge,Xe,Ie,{})}}export{Ke as component};
