import{s as xe,n as Ee,o as We}from"../chunks/scheduler.9bc65507.js";import{S as Qe,i as Se,g as a,s as r,u as B,A as Ae,h as l,f as t,c as o,j as m,v as N,y as h,k as i,o as p,a as n,w as F,d as I,t as x,x as E}from"../chunks/index.78c82d43.js";import{I as Me}from"../chunks/IconCopyLink.bedaa44d.js";import{C as ue}from"../chunks/CodeBlock.48618ec2.js";function Xe(je){let c,R,A,q,f,k,X,M,ke,W,ze="Use tokenizers from 🤗 Tokenizers",Y,j,ve=`The <a href="/docs/transformers/v4.33.3/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> depends on the <a href="https://huggingface.co/docs/tokenizers" rel="nofollow">🤗 Tokenizers</a> library. The tokenizers obtained from the 🤗 Tokenizers library can be
loaded very simply into 🤗 Transformers.`,K,z,be="Before getting in the specifics, let’s first start by creating a dummy tokenizer in a few lines:",O,v,D,b,Te=`We now have a tokenizer trained on the files we defined. We can either continue using it in that runtime, or save it to
a JSON file for future re-use.`,ee,d,y,G,T,ye,Q,we="Loading directly from the tokenizer object",te,w,$e=`Let’s see how to leverage this tokenizer object in the 🤗 Transformers library. The
<a href="/docs/transformers/v4.33.3/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> class allows for easy instantiation, by accepting the instantiated
<em>tokenizer</em> object as an argument:`,se,$,ne,U,Ue=`This object can now be used with all the methods shared by the 🤗 Transformers tokenizers! Head to <a href="main_classes/tokenizer">the tokenizer
page</a> for more information.`,ae,u,g,H,_,ge,S,_e="Loading from a JSON file",le,Z,Ze="In order to load a tokenizer from a JSON file, let’s first start by saving our tokenizer:",re,C,oe,J,Ce=`The path to which we saved this file can be passed to the <a href="/docs/transformers/v4.33.3/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> initialization
method using the <code>tokenizer_file</code> parameter:`,ie,V,pe,P,Je=`This object can now be used with all the methods shared by the 🤗 Transformers tokenizers! Head to <a href="main_classes/tokenizer">the tokenizer
page</a> for more information.`,me,L,he;return M=new Me({}),v=new ue({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`}}),T=new Me({}),$=new ue({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`}}),_=new Me({}),C=new ue({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),V=new ue({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`}}),{c(){c=a("meta"),R=r(),A=a("p"),q=r(),f=a("h1"),k=a("a"),X=a("span"),B(M.$$.fragment),ke=r(),W=a("span"),W.textContent=ze,Y=r(),j=a("p"),j.innerHTML=ve,K=r(),z=a("p"),z.textContent=be,O=r(),B(v.$$.fragment),D=r(),b=a("p"),b.textContent=Te,ee=r(),d=a("h2"),y=a("a"),G=a("span"),B(T.$$.fragment),ye=r(),Q=a("span"),Q.textContent=we,te=r(),w=a("p"),w.innerHTML=$e,se=r(),B($.$$.fragment),ne=r(),U=a("p"),U.innerHTML=Ue,ae=r(),u=a("h2"),g=a("a"),H=a("span"),B(_.$$.fragment),ge=r(),S=a("span"),S.textContent=_e,le=r(),Z=a("p"),Z.textContent=Ze,re=r(),B(C.$$.fragment),oe=r(),J=a("p"),J.innerHTML=Ce,ie=r(),B(V.$$.fragment),pe=r(),P=a("p"),P.innerHTML=Je,me=r(),L=a("p"),this.h()},l(e){const s=Ae("svelte-1phssyn",document.head);c=l(s,"META",{name:!0,content:!0}),s.forEach(t),R=o(e),A=l(e,"P",{}),m(A).forEach(t),q=o(e),f=l(e,"H1",{class:!0});var ce=m(f);k=l(ce,"A",{id:!0,class:!0,href:!0});var Ve=m(k);X=l(Ve,"SPAN",{});var Pe=m(X);N(M.$$.fragment,Pe),Pe.forEach(t),Ve.forEach(t),ke=o(ce),W=l(ce,"SPAN",{"data-svelte-h":!0}),h(W)!=="svelte-upgw2t"&&(W.textContent=ze),ce.forEach(t),Y=o(e),j=l(e,"P",{"data-svelte-h":!0}),h(j)!=="svelte-1yzmj32"&&(j.innerHTML=ve),K=o(e),z=l(e,"P",{"data-svelte-h":!0}),h(z)!=="svelte-1fy1h9g"&&(z.textContent=be),O=o(e),N(v.$$.fragment,e),D=o(e),b=l(e,"P",{"data-svelte-h":!0}),h(b)!=="svelte-rt9ca6"&&(b.textContent=Te),ee=o(e),d=l(e,"H2",{class:!0});var fe=m(d);y=l(fe,"A",{id:!0,class:!0,href:!0});var Be=m(y);G=l(Be,"SPAN",{});var Ne=m(G);N(T.$$.fragment,Ne),Ne.forEach(t),Be.forEach(t),ye=o(fe),Q=l(fe,"SPAN",{"data-svelte-h":!0}),h(Q)!=="svelte-17nzdd4"&&(Q.textContent=we),fe.forEach(t),te=o(e),w=l(e,"P",{"data-svelte-h":!0}),h(w)!=="svelte-1bo16p4"&&(w.innerHTML=$e),se=o(e),N($.$$.fragment,e),ne=o(e),U=l(e,"P",{"data-svelte-h":!0}),h(U)!=="svelte-mowuiv"&&(U.innerHTML=Ue),ae=o(e),u=l(e,"H2",{class:!0});var de=m(u);g=l(de,"A",{id:!0,class:!0,href:!0});var Fe=m(g);H=l(Fe,"SPAN",{});var Ie=m(H);N(_.$$.fragment,Ie),Ie.forEach(t),Fe.forEach(t),ge=o(de),S=l(de,"SPAN",{"data-svelte-h":!0}),h(S)!=="svelte-6gjiq0"&&(S.textContent=_e),de.forEach(t),le=o(e),Z=l(e,"P",{"data-svelte-h":!0}),h(Z)!=="svelte-kuw3zs"&&(Z.textContent=Ze),re=o(e),N(C.$$.fragment,e),oe=o(e),J=l(e,"P",{"data-svelte-h":!0}),h(J)!=="svelte-nnazr6"&&(J.innerHTML=Ce),ie=o(e),N(V.$$.fragment,e),pe=o(e),P=l(e,"P",{"data-svelte-h":!0}),h(P)!=="svelte-mowuiv"&&(P.innerHTML=Je),me=o(e),L=l(e,"P",{}),m(L).forEach(t),this.h()},h(){i(c,"name","hf:doc:metadata"),i(c,"content",JSON.stringify(Ge)),i(k,"id","use-tokenizers-from-tokenizers"),i(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(k,"href","#use-tokenizers-from-tokenizers"),i(f,"class","relative group"),i(y,"id","loading-directly-from-the-tokenizer-object"),i(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(y,"href","#loading-directly-from-the-tokenizer-object"),i(d,"class","relative group"),i(g,"id","loading-from-a-json-file"),i(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(g,"href","#loading-from-a-json-file"),i(u,"class","relative group")},m(e,s){p(document.head,c),n(e,R,s),n(e,A,s),n(e,q,s),n(e,f,s),p(f,k),p(k,X),F(M,X,null),p(f,ke),p(f,W),n(e,Y,s),n(e,j,s),n(e,K,s),n(e,z,s),n(e,O,s),F(v,e,s),n(e,D,s),n(e,b,s),n(e,ee,s),n(e,d,s),p(d,y),p(y,G),F(T,G,null),p(d,ye),p(d,Q),n(e,te,s),n(e,w,s),n(e,se,s),F($,e,s),n(e,ne,s),n(e,U,s),n(e,ae,s),n(e,u,s),p(u,g),p(g,H),F(_,H,null),p(u,ge),p(u,S),n(e,le,s),n(e,Z,s),n(e,re,s),F(C,e,s),n(e,oe,s),n(e,J,s),n(e,ie,s),F(V,e,s),n(e,pe,s),n(e,P,s),n(e,me,s),n(e,L,s),he=!0},p:Ee,i(e){he||(I(M.$$.fragment,e),I(v.$$.fragment,e),I(T.$$.fragment,e),I($.$$.fragment,e),I(_.$$.fragment,e),I(C.$$.fragment,e),I(V.$$.fragment,e),he=!0)},o(e){x(M.$$.fragment,e),x(v.$$.fragment,e),x(T.$$.fragment,e),x($.$$.fragment,e),x(_.$$.fragment,e),x(C.$$.fragment,e),x(V.$$.fragment,e),he=!1},d(e){e&&(t(R),t(A),t(q),t(f),t(Y),t(j),t(K),t(z),t(O),t(D),t(b),t(ee),t(d),t(te),t(w),t(se),t(ne),t(U),t(ae),t(u),t(le),t(Z),t(re),t(oe),t(J),t(ie),t(pe),t(P),t(me),t(L)),t(c),E(M),E(v,e),E(T),E($,e),E(_),E(C,e),E(V,e)}}}const Ge={local:"use-tokenizers-from-tokenizers",sections:[{local:"loading-directly-from-the-tokenizer-object",title:"Loading directly from the tokenizer object"},{local:"loading-from-a-json-file",title:"Loading from a JSON file"}],title:"Use tokenizers from 🤗 Tokenizers"};function He(je){return We(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ke extends Qe{constructor(c){super(),Se(this,c,He,Xe,xe,{})}}export{Ke as component};
