import{s as pe,n as fe,o as ce}from"../chunks/scheduler.9bc65507.js";import{S as he,i as de,g as r,s as a,r as w,A as ke,h as i,f as s,c as l,j as oe,u as U,x as m,k as me,y as ye,a as n,v as Z,d as J,t as V,w as _}from"../chunks/index.707bf1b6.js";import{C as K}from"../chunks/CodeBlock.54a9f38d.js";import{H as O}from"../chunks/Heading.342b1fa6.js";function ge(D){let o,F,v,C,p,I,f,ee=`The <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> depends on the <a href="https://huggingface.co/docs/tokenizers" rel="nofollow">🤗 Tokenizers</a> library. The tokenizers obtained from the 🤗 Tokenizers library can be
loaded very simply into 🤗 Transformers.`,P,c,te="Before getting in the specifics, let’s first start by creating a dummy tokenizer in a few lines:",W,h,N,d,se=`We now have a tokenizer trained on the files we defined. We can either continue using it in that runtime, or save it to
a JSON file for future re-use.`,Q,k,X,y,ne=`Let’s see how to leverage this tokenizer object in the 🤗 Transformers library. The
<a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> class allows for easy instantiation, by accepting the instantiated
<em>tokenizer</em> object as an argument:`,G,g,E,M,ae=`This object can now be used with all the methods shared by the 🤗 Transformers tokenizers! Head to <a href="main_classes/tokenizer">the tokenizer
page</a> for more information.`,H,j,L,u,le="In order to load a tokenizer from a JSON file, let’s first start by saving our tokenizer:",R,z,q,T,re=`The path to which we saved this file can be passed to the <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> initialization
method using the <code>tokenizer_file</code> parameter:`,x,b,S,$,ie=`This object can now be used with all the methods shared by the 🤗 Transformers tokenizers! Head to <a href="main_classes/tokenizer">the tokenizer
page</a> for more information.`,A,B,Y;return p=new O({props:{title:"Use tokenizers from 🤗 Tokenizers",local:"use-tokenizers-from--tokenizers",headingTag:"h1"}}),h=new K({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`,wrap:!1}}),k=new O({props:{title:"Loading directly from the tokenizer object",local:"loading-directly-from-the-tokenizer-object",headingTag:"h2"}}),g=new K({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`,wrap:!1}}),j=new O({props:{title:"Loading from a JSON file",local:"loading-from-a-json-file",headingTag:"h2"}}),z=new K({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)',wrap:!1}}),b=new K({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`,wrap:!1}}),{c(){o=r("meta"),F=a(),v=r("p"),C=a(),w(p.$$.fragment),I=a(),f=r("p"),f.innerHTML=ee,P=a(),c=r("p"),c.textContent=te,W=a(),w(h.$$.fragment),N=a(),d=r("p"),d.textContent=se,Q=a(),w(k.$$.fragment),X=a(),y=r("p"),y.innerHTML=ne,G=a(),w(g.$$.fragment),E=a(),M=r("p"),M.innerHTML=ae,H=a(),w(j.$$.fragment),L=a(),u=r("p"),u.textContent=le,R=a(),w(z.$$.fragment),q=a(),T=r("p"),T.innerHTML=re,x=a(),w(b.$$.fragment),S=a(),$=r("p"),$.innerHTML=ie,A=a(),B=r("p"),this.h()},l(e){const t=ke("svelte-u9bgzb",document.head);o=i(t,"META",{name:!0,content:!0}),t.forEach(s),F=l(e),v=i(e,"P",{}),oe(v).forEach(s),C=l(e),U(p.$$.fragment,e),I=l(e),f=i(e,"P",{"data-svelte-h":!0}),m(f)!=="svelte-l7e5e"&&(f.innerHTML=ee),P=l(e),c=i(e,"P",{"data-svelte-h":!0}),m(c)!=="svelte-1fy1h9g"&&(c.textContent=te),W=l(e),U(h.$$.fragment,e),N=l(e),d=i(e,"P",{"data-svelte-h":!0}),m(d)!=="svelte-rt9ca6"&&(d.textContent=se),Q=l(e),U(k.$$.fragment,e),X=l(e),y=i(e,"P",{"data-svelte-h":!0}),m(y)!=="svelte-tcba5g"&&(y.innerHTML=ne),G=l(e),U(g.$$.fragment,e),E=l(e),M=i(e,"P",{"data-svelte-h":!0}),m(M)!=="svelte-mowuiv"&&(M.innerHTML=ae),H=l(e),U(j.$$.fragment,e),L=l(e),u=i(e,"P",{"data-svelte-h":!0}),m(u)!=="svelte-kuw3zs"&&(u.textContent=le),R=l(e),U(z.$$.fragment,e),q=l(e),T=i(e,"P",{"data-svelte-h":!0}),m(T)!=="svelte-nfh1hs"&&(T.innerHTML=re),x=l(e),U(b.$$.fragment,e),S=l(e),$=i(e,"P",{"data-svelte-h":!0}),m($)!=="svelte-mowuiv"&&($.innerHTML=ie),A=l(e),B=i(e,"P",{}),oe(B).forEach(s),this.h()},h(){me(o,"name","hf:doc:metadata"),me(o,"content",Me)},m(e,t){ye(document.head,o),n(e,F,t),n(e,v,t),n(e,C,t),Z(p,e,t),n(e,I,t),n(e,f,t),n(e,P,t),n(e,c,t),n(e,W,t),Z(h,e,t),n(e,N,t),n(e,d,t),n(e,Q,t),Z(k,e,t),n(e,X,t),n(e,y,t),n(e,G,t),Z(g,e,t),n(e,E,t),n(e,M,t),n(e,H,t),Z(j,e,t),n(e,L,t),n(e,u,t),n(e,R,t),Z(z,e,t),n(e,q,t),n(e,T,t),n(e,x,t),Z(b,e,t),n(e,S,t),n(e,$,t),n(e,A,t),n(e,B,t),Y=!0},p:fe,i(e){Y||(J(p.$$.fragment,e),J(h.$$.fragment,e),J(k.$$.fragment,e),J(g.$$.fragment,e),J(j.$$.fragment,e),J(z.$$.fragment,e),J(b.$$.fragment,e),Y=!0)},o(e){V(p.$$.fragment,e),V(h.$$.fragment,e),V(k.$$.fragment,e),V(g.$$.fragment,e),V(j.$$.fragment,e),V(z.$$.fragment,e),V(b.$$.fragment,e),Y=!1},d(e){e&&(s(F),s(v),s(C),s(I),s(f),s(P),s(c),s(W),s(N),s(d),s(Q),s(X),s(y),s(G),s(E),s(M),s(H),s(L),s(u),s(R),s(q),s(T),s(x),s(S),s($),s(A),s(B)),s(o),_(p,e),_(h,e),_(k,e),_(g,e),_(j,e),_(z,e),_(b,e)}}}const Me='{"title":"Use tokenizers from 🤗 Tokenizers","local":"use-tokenizers-from--tokenizers","sections":[{"title":"Loading directly from the tokenizer object","local":"loading-directly-from-the-tokenizer-object","sections":[],"depth":2},{"title":"Loading from a JSON file","local":"loading-from-a-json-file","sections":[],"depth":2}],"depth":1}';function je(D){return ce(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $e extends he{constructor(o){super(),de(this,o,je,ge,pe,{})}}export{$e as component};
