import{s as oe,n as ce,o as fe}from"../chunks/scheduler.3e4cb221.js";import{S as Me,i as ye,g as r,s as n,r as U,A as je,h as p,f as s,c as a,j as me,u as Z,x as i,k as ie,y as ke,a as l,v as w,d as J,t as V,w as _}from"../chunks/index.bc693fcb.js";import{C as K}from"../chunks/CodeBlock.4b1f9331.js";import{H as O}from"../chunks/Heading.fa24041c.js";function ge(D){let m,C,B,F,o,I,c,ee='<code>PreTrainedTokenizerFast</code> 依赖于 <a href="https://huggingface.co/docs/tokenizers" rel="nofollow">🤗 Tokenizers</a> 库。从 🤗 Tokenizers 库获得的分词器可以被轻松地加载到 🤗 Transformers 中。',N,f,te="在了解具体内容之前，让我们先用几行代码创建一个虚拟的分词器：",P,M,Q,y,se="现在，我们拥有了一个针对我们定义的文件进行训练的分词器。我们可以在当前运行时中继续使用它，或者将其保存到一个 JSON 文件以供将来重复使用。",W,j,X,k,le="让我们看看如何利用 🤗 Transformers 库中的这个分词器对象。<code>PreTrainedTokenizerFast</code> 类允许通过接受已实例化的 <em>tokenizer</em> 对象作为参数，进行轻松实例化：",G,g,E,d,ne='现在可以使用这个对象，使用 🤗 Transformers 分词器共享的所有方法！前往<a href="main_classes/tokenizer">分词器页面</a>了解更多信息。',R,h,q,u,ae="为了从 JSON 文件中加载分词器，让我们先保存我们的分词器：",x,T,H,$,re="我们保存此文件的路径可以通过 <code>tokenizer_file</code> 参数传递给 <code>PreTrainedTokenizerFast</code> 初始化方法：",S,b,L,z,pe='现在可以使用这个对象，使用 🤗 Transformers 分词器共享的所有方法！前往<a href="main_classes/tokenizer">分词器页面</a>了解更多信息。',A,v,Y;return o=new O({props:{title:"使用 🤗 Tokenizers 中的分词器",local:"使用--tokenizers-中的分词器",headingTag:"h1"}}),M=new K({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`,wrap:!1}}),j=new O({props:{title:"直接从分词器对象加载",local:"直接从分词器对象加载",headingTag:"h2"}}),g=new K({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`,wrap:!1}}),h=new O({props:{title:"从 JSON 文件加载",local:"从-json-文件加载",headingTag:"h2"}}),T=new K({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)',wrap:!1}}),b=new K({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`,wrap:!1}}),{c(){m=r("meta"),C=n(),B=r("p"),F=n(),U(o.$$.fragment),I=n(),c=r("p"),c.innerHTML=ee,N=n(),f=r("p"),f.textContent=te,P=n(),U(M.$$.fragment),Q=n(),y=r("p"),y.textContent=se,W=n(),U(j.$$.fragment),X=n(),k=r("p"),k.innerHTML=le,G=n(),U(g.$$.fragment),E=n(),d=r("p"),d.innerHTML=ne,R=n(),U(h.$$.fragment),q=n(),u=r("p"),u.textContent=ae,x=n(),U(T.$$.fragment),H=n(),$=r("p"),$.innerHTML=re,S=n(),U(b.$$.fragment),L=n(),z=r("p"),z.innerHTML=pe,A=n(),v=r("p"),this.h()},l(e){const t=je("svelte-u9bgzb",document.head);m=p(t,"META",{name:!0,content:!0}),t.forEach(s),C=a(e),B=p(e,"P",{}),me(B).forEach(s),F=a(e),Z(o.$$.fragment,e),I=a(e),c=p(e,"P",{"data-svelte-h":!0}),i(c)!=="svelte-1n2yfjs"&&(c.innerHTML=ee),N=a(e),f=p(e,"P",{"data-svelte-h":!0}),i(f)!=="svelte-opvc5k"&&(f.textContent=te),P=a(e),Z(M.$$.fragment,e),Q=a(e),y=p(e,"P",{"data-svelte-h":!0}),i(y)!=="svelte-17x9tu2"&&(y.textContent=se),W=a(e),Z(j.$$.fragment,e),X=a(e),k=p(e,"P",{"data-svelte-h":!0}),i(k)!=="svelte-91ale0"&&(k.innerHTML=le),G=a(e),Z(g.$$.fragment,e),E=a(e),d=p(e,"P",{"data-svelte-h":!0}),i(d)!=="svelte-tb93lc"&&(d.innerHTML=ne),R=a(e),Z(h.$$.fragment,e),q=a(e),u=p(e,"P",{"data-svelte-h":!0}),i(u)!=="svelte-lr3arf"&&(u.textContent=ae),x=a(e),Z(T.$$.fragment,e),H=a(e),$=p(e,"P",{"data-svelte-h":!0}),i($)!=="svelte-18yuarv"&&($.innerHTML=re),S=a(e),Z(b.$$.fragment,e),L=a(e),z=p(e,"P",{"data-svelte-h":!0}),i(z)!=="svelte-tb93lc"&&(z.innerHTML=pe),A=a(e),v=p(e,"P",{}),me(v).forEach(s),this.h()},h(){ie(m,"name","hf:doc:metadata"),ie(m,"content",de)},m(e,t){ke(document.head,m),l(e,C,t),l(e,B,t),l(e,F,t),w(o,e,t),l(e,I,t),l(e,c,t),l(e,N,t),l(e,f,t),l(e,P,t),w(M,e,t),l(e,Q,t),l(e,y,t),l(e,W,t),w(j,e,t),l(e,X,t),l(e,k,t),l(e,G,t),w(g,e,t),l(e,E,t),l(e,d,t),l(e,R,t),w(h,e,t),l(e,q,t),l(e,u,t),l(e,x,t),w(T,e,t),l(e,H,t),l(e,$,t),l(e,S,t),w(b,e,t),l(e,L,t),l(e,z,t),l(e,A,t),l(e,v,t),Y=!0},p:ce,i(e){Y||(J(o.$$.fragment,e),J(M.$$.fragment,e),J(j.$$.fragment,e),J(g.$$.fragment,e),J(h.$$.fragment,e),J(T.$$.fragment,e),J(b.$$.fragment,e),Y=!0)},o(e){V(o.$$.fragment,e),V(M.$$.fragment,e),V(j.$$.fragment,e),V(g.$$.fragment,e),V(h.$$.fragment,e),V(T.$$.fragment,e),V(b.$$.fragment,e),Y=!1},d(e){e&&(s(C),s(B),s(F),s(I),s(c),s(N),s(f),s(P),s(Q),s(y),s(W),s(X),s(k),s(G),s(E),s(d),s(R),s(q),s(u),s(x),s(H),s($),s(S),s(L),s(z),s(A),s(v)),s(m),_(o,e),_(M,e),_(j,e),_(g,e),_(h,e),_(T,e),_(b,e)}}}const de='{"title":"使用 🤗 Tokenizers 中的分词器","local":"使用--tokenizers-中的分词器","sections":[{"title":"直接从分词器对象加载","local":"直接从分词器对象加载","sections":[],"depth":2},{"title":"从 JSON 文件加载","local":"从-json-文件加载","sections":[],"depth":2}],"depth":1}';function he(D){return fe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ze extends Me{constructor(m){super(),ye(this,m,he,ge,oe,{})}}export{ze as component};
