import{s as Ie,n as Fe,o as Se}from"../chunks/scheduler.889ebb13.js";import{S as Qe,i as We,g as n,s as r,u as x,A as Ae,h as a,f as t,c as o,j as m,v as B,y as c,k as i,o as p,a as l,w as P,d as E,t as I,x as F}from"../chunks/index.8980e686.js";import{I as ge}from"../chunks/IconCopyLink.357761cd.js";import{C as ue}from"../chunks/CodeBlock.25fc3430.js";function Xe(je){let h,q,A,L,f,k,X,g,ke,S,ve="🤗 Tokenizers 라이브러리의 토크나이저 사용하기",O,j,Te=`<code>PreTrainedTokenizerFast</code>는 <a href="https://huggingface.co/docs/tokenizers" rel="nofollow">🤗 Tokenizers</a> 라이브러리에 기반합니다. 🤗 Tokenizers 라이브러리의 토크나이저는
🤗 Transformers로 매우 간단하게 불러올 수 있습니다.`,Y,v,$e="구체적인 내용에 들어가기 전에, 몇 줄의 코드로 더미 토크나이저를 만들어 보겠습니다:",K,T,D,$,be="우리가 정의한 파일을 통해 이제 학습된 토크나이저를 갖게 되었습니다. 이 런타임에서 계속 사용하거나 JSON 파일로 저장하여 나중에 사용할 수 있습니다.",ee,d,y,G,b,ye,Q,ze="토크나이저 객체로부터 직접 불러오기",te,z,we=`🤗 Transformers 라이브러리에서 이 토크나이저 객체를 활용하는 방법을 살펴보겠습니다.
<code>PreTrainedTokenizerFast</code> 클래스는 인스턴스화된 <em>토크나이저</em> 객체를 인수로 받아 쉽게 인스턴스화할 수 있습니다:`,se,w,le,U,Ue='이제 <code>fast_tokenizer</code> 객체는 🤗 Transformers 토크나이저에서 공유하는 모든 메소드와 함께 사용할 수 있습니다! 자세한 내용은 <a href="main_classes/tokenizer">토크나이저 페이지</a>를 참조하세요.',ne,u,M,R,_,Me,W,_e="JSON 파일에서 불러오기",ae,J,Je="JSON 파일에서 토크나이저를 불러오기 위해, 먼저 토크나이저를 저장해 보겠습니다:",re,Z,oe,C,Ze="JSON 파일을 저장한 경로는 <code>tokenizer_file</code> 매개변수를 사용하여 <code>PreTrainedTokenizerFast</code> 초기화 메소드에 전달할 수 있습니다:",ie,V,pe,N,Ce='이제 <code>fast_tokenizer</code> 객체는 🤗 Transformers 토크나이저에서 공유하는 모든 메소드와 함께 사용할 수 있습니다! 자세한 내용은 <a href="main_classes/tokenizer">토크나이저 페이지</a>를 참조하세요.',me,H,ce;return g=new ge({}),T=new ue({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`}}),b=new ge({}),w=new ue({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`}}),_=new ge({}),Z=new ue({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),V=new ue({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`}}),{c(){h=n("meta"),q=r(),A=n("p"),L=r(),f=n("h1"),k=n("a"),X=n("span"),x(g.$$.fragment),ke=r(),S=n("span"),S.textContent=ve,O=r(),j=n("p"),j.innerHTML=Te,Y=r(),v=n("p"),v.textContent=$e,K=r(),x(T.$$.fragment),D=r(),$=n("p"),$.textContent=be,ee=r(),d=n("h2"),y=n("a"),G=n("span"),x(b.$$.fragment),ye=r(),Q=n("span"),Q.textContent=ze,te=r(),z=n("p"),z.innerHTML=we,se=r(),x(w.$$.fragment),le=r(),U=n("p"),U.innerHTML=Ue,ne=r(),u=n("h2"),M=n("a"),R=n("span"),x(_.$$.fragment),Me=r(),W=n("span"),W.textContent=_e,ae=r(),J=n("p"),J.textContent=Je,re=r(),x(Z.$$.fragment),oe=r(),C=n("p"),C.innerHTML=Ze,ie=r(),x(V.$$.fragment),pe=r(),N=n("p"),N.innerHTML=Ce,me=r(),H=n("p"),this.h()},l(e){const s=Ae("svelte-1phssyn",document.head);h=a(s,"META",{name:!0,content:!0}),s.forEach(t),q=o(e),A=a(e,"P",{}),m(A).forEach(t),L=o(e),f=a(e,"H1",{class:!0});var he=m(f);k=a(he,"A",{id:!0,class:!0,href:!0});var Ve=m(k);X=a(Ve,"SPAN",{});var Ne=m(X);B(g.$$.fragment,Ne),Ne.forEach(t),Ve.forEach(t),ke=o(he),S=a(he,"SPAN",{"data-svelte-h":!0}),c(S)!=="svelte-1lpeglb"&&(S.textContent=ve),he.forEach(t),O=o(e),j=a(e,"P",{"data-svelte-h":!0}),c(j)!=="svelte-1ilwdv2"&&(j.innerHTML=Te),Y=o(e),v=a(e,"P",{"data-svelte-h":!0}),c(v)!=="svelte-nobjxu"&&(v.textContent=$e),K=o(e),B(T.$$.fragment,e),D=o(e),$=a(e,"P",{"data-svelte-h":!0}),c($)!=="svelte-vmdasx"&&($.textContent=be),ee=o(e),d=a(e,"H2",{class:!0});var fe=m(d);y=a(fe,"A",{id:!0,class:!0,href:!0});var xe=m(y);G=a(xe,"SPAN",{});var Be=m(G);B(b.$$.fragment,Be),Be.forEach(t),xe.forEach(t),ye=o(fe),Q=a(fe,"SPAN",{"data-svelte-h":!0}),c(Q)!=="svelte-1d9ceki"&&(Q.textContent=ze),fe.forEach(t),te=o(e),z=a(e,"P",{"data-svelte-h":!0}),c(z)!=="svelte-10u78cn"&&(z.innerHTML=we),se=o(e),B(w.$$.fragment,e),le=o(e),U=a(e,"P",{"data-svelte-h":!0}),c(U)!=="svelte-tdf1x7"&&(U.innerHTML=Ue),ne=o(e),u=a(e,"H2",{class:!0});var de=m(u);M=a(de,"A",{id:!0,class:!0,href:!0});var Pe=m(M);R=a(Pe,"SPAN",{});var Ee=m(R);B(_.$$.fragment,Ee),Ee.forEach(t),Pe.forEach(t),Me=o(de),W=a(de,"SPAN",{"data-svelte-h":!0}),c(W)!=="svelte-s9ktfj"&&(W.textContent=_e),de.forEach(t),ae=o(e),J=a(e,"P",{"data-svelte-h":!0}),c(J)!=="svelte-16yucd6"&&(J.textContent=Je),re=o(e),B(Z.$$.fragment,e),oe=o(e),C=a(e,"P",{"data-svelte-h":!0}),c(C)!=="svelte-126md19"&&(C.innerHTML=Ze),ie=o(e),B(V.$$.fragment,e),pe=o(e),N=a(e,"P",{"data-svelte-h":!0}),c(N)!=="svelte-tdf1x7"&&(N.innerHTML=Ce),me=o(e),H=a(e,"P",{}),m(H).forEach(t),this.h()},h(){i(h,"name","hf:doc:metadata"),i(h,"content",JSON.stringify(Ge)),i(k,"id","use-tokenizers-from-tokenizers"),i(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(k,"href","#use-tokenizers-from-tokenizers"),i(f,"class","relative group"),i(y,"id","loading-directly-from-the-tokenizer-object"),i(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(y,"href","#loading-directly-from-the-tokenizer-object"),i(d,"class","relative group"),i(M,"id","loading-from-a-JSON-file"),i(M,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(M,"href","#loading-from-a-JSON-file"),i(u,"class","relative group")},m(e,s){p(document.head,h),l(e,q,s),l(e,A,s),l(e,L,s),l(e,f,s),p(f,k),p(k,X),P(g,X,null),p(f,ke),p(f,S),l(e,O,s),l(e,j,s),l(e,Y,s),l(e,v,s),l(e,K,s),P(T,e,s),l(e,D,s),l(e,$,s),l(e,ee,s),l(e,d,s),p(d,y),p(y,G),P(b,G,null),p(d,ye),p(d,Q),l(e,te,s),l(e,z,s),l(e,se,s),P(w,e,s),l(e,le,s),l(e,U,s),l(e,ne,s),l(e,u,s),p(u,M),p(M,R),P(_,R,null),p(u,Me),p(u,W),l(e,ae,s),l(e,J,s),l(e,re,s),P(Z,e,s),l(e,oe,s),l(e,C,s),l(e,ie,s),P(V,e,s),l(e,pe,s),l(e,N,s),l(e,me,s),l(e,H,s),ce=!0},p:Fe,i(e){ce||(E(g.$$.fragment,e),E(T.$$.fragment,e),E(b.$$.fragment,e),E(w.$$.fragment,e),E(_.$$.fragment,e),E(Z.$$.fragment,e),E(V.$$.fragment,e),ce=!0)},o(e){I(g.$$.fragment,e),I(T.$$.fragment,e),I(b.$$.fragment,e),I(w.$$.fragment,e),I(_.$$.fragment,e),I(Z.$$.fragment,e),I(V.$$.fragment,e),ce=!1},d(e){e&&(t(q),t(A),t(L),t(f),t(O),t(j),t(Y),t(v),t(K),t(D),t($),t(ee),t(d),t(te),t(z),t(se),t(le),t(U),t(ne),t(u),t(ae),t(J),t(re),t(oe),t(C),t(ie),t(pe),t(N),t(me),t(H)),t(h),F(g),F(T,e),F(b),F(w,e),F(_),F(Z,e),F(V,e)}}}const Ge={local:"use-tokenizers-from-tokenizers",sections:[{local:"loading-directly-from-the-tokenizer-object",title:"토크나이저 객체로부터 직접 불러오기"},{local:"loading-from-a-JSON-file",title:"JSON 파일에서 불러오기"}],title:"🤗 Tokenizers 라이브러리의 토크나이저 사용하기"};function Re(je){return Se(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ye extends Qe{constructor(h){super(),We(this,h,Re,Xe,Ie,{})}}export{Ye as component};
