import{s as A,n as S,o as F}from"../chunks/scheduler.37c15a92.js";import{S as G,i as K,g as f,s as c,r as B,A as N,h as u,f as n,c as p,j as E,u as H,x as g,k as L,y as R,a,v as M,d as U,t as j,w as q}from"../chunks/index.2bf4358c.js";import{C as W}from"../chunks/CourseFloatingBanner.15ba07e6.js";import{H as D}from"../chunks/Heading.8ada512a.js";function I(y){let i,_,h,$,s,k,o,b,l,C="Great job finishing this chapter!",x,r,P="After this deep dive into tokenizers, you should:",v,m,T="<li>Be able to train a new tokenizer using an old one as a template</li> <li>Understand how to use offsets to map tokens’ positions to their original span of text</li> <li>Know the differences between BPE, WordPiece, and Unigram</li> <li>Be able to mix and match the blocks provided by the 🤗 Tokenizers library to build your own tokenizer</li> <li>Be able to use that tokenizer inside the 🤗 Transformers library</li>",w,d,z;return s=new D({props:{title:"Tokenizers, check!",local:"tokenizers-check",headingTag:"h1"}}),o=new W({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),{c(){i=f("meta"),_=c(),h=f("p"),$=c(),B(s.$$.fragment),k=c(),B(o.$$.fragment),b=c(),l=f("p"),l.textContent=C,x=c(),r=f("p"),r.textContent=P,v=c(),m=f("ul"),m.innerHTML=T,w=c(),d=f("p"),this.h()},l(e){const t=N("svelte-u9bgzb",document.head);i=u(t,"META",{name:!0,content:!0}),t.forEach(n),_=p(e),h=u(e,"P",{}),E(h).forEach(n),$=p(e),H(s.$$.fragment,e),k=p(e),H(o.$$.fragment,e),b=p(e),l=u(e,"P",{"data-svelte-h":!0}),g(l)!=="svelte-qrdqcf"&&(l.textContent=C),x=p(e),r=u(e,"P",{"data-svelte-h":!0}),g(r)!=="svelte-ziaxv6"&&(r.textContent=P),v=p(e),m=u(e,"UL",{"data-svelte-h":!0}),g(m)!=="svelte-jl1wny"&&(m.innerHTML=T),w=p(e),d=u(e,"P",{}),E(d).forEach(n),this.h()},h(){L(i,"name","hf:doc:metadata"),L(i,"content",J)},m(e,t){R(document.head,i),a(e,_,t),a(e,h,t),a(e,$,t),M(s,e,t),a(e,k,t),M(o,e,t),a(e,b,t),a(e,l,t),a(e,x,t),a(e,r,t),a(e,v,t),a(e,m,t),a(e,w,t),a(e,d,t),z=!0},p:S,i(e){z||(U(s.$$.fragment,e),U(o.$$.fragment,e),z=!0)},o(e){j(s.$$.fragment,e),j(o.$$.fragment,e),z=!1},d(e){e&&(n(_),n(h),n($),n(k),n(b),n(l),n(x),n(r),n(v),n(m),n(w),n(d)),n(i),q(s,e),q(o,e)}}}const J='{"title":"Tokenizers, check!","local":"tokenizers-check","sections":[],"depth":1}';function O(y){return F(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Z extends G{constructor(i){super(),K(this,i,O,I,A,{})}}export{Z as component};
