import{s as B,n as q,o as F}from"../chunks/scheduler.37c15a92.js";import{S as N,i as R,g as u,s as p,r as L,A as W,h as f,f as a,c,j as M,u as j,x as T,k as z,y as D,a as n,v as S,d as U,t as k,w as A}from"../chunks/index.2bf4358c.js";import{C as G}from"../chunks/CourseFloatingBanner.06521e8b.js";import{H as I}from"../chunks/Heading.8ada512a.js";function J(E){let s,h,_,d,l,x,i,v,o,b="完成这一章，辛苦了！",C,r,y="在深入研究标记器之后，您应该：",P,m,H="<li>能够使用旧的标记器作为模板来训练新的标记器</li> <li>了解如何使用偏移量将标记的位置映射到其原始文本范围</li> <li>了解 BPE、WordPiece 和 Unigram 之间的区别</li> <li>能够混合和匹配 🤗 Tokenizers 库提供的块来构建您自己的标记器</li> <li>能够在 🤗 Transformers 库中使用该标记器</li>",g,$,w;return l=new I({props:{title:"标记器，回顾！",local:"标记器，回顾！",headingTag:"h1"}}),i=new G({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),{c(){s=u("meta"),h=p(),_=u("p"),d=p(),L(l.$$.fragment),x=p(),L(i.$$.fragment),v=p(),o=u("p"),o.textContent=b,C=p(),r=u("p"),r.textContent=y,P=p(),m=u("ul"),m.innerHTML=H,g=p(),$=u("p"),this.h()},l(t){const e=W("svelte-u9bgzb",document.head);s=f(e,"META",{name:!0,content:!0}),e.forEach(a),h=c(t),_=f(t,"P",{}),M(_).forEach(a),d=c(t),j(l.$$.fragment,t),x=c(t),j(i.$$.fragment,t),v=c(t),o=f(t,"P",{"data-svelte-h":!0}),T(o)!=="svelte-cpcc87"&&(o.textContent=b),C=c(t),r=f(t,"P",{"data-svelte-h":!0}),T(r)!=="svelte-1ajcmaj"&&(r.textContent=y),P=c(t),m=f(t,"UL",{"data-svelte-h":!0}),T(m)!=="svelte-13nu9mu"&&(m.innerHTML=H),g=c(t),$=f(t,"P",{}),M($).forEach(a),this.h()},h(){z(s,"name","hf:doc:metadata"),z(s,"content",K)},m(t,e){D(document.head,s),n(t,h,e),n(t,_,e),n(t,d,e),S(l,t,e),n(t,x,e),S(i,t,e),n(t,v,e),n(t,o,e),n(t,C,e),n(t,r,e),n(t,P,e),n(t,m,e),n(t,g,e),n(t,$,e),w=!0},p:q,i(t){w||(U(l.$$.fragment,t),U(i.$$.fragment,t),w=!0)},o(t){k(l.$$.fragment,t),k(i.$$.fragment,t),w=!1},d(t){t&&(a(h),a(_),a(d),a(x),a(v),a(o),a(C),a(r),a(P),a(m),a(g),a($)),a(s),A(l,t),A(i,t)}}}const K='{"title":"标记器，回顾！","local":"标记器，回顾！","sections":[],"depth":1}';function O(E){return F(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Z extends N{constructor(s){super(),R(this,s,O,J,B,{})}}export{Z as component};
