import{s as G,n as O,o as R}from"../chunks/scheduler.cc52f4b9.js";import{S as W,i as D,e as c,s as l,c as E,h as I,a as h,d as n,b as s,f as A,g as T,j as U,k as F,l as J,m as a,n as L,t as M,o as H,p as S}from"../chunks/index.9480bb2b.js";import{C as K,H as Q,E as V}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.009dc98d.js";import{C as X}from"../chunks/CourseFloatingBanner.6488bde3.js";function Y(j){let i,x,_,z,r,d,o,k,m,C,p,q="恭喜你完成了这一章！",b,f,B="在深入研究 tokenizer 之后，你应该：",v,u,N="<li>能够使用旧的 tokenizer 作为模板来训练新的 tokenizer</li> <li>了解如何使用偏移量将 tokens 的位置映射到其原始文本范围</li> <li>了解 BPE、WordPiece 和 Unigram 之间的区别</li> <li>能够混合使用 🤗 Tokenizers 库提供的块来构建你自己的 tokenizer</li> <li>能够在 🤗 Transformers 库中使用该 tokenizer</li>",P,$,w,g,y;return r=new K({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),o=new Q({props:{title:"tokenizer 回顾！",local:"tokenizer ，回顾！",headingTag:"h1"}}),m=new X({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),$=new V({props:{source:"https://github.com/huggingface/course/blob/main/chapters/zh-CN/chapter6/9.mdx"}}),{c(){i=c("meta"),x=l(),_=c("p"),z=l(),E(r.$$.fragment),d=l(),E(o.$$.fragment),k=l(),E(m.$$.fragment),C=l(),p=c("p"),p.textContent=q,b=l(),f=c("p"),f.textContent=B,v=l(),u=c("ul"),u.innerHTML=N,P=l(),E($.$$.fragment),w=l(),g=c("p"),this.h()},l(t){const e=I("svelte-u9bgzb",document.head);i=h(e,"META",{name:!0,content:!0}),e.forEach(n),x=s(t),_=h(t,"P",{}),A(_).forEach(n),z=s(t),T(r.$$.fragment,t),d=s(t),T(o.$$.fragment,t),k=s(t),T(m.$$.fragment,t),C=s(t),p=h(t,"P",{"data-svelte-h":!0}),U(p)!=="svelte-wo5o3f"&&(p.textContent=q),b=s(t),f=h(t,"P",{"data-svelte-h":!0}),U(f)!=="svelte-ykyhjl"&&(f.textContent=B),v=s(t),u=h(t,"UL",{"data-svelte-h":!0}),U(u)!=="svelte-7z1lnq"&&(u.innerHTML=N),P=s(t),T($.$$.fragment,t),w=s(t),g=h(t,"P",{}),A(g).forEach(n),this.h()},h(){F(i,"name","hf:doc:metadata"),F(i,"content",Z)},m(t,e){J(document.head,i),a(t,x,e),a(t,_,e),a(t,z,e),L(r,t,e),a(t,d,e),L(o,t,e),a(t,k,e),L(m,t,e),a(t,C,e),a(t,p,e),a(t,b,e),a(t,f,e),a(t,v,e),a(t,u,e),a(t,P,e),L($,t,e),a(t,w,e),a(t,g,e),y=!0},p:O,i(t){y||(M(r.$$.fragment,t),M(o.$$.fragment,t),M(m.$$.fragment,t),M($.$$.fragment,t),y=!0)},o(t){H(r.$$.fragment,t),H(o.$$.fragment,t),H(m.$$.fragment,t),H($.$$.fragment,t),y=!1},d(t){t&&(n(x),n(_),n(z),n(d),n(k),n(C),n(p),n(b),n(f),n(v),n(u),n(P),n(w),n(g)),n(i),S(r,t),S(o,t),S(m,t),S($,t)}}}const Z='{"title":"tokenizer 回顾！","local":"tokenizer ，回顾！","sections":[],"depth":1}';function tt(j){return R(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class lt extends W{constructor(i){super(),D(this,i,tt,Y,G,{})}}export{lt as component};
