import{s as U,n as j,o as F}from"../chunks/scheduler.37c15a92.js";import{S as N,i as R,g as u,s as c,r as L,A as W,h as f,f as n,c as p,j as M,u as E,x as T,k as y,y as D,a as i,v as q,d as A,t as B,w as S}from"../chunks/index.2bf4358c.js";import{C as G}from"../chunks/CourseFloatingBanner.15ba07e6.js";import{H as I}from"../chunks/Heading.8ada512a.js";function J(g){let a,_,d,$,r,v,o,b,s,w="Bon travail pour finir ce chapitre !",z,l,C="Après cette plongée en profondeur dans les <em>tokenizers</em>, vous devriez :",k,m,H="<li>être capable d’entraîner un nouveau tokenizer en utilisant un ancien tokenizer comme modèle,</li> <li>comprendre comment utiliser les <em>offsets</em> pour faire correspondre la position des <em>tokens</em> à l’étendue de texte d’origine,</li> <li>connaître les différences entre BPE, <em>WordPiece</em> et <em>Unigram</em>,</li> <li>être capable de combiner les blocs fournis par la bibliothèque 🤗 <em>Tokenizers</em> pour construire votre propre <em>tokenizer</em>,</li> <li>être capable d’utiliser ce <em>tokenizer</em> dans la bibliothèque 🤗 <em>Transformers</em>.</li>",x,h,P;return r=new I({props:{title:"<i> Tokenizer </i> , coché !",local:"i-tokenizer-i--coché-",headingTag:"h1"}}),o=new G({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),{c(){a=u("meta"),_=c(),d=u("p"),$=c(),L(r.$$.fragment),v=c(),L(o.$$.fragment),b=c(),s=u("p"),s.textContent=w,z=c(),l=u("p"),l.innerHTML=C,k=c(),m=u("ul"),m.innerHTML=H,x=c(),h=u("p"),this.h()},l(e){const t=W("svelte-u9bgzb",document.head);a=f(t,"META",{name:!0,content:!0}),t.forEach(n),_=p(e),d=f(e,"P",{}),M(d).forEach(n),$=p(e),E(r.$$.fragment,e),v=p(e),E(o.$$.fragment,e),b=p(e),s=f(e,"P",{"data-svelte-h":!0}),T(s)!=="svelte-1nu3oa1"&&(s.textContent=w),z=p(e),l=f(e,"P",{"data-svelte-h":!0}),T(l)!=="svelte-thw73"&&(l.innerHTML=C),k=p(e),m=f(e,"UL",{"data-svelte-h":!0}),T(m)!=="svelte-16sd9em"&&(m.innerHTML=H),x=p(e),h=f(e,"P",{}),M(h).forEach(n),this.h()},h(){y(a,"name","hf:doc:metadata"),y(a,"content",K)},m(e,t){D(document.head,a),i(e,_,t),i(e,d,t),i(e,$,t),q(r,e,t),i(e,v,t),q(o,e,t),i(e,b,t),i(e,s,t),i(e,z,t),i(e,l,t),i(e,k,t),i(e,m,t),i(e,x,t),i(e,h,t),P=!0},p:j,i(e){P||(A(r.$$.fragment,e),A(o.$$.fragment,e),P=!0)},o(e){B(r.$$.fragment,e),B(o.$$.fragment,e),P=!1},d(e){e&&(n(_),n(d),n($),n(v),n(b),n(s),n(z),n(l),n(k),n(m),n(x),n(h)),n(a),S(r,e),S(o,e)}}}const K='{"title":"<i> Tokenizer </i> , coché !","local":"i-tokenizer-i--coché-","sections":[],"depth":1}';function O(g){return F(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Z extends N{constructor(a){super(),R(this,a,O,J,U,{})}}export{Z as component};
