import{s as D,n as K,o as Q}from"../chunks/scheduler.cc52f4b9.js";import{S as V,i as X,e as s,s as i,c as I,h as Y,a as l,d as a,b as o,f as O,g as S,j as k,k as R,l as Z,m as n,n as q,t as F,o as N,p as U}from"../chunks/index.5b747fde.js";import{C as ee,H as te,E as ae}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.d38ac5a9.js";import{C as ne}from"../chunks/CourseFloatingBanner.b33fcd79.js";function ie(W){let r,v,$,x,h,y,p,_,f,T,m,j='In <a href="/course/chapter3">Chapter 3</a>, we looked at how to fine-tune a model on a given task. When we do that, we use the same tokenizer that the model was pretrained with — but what do we do when we want to train a model from scratch? In these cases, using a tokenizer that was pretrained on a corpus from another domain or language is typically suboptimal. For example, a tokenizer that’s trained on an English corpus will perform poorly on a corpus of Japanese texts because the use of spaces and punctuation is very different in the two languages.',z,u,A='In this chapter, you will learn how to train a brand new tokenizer on a corpus of texts, so it can then be used to pretrain a language model. This will all be done with the help of the <a href="https://github.com/huggingface/tokenizers" rel="nofollow">🤗 Tokenizers</a> library, which provides the “fast” tokenizers in the <a href="https://github.com/huggingface/transformers" rel="nofollow">🤗 Transformers</a> library. We’ll take a close look at the features that this library provides, and explore how the fast tokenizers differ from the “slow” versions.',C,c,B="Topics we will cover include:",L,d,G="<li>How to train a new tokenizer similar to the one used by a given checkpoint on a new corpus of texts</li> <li>The special features of fast tokenizers</li> <li>The differences between the three main subword tokenization algorithms used in NLP today</li> <li>How to build a tokenizer from scratch with the 🤗 Tokenizers library and train it on some data</li>",H,w,J='The techniques introduced in this chapter will prepare you for the section in <a href="/course/chapter7/6">Chapter 7</a> where we look at creating a language model for Python source code. Let’s start by looking at what it means to “train” a tokenizer in the first place.',M,g,P,b,E;return h=new ee({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),p=new te({props:{title:"Introduction",local:"introduction",headingTag:"h1"}}),f=new ne({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),g=new ae({props:{source:"https://github.com/huggingface/course/blob/main/chapters/en/chapter6/1.mdx"}}),{c(){r=s("meta"),v=i(),$=s("p"),x=i(),I(h.$$.fragment),y=i(),I(p.$$.fragment),_=i(),I(f.$$.fragment),T=i(),m=s("p"),m.innerHTML=j,z=i(),u=s("p"),u.innerHTML=A,C=i(),c=s("p"),c.textContent=B,L=i(),d=s("ul"),d.innerHTML=G,H=i(),w=s("p"),w.innerHTML=J,M=i(),I(g.$$.fragment),P=i(),b=s("p"),this.h()},l(e){const t=Y("svelte-u9bgzb",document.head);r=l(t,"META",{name:!0,content:!0}),t.forEach(a),v=o(e),$=l(e,"P",{}),O($).forEach(a),x=o(e),S(h.$$.fragment,e),y=o(e),S(p.$$.fragment,e),_=o(e),S(f.$$.fragment,e),T=o(e),m=l(e,"P",{"data-svelte-h":!0}),k(m)!=="svelte-1bhpfou"&&(m.innerHTML=j),z=o(e),u=l(e,"P",{"data-svelte-h":!0}),k(u)!=="svelte-cst5sb"&&(u.innerHTML=A),C=o(e),c=l(e,"P",{"data-svelte-h":!0}),k(c)!=="svelte-1v9fbl3"&&(c.textContent=B),L=o(e),d=l(e,"UL",{"data-svelte-h":!0}),k(d)!=="svelte-1mhak9"&&(d.innerHTML=G),H=o(e),w=l(e,"P",{"data-svelte-h":!0}),k(w)!=="svelte-l9t9z5"&&(w.innerHTML=J),M=o(e),S(g.$$.fragment,e),P=o(e),b=l(e,"P",{}),O(b).forEach(a),this.h()},h(){R(r,"name","hf:doc:metadata"),R(r,"content",oe)},m(e,t){Z(document.head,r),n(e,v,t),n(e,$,t),n(e,x,t),q(h,e,t),n(e,y,t),q(p,e,t),n(e,_,t),q(f,e,t),n(e,T,t),n(e,m,t),n(e,z,t),n(e,u,t),n(e,C,t),n(e,c,t),n(e,L,t),n(e,d,t),n(e,H,t),n(e,w,t),n(e,M,t),q(g,e,t),n(e,P,t),n(e,b,t),E=!0},p:K,i(e){E||(F(h.$$.fragment,e),F(p.$$.fragment,e),F(f.$$.fragment,e),F(g.$$.fragment,e),E=!0)},o(e){N(h.$$.fragment,e),N(p.$$.fragment,e),N(f.$$.fragment,e),N(g.$$.fragment,e),E=!1},d(e){e&&(a(v),a($),a(x),a(y),a(_),a(T),a(m),a(z),a(u),a(C),a(c),a(L),a(d),a(H),a(w),a(M),a(P),a(b)),a(r),U(h,e),U(p,e),U(f,e),U(g,e)}}}const oe='{"title":"Introduction","local":"introduction","sections":[],"depth":1}';function re(W){return Q(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class fe extends V{constructor(r){super(),X(this,r,re,ie,D,{})}}export{fe as component};
