import{s as J,n as R,o as D}from"../chunks/scheduler.37c15a92.js";import{S as G,i as K,g as s,s as i,r as q,A as O,h as l,f as a,c as r,j as A,u as F,x as b,k as N,y as Q,a as n,v as U,d as W,t as j,w as B}from"../chunks/index.2bf4358c.js";import{C as V}from"../chunks/CourseFloatingBanner.15ba07e6.js";import{H as X}from"../chunks/Heading.8ada512a.js";function Y(L){let o,k,w,v,h,_,u,y,p,P='In <a href="/course/chapter3">Chapter 3</a>, we looked at how to fine-tune a model on a given task. When we do that, we use the same tokenizer that the model was pretrained with — but what do we do when we want to train a model from scratch? In these cases, using a tokenizer that was pretrained on a corpus from another domain or language is typically suboptimal. For example, a tokenizer that’s trained on an English corpus will perform poorly on a corpus of Japanese texts because the use of spaces and punctuation is very different in the two languages.',T,c,M='In this chapter, you will learn how to train a brand new tokenizer on a corpus of texts, so it can then be used to pretrain a language model. This will all be done with the help of the <a href="https://github.com/huggingface/tokenizers" rel="nofollow">🤗 Tokenizers</a> library, which provides the “fast” tokenizers in the <a href="https://github.com/huggingface/transformers" rel="nofollow">🤗 Transformers</a> library. We’ll take a close look at the features that this library provides, and explore how the fast tokenizers differ from the “slow” versions.',$,f,E="Topics we will cover include:",z,m,I="<li>How to train a new tokenizer similar to the one used by a given checkpoint on a new corpus of texts</li> <li>The special features of fast tokenizers</li> <li>The differences between the three main subword tokenization algorithms used in NLP today</li> <li>How to build a tokenizer from scratch with the 🤗 Tokenizers library and train it on some data</li>",x,d,S='The techniques introduced in this chapter will prepare you for the section in <a href="/course/chapter7/6">Chapter 7</a> where we look at creating a language model for Python source code. Let’s start by looking at what it means to “train” a tokenizer in the first place.',C,g,H;return h=new X({props:{title:"Introduction",local:"introduction",headingTag:"h1"}}),u=new V({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),{c(){o=s("meta"),k=i(),w=s("p"),v=i(),q(h.$$.fragment),_=i(),q(u.$$.fragment),y=i(),p=s("p"),p.innerHTML=P,T=i(),c=s("p"),c.innerHTML=M,$=i(),f=s("p"),f.textContent=E,z=i(),m=s("ul"),m.innerHTML=I,x=i(),d=s("p"),d.innerHTML=S,C=i(),g=s("p"),this.h()},l(e){const t=O("svelte-u9bgzb",document.head);o=l(t,"META",{name:!0,content:!0}),t.forEach(a),k=r(e),w=l(e,"P",{}),A(w).forEach(a),v=r(e),F(h.$$.fragment,e),_=r(e),F(u.$$.fragment,e),y=r(e),p=l(e,"P",{"data-svelte-h":!0}),b(p)!=="svelte-1bhpfou"&&(p.innerHTML=P),T=r(e),c=l(e,"P",{"data-svelte-h":!0}),b(c)!=="svelte-cst5sb"&&(c.innerHTML=M),$=r(e),f=l(e,"P",{"data-svelte-h":!0}),b(f)!=="svelte-1v9fbl3"&&(f.textContent=E),z=r(e),m=l(e,"UL",{"data-svelte-h":!0}),b(m)!=="svelte-1mhak9"&&(m.innerHTML=I),x=r(e),d=l(e,"P",{"data-svelte-h":!0}),b(d)!=="svelte-l9t9z5"&&(d.innerHTML=S),C=r(e),g=l(e,"P",{}),A(g).forEach(a),this.h()},h(){N(o,"name","hf:doc:metadata"),N(o,"content",Z)},m(e,t){Q(document.head,o),n(e,k,t),n(e,w,t),n(e,v,t),U(h,e,t),n(e,_,t),U(u,e,t),n(e,y,t),n(e,p,t),n(e,T,t),n(e,c,t),n(e,$,t),n(e,f,t),n(e,z,t),n(e,m,t),n(e,x,t),n(e,d,t),n(e,C,t),n(e,g,t),H=!0},p:R,i(e){H||(W(h.$$.fragment,e),W(u.$$.fragment,e),H=!0)},o(e){j(h.$$.fragment,e),j(u.$$.fragment,e),H=!1},d(e){e&&(a(k),a(w),a(v),a(_),a(y),a(p),a(T),a(c),a($),a(f),a(z),a(m),a(x),a(d),a(C),a(g)),a(o),B(h,e),B(u,e)}}}const Z='{"title":"Introduction","local":"introduction","sections":[],"depth":1}';function ee(L){return D(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ie extends G{constructor(o){super(),K(this,o,ee,Y,J,{})}}export{ie as component};
