import{s as oa,a as ia,o as pa,n as cs}from"../chunks/scheduler.cc52f4b9.js";import{S as ra,i as Ma,e as T,s as i,c as w,h as ca,a as U,d as s,b as p,f as Jl,g as u,j as f,k as xe,l as ya,m as a,n as h,o as c,E as rt,t as M,p as d,A as sa,F as Mt}from"../chunks/index.5b747fde.js";import{T as ct}from"../chunks/Tip.f0925d57.js";import{C as ma,H as Us,E as wa}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.d38ac5a9.js";import{Y as Tl}from"../chunks/Youtube.8bb4154e.js";import{C as I}from"../chunks/CodeBlock.665bfc37.js";import{C as na}from"../chunks/CourseFloatingBanner.b33fcd79.js";import{F as ua}from"../chunks/FrameworkSwitchCourse.b3b657b3.js";function ha(C){let l,r;return l=new na({props:{chapter:7,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter7/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter7/section6_tf.ipynb"}]}}),{c(){w(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,y){h(l,n,y),r=!0},i(n){r||(M(l.$$.fragment,n),r=!0)},o(n){c(l.$$.fragment,n),r=!1},d(n){d(l,n)}}}function da(C){let l,r;return l=new na({props:{chapter:7,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter7/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter7/section6_pt.ipynb"}]}}),{c(){w(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,y){h(l,n,y),r=!0},i(n){r||(M(l.$$.fragment,n),r=!0)},o(n){c(l.$$.fragment,n),r=!1},d(n){d(l,n)}}}function Ja(C){let l,r="Pretraining the language model will take a while. We suggest that you first run the training loop on a sample of the data by uncommenting the two partial lines above, and make sure that the training successfully completes and the models are stored. Nothing is more frustrating than a training run failing at the last step because you forgot to create a folder or because there’s a typo at the end of the training loop!";return{c(){l=T("p"),l.textContent=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-11piewf"&&(l.textContent=r)},m(n,y){a(n,l,y)},p:cs,d(n){n&&s(l)}}}function Ta(C){let l,r="✏️ <strong>Try it out!</strong> Getting rid of all the chunks that are smaller than the context size wasn’t a big issue here because we’re using small context windows. As you increase the context size (or if you have a corpus of short documents), the fraction of chunks that are thrown away will also grow. A more efficient way to prepare the data is to join all the tokenized samples in a batch with an <code>eos_token_id</code> token in between, and then perform the chunking on the concatenated sequences. As an exercise, modify the <code>tokenize()</code> function to make use of that approach. Note that you’ll want to set <code>truncation=False</code> and remove the other arguments from the tokenizer to get the full sequence of token IDs.";return{c(){l=T("p"),l.innerHTML=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-n84vqf"&&(l.innerHTML=r)},m(n,y){a(n,l,y)},p:cs,d(n){n&&s(l)}}}function Ua(C){let l,r,n,y="With that configuration, we can load a new model. Note that this is the first time we don’t use the <code>from_pretrained()</code> function, since we’re actually initializing a model ourself:",g,b,$,k,Z;return l=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBURkdQVDJMTUhlYWRNb2RlbCUyQyUyMEF1dG9Db25maWclMEElMEFjb25maWclMjAlM0QlMjBBdXRvQ29uZmlnLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJncHQyJTIyJTJDJTBBJTIwJTIwJTIwJTIwdm9jYWJfc2l6ZSUzRGxlbih0b2tlbml6ZXIpJTJDJTBBJTIwJTIwJTIwJTIwbl9jdHglM0Rjb250ZXh0X2xlbmd0aCUyQyUwQSUyMCUyMCUyMCUyMGJvc190b2tlbl9pZCUzRHRva2VuaXplci5ib3NfdG9rZW5faWQlMkMlMEElMjAlMjAlMjAlMjBlb3NfdG9rZW5faWQlM0R0b2tlbml6ZXIuZW9zX3Rva2VuX2lkJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFGPT2LMHeadModel, AutoConfig

config = AutoConfig.from_pretrained(
    <span class="hljs-string">&quot;gpt2&quot;</span>,
    vocab_size=<span class="hljs-built_in">len</span>(tokenizer),
    n_ctx=context_length,
    bos_token_id=tokenizer.bos_token_id,
    eos_token_id=tokenizer.eos_token_id,
)`,wrap:!1}}),b=new I({props:{code:"bW9kZWwlMjAlM0QlMjBURkdQVDJMTUhlYWRNb2RlbChjb25maWcpJTBBbW9kZWwobW9kZWwuZHVtbXlfaW5wdXRzKSUyMCUyMCUyMyUyMEJ1aWxkcyUyMHRoZSUyMG1vZGVsJTBBbW9kZWwuc3VtbWFyeSgp",highlighted:`model = TFGPT2LMHeadModel(config)
model(model.dummy_inputs)  <span class="hljs-comment"># Builds the model</span>
model.summary()`,wrap:!1}}),k=new I({props:{code:"X19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX18lMEFMYXllciUyMCh0eXBlKSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME91dHB1dCUyMFNoYXBlJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwUGFyYW0lMjAlMjMlMjAlMjAlMjAlMEElM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMEF0cmFuc2Zvcm1lciUyMChURkdQVDJNYWluTGF5ZXIlMjBtdWx0aXBsZSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDEyNDI0MjQzMiUyMCUwQSUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUzRCUwQVRvdGFsJTIwcGFyYW1zJTNBJTIwMTI0JTJDMjQyJTJDNDMyJTBBVHJhaW5hYmxlJTIwcGFyYW1zJTNBJTIwMTI0JTJDMjQyJTJDNDMyJTBBTm9uLXRyYWluYWJsZSUyMHBhcmFtcyUzQSUyMDAlMEFfX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fXw==",highlighted:`_________________________________________________________________
Layer (<span class="hljs-built_in">type</span>)                 Output Shape              Param <span class="hljs-comment">#   </span>
=================================================================
transformer (TFGPT2MainLayer multiple                  <span class="hljs-number">124242432</span> 
=================================================================
Total params: <span class="hljs-number">124</span>,<span class="hljs-number">242</span>,<span class="hljs-number">432</span>
Trainable params: <span class="hljs-number">124</span>,<span class="hljs-number">242</span>,<span class="hljs-number">432</span>
Non-trainable params: <span class="hljs-number">0</span>
_________________________________________________________________`,wrap:!1}}),{c(){w(l.$$.fragment),r=i(),n=T("p"),n.innerHTML=y,g=i(),w(b.$$.fragment),$=i(),w(k.$$.fragment)},l(J){u(l.$$.fragment,J),r=p(J),n=U(J,"P",{"data-svelte-h":!0}),f(n)!=="svelte-1tlyd8t"&&(n.innerHTML=y),g=p(J),u(b.$$.fragment,J),$=p(J),u(k.$$.fragment,J)},m(J,j){h(l,J,j),a(J,r,j),a(J,n,j),a(J,g,j),h(b,J,j),a(J,$,j),h(k,J,j),Z=!0},i(J){Z||(M(l.$$.fragment,J),M(b.$$.fragment,J),M(k.$$.fragment,J),Z=!0)},o(J){c(l.$$.fragment,J),c(b.$$.fragment,J),c(k.$$.fragment,J),Z=!1},d(J){J&&(s(r),s(n),s(g),s($)),d(l,J),d(b,J),d(k,J)}}}function fa(C){let l,r,n,y="With that configuration, we can load a new model. Note that this is the first time we don’t use the <code>from_pretrained()</code> function, since we’re actually initializing a model ourself:",g,b,$,k,Z;return l=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBHUFQyTE1IZWFkTW9kZWwlMkMlMjBBdXRvQ29uZmlnJTBBJTBBY29uZmlnJTIwJTNEJTIwQXV0b0NvbmZpZy5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyZ3B0MiUyMiUyQyUwQSUyMCUyMCUyMCUyMHZvY2FiX3NpemUlM0RsZW4odG9rZW5pemVyKSUyQyUwQSUyMCUyMCUyMCUyMG5fY3R4JTNEY29udGV4dF9sZW5ndGglMkMlMEElMjAlMjAlMjAlMjBib3NfdG9rZW5faWQlM0R0b2tlbml6ZXIuYm9zX3Rva2VuX2lkJTJDJTBBJTIwJTIwJTIwJTIwZW9zX3Rva2VuX2lkJTNEdG9rZW5pemVyLmVvc190b2tlbl9pZCUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, GPT2LMHeadModel, AutoConfig

config = AutoConfig.from_pretrained(
    <span class="hljs-string">&quot;gpt2&quot;</span>,
    vocab_size=<span class="hljs-built_in">len</span>(tokenizer),
    n_ctx=context_length,
    bos_token_id=tokenizer.bos_token_id,
    eos_token_id=tokenizer.eos_token_id,
)`,wrap:!1}}),b=new I({props:{code:"bW9kZWwlMjAlM0QlMjBHUFQyTE1IZWFkTW9kZWwoY29uZmlnKSUwQW1vZGVsX3NpemUlMjAlM0QlMjBzdW0odC5udW1lbCgpJTIwZm9yJTIwdCUyMGluJTIwbW9kZWwucGFyYW1ldGVycygpKSUwQXByaW50KGYlMjJHUFQtMiUyMHNpemUlM0ElMjAlN0Jtb2RlbF9zaXplJTJGMTAwMCoqMiUzQS4xZiU3RE0lMjBwYXJhbWV0ZXJzJTIyKQ==",highlighted:`model = GPT2LMHeadModel(config)
model_size = <span class="hljs-built_in">sum</span>(t.numel() <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> model.parameters())
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;GPT-2 size: <span class="hljs-subst">{model_size/<span class="hljs-number">1000</span>**<span class="hljs-number">2</span>:<span class="hljs-number">.1</span>f}</span>M parameters&quot;</span>)`,wrap:!1}}),k=new I({props:{code:"R1BULTIlMjBzaXplJTNBJTIwMTI0LjJNJTIwcGFyYW1ldGVycw==",highlighted:'GPT-<span class="hljs-number">2</span> size: <span class="hljs-number">124.2</span>M parameters',wrap:!1}}),{c(){w(l.$$.fragment),r=i(),n=T("p"),n.innerHTML=y,g=i(),w(b.$$.fragment),$=i(),w(k.$$.fragment)},l(J){u(l.$$.fragment,J),r=p(J),n=U(J,"P",{"data-svelte-h":!0}),f(n)!=="svelte-1tlyd8t"&&(n.innerHTML=y),g=p(J),u(b.$$.fragment,J),$=p(J),u(k.$$.fragment,J)},m(J,j){h(l,J,j),a(J,r,j),a(J,n,j),a(J,g,j),h(b,J,j),a(J,$,j),h(k,J,j),Z=!0},i(J){Z||(M(l.$$.fragment,J),M(b.$$.fragment,J),M(k.$$.fragment,J),Z=!0)},o(J){c(l.$$.fragment,J),c(b.$$.fragment,J),c(k.$$.fragment,J),Z=!1},d(J){J&&(s(r),s(n),s(g),s($)),d(l,J),d(b,J),d(k,J)}}}function ja(C){let l,r;return l=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvckxhbmd1YWdlTW9kZWxpbmclMEElMEF0b2tlbml6ZXIucGFkX3Rva2VuJTIwJTNEJTIwdG9rZW5pemVyLmVvc190b2tlbiUwQWRhdGFfY29sbGF0b3IlMjAlM0QlMjBEYXRhQ29sbGF0b3JGb3JMYW5ndWFnZU1vZGVsaW5nKHRva2VuaXplciUyQyUyMG1sbSUzREZhbHNlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJ0ZiUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),{c(){w(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,y){h(l,n,y),r=!0},i(n){r||(M(l.$$.fragment,n),r=!0)},o(n){c(l.$$.fragment,n),r=!1},d(n){d(l,n)}}}function ba(C){let l,r;return l=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvckxhbmd1YWdlTW9kZWxpbmclMEElMEF0b2tlbml6ZXIucGFkX3Rva2VuJTIwJTNEJTIwdG9rZW5pemVyLmVvc190b2tlbiUwQWRhdGFfY29sbGF0b3IlMjAlM0QlMjBEYXRhQ29sbGF0b3JGb3JMYW5ndWFnZU1vZGVsaW5nKHRva2VuaXplciUyQyUyMG1sbSUzREZhbHNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=<span class="hljs-literal">False</span>)`,wrap:!1}}),{c(){w(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,y){h(l,n,y),r=!0},i(n){r||(M(l.$$.fragment,n),r=!0)},o(n){c(l.$$.fragment,n),r=!1},d(n){d(l,n)}}}function Ia(C){let l,r;return l=new I({props:{code:"aW5wdXRfaWRzJTIwc2hhcGUlM0ElMjAoNSUyQyUyMDEyOCklMEFhdHRlbnRpb25fbWFzayUyMHNoYXBlJTNBJTIwKDUlMkMlMjAxMjgpJTBBbGFiZWxzJTIwc2hhcGUlM0ElMjAoNSUyQyUyMDEyOCk=",highlighted:`input_ids shape: (<span class="hljs-number">5</span>, <span class="hljs-number">128</span>)
attention_mask shape: (<span class="hljs-number">5</span>, <span class="hljs-number">128</span>)
labels shape: (<span class="hljs-number">5</span>, <span class="hljs-number">128</span>)`,wrap:!1}}),{c(){w(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,y){h(l,n,y),r=!0},i(n){r||(M(l.$$.fragment,n),r=!0)},o(n){c(l.$$.fragment,n),r=!1},d(n){d(l,n)}}}function Ca(C){let l,r;return l=new I({props:{code:"aW5wdXRfaWRzJTIwc2hhcGUlM0ElMjB0b3JjaC5TaXplKCU1QjUlMkMlMjAxMjglNUQpJTBBYXR0ZW50aW9uX21hc2slMjBzaGFwZSUzQSUyMHRvcmNoLlNpemUoJTVCNSUyQyUyMDEyOCU1RCklMEFsYWJlbHMlMjBzaGFwZSUzQSUyMHRvcmNoLlNpemUoJTVCNSUyQyUyMDEyOCU1RCk=",highlighted:`input_ids shape: torch.Size([<span class="hljs-number">5</span>, <span class="hljs-number">128</span>])
attention_mask shape: torch.Size([<span class="hljs-number">5</span>, <span class="hljs-number">128</span>])
labels shape: torch.Size([<span class="hljs-number">5</span>, <span class="hljs-number">128</span>])`,wrap:!1}}),{c(){w(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,y){h(l,n,y),r=!0},i(n){r||(M(l.$$.fragment,n),r=!0)},o(n){c(l.$$.fragment,n),r=!1},d(n){d(l,n)}}}function la(C){let l,r="Now we can use the <code>prepare_tf_dataset()</code> method to convert our datasets to TensorFlow datasets with the data collator we created above:",n,y,g;return y=new I({props:{code:"dGZfdHJhaW5fZGF0YXNldCUyMCUzRCUyMG1vZGVsLnByZXBhcmVfdGZfZGF0YXNldCglMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfZGF0YXNldHMlNUIlMjJ0cmFpbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBJTIwJTIwJTIwJTIwc2h1ZmZsZSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNEMzIlMkMlMEEpJTBBdGZfZXZhbF9kYXRhc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF9kYXRhc2V0cyU1QiUyMnZhbGlkJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwY29sbGF0ZV9mbiUzRGRhdGFfY29sbGF0b3IlMkMlMEElMjAlMjAlMjAlMjBzaHVmZmxlJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNEMzIlMkMlMEEp",highlighted:`tf_train_dataset = model.prepare_tf_dataset(
    tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">32</span>,
)
tf_eval_dataset = model.prepare_tf_dataset(
    tokenized_datasets[<span class="hljs-string">&quot;valid&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">32</span>,
)`,wrap:!1}}),{c(){l=T("p"),l.innerHTML=r,n=i(),w(y.$$.fragment)},l(b){l=U(b,"P",{"data-svelte-h":!0}),f(l)!=="svelte-13jf88u"&&(l.innerHTML=r),n=p(b),u(y.$$.fragment,b)},m(b,$){a(b,l,$),a(b,n,$),h(y,b,$),g=!0},i(b){g||(M(y.$$.fragment,b),g=!0)},o(b){c(y.$$.fragment,b),g=!1},d(b){b&&(s(l),s(n)),d(y,b)}}}function ga(C){let l,r="⚠️ Shifting the inputs and labels to align them happens inside the model, so the data collator just copies the inputs to create the labels.";return{c(){l=T("p"),l.textContent=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-k25o8d"&&(l.textContent=r)},m(n,y){a(n,l,y)},p:cs,d(n){n&&s(l)}}}function _a(C){let l,r="All that’s left to do is configure the training hyperparameters and call <code>compile()</code> and <code>fit()</code>. We’ll use a learning rate schedule with some warmup to improve the stability of training:",n,y,g,b,$="Now we can just call <code>model.fit()</code> and wait for training to finish. Depending on whether you run it on the full or a subset of the training set this will take 20 or 2 hours, respectively, so grab a few coffees and a good book to read! After training completes we can push the model and tokenizer to the Hub:",k,Z,J;return y=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEFpbXBvcnQlMjB0ZW5zb3JmbG93JTIwYXMlMjB0ZiUwQSUwQW51bV90cmFpbl9zdGVwcyUyMCUzRCUyMGxlbih0Zl90cmFpbl9kYXRhc2V0KSUwQW9wdGltaXplciUyQyUyMHNjaGVkdWxlJTIwJTNEJTIwY3JlYXRlX29wdGltaXplciglMEElMjAlMjAlMjAlMjBpbml0X2xyJTNENWUtNSUyQyUwQSUyMCUyMCUyMCUyMG51bV93YXJtdXBfc3RlcHMlM0QxXzAwMCUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9zdGVwcyUzRG51bV90cmFpbl9zdGVwcyUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheV9yYXRlJTNEMC4wMSUyQyUwQSklMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplciklMEElMEElMjMlMjBUcmFpbiUyMGluJTIwbWl4ZWQtcHJlY2lzaW9uJTIwZmxvYXQxNiUwQXRmLmtlcmFzLm1peGVkX3ByZWNpc2lvbi5zZXRfZ2xvYmFsX3BvbGljeSglMjJtaXhlZF9mbG9hdDE2JTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

num_train_steps = <span class="hljs-built_in">len</span>(tf_train_dataset)
optimizer, schedule = create_optimizer(
    init_lr=<span class="hljs-number">5e-5</span>,
    num_warmup_steps=<span class="hljs-number">1_000</span>,
    num_train_steps=num_train_steps,
    weight_decay_rate=<span class="hljs-number">0.01</span>,
)
model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)

<span class="hljs-comment"># Train in mixed-precision float16</span>
tf.keras.mixed_precision.set_global_policy(<span class="hljs-string">&quot;mixed_float16&quot;</span>)`,wrap:!1}}),Z=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBQdXNoVG9IdWJDYWxsYmFjayUwQSUwQWNhbGxiYWNrJTIwJTNEJTIwUHVzaFRvSHViQ2FsbGJhY2sob3V0cHV0X2RpciUzRCUyMmNvZGVwYXJyb3QtZHMlMjIlMkMlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIpJTBBJTBBbW9kZWwuZml0KHRmX3RyYWluX2RhdGFzZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl9ldmFsX2RhdGFzZXQlMkMlMjBjYWxsYmFja3MlM0QlNUJjYWxsYmFjayU1RCk=",highlighted:`<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

callback = PushToHubCallback(output_dir=<span class="hljs-string">&quot;codeparrot-ds&quot;</span>, tokenizer=tokenizer)

model.fit(tf_train_dataset, validation_data=tf_eval_dataset, callbacks=[callback])`,wrap:!1}}),{c(){l=T("p"),l.innerHTML=r,n=i(),w(y.$$.fragment),g=i(),b=T("p"),b.innerHTML=$,k=i(),w(Z.$$.fragment)},l(j){l=U(j,"P",{"data-svelte-h":!0}),f(l)!=="svelte-mj1jm"&&(l.innerHTML=r),n=p(j),u(y.$$.fragment,j),g=p(j),b=U(j,"P",{"data-svelte-h":!0}),f(b)!=="svelte-18w1tu4"&&(b.innerHTML=$),k=p(j),u(Z.$$.fragment,j)},m(j,G){a(j,l,G),a(j,n,G),h(y,j,G),a(j,g,G),a(j,b,G),a(j,k,G),h(Z,j,G),J=!0},i(j){J||(M(y.$$.fragment,j),M(Z.$$.fragment,j),J=!0)},o(j){c(y.$$.fragment,j),c(Z.$$.fragment,j),J=!1},d(j){j&&(s(l),s(n),s(g),s(b),s(k)),d(y,j),d(Z,j)}}}function ka(C){let l,r="All that’s left to do is configure the training arguments and fire up the <code>Trainer</code>. We’ll use a cosine learning rate schedule with some warmup and an effective batch size of 256 (<code>per_device_train_batch_size</code> * <code>gradient_accumulation_steps</code>). Gradient accumulation is used when a single batch does not fit into memory, and incrementally builds up the gradient through several forward/backward passes. We’ll see this in action when we create the training loop with 🤗 Accelerate.",n,y,g,b,$="Now we can just start the <code>Trainer</code> and wait for training to finish. Depending on whether you run it on the full or a subset of the training set this will take 20 or 2 hours, respectively, so grab a few coffees and a good book to read!",k,Z,J,j,G="After training completes, we can push the model and tokenizer to the Hub:",R,X,N;return y=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMkMlMjBUcmFpbmluZ0FyZ3VtZW50cyUwQSUwQWFyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEJTIyY29kZXBhcnJvdC1kcyUyMiUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDMyJTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0QzMiUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fc3RyYXRlZ3klM0QlMjJzdGVwcyUyMiUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfc3RlcHMlM0Q1XzAwMCUyQyUwQSUyMCUyMCUyMCUyMGxvZ2dpbmdfc3RlcHMlM0Q1XzAwMCUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzRDglMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMSUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRDAuMSUyQyUwQSUyMCUyMCUyMCUyMHdhcm11cF9zdGVwcyUzRDFfMDAwJTJDJTBBJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyX3R5cGUlM0QlMjJjb3NpbmUlMjIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNENWUtNCUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfc3RlcHMlM0Q1XzAwMCUyQyUwQSUyMCUyMCUyMCUyMGZwMTYlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEYXJncyUyQyUwQSUyMCUyMCUyMCUyMGRhdGFfY29sbGF0b3IlM0RkYXRhX2NvbGxhdG9yJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRHRva2VuaXplZF9kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX2RhdGFzZXRzJTVCJTIydmFsaWQlMjIlNUQlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainingArguments

args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;codeparrot-ds&quot;</span>,
    per_device_train_batch_size=<span class="hljs-number">32</span>,
    per_device_eval_batch_size=<span class="hljs-number">32</span>,
    evaluation_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
    eval_steps=<span class="hljs-number">5_000</span>,
    logging_steps=<span class="hljs-number">5_000</span>,
    gradient_accumulation_steps=<span class="hljs-number">8</span>,
    num_train_epochs=<span class="hljs-number">1</span>,
    weight_decay=<span class="hljs-number">0.1</span>,
    warmup_steps=<span class="hljs-number">1_000</span>,
    lr_scheduler_type=<span class="hljs-string">&quot;cosine&quot;</span>,
    learning_rate=<span class="hljs-number">5e-4</span>,
    save_steps=<span class="hljs-number">5_000</span>,
    fp16=<span class="hljs-literal">True</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
)

trainer = Trainer(
    model=model,
    tokenizer=tokenizer,
    args=args,
    data_collator=data_collator,
    train_dataset=tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=tokenized_datasets[<span class="hljs-string">&quot;valid&quot;</span>],
)`,wrap:!1}}),Z=new I({props:{code:"dHJhaW5lci50cmFpbigp",highlighted:"trainer.train()",wrap:!1}}),X=new I({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:"trainer.push_to_hub()",wrap:!1}}),{c(){l=T("p"),l.innerHTML=r,n=i(),w(y.$$.fragment),g=i(),b=T("p"),b.innerHTML=$,k=i(),w(Z.$$.fragment),J=i(),j=T("p"),j.textContent=G,R=i(),w(X.$$.fragment)},l(_){l=U(_,"P",{"data-svelte-h":!0}),f(l)!=="svelte-3hmwum"&&(l.innerHTML=r),n=p(_),u(y.$$.fragment,_),g=p(_),b=U(_,"P",{"data-svelte-h":!0}),f(b)!=="svelte-7ziduw"&&(b.innerHTML=$),k=p(_),u(Z.$$.fragment,_),J=p(_),j=U(_,"P",{"data-svelte-h":!0}),f(j)!=="svelte-16pwd35"&&(j.textContent=G),R=p(_),u(X.$$.fragment,_)},m(_,B){a(_,l,B),a(_,n,B),h(y,_,B),a(_,g,B),a(_,b,B),a(_,k,B),h(Z,_,B),a(_,J,B),a(_,j,B),a(_,R,B),h(X,_,B),N=!0},i(_){N||(M(y.$$.fragment,_),M(Z.$$.fragment,_),M(X.$$.fragment,_),N=!0)},o(_){c(y.$$.fragment,_),c(Z.$$.fragment,_),c(X.$$.fragment,_),N=!1},d(_){_&&(s(l),s(n),s(g),s(b),s(k),s(J),s(j),s(R)),d(y,_),d(Z,_),d(X,_)}}}function Za(C){let l,r="✏️ <strong>Try it out!</strong> It only took us about 30 lines of code in addition to the <code>TrainingArguments</code> to get from raw texts to training GPT-2. Try it out with your own dataset and see if you can get good results!";return{c(){l=T("p"),l.innerHTML=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-ayzxue"&&(l.innerHTML=r)},m(n,y){a(n,l,y)},p:cs,d(n){n&&s(l)}}}function $a(C){let l,r='💡 If you have access to a machine with multiple GPUs, you can try using a <code>MirroredStrategy</code> context to substantially speed up training. You’ll need to create a <code>tf.distribute.MirroredStrategy</code> object, and make sure that any <code>to_tf_dataset()</code> or <code>prepare_tf_dataset()</code> methods as well as model creation and the call to <code>fit()</code> are all run in its <code>scope()</code> context. You can see documentation on this <a href="https://www.tensorflow.org/guide/distributed_training#use_tfdistributestrategy_with_keras_modelfit" rel="nofollow">here</a>.';return{c(){l=T("p"),l.innerHTML=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-cn7tf5"&&(l.innerHTML=r)},m(n,y){a(n,l,y)},d(n){n&&s(l)}}}function Ga(C){let l,r="💡 If you have access to a machine with multiple GPUs, try to run the code there. The <code>Trainer</code> automatically manages multiple machines, and this can speed up training tremendously.";return{c(){l=T("p"),l.innerHTML=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-156ysgv"&&(l.innerHTML=r)},m(n,y){a(n,l,y)},d(n){n&&s(l)}}}function Ba(C){let l;function r(g,b){return g[0]==="pt"?Ga:$a}let n=r(C),y=n(C);return{c(){y.c(),l=sa()},l(g){y.l(g),l=sa()},m(g,b){y.m(g,b),a(g,l,b)},p(g,b){n!==(n=r(g))&&(y.d(1),y=n(g),y&&(y.c(),y.m(l.parentNode,l)))},d(g){g&&s(l),y.d(g)}}}function Xa(C){let l,r;return l=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBY291cnNlX21vZGVsJTIwJTNEJTIwVEZHUFQyTE1IZWFkTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMmh1Z2dpbmdmYWNlLWNvdXJzZSUyRmNvZGVwYXJyb3QtZHMlMjIpJTBBY291cnNlX3Rva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmh1Z2dpbmdmYWNlLWNvdXJzZSUyRmNvZGVwYXJyb3QtZHMlMjIpJTBBcGlwZSUyMCUzRCUyMHBpcGVsaW5lKCUwQSUyMCUyMCUyMCUyMCUyMnRleHQtZ2VuZXJhdGlvbiUyMiUyQyUyMG1vZGVsJTNEY291cnNlX21vZGVsJTJDJTIwdG9rZW5pemVyJTNEY291cnNlX3Rva2VuaXplciUyQyUyMGRldmljZSUzRDAlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

course_model = TFGPT2LMHeadModel.from_pretrained(<span class="hljs-string">&quot;huggingface-course/codeparrot-ds&quot;</span>)
course_tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;huggingface-course/codeparrot-ds&quot;</span>)
pipe = pipeline(
    <span class="hljs-string">&quot;text-generation&quot;</span>, model=course_model, tokenizer=course_tokenizer, device=<span class="hljs-number">0</span>
)`,wrap:!1}}),{c(){w(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,y){h(l,n,y),r=!0},i(n){r||(M(l.$$.fragment,n),r=!0)},o(n){c(l.$$.fragment,n),r=!1},d(n){d(l,n)}}}function Ra(C){let l,r;return l=new I({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwcGlwZWxpbmUlMEElMEFkZXZpY2UlMjAlM0QlMjB0b3JjaC5kZXZpY2UoJTIyY3VkYSUyMiklMjBpZiUyMHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklMjBlbHNlJTIwdG9yY2guZGV2aWNlKCUyMmNwdSUyMiklMEFwaXBlJTIwJTNEJTIwcGlwZWxpbmUoJTBBJTIwJTIwJTIwJTIwJTIydGV4dC1nZW5lcmF0aW9uJTIyJTJDJTIwbW9kZWwlM0QlMjJodWdnaW5nZmFjZS1jb3Vyc2UlMkZjb2RlcGFycm90LWRzJTIyJTJDJTIwZGV2aWNlJTNEZGV2aWNlJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>) <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)
pipe = pipeline(
    <span class="hljs-string">&quot;text-generation&quot;</span>, model=<span class="hljs-string">&quot;huggingface-course/codeparrot-ds&quot;</span>, device=device
)`,wrap:!1}}),{c(){w(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,y){h(l,n,y),r=!0},i(n){r||(M(l.$$.fragment,n),r=!0)},o(n){c(l.$$.fragment,n),r=!1},d(n){d(l,n)}}}function Wa(C){let l,r="Looking at these few examples, it seems that the model has learned some of the syntax of the Python data science stack (of course, we would need to evaluate it more thoroughly before deploying the model in the real world). Sometimes it requires more customization of the model training to achieve the necessary performance for a given use case, however. For example, what if we would like to dynamically update the batch size or have a conditional training loop that skips bad examples on the fly? One option would be to subclass the <code>Trainer</code> and add the necessary changes, but sometimes it’s simpler to write the training loop from scratch. That’s where 🤗 Accelerate comes in.";return{c(){l=T("p"),l.innerHTML=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-n4fvox"&&(l.innerHTML=r)},m(n,y){a(n,l,y)},d(n){n&&s(l)}}}function va(C){let l,r="Looking at these few examples, it seems that the model has learned some of the syntax of the Python data science stack. Of course, we would need to evaluate the model more thoroughly before deploying it in the real world, but this is still an impressive prototype.";return{c(){l=T("p"),l.textContent=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-1kfbi7i"&&(l.textContent=r)},m(n,y){a(n,l,y)},d(n){n&&s(l)}}}function aa(C){let l,r,n,y="We’ve seen how to train a model with the <code>Trainer</code>, which can allow for some customization. However, sometimes we want full control over the training loop, or we want to make some exotic changes. In this case 🤗 Accelerate is a great choice, and in this section we’ll go through the steps to use it to train our model. To make things more interesting, we’ll also add a twist to the training loop.",g,b,$,k,Z="Since we are mainly interested in sensible autocompletion for the the data science libraries, it makes sense to give more weight to training samples that make more use of these libraries. We can easily identify these examples through the use of keywords such as <code>plt</code>, <code>pd</code>, <code>sk</code>, <code>fit</code>, and <code>predict</code>, which are the most frequent import names for <code>matplotlib.pyplot</code>, <code>pandas</code>, and <code>sklearn</code> as well as the fit/predict pattern of the latter. If these are each represented as a single token, we can easily check if they occur in the input sequence. Tokens can have a whitespace prefix, so we’ll also check for those versions in the tokenizer vocabulary. To verify that it works, we’ll add one test token which should be split into multiple tokens:",J,j,G,R,X,N,_="Great, that seems to work nicely! We can now write a custom loss function that takes the input sequence, the logits, and the key tokens we just selected as inputs. First we need to align the logits and inputs: the input sequence shifted by one to the right forms the labels, since the next token is the label for the current token. We can achieve this by starting the labels from the second token of the input sequence, since the model does not make a prediction for the first token anyway. Then we cut off the last logit, as we don’t have a label for the token that follows the full input sequence. With that we can compute the loss per sample and count the occurrences of all keywords in each sample. Finally, we calculate the weighted average over all samples using the occurrences as weights. Since we don’t want to throw away all the samples that have no keywords, we add 1 to the weights:",B,ke,Ee,V,yt="Before we can start training with this awesome new loss function, we need to prepare a few things:",se,Ze,mt="<li>We need dataloaders to load the data in batches.</li> <li>We need to set up weight decay parameters.</li> <li>From time to time we want to evaluate, so it makes sense to wrap the evaluation code in a function.</li>",Q,$e,wt="Let’s start with the dataloaders. We only need to set the dataset’s format to <code>&quot;torch&quot;</code>, and then we can pass it to a PyTorch <code>DataLoader</code> with the appropriate batch size:",le,Ge,Ae,x,ut="Next, we group the parameters so that the optimizer knows which ones will get an additional weight decay. Usually, all bias and LayerNorm weights terms are exempt from this; here’s how we can do this:",ae,Be,Fe,E,fs="Since we want to evaluate the model regularly on the validation set during training, let’s write a function for that as well. It just runs through the evaluation dataloader and gathers all the losses across processes:",Ye,A,He,F,js='With the <code>evaluate()</code> function we can report loss and <a href="/course/chapter7/3">perplexity</a> at regular intervals. Next, we redefine our model to make sure we train from scratch again:',Se,Y,qe,H,ht="We can then define our optimizer, using the function from before to split the parameters for weight decay:",ne,Xe,De,S,dt="Now let’s prepare the model, optimizer, and dataloaders so we can start training:",oe,Re,Le,q,Ke,D,Jt="Now that we have sent our <code>train_dataloader</code> to <code>accelerator.prepare()</code>, we can use its length to compute the number of training steps. Remember that we should always do this after preparing the dataloader, as that method will change its length. We use a classic linear schedule from the learning rate to 0:",ie,We,Pe,L,bs="Lastly, to push our model to the Hub, we will need to create a <code>Repository</code> object in a working folder. First log in to the Hugging Face Hub, if you aren’t logged in already. We’ll determine the repository name from the model ID we want to give our model (feel free to replace the <code>repo_name</code> with your own choice; it just needs to contain your username, which is what the function <code>get_full_repo_name()</code> does):",Oe,K,et,P,tt,z,Tt="Then we can clone that repository in a local folder. If it already exists, this local folder should be an existing clone of the repository we are working with:",pe,ve,st,O,Ut="We can now upload anything we save in <code>output_dir</code> by calling the <code>repo.push_to_hub()</code> method. This will help us upload the intermediate models at the end of each epoch.",re,Me,Qe="Before we train, let’s run a quick test to see if the evaluation function works properly:",ft,ce,ye,me,we,ue,ze="Those are very high values for loss and perplexity, but that’s not surprising as we haven’t trained the model yet. With that, we have everything prepared to write the core part of the training script: the training loop. In the training loop we iterate over the dataloader and pass the batches to the model. With the logits, we can then evaluate our custom loss function. We scale the loss by the number of gradient accumulation steps so as not to create larger losses when aggregating more steps. Before we optimize, we also clip the gradients for better convergence. Finally, every few steps we evaluate the model on the evaluation set with our new <code>evaluate()</code> function:",jt,he,de,Ne,bt="And that’s it — you now have your own custom training loop for causal language models such as GPT-2 that you can further customize to your needs.",te,Ve,lt,ee,It;return l=new Us({props:{title:"Training with 🤗 Accelerate",local:"training-with-accelerate",headingTag:"h2"}}),b=new Tl({props:{id:"Hm8_PgVTFuc"}}),j=new I({props:{code:"a2V5dG9rZW5faWRzJTIwJTNEJTIwJTVCJTVEJTBBZm9yJTIwa2V5d29yZCUyMGluJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIycGx0JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIycGQlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJzayUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmZpdCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMnByZWRpY3QlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjIlMjBwbHQlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjIlMjBwZCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMiUyMHNrJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyJTIwZml0JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyJTIwcHJlZGljdCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMnRlc3R0ZXN0JTIyJTJDJTBBJTVEJTNBJTBBJTIwJTIwJTIwJTIwaWRzJTIwJTNEJTIwdG9rZW5pemVyKCU1QmtleXdvcmQlNUQpLmlucHV0X2lkcyU1QjAlNUQlMEElMjAlMjAlMjAlMjBpZiUyMGxlbihpZHMpJTIwJTNEJTNEJTIwMSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGtleXRva2VuX2lkcy5hcHBlbmQoaWRzJTVCMCU1RCklMEElMjAlMjAlMjAlMjBlbHNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZiUyMktleXdvcmQlMjBoYXMlMjBub3QlMjBzaW5nbGUlMjB0b2tlbiUzQSUyMCU3QmtleXdvcmQlN0QlMjIp",highlighted:`keytoken_ids = []
<span class="hljs-keyword">for</span> keyword <span class="hljs-keyword">in</span> [
    <span class="hljs-string">&quot;plt&quot;</span>,
    <span class="hljs-string">&quot;pd&quot;</span>,
    <span class="hljs-string">&quot;sk&quot;</span>,
    <span class="hljs-string">&quot;fit&quot;</span>,
    <span class="hljs-string">&quot;predict&quot;</span>,
    <span class="hljs-string">&quot; plt&quot;</span>,
    <span class="hljs-string">&quot; pd&quot;</span>,
    <span class="hljs-string">&quot; sk&quot;</span>,
    <span class="hljs-string">&quot; fit&quot;</span>,
    <span class="hljs-string">&quot; predict&quot;</span>,
    <span class="hljs-string">&quot;testtest&quot;</span>,
]:
    ids = tokenizer([keyword]).input_ids[<span class="hljs-number">0</span>]
    <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(ids) == <span class="hljs-number">1</span>:
        keytoken_ids.append(ids[<span class="hljs-number">0</span>])
    <span class="hljs-keyword">else</span>:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Keyword has not single token: <span class="hljs-subst">{keyword}</span>&quot;</span>)`,wrap:!1}}),R=new I({props:{code:"J0tleXdvcmQlMjBoYXMlMjBub3QlMjBzaW5nbGUlMjB0b2tlbiUzQSUyMHRlc3R0ZXN0Jw==",highlighted:'<span class="hljs-string">&#x27;Keyword has not single token: testtest&#x27;</span>',wrap:!1}}),ke=new I({props:{code:"ZnJvbSUyMHRvcmNoLm5uJTIwaW1wb3J0JTIwQ3Jvc3NFbnRyb3B5TG9zcyUwQWltcG9ydCUyMHRvcmNoJTBBJTBBJTBBZGVmJTIwa2V5dG9rZW5fd2VpZ2h0ZWRfbG9zcyhpbnB1dHMlMkMlMjBsb2dpdHMlMkMlMjBrZXl0b2tlbl9pZHMlMkMlMjBhbHBoYSUzRDEuMCklM0ElMEElMjAlMjAlMjAlMjAlMjMlMjBTaGlmdCUyMHNvJTIwdGhhdCUyMHRva2VucyUyMCUzQyUyMG4lMjBwcmVkaWN0JTIwbiUwQSUyMCUyMCUyMCUyMHNoaWZ0X2xhYmVscyUyMCUzRCUyMGlucHV0cyU1Qi4uLiUyQyUyMDElM0ElNUQuY29udGlndW91cygpJTBBJTIwJTIwJTIwJTIwc2hpZnRfbG9naXRzJTIwJTNEJTIwbG9naXRzJTVCLi4uJTJDJTIwJTNBLTElMkMlMjAlM0ElNUQuY29udGlndW91cygpJTBBJTIwJTIwJTIwJTIwJTIzJTIwQ2FsY3VsYXRlJTIwcGVyLXRva2VuJTIwbG9zcyUwQSUyMCUyMCUyMCUyMGxvc3NfZmN0JTIwJTNEJTIwQ3Jvc3NFbnRyb3B5TG9zcyhyZWR1Y2UlM0RGYWxzZSklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mY3Qoc2hpZnRfbG9naXRzLnZpZXcoLTElMkMlMjBzaGlmdF9sb2dpdHMuc2l6ZSgtMSkpJTJDJTIwc2hpZnRfbGFiZWxzLnZpZXcoLTEpKSUwQSUyMCUyMCUyMCUyMCUyMyUyMFJlc2l6ZSUyMGFuZCUyMGF2ZXJhZ2UlMjBsb3NzJTIwcGVyJTIwc2FtcGxlJTBBJTIwJTIwJTIwJTIwbG9zc19wZXJfc2FtcGxlJTIwJTNEJTIwbG9zcy52aWV3KHNoaWZ0X2xvZ2l0cy5zaXplKDApJTJDJTIwc2hpZnRfbG9naXRzLnNpemUoMSkpLm1lYW4oYXhpcyUzRDEpJTBBJTIwJTIwJTIwJTIwJTIzJTIwQ2FsY3VsYXRlJTIwYW5kJTIwc2NhbGUlMjB3ZWlnaHRpbmclMEElMjAlMjAlMjAlMjB3ZWlnaHRzJTIwJTNEJTIwdG9yY2guc3RhY2soJTVCKGlucHV0cyUyMCUzRCUzRCUyMGt0KS5mbG9hdCgpJTIwZm9yJTIwa3QlMjBpbiUyMGtleXRva2VuX2lkcyU1RCkuc3VtKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGF4aXMlM0QlNUIwJTJDJTIwMiU1RCUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjB3ZWlnaHRzJTIwJTNEJTIwYWxwaGElMjAqJTIwKDEuMCUyMCUyQiUyMHdlaWdodHMpJTBBJTIwJTIwJTIwJTIwJTIzJTIwQ2FsY3VsYXRlJTIwd2VpZ2h0ZWQlMjBhdmVyYWdlJTBBJTIwJTIwJTIwJTIwd2VpZ2h0ZWRfbG9zcyUyMCUzRCUyMChsb3NzX3Blcl9zYW1wbGUlMjAqJTIwd2VpZ2h0cykubWVhbigpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwd2VpZ2h0ZWRfbG9zcw==",highlighted:`<span class="hljs-keyword">from</span> torch.nn <span class="hljs-keyword">import</span> CrossEntropyLoss
<span class="hljs-keyword">import</span> torch


<span class="hljs-keyword">def</span> <span class="hljs-title function_">keytoken_weighted_loss</span>(<span class="hljs-params">inputs, logits, keytoken_ids, alpha=<span class="hljs-number">1.0</span></span>):
    <span class="hljs-comment"># Shift so that tokens &lt; n predict n</span>
    shift_labels = inputs[..., <span class="hljs-number">1</span>:].contiguous()
    shift_logits = logits[..., :-<span class="hljs-number">1</span>, :].contiguous()
    <span class="hljs-comment"># Calculate per-token loss</span>
    loss_fct = CrossEntropyLoss(reduce=<span class="hljs-literal">False</span>)
    loss = loss_fct(shift_logits.view(-<span class="hljs-number">1</span>, shift_logits.size(-<span class="hljs-number">1</span>)), shift_labels.view(-<span class="hljs-number">1</span>))
    <span class="hljs-comment"># Resize and average loss per sample</span>
    loss_per_sample = loss.view(shift_logits.size(<span class="hljs-number">0</span>), shift_logits.size(<span class="hljs-number">1</span>)).mean(axis=<span class="hljs-number">1</span>)
    <span class="hljs-comment"># Calculate and scale weighting</span>
    weights = torch.stack([(inputs == kt).<span class="hljs-built_in">float</span>() <span class="hljs-keyword">for</span> kt <span class="hljs-keyword">in</span> keytoken_ids]).<span class="hljs-built_in">sum</span>(
        axis=[<span class="hljs-number">0</span>, <span class="hljs-number">2</span>]
    )
    weights = alpha * (<span class="hljs-number">1.0</span> + weights)
    <span class="hljs-comment"># Calculate weighted average</span>
    weighted_loss = (loss_per_sample * weights).mean()
    <span class="hljs-keyword">return</span> weighted_loss`,wrap:!1}}),Ge=new I({props:{code:"ZnJvbSUyMHRvcmNoLnV0aWxzLmRhdGEuZGF0YWxvYWRlciUyMGltcG9ydCUyMERhdGFMb2FkZXIlMEElMEF0b2tlbml6ZWRfZGF0YXNldHMuc2V0X2Zvcm1hdCglMjJ0b3JjaCUyMiklMEF0cmFpbl9kYXRhbG9hZGVyJTIwJTNEJTIwRGF0YUxvYWRlcih0b2tlbml6ZWRfZGF0YXNldHMlNUIlMjJ0cmFpbiUyMiU1RCUyQyUyMGJhdGNoX3NpemUlM0QzMiUyQyUyMHNodWZmbGUlM0RUcnVlKSUwQWV2YWxfZGF0YWxvYWRlciUyMCUzRCUyMERhdGFMb2FkZXIodG9rZW5pemVkX2RhdGFzZXRzJTVCJTIydmFsaWQlMjIlNUQlMkMlMjBiYXRjaF9zaXplJTNEMzIp",highlighted:`<span class="hljs-keyword">from</span> torch.utils.data.dataloader <span class="hljs-keyword">import</span> DataLoader

tokenized_datasets.set_format(<span class="hljs-string">&quot;torch&quot;</span>)
train_dataloader = DataLoader(tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>], batch_size=<span class="hljs-number">32</span>, shuffle=<span class="hljs-literal">True</span>)
eval_dataloader = DataLoader(tokenized_datasets[<span class="hljs-string">&quot;valid&quot;</span>], batch_size=<span class="hljs-number">32</span>)`,wrap:!1}}),Be=new I({props:{code:"d2VpZ2h0X2RlY2F5JTIwJTNEJTIwMC4xJTBBJTBBJTBBZGVmJTIwZ2V0X2dyb3VwZWRfcGFyYW1zKG1vZGVsJTJDJTIwbm9fZGVjYXklM0QlNUIlMjJiaWFzJTIyJTJDJTIwJTIyTGF5ZXJOb3JtLndlaWdodCUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjBwYXJhbXNfd2l0aF93ZCUyQyUyMHBhcmFtc193aXRob3V0X3dkJTIwJTNEJTIwJTVCJTVEJTJDJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwbiUyQyUyMHAlMjBpbiUyMG1vZGVsLm5hbWVkX3BhcmFtZXRlcnMoKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwYW55KG5kJTIwaW4lMjBuJTIwZm9yJTIwbmQlMjBpbiUyMG5vX2RlY2F5KSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBhcmFtc193aXRob3V0X3dkLmFwcGVuZChwKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwYXJhbXNfd2l0aF93ZC5hcHBlbmQocCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJwYXJhbXMlMjIlM0ElMjBwYXJhbXNfd2l0aF93ZCUyQyUyMCUyMndlaWdodF9kZWNheSUyMiUzQSUyMHdlaWdodF9kZWNheSU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMnBhcmFtcyUyMiUzQSUyMHBhcmFtc193aXRob3V0X3dkJTJDJTIwJTIyd2VpZ2h0X2RlY2F5JTIyJTNBJTIwMC4wJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTVE",highlighted:`weight_decay = <span class="hljs-number">0.1</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_grouped_params</span>(<span class="hljs-params">model, no_decay=[<span class="hljs-string">&quot;bias&quot;</span>, <span class="hljs-string">&quot;LayerNorm.weight&quot;</span>]</span>):
    params_with_wd, params_without_wd = [], []
    <span class="hljs-keyword">for</span> n, p <span class="hljs-keyword">in</span> model.named_parameters():
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">any</span>(nd <span class="hljs-keyword">in</span> n <span class="hljs-keyword">for</span> nd <span class="hljs-keyword">in</span> no_decay):
            params_without_wd.append(p)
        <span class="hljs-keyword">else</span>:
            params_with_wd.append(p)
    <span class="hljs-keyword">return</span> [
        {<span class="hljs-string">&quot;params&quot;</span>: params_with_wd, <span class="hljs-string">&quot;weight_decay&quot;</span>: weight_decay},
        {<span class="hljs-string">&quot;params&quot;</span>: params_without_wd, <span class="hljs-string">&quot;weight_decay&quot;</span>: <span class="hljs-number">0.0</span>},
    ]`,wrap:!1}}),A=new I({props:{code:"ZGVmJTIwZXZhbHVhdGUoKSUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsLmV2YWwoKSUwQSUyMCUyMCUyMCUyMGxvc3NlcyUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMHN0ZXAlMkMlMjBiYXRjaCUyMGluJTIwZW51bWVyYXRlKGV2YWxfZGF0YWxvYWRlciklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGJhdGNoJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTJDJTIwbGFiZWxzJTNEYmF0Y2glNUIlMjJpbnB1dF9pZHMlMjIlNUQpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbG9zc2VzLmFwcGVuZChhY2NlbGVyYXRvci5nYXRoZXIob3V0cHV0cy5sb3NzKSklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwdG9yY2gubWVhbih0b3JjaC5jYXQobG9zc2VzKSklMEElMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwZXJwbGV4aXR5JTIwJTNEJTIwdG9yY2guZXhwKGxvc3MpJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwT3ZlcmZsb3dFcnJvciUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBlcnBsZXhpdHklMjAlM0QlMjBmbG9hdCglMjJpbmYlMjIpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwbG9zcy5pdGVtKCklMkMlMjBwZXJwbGV4aXR5Lml0ZW0oKQ==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">evaluate</span>():
    model.<span class="hljs-built_in">eval</span>()
    losses = []
    <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(eval_dataloader):
        <span class="hljs-keyword">with</span> torch.no_grad():
            outputs = model(batch[<span class="hljs-string">&quot;input_ids&quot;</span>], labels=batch[<span class="hljs-string">&quot;input_ids&quot;</span>])

        losses.append(accelerator.gather(outputs.loss))
    loss = torch.mean(torch.cat(losses))
    <span class="hljs-keyword">try</span>:
        perplexity = torch.exp(loss)
    <span class="hljs-keyword">except</span> OverflowError:
        perplexity = <span class="hljs-built_in">float</span>(<span class="hljs-string">&quot;inf&quot;</span>)
    <span class="hljs-keyword">return</span> loss.item(), perplexity.item()`,wrap:!1}}),Y=new I({props:{code:"bW9kZWwlMjAlM0QlMjBHUFQyTE1IZWFkTW9kZWwoY29uZmlnKQ==",highlighted:"model = GPT2LMHeadModel(config)",wrap:!1}}),Xe=new I({props:{code:"ZnJvbSUyMHRvcmNoLm9wdGltJTIwaW1wb3J0JTIwQWRhbVclMEElMEFvcHRpbWl6ZXIlMjAlM0QlMjBBZGFtVyhnZXRfZ3JvdXBlZF9wYXJhbXMobW9kZWwpJTJDJTIwbHIlM0Q1ZS00KQ==",highlighted:`<span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW

optimizer = AdamW(get_grouped_params(model), lr=<span class="hljs-number">5e-4</span>)`,wrap:!1}}),Re=new I({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoZnAxNiUzRFRydWUpJTBBJTBBbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjB0cmFpbl9kYXRhbG9hZGVyJTJDJTIwZXZhbF9kYXRhbG9hZGVyJTIwJTNEJTIwYWNjZWxlcmF0b3IucHJlcGFyZSglMEElMjAlMjAlMjAlMjBtb2RlbCUyQyUyMG9wdGltaXplciUyQyUyMHRyYWluX2RhdGFsb2FkZXIlMkMlMjBldmFsX2RhdGFsb2FkZXIlMEEp",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator(fp16=<span class="hljs-literal">True</span>)

model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`,wrap:!1}}),q=new ct({props:{$$slots:{default:[Qa]},$$scope:{ctx:C}}}),We=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGdldF9zY2hlZHVsZXIlMEElMEFudW1fdHJhaW5fZXBvY2hzJTIwJTNEJTIwMSUwQW51bV91cGRhdGVfc3RlcHNfcGVyX2Vwb2NoJTIwJTNEJTIwbGVuKHRyYWluX2RhdGFsb2FkZXIpJTBBbnVtX3RyYWluaW5nX3N0ZXBzJTIwJTNEJTIwbnVtX3RyYWluX2Vwb2NocyUyMColMjBudW1fdXBkYXRlX3N0ZXBzX3Blcl9lcG9jaCUwQSUwQWxyX3NjaGVkdWxlciUyMCUzRCUyMGdldF9zY2hlZHVsZXIoJTBBJTIwJTIwJTIwJTIwbmFtZSUzRCUyMmxpbmVhciUyMiUyQyUwQSUyMCUyMCUyMCUyMG9wdGltaXplciUzRG9wdGltaXplciUyQyUwQSUyMCUyMCUyMCUyMG51bV93YXJtdXBfc3RlcHMlM0QxXzAwMCUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbmluZ19zdGVwcyUzRG51bV90cmFpbmluZ19zdGVwcyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

num_train_epochs = <span class="hljs-number">1</span>
num_update_steps_per_epoch = <span class="hljs-built_in">len</span>(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    name=<span class="hljs-string">&quot;linear&quot;</span>,
    optimizer=optimizer,
    num_warmup_steps=<span class="hljs-number">1_000</span>,
    num_training_steps=num_training_steps,
)`,wrap:!1}}),K=new I({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMFJlcG9zaXRvcnklMkMlMjBnZXRfZnVsbF9yZXBvX25hbWUlMEElMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyY29kZXBhcnJvdC1kcy1hY2NlbGVyYXRlJTIyJTBBcmVwb19uYW1lJTIwJTNEJTIwZ2V0X2Z1bGxfcmVwb19uYW1lKG1vZGVsX25hbWUpJTBBcmVwb19uYW1l",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository, get_full_repo_name

model_name = <span class="hljs-string">&quot;codeparrot-ds-accelerate&quot;</span>
repo_name = get_full_repo_name(model_name)
repo_name`,wrap:!1}}),P=new I({props:{code:"J3NndWdnZXIlMkZjb2RlcGFycm90LWRzLWFjY2VsZXJhdGUn",highlighted:'<span class="hljs-string">&#x27;sgugger/codeparrot-ds-accelerate&#x27;</span>',wrap:!1}}),ve=new I({props:{code:"b3V0cHV0X2RpciUyMCUzRCUyMCUyMmNvZGVwYXJyb3QtZHMtYWNjZWxlcmF0ZSUyMiUwQXJlcG8lMjAlM0QlMjBSZXBvc2l0b3J5KG91dHB1dF9kaXIlMkMlMjBjbG9uZV9mcm9tJTNEcmVwb19uYW1lKQ==",highlighted:`output_dir = <span class="hljs-string">&quot;codeparrot-ds-accelerate&quot;</span>
repo = Repository(output_dir, clone_from=repo_name)`,wrap:!1}}),ce=new I({props:{code:"ZXZhbHVhdGUoKQ==",highlighted:"evaluate()",wrap:!1}}),me=new I({props:{code:"KDEwLjkzNDEyNjg1Mzk0Mjg3MSUyQyUyMDU2MDU3LjE0NDUzMTI1KQ==",highlighted:'(<span class="hljs-number">10.934126853942871</span>, <span class="hljs-number">56057.14453125</span>)',wrap:!1}}),he=new I({props:{code:"ZnJvbSUyMHRxZG0ubm90ZWJvb2slMjBpbXBvcnQlMjB0cWRtJTBBJTBBZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIwJTNEJTIwOCUwQWV2YWxfc3RlcHMlMjAlM0QlMjA1XzAwMCUwQSUwQW1vZGVsLnRyYWluKCklMEFjb21wbGV0ZWRfc3RlcHMlMjAlM0QlMjAwJTBBZm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKG51bV90cmFpbl9lcG9jaHMpJTNBJTBBJTIwJTIwJTIwJTIwZm9yJTIwc3RlcCUyQyUyMGJhdGNoJTIwaW4lMjB0cWRtKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVudW1lcmF0ZSh0cmFpbl9kYXRhbG9hZGVyJTJDJTIwc3RhcnQlM0QxKSUyQyUyMHRvdGFsJTNEbnVtX3RyYWluaW5nX3N0ZXBzJTBBJTIwJTIwJTIwJTIwKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvZ2l0cyUyMCUzRCUyMG1vZGVsKGJhdGNoJTVCJTIyaW5wdXRfaWRzJTIyJTVEKS5sb2dpdHMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwa2V5dG9rZW5fd2VpZ2h0ZWRfbG9zcyhiYXRjaCU1QiUyMmlucHV0X2lkcyUyMiU1RCUyQyUyMGxvZ2l0cyUyQyUyMGtleXRva2VuX2lkcyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHN0ZXAlMjAlMjUlMjAxMDAlMjAlM0QlM0QlMjAwJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IucHJpbnQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyc2FtcGxlcyUyMiUzQSUyMHN0ZXAlMjAqJTIwc2FtcGxlc19wZXJfc3RlcCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0ZXBzJTIyJTNBJTIwY29tcGxldGVkX3N0ZXBzJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybG9zcyUyRnRyYWluJTIyJTNBJTIwbG9zcy5pdGVtKCklMjAqJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvc3MlMjAlM0QlMjBsb3NzJTIwJTJGJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IuYmFja3dhcmQobG9zcyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHN0ZXAlMjAlMjUlMjBncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlMjAlM0QlM0QlMjAwJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IuY2xpcF9ncmFkX25vcm1fKG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMDEuMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplci56ZXJvX2dyYWQoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbXBsZXRlZF9zdGVwcyUyMCUyQiUzRCUyMDElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMChzdGVwJTIwJTI1JTIwKGV2YWxfc3RlcHMlMjAqJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzKSklMjAlM0QlM0QlMjAwJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXZhbF9sb3NzJTJDJTIwcGVycGxleGl0eSUyMCUzRCUyMGV2YWx1YXRlKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhY2NlbGVyYXRvci5wcmludCglN0IlMjJsb3NzJTJGZXZhbCUyMiUzQSUyMGV2YWxfbG9zcyUyQyUyMCUyMnBlcnBsZXhpdHklMjIlM0ElMjBwZXJwbGV4aXR5JTdEKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1vZGVsLnRyYWluKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhY2NlbGVyYXRvci53YWl0X2Zvcl9ldmVyeW9uZSgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdW53cmFwcGVkX21vZGVsJTIwJTNEJTIwYWNjZWxlcmF0b3IudW53cmFwX21vZGVsKG1vZGVsKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHVud3JhcHBlZF9tb2RlbC5zYXZlX3ByZXRyYWluZWQob3V0cHV0X2RpciUyQyUyMHNhdmVfZnVuY3Rpb24lM0RhY2NlbGVyYXRvci5zYXZlKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwYWNjZWxlcmF0b3IuaXNfbWFpbl9wcm9jZXNzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdG9rZW5pemVyLnNhdmVfcHJldHJhaW5lZChvdXRwdXRfZGlyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlcG8ucHVzaF90b19odWIoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY29tbWl0X21lc3NhZ2UlM0RmJTIyVHJhaW5pbmclMjBpbiUyMHByb2dyZXNzJTIwc3RlcCUyMCU3QnN0ZXAlN0QlMjIlMkMlMjBibG9ja2luZyUzREZhbHNlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKQ==",highlighted:`<span class="hljs-keyword">from</span> tqdm.notebook <span class="hljs-keyword">import</span> tqdm

gradient_accumulation_steps = <span class="hljs-number">8</span>
eval_steps = <span class="hljs-number">5_000</span>

model.train()
completed_steps = <span class="hljs-number">0</span>
<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_train_epochs):
    <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> tqdm(
        <span class="hljs-built_in">enumerate</span>(train_dataloader, start=<span class="hljs-number">1</span>), total=num_training_steps
    ):
        logits = model(batch[<span class="hljs-string">&quot;input_ids&quot;</span>]).logits
        loss = keytoken_weighted_loss(batch[<span class="hljs-string">&quot;input_ids&quot;</span>], logits, keytoken_ids)
        <span class="hljs-keyword">if</span> step % <span class="hljs-number">100</span> == <span class="hljs-number">0</span>:
            accelerator.<span class="hljs-built_in">print</span>(
                {
                    <span class="hljs-string">&quot;samples&quot;</span>: step * samples_per_step,
                    <span class="hljs-string">&quot;steps&quot;</span>: completed_steps,
                    <span class="hljs-string">&quot;loss/train&quot;</span>: loss.item() * gradient_accumulation_steps,
                }
            )
        loss = loss / gradient_accumulation_steps
        accelerator.backward(loss)
        <span class="hljs-keyword">if</span> step % gradient_accumulation_steps == <span class="hljs-number">0</span>:
            accelerator.clip_grad_norm_(model.parameters(), <span class="hljs-number">1.0</span>)
            optimizer.step()
            lr_scheduler.step()
            optimizer.zero_grad()
            completed_steps += <span class="hljs-number">1</span>
        <span class="hljs-keyword">if</span> (step % (eval_steps * gradient_accumulation_steps)) == <span class="hljs-number">0</span>:
            eval_loss, perplexity = evaluate()
            accelerator.<span class="hljs-built_in">print</span>({<span class="hljs-string">&quot;loss/eval&quot;</span>: eval_loss, <span class="hljs-string">&quot;perplexity&quot;</span>: perplexity})
            model.train()
            accelerator.wait_for_everyone()
            unwrapped_model = accelerator.unwrap_model(model)
            unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
            <span class="hljs-keyword">if</span> accelerator.is_main_process:
                tokenizer.save_pretrained(output_dir)
                repo.push_to_hub(
                    commit_message=<span class="hljs-string">f&quot;Training in progress step <span class="hljs-subst">{step}</span>&quot;</span>, blocking=<span class="hljs-literal">False</span>
                )`,wrap:!1}}),Ve=new ct({props:{$$slots:{default:[za]},$$scope:{ctx:C}}}),ee=new ct({props:{$$slots:{default:[Na]},$$scope:{ctx:C}}}),{c(){w(l.$$.fragment),r=i(),n=T("p"),n.innerHTML=y,g=i(),w(b.$$.fragment),$=i(),k=T("p"),k.innerHTML=Z,J=i(),w(j.$$.fragment),G=i(),w(R.$$.fragment),X=i(),N=T("p"),N.textContent=_,B=i(),w(ke.$$.fragment),Ee=i(),V=T("p"),V.textContent=yt,se=i(),Ze=T("ul"),Ze.innerHTML=mt,Q=i(),$e=T("p"),$e.innerHTML=wt,le=i(),w(Ge.$$.fragment),Ae=i(),x=T("p"),x.textContent=ut,ae=i(),w(Be.$$.fragment),Fe=i(),E=T("p"),E.textContent=fs,Ye=i(),w(A.$$.fragment),He=i(),F=T("p"),F.innerHTML=js,Se=i(),w(Y.$$.fragment),qe=i(),H=T("p"),H.textContent=ht,ne=i(),w(Xe.$$.fragment),De=i(),S=T("p"),S.textContent=dt,oe=i(),w(Re.$$.fragment),Le=i(),w(q.$$.fragment),Ke=i(),D=T("p"),D.innerHTML=Jt,ie=i(),w(We.$$.fragment),Pe=i(),L=T("p"),L.innerHTML=bs,Oe=i(),w(K.$$.fragment),et=i(),w(P.$$.fragment),tt=i(),z=T("p"),z.textContent=Tt,pe=i(),w(ve.$$.fragment),st=i(),O=T("p"),O.innerHTML=Ut,re=i(),Me=T("p"),Me.textContent=Qe,ft=i(),w(ce.$$.fragment),ye=i(),w(me.$$.fragment),we=i(),ue=T("p"),ue.innerHTML=ze,jt=i(),w(he.$$.fragment),de=i(),Ne=T("p"),Ne.textContent=bt,te=i(),w(Ve.$$.fragment),lt=i(),w(ee.$$.fragment)},l(t){u(l.$$.fragment,t),r=p(t),n=U(t,"P",{"data-svelte-h":!0}),f(n)!=="svelte-qf6luh"&&(n.innerHTML=y),g=p(t),u(b.$$.fragment,t),$=p(t),k=U(t,"P",{"data-svelte-h":!0}),f(k)!=="svelte-dv4zfi"&&(k.innerHTML=Z),J=p(t),u(j.$$.fragment,t),G=p(t),u(R.$$.fragment,t),X=p(t),N=U(t,"P",{"data-svelte-h":!0}),f(N)!=="svelte-1xv4c43"&&(N.textContent=_),B=p(t),u(ke.$$.fragment,t),Ee=p(t),V=U(t,"P",{"data-svelte-h":!0}),f(V)!=="svelte-1gmwrn7"&&(V.textContent=yt),se=p(t),Ze=U(t,"UL",{"data-svelte-h":!0}),f(Ze)!=="svelte-zoq68e"&&(Ze.innerHTML=mt),Q=p(t),$e=U(t,"P",{"data-svelte-h":!0}),f($e)!=="svelte-qghkub"&&($e.innerHTML=wt),le=p(t),u(Ge.$$.fragment,t),Ae=p(t),x=U(t,"P",{"data-svelte-h":!0}),f(x)!=="svelte-asmh6m"&&(x.textContent=ut),ae=p(t),u(Be.$$.fragment,t),Fe=p(t),E=U(t,"P",{"data-svelte-h":!0}),f(E)!=="svelte-ig8se8"&&(E.textContent=fs),Ye=p(t),u(A.$$.fragment,t),He=p(t),F=U(t,"P",{"data-svelte-h":!0}),f(F)!=="svelte-11o4vzt"&&(F.innerHTML=js),Se=p(t),u(Y.$$.fragment,t),qe=p(t),H=U(t,"P",{"data-svelte-h":!0}),f(H)!=="svelte-1n65zke"&&(H.textContent=ht),ne=p(t),u(Xe.$$.fragment,t),De=p(t),S=U(t,"P",{"data-svelte-h":!0}),f(S)!=="svelte-yqp0m0"&&(S.textContent=dt),oe=p(t),u(Re.$$.fragment,t),Le=p(t),u(q.$$.fragment,t),Ke=p(t),D=U(t,"P",{"data-svelte-h":!0}),f(D)!=="svelte-1409oc7"&&(D.innerHTML=Jt),ie=p(t),u(We.$$.fragment,t),Pe=p(t),L=U(t,"P",{"data-svelte-h":!0}),f(L)!=="svelte-ppazi4"&&(L.innerHTML=bs),Oe=p(t),u(K.$$.fragment,t),et=p(t),u(P.$$.fragment,t),tt=p(t),z=U(t,"P",{"data-svelte-h":!0}),f(z)!=="svelte-lt5x8r"&&(z.textContent=Tt),pe=p(t),u(ve.$$.fragment,t),st=p(t),O=U(t,"P",{"data-svelte-h":!0}),f(O)!=="svelte-1mcbsiy"&&(O.innerHTML=Ut),re=p(t),Me=U(t,"P",{"data-svelte-h":!0}),f(Me)!=="svelte-983pzc"&&(Me.textContent=Qe),ft=p(t),u(ce.$$.fragment,t),ye=p(t),u(me.$$.fragment,t),we=p(t),ue=U(t,"P",{"data-svelte-h":!0}),f(ue)!=="svelte-9b8c5q"&&(ue.innerHTML=ze),jt=p(t),u(he.$$.fragment,t),de=p(t),Ne=U(t,"P",{"data-svelte-h":!0}),f(Ne)!=="svelte-1sn049p"&&(Ne.textContent=bt),te=p(t),u(Ve.$$.fragment,t),lt=p(t),u(ee.$$.fragment,t)},m(t,m){h(l,t,m),a(t,r,m),a(t,n,m),a(t,g,m),h(b,t,m),a(t,$,m),a(t,k,m),a(t,J,m),h(j,t,m),a(t,G,m),h(R,t,m),a(t,X,m),a(t,N,m),a(t,B,m),h(ke,t,m),a(t,Ee,m),a(t,V,m),a(t,se,m),a(t,Ze,m),a(t,Q,m),a(t,$e,m),a(t,le,m),h(Ge,t,m),a(t,Ae,m),a(t,x,m),a(t,ae,m),h(Be,t,m),a(t,Fe,m),a(t,E,m),a(t,Ye,m),h(A,t,m),a(t,He,m),a(t,F,m),a(t,Se,m),h(Y,t,m),a(t,qe,m),a(t,H,m),a(t,ne,m),h(Xe,t,m),a(t,De,m),a(t,S,m),a(t,oe,m),h(Re,t,m),a(t,Le,m),h(q,t,m),a(t,Ke,m),a(t,D,m),a(t,ie,m),h(We,t,m),a(t,Pe,m),a(t,L,m),a(t,Oe,m),h(K,t,m),a(t,et,m),h(P,t,m),a(t,tt,m),a(t,z,m),a(t,pe,m),h(ve,t,m),a(t,st,m),a(t,O,m),a(t,re,m),a(t,Me,m),a(t,ft,m),h(ce,t,m),a(t,ye,m),h(me,t,m),a(t,we,m),a(t,ue,m),a(t,jt,m),h(he,t,m),a(t,de,m),a(t,Ne,m),a(t,te,m),h(Ve,t,m),a(t,lt,m),h(ee,t,m),It=!0},i(t){It||(M(l.$$.fragment,t),M(b.$$.fragment,t),M(j.$$.fragment,t),M(R.$$.fragment,t),M(ke.$$.fragment,t),M(Ge.$$.fragment,t),M(Be.$$.fragment,t),M(A.$$.fragment,t),M(Y.$$.fragment,t),M(Xe.$$.fragment,t),M(Re.$$.fragment,t),M(q.$$.fragment,t),M(We.$$.fragment,t),M(K.$$.fragment,t),M(P.$$.fragment,t),M(ve.$$.fragment,t),M(ce.$$.fragment,t),M(me.$$.fragment,t),M(he.$$.fragment,t),M(Ve.$$.fragment,t),M(ee.$$.fragment,t),It=!0)},o(t){c(l.$$.fragment,t),c(b.$$.fragment,t),c(j.$$.fragment,t),c(R.$$.fragment,t),c(ke.$$.fragment,t),c(Ge.$$.fragment,t),c(Be.$$.fragment,t),c(A.$$.fragment,t),c(Y.$$.fragment,t),c(Xe.$$.fragment,t),c(Re.$$.fragment,t),c(q.$$.fragment,t),c(We.$$.fragment,t),c(K.$$.fragment,t),c(P.$$.fragment,t),c(ve.$$.fragment,t),c(ce.$$.fragment,t),c(me.$$.fragment,t),c(he.$$.fragment,t),c(Ve.$$.fragment,t),c(ee.$$.fragment,t),It=!1},d(t){t&&(s(r),s(n),s(g),s($),s(k),s(J),s(G),s(X),s(N),s(B),s(Ee),s(V),s(se),s(Ze),s(Q),s($e),s(le),s(Ae),s(x),s(ae),s(Fe),s(E),s(Ye),s(He),s(F),s(Se),s(qe),s(H),s(ne),s(De),s(S),s(oe),s(Le),s(Ke),s(D),s(ie),s(Pe),s(L),s(Oe),s(et),s(tt),s(z),s(pe),s(st),s(O),s(re),s(Me),s(ft),s(ye),s(we),s(ue),s(jt),s(de),s(Ne),s(te),s(lt)),d(l,t),d(b,t),d(j,t),d(R,t),d(ke,t),d(Ge,t),d(Be,t),d(A,t),d(Y,t),d(Xe,t),d(Re,t),d(q,t),d(We,t),d(K,t),d(P,t),d(ve,t),d(ce,t),d(me,t),d(he,t),d(Ve,t),d(ee,t)}}}function Qa(C){let l,r='🚨 If you’re training on a TPU, you’ll need to move all the code starting at the cell above into a dedicated training function. See <a href="/course/chapter3">Chapter 3</a> for more details.';return{c(){l=T("p"),l.innerHTML=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-185qbwt"&&(l.innerHTML=r)},m(n,y){a(n,l,y)},p:cs,d(n){n&&s(l)}}}function za(C){let l,r="✏️ <strong>Try it out!</strong> Either create your own custom loss function tailored to your use case, or add another custom step into the training loop.";return{c(){l=T("p"),l.innerHTML=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-mw9suu"&&(l.innerHTML=r)},m(n,y){a(n,l,y)},p:cs,d(n){n&&s(l)}}}function Na(C){let l,r="✏️ <strong>Try it out!</strong> When running long training experiments it’s a good idea to log important metrics using tools such as TensorBoard or Weights &amp; Biases. Add proper logging to the training loop so you can always check how the training is going.";return{c(){l=T("p"),l.innerHTML=r},l(n){l=U(n,"P",{"data-svelte-h":!0}),f(l)!=="svelte-gp1epm"&&(l.innerHTML=r)},m(n,y){a(n,l,y)},p:cs,d(n){n&&s(l)}}}function Va(C){let l,r,n,y,g,b,$,k,Z,J,j,G,R,X,N='Up until now, we’ve mostly been using pretrained models and fine-tuning them for new use cases by reusing the weights from pretraining. As we saw in <a href="/course/chapter1">Chapter 1</a>, this is commonly referred to as <em>transfer learning</em>, and it’s a very successful strategy for applying Transformer models to most real-world use cases where labeled data is sparse. In this chapter, we’ll take a different approach and train a completely new model from scratch. This is a good approach to take if you have a lot of data and it is very different from the pretraining data used for the available models. However, it also requires considerably more compute resources to pretrain a language model than just to fine-tune an existing one. Examples where it can make sense to train a new model include for datasets consisting of musical notes, molecular sequences such as DNA, or programming languages. The latter have recently gained traction thanks to tools such as TabNine and GitHub’s Copilot, powered by OpenAI’s Codex model, that can generate long sequences of code. This task of text generation is best addressed with auto-regressive or causal language models such as GPT-2.',_,B,ke="In this section we will build a scaled-down version of a code generation model: we’ll focus on one-line completions instead of full functions or classes, using a subset of Python code. When working with data in Python you are in frequent contact with the Python data science stack, consisting of the <code>matplotlib</code>, <code>seaborn</code>, <code>pandas</code>, and <code>scikit-learn</code> libraries. When using those frameworks it’s common to need to look up specific commands, so it would be nice if we could use a model to complete these calls for us.",Ee,V,yt,se,Ze='In <a href="/course/chapter6">Chapter 6</a> we created an efficient tokenizer to process Python source code, but what we still need is a large-scale dataset to pretrain a model on. Here, we’ll apply our tokenizer to a corpus of Python code derived from GitHub repositories. We will then use the <code>Trainer</code> API and 🤗 Accelerate to train the model. Let’s get to it!',mt,Q,$e,wt,le,Ge='This is actually showcasing the model that was trained and uploaded to the Hub using the code shown in this section. You can find it <a href="https://huggingface.co/huggingface-course/codeparrot-ds?text=plt.imshow%28" rel="nofollow">here</a>. Note that since there is some randomization happening in the text generation, you will probably get a slightly different result.',Ae,x,ut,ae,Be='Python code is abundantly available from code repositories such as GitHub, which we can use to create a dataset by scraping for every Python repository. This was the approach taken in the <a href="https://learning.oreilly.com/library/view/natural-language-processing/9781098136789/" rel="nofollow">Transformers textbook</a> to pretrain a large GPT-2 model. Using a GitHub dump of about 180 GB containing roughly 20 million Python files called <code>codeparrot</code>, the authors built a dataset that they then shared on the <a href="https://huggingface.co/datasets/transformersbook/codeparrot" rel="nofollow">Hugging Face Hub</a>.',Fe,E,fs="However, training on the full corpus is time- and compute-consuming, and we only need the subset of the dataset concerned with the Python data science stack. So, let’s start by filtering the <code>codeparrot</code> dataset for all files that include any of the libraries in this stack. Because of the dataset’s size, we want to avoid downloading it; instead, we’ll use the streaming feature to filter it on the fly. To help us filter the code samples using the libraries we mentioned earlier, we’ll use the following function:",Ye,A,He,F,js="Let’s test it on two examples:",Se,Y,qe,H,ht,ne,Xe="We can use this to create a function that will stream the dataset and filter the elements we want:",De,S,dt,oe,Re="Then we can simply apply this function to the streaming dataset:",Le,q,Ke,D,Jt,ie,We="This leaves us with about 3% of the original dataset, which is still quite sizable — the resulting dataset is 6 GB and consists of 600,000 Python scripts!",Pe,L,bs="Filtering the full dataset can take 2-3h depending on your machine and bandwidth. If you don’t want to go through this lengthy process yourself, we provide the filtered dataset on the Hub for you to download:",Oe,K,et,P,tt,z,Tt,pe,ve="Let’s look at an example from the dataset. We’ll just show the first 200 characters of each field:",st,O,Ut,re,Me,Qe,ft="We can see that the <code>content</code> field contains the code that we want our model to train on. Now that we have a dataset, we need to prepare the texts so they’re in a format suitable for pretraining.",ce,ye,me,we,ue,ze,jt="The first step will be to tokenize the data, so we can use it for training. Since our goal is to mainly autocomplete short function calls, we can keep the context size relatively small. This has the benefit that we can train the model much faster and it requires significantly less memory. If it is important for your application to have more context (for example, if you want the model to write unit tests based on a file with the function definition), make sure you increase that number, but also keep in mind that this comes with a greater GPU memory footprint. For now, let’s fix the context size at 128 tokens, as opposed to the 1,024 or 2,048 used in GPT-2 or GPT-3, respectively.",he,de,Ne='Most documents contain many more than 128 tokens, so simply truncating the inputs to the maximum length would eliminate a large fraction of our dataset. Instead, we’ll use the <code>return_overflowing_tokens</code> option to tokenize the whole input and split it into several chunks, as we did in <a href="/course/chapter6/4">Chapter 6</a>. We’ll also use the <code>return_length</code> option to return the length of each created chunk automatically. Often the last chunk will be smaller than the context size, and we’ll get rid of these pieces to avoid padding issues; we don’t really need them as we have plenty of data anyway.',bt,te,Ve='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/chunking_texts.svg" alt="Chunking a large texts in several pieces."/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/chunking_texts-dark.svg" alt="Chunking a large texts in several pieces."/>',lt,ee,It="Let’s see exactly how this works by looking at the first two examples:",t,m,Cs,Ct,gs,gt,Ul="We can see that we get 34 segments in total from those two examples. Looking at the chunk lengths, we can see that the chunks at the ends of both documents have less than 128 tokens (117 and 41, respectively). These represent just a small fraction of the total chunks that we have, so we can safely throw them away. With the <code>overflow_to_sample_mapping</code> field, we can also reconstruct which chunks belonged to which input samples.",_s,_t,fl='With this operation we’re using a handy feature of the <code>Dataset.map()</code> function in 🤗 Datasets, which is that it does not require one-to-one maps; as we saw in <a href="/course/chapter7/3">section 3</a>, we can create batches with more or fewer elements than the input batch. This is useful when doing operations like data augmentation or data filtering that change the number of elements. In our case, when tokenizing each element into chunks of the specified context size, we create many samples from each document. We just need to make sure to delete the existing columns, since they have a conflicting size. If we wanted to keep them, we could repeat them appropriately and return them within the <code>Dataset.map()</code> call:',ks,kt,Zs,Zt,$s,$t,jl="We now have 16.7 million examples with 128 tokens each, which corresponds to about 2.1 billion tokens in total. For reference, OpenAI’s GPT-3 and Codex models are trained on 300 and 100 billion tokens, respectively, where the Codex models are initialized from the GPT-3 checkpoints. Our goal in this section is not to compete with these models, which can generate long, coherent texts, but to create a scaled-down version providing a quick autocomplete function for data scientists.",Gs,Gt,bl="Now that we have the dataset ready, let’s set up the model!",Bs,nt,Xs,Bt,Rs,Xt,Il="Our first step is to freshly initialize a GPT-2 model. We’ll use the same configuration for our model as for the small GPT-2 model, so we load the pretrained configuration, make sure that the tokenizer size matches the model vocabulary size and pass the <code>bos</code> and <code>eos</code> (beginning and end of sequence) token IDs:",Ws,Je,Te,ys,Rt,Cl="Our model has 124M parameters that we’ll have to tune. Before we can start training, we need to set up a data collator that will take care of creating the batches. We can use the <code>DataCollatorForLanguageModeling</code> collator, which is designed specifically for language modeling (as the name subtly suggests). Besides stacking and padding batches, it also takes care of creating the language model labels — in causal language modeling the inputs serve as labels too (just shifted by one element), and this data collator creates them on the fly during training so we don’t need to duplicate the <code>input_ids</code>.",vs,Wt,gl="Note that <code>DataCollatorForLanguageModeling</code> supports both masked language modeling (MLM) and causal language modeling (CLM). By default it prepares data for MLM, but we can switch to CLM by setting the argument <code>mlm=False</code>:",Qs,Ue,fe,ms,vt,_l="Let’s have a look at an example:",zs,Qt,Ns,je,be,ws,zt,kl="We can see that the examples have been stacked and all the tensors have the same shape.",Vs,us,ot,xs,Nt,Zl="Now we have everything in place to actually train our model — that wasn’t so much work after all! Before we start training we should log in to Hugging Face. If you’re working in a notebook, you can do so with the following utility function:",Es,Vt,As,xt,$l="This will display a widget where you can enter your Hugging Face login credentials.",Fs,Et,Gl="If you aren’t working in a notebook, just type the following line in your terminal:",Ys,At,Hs,Ie,Ce,hs,it,Ss,pt,qs,Ft,Ds,Yt,Bl="Now is the moment of truth: let’s see how well the trained model actually works! We can see in the logs that the loss went down steadily, but to put the model to the test let’s take a look at how well it works on some prompts. To do that we’ll wrap the model in a text generation <code>pipeline</code>, and we’ll put it on the GPU for fast generations if there is one available:",Ls,ge,_e,ds,Ht,Xl="Let’s start with the simple task of creating a scatter plot:",Ks,St,Ps,qt,Os,Dt,Rl="The result looks correct. Does it also work for a <code>pandas</code> operation? Let’s see if we can create a <code>DataFrame</code> from two arrays:",el,Lt,tl,Kt,sl,Pt,Wl="Nice, that’s the correct answer — although it then inserts the column <code>x</code> again. Since the number of generated tokens is limited, the following <code>for</code> loop is cut off. Let’s see if we can do something a bit more complex and have the model help us use the <code>groupby</code> operation:",ll,Ot,al,es,nl,ts,vl="Not bad; that’s the right way to do it. Finally, let’s see if we can also use it for <code>scikit-learn</code> and set up a Random Forest model:",ol,ss,il,ls,pl,Js,Ts,as,rl,Is,Ml;g=new ua({props:{fw:C[0]}}),$=new ma({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),Z=new Us({props:{title:"Training a causal language model from scratch",local:"training-a-causal-language-model-from-scratch",headingTag:"h1"}});const Ql=[da,ha],ns=[];function zl(e,o){return e[0]==="pt"?0:1}j=zl(C),G=ns[j]=Ql[j](C),V=new Tl({props:{id:"Vpjb1lu0MDk"}}),x=new Us({props:{title:"Gathering the data",local:"gathering-the-data",headingTag:"h2"}}),A=new I({props:{code:"ZGVmJTIwYW55X2tleXdvcmRfaW5fc3RyaW5nKHN0cmluZyUyQyUyMGtleXdvcmRzKSUzQSUwQSUyMCUyMCUyMCUyMGZvciUyMGtleXdvcmQlMjBpbiUyMGtleXdvcmRzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBrZXl3b3JkJTIwaW4lMjBzdHJpbmclM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBUcnVlJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwRmFsc2U=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">any_keyword_in_string</span>(<span class="hljs-params">string, keywords</span>):
    <span class="hljs-keyword">for</span> keyword <span class="hljs-keyword">in</span> keywords:
        <span class="hljs-keyword">if</span> keyword <span class="hljs-keyword">in</span> string:
            <span class="hljs-keyword">return</span> <span class="hljs-literal">True</span>
    <span class="hljs-keyword">return</span> <span class="hljs-literal">False</span>`,wrap:!1}}),Y=new I({props:{code:"ZmlsdGVycyUyMCUzRCUyMCU1QiUyMnBhbmRhcyUyMiUyQyUyMCUyMnNrbGVhcm4lMjIlMkMlMjAlMjJtYXRwbG90bGliJTIyJTJDJTIwJTIyc2VhYm9ybiUyMiU1RCUwQWV4YW1wbGVfMSUyMCUzRCUyMCUyMmltcG9ydCUyMG51bXB5JTIwYXMlMjBucCUyMiUwQWV4YW1wbGVfMiUyMCUzRCUyMCUyMmltcG9ydCUyMHBhbmRhcyUyMGFzJTIwcGQlMjIlMEElMEFwcmludCglMEElMjAlMjAlMjAlMjBhbnlfa2V5d29yZF9pbl9zdHJpbmcoZXhhbXBsZV8xJTJDJTIwZmlsdGVycyklMkMlMjBhbnlfa2V5d29yZF9pbl9zdHJpbmcoZXhhbXBsZV8yJTJDJTIwZmlsdGVycyklMEEp",highlighted:`filters = [<span class="hljs-string">&quot;pandas&quot;</span>, <span class="hljs-string">&quot;sklearn&quot;</span>, <span class="hljs-string">&quot;matplotlib&quot;</span>, <span class="hljs-string">&quot;seaborn&quot;</span>]
example_1 = <span class="hljs-string">&quot;import numpy as np&quot;</span>
example_2 = <span class="hljs-string">&quot;import pandas as pd&quot;</span>

<span class="hljs-built_in">print</span>(
    any_keyword_in_string(example_1, filters), any_keyword_in_string(example_2, filters)
)`,wrap:!1}}),H=new I({props:{code:"RmFsc2UlMjBUcnVl",highlighted:'<span class="hljs-literal">False</span> <span class="hljs-literal">True</span>',wrap:!1}}),S=new I({props:{code:"ZnJvbSUyMGNvbGxlY3Rpb25zJTIwaW1wb3J0JTIwZGVmYXVsdGRpY3QlMEFmcm9tJTIwdHFkbSUyMGltcG9ydCUyMHRxZG0lMEFmcm9tJTIwZGF0YXNldHMlMjBpbXBvcnQlMjBEYXRhc2V0JTBBJTBBJTBBZGVmJTIwZmlsdGVyX3N0cmVhbWluZ19kYXRhc2V0KGRhdGFzZXQlMkMlMjBmaWx0ZXJzKSUzQSUwQSUyMCUyMCUyMCUyMGZpbHRlcmVkX2RpY3QlMjAlM0QlMjBkZWZhdWx0ZGljdChsaXN0KSUwQSUyMCUyMCUyMCUyMHRvdGFsJTIwJTNEJTIwMCUwQSUyMCUyMCUyMCUyMGZvciUyMHNhbXBsZSUyMGluJTIwdHFkbShpdGVyKGRhdGFzZXQpKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvdGFsJTIwJTJCJTNEJTIwMSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwYW55X2tleXdvcmRfaW5fc3RyaW5nKHNhbXBsZSU1QiUyMmNvbnRlbnQlMjIlNUQlMkMlMjBmaWx0ZXJzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMGslMkMlMjB2JTIwaW4lMjBzYW1wbGUuaXRlbXMoKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZpbHRlcmVkX2RpY3QlNUJrJTVELmFwcGVuZCh2KSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlN0JsZW4oZmlsdGVyZWRfZGljdCU1Qidjb250ZW50JyU1RCklMkZ0b3RhbCUzQS4yJTI1JTdEJTIwb2YlMjBkYXRhJTIwYWZ0ZXIlMjBmaWx0ZXJpbmcuJTIyKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMERhdGFzZXQuZnJvbV9kaWN0KGZpbHRlcmVkX2RpY3Qp",highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict
<span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset


<span class="hljs-keyword">def</span> <span class="hljs-title function_">filter_streaming_dataset</span>(<span class="hljs-params">dataset, filters</span>):
    filtered_dict = defaultdict(<span class="hljs-built_in">list</span>)
    total = <span class="hljs-number">0</span>
    <span class="hljs-keyword">for</span> sample <span class="hljs-keyword">in</span> tqdm(<span class="hljs-built_in">iter</span>(dataset)):
        total += <span class="hljs-number">1</span>
        <span class="hljs-keyword">if</span> any_keyword_in_string(sample[<span class="hljs-string">&quot;content&quot;</span>], filters):
            <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> sample.items():
                filtered_dict[k].append(v)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{<span class="hljs-built_in">len</span>(filtered_dict[<span class="hljs-string">&#x27;content&#x27;</span>])/total:<span class="hljs-number">.2</span>%}</span> of data after filtering.&quot;</span>)
    <span class="hljs-keyword">return</span> Dataset.from_dict(filtered_dict)`,wrap:!1}}),q=new I({props:{code:"JTIzJTIwVGhpcyUyMGNlbGwlMjB3aWxsJTIwdGFrZSUyMGElMjB2ZXJ5JTIwbG9uZyUyMHRpbWUlMjB0byUyMGV4ZWN1dGUlMkMlMjBzbyUyMHlvdSUyMHNob3VsZCUyMHNraXAlMjBpdCUyMGFuZCUyMGdvJTIwdG8lMEElMjMlMjB0aGUlMjBuZXh0JTIwb25lISUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQSUwQXNwbGl0JTIwJTNEJTIwJTIydHJhaW4lMjIlMjAlMjAlMjMlMjAlMjJ2YWxpZCUyMiUwQWZpbHRlcnMlMjAlM0QlMjAlNUIlMjJwYW5kYXMlMjIlMkMlMjAlMjJza2xlYXJuJTIyJTJDJTIwJTIybWF0cGxvdGxpYiUyMiUyQyUyMCUyMnNlYWJvcm4lMjIlNUQlMEElMEFkYXRhJTIwJTNEJTIwbG9hZF9kYXRhc2V0KGYlMjJ0cmFuc2Zvcm1lcnNib29rJTJGY29kZXBhcnJvdC0lN0JzcGxpdCU3RCUyMiUyQyUyMHNwbGl0JTNEc3BsaXQlMkMlMjBzdHJlYW1pbmclM0RUcnVlKSUwQWZpbHRlcmVkX2RhdGElMjAlM0QlMjBmaWx0ZXJfc3RyZWFtaW5nX2RhdGFzZXQoZGF0YSUyQyUyMGZpbHRlcnMp",highlighted:`<span class="hljs-comment"># This cell will take a very long time to execute, so you should skip it and go to</span>
<span class="hljs-comment"># the next one!</span>
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

split = <span class="hljs-string">&quot;train&quot;</span>  <span class="hljs-comment"># &quot;valid&quot;</span>
filters = [<span class="hljs-string">&quot;pandas&quot;</span>, <span class="hljs-string">&quot;sklearn&quot;</span>, <span class="hljs-string">&quot;matplotlib&quot;</span>, <span class="hljs-string">&quot;seaborn&quot;</span>]

data = load_dataset(<span class="hljs-string">f&quot;transformersbook/codeparrot-<span class="hljs-subst">{split}</span>&quot;</span>, split=split, streaming=<span class="hljs-literal">True</span>)
filtered_data = filter_streaming_dataset(data, filters)`,wrap:!1}}),D=new I({props:{code:"My4yNiUyNSUyMG9mJTIwZGF0YSUyMGFmdGVyJTIwZmlsdGVyaW5nLg==",highlighted:'<span class="hljs-number">3.26</span>% of data after filtering.',wrap:!1}}),K=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTJDJTIwRGF0YXNldERpY3QlMEElMEFkc190cmFpbiUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJodWdnaW5nZmFjZS1jb3Vyc2UlMkZjb2RlcGFycm90LWRzLXRyYWluJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEFkc192YWxpZCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJodWdnaW5nZmFjZS1jb3Vyc2UlMkZjb2RlcGFycm90LWRzLXZhbGlkJTIyJTJDJTIwc3BsaXQlM0QlMjJ2YWxpZGF0aW9uJTIyKSUwQSUwQXJhd19kYXRhc2V0cyUyMCUzRCUyMERhdGFzZXREaWN0KCUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRyYWluJTIyJTNBJTIwZHNfdHJhaW4lMkMlMjAlMjAlMjMlMjAuc2h1ZmZsZSgpLnNlbGVjdChyYW5nZSg1MDAwMCkpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydmFsaWQlMjIlM0ElMjBkc192YWxpZCUyQyUyMCUyMCUyMyUyMC5zaHVmZmxlKCkuc2VsZWN0KHJhbmdlKDUwMCkpJTBBJTIwJTIwJTIwJTIwJTdEJTBBKSUwQSUwQXJhd19kYXRhc2V0cw==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, DatasetDict

ds_train = load_dataset(<span class="hljs-string">&quot;huggingface-course/codeparrot-ds-train&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
ds_valid = load_dataset(<span class="hljs-string">&quot;huggingface-course/codeparrot-ds-valid&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>)

raw_datasets = DatasetDict(
    {
        <span class="hljs-string">&quot;train&quot;</span>: ds_train,  <span class="hljs-comment"># .shuffle().select(range(50000)),</span>
        <span class="hljs-string">&quot;valid&quot;</span>: ds_valid,  <span class="hljs-comment"># .shuffle().select(range(500))</span>
    }
)

raw_datasets`,wrap:!1}}),P=new I({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ3JlcG9fbmFtZSclMkMlMjAncGF0aCclMkMlMjAnY29waWVzJyUyQyUyMCdzaXplJyUyQyUyMCdjb250ZW50JyUyQyUyMCdsaWNlbnNlJyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwNjA2NzIwJTBBJTIwJTIwJTIwJTIwJTdEKSUwQSUyMCUyMCUyMCUyMHZhbGlkJTNBJTIwRGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidyZXBvX25hbWUnJTJDJTIwJ3BhdGgnJTJDJTIwJ2NvcGllcyclMkMlMjAnc2l6ZSclMkMlMjAnY29udGVudCclMkMlMjAnbGljZW5zZSclNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBudW1fcm93cyUzQSUyMDMzMjIlMEElMjAlMjAlMjAlMjAlN0QpJTBBJTdEKQ==",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;repo_name&#x27;</span>, <span class="hljs-string">&#x27;path&#x27;</span>, <span class="hljs-string">&#x27;copies&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;content&#x27;</span>, <span class="hljs-string">&#x27;license&#x27;</span>],
        num_rows: <span class="hljs-number">606720</span>
    })
    valid: Dataset({
        features: [<span class="hljs-string">&#x27;repo_name&#x27;</span>, <span class="hljs-string">&#x27;path&#x27;</span>, <span class="hljs-string">&#x27;copies&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;content&#x27;</span>, <span class="hljs-string">&#x27;license&#x27;</span>],
        num_rows: <span class="hljs-number">3322</span>
    })
})`,wrap:!1}}),z=new ct({props:{$$slots:{default:[Ja]},$$scope:{ctx:C}}}),O=new I({props:{code:"Zm9yJTIwa2V5JTIwaW4lMjByYXdfZGF0YXNldHMlNUIlMjJ0cmFpbiUyMiU1RCU1QjAlNUQlM0ElMEElMjAlMjAlMjAlMjBwcmludChmJTIyJTdCa2V5LnVwcGVyKCklN0QlM0ElMjAlN0JyYXdfZGF0YXNldHMlNUIndHJhaW4nJTVEJTVCMCU1RCU1QmtleSU1RCU1QiUzQTIwMCU1RCU3RCUyMik=",highlighted:`<span class="hljs-keyword">for</span> key <span class="hljs-keyword">in</span> raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key.upper()}</span>: <span class="hljs-subst">{raw_datasets[<span class="hljs-string">&#x27;train&#x27;</span>][<span class="hljs-number">0</span>][key][:<span class="hljs-number">200</span>]}</span>&quot;</span>)`,wrap:!1}}),re=new I({props:{code:"J1JFUE9fTkFNRSUzQSUyMGttaWtlJTJGc2Npa2l0LWxlYXJuJyUwQSdQQVRIJTNBJTIwc2tsZWFybiUyRnV0aWxzJTJGX19pbml0X18ucHknJTBBJ0NPUElFUyUzQSUyMDMnJTBBJ1NJWkUlM0ElMjAxMDA5NCclMEEnJydDT05URU5UJTNBJTIwJTIyJTIyJTIyJTBBVGhlJTIwJTNBbW9kJTNBJTYwc2tsZWFybi51dGlscyU2MCUyMG1vZHVsZSUyMGluY2x1ZGVzJTIwdmFyaW91cyUyMHV0aWxpdGVzLiUwQSUyMiUyMiUyMiUwQSUwQWZyb20lMjBjb2xsZWN0aW9ucyUyMGltcG9ydCUyMFNlcXVlbmNlJTBBJTBBaW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBZnJvbSUyMHNjaXB5LnNwYXJzZSUyMGltcG9ydCUyMGlzc3BhcnNlJTBBaW1wb3J0JTIwd2FybmluZ3MlMEElMEFmcm9tJTIwLm11cm11cmhhc2glMjBpbXBvcnQlMjBtdXJtJTBBTElDRU5TRSUzQSUyMGJzZC0zLWNsYXVzZScnJw==",highlighted:`<span class="hljs-string">&#x27;REPO_NAME: kmike/scikit-learn&#x27;</span>
<span class="hljs-string">&#x27;PATH: sklearn/utils/__init__.py&#x27;</span>
<span class="hljs-string">&#x27;COPIES: 3&#x27;</span>
<span class="hljs-string">&#x27;SIZE: 10094&#x27;</span>
<span class="hljs-string">&#x27;&#x27;&#x27;CONTENT: &quot;&quot;&quot;
The :mod:\`sklearn.utils\` module includes various utilites.
&quot;&quot;&quot;

from collections import Sequence

import numpy as np
from scipy.sparse import issparse
import warnings

from .murmurhash import murm
LICENSE: bsd-3-clause&#x27;&#x27;&#x27;</span>`,wrap:!1}}),ye=new Us({props:{title:"Preparing the dataset",local:"preparing-the-dataset",headingTag:"h2"}}),we=new Tl({props:{id:"ma1TrR7gE7I"}}),m=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjb250ZXh0X2xlbmd0aCUyMCUzRCUyMDEyOCUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmh1Z2dpbmdmYWNlLWNvdXJzZSUyRmNvZGUtc2VhcmNoLW5ldC10b2tlbml6ZXIlMjIpJTBBJTBBb3V0cHV0cyUyMCUzRCUyMHRva2VuaXplciglMEElMjAlMjAlMjAlMjByYXdfZGF0YXNldHMlNUIlMjJ0cmFpbiUyMiU1RCU1QiUzQTIlNUQlNUIlMjJjb250ZW50JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBtYXhfbGVuZ3RoJTNEY29udGV4dF9sZW5ndGglMkMlMEElMjAlMjAlMjAlMjByZXR1cm5fb3ZlcmZsb3dpbmdfdG9rZW5zJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHJldHVybl9sZW5ndGglM0RUcnVlJTJDJTBBKSUwQSUwQXByaW50KGYlMjJJbnB1dCUyMElEcyUyMGxlbmd0aCUzQSUyMCU3QmxlbihvdXRwdXRzJTVCJ2lucHV0X2lkcyclNUQpJTdEJTIyKSUwQXByaW50KGYlMjJJbnB1dCUyMGNodW5rJTIwbGVuZ3RocyUzQSUyMCU3QihvdXRwdXRzJTVCJ2xlbmd0aCclNUQpJTdEJTIyKSUwQXByaW50KGYlMjJDaHVuayUyMG1hcHBpbmclM0ElMjAlN0JvdXRwdXRzJTVCJ292ZXJmbG93X3RvX3NhbXBsZV9tYXBwaW5nJyU1RCU3RCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

context_length = <span class="hljs-number">128</span>
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;huggingface-course/code-search-net-tokenizer&quot;</span>)

outputs = tokenizer(
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][:<span class="hljs-number">2</span>][<span class="hljs-string">&quot;content&quot;</span>],
    truncation=<span class="hljs-literal">True</span>,
    max_length=context_length,
    return_overflowing_tokens=<span class="hljs-literal">True</span>,
    return_length=<span class="hljs-literal">True</span>,
)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Input IDs length: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(outputs[<span class="hljs-string">&#x27;input_ids&#x27;</span>])}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Input chunk lengths: <span class="hljs-subst">{(outputs[<span class="hljs-string">&#x27;length&#x27;</span>])}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Chunk mapping: <span class="hljs-subst">{outputs[<span class="hljs-string">&#x27;overflow_to_sample_mapping&#x27;</span>]}</span>&quot;</span>)`,wrap:!1}}),Ct=new I({props:{code:"SW5wdXQlMjBJRHMlMjBsZW5ndGglM0ElMjAzNCUwQUlucHV0JTIwY2h1bmslMjBsZW5ndGhzJTNBJTIwJTVCMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTE3JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwMTI4JTJDJTIwNDElNUQlMEFDaHVuayUyMG1hcHBpbmclM0ElMjAlNUIwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTJDJTIwMSUyQyUyMDElMkMlMjAxJTVE",highlighted:`Input IDs length: <span class="hljs-number">34</span>
Input chunk lengths: [<span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">117</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">128</span>, <span class="hljs-number">41</span>]
Chunk mapping: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]`,wrap:!1}}),kt=new I({props:{code:"ZGVmJTIwdG9rZW5pemUoZWxlbWVudCklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsZW1lbnQlNUIlMjJjb250ZW50JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXhfbGVuZ3RoJTNEY29udGV4dF9sZW5ndGglMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm5fb3ZlcmZsb3dpbmdfdG9rZW5zJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybl9sZW5ndGglM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMGlucHV0X2JhdGNoJTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwbGVuZ3RoJTJDJTIwaW5wdXRfaWRzJTIwaW4lMjB6aXAob3V0cHV0cyU1QiUyMmxlbmd0aCUyMiU1RCUyQyUyMG91dHB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBsZW5ndGglMjAlM0QlM0QlMjBjb250ZXh0X2xlbmd0aCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0X2JhdGNoLmFwcGVuZChpbnB1dF9pZHMpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTdCJTIyaW5wdXRfaWRzJTIyJTNBJTIwaW5wdXRfYmF0Y2glN0QlMEElMEElMEF0b2tlbml6ZWRfZGF0YXNldHMlMjAlM0QlMjByYXdfZGF0YXNldHMubWFwKCUwQSUyMCUyMCUyMCUyMHRva2VuaXplJTJDJTIwYmF0Y2hlZCUzRFRydWUlMkMlMjByZW1vdmVfY29sdW1ucyUzRHJhd19kYXRhc2V0cyU1QiUyMnRyYWluJTIyJTVELmNvbHVtbl9uYW1lcyUwQSklMEF0b2tlbml6ZWRfZGF0YXNldHM=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">element</span>):
    outputs = tokenizer(
        element[<span class="hljs-string">&quot;content&quot;</span>],
        truncation=<span class="hljs-literal">True</span>,
        max_length=context_length,
        return_overflowing_tokens=<span class="hljs-literal">True</span>,
        return_length=<span class="hljs-literal">True</span>,
    )
    input_batch = []
    <span class="hljs-keyword">for</span> length, input_ids <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(outputs[<span class="hljs-string">&quot;length&quot;</span>], outputs[<span class="hljs-string">&quot;input_ids&quot;</span>]):
        <span class="hljs-keyword">if</span> length == context_length:
            input_batch.append(input_ids)
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;input_ids&quot;</span>: input_batch}


tokenized_datasets = raw_datasets.<span class="hljs-built_in">map</span>(
    tokenize, batched=<span class="hljs-literal">True</span>, remove_columns=raw_datasets[<span class="hljs-string">&quot;train&quot;</span>].column_names
)
tokenized_datasets`,wrap:!1}}),Zt=new I({props:{code:"RGF0YXNldERpY3QoJTdCJTBBJTIwJTIwJTIwJTIwdHJhaW4lM0ElMjBEYXRhc2V0KCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlYXR1cmVzJTNBJTIwJTVCJ2lucHV0X2lkcyclNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBudW1fcm93cyUzQSUyMDE2NzAyMDYxJTBBJTIwJTIwJTIwJTIwJTdEKSUwQSUyMCUyMCUyMCUyMHZhbGlkJTNBJTIwRGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidpbnB1dF9pZHMnJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3Jvd3MlM0ElMjA5MzE2NCUwQSUyMCUyMCUyMCUyMCU3RCklMEElN0Qp",highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;input_ids&#x27;</span>],
        num_rows: <span class="hljs-number">16702061</span>
    })
    valid: Dataset({
        features: [<span class="hljs-string">&#x27;input_ids&#x27;</span>],
        num_rows: <span class="hljs-number">93164</span>
    })
})`,wrap:!1}}),nt=new ct({props:{$$slots:{default:[Ta]},$$scope:{ctx:C}}}),Bt=new Us({props:{title:"Initializing a new model",local:"initializing-a-new-model",headingTag:"h2"}});const Nl=[fa,Ua],os=[];function Vl(e,o){return e[0]==="pt"?0:1}Je=Vl(C),Te=os[Je]=Nl[Je](C);const xl=[ba,ja],is=[];function El(e,o){return e[0]==="pt"?0:1}Ue=El(C),fe=is[Ue]=xl[Ue](C),Qt=new I({props:{code:"b3V0JTIwJTNEJTIwZGF0YV9jb2xsYXRvciglNUJ0b2tlbml6ZWRfZGF0YXNldHMlNUIlMjJ0cmFpbiUyMiU1RCU1QmklNUQlMjBmb3IlMjBpJTIwaW4lMjByYW5nZSg1KSU1RCklMEFmb3IlMjBrZXklMjBpbiUyMG91dCUzQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjIlN0JrZXklN0QlMjBzaGFwZSUzQSUyMCU3Qm91dCU1QmtleSU1RC5zaGFwZSU3RCUyMik=",highlighted:`out = data_collator([tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">5</span>)])
<span class="hljs-keyword">for</span> key <span class="hljs-keyword">in</span> out:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span> shape: <span class="hljs-subst">{out[key].shape}</span>&quot;</span>)`,wrap:!1}});const Al=[Ca,Ia],ps=[];function Fl(e,o){return e[0]==="pt"?0:1}je=Fl(C),be=ps[je]=Al[je](C);let W=C[0]==="tf"&&la();ot=new ct({props:{warning:!0,$$slots:{default:[ga]},$$scope:{ctx:C}}}),Vt=new I({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,wrap:!1}}),At=new I({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:"huggingface-cli login",wrap:!1}});const Yl=[ka,_a],rs=[];function Hl(e,o){return e[0]==="pt"?0:1}Ie=Hl(C),Ce=rs[Ie]=Yl[Ie](C),it=new ct({props:{$$slots:{default:[Za]},$$scope:{ctx:C}}}),pt=new ct({props:{$$slots:{default:[Ba]},$$scope:{ctx:C}}}),Ft=new Us({props:{title:"Code generation with a pipeline",local:"code-generation-with-a-pipeline",headingTag:"h2"}});const Sl=[Ra,Xa],Ms=[];function ql(e,o){return e[0]==="pt"?0:1}ge=ql(C),_e=Ms[ge]=Sl[ge](C),St=new I({props:{code:"dHh0JTIwJTNEJTIwJTIyJTIyJTIyJTVDJTBBJTIzJTIwY3JlYXRlJTIwc29tZSUyMGRhdGElMEF4JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEF5JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEElMEElMjMlMjBjcmVhdGUlMjBzY2F0dGVyJTIwcGxvdCUyMHdpdGglMjB4JTJDJTIweSUwQSUyMiUyMiUyMiUwQXByaW50KHBpcGUodHh0JTJDJTIwbnVtX3JldHVybl9zZXF1ZW5jZXMlM0QxKSU1QjAlNUQlNUIlMjJnZW5lcmF0ZWRfdGV4dCUyMiU1RCk=",highlighted:`txt = <span class="hljs-string">&quot;&quot;&quot;\\
# create some data
x = np.random.randn(100)
y = np.random.randn(100)

# create scatter plot with x, y
&quot;&quot;&quot;</span>
<span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>])`,wrap:!1}}),qt=new I({props:{code:"JTIzJTIwY3JlYXRlJTIwc29tZSUyMGRhdGElMEF4JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEF5JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEElMEElMjMlMjBjcmVhdGUlMjBzY2F0dGVyJTIwcGxvdCUyMHdpdGglMjB4JTJDJTIweSUwQXBsdC5zY2F0dGVyKHglMkMlMjB5KSUwQSUwQSUyMyUyMGNyZWF0ZSUyMHNjYXR0ZXI=",highlighted:`<span class="hljs-comment"># create some data</span>
x = np.random.randn(<span class="hljs-number">100</span>)
y = np.random.randn(<span class="hljs-number">100</span>)

<span class="hljs-comment"># create scatter plot with x, y</span>
plt.scatter(x, y)

<span class="hljs-comment"># create scatter</span>`,wrap:!1}}),Lt=new I({props:{code:"dHh0JTIwJTNEJTIwJTIyJTIyJTIyJTVDJTBBJTIzJTIwY3JlYXRlJTIwc29tZSUyMGRhdGElMEF4JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEF5JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEElMEElMjMlMjBjcmVhdGUlMjBkYXRhZnJhbWUlMjBmcm9tJTIweCUyMGFuZCUyMHklMEElMjIlMjIlMjIlMEFwcmludChwaXBlKHR4dCUyQyUyMG51bV9yZXR1cm5fc2VxdWVuY2VzJTNEMSklNUIwJTVEJTVCJTIyZ2VuZXJhdGVkX3RleHQlMjIlNUQp",highlighted:`txt = <span class="hljs-string">&quot;&quot;&quot;\\
# create some data
x = np.random.randn(100)
y = np.random.randn(100)

# create dataframe from x and y
&quot;&quot;&quot;</span>
<span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>])`,wrap:!1}}),Kt=new I({props:{code:"JTIzJTIwY3JlYXRlJTIwc29tZSUyMGRhdGElMEF4JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEF5JTIwJTNEJTIwbnAucmFuZG9tLnJhbmRuKDEwMCklMEElMEElMjMlMjBjcmVhdGUlMjBkYXRhZnJhbWUlMjBmcm9tJTIweCUyMGFuZCUyMHklMEFkZiUyMCUzRCUyMHBkLkRhdGFGcmFtZSglN0IneCclM0ElMjB4JTJDJTIwJ3knJTNBJTIweSU3RCklMEFkZi5pbnNlcnQoMCUyQyd4JyUyQyUyMHgpJTBBZm9y",highlighted:`<span class="hljs-comment"># create some data</span>
x = np.random.randn(<span class="hljs-number">100</span>)
y = np.random.randn(<span class="hljs-number">100</span>)

<span class="hljs-comment"># create dataframe from x and y</span>
df = pd.DataFrame({<span class="hljs-string">&#x27;x&#x27;</span>: x, <span class="hljs-string">&#x27;y&#x27;</span>: y})
df.insert(<span class="hljs-number">0</span>,<span class="hljs-string">&#x27;x&#x27;</span>, x)
<span class="hljs-keyword">for</span>`,wrap:!1}}),Ot=new I({props:{code:"dHh0JTIwJTNEJTIwJTIyJTIyJTIyJTVDJTBBJTIzJTIwZGF0YWZyYW1lJTIwd2l0aCUyMHByb2Zlc3Npb24lMkMlMjBpbmNvbWUlMjBhbmQlMjBuYW1lJTBBZGYlMjAlM0QlMjBwZC5EYXRhRnJhbWUoJTdCJ3Byb2Zlc3Npb24nJTNBJTIweCUyQyUyMCdpbmNvbWUnJTNBeSUyQyUyMCduYW1lJyUzQSUyMHolN0QpJTBBJTBBJTIzJTIwY2FsY3VsYXRlJTIwdGhlJTIwbWVhbiUyMGluY29tZSUyMHBlciUyMHByb2Zlc3Npb24lMEElMjIlMjIlMjIlMEFwcmludChwaXBlKHR4dCUyQyUyMG51bV9yZXR1cm5fc2VxdWVuY2VzJTNEMSklNUIwJTVEJTVCJTIyZ2VuZXJhdGVkX3RleHQlMjIlNUQp",highlighted:`txt = <span class="hljs-string">&quot;&quot;&quot;\\
# dataframe with profession, income and name
df = pd.DataFrame({&#x27;profession&#x27;: x, &#x27;income&#x27;:y, &#x27;name&#x27;: z})

# calculate the mean income per profession
&quot;&quot;&quot;</span>
<span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>])`,wrap:!1}}),es=new I({props:{code:"JTIzJTIwZGF0YWZyYW1lJTIwd2l0aCUyMHByb2Zlc3Npb24lMkMlMjBpbmNvbWUlMjBhbmQlMjBuYW1lJTBBZGYlMjAlM0QlMjBwZC5EYXRhRnJhbWUoJTdCJ3Byb2Zlc3Npb24nJTNBJTIweCUyQyUyMCdpbmNvbWUnJTNBeSUyQyUyMCduYW1lJyUzQSUyMHolN0QpJTBBJTBBJTIzJTIwY2FsY3VsYXRlJTIwdGhlJTIwbWVhbiUyMGluY29tZSUyMHBlciUyMHByb2Zlc3Npb24lMEFwcm9mZXNzaW9uJTIwJTNEJTIwZGYuZ3JvdXBieSglNUIncHJvZmVzc2lvbiclNUQpLm1lYW4oKSUwQSUwQSUyMyUyMGNvbXB1dGUlMjB0aGU=",highlighted:`<span class="hljs-comment"># dataframe with profession, income and name</span>
df = pd.DataFrame({<span class="hljs-string">&#x27;profession&#x27;</span>: x, <span class="hljs-string">&#x27;income&#x27;</span>:y, <span class="hljs-string">&#x27;name&#x27;</span>: z})

<span class="hljs-comment"># calculate the mean income per profession</span>
profession = df.groupby([<span class="hljs-string">&#x27;profession&#x27;</span>]).mean()

<span class="hljs-comment"># compute the</span>`,wrap:!1}}),ss=new I({props:{code:"dHh0JTIwJTNEJTIwJTIyJTIyJTIyJTBBJTIzJTIwaW1wb3J0JTIwcmFuZG9tJTIwZm9yZXN0JTIwcmVncmVzc29yJTIwZnJvbSUyMHNjaWtpdC1sZWFybiUwQWZyb20lMjBza2xlYXJuLmVuc2VtYmxlJTIwaW1wb3J0JTIwUmFuZG9tRm9yZXN0UmVncmVzc29yJTBBJTBBJTIzJTIwZml0JTIwcmFuZG9tJTIwZm9yZXN0JTIwbW9kZWwlMjB3aXRoJTIwMzAwJTIwZXN0aW1hdG9ycyUyMG9uJTIwWCUyQyUyMHklM0ElMEElMjIlMjIlMjIlMEFwcmludChwaXBlKHR4dCUyQyUyMG51bV9yZXR1cm5fc2VxdWVuY2VzJTNEMSklNUIwJTVEJTVCJTIyZ2VuZXJhdGVkX3RleHQlMjIlNUQp",highlighted:`txt = <span class="hljs-string">&quot;&quot;&quot;
# import random forest regressor from scikit-learn
from sklearn.ensemble import RandomForestRegressor

# fit random forest model with 300 estimators on X, y:
&quot;&quot;&quot;</span>
<span class="hljs-built_in">print</span>(pipe(txt, num_return_sequences=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>])`,wrap:!1}}),ls=new I({props:{code:"JTIzJTIwaW1wb3J0JTIwcmFuZG9tJTIwZm9yZXN0JTIwcmVncmVzc29yJTIwZnJvbSUyMHNjaWtpdC1sZWFybiUwQWZyb20lMjBza2xlYXJuLmVuc2VtYmxlJTIwaW1wb3J0JTIwUmFuZG9tRm9yZXN0UmVncmVzc29yJTBBJTBBJTIzJTIwZml0JTIwcmFuZG9tJTIwZm9yZXN0JTIwbW9kZWwlMjB3aXRoJTIwMzAwJTIwZXN0aW1hdG9ycyUyMG9uJTIwWCUyQyUyMHklM0ElMEFyZiUyMCUzRCUyMFJhbmRvbUZvcmVzdFJlZ3Jlc3NvcihuX2VzdGltYXRvcnMlM0QzMDAlMkMlMjByYW5kb21fc3RhdGUlM0RyYW5kb21fc3RhdGUlMkMlMjBtYXhfZGVwdGglM0QzKSUwQXJmLmZpdChYJTJDJTIweSklMEFyZg==",highlighted:`<span class="hljs-comment"># import random forest regressor from scikit-learn</span>
<span class="hljs-keyword">from</span> sklearn.ensemble <span class="hljs-keyword">import</span> RandomForestRegressor

<span class="hljs-comment"># fit random forest model with 300 estimators on X, y:</span>
rf = RandomForestRegressor(n_estimators=<span class="hljs-number">300</span>, random_state=random_state, max_depth=<span class="hljs-number">3</span>)
rf.fit(X, y)
rf`,wrap:!1}});function Dl(e,o){return e[0]==="tf"?va:Wa}let cl=Dl(C),at=cl(C),v=C[0]==="pt"&&aa(C);return as=new wa({props:{source:"https://github.com/huggingface/course/blob/main/chapters/en/chapter7/6.mdx"}}),{c(){l=T("meta"),r=i(),n=T("p"),y=i(),w(g.$$.fragment),b=i(),w($.$$.fragment),k=i(),w(Z.$$.fragment),J=i(),G.c(),R=i(),X=T("p"),X.innerHTML=N,_=i(),B=T("p"),B.innerHTML=ke,Ee=i(),w(V.$$.fragment),yt=i(),se=T("p"),se.innerHTML=Ze,mt=i(),Q=T("iframe"),wt=i(),le=T("p"),le.innerHTML=Ge,Ae=i(),w(x.$$.fragment),ut=i(),ae=T("p"),ae.innerHTML=Be,Fe=i(),E=T("p"),E.innerHTML=fs,Ye=i(),w(A.$$.fragment),He=i(),F=T("p"),F.textContent=js,Se=i(),w(Y.$$.fragment),qe=i(),w(H.$$.fragment),ht=i(),ne=T("p"),ne.textContent=Xe,De=i(),w(S.$$.fragment),dt=i(),oe=T("p"),oe.textContent=Re,Le=i(),w(q.$$.fragment),Ke=i(),w(D.$$.fragment),Jt=i(),ie=T("p"),ie.textContent=We,Pe=i(),L=T("p"),L.textContent=bs,Oe=i(),w(K.$$.fragment),et=i(),w(P.$$.fragment),tt=i(),w(z.$$.fragment),Tt=i(),pe=T("p"),pe.textContent=ve,st=i(),w(O.$$.fragment),Ut=i(),w(re.$$.fragment),Me=i(),Qe=T("p"),Qe.innerHTML=ft,ce=i(),w(ye.$$.fragment),me=i(),w(we.$$.fragment),ue=i(),ze=T("p"),ze.textContent=jt,he=i(),de=T("p"),de.innerHTML=Ne,bt=i(),te=T("div"),te.innerHTML=Ve,lt=i(),ee=T("p"),ee.textContent=It,t=i(),w(m.$$.fragment),Cs=i(),w(Ct.$$.fragment),gs=i(),gt=T("p"),gt.innerHTML=Ul,_s=i(),_t=T("p"),_t.innerHTML=fl,ks=i(),w(kt.$$.fragment),Zs=i(),w(Zt.$$.fragment),$s=i(),$t=T("p"),$t.textContent=jl,Gs=i(),Gt=T("p"),Gt.textContent=bl,Bs=i(),w(nt.$$.fragment),Xs=i(),w(Bt.$$.fragment),Rs=i(),Xt=T("p"),Xt.innerHTML=Il,Ws=i(),Te.c(),ys=i(),Rt=T("p"),Rt.innerHTML=Cl,vs=i(),Wt=T("p"),Wt.innerHTML=gl,Qs=i(),fe.c(),ms=i(),vt=T("p"),vt.textContent=_l,zs=i(),w(Qt.$$.fragment),Ns=i(),be.c(),ws=i(),zt=T("p"),zt.textContent=kl,Vs=i(),W&&W.c(),us=i(),w(ot.$$.fragment),xs=i(),Nt=T("p"),Nt.textContent=Zl,Es=i(),w(Vt.$$.fragment),As=i(),xt=T("p"),xt.textContent=$l,Fs=i(),Et=T("p"),Et.textContent=Gl,Ys=i(),w(At.$$.fragment),Hs=i(),Ce.c(),hs=i(),w(it.$$.fragment),Ss=i(),w(pt.$$.fragment),qs=i(),w(Ft.$$.fragment),Ds=i(),Yt=T("p"),Yt.innerHTML=Bl,Ls=i(),_e.c(),ds=i(),Ht=T("p"),Ht.textContent=Xl,Ks=i(),w(St.$$.fragment),Ps=i(),w(qt.$$.fragment),Os=i(),Dt=T("p"),Dt.innerHTML=Rl,el=i(),w(Lt.$$.fragment),tl=i(),w(Kt.$$.fragment),sl=i(),Pt=T("p"),Pt.innerHTML=Wl,ll=i(),w(Ot.$$.fragment),al=i(),w(es.$$.fragment),nl=i(),ts=T("p"),ts.innerHTML=vl,ol=i(),w(ss.$$.fragment),il=i(),w(ls.$$.fragment),pl=i(),at.c(),Js=i(),v&&v.c(),Ts=i(),w(as.$$.fragment),rl=i(),Is=T("p"),this.h()},l(e){const o=ca("svelte-u9bgzb",document.head);l=U(o,"META",{name:!0,content:!0}),o.forEach(s),r=p(e),n=U(e,"P",{}),Jl(n).forEach(s),y=p(e),u(g.$$.fragment,e),b=p(e),u($.$$.fragment,e),k=p(e),u(Z.$$.fragment,e),J=p(e),G.l(e),R=p(e),X=U(e,"P",{"data-svelte-h":!0}),f(X)!=="svelte-n23fcu"&&(X.innerHTML=N),_=p(e),B=U(e,"P",{"data-svelte-h":!0}),f(B)!=="svelte-1pnwlne"&&(B.innerHTML=ke),Ee=p(e),u(V.$$.fragment,e),yt=p(e),se=U(e,"P",{"data-svelte-h":!0}),f(se)!=="svelte-196thle"&&(se.innerHTML=Ze),mt=p(e),Q=U(e,"IFRAME",{src:!0,frameborder:!0,height:!0,title:!0,class:!0,allow:!0,sandbox:!0}),Jl(Q).forEach(s),wt=p(e),le=U(e,"P",{"data-svelte-h":!0}),f(le)!=="svelte-ab9fd4"&&(le.innerHTML=Ge),Ae=p(e),u(x.$$.fragment,e),ut=p(e),ae=U(e,"P",{"data-svelte-h":!0}),f(ae)!=="svelte-tpjiv"&&(ae.innerHTML=Be),Fe=p(e),E=U(e,"P",{"data-svelte-h":!0}),f(E)!=="svelte-c22pqj"&&(E.innerHTML=fs),Ye=p(e),u(A.$$.fragment,e),He=p(e),F=U(e,"P",{"data-svelte-h":!0}),f(F)!=="svelte-1dvnz8o"&&(F.textContent=js),Se=p(e),u(Y.$$.fragment,e),qe=p(e),u(H.$$.fragment,e),ht=p(e),ne=U(e,"P",{"data-svelte-h":!0}),f(ne)!=="svelte-1aeifay"&&(ne.textContent=Xe),De=p(e),u(S.$$.fragment,e),dt=p(e),oe=U(e,"P",{"data-svelte-h":!0}),f(oe)!=="svelte-1ekmk11"&&(oe.textContent=Re),Le=p(e),u(q.$$.fragment,e),Ke=p(e),u(D.$$.fragment,e),Jt=p(e),ie=U(e,"P",{"data-svelte-h":!0}),f(ie)!=="svelte-bf1ont"&&(ie.textContent=We),Pe=p(e),L=U(e,"P",{"data-svelte-h":!0}),f(L)!=="svelte-zq0s8o"&&(L.textContent=bs),Oe=p(e),u(K.$$.fragment,e),et=p(e),u(P.$$.fragment,e),tt=p(e),u(z.$$.fragment,e),Tt=p(e),pe=U(e,"P",{"data-svelte-h":!0}),f(pe)!=="svelte-yay142"&&(pe.textContent=ve),st=p(e),u(O.$$.fragment,e),Ut=p(e),u(re.$$.fragment,e),Me=p(e),Qe=U(e,"P",{"data-svelte-h":!0}),f(Qe)!=="svelte-tg22kr"&&(Qe.innerHTML=ft),ce=p(e),u(ye.$$.fragment,e),me=p(e),u(we.$$.fragment,e),ue=p(e),ze=U(e,"P",{"data-svelte-h":!0}),f(ze)!=="svelte-snql8t"&&(ze.textContent=jt),he=p(e),de=U(e,"P",{"data-svelte-h":!0}),f(de)!=="svelte-4oqi89"&&(de.innerHTML=Ne),bt=p(e),te=U(e,"DIV",{class:!0,"data-svelte-h":!0}),f(te)!=="svelte-1mf8pz"&&(te.innerHTML=Ve),lt=p(e),ee=U(e,"P",{"data-svelte-h":!0}),f(ee)!=="svelte-1k3fmn9"&&(ee.textContent=It),t=p(e),u(m.$$.fragment,e),Cs=p(e),u(Ct.$$.fragment,e),gs=p(e),gt=U(e,"P",{"data-svelte-h":!0}),f(gt)!=="svelte-4fmrv9"&&(gt.innerHTML=Ul),_s=p(e),_t=U(e,"P",{"data-svelte-h":!0}),f(_t)!=="svelte-1ro02u6"&&(_t.innerHTML=fl),ks=p(e),u(kt.$$.fragment,e),Zs=p(e),u(Zt.$$.fragment,e),$s=p(e),$t=U(e,"P",{"data-svelte-h":!0}),f($t)!=="svelte-4o3hzi"&&($t.textContent=jl),Gs=p(e),Gt=U(e,"P",{"data-svelte-h":!0}),f(Gt)!=="svelte-17t6lwy"&&(Gt.textContent=bl),Bs=p(e),u(nt.$$.fragment,e),Xs=p(e),u(Bt.$$.fragment,e),Rs=p(e),Xt=U(e,"P",{"data-svelte-h":!0}),f(Xt)!=="svelte-1hjqc59"&&(Xt.innerHTML=Il),Ws=p(e),Te.l(e),ys=p(e),Rt=U(e,"P",{"data-svelte-h":!0}),f(Rt)!=="svelte-1hk8ix6"&&(Rt.innerHTML=Cl),vs=p(e),Wt=U(e,"P",{"data-svelte-h":!0}),f(Wt)!=="svelte-z4ltmo"&&(Wt.innerHTML=gl),Qs=p(e),fe.l(e),ms=p(e),vt=U(e,"P",{"data-svelte-h":!0}),f(vt)!=="svelte-d5rqk1"&&(vt.textContent=_l),zs=p(e),u(Qt.$$.fragment,e),Ns=p(e),be.l(e),ws=p(e),zt=U(e,"P",{"data-svelte-h":!0}),f(zt)!=="svelte-1qzksrg"&&(zt.textContent=kl),Vs=p(e),W&&W.l(e),us=p(e),u(ot.$$.fragment,e),xs=p(e),Nt=U(e,"P",{"data-svelte-h":!0}),f(Nt)!=="svelte-1vtzchd"&&(Nt.textContent=Zl),Es=p(e),u(Vt.$$.fragment,e),As=p(e),xt=U(e,"P",{"data-svelte-h":!0}),f(xt)!=="svelte-1ied0vh"&&(xt.textContent=$l),Fs=p(e),Et=U(e,"P",{"data-svelte-h":!0}),f(Et)!=="svelte-648vlf"&&(Et.textContent=Gl),Ys=p(e),u(At.$$.fragment,e),Hs=p(e),Ce.l(e),hs=p(e),u(it.$$.fragment,e),Ss=p(e),u(pt.$$.fragment,e),qs=p(e),u(Ft.$$.fragment,e),Ds=p(e),Yt=U(e,"P",{"data-svelte-h":!0}),f(Yt)!=="svelte-cktwb7"&&(Yt.innerHTML=Bl),Ls=p(e),_e.l(e),ds=p(e),Ht=U(e,"P",{"data-svelte-h":!0}),f(Ht)!=="svelte-14oror3"&&(Ht.textContent=Xl),Ks=p(e),u(St.$$.fragment,e),Ps=p(e),u(qt.$$.fragment,e),Os=p(e),Dt=U(e,"P",{"data-svelte-h":!0}),f(Dt)!=="svelte-1htu6p0"&&(Dt.innerHTML=Rl),el=p(e),u(Lt.$$.fragment,e),tl=p(e),u(Kt.$$.fragment,e),sl=p(e),Pt=U(e,"P",{"data-svelte-h":!0}),f(Pt)!=="svelte-ls0a4d"&&(Pt.innerHTML=Wl),ll=p(e),u(Ot.$$.fragment,e),al=p(e),u(es.$$.fragment,e),nl=p(e),ts=U(e,"P",{"data-svelte-h":!0}),f(ts)!=="svelte-15s17ie"&&(ts.innerHTML=vl),ol=p(e),u(ss.$$.fragment,e),il=p(e),u(ls.$$.fragment,e),pl=p(e),at.l(e),Js=p(e),v&&v.l(e),Ts=p(e),u(as.$$.fragment,e),rl=p(e),Is=U(e,"P",{}),Jl(Is).forEach(s),this.h()},h(){xe(l,"name","hf:doc:metadata"),xe(l,"content",xa),ia(Q.src,$e="https://course-demos-codeparrot-ds.hf.space")||xe(Q,"src",$e),xe(Q,"frameborder","0"),xe(Q,"height","300"),xe(Q,"title","Gradio app"),xe(Q,"class","block dark:hidden container p-0 flex-grow space-iframe"),xe(Q,"allow","accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking"),xe(Q,"sandbox","allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"),xe(te,"class","flex justify-center")},m(e,o){ya(document.head,l),a(e,r,o),a(e,n,o),a(e,y,o),h(g,e,o),a(e,b,o),h($,e,o),a(e,k,o),h(Z,e,o),a(e,J,o),ns[j].m(e,o),a(e,R,o),a(e,X,o),a(e,_,o),a(e,B,o),a(e,Ee,o),h(V,e,o),a(e,yt,o),a(e,se,o),a(e,mt,o),a(e,Q,o),a(e,wt,o),a(e,le,o),a(e,Ae,o),h(x,e,o),a(e,ut,o),a(e,ae,o),a(e,Fe,o),a(e,E,o),a(e,Ye,o),h(A,e,o),a(e,He,o),a(e,F,o),a(e,Se,o),h(Y,e,o),a(e,qe,o),h(H,e,o),a(e,ht,o),a(e,ne,o),a(e,De,o),h(S,e,o),a(e,dt,o),a(e,oe,o),a(e,Le,o),h(q,e,o),a(e,Ke,o),h(D,e,o),a(e,Jt,o),a(e,ie,o),a(e,Pe,o),a(e,L,o),a(e,Oe,o),h(K,e,o),a(e,et,o),h(P,e,o),a(e,tt,o),h(z,e,o),a(e,Tt,o),a(e,pe,o),a(e,st,o),h(O,e,o),a(e,Ut,o),h(re,e,o),a(e,Me,o),a(e,Qe,o),a(e,ce,o),h(ye,e,o),a(e,me,o),h(we,e,o),a(e,ue,o),a(e,ze,o),a(e,he,o),a(e,de,o),a(e,bt,o),a(e,te,o),a(e,lt,o),a(e,ee,o),a(e,t,o),h(m,e,o),a(e,Cs,o),h(Ct,e,o),a(e,gs,o),a(e,gt,o),a(e,_s,o),a(e,_t,o),a(e,ks,o),h(kt,e,o),a(e,Zs,o),h(Zt,e,o),a(e,$s,o),a(e,$t,o),a(e,Gs,o),a(e,Gt,o),a(e,Bs,o),h(nt,e,o),a(e,Xs,o),h(Bt,e,o),a(e,Rs,o),a(e,Xt,o),a(e,Ws,o),os[Je].m(e,o),a(e,ys,o),a(e,Rt,o),a(e,vs,o),a(e,Wt,o),a(e,Qs,o),is[Ue].m(e,o),a(e,ms,o),a(e,vt,o),a(e,zs,o),h(Qt,e,o),a(e,Ns,o),ps[je].m(e,o),a(e,ws,o),a(e,zt,o),a(e,Vs,o),W&&W.m(e,o),a(e,us,o),h(ot,e,o),a(e,xs,o),a(e,Nt,o),a(e,Es,o),h(Vt,e,o),a(e,As,o),a(e,xt,o),a(e,Fs,o),a(e,Et,o),a(e,Ys,o),h(At,e,o),a(e,Hs,o),rs[Ie].m(e,o),a(e,hs,o),h(it,e,o),a(e,Ss,o),h(pt,e,o),a(e,qs,o),h(Ft,e,o),a(e,Ds,o),a(e,Yt,o),a(e,Ls,o),Ms[ge].m(e,o),a(e,ds,o),a(e,Ht,o),a(e,Ks,o),h(St,e,o),a(e,Ps,o),h(qt,e,o),a(e,Os,o),a(e,Dt,o),a(e,el,o),h(Lt,e,o),a(e,tl,o),h(Kt,e,o),a(e,sl,o),a(e,Pt,o),a(e,ll,o),h(Ot,e,o),a(e,al,o),h(es,e,o),a(e,nl,o),a(e,ts,o),a(e,ol,o),h(ss,e,o),a(e,il,o),h(ls,e,o),a(e,pl,o),at.m(e,o),a(e,Js,o),v&&v.m(e,o),a(e,Ts,o),h(as,e,o),a(e,rl,o),a(e,Is,o),Ml=!0},p(e,[o]){const Ll={};o&1&&(Ll.fw=e[0]),g.$set(Ll);let yl=j;j=zl(e),j!==yl&&(Mt(),c(ns[yl],1,1,()=>{ns[yl]=null}),rt(),G=ns[j],G||(G=ns[j]=Ql[j](e),G.c()),M(G,1),G.m(R.parentNode,R));const Kl={};o&2&&(Kl.$$scope={dirty:o,ctx:e}),z.$set(Kl);const Pl={};o&2&&(Pl.$$scope={dirty:o,ctx:e}),nt.$set(Pl);let ml=Je;Je=Vl(e),Je!==ml&&(Mt(),c(os[ml],1,1,()=>{os[ml]=null}),rt(),Te=os[Je],Te||(Te=os[Je]=Nl[Je](e),Te.c()),M(Te,1),Te.m(ys.parentNode,ys));let wl=Ue;Ue=El(e),Ue!==wl&&(Mt(),c(is[wl],1,1,()=>{is[wl]=null}),rt(),fe=is[Ue],fe||(fe=is[Ue]=xl[Ue](e),fe.c()),M(fe,1),fe.m(ms.parentNode,ms));let ul=je;je=Fl(e),je!==ul&&(Mt(),c(ps[ul],1,1,()=>{ps[ul]=null}),rt(),be=ps[je],be||(be=ps[je]=Al[je](e),be.c()),M(be,1),be.m(ws.parentNode,ws)),e[0]==="tf"?W?o&1&&M(W,1):(W=la(),W.c(),M(W,1),W.m(us.parentNode,us)):W&&(Mt(),c(W,1,1,()=>{W=null}),rt());const Ol={};o&2&&(Ol.$$scope={dirty:o,ctx:e}),ot.$set(Ol);let hl=Ie;Ie=Hl(e),Ie!==hl&&(Mt(),c(rs[hl],1,1,()=>{rs[hl]=null}),rt(),Ce=rs[Ie],Ce||(Ce=rs[Ie]=Yl[Ie](e),Ce.c()),M(Ce,1),Ce.m(hs.parentNode,hs));const ea={};o&2&&(ea.$$scope={dirty:o,ctx:e}),it.$set(ea);const ta={};o&3&&(ta.$$scope={dirty:o,ctx:e}),pt.$set(ta);let dl=ge;ge=ql(e),ge!==dl&&(Mt(),c(Ms[dl],1,1,()=>{Ms[dl]=null}),rt(),_e=Ms[ge],_e||(_e=Ms[ge]=Sl[ge](e),_e.c()),M(_e,1),_e.m(ds.parentNode,ds)),cl!==(cl=Dl(e))&&(at.d(1),at=cl(e),at&&(at.c(),at.m(Js.parentNode,Js))),e[0]==="pt"?v?o&1&&M(v,1):(v=aa(e),v.c(),M(v,1),v.m(Ts.parentNode,Ts)):v&&(Mt(),c(v,1,1,()=>{v=null}),rt())},i(e){Ml||(M(g.$$.fragment,e),M($.$$.fragment,e),M(Z.$$.fragment,e),M(G),M(V.$$.fragment,e),M(x.$$.fragment,e),M(A.$$.fragment,e),M(Y.$$.fragment,e),M(H.$$.fragment,e),M(S.$$.fragment,e),M(q.$$.fragment,e),M(D.$$.fragment,e),M(K.$$.fragment,e),M(P.$$.fragment,e),M(z.$$.fragment,e),M(O.$$.fragment,e),M(re.$$.fragment,e),M(ye.$$.fragment,e),M(we.$$.fragment,e),M(m.$$.fragment,e),M(Ct.$$.fragment,e),M(kt.$$.fragment,e),M(Zt.$$.fragment,e),M(nt.$$.fragment,e),M(Bt.$$.fragment,e),M(Te),M(fe),M(Qt.$$.fragment,e),M(be),M(W),M(ot.$$.fragment,e),M(Vt.$$.fragment,e),M(At.$$.fragment,e),M(Ce),M(it.$$.fragment,e),M(pt.$$.fragment,e),M(Ft.$$.fragment,e),M(_e),M(St.$$.fragment,e),M(qt.$$.fragment,e),M(Lt.$$.fragment,e),M(Kt.$$.fragment,e),M(Ot.$$.fragment,e),M(es.$$.fragment,e),M(ss.$$.fragment,e),M(ls.$$.fragment,e),M(v),M(as.$$.fragment,e),Ml=!0)},o(e){c(g.$$.fragment,e),c($.$$.fragment,e),c(Z.$$.fragment,e),c(G),c(V.$$.fragment,e),c(x.$$.fragment,e),c(A.$$.fragment,e),c(Y.$$.fragment,e),c(H.$$.fragment,e),c(S.$$.fragment,e),c(q.$$.fragment,e),c(D.$$.fragment,e),c(K.$$.fragment,e),c(P.$$.fragment,e),c(z.$$.fragment,e),c(O.$$.fragment,e),c(re.$$.fragment,e),c(ye.$$.fragment,e),c(we.$$.fragment,e),c(m.$$.fragment,e),c(Ct.$$.fragment,e),c(kt.$$.fragment,e),c(Zt.$$.fragment,e),c(nt.$$.fragment,e),c(Bt.$$.fragment,e),c(Te),c(fe),c(Qt.$$.fragment,e),c(be),c(W),c(ot.$$.fragment,e),c(Vt.$$.fragment,e),c(At.$$.fragment,e),c(Ce),c(it.$$.fragment,e),c(pt.$$.fragment,e),c(Ft.$$.fragment,e),c(_e),c(St.$$.fragment,e),c(qt.$$.fragment,e),c(Lt.$$.fragment,e),c(Kt.$$.fragment,e),c(Ot.$$.fragment,e),c(es.$$.fragment,e),c(ss.$$.fragment,e),c(ls.$$.fragment,e),c(v),c(as.$$.fragment,e),Ml=!1},d(e){e&&(s(r),s(n),s(y),s(b),s(k),s(J),s(R),s(X),s(_),s(B),s(Ee),s(yt),s(se),s(mt),s(Q),s(wt),s(le),s(Ae),s(ut),s(ae),s(Fe),s(E),s(Ye),s(He),s(F),s(Se),s(qe),s(ht),s(ne),s(De),s(dt),s(oe),s(Le),s(Ke),s(Jt),s(ie),s(Pe),s(L),s(Oe),s(et),s(tt),s(Tt),s(pe),s(st),s(Ut),s(Me),s(Qe),s(ce),s(me),s(ue),s(ze),s(he),s(de),s(bt),s(te),s(lt),s(ee),s(t),s(Cs),s(gs),s(gt),s(_s),s(_t),s(ks),s(Zs),s($s),s($t),s(Gs),s(Gt),s(Bs),s(Xs),s(Rs),s(Xt),s(Ws),s(ys),s(Rt),s(vs),s(Wt),s(Qs),s(ms),s(vt),s(zs),s(Ns),s(ws),s(zt),s(Vs),s(us),s(xs),s(Nt),s(Es),s(As),s(xt),s(Fs),s(Et),s(Ys),s(Hs),s(hs),s(Ss),s(qs),s(Ds),s(Yt),s(Ls),s(ds),s(Ht),s(Ks),s(Ps),s(Os),s(Dt),s(el),s(tl),s(sl),s(Pt),s(ll),s(al),s(nl),s(ts),s(ol),s(il),s(pl),s(Js),s(Ts),s(rl),s(Is)),s(l),d(g,e),d($,e),d(Z,e),ns[j].d(e),d(V,e),d(x,e),d(A,e),d(Y,e),d(H,e),d(S,e),d(q,e),d(D,e),d(K,e),d(P,e),d(z,e),d(O,e),d(re,e),d(ye,e),d(we,e),d(m,e),d(Ct,e),d(kt,e),d(Zt,e),d(nt,e),d(Bt,e),os[Je].d(e),is[Ue].d(e),d(Qt,e),ps[je].d(e),W&&W.d(e),d(ot,e),d(Vt,e),d(At,e),rs[Ie].d(e),d(it,e),d(pt,e),d(Ft,e),Ms[ge].d(e),d(St,e),d(qt,e),d(Lt,e),d(Kt,e),d(Ot,e),d(es,e),d(ss,e),d(ls,e),at.d(e),v&&v.d(e),d(as,e)}}}const xa='{"title":"Training a causal language model from scratch","local":"training-a-causal-language-model-from-scratch","sections":[{"title":"Gathering the data","local":"gathering-the-data","sections":[],"depth":2},{"title":"Preparing the dataset","local":"preparing-the-dataset","sections":[],"depth":2},{"title":"Initializing a new model","local":"initializing-a-new-model","sections":[],"depth":2},{"title":"Code generation with a pipeline","local":"code-generation-with-a-pipeline","sections":[],"depth":2},{"title":"Training with 🤗 Accelerate","local":"training-with-accelerate","sections":[],"depth":2}],"depth":1}';function Ea(C,l,r){let n="pt";return pa(()=>{const y=new URLSearchParams(window.location.search);r(0,n=y.get("fw")||"pt")}),[n]}class Ka extends ra{constructor(l){super(),Ma(this,l,Ea,Va,oa,{})}}export{Ka as component};
