import{s as zl,o as Hl,n as v}from"../chunks/scheduler.9bc65507.js";import{S as ql,i as Rl,g as m,s as l,r as h,A as Xl,h as c,f as n,c as t,j as Bl,u as o,x as u,k as _,y as Yl,a as e,v as i,d as j,t as b,w as d}from"../chunks/index.707bf1b6.js";import{T as Ia}from"../chunks/Tip.c2ecdbf4.js";import{Y as Nl}from"../chunks/Youtube.e1129c6f.js";import{C as M}from"../chunks/CodeBlock.54a9f38d.js";import{D as Fl}from"../chunks/DocNotebookDropdown.3e6b3817.js";import{F as Ll,M as Wl}from"../chunks/Markdown.fef84341.js";import{H as I}from"../chunks/Heading.342b1fa6.js";function Al(y){let p,f="<code>AutoProcessor</code> <strong>always</strong> works and automatically chooses the correct class for the model you’re using, whether you’re using a tokenizer, image processor, feature extractor or processor.";return{c(){p=m("p"),p.innerHTML=f},l(r){p=c(r,"P",{"data-svelte-h":!0}),u(p)!=="svelte-15om5of"&&(p.innerHTML=f)},m(r,g){e(r,p,g)},p:v,d(r){r&&n(p)}}}function Ql(y){let p,f="If you plan on using a pretrained model, it’s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referred to as the <em>vocab</em>) during pretraining.";return{c(){p=m("p"),p.innerHTML=f},l(r){p=c(r,"P",{"data-svelte-h":!0}),u(p)!=="svelte-1irmkk5"&&(p.innerHTML=f)},m(r,g){e(r,p,g)},p:v,d(r){r&&n(p)}}}function El(y){let p,f='Check out the <a href="./pad_truncation">Padding and truncation</a> concept guide to learn more different padding and truncation arguments.';return{c(){p=m("p"),p.innerHTML=f},l(r){p=c(r,"P",{"data-svelte-h":!0}),u(p)!=="svelte-4vcsqp"&&(p.innerHTML=f)},m(r,g){e(r,p,g)},p:v,d(r){r&&n(p)}}}function Pl(y){let p,f;return p=new M({props:{code:"YmF0Y2hfc2VudGVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyQnV0JTIwd2hhdCUyMGFib3V0JTIwc2Vjb25kJTIwYnJlYWtmYXN0JTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyRG9uJ3QlMjB0aGluayUyMGhlJTIwa25vd3MlMjBhYm91dCUyMHNlY29uZCUyMGJyZWFrZmFzdCUyQyUyMFBpcC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwYWJvdXQlMjBlbGV2ZW5zaWVzJTNGJTIyJTJDJTBBJTVEJTBBZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplcihiYXRjaF9zZW50ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFwcmludChlbmNvZGVkX2lucHV0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]),
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]),
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]])}`,wrap:!1}}),{c(){h(p.$$.fragment)},l(r){o(p.$$.fragment,r)},m(r,g){i(p,r,g),f=!0},p:v,i(r){f||(j(p.$$.fragment,r),f=!0)},o(r){b(p.$$.fragment,r),f=!1},d(r){d(p,r)}}}function Sl(y){let p,f;return p=new Wl({props:{$$slots:{default:[Pl]},$$scope:{ctx:y}}}),{c(){h(p.$$.fragment)},l(r){o(p.$$.fragment,r)},m(r,g){i(p,r,g),f=!0},p(r,g){const J={};g&2&&(J.$$scope={dirty:g,ctx:r}),p.$set(J)},i(r){f||(j(p.$$.fragment,r),f=!0)},o(r){b(p.$$.fragment,r),f=!1},d(r){d(p,r)}}}function Dl(y){let p,f;return p=new M({props:{code:"YmF0Y2hfc2VudGVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyQnV0JTIwd2hhdCUyMGFib3V0JTIwc2Vjb25kJTIwYnJlYWtmYXN0JTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyRG9uJ3QlMjB0aGluayUyMGhlJTIwa25vd3MlMjBhYm91dCUyMHNlY29uZCUyMGJyZWFrZmFzdCUyQyUyMFBpcC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwYWJvdXQlMjBlbGV2ZW5zaWVzJTNGJTIyJTJDJTBBJTVEJTBBZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplcihiYXRjaF9zZW50ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJ0ZiUyMiklMEFwcmludChlbmNvZGVkX2lucHV0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
      dtype=int32)&gt;,
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;,
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;}`,wrap:!1}}),{c(){h(p.$$.fragment)},l(r){o(p.$$.fragment,r)},m(r,g){i(p,r,g),f=!0},p:v,i(r){f||(j(p.$$.fragment,r),f=!0)},o(r){b(p.$$.fragment,r),f=!1},d(r){d(p,r)}}}function Kl(y){let p,f;return p=new Wl({props:{$$slots:{default:[Dl]},$$scope:{ctx:y}}}),{c(){h(p.$$.fragment)},l(r){o(p.$$.fragment,r)},m(r,g){i(p,r,g),f=!0},p(r,g){const J={};g&2&&(J.$$scope={dirty:g,ctx:r}),p.$set(J)},i(r){f||(j(p.$$.fragment,r),f=!0)},o(r){b(p.$$.fragment,r),f=!1},d(r){d(p,r)}}}function Ol(y){let p,f=`Image preprocessing often follows some form of image augmentation. Both image preprocessing and image augmentation
transform image data, but they serve different purposes:`,r,g,J="<li>Image augmentation alters images in a way that can help prevent overfitting and increase the robustness of the model. You can get creative in how you augment your data - adjust brightness and colors, crop, rotate, resize, zoom, etc. However, be mindful not to change the meaning of the images with your augmentations.</li> <li>Image preprocessing guarantees that the images match the model’s expected input format. When fine-tuning a computer vision model, images must be preprocessed exactly as when the model was initially trained.</li>",w,$,F="You can use any library you like for image augmentation. For image preprocessing, use the <code>ImageProcessor</code> associated with the model.";return{c(){p=m("p"),p.textContent=f,r=l(),g=m("ul"),g.innerHTML=J,w=l(),$=m("p"),$.innerHTML=F},l(T){p=c(T,"P",{"data-svelte-h":!0}),u(p)!=="svelte-aepj6y"&&(p.textContent=f),r=t(T),g=c(T,"UL",{"data-svelte-h":!0}),u(g)!=="svelte-mzhu6i"&&(g.innerHTML=J),w=t(T),$=c(T,"P",{"data-svelte-h":!0}),u($)!=="svelte-dwyc8n"&&($.innerHTML=F)},m(T,k){e(T,p,k),e(T,r,k),e(T,g,k),e(T,w,k),e(T,$,k)},p:v,d(T){T&&(n(p),n(r),n(g),n(w),n($))}}}function st(y){let p,f="Use 🤗 Datasets <code>split</code> parameter to only load a small sample from the training split since the dataset is quite large!";return{c(){p=m("p"),p.innerHTML=f},l(r){p=c(r,"P",{"data-svelte-h":!0}),u(p)!=="svelte-14ltu7c"&&(p.innerHTML=f)},m(r,g){e(r,p,g)},p:v,d(r){r&&n(p)}}}function at(y){let p,f=`In the example above we set <code>do_resize=False</code> because we have already resized the images in the image augmentation transformation,
and leveraged the <code>size</code> attribute from the appropriate <code>image_processor</code>. If you do not resize images during image augmentation,
leave this parameter out. By default, <code>ImageProcessor</code> will handle the resizing.`,r,g,J=`If you wish to normalize images as a part of the augmentation transformation, use the <code>image_processor.image_mean</code>,
and <code>image_processor.image_std</code> values.`;return{c(){p=m("p"),p.innerHTML=f,r=l(),g=m("p"),g.innerHTML=J},l(w){p=c(w,"P",{"data-svelte-h":!0}),u(p)!=="svelte-nqgnl7"&&(p.innerHTML=f),r=t(w),g=c(w,"P",{"data-svelte-h":!0}),u(g)!=="svelte-13w97bd"&&(g.innerHTML=J)},m(w,$){e(w,p,$),e(w,r,$),e(w,g,$)},p:v,d(w){w&&(n(p),n(r),n(g))}}}function nt(y){let p,f=`For tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, <code>ImageProcessor</code>
offers post processing methods. These methods convert model’s raw outputs into meaningful predictions such as bounding boxes,
or segmentation maps.`;return{c(){p=m("p"),p.innerHTML=f},l(r){p=c(r,"P",{"data-svelte-h":!0}),u(p)!=="svelte-p4rlwj"&&(p.innerHTML=f)},m(r,g){e(r,p,g)},p:v,d(r){r&&n(p)}}}function et(y){let p,f,r,g,J,w,$,F,T,k="Before you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. 🤗 Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you’ll learn that for:",ka,L,ve='<li>Text, use a <a href="./main_classes/tokenizer">Tokenizer</a> to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors.</li> <li>Speech and audio, use a <a href="./main_classes/feature_extractor">Feature extractor</a> to extract sequential features from audio waveforms and convert them into tensors.</li> <li>Image inputs use a <a href="./main_classes/image">ImageProcessor</a> to convert images into tensors.</li> <li>Multimodal inputs, use a <a href="./main_classes/processors">Processor</a> to combine a tokenizer and a feature extractor or image processor.</li>',Ua,U,xa,A,ke="Before you begin, install 🤗 Datasets so you can load some datasets to experiment with:",Ca,Q,Za,E,Ga,P,Va,S,Ue='The main tool for preprocessing textual data is a <a href="main_classes/tokenizer">tokenizer</a>. A tokenizer splits text into <em>tokens</em> according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer.',Ba,x,Wa,D,xe='Get started by loading a pretrained tokenizer with the <a href="/docs/transformers/v4.36.0/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained">AutoTokenizer.from_pretrained()</a> method. This downloads the <em>vocab</em> a model was pretrained with:',za,K,Ha,O,Ce="Then pass your text to the tokenizer:",qa,ss,Ra,as,Ze="The tokenizer returns a dictionary with three important items:",Xa,ns,Ge='<li><a href="glossary#input-ids">input_ids</a> are the indices corresponding to each token in the sentence.</li> <li><a href="glossary#attention-mask">attention_mask</a> indicates whether a token should be attended to or not.</li> <li><a href="glossary#token-type-ids">token_type_ids</a> identifies which sequence a token belongs to when there is more than one sequence.</li>',Ya,es,Ve="Return your input by decoding the <code>input_ids</code>:",Na,ls,Fa,ts,Be=`As you can see, the tokenizer added two special tokens - <code>CLS</code> and <code>SEP</code> (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer automatically adds them for you.`,La,ps,We="If there are several sentences you want to preprocess, pass them as a list to the tokenizer:",Aa,rs,Qa,ms,Ea,cs,ze="Sentences aren’t always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special <em>padding token</em> to shorter sentences.",Pa,us,He="Set the <code>padding</code> parameter to <code>True</code> to pad the shorter sequences in the batch to match the longest sequence:",Sa,hs,Da,os,qe="The first and third sentences are now padded with <code>0</code>’s because they are shorter.",Ka,is,Oa,js,Re="On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you’ll need to truncate the sequence to a shorter length.",sn,bs,Xe="Set the <code>truncation</code> parameter to <code>True</code> to truncate a sequence to the maximum length accepted by the model:",an,ds,nn,C,en,fs,ln,gs,Ye="Finally, you want the tokenizer to return the actual tensors that get fed to the model.",tn,Ms,Ne="Set the <code>return_tensors</code> parameter to either <code>pt</code> for PyTorch, or <code>tf</code> for TensorFlow:",pn,Z,rn,ys,mn,Ts,Fe='For audio tasks, you’ll need a <a href="main_classes/feature_extractor">feature extractor</a> to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors.',cn,Js,Le='Load the <a href="https://huggingface.co/datasets/PolyAI/minds14" rel="nofollow">MInDS-14</a> dataset (see the 🤗 <a href="https://huggingface.co/docs/datasets/load_hub" rel="nofollow">Datasets tutorial</a> for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:',un,ws,hn,$s,Ae="Access the first element of the <code>audio</code> column to take a look at the input. Calling the <code>audio</code> column automatically loads and resamples the audio file:",on,_s,jn,Is,Qe="This returns three items:",bn,vs,Ee="<li><code>array</code> is the speech signal loaded - and potentially resampled - as a 1D array.</li> <li><code>path</code> points to the location of the audio file.</li> <li><code>sampling_rate</code> refers to how many data points in the speech signal are measured per second.</li>",dn,ks,Pe='For this tutorial, you’ll use the <a href="https://huggingface.co/facebook/wav2vec2-base" rel="nofollow">Wav2Vec2</a> model. Take a look at the model card, and you’ll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data’s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data’s sampling rate isn’t the same, then you need to resample your data.',fn,Us,Se='<li>Use 🤗 Datasets’ <a href="https://huggingface.co/docs/datasets/v2.15.0/en/package_reference/main_classes#datasets.Dataset.cast_column" rel="nofollow">cast_column</a> method to upsample the sampling rate to 16kHz:</li>',gn,xs,Mn,G,De="<li>Call the <code>audio</code> column again to resample the audio file:</li>",yn,Cs,Tn,Zs,Ke="Next, load a feature extractor to normalize and pad the input. When padding textual data, a <code>0</code> is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a <code>0</code> - interpreted as silence - to <code>array</code>.",Jn,Gs,Oe='Load the feature extractor with <a href="/docs/transformers/v4.36.0/en/model_doc/auto#transformers.AutoFeatureExtractor.from_pretrained">AutoFeatureExtractor.from_pretrained()</a>:',wn,Vs,$n,Bs,sl="Pass the audio <code>array</code> to the feature extractor. We also recommend adding the <code>sampling_rate</code> argument in the feature extractor in order to better debug any silent errors that may occur.",_n,Ws,In,zs,al="Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:",vn,Hs,kn,qs,nl="Create a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:",Un,Rs,xn,Xs,el="Apply the <code>preprocess_function</code> to the first few examples in the dataset:",Cn,Ys,Zn,Ns,ll="The sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!",Gn,Fs,Vn,Ls,Bn,As,tl=`For computer vision tasks, you’ll need an <a href="main_classes/image_processor">image processor</a> to prepare your dataset for the model.
Image preprocessing consists of several steps that convert images into the input expected by the model. These steps
include but are not limited to resizing, normalizing, color channel correction, and converting images to tensors.`,Wn,V,zn,Qs,pl='Load the <a href="https://huggingface.co/datasets/food101" rel="nofollow">food101</a> dataset (see the 🤗 <a href="https://huggingface.co/docs/datasets/load_hub" rel="nofollow">Datasets tutorial</a> for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:',Hn,B,qn,Es,Rn,Ps,rl='Next, take a look at the image with 🤗 Datasets <a href="https://huggingface.co/docs/datasets/package_reference/main_classes?highlight=image#datasets.Image" rel="nofollow"><code>Image</code></a> feature:',Xn,Ss,Yn,W,ml='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png"/>',Nn,Ds,cl='Load the image processor with <a href="/docs/transformers/v4.36.0/en/model_doc/auto#transformers.AutoImageProcessor.from_pretrained">AutoImageProcessor.from_pretrained()</a>:',Fn,Ks,Ln,Os,ul='First, let’s add some image augmentation. You can use any library you prefer, but in this tutorial, we’ll use torchvision’s <a href="https://pytorch.org/vision/stable/transforms.html" rel="nofollow"><code>transforms</code></a> module. If you’re interested in using another data augmentation library, learn how in the <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb" rel="nofollow">Albumentations</a> or <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb" rel="nofollow">Kornia notebooks</a>.',An,sa,hl=`<li>Here we use <a href="https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html" rel="nofollow"><code>Compose</code></a> to chain together a couple of
transforms - <a href="https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html" rel="nofollow"><code>RandomResizedCrop</code></a> and <a href="https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html" rel="nofollow"><code>ColorJitter</code></a>.
Note that for resizing, we can get the image size requirements from the <code>image_processor</code>. For some models, an exact height and
width are expected, for others only the <code>shortest_edge</code> is defined.</li>`,Qn,aa,En,z,ol=`<li>The model accepts <a href="model_doc/visionencoderdecoder#transformers.VisionEncoderDecoderModel.forward.pixel_values"><code>pixel_values</code></a>
as its input. <code>ImageProcessor</code> can take care of normalizing the images, and generating appropriate tensors.
Create a function that combines image augmentation and image preprocessing for a batch of images and generates <code>pixel_values</code>:</li>`,Pn,na,Sn,H,Dn,q,il='<li>Then use 🤗 Datasets<a href="https://huggingface.co/docs/datasets/v2.15.0/en/package_reference/main_classes#datasets.Dataset.set_transform" rel="nofollow">set_transform</a> to apply the transforms on the fly:</li>',Kn,ea,On,R,jl="<li>Now when you access the image, you’ll notice the image processor has added <code>pixel_values</code>. You can pass your processed dataset to the model now!</li>",se,la,ae,ta,bl="Here is what the image looks like after the transforms are applied. The image has been randomly cropped and it’s color properties are different.",ne,pa,ee,X,dl='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png"/>',le,Y,te,ra,pe,ma,fl=`In some cases, for instance, when fine-tuning <a href="./model_doc/detr">DETR</a>, the model applies scale augmentation at training
time. This may cause images to be different sizes in a batch. You can use <code>DetrImageProcessor.pad()</code>
from <a href="/docs/transformers/v4.36.0/en/model_doc/detr#transformers.DetrImageProcessor">DetrImageProcessor</a> and define a custom <code>collate_fn</code> to batch images together.`,re,ca,me,ua,ce,ha,gl='For tasks involving multimodal inputs, you’ll need a <a href="main_classes/processors">processor</a> to prepare your dataset for the model. A processor couples together two processing objects such as as tokenizer and feature extractor.',ue,oa,Ml='Load the <a href="https://huggingface.co/datasets/lj_speech" rel="nofollow">LJ Speech</a> dataset (see the 🤗 <a href="https://huggingface.co/docs/datasets/load_hub" rel="nofollow">Datasets tutorial</a> for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):',he,ia,oe,ja,yl="For ASR, you’re mainly focused on <code>audio</code> and <code>text</code> so you can remove the other columns:",ie,ba,je,da,Tl="Now take a look at the <code>audio</code> and <code>text</code> columns:",be,fa,de,ga,Jl='Remember you should always <a href="preprocessing#audio">resample</a> your audio dataset’s sampling rate to match the sampling rate of the dataset used to pretrain a model!',fe,Ma,ge,ya,wl='Load a processor with <a href="/docs/transformers/v4.36.0/en/model_doc/auto#transformers.AutoProcessor.from_pretrained">AutoProcessor.from_pretrained()</a>:',Me,Ta,ye,Ja,$l="<li>Create a function to process the audio data contained in <code>array</code> to <code>input_values</code>, and tokenize <code>text</code> to <code>labels</code>. These are the inputs to the model:</li>",Te,wa,Je,N,_l="<li>Apply the <code>prepare_dataset</code> function to a sample:</li>",we,$a,$e,_a,Il="The processor has now added <code>input_values</code> and <code>labels</code>, and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!",_e,va,Ie;return J=new I({props:{title:"Preprocess",local:"preprocess",headingTag:"h1"}}),$=new Fl({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"}]}}),U=new Ia({props:{$$slots:{default:[Al]},$$scope:{ctx:y}}}),Q=new M({props:{code:"cGlwJTIwaW5zdGFsbCUyMGRhdGFzZXRz",highlighted:"pip install datasets",wrap:!1}}),E=new I({props:{title:"Natural Language Processing",local:"natural-language-processing",headingTag:"h2"}}),P=new Nl({props:{id:"Yffk5aydLzg"}}),x=new Ia({props:{$$slots:{default:[Ql]},$$scope:{ctx:y}}}),K=new M({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`,wrap:!1}}),ss=new M({props:{code:"ZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplciglMjJEbyUyMG5vdCUyMG1lZGRsZSUyMGluJTIwdGhlJTIwYWZmYWlycyUyMG9mJTIwd2l6YXJkcyUyQyUyMGZvciUyMHRoZXklMjBhcmUlMjBzdWJ0bGUlMjBhbmQlMjBxdWljayUyMHRvJTIwYW5nZXIuJTIyKSUwQXByaW50KGVuY29kZWRfaW5wdXQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(<span class="hljs-string">&quot;Do not meddle in the affairs of wizards, for they are subtle and quick to anger.&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2079</span>, <span class="hljs-number">2025</span>, <span class="hljs-number">19960</span>, <span class="hljs-number">10362</span>, <span class="hljs-number">1999</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">3821</span>, <span class="hljs-number">1997</span>, <span class="hljs-number">16657</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">2027</span>, <span class="hljs-number">2024</span>, <span class="hljs-number">11259</span>, <span class="hljs-number">1998</span>, <span class="hljs-number">4248</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">4963</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>],
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`,wrap:!1}}),ls=new M({props:{code:"dG9rZW5pemVyLmRlY29kZShlbmNvZGVkX2lucHV0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]&#x27;</span>`,wrap:!1}}),rs=new M({props:{code:"YmF0Y2hfc2VudGVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyQnV0JTIwd2hhdCUyMGFib3V0JTIwc2Vjb25kJTIwYnJlYWtmYXN0JTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyRG9uJ3QlMjB0aGluayUyMGhlJTIwa25vd3MlMjBhYm91dCUyMHNlY29uZCUyMGJyZWFrZmFzdCUyQyUyMFBpcC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwYWJvdXQlMjBlbGV2ZW5zaWVzJTNGJTIyJTJDJTBBJTVEJTBBZW5jb2RlZF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoYmF0Y2hfc2VudGVuY2VzKSUwQXByaW50KGVuY29kZWRfaW5wdXRzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_inputs = tokenizer(batch_sentences)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_inputs)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>],
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]],
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]}`,wrap:!1}}),ms=new I({props:{title:"Pad",local:"pad",headingTag:"h3"}}),hs=new M({props:{code:"YmF0Y2hfc2VudGVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyQnV0JTIwd2hhdCUyMGFib3V0JTIwc2Vjb25kJTIwYnJlYWtmYXN0JTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyRG9uJ3QlMjB0aGluayUyMGhlJTIwa25vd3MlMjBhYm91dCUyMHNlY29uZCUyMGJyZWFrZmFzdCUyQyUyMFBpcC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwYWJvdXQlMjBlbGV2ZW5zaWVzJTNGJTIyJTJDJTBBJTVEJTBBZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplcihiYXRjaF9zZW50ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSklMEFwcmludChlbmNvZGVkX2lucHV0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`,wrap:!1}}),is=new I({props:{title:"Truncation",local:"truncation",headingTag:"h3"}}),ds=new M({props:{code:"YmF0Y2hfc2VudGVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyQnV0JTIwd2hhdCUyMGFib3V0JTIwc2Vjb25kJTIwYnJlYWtmYXN0JTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyRG9uJ3QlMjB0aGluayUyMGhlJTIwa25vd3MlMjBhYm91dCUyMHNlY29uZCUyMGJyZWFrZmFzdCUyQyUyMFBpcC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwYWJvdXQlMjBlbGV2ZW5zaWVzJTNGJTIyJTJDJTBBJTVEJTBBZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplcihiYXRjaF9zZW50ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlKSUwQXByaW50KGVuY29kZWRfaW5wdXQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`,wrap:!1}}),C=new Ia({props:{$$slots:{default:[El]},$$scope:{ctx:y}}}),fs=new I({props:{title:"Build tensors",local:"build-tensors",headingTag:"h3"}}),Z=new Ll({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Kl],pytorch:[Sl]},$$scope:{ctx:y}}}),ys=new I({props:{title:"Audio",local:"audio",headingTag:"h2"}}),ws=new M({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTJDJTIwQXVkaW8lMEElMEFkYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMlBvbHlBSSUyRm1pbmRzMTQlMjIlMkMlMjBuYW1lJTNEJTIyZW4tVVMlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`,wrap:!1}}),_s=new M({props:{code:"ZGF0YXNldCU1QjAlNUQlNUIlMjJhdWRpbyUyMiU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`,wrap:!1}}),xs=new M({props:{code:"ZGF0YXNldCUyMCUzRCUyMGRhdGFzZXQuY2FzdF9jb2x1bW4oJTIyYXVkaW8lMjIlMkMlMjBBdWRpbyhzYW1wbGluZ19yYXRlJTNEMTZfMDAwKSk=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))',wrap:!1}}),Cs=new M({props:{code:"ZGF0YXNldCU1QjAlNUQlNUIlMjJhdWRpbyUyMiU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">2.3443763e-05</span>,  <span class="hljs-number">2.1729663e-04</span>,  <span class="hljs-number">2.2145823e-04</span>, ...,
         <span class="hljs-number">3.8356509e-05</span>, -<span class="hljs-number">7.3497440e-06</span>, -<span class="hljs-number">2.1754686e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`,wrap:!1}}),Vs=new M({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9GZWF0dXJlRXh0cmFjdG9yJTBBJTBBZmVhdHVyZV9leHRyYWN0b3IlMjAlM0QlMjBBdXRvRmVhdHVyZUV4dHJhY3Rvci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZ3YXYydmVjMi1iYXNlJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)`,wrap:!1}}),Ws=new M({props:{code:"YXVkaW9faW5wdXQlMjAlM0QlMjAlNUJkYXRhc2V0JTVCMCU1RCU1QiUyMmF1ZGlvJTIyJTVEJTVCJTIyYXJyYXklMjIlNUQlNUQlMEFmZWF0dXJlX2V4dHJhY3RvcihhdWRpb19pbnB1dCUyQyUyMHNhbXBsaW5nX3JhdGUlM0QxNjAwMCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>audio_input = [dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor(audio_input, sampling_rate=<span class="hljs-number">16000</span>)
{<span class="hljs-string">&#x27;input_values&#x27;</span>: [array([ <span class="hljs-number">3.8106556e-04</span>,  <span class="hljs-number">2.7506407e-03</span>,  <span class="hljs-number">2.8015103e-03</span>, ...,
        <span class="hljs-number">5.6335266e-04</span>,  <span class="hljs-number">4.6588284e-06</span>, -<span class="hljs-number">1.7142107e-04</span>], dtype=float32)]}`,wrap:!1}}),Hs=new M({props:{code:"ZGF0YXNldCU1QjAlNUQlNUIlMjJhdWRpbyUyMiU1RCU1QiUyMmFycmF5JTIyJTVELnNoYXBlJTBBJTBBZGF0YXNldCU1QjElNUQlNUIlMjJhdWRpbyUyMiU1RCU1QiUyMmFycmF5JTIyJTVELnNoYXBl",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">173398</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">106496</span>,)`,wrap:!1}}),Rs=new M({props:{code:"ZGVmJTIwcHJlcHJvY2Vzc19mdW5jdGlvbihleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjBhdWRpb19hcnJheXMlMjAlM0QlMjAlNUJ4JTVCJTIyYXJyYXklMjIlNUQlMjBmb3IlMjB4JTIwaW4lMjBleGFtcGxlcyU1QiUyMmF1ZGlvJTIyJTVEJTVEJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTIwJTNEJTIwZmVhdHVyZV9leHRyYWN0b3IoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYXVkaW9fYXJyYXlzJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2FtcGxpbmdfcmF0ZSUzRDE2MDAwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGFkZGluZyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXhfbGVuZ3RoJTNEMTAwMDAwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwaW5wdXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays,
<span class="hljs-meta">... </span>        sampling_rate=<span class="hljs-number">16000</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">100000</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`,wrap:!1}}),Ys=new M({props:{code:"cHJvY2Vzc2VkX2RhdGFzZXQlMjAlM0QlMjBwcmVwcm9jZXNzX2Z1bmN0aW9uKGRhdGFzZXQlNUIlM0E1JTVEKQ==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset = preprocess_function(dataset[:<span class="hljs-number">5</span>])',wrap:!1}}),Fs=new M({props:{code:"cHJvY2Vzc2VkX2RhdGFzZXQlNUIlMjJpbnB1dF92YWx1ZXMlMjIlNUQlNUIwJTVELnNoYXBlJTBBJTBBcHJvY2Vzc2VkX2RhdGFzZXQlNUIlMjJpbnB1dF92YWx1ZXMlMjIlNUQlNUIxJTVELnNoYXBl",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">0</span>].shape
(<span class="hljs-number">100000</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">1</span>].shape
(<span class="hljs-number">100000</span>,)`,wrap:!1}}),Ls=new I({props:{title:"Computer vision",local:"computer-vision",headingTag:"h2"}}),V=new Ia({props:{$$slots:{default:[Ol]},$$scope:{ctx:y}}}),B=new Ia({props:{$$slots:{default:[st]},$$scope:{ctx:y}}}),Es=new M({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJmb29kMTAxJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiU1QiUzQTEwMCU1RCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;food101&quot;</span>, split=<span class="hljs-string">&quot;train[:100]&quot;</span>)`,wrap:!1}}),Ss=new M({props:{code:"ZGF0YXNldCU1QjAlNUQlNUIlMjJpbWFnZSUyMiU1RA==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]',wrap:!1}}),Ks=new M({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9JbWFnZVByb2Nlc3NvciUwQSUwQWltYWdlX3Byb2Nlc3NvciUyMCUzRCUyMEF1dG9JbWFnZVByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGdml0LWJhc2UtcGF0Y2gxNi0yMjQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoImageProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>image_processor = AutoImageProcessor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224&quot;</span>)`,wrap:!1}}),aa=new M({props:{code:"ZnJvbSUyMHRvcmNodmlzaW9uLnRyYW5zZm9ybXMlMjBpbXBvcnQlMjBSYW5kb21SZXNpemVkQ3JvcCUyQyUyMENvbG9ySml0dGVyJTJDJTIwQ29tcG9zZSUwQSUwQXNpemUlMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwaW1hZ2VfcHJvY2Vzc29yLnNpemUlNUIlMjJzaG9ydGVzdF9lZGdlJTIyJTVEJTBBJTIwJTIwJTIwJTIwaWYlMjAlMjJzaG9ydGVzdF9lZGdlJTIyJTIwaW4lMjBpbWFnZV9wcm9jZXNzb3Iuc2l6ZSUwQSUyMCUyMCUyMCUyMGVsc2UlMjAoaW1hZ2VfcHJvY2Vzc29yLnNpemUlNUIlMjJoZWlnaHQlMjIlNUQlMkMlMjBpbWFnZV9wcm9jZXNzb3Iuc2l6ZSU1QiUyMndpZHRoJTIyJTVEKSUwQSklMEElMEFfdHJhbnNmb3JtcyUyMCUzRCUyMENvbXBvc2UoJTVCUmFuZG9tUmVzaXplZENyb3Aoc2l6ZSklMkMlMjBDb2xvckppdHRlcihicmlnaHRuZXNzJTNEMC41JTJDJTIwaHVlJTNEMC41KSU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> RandomResizedCrop, ColorJitter, Compose

<span class="hljs-meta">&gt;&gt;&gt; </span>size = (
<span class="hljs-meta">... </span>    image_processor.size[<span class="hljs-string">&quot;shortest_edge&quot;</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;shortest_edge&quot;</span> <span class="hljs-keyword">in</span> image_processor.size
<span class="hljs-meta">... </span>    <span class="hljs-keyword">else</span> (image_processor.size[<span class="hljs-string">&quot;height&quot;</span>], image_processor.size[<span class="hljs-string">&quot;width&quot;</span>])
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>_transforms = Compose([RandomResizedCrop(size), ColorJitter(brightness=<span class="hljs-number">0.5</span>, hue=<span class="hljs-number">0.5</span>)])`,wrap:!1}}),na=new M({props:{code:"ZGVmJTIwdHJhbnNmb3JtcyhleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjBpbWFnZXMlMjAlM0QlMjAlNUJfdHJhbnNmb3JtcyhpbWcuY29udmVydCglMjJSR0IlMjIpKSUyMGZvciUyMGltZyUyMGluJTIwZXhhbXBsZXMlNUIlMjJpbWFnZSUyMiU1RCU1RCUwQSUyMCUyMCUyMCUyMGV4YW1wbGVzJTVCJTIycGl4ZWxfdmFsdWVzJTIyJTVEJTIwJTNEJTIwaW1hZ2VfcHJvY2Vzc29yKGltYWdlcyUyQyUyMGRvX3Jlc2l6ZSUzREZhbHNlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklNUIlMjJwaXhlbF92YWx1ZXMlMjIlNUQlMEElMjAlMjAlMjAlMjByZXR1cm4lMjBleGFtcGxlcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    images = [_transforms(img.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> img <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = image_processor(images, do_resize=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`,wrap:!1}}),H=new Ia({props:{$$slots:{default:[at]},$$scope:{ctx:y}}}),ea=new M({props:{code:"ZGF0YXNldC5zZXRfdHJhbnNmb3JtKHRyYW5zZm9ybXMp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_transform(transforms)',wrap:!1}}),la=new M({props:{code:"ZGF0YXNldCU1QjAlNUQua2V5cygp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>].keys()',wrap:!1}}),pa=new M({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBaW1wb3J0JTIwbWF0cGxvdGxpYi5weXBsb3QlMjBhcyUyMHBsdCUwQSUwQWltZyUyMCUzRCUyMGRhdGFzZXQlNUIwJTVEJTVCJTIycGl4ZWxfdmFsdWVzJTIyJTVEJTBBcGx0Lmltc2hvdyhpbWcucGVybXV0ZSgxJTJDJTIwMiUyQyUyMDApKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt

<span class="hljs-meta">&gt;&gt;&gt; </span>img = dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>plt.imshow(img.permute(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>))`,wrap:!1}}),Y=new Ia({props:{$$slots:{default:[nt]},$$scope:{ctx:y}}}),ra=new I({props:{title:"Pad",local:"pad",headingTag:"h3"}}),ca=new M({props:{code:"ZGVmJTIwY29sbGF0ZV9mbihiYXRjaCklM0ElMEElMjAlMjAlMjAlMjBwaXhlbF92YWx1ZXMlMjAlM0QlMjAlNUJpdGVtJTVCJTIycGl4ZWxfdmFsdWVzJTIyJTVEJTIwZm9yJTIwaXRlbSUyMGluJTIwYmF0Y2glNUQlMEElMjAlMjAlMjAlMjBlbmNvZGluZyUyMCUzRCUyMGltYWdlX3Byb2Nlc3Nvci5wYWQocGl4ZWxfdmFsdWVzJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMjAlMjAlMjAlMjBsYWJlbHMlMjAlM0QlMjAlNUJpdGVtJTVCJTIybGFiZWxzJTIyJTVEJTIwZm9yJTIwaXRlbSUyMGluJTIwYmF0Y2glNUQlMEElMjAlMjAlMjAlMjBiYXRjaCUyMCUzRCUyMCU3QiU3RCUwQSUyMCUyMCUyMCUyMGJhdGNoJTVCJTIycGl4ZWxfdmFsdWVzJTIyJTVEJTIwJTNEJTIwZW5jb2RpbmclNUIlMjJwaXhlbF92YWx1ZXMlMjIlNUQlMEElMjAlMjAlMjAlMjBiYXRjaCU1QiUyMnBpeGVsX21hc2slMjIlNUQlMjAlM0QlMjBlbmNvZGluZyU1QiUyMnBpeGVsX21hc2slMjIlNUQlMEElMjAlMjAlMjAlMjBiYXRjaCU1QiUyMmxhYmVscyUyMiU1RCUyMCUzRCUyMGxhYmVscyUwQSUyMCUyMCUyMCUyMHJldHVybiUyMGJhdGNo",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">collate_fn</span>(<span class="hljs-params">batch</span>):
<span class="hljs-meta">... </span>    pixel_values = [item[<span class="hljs-string">&quot;pixel_values&quot;</span>] <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> batch]
<span class="hljs-meta">... </span>    encoding = image_processor.pad(pixel_values, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">... </span>    labels = [item[<span class="hljs-string">&quot;labels&quot;</span>] <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> batch]
<span class="hljs-meta">... </span>    batch = {}
<span class="hljs-meta">... </span>    batch[<span class="hljs-string">&quot;pixel_values&quot;</span>] = encoding[<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">... </span>    batch[<span class="hljs-string">&quot;pixel_mask&quot;</span>] = encoding[<span class="hljs-string">&quot;pixel_mask&quot;</span>]
<span class="hljs-meta">... </span>    batch[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> batch`,wrap:!1}}),ua=new I({props:{title:"Multimodal",local:"multimodal",headingTag:"h2"}}),ia=new M({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBbGpfc3BlZWNoJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmxqX3NwZWVjaCUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = load_dataset(<span class="hljs-string">&quot;lj_speech&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`,wrap:!1}}),ba=new M({props:{code:"bGpfc3BlZWNoJTIwJTNEJTIwbGpfc3BlZWNoLm1hcChyZW1vdmVfY29sdW1ucyUzRCU1QiUyMmZpbGUlMjIlMkMlMjAlMjJpZCUyMiUyQyUyMCUyMm5vcm1hbGl6ZWRfdGV4dCUyMiU1RCk=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.<span class="hljs-built_in">map</span>(remove_columns=[<span class="hljs-string">&quot;file&quot;</span>, <span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;normalized_text&quot;</span>])',wrap:!1}}),fa=new M({props:{code:"bGpfc3BlZWNoJTVCMCU1RCU1QiUyMmF1ZGlvJTIyJTVEJTBBJTBBbGpfc3BlZWNoJTVCMCU1RCU1QiUyMnRleHQlMjIlNUQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">7.3242188e-04</span>, -<span class="hljs-number">7.6293945e-04</span>, -<span class="hljs-number">6.4086914e-04</span>, ...,
         <span class="hljs-number">7.3242188e-04</span>,  <span class="hljs-number">2.1362305e-04</span>,  <span class="hljs-number">6.1035156e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">22050</span>}

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]
<span class="hljs-string">&#x27;Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition&#x27;</span>`,wrap:!1}}),Ma=new M({props:{code:"bGpfc3BlZWNoJTIwJTNEJTIwbGpfc3BlZWNoLmNhc3RfY29sdW1uKCUyMmF1ZGlvJTIyJTJDJTIwQXVkaW8oc2FtcGxpbmdfcmF0ZSUzRDE2XzAwMCkp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))',wrap:!1}}),Ta=new M({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRndhdjJ2ZWMyLWJhc2UtOTYwaCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>)`,wrap:!1}}),wa=new M({props:{code:"ZGVmJTIwcHJlcGFyZV9kYXRhc2V0KGV4YW1wbGUpJTNBJTBBJTIwJTIwJTIwJTIwYXVkaW8lMjAlM0QlMjBleGFtcGxlJTVCJTIyYXVkaW8lMjIlNUQlMEElMEElMjAlMjAlMjAlMjBleGFtcGxlLnVwZGF0ZShwcm9jZXNzb3IoYXVkaW8lM0RhdWRpbyU1QiUyMmFycmF5JTIyJTVEJTJDJTIwdGV4dCUzRGV4YW1wbGUlNUIlMjJ0ZXh0JTIyJTVEJTJDJTIwc2FtcGxpbmdfcmF0ZSUzRDE2MDAwKSklMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBleGFtcGxl",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">prepare_dataset</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    audio = example[<span class="hljs-string">&quot;audio&quot;</span>]

<span class="hljs-meta">... </span>    example.update(processor(audio=audio[<span class="hljs-string">&quot;array&quot;</span>], text=example[<span class="hljs-string">&quot;text&quot;</span>], sampling_rate=<span class="hljs-number">16000</span>))

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> example`,wrap:!1}}),$a=new M({props:{code:"cHJlcGFyZV9kYXRhc2V0KGxqX3NwZWVjaCU1QjAlNUQp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prepare_dataset(lj_speech[<span class="hljs-number">0</span>])',wrap:!1}}),{c(){p=m("meta"),f=l(),r=m("p"),g=l(),h(J.$$.fragment),w=l(),h($.$$.fragment),F=l(),T=m("p"),T.textContent=k,ka=l(),L=m("ul"),L.innerHTML=ve,Ua=l(),h(U.$$.fragment),xa=l(),A=m("p"),A.textContent=ke,Ca=l(),h(Q.$$.fragment),Za=l(),h(E.$$.fragment),Ga=l(),h(P.$$.fragment),Va=l(),S=m("p"),S.innerHTML=Ue,Ba=l(),h(x.$$.fragment),Wa=l(),D=m("p"),D.innerHTML=xe,za=l(),h(K.$$.fragment),Ha=l(),O=m("p"),O.textContent=Ce,qa=l(),h(ss.$$.fragment),Ra=l(),as=m("p"),as.textContent=Ze,Xa=l(),ns=m("ul"),ns.innerHTML=Ge,Ya=l(),es=m("p"),es.innerHTML=Ve,Na=l(),h(ls.$$.fragment),Fa=l(),ts=m("p"),ts.innerHTML=Be,La=l(),ps=m("p"),ps.textContent=We,Aa=l(),h(rs.$$.fragment),Qa=l(),h(ms.$$.fragment),Ea=l(),cs=m("p"),cs.innerHTML=ze,Pa=l(),us=m("p"),us.innerHTML=He,Sa=l(),h(hs.$$.fragment),Da=l(),os=m("p"),os.innerHTML=qe,Ka=l(),h(is.$$.fragment),Oa=l(),js=m("p"),js.textContent=Re,sn=l(),bs=m("p"),bs.innerHTML=Xe,an=l(),h(ds.$$.fragment),nn=l(),h(C.$$.fragment),en=l(),h(fs.$$.fragment),ln=l(),gs=m("p"),gs.textContent=Ye,tn=l(),Ms=m("p"),Ms.innerHTML=Ne,pn=l(),h(Z.$$.fragment),rn=l(),h(ys.$$.fragment),mn=l(),Ts=m("p"),Ts.innerHTML=Fe,cn=l(),Js=m("p"),Js.innerHTML=Le,un=l(),h(ws.$$.fragment),hn=l(),$s=m("p"),$s.innerHTML=Ae,on=l(),h(_s.$$.fragment),jn=l(),Is=m("p"),Is.textContent=Qe,bn=l(),vs=m("ul"),vs.innerHTML=Ee,dn=l(),ks=m("p"),ks.innerHTML=Pe,fn=l(),Us=m("ol"),Us.innerHTML=Se,gn=l(),h(xs.$$.fragment),Mn=l(),G=m("ol"),G.innerHTML=De,yn=l(),h(Cs.$$.fragment),Tn=l(),Zs=m("p"),Zs.innerHTML=Ke,Jn=l(),Gs=m("p"),Gs.innerHTML=Oe,wn=l(),h(Vs.$$.fragment),$n=l(),Bs=m("p"),Bs.innerHTML=sl,_n=l(),h(Ws.$$.fragment),In=l(),zs=m("p"),zs.textContent=al,vn=l(),h(Hs.$$.fragment),kn=l(),qs=m("p"),qs.textContent=nl,Un=l(),h(Rs.$$.fragment),xn=l(),Xs=m("p"),Xs.innerHTML=el,Cn=l(),h(Ys.$$.fragment),Zn=l(),Ns=m("p"),Ns.textContent=ll,Gn=l(),h(Fs.$$.fragment),Vn=l(),h(Ls.$$.fragment),Bn=l(),As=m("p"),As.innerHTML=tl,Wn=l(),h(V.$$.fragment),zn=l(),Qs=m("p"),Qs.innerHTML=pl,Hn=l(),h(B.$$.fragment),qn=l(),h(Es.$$.fragment),Rn=l(),Ps=m("p"),Ps.innerHTML=rl,Xn=l(),h(Ss.$$.fragment),Yn=l(),W=m("div"),W.innerHTML=ml,Nn=l(),Ds=m("p"),Ds.innerHTML=cl,Fn=l(),h(Ks.$$.fragment),Ln=l(),Os=m("p"),Os.innerHTML=ul,An=l(),sa=m("ol"),sa.innerHTML=hl,Qn=l(),h(aa.$$.fragment),En=l(),z=m("ol"),z.innerHTML=ol,Pn=l(),h(na.$$.fragment),Sn=l(),h(H.$$.fragment),Dn=l(),q=m("ol"),q.innerHTML=il,Kn=l(),h(ea.$$.fragment),On=l(),R=m("ol"),R.innerHTML=jl,se=l(),h(la.$$.fragment),ae=l(),ta=m("p"),ta.textContent=bl,ne=l(),h(pa.$$.fragment),ee=l(),X=m("div"),X.innerHTML=dl,le=l(),h(Y.$$.fragment),te=l(),h(ra.$$.fragment),pe=l(),ma=m("p"),ma.innerHTML=fl,re=l(),h(ca.$$.fragment),me=l(),h(ua.$$.fragment),ce=l(),ha=m("p"),ha.innerHTML=gl,ue=l(),oa=m("p"),oa.innerHTML=Ml,he=l(),h(ia.$$.fragment),oe=l(),ja=m("p"),ja.innerHTML=yl,ie=l(),h(ba.$$.fragment),je=l(),da=m("p"),da.innerHTML=Tl,be=l(),h(fa.$$.fragment),de=l(),ga=m("p"),ga.innerHTML=Jl,fe=l(),h(Ma.$$.fragment),ge=l(),ya=m("p"),ya.innerHTML=wl,Me=l(),h(Ta.$$.fragment),ye=l(),Ja=m("ol"),Ja.innerHTML=$l,Te=l(),h(wa.$$.fragment),Je=l(),N=m("ol"),N.innerHTML=_l,we=l(),h($a.$$.fragment),$e=l(),_a=m("p"),_a.innerHTML=Il,_e=l(),va=m("p"),this.h()},l(s){const a=Xl("svelte-u9bgzb",document.head);p=c(a,"META",{name:!0,content:!0}),a.forEach(n),f=t(s),r=c(s,"P",{}),Bl(r).forEach(n),g=t(s),o(J.$$.fragment,s),w=t(s),o($.$$.fragment,s),F=t(s),T=c(s,"P",{"data-svelte-h":!0}),u(T)!=="svelte-16m3n59"&&(T.textContent=k),ka=t(s),L=c(s,"UL",{"data-svelte-h":!0}),u(L)!=="svelte-97f948"&&(L.innerHTML=ve),Ua=t(s),o(U.$$.fragment,s),xa=t(s),A=c(s,"P",{"data-svelte-h":!0}),u(A)!=="svelte-miijey"&&(A.textContent=ke),Ca=t(s),o(Q.$$.fragment,s),Za=t(s),o(E.$$.fragment,s),Ga=t(s),o(P.$$.fragment,s),Va=t(s),S=c(s,"P",{"data-svelte-h":!0}),u(S)!=="svelte-1kdxp2p"&&(S.innerHTML=Ue),Ba=t(s),o(x.$$.fragment,s),Wa=t(s),D=c(s,"P",{"data-svelte-h":!0}),u(D)!=="svelte-sa48uu"&&(D.innerHTML=xe),za=t(s),o(K.$$.fragment,s),Ha=t(s),O=c(s,"P",{"data-svelte-h":!0}),u(O)!=="svelte-uv8q8p"&&(O.textContent=Ce),qa=t(s),o(ss.$$.fragment,s),Ra=t(s),as=c(s,"P",{"data-svelte-h":!0}),u(as)!=="svelte-19z6qke"&&(as.textContent=Ze),Xa=t(s),ns=c(s,"UL",{"data-svelte-h":!0}),u(ns)!=="svelte-1l83r3n"&&(ns.innerHTML=Ge),Ya=t(s),es=c(s,"P",{"data-svelte-h":!0}),u(es)!=="svelte-1nvt1pa"&&(es.innerHTML=Ve),Na=t(s),o(ls.$$.fragment,s),Fa=t(s),ts=c(s,"P",{"data-svelte-h":!0}),u(ts)!=="svelte-1b2ebhq"&&(ts.innerHTML=Be),La=t(s),ps=c(s,"P",{"data-svelte-h":!0}),u(ps)!=="svelte-838qsc"&&(ps.textContent=We),Aa=t(s),o(rs.$$.fragment,s),Qa=t(s),o(ms.$$.fragment,s),Ea=t(s),cs=c(s,"P",{"data-svelte-h":!0}),u(cs)!=="svelte-uq7hrd"&&(cs.innerHTML=ze),Pa=t(s),us=c(s,"P",{"data-svelte-h":!0}),u(us)!=="svelte-1ahwezm"&&(us.innerHTML=He),Sa=t(s),o(hs.$$.fragment,s),Da=t(s),os=c(s,"P",{"data-svelte-h":!0}),u(os)!=="svelte-375nfh"&&(os.innerHTML=qe),Ka=t(s),o(is.$$.fragment,s),Oa=t(s),js=c(s,"P",{"data-svelte-h":!0}),u(js)!=="svelte-1pf21dx"&&(js.textContent=Re),sn=t(s),bs=c(s,"P",{"data-svelte-h":!0}),u(bs)!=="svelte-1nn988l"&&(bs.innerHTML=Xe),an=t(s),o(ds.$$.fragment,s),nn=t(s),o(C.$$.fragment,s),en=t(s),o(fs.$$.fragment,s),ln=t(s),gs=c(s,"P",{"data-svelte-h":!0}),u(gs)!=="svelte-190ympp"&&(gs.textContent=Ye),tn=t(s),Ms=c(s,"P",{"data-svelte-h":!0}),u(Ms)!=="svelte-ynk6ab"&&(Ms.innerHTML=Ne),pn=t(s),o(Z.$$.fragment,s),rn=t(s),o(ys.$$.fragment,s),mn=t(s),Ts=c(s,"P",{"data-svelte-h":!0}),u(Ts)!=="svelte-1h9zu4s"&&(Ts.innerHTML=Fe),cn=t(s),Js=c(s,"P",{"data-svelte-h":!0}),u(Js)!=="svelte-16xmqn"&&(Js.innerHTML=Le),un=t(s),o(ws.$$.fragment,s),hn=t(s),$s=c(s,"P",{"data-svelte-h":!0}),u($s)!=="svelte-1hq7vhu"&&($s.innerHTML=Ae),on=t(s),o(_s.$$.fragment,s),jn=t(s),Is=c(s,"P",{"data-svelte-h":!0}),u(Is)!=="svelte-1c6azkv"&&(Is.textContent=Qe),bn=t(s),vs=c(s,"UL",{"data-svelte-h":!0}),u(vs)!=="svelte-ykpien"&&(vs.innerHTML=Ee),dn=t(s),ks=c(s,"P",{"data-svelte-h":!0}),u(ks)!=="svelte-6giqvb"&&(ks.innerHTML=Pe),fn=t(s),Us=c(s,"OL",{"data-svelte-h":!0}),u(Us)!=="svelte-t1vsm2"&&(Us.innerHTML=Se),gn=t(s),o(xs.$$.fragment,s),Mn=t(s),G=c(s,"OL",{start:!0,"data-svelte-h":!0}),u(G)!=="svelte-1965xun"&&(G.innerHTML=De),yn=t(s),o(Cs.$$.fragment,s),Tn=t(s),Zs=c(s,"P",{"data-svelte-h":!0}),u(Zs)!=="svelte-1a0ukdn"&&(Zs.innerHTML=Ke),Jn=t(s),Gs=c(s,"P",{"data-svelte-h":!0}),u(Gs)!=="svelte-1afzlfu"&&(Gs.innerHTML=Oe),wn=t(s),o(Vs.$$.fragment,s),$n=t(s),Bs=c(s,"P",{"data-svelte-h":!0}),u(Bs)!=="svelte-1yb546a"&&(Bs.innerHTML=sl),_n=t(s),o(Ws.$$.fragment,s),In=t(s),zs=c(s,"P",{"data-svelte-h":!0}),u(zs)!=="svelte-bo476j"&&(zs.textContent=al),vn=t(s),o(Hs.$$.fragment,s),kn=t(s),qs=c(s,"P",{"data-svelte-h":!0}),u(qs)!=="svelte-1407gba"&&(qs.textContent=nl),Un=t(s),o(Rs.$$.fragment,s),xn=t(s),Xs=c(s,"P",{"data-svelte-h":!0}),u(Xs)!=="svelte-mxr6t4"&&(Xs.innerHTML=el),Cn=t(s),o(Ys.$$.fragment,s),Zn=t(s),Ns=c(s,"P",{"data-svelte-h":!0}),u(Ns)!=="svelte-1n00f7l"&&(Ns.textContent=ll),Gn=t(s),o(Fs.$$.fragment,s),Vn=t(s),o(Ls.$$.fragment,s),Bn=t(s),As=c(s,"P",{"data-svelte-h":!0}),u(As)!=="svelte-14l1a6u"&&(As.innerHTML=tl),Wn=t(s),o(V.$$.fragment,s),zn=t(s),Qs=c(s,"P",{"data-svelte-h":!0}),u(Qs)!=="svelte-1aqi6mt"&&(Qs.innerHTML=pl),Hn=t(s),o(B.$$.fragment,s),qn=t(s),o(Es.$$.fragment,s),Rn=t(s),Ps=c(s,"P",{"data-svelte-h":!0}),u(Ps)!=="svelte-9ffc4n"&&(Ps.innerHTML=rl),Xn=t(s),o(Ss.$$.fragment,s),Yn=t(s),W=c(s,"DIV",{class:!0,"data-svelte-h":!0}),u(W)!=="svelte-1jzdron"&&(W.innerHTML=ml),Nn=t(s),Ds=c(s,"P",{"data-svelte-h":!0}),u(Ds)!=="svelte-dk7exh"&&(Ds.innerHTML=cl),Fn=t(s),o(Ks.$$.fragment,s),Ln=t(s),Os=c(s,"P",{"data-svelte-h":!0}),u(Os)!=="svelte-1k4l269"&&(Os.innerHTML=ul),An=t(s),sa=c(s,"OL",{"data-svelte-h":!0}),u(sa)!=="svelte-iu2fmq"&&(sa.innerHTML=hl),Qn=t(s),o(aa.$$.fragment,s),En=t(s),z=c(s,"OL",{start:!0,"data-svelte-h":!0}),u(z)!=="svelte-pnq83u"&&(z.innerHTML=ol),Pn=t(s),o(na.$$.fragment,s),Sn=t(s),o(H.$$.fragment,s),Dn=t(s),q=c(s,"OL",{start:!0,"data-svelte-h":!0}),u(q)!=="svelte-nuhaiq"&&(q.innerHTML=il),Kn=t(s),o(ea.$$.fragment,s),On=t(s),R=c(s,"OL",{start:!0,"data-svelte-h":!0}),u(R)!=="svelte-z5mlcc"&&(R.innerHTML=jl),se=t(s),o(la.$$.fragment,s),ae=t(s),ta=c(s,"P",{"data-svelte-h":!0}),u(ta)!=="svelte-trcbp4"&&(ta.textContent=bl),ne=t(s),o(pa.$$.fragment,s),ee=t(s),X=c(s,"DIV",{class:!0,"data-svelte-h":!0}),u(X)!=="svelte-mp8qzc"&&(X.innerHTML=dl),le=t(s),o(Y.$$.fragment,s),te=t(s),o(ra.$$.fragment,s),pe=t(s),ma=c(s,"P",{"data-svelte-h":!0}),u(ma)!=="svelte-16xhz32"&&(ma.innerHTML=fl),re=t(s),o(ca.$$.fragment,s),me=t(s),o(ua.$$.fragment,s),ce=t(s),ha=c(s,"P",{"data-svelte-h":!0}),u(ha)!=="svelte-199ztpc"&&(ha.innerHTML=gl),ue=t(s),oa=c(s,"P",{"data-svelte-h":!0}),u(oa)!=="svelte-d9ps8f"&&(oa.innerHTML=Ml),he=t(s),o(ia.$$.fragment,s),oe=t(s),ja=c(s,"P",{"data-svelte-h":!0}),u(ja)!=="svelte-1k5hdjd"&&(ja.innerHTML=yl),ie=t(s),o(ba.$$.fragment,s),je=t(s),da=c(s,"P",{"data-svelte-h":!0}),u(da)!=="svelte-115u1vs"&&(da.innerHTML=Tl),be=t(s),o(fa.$$.fragment,s),de=t(s),ga=c(s,"P",{"data-svelte-h":!0}),u(ga)!=="svelte-mn9i5v"&&(ga.innerHTML=Jl),fe=t(s),o(Ma.$$.fragment,s),ge=t(s),ya=c(s,"P",{"data-svelte-h":!0}),u(ya)!=="svelte-1ekdgxw"&&(ya.innerHTML=wl),Me=t(s),o(Ta.$$.fragment,s),ye=t(s),Ja=c(s,"OL",{"data-svelte-h":!0}),u(Ja)!=="svelte-1rrl7ud"&&(Ja.innerHTML=$l),Te=t(s),o(wa.$$.fragment,s),Je=t(s),N=c(s,"OL",{start:!0,"data-svelte-h":!0}),u(N)!=="svelte-ub4yka"&&(N.innerHTML=_l),we=t(s),o($a.$$.fragment,s),$e=t(s),_a=c(s,"P",{"data-svelte-h":!0}),u(_a)!=="svelte-1fqvd5l"&&(_a.innerHTML=Il),_e=t(s),va=c(s,"P",{}),Bl(va).forEach(n),this.h()},h(){_(p,"name","hf:doc:metadata"),_(p,"content",lt),_(G,"start","2"),_(W,"class","flex justify-center"),_(z,"start","2"),_(q,"start","3"),_(R,"start","4"),_(X,"class","flex justify-center"),_(N,"start","2")},m(s,a){Yl(document.head,p),e(s,f,a),e(s,r,a),e(s,g,a),i(J,s,a),e(s,w,a),i($,s,a),e(s,F,a),e(s,T,a),e(s,ka,a),e(s,L,a),e(s,Ua,a),i(U,s,a),e(s,xa,a),e(s,A,a),e(s,Ca,a),i(Q,s,a),e(s,Za,a),i(E,s,a),e(s,Ga,a),i(P,s,a),e(s,Va,a),e(s,S,a),e(s,Ba,a),i(x,s,a),e(s,Wa,a),e(s,D,a),e(s,za,a),i(K,s,a),e(s,Ha,a),e(s,O,a),e(s,qa,a),i(ss,s,a),e(s,Ra,a),e(s,as,a),e(s,Xa,a),e(s,ns,a),e(s,Ya,a),e(s,es,a),e(s,Na,a),i(ls,s,a),e(s,Fa,a),e(s,ts,a),e(s,La,a),e(s,ps,a),e(s,Aa,a),i(rs,s,a),e(s,Qa,a),i(ms,s,a),e(s,Ea,a),e(s,cs,a),e(s,Pa,a),e(s,us,a),e(s,Sa,a),i(hs,s,a),e(s,Da,a),e(s,os,a),e(s,Ka,a),i(is,s,a),e(s,Oa,a),e(s,js,a),e(s,sn,a),e(s,bs,a),e(s,an,a),i(ds,s,a),e(s,nn,a),i(C,s,a),e(s,en,a),i(fs,s,a),e(s,ln,a),e(s,gs,a),e(s,tn,a),e(s,Ms,a),e(s,pn,a),i(Z,s,a),e(s,rn,a),i(ys,s,a),e(s,mn,a),e(s,Ts,a),e(s,cn,a),e(s,Js,a),e(s,un,a),i(ws,s,a),e(s,hn,a),e(s,$s,a),e(s,on,a),i(_s,s,a),e(s,jn,a),e(s,Is,a),e(s,bn,a),e(s,vs,a),e(s,dn,a),e(s,ks,a),e(s,fn,a),e(s,Us,a),e(s,gn,a),i(xs,s,a),e(s,Mn,a),e(s,G,a),e(s,yn,a),i(Cs,s,a),e(s,Tn,a),e(s,Zs,a),e(s,Jn,a),e(s,Gs,a),e(s,wn,a),i(Vs,s,a),e(s,$n,a),e(s,Bs,a),e(s,_n,a),i(Ws,s,a),e(s,In,a),e(s,zs,a),e(s,vn,a),i(Hs,s,a),e(s,kn,a),e(s,qs,a),e(s,Un,a),i(Rs,s,a),e(s,xn,a),e(s,Xs,a),e(s,Cn,a),i(Ys,s,a),e(s,Zn,a),e(s,Ns,a),e(s,Gn,a),i(Fs,s,a),e(s,Vn,a),i(Ls,s,a),e(s,Bn,a),e(s,As,a),e(s,Wn,a),i(V,s,a),e(s,zn,a),e(s,Qs,a),e(s,Hn,a),i(B,s,a),e(s,qn,a),i(Es,s,a),e(s,Rn,a),e(s,Ps,a),e(s,Xn,a),i(Ss,s,a),e(s,Yn,a),e(s,W,a),e(s,Nn,a),e(s,Ds,a),e(s,Fn,a),i(Ks,s,a),e(s,Ln,a),e(s,Os,a),e(s,An,a),e(s,sa,a),e(s,Qn,a),i(aa,s,a),e(s,En,a),e(s,z,a),e(s,Pn,a),i(na,s,a),e(s,Sn,a),i(H,s,a),e(s,Dn,a),e(s,q,a),e(s,Kn,a),i(ea,s,a),e(s,On,a),e(s,R,a),e(s,se,a),i(la,s,a),e(s,ae,a),e(s,ta,a),e(s,ne,a),i(pa,s,a),e(s,ee,a),e(s,X,a),e(s,le,a),i(Y,s,a),e(s,te,a),i(ra,s,a),e(s,pe,a),e(s,ma,a),e(s,re,a),i(ca,s,a),e(s,me,a),i(ua,s,a),e(s,ce,a),e(s,ha,a),e(s,ue,a),e(s,oa,a),e(s,he,a),i(ia,s,a),e(s,oe,a),e(s,ja,a),e(s,ie,a),i(ba,s,a),e(s,je,a),e(s,da,a),e(s,be,a),i(fa,s,a),e(s,de,a),e(s,ga,a),e(s,fe,a),i(Ma,s,a),e(s,ge,a),e(s,ya,a),e(s,Me,a),i(Ta,s,a),e(s,ye,a),e(s,Ja,a),e(s,Te,a),i(wa,s,a),e(s,Je,a),e(s,N,a),e(s,we,a),i($a,s,a),e(s,$e,a),e(s,_a,a),e(s,_e,a),e(s,va,a),Ie=!0},p(s,[a]){const vl={};a&2&&(vl.$$scope={dirty:a,ctx:s}),U.$set(vl);const kl={};a&2&&(kl.$$scope={dirty:a,ctx:s}),x.$set(kl);const Ul={};a&2&&(Ul.$$scope={dirty:a,ctx:s}),C.$set(Ul);const xl={};a&2&&(xl.$$scope={dirty:a,ctx:s}),Z.$set(xl);const Cl={};a&2&&(Cl.$$scope={dirty:a,ctx:s}),V.$set(Cl);const Zl={};a&2&&(Zl.$$scope={dirty:a,ctx:s}),B.$set(Zl);const Gl={};a&2&&(Gl.$$scope={dirty:a,ctx:s}),H.$set(Gl);const Vl={};a&2&&(Vl.$$scope={dirty:a,ctx:s}),Y.$set(Vl)},i(s){Ie||(j(J.$$.fragment,s),j($.$$.fragment,s),j(U.$$.fragment,s),j(Q.$$.fragment,s),j(E.$$.fragment,s),j(P.$$.fragment,s),j(x.$$.fragment,s),j(K.$$.fragment,s),j(ss.$$.fragment,s),j(ls.$$.fragment,s),j(rs.$$.fragment,s),j(ms.$$.fragment,s),j(hs.$$.fragment,s),j(is.$$.fragment,s),j(ds.$$.fragment,s),j(C.$$.fragment,s),j(fs.$$.fragment,s),j(Z.$$.fragment,s),j(ys.$$.fragment,s),j(ws.$$.fragment,s),j(_s.$$.fragment,s),j(xs.$$.fragment,s),j(Cs.$$.fragment,s),j(Vs.$$.fragment,s),j(Ws.$$.fragment,s),j(Hs.$$.fragment,s),j(Rs.$$.fragment,s),j(Ys.$$.fragment,s),j(Fs.$$.fragment,s),j(Ls.$$.fragment,s),j(V.$$.fragment,s),j(B.$$.fragment,s),j(Es.$$.fragment,s),j(Ss.$$.fragment,s),j(Ks.$$.fragment,s),j(aa.$$.fragment,s),j(na.$$.fragment,s),j(H.$$.fragment,s),j(ea.$$.fragment,s),j(la.$$.fragment,s),j(pa.$$.fragment,s),j(Y.$$.fragment,s),j(ra.$$.fragment,s),j(ca.$$.fragment,s),j(ua.$$.fragment,s),j(ia.$$.fragment,s),j(ba.$$.fragment,s),j(fa.$$.fragment,s),j(Ma.$$.fragment,s),j(Ta.$$.fragment,s),j(wa.$$.fragment,s),j($a.$$.fragment,s),Ie=!0)},o(s){b(J.$$.fragment,s),b($.$$.fragment,s),b(U.$$.fragment,s),b(Q.$$.fragment,s),b(E.$$.fragment,s),b(P.$$.fragment,s),b(x.$$.fragment,s),b(K.$$.fragment,s),b(ss.$$.fragment,s),b(ls.$$.fragment,s),b(rs.$$.fragment,s),b(ms.$$.fragment,s),b(hs.$$.fragment,s),b(is.$$.fragment,s),b(ds.$$.fragment,s),b(C.$$.fragment,s),b(fs.$$.fragment,s),b(Z.$$.fragment,s),b(ys.$$.fragment,s),b(ws.$$.fragment,s),b(_s.$$.fragment,s),b(xs.$$.fragment,s),b(Cs.$$.fragment,s),b(Vs.$$.fragment,s),b(Ws.$$.fragment,s),b(Hs.$$.fragment,s),b(Rs.$$.fragment,s),b(Ys.$$.fragment,s),b(Fs.$$.fragment,s),b(Ls.$$.fragment,s),b(V.$$.fragment,s),b(B.$$.fragment,s),b(Es.$$.fragment,s),b(Ss.$$.fragment,s),b(Ks.$$.fragment,s),b(aa.$$.fragment,s),b(na.$$.fragment,s),b(H.$$.fragment,s),b(ea.$$.fragment,s),b(la.$$.fragment,s),b(pa.$$.fragment,s),b(Y.$$.fragment,s),b(ra.$$.fragment,s),b(ca.$$.fragment,s),b(ua.$$.fragment,s),b(ia.$$.fragment,s),b(ba.$$.fragment,s),b(fa.$$.fragment,s),b(Ma.$$.fragment,s),b(Ta.$$.fragment,s),b(wa.$$.fragment,s),b($a.$$.fragment,s),Ie=!1},d(s){s&&(n(f),n(r),n(g),n(w),n(F),n(T),n(ka),n(L),n(Ua),n(xa),n(A),n(Ca),n(Za),n(Ga),n(Va),n(S),n(Ba),n(Wa),n(D),n(za),n(Ha),n(O),n(qa),n(Ra),n(as),n(Xa),n(ns),n(Ya),n(es),n(Na),n(Fa),n(ts),n(La),n(ps),n(Aa),n(Qa),n(Ea),n(cs),n(Pa),n(us),n(Sa),n(Da),n(os),n(Ka),n(Oa),n(js),n(sn),n(bs),n(an),n(nn),n(en),n(ln),n(gs),n(tn),n(Ms),n(pn),n(rn),n(mn),n(Ts),n(cn),n(Js),n(un),n(hn),n($s),n(on),n(jn),n(Is),n(bn),n(vs),n(dn),n(ks),n(fn),n(Us),n(gn),n(Mn),n(G),n(yn),n(Tn),n(Zs),n(Jn),n(Gs),n(wn),n($n),n(Bs),n(_n),n(In),n(zs),n(vn),n(kn),n(qs),n(Un),n(xn),n(Xs),n(Cn),n(Zn),n(Ns),n(Gn),n(Vn),n(Bn),n(As),n(Wn),n(zn),n(Qs),n(Hn),n(qn),n(Rn),n(Ps),n(Xn),n(Yn),n(W),n(Nn),n(Ds),n(Fn),n(Ln),n(Os),n(An),n(sa),n(Qn),n(En),n(z),n(Pn),n(Sn),n(Dn),n(q),n(Kn),n(On),n(R),n(se),n(ae),n(ta),n(ne),n(ee),n(X),n(le),n(te),n(pe),n(ma),n(re),n(me),n(ce),n(ha),n(ue),n(oa),n(he),n(oe),n(ja),n(ie),n(je),n(da),n(be),n(de),n(ga),n(fe),n(ge),n(ya),n(Me),n(ye),n(Ja),n(Te),n(Je),n(N),n(we),n($e),n(_a),n(_e),n(va)),n(p),d(J,s),d($,s),d(U,s),d(Q,s),d(E,s),d(P,s),d(x,s),d(K,s),d(ss,s),d(ls,s),d(rs,s),d(ms,s),d(hs,s),d(is,s),d(ds,s),d(C,s),d(fs,s),d(Z,s),d(ys,s),d(ws,s),d(_s,s),d(xs,s),d(Cs,s),d(Vs,s),d(Ws,s),d(Hs,s),d(Rs,s),d(Ys,s),d(Fs,s),d(Ls,s),d(V,s),d(B,s),d(Es,s),d(Ss,s),d(Ks,s),d(aa,s),d(na,s),d(H,s),d(ea,s),d(la,s),d(pa,s),d(Y,s),d(ra,s),d(ca,s),d(ua,s),d(ia,s),d(ba,s),d(fa,s),d(Ma,s),d(Ta,s),d(wa,s),d($a,s)}}}const lt='{"title":"Preprocess","local":"preprocess","sections":[{"title":"Natural Language Processing","local":"natural-language-processing","sections":[{"title":"Pad","local":"pad","sections":[],"depth":3},{"title":"Truncation","local":"truncation","sections":[],"depth":3},{"title":"Build tensors","local":"build-tensors","sections":[],"depth":3}],"depth":2},{"title":"Audio","local":"audio","sections":[],"depth":2},{"title":"Computer vision","local":"computer-vision","sections":[{"title":"Pad","local":"pad","sections":[],"depth":3}],"depth":2},{"title":"Multimodal","local":"multimodal","sections":[],"depth":2}],"depth":1}';function tt(y){return Hl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jt extends ql{constructor(p){super(),Rl(this,p,tt,et,zl,{})}}export{jt as component};
