import{S as hd,i as md,s as ud,e as p,k as h,w as j,t as n,M as id,c as r,d as e,m,a as o,x as f,h as l,b as i,N as rd,G as a,g as c,y as b,q as g,o as y,B as w,v as dd,L as od}from"../chunks/vendor-hf-doc-builder.js";import{T as Ie}from"../chunks/Tip-hf-doc-builder.js";import{Y as jd}from"../chunks/Youtube-hf-doc-builder.js";import{I as ks}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as E}from"../chunks/CodeBlock-hf-doc-builder.js";import{D as fd}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as bd,M as cd}from"../chunks/Markdown-hf-doc-builder.js";function gd(k){let u,_,d,J,$,v,T;return{c(){u=p("p"),_=p("code"),d=n("AutoProcessor"),J=h(),$=p("strong"),v=n("always"),T=n(" works and automatically chooses the correct class for the model you\u2019re using, whether you\u2019re using a tokenizer, image processor, feature extractor or processor.")},l(I){u=r(I,"P",{});var x=o(u);_=r(x,"CODE",{});var V=o(_);d=l(V,"AutoProcessor"),V.forEach(e),J=m(x),$=r(x,"STRONG",{});var U=o($);v=l(U,"always"),U.forEach(e),T=l(x," works and automatically chooses the correct class for the model you\u2019re using, whether you\u2019re using a tokenizer, image processor, feature extractor or processor."),x.forEach(e)},m(I,x){c(I,u,x),a(u,_),a(_,d),a(u,J),a(u,$),a($,v),a(u,T)},d(I){I&&e(u)}}}function yd(k){let u,_,d,J,$;return{c(){u=p("p"),_=n("If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referred to as the "),d=p("em"),J=n("vocab"),$=n(") during pretraining.")},l(v){u=r(v,"P",{});var T=o(u);_=l(T,"If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referred to as the "),d=r(T,"EM",{});var I=o(d);J=l(I,"vocab"),I.forEach(e),$=l(T,") during pretraining."),T.forEach(e)},m(v,T){c(v,u,T),a(u,_),a(u,d),a(d,J),a(u,$)},d(v){v&&e(u)}}}function wd(k){let u,_,d,J,$;return{c(){u=p("p"),_=n("Check out the "),d=p("a"),J=n("Padding and truncation"),$=n(" concept guide to learn more different padding and truncation arguments."),this.h()},l(v){u=r(v,"P",{});var T=o(u);_=l(T,"Check out the "),d=r(T,"A",{href:!0});var I=o(d);J=l(I,"Padding and truncation"),I.forEach(e),$=l(T," concept guide to learn more different padding and truncation arguments."),T.forEach(e),this.h()},h(){i(d,"href","./pad_truncation")},m(v,T){c(v,u,T),a(u,_),a(u,d),a(d,J),a(u,$)},d(v){v&&e(u)}}}function Jd(k){let u,_;return u=new E({props:{code:"YmF0Y2hfc2VudGVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyQnV0JTIwd2hhdCUyMGFib3V0JTIwc2Vjb25kJTIwYnJlYWtmYXN0JTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyRG9uJ3QlMjB0aGluayUyMGhlJTIwa25vd3MlMjBhYm91dCUyMHNlY29uZCUyMGJyZWFrZmFzdCUyQyUyMFBpcC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwYWJvdXQlMjBlbGV2ZW5zaWVzJTNGJTIyJTJDJTBBJTVEJTBBZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplcihiYXRjaF9zZW50ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFwcmludChlbmNvZGVkX2lucHV0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]])}`}}),{c(){j(u.$$.fragment)},l(d){f(u.$$.fragment,d)},m(d,J){b(u,d,J),_=!0},p:od,i(d){_||(g(u.$$.fragment,d),_=!0)},o(d){y(u.$$.fragment,d),_=!1},d(d){w(u,d)}}}function _d(k){let u,_;return u=new cd({props:{$$slots:{default:[Jd]},$$scope:{ctx:k}}}),{c(){j(u.$$.fragment)},l(d){f(u.$$.fragment,d)},m(d,J){b(u,d,J),_=!0},p(d,J){const $={};J&2&&($.$$scope={dirty:J,ctx:d}),u.$set($)},i(d){_||(g(u.$$.fragment,d),_=!0)},o(d){y(u.$$.fragment,d),_=!1},d(d){w(u,d)}}}function vd(k){let u,_;return u=new E({props:{code:"YmF0Y2hfc2VudGVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyQnV0JTIwd2hhdCUyMGFib3V0JTIwc2Vjb25kJTIwYnJlYWtmYXN0JTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyRG9uJ3QlMjB0aGluayUyMGhlJTIwa25vd3MlMjBhYm91dCUyMHNlY29uZCUyMGJyZWFrZmFzdCUyQyUyMFBpcC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwYWJvdXQlMjBlbGV2ZW5zaWVzJTNGJTIyJTJDJTBBJTVEJTBBZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplcihiYXRjaF9zZW50ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJ0ZiUyMiklMEFwcmludChlbmNvZGVkX2lucHV0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
      dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;}`}}),{c(){j(u.$$.fragment)},l(d){f(u.$$.fragment,d)},m(d,J){b(u,d,J),_=!0},p:od,i(d){_||(g(u.$$.fragment,d),_=!0)},o(d){y(u.$$.fragment,d),_=!1},d(d){w(u,d)}}}function $d(k){let u,_;return u=new cd({props:{$$slots:{default:[vd]},$$scope:{ctx:k}}}),{c(){j(u.$$.fragment)},l(d){f(u.$$.fragment,d)},m(d,J){b(u,d,J),_=!0},p(d,J){const $={};J&2&&($.$$scope={dirty:J,ctx:d}),u.$set($)},i(d){_||(g(u.$$.fragment,d),_=!0)},o(d){y(u.$$.fragment,d),_=!1},d(d){w(u,d)}}}function Td(k){let u,_,d,J,$,v,T,I,x,V,U,z,B,S,R;return{c(){u=p("p"),_=n(`Image preprocessing often follows some form of image augmentation. Both image preprocessing and image augmentation
transform image data, but they serve different purposes:`),d=h(),J=p("ul"),$=p("li"),v=n("Image augmentation alters images in a way that can help prevent overfitting and increase the robustness of the model. You can get creative in how you augment your data - adjust brightness and colors, crop, rotate, resize, zoom, etc. However, be mindful not to change the meaning of the images with your augmentations."),T=h(),I=p("li"),x=n("Image preprocessing guarantees that the images match the model\u2019s expected input format. When fine-tuning a computer vision model, images must be preprocessed exactly as when the model was initially trained."),V=h(),U=p("p"),z=n("You can use any library you like for image augmentation. For image preprocessing, use the "),B=p("code"),S=n("ImageProcessor"),R=n(" associated with the model.")},l(M){u=r(M,"P",{});var Z=o(u);_=l(Z,`Image preprocessing often follows some form of image augmentation. Both image preprocessing and image augmentation
transform image data, but they serve different purposes:`),Z.forEach(e),d=m(M),J=r(M,"UL",{});var W=o(J);$=r(W,"LI",{});var F=o($);v=l(F,"Image augmentation alters images in a way that can help prevent overfitting and increase the robustness of the model. You can get creative in how you augment your data - adjust brightness and colors, crop, rotate, resize, zoom, etc. However, be mindful not to change the meaning of the images with your augmentations."),F.forEach(e),T=m(W),I=r(W,"LI",{});var K=o(I);x=l(K,"Image preprocessing guarantees that the images match the model\u2019s expected input format. When fine-tuning a computer vision model, images must be preprocessed exactly as when the model was initially trained."),K.forEach(e),W.forEach(e),V=m(M),U=r(M,"P",{});var N=o(U);z=l(N,"You can use any library you like for image augmentation. For image preprocessing, use the "),B=r(N,"CODE",{});var ss=o(B);S=l(ss,"ImageProcessor"),ss.forEach(e),R=l(N," associated with the model."),N.forEach(e)},m(M,Z){c(M,u,Z),a(u,_),c(M,d,Z),c(M,J,Z),a(J,$),a($,v),a(J,T),a(J,I),a(I,x),c(M,V,Z),c(M,U,Z),a(U,z),a(U,B),a(B,S),a(U,R)},d(M){M&&e(u),M&&e(d),M&&e(J),M&&e(V),M&&e(U)}}}function Md(k){let u,_,d,J,$;return{c(){u=p("p"),_=n("Use \u{1F917} Datasets "),d=p("code"),J=n("split"),$=n(" parameter to only load a small sample from the training split since the dataset is quite large!")},l(v){u=r(v,"P",{});var T=o(u);_=l(T,"Use \u{1F917} Datasets "),d=r(T,"CODE",{});var I=o(d);J=l(I,"split"),I.forEach(e),$=l(T," parameter to only load a small sample from the training split since the dataset is quite large!"),T.forEach(e)},m(v,T){c(v,u,T),a(u,_),a(u,d),a(d,J),a(u,$)},d(v){v&&e(u)}}}function Ed(k){let u,_,d,J,$,v,T,I,x,V,U,z,B,S,R,M,Z,W,F,K,N,ss,H;return{c(){u=p("p"),_=n("In the example above we set "),d=p("code"),J=n("do_resize=False"),$=n(` because we have already resized the images in the image augmentation transformation,
and leveraged the `),v=p("code"),T=n("size"),I=n(" attribute from the appropriate "),x=p("code"),V=n("image_processor"),U=n(`. If you do not resize images during image augmentation,
leave this parameter out. By default, `),z=p("code"),B=n("ImageProcessor"),S=n(" will handle the resizing."),R=h(),M=p("p"),Z=n("If you wish to normalize images as a part of the augmentation transformation, use the "),W=p("code"),F=n("image_processor.image_mean"),K=n(`,
and `),N=p("code"),ss=n("image_processor.image_std"),H=n(" values.")},l(G){u=r(G,"P",{});var C=o(u);_=l(C,"In the example above we set "),d=r(C,"CODE",{});var ke=o(d);J=l(ke,"do_resize=False"),ke.forEach(e),$=l(C,` because we have already resized the images in the image augmentation transformation,
and leveraged the `),v=r(C,"CODE",{});var Ue=o(v);T=l(Ue,"size"),Ue.forEach(e),I=l(C," attribute from the appropriate "),x=r(C,"CODE",{});var xe=o(x);V=l(xe,"image_processor"),xe.forEach(e),U=l(C,`. If you do not resize images during image augmentation,
leave this parameter out. By default, `),z=r(C,"CODE",{});var as=o(z);B=l(as,"ImageProcessor"),as.forEach(e),S=l(C," will handle the resizing."),C.forEach(e),R=m(G),M=r(G,"P",{});var es=o(M);Z=l(es,"If you wish to normalize images as a part of the augmentation transformation, use the "),W=r(es,"CODE",{});var ws=o(W);F=l(ws,"image_processor.image_mean"),ws.forEach(e),K=l(es,`,
and `),N=r(es,"CODE",{});var Ce=o(N);ss=l(Ce,"image_processor.image_std"),Ce.forEach(e),H=l(es," values."),es.forEach(e)},m(G,C){c(G,u,C),a(u,_),a(u,d),a(d,J),a(u,$),a(u,v),a(v,T),a(u,I),a(u,x),a(x,V),a(u,U),a(u,z),a(z,B),a(u,S),c(G,R,C),c(G,M,C),a(M,Z),a(M,W),a(W,F),a(M,K),a(M,N),a(N,ss),a(M,H)},d(G){G&&e(u),G&&e(R),G&&e(M)}}}function Id(k){let u,_,d,J,$;return{c(){u=p("p"),_=n("For tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, "),d=p("code"),J=n("ImageProcessor"),$=n(`
offers post processing methods. These methods convert model\u2019s raw outputs into meaningful predictions such as bounding boxes,
or segmentation maps.`)},l(v){u=r(v,"P",{});var T=o(u);_=l(T,"For tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, "),d=r(T,"CODE",{});var I=o(d);J=l(I,"ImageProcessor"),I.forEach(e),$=l(T,`
offers post processing methods. These methods convert model\u2019s raw outputs into meaningful predictions such as bounding boxes,
or segmentation maps.`),T.forEach(e)},m(v,T){c(v,u,T),a(u,_),a(u,d),a(d,J),a(u,$)},d(v){v&&e(u)}}}function kd(k){let u,_,d,J,$,v,T,I,x,V,U,z,B,S,R,M,Z,W,F,K,N,ss,H,G,C,ke,Ue,xe,as,es,ws,Ce,_r,vr,ca,$r,Ze,Tr,Mr,Ll,Us,Ol,Ve,Er,Kl,ha,st,Js,xs,Cn,ma,Ir,Zn,kr,at,ua,et,ns,Ur,Be,xr,Cr,Vn,Zr,Vr,nt,Cs,lt,ls,Br,Ge,Gr,Ar,Bn,zr,Wr,tt,ia,pt,Ae,Nr,rt,da,ot,ze,Yr,ct,ts,We,Ne,Rr,Fr,qr,Ye,Re,Xr,Pr,Qr,Fe,qe,Dr,Sr,ht,Zs,Hr,Gn,Lr,Or,mt,ja,ut,ps,Kr,An,so,ao,zn,eo,no,it,Xe,lo,dt,fa,jt,_s,Vs,Wn,ba,to,Nn,po,ft,Bs,ro,Yn,oo,co,bt,rs,ho,Rn,mo,uo,Fn,io,jo,gt,ga,yt,Gs,fo,qn,bo,go,wt,vs,As,Xn,ya,yo,Pn,wo,Jt,Pe,Jo,_t,os,_o,Qn,vo,$o,Dn,To,Mo,vt,wa,$t,zs,Tt,$s,Ws,Sn,Ja,Eo,Hn,Io,Mt,Qe,ko,Et,X,Uo,Ln,xo,Co,On,Zo,Vo,Kn,Bo,Go,It,Ns,kt,Ts,Ys,sl,_a,Ao,al,zo,Ut,Rs,Wo,De,No,Yo,xt,cs,Ro,va,Fo,qo,$a,Xo,Po,Ct,Ta,Zt,hs,Qo,el,Do,So,nl,Ho,Lo,Vt,Ma,Bt,Se,Oo,Gt,ms,He,ll,Ko,sc,ac,Le,tl,ec,nc,lc,Oe,pl,tc,pc,At,Fs,rc,Ea,oc,cc,zt,Ke,Ia,hc,ka,mc,uc,Wt,Ua,Nt,xa,Ca,ic,rl,dc,jc,Yt,Za,Rt,P,fc,ol,bc,gc,cl,yc,wc,hl,Jc,_c,Ft,qs,vc,sn,$c,Tc,qt,Va,Xt,us,Mc,ml,Ec,Ic,ul,kc,Uc,Pt,Ba,Qt,an,xc,Dt,Ga,St,en,Cc,Ht,Aa,Lt,Xs,Zc,il,Vc,Bc,Ot,za,Kt,nn,Gc,sp,Wa,ap,Ms,Ps,dl,Na,Ac,jl,zc,ep,Qs,Wc,ln,Nc,Yc,np,Ds,lp,is,Rc,Ya,Fc,qc,Ra,Xc,Pc,tp,Ss,pp,Fa,rp,Hs,Qc,qa,fl,Dc,Sc,op,Xa,cp,Pa,bl,Am,hp,Ls,Hc,tn,Lc,Oc,mp,Qa,up,Q,Kc,Da,gl,sh,ah,Sa,eh,nh,Ha,lh,th,ip,pn,A,ph,La,yl,rh,oh,Oa,wl,ch,hh,Ka,Jl,mh,uh,_l,ih,dh,vl,jh,fh,dp,se,jp,ae,L,bh,rn,$l,gh,yh,Tl,wh,Jh,Ml,_h,vh,fp,ee,bp,Os,gp,ne,le,$h,te,El,Th,Mh,yp,pe,wp,re,oe,Eh,Il,Ih,kh,Jp,ce,_p,on,Uh,vp,he,$p,me,kl,zm,Tp,Ks,Mp,Es,sa,Ul,ue,xh,xl,Ch,Ep,Y,Zh,cn,Vh,Bh,hn,Gh,Ah,mn,zh,Wh,Cl,Nh,Yh,Ip,ie,kp,Is,aa,Zl,de,Rh,Vl,Fh,Up,ea,qh,un,Xh,Ph,xp,ds,Qh,je,Dh,Sh,fe,Hh,Lh,Cp,be,Zp,js,Oh,Bl,Kh,sm,Gl,am,em,Vp,ge,Bp,fs,nm,Al,lm,tm,zl,pm,rm,Gp,ye,Ap,na,om,dn,cm,hm,zp,we,Wp,la,mm,jn,um,im,Np,Je,Yp,fn,q,dm,Wl,jm,fm,Nl,bm,gm,Yl,ym,wm,Rl,Jm,_m,Rp,_e,Fp,ve,$e,vm,Fl,$m,Tm,qp,Te,Xp,bs,Mm,ql,Em,Im,Xl,km,Um,Pp;return v=new ks({}),U=new fd({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"}]}}),Us=new Ie({props:{$$slots:{default:[gd]},$$scope:{ctx:k}}}),ha=new E({props:{code:"cGlwJTIwaW5zdGFsbCUyMGRhdGFzZXRz",highlighted:"pip install datasets"}}),ma=new ks({}),ua=new jd({props:{id:"Yffk5aydLzg"}}),Cs=new Ie({props:{$$slots:{default:[yd]},$$scope:{ctx:k}}}),ia=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),da=new E({props:{code:"ZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplciglMjJEbyUyMG5vdCUyMG1lZGRsZSUyMGluJTIwdGhlJTIwYWZmYWlycyUyMG9mJTIwd2l6YXJkcyUyQyUyMGZvciUyMHRoZXklMjBhcmUlMjBzdWJ0bGUlMjBhbmQlMjBxdWljayUyMHRvJTIwYW5nZXIuJTIyKSUwQXByaW50KGVuY29kZWRfaW5wdXQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(<span class="hljs-string">&quot;Do not meddle in the affairs of wizards, for they are subtle and quick to anger.&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2079</span>, <span class="hljs-number">2025</span>, <span class="hljs-number">19960</span>, <span class="hljs-number">10362</span>, <span class="hljs-number">1999</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">3821</span>, <span class="hljs-number">1997</span>, <span class="hljs-number">16657</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">2027</span>, <span class="hljs-number">2024</span>, <span class="hljs-number">11259</span>, <span class="hljs-number">1998</span>, <span class="hljs-number">4248</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">4963</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),ja=new E({props:{code:"dG9rZW5pemVyLmRlY29kZShlbmNvZGVkX2lucHV0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]&#x27;</span>`}}),fa=new E({props:{code:"YmF0Y2hfc2VudGVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyQnV0JTIwd2hhdCUyMGFib3V0JTIwc2Vjb25kJTIwYnJlYWtmYXN0JTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyRG9uJ3QlMjB0aGluayUyMGhlJTIwa25vd3MlMjBhYm91dCUyMHNlY29uZCUyMGJyZWFrZmFzdCUyQyUyMFBpcC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwYWJvdXQlMjBlbGV2ZW5zaWVzJTNGJTIyJTJDJTBBJTVEJTBBZW5jb2RlZF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoYmF0Y2hfc2VudGVuY2VzKSUwQXByaW50KGVuY29kZWRfaW5wdXRzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_inputs = tokenizer(batch_sentences)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_inputs)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]}`}}),ba=new ks({}),ga=new E({props:{code:"YmF0Y2hfc2VudGVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyQnV0JTIwd2hhdCUyMGFib3V0JTIwc2Vjb25kJTIwYnJlYWtmYXN0JTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyRG9uJ3QlMjB0aGluayUyMGhlJTIwa25vd3MlMjBhYm91dCUyMHNlY29uZCUyMGJyZWFrZmFzdCUyQyUyMFBpcC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwYWJvdXQlMjBlbGV2ZW5zaWVzJTNGJTIyJTJDJTBBJTVEJTBBZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplcihiYXRjaF9zZW50ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSklMEFwcmludChlbmNvZGVkX2lucHV0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),ya=new ks({}),wa=new E({props:{code:"YmF0Y2hfc2VudGVuY2VzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyQnV0JTIwd2hhdCUyMGFib3V0JTIwc2Vjb25kJTIwYnJlYWtmYXN0JTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyRG9uJ3QlMjB0aGluayUyMGhlJTIwa25vd3MlMjBhYm91dCUyMHNlY29uZCUyMGJyZWFrZmFzdCUyQyUyMFBpcC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwYWJvdXQlMjBlbGV2ZW5zaWVzJTNGJTIyJTJDJTBBJTVEJTBBZW5jb2RlZF9pbnB1dCUyMCUzRCUyMHRva2VuaXplcihiYXRjaF9zZW50ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlKSUwQXByaW50KGVuY29kZWRfaW5wdXQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),zs=new Ie({props:{$$slots:{default:[wd]},$$scope:{ctx:k}}}),Ja=new ks({}),Ns=new bd({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[$d],pytorch:[_d]},$$scope:{ctx:k}}}),_a=new ks({}),Ta=new E({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTJDJTIwQXVkaW8lMEElMEFkYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMlBvbHlBSSUyRm1pbmRzMTQlMjIlMkMlMjBuYW1lJTNEJTIyZW4tVVMlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),Ma=new E({props:{code:"ZGF0YXNldCU1QjAlNUQlNUIlMjJhdWRpbyUyMiU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),Ua=new E({props:{code:"ZGF0YXNldCUyMCUzRCUyMGRhdGFzZXQuY2FzdF9jb2x1bW4oJTIyYXVkaW8lMjIlMkMlMjBBdWRpbyhzYW1wbGluZ19yYXRlJTNEMTZfMDAwKSk=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),Za=new E({props:{code:"ZGF0YXNldCU1QjAlNUQlNUIlMjJhdWRpbyUyMiU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">2.3443763e-05</span>,  <span class="hljs-number">2.1729663e-04</span>,  <span class="hljs-number">2.2145823e-04</span>, ...,
         <span class="hljs-number">3.8356509e-05</span>, -<span class="hljs-number">7.3497440e-06</span>, -<span class="hljs-number">2.1754686e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),Va=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9GZWF0dXJlRXh0cmFjdG9yJTBBJTBBZmVhdHVyZV9leHRyYWN0b3IlMjAlM0QlMjBBdXRvRmVhdHVyZUV4dHJhY3Rvci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZ3YXYydmVjMi1iYXNlJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)`}}),Ba=new E({props:{code:"YXVkaW9faW5wdXQlMjAlM0QlMjAlNUJkYXRhc2V0JTVCMCU1RCU1QiUyMmF1ZGlvJTIyJTVEJTVCJTIyYXJyYXklMjIlNUQlNUQlMEFmZWF0dXJlX2V4dHJhY3RvcihhdWRpb19pbnB1dCUyQyUyMHNhbXBsaW5nX3JhdGUlM0QxNjAwMCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>audio_input = [dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor(audio_input, sampling_rate=<span class="hljs-number">16000</span>)
{<span class="hljs-string">&#x27;input_values&#x27;</span>: [array([ <span class="hljs-number">3.8106556e-04</span>,  <span class="hljs-number">2.7506407e-03</span>,  <span class="hljs-number">2.8015103e-03</span>, ...,
        <span class="hljs-number">5.6335266e-04</span>,  <span class="hljs-number">4.6588284e-06</span>, -<span class="hljs-number">1.7142107e-04</span>], dtype=float32)]}`}}),Ga=new E({props:{code:"ZGF0YXNldCU1QjAlNUQlNUIlMjJhdWRpbyUyMiU1RCU1QiUyMmFycmF5JTIyJTVELnNoYXBlJTBBJTBBZGF0YXNldCU1QjElNUQlNUIlMjJhdWRpbyUyMiU1RCU1QiUyMmFycmF5JTIyJTVELnNoYXBl",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">173398</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">106496</span>,)`}}),Aa=new E({props:{code:"ZGVmJTIwcHJlcHJvY2Vzc19mdW5jdGlvbihleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjBhdWRpb19hcnJheXMlMjAlM0QlMjAlNUJ4JTVCJTIyYXJyYXklMjIlNUQlMjBmb3IlMjB4JTIwaW4lMjBleGFtcGxlcyU1QiUyMmF1ZGlvJTIyJTVEJTVEJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTIwJTNEJTIwZmVhdHVyZV9leHRyYWN0b3IoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYXVkaW9fYXJyYXlzJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2FtcGxpbmdfcmF0ZSUzRDE2MDAwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGFkZGluZyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXhfbGVuZ3RoJTNEMTAwMDAwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwaW5wdXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays,
<span class="hljs-meta">... </span>        sampling_rate=<span class="hljs-number">16000</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">100000</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`}}),za=new E({props:{code:"cHJvY2Vzc2VkX2RhdGFzZXQlMjAlM0QlMjBwcmVwcm9jZXNzX2Z1bmN0aW9uKGRhdGFzZXQlNUIlM0E1JTVEKQ==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset = preprocess_function(dataset[:<span class="hljs-number">5</span>])'}}),Wa=new E({props:{code:"cHJvY2Vzc2VkX2RhdGFzZXQlNUIlMjJpbnB1dF92YWx1ZXMlMjIlNUQlNUIwJTVELnNoYXBlJTBBJTBBcHJvY2Vzc2VkX2RhdGFzZXQlNUIlMjJpbnB1dF92YWx1ZXMlMjIlNUQlNUIxJTVELnNoYXBl",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">0</span>].shape
(<span class="hljs-number">100000</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">1</span>].shape
(<span class="hljs-number">100000</span>,)`}}),Na=new ks({}),Ds=new Ie({props:{$$slots:{default:[Td]},$$scope:{ctx:k}}}),Ss=new Ie({props:{$$slots:{default:[Md]},$$scope:{ctx:k}}}),Fa=new E({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJmb29kMTAxJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiU1QiUzQTEwMCU1RCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;food101&quot;</span>, split=<span class="hljs-string">&quot;train[:100]&quot;</span>)`}}),Xa=new E({props:{code:"ZGF0YXNldCU1QjAlNUQlNUIlMjJpbWFnZSUyMiU1RA==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]'}}),Qa=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9JbWFnZVByb2Nlc3NvciUwQSUwQWltYWdlX3Byb2Nlc3NvciUyMCUzRCUyMEF1dG9JbWFnZVByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGdml0LWJhc2UtcGF0Y2gxNi0yMjQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoImageProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>image_processor = AutoImageProcessor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224&quot;</span>)`}}),se=new E({props:{code:"ZnJvbSUyMHRvcmNodmlzaW9uLnRyYW5zZm9ybXMlMjBpbXBvcnQlMjBSYW5kb21SZXNpemVkQ3JvcCUyQyUyMENvbG9ySml0dGVyJTJDJTIwQ29tcG9zZSUwQSUwQXNpemUlMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwaW1hZ2VfcHJvY2Vzc29yLnNpemUlNUIlMjJzaG9ydGVzdF9lZGdlJTIyJTVEJTBBJTIwJTIwJTIwJTIwaWYlMjAlMjJzaG9ydGVzdF9lZGdlJTIyJTIwaW4lMjBpbWFnZV9wcm9jZXNzb3Iuc2l6ZSUwQSUyMCUyMCUyMCUyMGVsc2UlMjAoaW1hZ2VfcHJvY2Vzc29yLnNpemUlNUIlMjJoZWlnaHQlMjIlNUQlMkMlMjBpbWFnZV9wcm9jZXNzb3Iuc2l6ZSU1QiUyMndpZHRoJTIyJTVEKSUwQSklMEElMEFfdHJhbnNmb3JtcyUyMCUzRCUyMENvbXBvc2UoJTVCUmFuZG9tUmVzaXplZENyb3Aoc2l6ZSklMkMlMjBDb2xvckppdHRlcihicmlnaHRuZXNzJTNEMC41JTJDJTIwaHVlJTNEMC41KSU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> RandomResizedCrop, ColorJitter, Compose

<span class="hljs-meta">&gt;&gt;&gt; </span>size = (
<span class="hljs-meta">... </span>    image_processor.size[<span class="hljs-string">&quot;shortest_edge&quot;</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;shortest_edge&quot;</span> <span class="hljs-keyword">in</span> image_processor.size
<span class="hljs-meta">... </span>    <span class="hljs-keyword">else</span> (image_processor.size[<span class="hljs-string">&quot;height&quot;</span>], image_processor.size[<span class="hljs-string">&quot;width&quot;</span>])
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>_transforms = Compose([RandomResizedCrop(size), ColorJitter(brightness=<span class="hljs-number">0.5</span>, hue=<span class="hljs-number">0.5</span>)])`}}),ee=new E({props:{code:"ZGVmJTIwdHJhbnNmb3JtcyhleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjBpbWFnZXMlMjAlM0QlMjAlNUJfdHJhbnNmb3JtcyhpbWcuY29udmVydCglMjJSR0IlMjIpKSUyMGZvciUyMGltZyUyMGluJTIwZXhhbXBsZXMlNUIlMjJpbWFnZSUyMiU1RCU1RCUwQSUyMCUyMCUyMCUyMGV4YW1wbGVzJTVCJTIycGl4ZWxfdmFsdWVzJTIyJTVEJTIwJTNEJTIwaW1hZ2VfcHJvY2Vzc29yKGltYWdlcyUyQyUyMGRvX3Jlc2l6ZSUzREZhbHNlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklNUIlMjJwaXhlbF92YWx1ZXMlMjIlNUQlMEElMjAlMjAlMjAlMjByZXR1cm4lMjBleGFtcGxlcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    images = [_transforms(img.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> img <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = image_processor(images, do_resize=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`}}),Os=new Ie({props:{$$slots:{default:[Ed]},$$scope:{ctx:k}}}),pe=new E({props:{code:"ZGF0YXNldC5zZXRfdHJhbnNmb3JtKHRyYW5zZm9ybXMp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_transform(transforms)'}}),ce=new E({props:{code:"ZGF0YXNldCU1QjAlNUQua2V5cygp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>].keys()'}}),he=new E({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBaW1wb3J0JTIwbWF0cGxvdGxpYi5weXBsb3QlMjBhcyUyMHBsdCUwQSUwQWltZyUyMCUzRCUyMGRhdGFzZXQlNUIwJTVEJTVCJTIycGl4ZWxfdmFsdWVzJTIyJTVEJTBBcGx0Lmltc2hvdyhpbWcucGVybXV0ZSgxJTJDJTIwMiUyQyUyMDApKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt

<span class="hljs-meta">&gt;&gt;&gt; </span>img = dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>plt.imshow(img.permute(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>))`}}),Ks=new Ie({props:{$$slots:{default:[Id]},$$scope:{ctx:k}}}),ue=new ks({}),ie=new E({props:{code:"ZGVmJTIwY29sbGF0ZV9mbihiYXRjaCklM0ElMEElMjAlMjAlMjAlMjBwaXhlbF92YWx1ZXMlMjAlM0QlMjAlNUJpdGVtJTVCJTIycGl4ZWxfdmFsdWVzJTIyJTVEJTIwZm9yJTIwaXRlbSUyMGluJTIwYmF0Y2glNUQlMEElMjAlMjAlMjAlMjBlbmNvZGluZyUyMCUzRCUyMGltYWdlX3Byb2Nlc3Nvci5wYWRfYW5kX2NyZWF0ZV9waXhlbF9tYXNrKHBpeGVsX3ZhbHVlcyUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTIwJTIwJTIwJTIwbGFiZWxzJTIwJTNEJTIwJTVCaXRlbSU1QiUyMmxhYmVscyUyMiU1RCUyMGZvciUyMGl0ZW0lMjBpbiUyMGJhdGNoJTVEJTBBJTIwJTIwJTIwJTIwYmF0Y2glMjAlM0QlMjAlN0IlN0QlMEElMjAlMjAlMjAlMjBiYXRjaCU1QiUyMnBpeGVsX3ZhbHVlcyUyMiU1RCUyMCUzRCUyMGVuY29kaW5nJTVCJTIycGl4ZWxfdmFsdWVzJTIyJTVEJTBBJTIwJTIwJTIwJTIwYmF0Y2glNUIlMjJwaXhlbF9tYXNrJTIyJTVEJTIwJTNEJTIwZW5jb2RpbmclNUIlMjJwaXhlbF9tYXNrJTIyJTVEJTBBJTIwJTIwJTIwJTIwYmF0Y2glNUIlMjJsYWJlbHMlMjIlNUQlMjAlM0QlMjBsYWJlbHMlMEElMjAlMjAlMjAlMjByZXR1cm4lMjBiYXRjaA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">collate_fn</span>(<span class="hljs-params">batch</span>):
<span class="hljs-meta">... </span>    pixel_values = [item[<span class="hljs-string">&quot;pixel_values&quot;</span>] <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> batch]
<span class="hljs-meta">... </span>    encoding = image_processor.pad_and_create_pixel_mask(pixel_values, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">... </span>    labels = [item[<span class="hljs-string">&quot;labels&quot;</span>] <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> batch]
<span class="hljs-meta">... </span>    batch = {}
<span class="hljs-meta">... </span>    batch[<span class="hljs-string">&quot;pixel_values&quot;</span>] = encoding[<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">... </span>    batch[<span class="hljs-string">&quot;pixel_mask&quot;</span>] = encoding[<span class="hljs-string">&quot;pixel_mask&quot;</span>]
<span class="hljs-meta">... </span>    batch[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> batch`}}),de=new ks({}),be=new E({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBbGpfc3BlZWNoJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmxqX3NwZWVjaCUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = load_dataset(<span class="hljs-string">&quot;lj_speech&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),ge=new E({props:{code:"bGpfc3BlZWNoJTIwJTNEJTIwbGpfc3BlZWNoLm1hcChyZW1vdmVfY29sdW1ucyUzRCU1QiUyMmZpbGUlMjIlMkMlMjAlMjJpZCUyMiUyQyUyMCUyMm5vcm1hbGl6ZWRfdGV4dCUyMiU1RCk=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.<span class="hljs-built_in">map</span>(remove_columns=[<span class="hljs-string">&quot;file&quot;</span>, <span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;normalized_text&quot;</span>])'}}),ye=new E({props:{code:"bGpfc3BlZWNoJTVCMCU1RCU1QiUyMmF1ZGlvJTIyJTVEJTBBJTBBbGpfc3BlZWNoJTVCMCU1RCU1QiUyMnRleHQlMjIlNUQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">7.3242188e-04</span>, -<span class="hljs-number">7.6293945e-04</span>, -<span class="hljs-number">6.4086914e-04</span>, ...,
         <span class="hljs-number">7.3242188e-04</span>,  <span class="hljs-number">2.1362305e-04</span>,  <span class="hljs-number">6.1035156e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">22050</span>}

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]
<span class="hljs-string">&#x27;Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition&#x27;</span>`}}),we=new E({props:{code:"bGpfc3BlZWNoJTIwJTNEJTIwbGpfc3BlZWNoLmNhc3RfY29sdW1uKCUyMmF1ZGlvJTIyJTJDJTIwQXVkaW8oc2FtcGxpbmdfcmF0ZSUzRDE2XzAwMCkp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),Je=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRndhdjJ2ZWMyLWJhc2UtOTYwaCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>)`}}),_e=new E({props:{code:"ZGVmJTIwcHJlcGFyZV9kYXRhc2V0KGV4YW1wbGUpJTNBJTBBJTIwJTIwJTIwJTIwYXVkaW8lMjAlM0QlMjBleGFtcGxlJTVCJTIyYXVkaW8lMjIlNUQlMEElMEElMjAlMjAlMjAlMjBleGFtcGxlLnVwZGF0ZShwcm9jZXNzb3IoYXVkaW8lM0RhdWRpbyU1QiUyMmFycmF5JTIyJTVEJTJDJTIwdGV4dCUzRGV4YW1wbGUlNUIlMjJ0ZXh0JTIyJTVEJTJDJTIwc2FtcGxpbmdfcmF0ZSUzRDE2MDAwKSklMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBleGFtcGxl",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">prepare_dataset</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    audio = example[<span class="hljs-string">&quot;audio&quot;</span>]

<span class="hljs-meta">... </span>    example.update(processor(audio=audio[<span class="hljs-string">&quot;array&quot;</span>], text=example[<span class="hljs-string">&quot;text&quot;</span>], sampling_rate=<span class="hljs-number">16000</span>))

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> example`}}),Te=new E({props:{code:"cHJlcGFyZV9kYXRhc2V0KGxqX3NwZWVjaCU1QjAlNUQp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prepare_dataset(lj_speech[<span class="hljs-number">0</span>])'}}),{c(){u=p("meta"),_=h(),d=p("h1"),J=p("a"),$=p("span"),j(v.$$.fragment),T=h(),I=p("span"),x=n("Preprocess"),V=h(),j(U.$$.fragment),z=h(),B=p("p"),S=n("Before you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. \u{1F917} Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you\u2019ll learn that for:"),R=h(),M=p("ul"),Z=p("li"),W=n("Text, use a "),F=p("a"),K=n("Tokenizer"),N=n(" to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors."),ss=h(),H=p("li"),G=n("Speech and audio, use a "),C=p("a"),ke=n("Feature extractor"),Ue=n(" to extract sequential features from audio waveforms and convert them into tensors."),xe=h(),as=p("li"),es=n("Image inputs use a "),ws=p("a"),Ce=n("ImageProcessor"),_r=n(" to convert images into tensors."),vr=h(),ca=p("li"),$r=n("Multimodal inputs, use a "),Ze=p("a"),Tr=n("Processor"),Mr=n(" to combine a tokenizer and a feature extractor or image processor."),Ll=h(),j(Us.$$.fragment),Ol=h(),Ve=p("p"),Er=n("Before you begin, install \u{1F917} Datasets so you can load some datasets to experiment with:"),Kl=h(),j(ha.$$.fragment),st=h(),Js=p("h2"),xs=p("a"),Cn=p("span"),j(ma.$$.fragment),Ir=h(),Zn=p("span"),kr=n("Natural Language Processing"),at=h(),j(ua.$$.fragment),et=h(),ns=p("p"),Ur=n("The main tool for preprocessing textual data is a "),Be=p("a"),xr=n("tokenizer"),Cr=n(". A tokenizer splits text into "),Vn=p("em"),Zr=n("tokens"),Vr=n(" according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer."),nt=h(),j(Cs.$$.fragment),lt=h(),ls=p("p"),Br=n("Get started by loading a pretrained tokenizer with the "),Ge=p("a"),Gr=n("AutoTokenizer.from_pretrained()"),Ar=n(" method. This downloads the "),Bn=p("em"),zr=n("vocab"),Wr=n(" a model was pretrained with:"),tt=h(),j(ia.$$.fragment),pt=h(),Ae=p("p"),Nr=n("Then pass your text to the tokenizer:"),rt=h(),j(da.$$.fragment),ot=h(),ze=p("p"),Yr=n("The tokenizer returns a dictionary with three important items:"),ct=h(),ts=p("ul"),We=p("li"),Ne=p("a"),Rr=n("input_ids"),Fr=n(" are the indices corresponding to each token in the sentence."),qr=h(),Ye=p("li"),Re=p("a"),Xr=n("attention_mask"),Pr=n(" indicates whether a token should be attended to or not."),Qr=h(),Fe=p("li"),qe=p("a"),Dr=n("token_type_ids"),Sr=n(" identifies which sequence a token belongs to when there is more than one sequence."),ht=h(),Zs=p("p"),Hr=n("Return your input by decoding the "),Gn=p("code"),Lr=n("input_ids"),Or=n(":"),mt=h(),j(ja.$$.fragment),ut=h(),ps=p("p"),Kr=n("As you can see, the tokenizer added two special tokens - "),An=p("code"),so=n("CLS"),ao=n(" and "),zn=p("code"),eo=n("SEP"),no=n(` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer automatically adds them for you.`),it=h(),Xe=p("p"),lo=n("If there are several sentences you want to preprocess, pass them as a list to the tokenizer:"),dt=h(),j(fa.$$.fragment),jt=h(),_s=p("h3"),Vs=p("a"),Wn=p("span"),j(ba.$$.fragment),to=h(),Nn=p("span"),po=n("Pad"),ft=h(),Bs=p("p"),ro=n("Sentences aren\u2019t always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),Yn=p("em"),oo=n("padding token"),co=n(" to shorter sentences."),bt=h(),rs=p("p"),ho=n("Set the "),Rn=p("code"),mo=n("padding"),uo=n(" parameter to "),Fn=p("code"),io=n("True"),jo=n(" to pad the shorter sequences in the batch to match the longest sequence:"),gt=h(),j(ga.$$.fragment),yt=h(),Gs=p("p"),fo=n("The first and third sentences are now padded with "),qn=p("code"),bo=n("0"),go=n("\u2019s because they are shorter."),wt=h(),vs=p("h3"),As=p("a"),Xn=p("span"),j(ya.$$.fragment),yo=h(),Pn=p("span"),wo=n("Truncation"),Jt=h(),Pe=p("p"),Jo=n("On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you\u2019ll need to truncate the sequence to a shorter length."),_t=h(),os=p("p"),_o=n("Set the "),Qn=p("code"),vo=n("truncation"),$o=n(" parameter to "),Dn=p("code"),To=n("True"),Mo=n(" to truncate a sequence to the maximum length accepted by the model:"),vt=h(),j(wa.$$.fragment),$t=h(),j(zs.$$.fragment),Tt=h(),$s=p("h3"),Ws=p("a"),Sn=p("span"),j(Ja.$$.fragment),Eo=h(),Hn=p("span"),Io=n("Build tensors"),Mt=h(),Qe=p("p"),ko=n("Finally, you want the tokenizer to return the actual tensors that get fed to the model."),Et=h(),X=p("p"),Uo=n("Set the "),Ln=p("code"),xo=n("return_tensors"),Co=n(" parameter to either "),On=p("code"),Zo=n("pt"),Vo=n(" for PyTorch, or "),Kn=p("code"),Bo=n("tf"),Go=n(" for TensorFlow:"),It=h(),j(Ns.$$.fragment),kt=h(),Ts=p("h2"),Ys=p("a"),sl=p("span"),j(_a.$$.fragment),Ao=h(),al=p("span"),zo=n("Audio"),Ut=h(),Rs=p("p"),Wo=n("For audio tasks, you\u2019ll need a "),De=p("a"),No=n("feature extractor"),Yo=n(" to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors."),xt=h(),cs=p("p"),Ro=n("Load the "),va=p("a"),Fo=n("MInDS-14"),qo=n(" dataset (see the \u{1F917} "),$a=p("a"),Xo=n("Datasets tutorial"),Po=n(" for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:"),Ct=h(),j(Ta.$$.fragment),Zt=h(),hs=p("p"),Qo=n("Access the first element of the "),el=p("code"),Do=n("audio"),So=n(" column to take a look at the input. Calling the "),nl=p("code"),Ho=n("audio"),Lo=n(" column automatically loads and resamples the audio file:"),Vt=h(),j(Ma.$$.fragment),Bt=h(),Se=p("p"),Oo=n("This returns three items:"),Gt=h(),ms=p("ul"),He=p("li"),ll=p("code"),Ko=n("array"),sc=n(" is the speech signal loaded - and potentially resampled - as a 1D array."),ac=h(),Le=p("li"),tl=p("code"),ec=n("path"),nc=n(" points to the location of the audio file."),lc=h(),Oe=p("li"),pl=p("code"),tc=n("sampling_rate"),pc=n(" refers to how many data points in the speech signal are measured per second."),At=h(),Fs=p("p"),rc=n("For this tutorial, you\u2019ll use the "),Ea=p("a"),oc=n("Wav2Vec2"),cc=n(" model. Take a look at the model card, and you\u2019ll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your data."),zt=h(),Ke=p("ol"),Ia=p("li"),hc=n("Use \u{1F917} Datasets\u2019 "),ka=p("a"),mc=n("cast_column"),uc=n(" method to upsample the sampling rate to 16kHz:"),Wt=h(),j(Ua.$$.fragment),Nt=h(),xa=p("ol"),Ca=p("li"),ic=n("Call the "),rl=p("code"),dc=n("audio"),jc=n(" column again to resample the audio file:"),Yt=h(),j(Za.$$.fragment),Rt=h(),P=p("p"),fc=n("Next, load a feature extractor to normalize and pad the input. When padding textual data, a "),ol=p("code"),bc=n("0"),gc=n(" is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a "),cl=p("code"),yc=n("0"),wc=n(" - interpreted as silence - to "),hl=p("code"),Jc=n("array"),_c=n("."),Ft=h(),qs=p("p"),vc=n("Load the feature extractor with "),sn=p("a"),$c=n("AutoFeatureExtractor.from_pretrained()"),Tc=n(":"),qt=h(),j(Va.$$.fragment),Xt=h(),us=p("p"),Mc=n("Pass the audio "),ml=p("code"),Ec=n("array"),Ic=n(" to the feature extractor. We also recommend adding the "),ul=p("code"),kc=n("sampling_rate"),Uc=n(" argument in the feature extractor in order to better debug any silent errors that may occur."),Pt=h(),j(Ba.$$.fragment),Qt=h(),an=p("p"),xc=n("Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),Dt=h(),j(Ga.$$.fragment),St=h(),en=p("p"),Cc=n("Create a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),Ht=h(),j(Aa.$$.fragment),Lt=h(),Xs=p("p"),Zc=n("Apply the "),il=p("code"),Vc=n("preprocess_function"),Bc=n(" to the the first few examples in the dataset:"),Ot=h(),j(za.$$.fragment),Kt=h(),nn=p("p"),Gc=n("The sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!"),sp=h(),j(Wa.$$.fragment),ap=h(),Ms=p("h2"),Ps=p("a"),dl=p("span"),j(Na.$$.fragment),Ac=h(),jl=p("span"),zc=n("Computer vision"),ep=h(),Qs=p("p"),Wc=n("For computer vision tasks, you\u2019ll need an "),ln=p("a"),Nc=n("image processor"),Yc=n(` to prepare your dataset for the model.
Image preprocessing consists of several steps that convert images into the input expected by the model. These steps
include but are not limited to resizing, normalizing, color channel correction, and converting images to tensors.`),np=h(),j(Ds.$$.fragment),lp=h(),is=p("p"),Rc=n("Load the "),Ya=p("a"),Fc=n("food101"),qc=n(" dataset (see the \u{1F917} "),Ra=p("a"),Xc=n("Datasets tutorial"),Pc=n(" for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:"),tp=h(),j(Ss.$$.fragment),pp=h(),j(Fa.$$.fragment),rp=h(),Hs=p("p"),Qc=n("Next, take a look at the image with \u{1F917} Datasets "),qa=p("a"),fl=p("code"),Dc=n("Image"),Sc=n(" feature:"),op=h(),j(Xa.$$.fragment),cp=h(),Pa=p("div"),bl=p("img"),hp=h(),Ls=p("p"),Hc=n("Load the image processor with "),tn=p("a"),Lc=n("AutoImageProcessor.from_pretrained()"),Oc=n(":"),mp=h(),j(Qa.$$.fragment),up=h(),Q=p("p"),Kc=n("First, let\u2019s add some image augmentation. You can use any library you prefer, but in this tutorial, we\u2019ll use torchvision\u2019s "),Da=p("a"),gl=p("code"),sh=n("transforms"),ah=n(" module. If you\u2019re interested in using another data augmentation library, learn how in the "),Sa=p("a"),eh=n("Albumentations"),nh=n(" or "),Ha=p("a"),lh=n("Kornia notebooks"),th=n("."),ip=h(),pn=p("ol"),A=p("li"),ph=n("Here we use "),La=p("a"),yl=p("code"),rh=n("Compose"),oh=n(` to chain together a couple of
transforms - `),Oa=p("a"),wl=p("code"),ch=n("RandomResizedCrop"),hh=n(" and "),Ka=p("a"),Jl=p("code"),mh=n("ColorJitter"),uh=n(`.
Note that for resizing, we can get the image size requirements from the `),_l=p("code"),ih=n("image_processor"),dh=n(`. For some models, an exact height and
width are expected, for others only the `),vl=p("code"),jh=n("shortest_edge"),fh=n(" is defined."),dp=h(),j(se.$$.fragment),jp=h(),ae=p("ol"),L=p("li"),bh=n("The model accepts "),rn=p("a"),$l=p("code"),gh=n("pixel_values"),yh=n(`
as its input. `),Tl=p("code"),wh=n("ImageProcessor"),Jh=n(` can take care of normalizing the images, and generating appropriate tensors.
Create a function that combines image augmentation and image preprocessing for a batch of images and generates `),Ml=p("code"),_h=n("pixel_values"),vh=n(":"),fp=h(),j(ee.$$.fragment),bp=h(),j(Os.$$.fragment),gp=h(),ne=p("ol"),le=p("li"),$h=n("Then use \u{1F917} Datasets "),te=p("a"),El=p("code"),Th=n("set_transform"),Mh=n(" to apply the transforms on the fly:"),yp=h(),j(pe.$$.fragment),wp=h(),re=p("ol"),oe=p("li"),Eh=n("Now when you access the image, you\u2019ll notice the image processor has added "),Il=p("code"),Ih=n("pixel_values"),kh=n(". You can pass your processed dataset to the model now!"),Jp=h(),j(ce.$$.fragment),_p=h(),on=p("p"),Uh=n("Here is what the image looks like after the transforms are applied. The image has been randomly cropped and it\u2019s color properties are different."),vp=h(),j(he.$$.fragment),$p=h(),me=p("div"),kl=p("img"),Tp=h(),j(Ks.$$.fragment),Mp=h(),Es=p("h3"),sa=p("a"),Ul=p("span"),j(ue.$$.fragment),xh=h(),xl=p("span"),Ch=n("Pad"),Ep=h(),Y=p("p"),Zh=n("In some cases, for instance, when fine-tuning "),cn=p("a"),Vh=n("DETR"),Bh=n(`, the model applies scale augmentation at training
time. This may cause images to be different sizes in a batch. You can use `),hn=p("a"),Gh=n("DetrImageProcessor.pad_and_create_pixel_mask()"),Ah=n(`
from `),mn=p("a"),zh=n("DetrImageProcessor"),Wh=n(" and define a custom "),Cl=p("code"),Nh=n("collate_fn"),Yh=n(" to batch images together."),Ip=h(),j(ie.$$.fragment),kp=h(),Is=p("h2"),aa=p("a"),Zl=p("span"),j(de.$$.fragment),Rh=h(),Vl=p("span"),Fh=n("Multimodal"),Up=h(),ea=p("p"),qh=n("For tasks involving multimodal inputs, you\u2019ll need a "),un=p("a"),Xh=n("processor"),Ph=n(" to prepare your dataset for the model. A processor couples together two processing objects such as as tokenizer and feature extractor."),xp=h(),ds=p("p"),Qh=n("Load the "),je=p("a"),Dh=n("LJ Speech"),Sh=n(" dataset (see the \u{1F917} "),fe=p("a"),Hh=n("Datasets tutorial"),Lh=n(" for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):"),Cp=h(),j(be.$$.fragment),Zp=h(),js=p("p"),Oh=n("For ASR, you\u2019re mainly focused on "),Bl=p("code"),Kh=n("audio"),sm=n(" and "),Gl=p("code"),am=n("text"),em=n(" so you can remove the other columns:"),Vp=h(),j(ge.$$.fragment),Bp=h(),fs=p("p"),nm=n("Now take a look at the "),Al=p("code"),lm=n("audio"),tm=n(" and "),zl=p("code"),pm=n("text"),rm=n(" columns:"),Gp=h(),j(ye.$$.fragment),Ap=h(),na=p("p"),om=n("Remember you should always "),dn=p("a"),cm=n("resample"),hm=n(" your audio dataset\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model!"),zp=h(),j(we.$$.fragment),Wp=h(),la=p("p"),mm=n("Load a processor with "),jn=p("a"),um=n("AutoProcessor.from_pretrained()"),im=n(":"),Np=h(),j(Je.$$.fragment),Yp=h(),fn=p("ol"),q=p("li"),dm=n("Create a function to process the audio data contained in "),Wl=p("code"),jm=n("array"),fm=n(" to "),Nl=p("code"),bm=n("input_values"),gm=n(", and tokenize "),Yl=p("code"),ym=n("text"),wm=n(" to "),Rl=p("code"),Jm=n("labels"),_m=n(". These are the inputs to the model:"),Rp=h(),j(_e.$$.fragment),Fp=h(),ve=p("ol"),$e=p("li"),vm=n("Apply the "),Fl=p("code"),$m=n("prepare_dataset"),Tm=n(" function to a sample:"),qp=h(),j(Te.$$.fragment),Xp=h(),bs=p("p"),Mm=n("The processor has now added "),ql=p("code"),Em=n("input_values"),Im=n(" and "),Xl=p("code"),km=n("labels"),Um=n(", and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!"),this.h()},l(s){const t=id('[data-svelte="svelte-1phssyn"]',document.head);u=r(t,"META",{name:!0,content:!0}),t.forEach(e),_=m(s),d=r(s,"H1",{class:!0});var Me=o(d);J=r(Me,"A",{id:!0,class:!0,href:!0});var Pl=o(J);$=r(Pl,"SPAN",{});var Ql=o($);f(v.$$.fragment,Ql),Ql.forEach(e),Pl.forEach(e),T=m(Me),I=r(Me,"SPAN",{});var Dl=o(I);x=l(Dl,"Preprocess"),Dl.forEach(e),Me.forEach(e),V=m(s),f(U.$$.fragment,s),z=m(s),B=r(s,"P",{});var Sl=o(B);S=l(Sl,"Before you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. \u{1F917} Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you\u2019ll learn that for:"),Sl.forEach(e),R=m(s),M=r(s,"UL",{});var O=o(M);Z=r(O,"LI",{});var Ee=o(Z);W=l(Ee,"Text, use a "),F=r(Ee,"A",{href:!0});var Hl=o(F);K=l(Hl,"Tokenizer"),Hl.forEach(e),N=l(Ee," to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors."),Ee.forEach(e),ss=m(O),H=r(O,"LI",{});var Qp=o(H);G=l(Qp,"Speech and audio, use a "),C=r(Qp,"A",{href:!0});var Wm=o(C);ke=l(Wm,"Feature extractor"),Wm.forEach(e),Ue=l(Qp," to extract sequential features from audio waveforms and convert them into tensors."),Qp.forEach(e),xe=m(O),as=r(O,"LI",{});var Dp=o(as);es=l(Dp,"Image inputs use a "),ws=r(Dp,"A",{href:!0});var Nm=o(ws);Ce=l(Nm,"ImageProcessor"),Nm.forEach(e),_r=l(Dp," to convert images into tensors."),Dp.forEach(e),vr=m(O),ca=r(O,"LI",{});var Sp=o(ca);$r=l(Sp,"Multimodal inputs, use a "),Ze=r(Sp,"A",{href:!0});var Ym=o(Ze);Tr=l(Ym,"Processor"),Ym.forEach(e),Mr=l(Sp," to combine a tokenizer and a feature extractor or image processor."),Sp.forEach(e),O.forEach(e),Ll=m(s),f(Us.$$.fragment,s),Ol=m(s),Ve=r(s,"P",{});var Rm=o(Ve);Er=l(Rm,"Before you begin, install \u{1F917} Datasets so you can load some datasets to experiment with:"),Rm.forEach(e),Kl=m(s),f(ha.$$.fragment,s),st=m(s),Js=r(s,"H2",{class:!0});var Hp=o(Js);xs=r(Hp,"A",{id:!0,class:!0,href:!0});var Fm=o(xs);Cn=r(Fm,"SPAN",{});var qm=o(Cn);f(ma.$$.fragment,qm),qm.forEach(e),Fm.forEach(e),Ir=m(Hp),Zn=r(Hp,"SPAN",{});var Xm=o(Zn);kr=l(Xm,"Natural Language Processing"),Xm.forEach(e),Hp.forEach(e),at=m(s),f(ua.$$.fragment,s),et=m(s),ns=r(s,"P",{});var bn=o(ns);Ur=l(bn,"The main tool for preprocessing textual data is a "),Be=r(bn,"A",{href:!0});var Pm=o(Be);xr=l(Pm,"tokenizer"),Pm.forEach(e),Cr=l(bn,". A tokenizer splits text into "),Vn=r(bn,"EM",{});var Qm=o(Vn);Zr=l(Qm,"tokens"),Qm.forEach(e),Vr=l(bn," according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer."),bn.forEach(e),nt=m(s),f(Cs.$$.fragment,s),lt=m(s),ls=r(s,"P",{});var gn=o(ls);Br=l(gn,"Get started by loading a pretrained tokenizer with the "),Ge=r(gn,"A",{href:!0});var Dm=o(Ge);Gr=l(Dm,"AutoTokenizer.from_pretrained()"),Dm.forEach(e),Ar=l(gn," method. This downloads the "),Bn=r(gn,"EM",{});var Sm=o(Bn);zr=l(Sm,"vocab"),Sm.forEach(e),Wr=l(gn," a model was pretrained with:"),gn.forEach(e),tt=m(s),f(ia.$$.fragment,s),pt=m(s),Ae=r(s,"P",{});var Hm=o(Ae);Nr=l(Hm,"Then pass your text to the tokenizer:"),Hm.forEach(e),rt=m(s),f(da.$$.fragment,s),ot=m(s),ze=r(s,"P",{});var Lm=o(ze);Yr=l(Lm,"The tokenizer returns a dictionary with three important items:"),Lm.forEach(e),ct=m(s),ts=r(s,"UL",{});var yn=o(ts);We=r(yn,"LI",{});var xm=o(We);Ne=r(xm,"A",{href:!0});var Om=o(Ne);Rr=l(Om,"input_ids"),Om.forEach(e),Fr=l(xm," are the indices corresponding to each token in the sentence."),xm.forEach(e),qr=m(yn),Ye=r(yn,"LI",{});var Cm=o(Ye);Re=r(Cm,"A",{href:!0});var Km=o(Re);Xr=l(Km,"attention_mask"),Km.forEach(e),Pr=l(Cm," indicates whether a token should be attended to or not."),Cm.forEach(e),Qr=m(yn),Fe=r(yn,"LI",{});var Zm=o(Fe);qe=r(Zm,"A",{href:!0});var su=o(qe);Dr=l(su,"token_type_ids"),su.forEach(e),Sr=l(Zm," identifies which sequence a token belongs to when there is more than one sequence."),Zm.forEach(e),yn.forEach(e),ht=m(s),Zs=r(s,"P",{});var Lp=o(Zs);Hr=l(Lp,"Return your input by decoding the "),Gn=r(Lp,"CODE",{});var au=o(Gn);Lr=l(au,"input_ids"),au.forEach(e),Or=l(Lp,":"),Lp.forEach(e),mt=m(s),f(ja.$$.fragment,s),ut=m(s),ps=r(s,"P",{});var wn=o(ps);Kr=l(wn,"As you can see, the tokenizer added two special tokens - "),An=r(wn,"CODE",{});var eu=o(An);so=l(eu,"CLS"),eu.forEach(e),ao=l(wn," and "),zn=r(wn,"CODE",{});var nu=o(zn);eo=l(nu,"SEP"),nu.forEach(e),no=l(wn,` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer automatically adds them for you.`),wn.forEach(e),it=m(s),Xe=r(s,"P",{});var lu=o(Xe);lo=l(lu,"If there are several sentences you want to preprocess, pass them as a list to the tokenizer:"),lu.forEach(e),dt=m(s),f(fa.$$.fragment,s),jt=m(s),_s=r(s,"H3",{class:!0});var Op=o(_s);Vs=r(Op,"A",{id:!0,class:!0,href:!0});var tu=o(Vs);Wn=r(tu,"SPAN",{});var pu=o(Wn);f(ba.$$.fragment,pu),pu.forEach(e),tu.forEach(e),to=m(Op),Nn=r(Op,"SPAN",{});var ru=o(Nn);po=l(ru,"Pad"),ru.forEach(e),Op.forEach(e),ft=m(s),Bs=r(s,"P",{});var Kp=o(Bs);ro=l(Kp,"Sentences aren\u2019t always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),Yn=r(Kp,"EM",{});var ou=o(Yn);oo=l(ou,"padding token"),ou.forEach(e),co=l(Kp," to shorter sentences."),Kp.forEach(e),bt=m(s),rs=r(s,"P",{});var Jn=o(rs);ho=l(Jn,"Set the "),Rn=r(Jn,"CODE",{});var cu=o(Rn);mo=l(cu,"padding"),cu.forEach(e),uo=l(Jn," parameter to "),Fn=r(Jn,"CODE",{});var hu=o(Fn);io=l(hu,"True"),hu.forEach(e),jo=l(Jn," to pad the shorter sequences in the batch to match the longest sequence:"),Jn.forEach(e),gt=m(s),f(ga.$$.fragment,s),yt=m(s),Gs=r(s,"P",{});var sr=o(Gs);fo=l(sr,"The first and third sentences are now padded with "),qn=r(sr,"CODE",{});var mu=o(qn);bo=l(mu,"0"),mu.forEach(e),go=l(sr,"\u2019s because they are shorter."),sr.forEach(e),wt=m(s),vs=r(s,"H3",{class:!0});var ar=o(vs);As=r(ar,"A",{id:!0,class:!0,href:!0});var uu=o(As);Xn=r(uu,"SPAN",{});var iu=o(Xn);f(ya.$$.fragment,iu),iu.forEach(e),uu.forEach(e),yo=m(ar),Pn=r(ar,"SPAN",{});var du=o(Pn);wo=l(du,"Truncation"),du.forEach(e),ar.forEach(e),Jt=m(s),Pe=r(s,"P",{});var ju=o(Pe);Jo=l(ju,"On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you\u2019ll need to truncate the sequence to a shorter length."),ju.forEach(e),_t=m(s),os=r(s,"P",{});var _n=o(os);_o=l(_n,"Set the "),Qn=r(_n,"CODE",{});var fu=o(Qn);vo=l(fu,"truncation"),fu.forEach(e),$o=l(_n," parameter to "),Dn=r(_n,"CODE",{});var bu=o(Dn);To=l(bu,"True"),bu.forEach(e),Mo=l(_n," to truncate a sequence to the maximum length accepted by the model:"),_n.forEach(e),vt=m(s),f(wa.$$.fragment,s),$t=m(s),f(zs.$$.fragment,s),Tt=m(s),$s=r(s,"H3",{class:!0});var er=o($s);Ws=r(er,"A",{id:!0,class:!0,href:!0});var gu=o(Ws);Sn=r(gu,"SPAN",{});var yu=o(Sn);f(Ja.$$.fragment,yu),yu.forEach(e),gu.forEach(e),Eo=m(er),Hn=r(er,"SPAN",{});var wu=o(Hn);Io=l(wu,"Build tensors"),wu.forEach(e),er.forEach(e),Mt=m(s),Qe=r(s,"P",{});var Ju=o(Qe);ko=l(Ju,"Finally, you want the tokenizer to return the actual tensors that get fed to the model."),Ju.forEach(e),Et=m(s),X=r(s,"P",{});var ta=o(X);Uo=l(ta,"Set the "),Ln=r(ta,"CODE",{});var _u=o(Ln);xo=l(_u,"return_tensors"),_u.forEach(e),Co=l(ta," parameter to either "),On=r(ta,"CODE",{});var vu=o(On);Zo=l(vu,"pt"),vu.forEach(e),Vo=l(ta," for PyTorch, or "),Kn=r(ta,"CODE",{});var $u=o(Kn);Bo=l($u,"tf"),$u.forEach(e),Go=l(ta," for TensorFlow:"),ta.forEach(e),It=m(s),f(Ns.$$.fragment,s),kt=m(s),Ts=r(s,"H2",{class:!0});var nr=o(Ts);Ys=r(nr,"A",{id:!0,class:!0,href:!0});var Tu=o(Ys);sl=r(Tu,"SPAN",{});var Mu=o(sl);f(_a.$$.fragment,Mu),Mu.forEach(e),Tu.forEach(e),Ao=m(nr),al=r(nr,"SPAN",{});var Eu=o(al);zo=l(Eu,"Audio"),Eu.forEach(e),nr.forEach(e),Ut=m(s),Rs=r(s,"P",{});var lr=o(Rs);Wo=l(lr,"For audio tasks, you\u2019ll need a "),De=r(lr,"A",{href:!0});var Iu=o(De);No=l(Iu,"feature extractor"),Iu.forEach(e),Yo=l(lr," to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors."),lr.forEach(e),xt=m(s),cs=r(s,"P",{});var vn=o(cs);Ro=l(vn,"Load the "),va=r(vn,"A",{href:!0,rel:!0});var ku=o(va);Fo=l(ku,"MInDS-14"),ku.forEach(e),qo=l(vn," dataset (see the \u{1F917} "),$a=r(vn,"A",{href:!0,rel:!0});var Uu=o($a);Xo=l(Uu,"Datasets tutorial"),Uu.forEach(e),Po=l(vn," for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:"),vn.forEach(e),Ct=m(s),f(Ta.$$.fragment,s),Zt=m(s),hs=r(s,"P",{});var $n=o(hs);Qo=l($n,"Access the first element of the "),el=r($n,"CODE",{});var xu=o(el);Do=l(xu,"audio"),xu.forEach(e),So=l($n," column to take a look at the input. Calling the "),nl=r($n,"CODE",{});var Cu=o(nl);Ho=l(Cu,"audio"),Cu.forEach(e),Lo=l($n," column automatically loads and resamples the audio file:"),$n.forEach(e),Vt=m(s),f(Ma.$$.fragment,s),Bt=m(s),Se=r(s,"P",{});var Zu=o(Se);Oo=l(Zu,"This returns three items:"),Zu.forEach(e),Gt=m(s),ms=r(s,"UL",{});var Tn=o(ms);He=r(Tn,"LI",{});var Vm=o(He);ll=r(Vm,"CODE",{});var Vu=o(ll);Ko=l(Vu,"array"),Vu.forEach(e),sc=l(Vm," is the speech signal loaded - and potentially resampled - as a 1D array."),Vm.forEach(e),ac=m(Tn),Le=r(Tn,"LI",{});var Bm=o(Le);tl=r(Bm,"CODE",{});var Bu=o(tl);ec=l(Bu,"path"),Bu.forEach(e),nc=l(Bm," points to the location of the audio file."),Bm.forEach(e),lc=m(Tn),Oe=r(Tn,"LI",{});var Gm=o(Oe);pl=r(Gm,"CODE",{});var Gu=o(pl);tc=l(Gu,"sampling_rate"),Gu.forEach(e),pc=l(Gm," refers to how many data points in the speech signal are measured per second."),Gm.forEach(e),Tn.forEach(e),At=m(s),Fs=r(s,"P",{});var tr=o(Fs);rc=l(tr,"For this tutorial, you\u2019ll use the "),Ea=r(tr,"A",{href:!0,rel:!0});var Au=o(Ea);oc=l(Au,"Wav2Vec2"),Au.forEach(e),cc=l(tr," model. Take a look at the model card, and you\u2019ll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your data."),tr.forEach(e),zt=m(s),Ke=r(s,"OL",{});var zu=o(Ke);Ia=r(zu,"LI",{});var pr=o(Ia);hc=l(pr,"Use \u{1F917} Datasets\u2019 "),ka=r(pr,"A",{href:!0,rel:!0});var Wu=o(ka);mc=l(Wu,"cast_column"),Wu.forEach(e),uc=l(pr," method to upsample the sampling rate to 16kHz:"),pr.forEach(e),zu.forEach(e),Wt=m(s),f(Ua.$$.fragment,s),Nt=m(s),xa=r(s,"OL",{start:!0});var Nu=o(xa);Ca=r(Nu,"LI",{});var rr=o(Ca);ic=l(rr,"Call the "),rl=r(rr,"CODE",{});var Yu=o(rl);dc=l(Yu,"audio"),Yu.forEach(e),jc=l(rr," column again to resample the audio file:"),rr.forEach(e),Nu.forEach(e),Yt=m(s),f(Za.$$.fragment,s),Rt=m(s),P=r(s,"P",{});var pa=o(P);fc=l(pa,"Next, load a feature extractor to normalize and pad the input. When padding textual data, a "),ol=r(pa,"CODE",{});var Ru=o(ol);bc=l(Ru,"0"),Ru.forEach(e),gc=l(pa," is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a "),cl=r(pa,"CODE",{});var Fu=o(cl);yc=l(Fu,"0"),Fu.forEach(e),wc=l(pa," - interpreted as silence - to "),hl=r(pa,"CODE",{});var qu=o(hl);Jc=l(qu,"array"),qu.forEach(e),_c=l(pa,"."),pa.forEach(e),Ft=m(s),qs=r(s,"P",{});var or=o(qs);vc=l(or,"Load the feature extractor with "),sn=r(or,"A",{href:!0});var Xu=o(sn);$c=l(Xu,"AutoFeatureExtractor.from_pretrained()"),Xu.forEach(e),Tc=l(or,":"),or.forEach(e),qt=m(s),f(Va.$$.fragment,s),Xt=m(s),us=r(s,"P",{});var Mn=o(us);Mc=l(Mn,"Pass the audio "),ml=r(Mn,"CODE",{});var Pu=o(ml);Ec=l(Pu,"array"),Pu.forEach(e),Ic=l(Mn," to the feature extractor. We also recommend adding the "),ul=r(Mn,"CODE",{});var Qu=o(ul);kc=l(Qu,"sampling_rate"),Qu.forEach(e),Uc=l(Mn," argument in the feature extractor in order to better debug any silent errors that may occur."),Mn.forEach(e),Pt=m(s),f(Ba.$$.fragment,s),Qt=m(s),an=r(s,"P",{});var Du=o(an);xc=l(Du,"Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),Du.forEach(e),Dt=m(s),f(Ga.$$.fragment,s),St=m(s),en=r(s,"P",{});var Su=o(en);Cc=l(Su,"Create a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),Su.forEach(e),Ht=m(s),f(Aa.$$.fragment,s),Lt=m(s),Xs=r(s,"P",{});var cr=o(Xs);Zc=l(cr,"Apply the "),il=r(cr,"CODE",{});var Hu=o(il);Vc=l(Hu,"preprocess_function"),Hu.forEach(e),Bc=l(cr," to the the first few examples in the dataset:"),cr.forEach(e),Ot=m(s),f(za.$$.fragment,s),Kt=m(s),nn=r(s,"P",{});var Lu=o(nn);Gc=l(Lu,"The sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!"),Lu.forEach(e),sp=m(s),f(Wa.$$.fragment,s),ap=m(s),Ms=r(s,"H2",{class:!0});var hr=o(Ms);Ps=r(hr,"A",{id:!0,class:!0,href:!0});var Ou=o(Ps);dl=r(Ou,"SPAN",{});var Ku=o(dl);f(Na.$$.fragment,Ku),Ku.forEach(e),Ou.forEach(e),Ac=m(hr),jl=r(hr,"SPAN",{});var si=o(jl);zc=l(si,"Computer vision"),si.forEach(e),hr.forEach(e),ep=m(s),Qs=r(s,"P",{});var mr=o(Qs);Wc=l(mr,"For computer vision tasks, you\u2019ll need an "),ln=r(mr,"A",{href:!0});var ai=o(ln);Nc=l(ai,"image processor"),ai.forEach(e),Yc=l(mr,` to prepare your dataset for the model.
Image preprocessing consists of several steps that convert images into the input expected by the model. These steps
include but are not limited to resizing, normalizing, color channel correction, and converting images to tensors.`),mr.forEach(e),np=m(s),f(Ds.$$.fragment,s),lp=m(s),is=r(s,"P",{});var En=o(is);Rc=l(En,"Load the "),Ya=r(En,"A",{href:!0,rel:!0});var ei=o(Ya);Fc=l(ei,"food101"),ei.forEach(e),qc=l(En," dataset (see the \u{1F917} "),Ra=r(En,"A",{href:!0,rel:!0});var ni=o(Ra);Xc=l(ni,"Datasets tutorial"),ni.forEach(e),Pc=l(En," for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:"),En.forEach(e),tp=m(s),f(Ss.$$.fragment,s),pp=m(s),f(Fa.$$.fragment,s),rp=m(s),Hs=r(s,"P",{});var ur=o(Hs);Qc=l(ur,"Next, take a look at the image with \u{1F917} Datasets "),qa=r(ur,"A",{href:!0,rel:!0});var li=o(qa);fl=r(li,"CODE",{});var ti=o(fl);Dc=l(ti,"Image"),ti.forEach(e),li.forEach(e),Sc=l(ur," feature:"),ur.forEach(e),op=m(s),f(Xa.$$.fragment,s),cp=m(s),Pa=r(s,"DIV",{class:!0});var pi=o(Pa);bl=r(pi,"IMG",{src:!0}),pi.forEach(e),hp=m(s),Ls=r(s,"P",{});var ir=o(Ls);Hc=l(ir,"Load the image processor with "),tn=r(ir,"A",{href:!0});var ri=o(tn);Lc=l(ri,"AutoImageProcessor.from_pretrained()"),ri.forEach(e),Oc=l(ir,":"),ir.forEach(e),mp=m(s),f(Qa.$$.fragment,s),up=m(s),Q=r(s,"P",{});var ra=o(Q);Kc=l(ra,"First, let\u2019s add some image augmentation. You can use any library you prefer, but in this tutorial, we\u2019ll use torchvision\u2019s "),Da=r(ra,"A",{href:!0,rel:!0});var oi=o(Da);gl=r(oi,"CODE",{});var ci=o(gl);sh=l(ci,"transforms"),ci.forEach(e),oi.forEach(e),ah=l(ra," module. If you\u2019re interested in using another data augmentation library, learn how in the "),Sa=r(ra,"A",{href:!0,rel:!0});var hi=o(Sa);eh=l(hi,"Albumentations"),hi.forEach(e),nh=l(ra," or "),Ha=r(ra,"A",{href:!0,rel:!0});var mi=o(Ha);lh=l(mi,"Kornia notebooks"),mi.forEach(e),th=l(ra,"."),ra.forEach(e),ip=m(s),pn=r(s,"OL",{});var ui=o(pn);A=r(ui,"LI",{});var D=o(A);ph=l(D,"Here we use "),La=r(D,"A",{href:!0,rel:!0});var ii=o(La);yl=r(ii,"CODE",{});var di=o(yl);rh=l(di,"Compose"),di.forEach(e),ii.forEach(e),oh=l(D,` to chain together a couple of
transforms - `),Oa=r(D,"A",{href:!0,rel:!0});var ji=o(Oa);wl=r(ji,"CODE",{});var fi=o(wl);ch=l(fi,"RandomResizedCrop"),fi.forEach(e),ji.forEach(e),hh=l(D," and "),Ka=r(D,"A",{href:!0,rel:!0});var bi=o(Ka);Jl=r(bi,"CODE",{});var gi=o(Jl);mh=l(gi,"ColorJitter"),gi.forEach(e),bi.forEach(e),uh=l(D,`.
Note that for resizing, we can get the image size requirements from the `),_l=r(D,"CODE",{});var yi=o(_l);ih=l(yi,"image_processor"),yi.forEach(e),dh=l(D,`. For some models, an exact height and
width are expected, for others only the `),vl=r(D,"CODE",{});var wi=o(vl);jh=l(wi,"shortest_edge"),wi.forEach(e),fh=l(D," is defined."),D.forEach(e),ui.forEach(e),dp=m(s),f(se.$$.fragment,s),jp=m(s),ae=r(s,"OL",{start:!0});var Ji=o(ae);L=r(Ji,"LI",{});var oa=o(L);bh=l(oa,"The model accepts "),rn=r(oa,"A",{href:!0});var _i=o(rn);$l=r(_i,"CODE",{});var vi=o($l);gh=l(vi,"pixel_values"),vi.forEach(e),_i.forEach(e),yh=l(oa,`
as its input. `),Tl=r(oa,"CODE",{});var $i=o(Tl);wh=l($i,"ImageProcessor"),$i.forEach(e),Jh=l(oa,` can take care of normalizing the images, and generating appropriate tensors.
Create a function that combines image augmentation and image preprocessing for a batch of images and generates `),Ml=r(oa,"CODE",{});var Ti=o(Ml);_h=l(Ti,"pixel_values"),Ti.forEach(e),vh=l(oa,":"),oa.forEach(e),Ji.forEach(e),fp=m(s),f(ee.$$.fragment,s),bp=m(s),f(Os.$$.fragment,s),gp=m(s),ne=r(s,"OL",{start:!0});var Mi=o(ne);le=r(Mi,"LI",{});var dr=o(le);$h=l(dr,"Then use \u{1F917} Datasets "),te=r(dr,"A",{href:!0,rel:!0});var Ei=o(te);El=r(Ei,"CODE",{});var Ii=o(El);Th=l(Ii,"set_transform"),Ii.forEach(e),Ei.forEach(e),Mh=l(dr," to apply the transforms on the fly:"),dr.forEach(e),Mi.forEach(e),yp=m(s),f(pe.$$.fragment,s),wp=m(s),re=r(s,"OL",{start:!0});var ki=o(re);oe=r(ki,"LI",{});var jr=o(oe);Eh=l(jr,"Now when you access the image, you\u2019ll notice the image processor has added "),Il=r(jr,"CODE",{});var Ui=o(Il);Ih=l(Ui,"pixel_values"),Ui.forEach(e),kh=l(jr,". You can pass your processed dataset to the model now!"),jr.forEach(e),ki.forEach(e),Jp=m(s),f(ce.$$.fragment,s),_p=m(s),on=r(s,"P",{});var xi=o(on);Uh=l(xi,"Here is what the image looks like after the transforms are applied. The image has been randomly cropped and it\u2019s color properties are different."),xi.forEach(e),vp=m(s),f(he.$$.fragment,s),$p=m(s),me=r(s,"DIV",{class:!0});var Ci=o(me);kl=r(Ci,"IMG",{src:!0}),Ci.forEach(e),Tp=m(s),f(Ks.$$.fragment,s),Mp=m(s),Es=r(s,"H3",{class:!0});var fr=o(Es);sa=r(fr,"A",{id:!0,class:!0,href:!0});var Zi=o(sa);Ul=r(Zi,"SPAN",{});var Vi=o(Ul);f(ue.$$.fragment,Vi),Vi.forEach(e),Zi.forEach(e),xh=m(fr),xl=r(fr,"SPAN",{});var Bi=o(xl);Ch=l(Bi,"Pad"),Bi.forEach(e),fr.forEach(e),Ep=m(s),Y=r(s,"P",{});var gs=o(Y);Zh=l(gs,"In some cases, for instance, when fine-tuning "),cn=r(gs,"A",{href:!0});var Gi=o(cn);Vh=l(Gi,"DETR"),Gi.forEach(e),Bh=l(gs,`, the model applies scale augmentation at training
time. This may cause images to be different sizes in a batch. You can use `),hn=r(gs,"A",{href:!0});var Ai=o(hn);Gh=l(Ai,"DetrImageProcessor.pad_and_create_pixel_mask()"),Ai.forEach(e),Ah=l(gs,`
from `),mn=r(gs,"A",{href:!0});var zi=o(mn);zh=l(zi,"DetrImageProcessor"),zi.forEach(e),Wh=l(gs," and define a custom "),Cl=r(gs,"CODE",{});var Wi=o(Cl);Nh=l(Wi,"collate_fn"),Wi.forEach(e),Yh=l(gs," to batch images together."),gs.forEach(e),Ip=m(s),f(ie.$$.fragment,s),kp=m(s),Is=r(s,"H2",{class:!0});var br=o(Is);aa=r(br,"A",{id:!0,class:!0,href:!0});var Ni=o(aa);Zl=r(Ni,"SPAN",{});var Yi=o(Zl);f(de.$$.fragment,Yi),Yi.forEach(e),Ni.forEach(e),Rh=m(br),Vl=r(br,"SPAN",{});var Ri=o(Vl);Fh=l(Ri,"Multimodal"),Ri.forEach(e),br.forEach(e),Up=m(s),ea=r(s,"P",{});var gr=o(ea);qh=l(gr,"For tasks involving multimodal inputs, you\u2019ll need a "),un=r(gr,"A",{href:!0});var Fi=o(un);Xh=l(Fi,"processor"),Fi.forEach(e),Ph=l(gr," to prepare your dataset for the model. A processor couples together two processing objects such as as tokenizer and feature extractor."),gr.forEach(e),xp=m(s),ds=r(s,"P",{});var In=o(ds);Qh=l(In,"Load the "),je=r(In,"A",{href:!0,rel:!0});var qi=o(je);Dh=l(qi,"LJ Speech"),qi.forEach(e),Sh=l(In," dataset (see the \u{1F917} "),fe=r(In,"A",{href:!0,rel:!0});var Xi=o(fe);Hh=l(Xi,"Datasets tutorial"),Xi.forEach(e),Lh=l(In," for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):"),In.forEach(e),Cp=m(s),f(be.$$.fragment,s),Zp=m(s),js=r(s,"P",{});var kn=o(js);Oh=l(kn,"For ASR, you\u2019re mainly focused on "),Bl=r(kn,"CODE",{});var Pi=o(Bl);Kh=l(Pi,"audio"),Pi.forEach(e),sm=l(kn," and "),Gl=r(kn,"CODE",{});var Qi=o(Gl);am=l(Qi,"text"),Qi.forEach(e),em=l(kn," so you can remove the other columns:"),kn.forEach(e),Vp=m(s),f(ge.$$.fragment,s),Bp=m(s),fs=r(s,"P",{});var Un=o(fs);nm=l(Un,"Now take a look at the "),Al=r(Un,"CODE",{});var Di=o(Al);lm=l(Di,"audio"),Di.forEach(e),tm=l(Un," and "),zl=r(Un,"CODE",{});var Si=o(zl);pm=l(Si,"text"),Si.forEach(e),rm=l(Un," columns:"),Un.forEach(e),Gp=m(s),f(ye.$$.fragment,s),Ap=m(s),na=r(s,"P",{});var yr=o(na);om=l(yr,"Remember you should always "),dn=r(yr,"A",{href:!0});var Hi=o(dn);cm=l(Hi,"resample"),Hi.forEach(e),hm=l(yr," your audio dataset\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model!"),yr.forEach(e),zp=m(s),f(we.$$.fragment,s),Wp=m(s),la=r(s,"P",{});var wr=o(la);mm=l(wr,"Load a processor with "),jn=r(wr,"A",{href:!0});var Li=o(jn);um=l(Li,"AutoProcessor.from_pretrained()"),Li.forEach(e),im=l(wr,":"),wr.forEach(e),Np=m(s),f(Je.$$.fragment,s),Yp=m(s),fn=r(s,"OL",{});var Oi=o(fn);q=r(Oi,"LI",{});var ys=o(q);dm=l(ys,"Create a function to process the audio data contained in "),Wl=r(ys,"CODE",{});var Ki=o(Wl);jm=l(Ki,"array"),Ki.forEach(e),fm=l(ys," to "),Nl=r(ys,"CODE",{});var sd=o(Nl);bm=l(sd,"input_values"),sd.forEach(e),gm=l(ys,", and tokenize "),Yl=r(ys,"CODE",{});var ad=o(Yl);ym=l(ad,"text"),ad.forEach(e),wm=l(ys," to "),Rl=r(ys,"CODE",{});var ed=o(Rl);Jm=l(ed,"labels"),ed.forEach(e),_m=l(ys,". These are the inputs to the model:"),ys.forEach(e),Oi.forEach(e),Rp=m(s),f(_e.$$.fragment,s),Fp=m(s),ve=r(s,"OL",{start:!0});var nd=o(ve);$e=r(nd,"LI",{});var Jr=o($e);vm=l(Jr,"Apply the "),Fl=r(Jr,"CODE",{});var ld=o(Fl);$m=l(ld,"prepare_dataset"),ld.forEach(e),Tm=l(Jr," function to a sample:"),Jr.forEach(e),nd.forEach(e),qp=m(s),f(Te.$$.fragment,s),Xp=m(s),bs=r(s,"P",{});var xn=o(bs);Mm=l(xn,"The processor has now added "),ql=r(xn,"CODE",{});var td=o(ql);Em=l(td,"input_values"),td.forEach(e),Im=l(xn," and "),Xl=r(xn,"CODE",{});var pd=o(Xl);km=l(pd,"labels"),pd.forEach(e),Um=l(xn,", and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!"),xn.forEach(e),this.h()},h(){i(u,"name","hf:doc:metadata"),i(u,"content",JSON.stringify(Ud)),i(J,"id","preprocess"),i(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(J,"href","#preprocess"),i(d,"class","relative group"),i(F,"href","./main_classes/tokenizer"),i(C,"href","./main_classes/feature_extractor"),i(ws,"href","./main_classes/image"),i(Ze,"href","./main_classes/processors"),i(xs,"id","natural-language-processing"),i(xs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(xs,"href","#natural-language-processing"),i(Js,"class","relative group"),i(Be,"href","main_classes/tokenizer"),i(Ge,"href","/docs/transformers/v4.30.0/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained"),i(Ne,"href","glossary#input-ids"),i(Re,"href","glossary#attention-mask"),i(qe,"href","glossary#token-type-ids"),i(Vs,"id","pad"),i(Vs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(Vs,"href","#pad"),i(_s,"class","relative group"),i(As,"id","truncation"),i(As,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(As,"href","#truncation"),i(vs,"class","relative group"),i(Ws,"id","build-tensors"),i(Ws,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(Ws,"href","#build-tensors"),i($s,"class","relative group"),i(Ys,"id","audio"),i(Ys,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(Ys,"href","#audio"),i(Ts,"class","relative group"),i(De,"href","main_classes/feature_extractor"),i(va,"href","https://huggingface.co/datasets/PolyAI/minds14"),i(va,"rel","nofollow"),i($a,"href","https://huggingface.co/docs/datasets/load_hub.html"),i($a,"rel","nofollow"),i(Ea,"href","https://huggingface.co/facebook/wav2vec2-base"),i(Ea,"rel","nofollow"),i(ka,"href","https://huggingface.co/docs/datasets/v2.12.0/en/package_reference/main_classes#datasets.Dataset.cast_column"),i(ka,"rel","nofollow"),i(xa,"start","2"),i(sn,"href","/docs/transformers/v4.30.0/en/model_doc/auto#transformers.AutoFeatureExtractor.from_pretrained"),i(Ps,"id","computer-vision"),i(Ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(Ps,"href","#computer-vision"),i(Ms,"class","relative group"),i(ln,"href","main_classes/image_processor"),i(Ya,"href","https://huggingface.co/datasets/food101"),i(Ya,"rel","nofollow"),i(Ra,"href","https://huggingface.co/docs/datasets/load_hub.html"),i(Ra,"rel","nofollow"),i(qa,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=image#datasets.Image"),i(qa,"rel","nofollow"),rd(bl.src,Am="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png")||i(bl,"src",Am),i(Pa,"class","flex justify-center"),i(tn,"href","/docs/transformers/v4.30.0/en/model_doc/auto#transformers.AutoImageProcessor.from_pretrained"),i(Da,"href","https://pytorch.org/vision/stable/transforms.html"),i(Da,"rel","nofollow"),i(Sa,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb"),i(Sa,"rel","nofollow"),i(Ha,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb"),i(Ha,"rel","nofollow"),i(La,"href","https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html"),i(La,"rel","nofollow"),i(Oa,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html"),i(Oa,"rel","nofollow"),i(Ka,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html"),i(Ka,"rel","nofollow"),i(rn,"href","model_doc/visionencoderdecoder#transformers.VisionEncoderDecoderModel.forward.pixel_values"),i(ae,"start","2"),i(te,"href","https://huggingface.co/docs/datasets/process.html#format-transform"),i(te,"rel","nofollow"),i(ne,"start","3"),i(re,"start","4"),rd(kl.src,zm="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png")||i(kl,"src",zm),i(me,"class","flex justify-center"),i(sa,"id","pad"),i(sa,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(sa,"href","#pad"),i(Es,"class","relative group"),i(cn,"href","./model_doc/detr"),i(hn,"href","/docs/transformers/v4.30.0/en/model_doc/detr#transformers.DetrFeatureExtractor.pad_and_create_pixel_mask"),i(mn,"href","/docs/transformers/v4.30.0/en/model_doc/detr#transformers.DetrImageProcessor"),i(aa,"id","multimodal"),i(aa,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(aa,"href","#multimodal"),i(Is,"class","relative group"),i(un,"href","main_classes/processors"),i(je,"href","https://huggingface.co/datasets/lj_speech"),i(je,"rel","nofollow"),i(fe,"href","https://huggingface.co/docs/datasets/load_hub.html"),i(fe,"rel","nofollow"),i(dn,"href","preprocessing#audio"),i(jn,"href","/docs/transformers/v4.30.0/en/model_doc/auto#transformers.AutoProcessor.from_pretrained"),i(ve,"start","2")},m(s,t){a(document.head,u),c(s,_,t),c(s,d,t),a(d,J),a(J,$),b(v,$,null),a(d,T),a(d,I),a(I,x),c(s,V,t),b(U,s,t),c(s,z,t),c(s,B,t),a(B,S),c(s,R,t),c(s,M,t),a(M,Z),a(Z,W),a(Z,F),a(F,K),a(Z,N),a(M,ss),a(M,H),a(H,G),a(H,C),a(C,ke),a(H,Ue),a(M,xe),a(M,as),a(as,es),a(as,ws),a(ws,Ce),a(as,_r),a(M,vr),a(M,ca),a(ca,$r),a(ca,Ze),a(Ze,Tr),a(ca,Mr),c(s,Ll,t),b(Us,s,t),c(s,Ol,t),c(s,Ve,t),a(Ve,Er),c(s,Kl,t),b(ha,s,t),c(s,st,t),c(s,Js,t),a(Js,xs),a(xs,Cn),b(ma,Cn,null),a(Js,Ir),a(Js,Zn),a(Zn,kr),c(s,at,t),b(ua,s,t),c(s,et,t),c(s,ns,t),a(ns,Ur),a(ns,Be),a(Be,xr),a(ns,Cr),a(ns,Vn),a(Vn,Zr),a(ns,Vr),c(s,nt,t),b(Cs,s,t),c(s,lt,t),c(s,ls,t),a(ls,Br),a(ls,Ge),a(Ge,Gr),a(ls,Ar),a(ls,Bn),a(Bn,zr),a(ls,Wr),c(s,tt,t),b(ia,s,t),c(s,pt,t),c(s,Ae,t),a(Ae,Nr),c(s,rt,t),b(da,s,t),c(s,ot,t),c(s,ze,t),a(ze,Yr),c(s,ct,t),c(s,ts,t),a(ts,We),a(We,Ne),a(Ne,Rr),a(We,Fr),a(ts,qr),a(ts,Ye),a(Ye,Re),a(Re,Xr),a(Ye,Pr),a(ts,Qr),a(ts,Fe),a(Fe,qe),a(qe,Dr),a(Fe,Sr),c(s,ht,t),c(s,Zs,t),a(Zs,Hr),a(Zs,Gn),a(Gn,Lr),a(Zs,Or),c(s,mt,t),b(ja,s,t),c(s,ut,t),c(s,ps,t),a(ps,Kr),a(ps,An),a(An,so),a(ps,ao),a(ps,zn),a(zn,eo),a(ps,no),c(s,it,t),c(s,Xe,t),a(Xe,lo),c(s,dt,t),b(fa,s,t),c(s,jt,t),c(s,_s,t),a(_s,Vs),a(Vs,Wn),b(ba,Wn,null),a(_s,to),a(_s,Nn),a(Nn,po),c(s,ft,t),c(s,Bs,t),a(Bs,ro),a(Bs,Yn),a(Yn,oo),a(Bs,co),c(s,bt,t),c(s,rs,t),a(rs,ho),a(rs,Rn),a(Rn,mo),a(rs,uo),a(rs,Fn),a(Fn,io),a(rs,jo),c(s,gt,t),b(ga,s,t),c(s,yt,t),c(s,Gs,t),a(Gs,fo),a(Gs,qn),a(qn,bo),a(Gs,go),c(s,wt,t),c(s,vs,t),a(vs,As),a(As,Xn),b(ya,Xn,null),a(vs,yo),a(vs,Pn),a(Pn,wo),c(s,Jt,t),c(s,Pe,t),a(Pe,Jo),c(s,_t,t),c(s,os,t),a(os,_o),a(os,Qn),a(Qn,vo),a(os,$o),a(os,Dn),a(Dn,To),a(os,Mo),c(s,vt,t),b(wa,s,t),c(s,$t,t),b(zs,s,t),c(s,Tt,t),c(s,$s,t),a($s,Ws),a(Ws,Sn),b(Ja,Sn,null),a($s,Eo),a($s,Hn),a(Hn,Io),c(s,Mt,t),c(s,Qe,t),a(Qe,ko),c(s,Et,t),c(s,X,t),a(X,Uo),a(X,Ln),a(Ln,xo),a(X,Co),a(X,On),a(On,Zo),a(X,Vo),a(X,Kn),a(Kn,Bo),a(X,Go),c(s,It,t),b(Ns,s,t),c(s,kt,t),c(s,Ts,t),a(Ts,Ys),a(Ys,sl),b(_a,sl,null),a(Ts,Ao),a(Ts,al),a(al,zo),c(s,Ut,t),c(s,Rs,t),a(Rs,Wo),a(Rs,De),a(De,No),a(Rs,Yo),c(s,xt,t),c(s,cs,t),a(cs,Ro),a(cs,va),a(va,Fo),a(cs,qo),a(cs,$a),a($a,Xo),a(cs,Po),c(s,Ct,t),b(Ta,s,t),c(s,Zt,t),c(s,hs,t),a(hs,Qo),a(hs,el),a(el,Do),a(hs,So),a(hs,nl),a(nl,Ho),a(hs,Lo),c(s,Vt,t),b(Ma,s,t),c(s,Bt,t),c(s,Se,t),a(Se,Oo),c(s,Gt,t),c(s,ms,t),a(ms,He),a(He,ll),a(ll,Ko),a(He,sc),a(ms,ac),a(ms,Le),a(Le,tl),a(tl,ec),a(Le,nc),a(ms,lc),a(ms,Oe),a(Oe,pl),a(pl,tc),a(Oe,pc),c(s,At,t),c(s,Fs,t),a(Fs,rc),a(Fs,Ea),a(Ea,oc),a(Fs,cc),c(s,zt,t),c(s,Ke,t),a(Ke,Ia),a(Ia,hc),a(Ia,ka),a(ka,mc),a(Ia,uc),c(s,Wt,t),b(Ua,s,t),c(s,Nt,t),c(s,xa,t),a(xa,Ca),a(Ca,ic),a(Ca,rl),a(rl,dc),a(Ca,jc),c(s,Yt,t),b(Za,s,t),c(s,Rt,t),c(s,P,t),a(P,fc),a(P,ol),a(ol,bc),a(P,gc),a(P,cl),a(cl,yc),a(P,wc),a(P,hl),a(hl,Jc),a(P,_c),c(s,Ft,t),c(s,qs,t),a(qs,vc),a(qs,sn),a(sn,$c),a(qs,Tc),c(s,qt,t),b(Va,s,t),c(s,Xt,t),c(s,us,t),a(us,Mc),a(us,ml),a(ml,Ec),a(us,Ic),a(us,ul),a(ul,kc),a(us,Uc),c(s,Pt,t),b(Ba,s,t),c(s,Qt,t),c(s,an,t),a(an,xc),c(s,Dt,t),b(Ga,s,t),c(s,St,t),c(s,en,t),a(en,Cc),c(s,Ht,t),b(Aa,s,t),c(s,Lt,t),c(s,Xs,t),a(Xs,Zc),a(Xs,il),a(il,Vc),a(Xs,Bc),c(s,Ot,t),b(za,s,t),c(s,Kt,t),c(s,nn,t),a(nn,Gc),c(s,sp,t),b(Wa,s,t),c(s,ap,t),c(s,Ms,t),a(Ms,Ps),a(Ps,dl),b(Na,dl,null),a(Ms,Ac),a(Ms,jl),a(jl,zc),c(s,ep,t),c(s,Qs,t),a(Qs,Wc),a(Qs,ln),a(ln,Nc),a(Qs,Yc),c(s,np,t),b(Ds,s,t),c(s,lp,t),c(s,is,t),a(is,Rc),a(is,Ya),a(Ya,Fc),a(is,qc),a(is,Ra),a(Ra,Xc),a(is,Pc),c(s,tp,t),b(Ss,s,t),c(s,pp,t),b(Fa,s,t),c(s,rp,t),c(s,Hs,t),a(Hs,Qc),a(Hs,qa),a(qa,fl),a(fl,Dc),a(Hs,Sc),c(s,op,t),b(Xa,s,t),c(s,cp,t),c(s,Pa,t),a(Pa,bl),c(s,hp,t),c(s,Ls,t),a(Ls,Hc),a(Ls,tn),a(tn,Lc),a(Ls,Oc),c(s,mp,t),b(Qa,s,t),c(s,up,t),c(s,Q,t),a(Q,Kc),a(Q,Da),a(Da,gl),a(gl,sh),a(Q,ah),a(Q,Sa),a(Sa,eh),a(Q,nh),a(Q,Ha),a(Ha,lh),a(Q,th),c(s,ip,t),c(s,pn,t),a(pn,A),a(A,ph),a(A,La),a(La,yl),a(yl,rh),a(A,oh),a(A,Oa),a(Oa,wl),a(wl,ch),a(A,hh),a(A,Ka),a(Ka,Jl),a(Jl,mh),a(A,uh),a(A,_l),a(_l,ih),a(A,dh),a(A,vl),a(vl,jh),a(A,fh),c(s,dp,t),b(se,s,t),c(s,jp,t),c(s,ae,t),a(ae,L),a(L,bh),a(L,rn),a(rn,$l),a($l,gh),a(L,yh),a(L,Tl),a(Tl,wh),a(L,Jh),a(L,Ml),a(Ml,_h),a(L,vh),c(s,fp,t),b(ee,s,t),c(s,bp,t),b(Os,s,t),c(s,gp,t),c(s,ne,t),a(ne,le),a(le,$h),a(le,te),a(te,El),a(El,Th),a(le,Mh),c(s,yp,t),b(pe,s,t),c(s,wp,t),c(s,re,t),a(re,oe),a(oe,Eh),a(oe,Il),a(Il,Ih),a(oe,kh),c(s,Jp,t),b(ce,s,t),c(s,_p,t),c(s,on,t),a(on,Uh),c(s,vp,t),b(he,s,t),c(s,$p,t),c(s,me,t),a(me,kl),c(s,Tp,t),b(Ks,s,t),c(s,Mp,t),c(s,Es,t),a(Es,sa),a(sa,Ul),b(ue,Ul,null),a(Es,xh),a(Es,xl),a(xl,Ch),c(s,Ep,t),c(s,Y,t),a(Y,Zh),a(Y,cn),a(cn,Vh),a(Y,Bh),a(Y,hn),a(hn,Gh),a(Y,Ah),a(Y,mn),a(mn,zh),a(Y,Wh),a(Y,Cl),a(Cl,Nh),a(Y,Yh),c(s,Ip,t),b(ie,s,t),c(s,kp,t),c(s,Is,t),a(Is,aa),a(aa,Zl),b(de,Zl,null),a(Is,Rh),a(Is,Vl),a(Vl,Fh),c(s,Up,t),c(s,ea,t),a(ea,qh),a(ea,un),a(un,Xh),a(ea,Ph),c(s,xp,t),c(s,ds,t),a(ds,Qh),a(ds,je),a(je,Dh),a(ds,Sh),a(ds,fe),a(fe,Hh),a(ds,Lh),c(s,Cp,t),b(be,s,t),c(s,Zp,t),c(s,js,t),a(js,Oh),a(js,Bl),a(Bl,Kh),a(js,sm),a(js,Gl),a(Gl,am),a(js,em),c(s,Vp,t),b(ge,s,t),c(s,Bp,t),c(s,fs,t),a(fs,nm),a(fs,Al),a(Al,lm),a(fs,tm),a(fs,zl),a(zl,pm),a(fs,rm),c(s,Gp,t),b(ye,s,t),c(s,Ap,t),c(s,na,t),a(na,om),a(na,dn),a(dn,cm),a(na,hm),c(s,zp,t),b(we,s,t),c(s,Wp,t),c(s,la,t),a(la,mm),a(la,jn),a(jn,um),a(la,im),c(s,Np,t),b(Je,s,t),c(s,Yp,t),c(s,fn,t),a(fn,q),a(q,dm),a(q,Wl),a(Wl,jm),a(q,fm),a(q,Nl),a(Nl,bm),a(q,gm),a(q,Yl),a(Yl,ym),a(q,wm),a(q,Rl),a(Rl,Jm),a(q,_m),c(s,Rp,t),b(_e,s,t),c(s,Fp,t),c(s,ve,t),a(ve,$e),a($e,vm),a($e,Fl),a(Fl,$m),a($e,Tm),c(s,qp,t),b(Te,s,t),c(s,Xp,t),c(s,bs,t),a(bs,Mm),a(bs,ql),a(ql,Em),a(bs,Im),a(bs,Xl),a(Xl,km),a(bs,Um),Pp=!0},p(s,[t]){const Me={};t&2&&(Me.$$scope={dirty:t,ctx:s}),Us.$set(Me);const Pl={};t&2&&(Pl.$$scope={dirty:t,ctx:s}),Cs.$set(Pl);const Ql={};t&2&&(Ql.$$scope={dirty:t,ctx:s}),zs.$set(Ql);const Dl={};t&2&&(Dl.$$scope={dirty:t,ctx:s}),Ns.$set(Dl);const Sl={};t&2&&(Sl.$$scope={dirty:t,ctx:s}),Ds.$set(Sl);const O={};t&2&&(O.$$scope={dirty:t,ctx:s}),Ss.$set(O);const Ee={};t&2&&(Ee.$$scope={dirty:t,ctx:s}),Os.$set(Ee);const Hl={};t&2&&(Hl.$$scope={dirty:t,ctx:s}),Ks.$set(Hl)},i(s){Pp||(g(v.$$.fragment,s),g(U.$$.fragment,s),g(Us.$$.fragment,s),g(ha.$$.fragment,s),g(ma.$$.fragment,s),g(ua.$$.fragment,s),g(Cs.$$.fragment,s),g(ia.$$.fragment,s),g(da.$$.fragment,s),g(ja.$$.fragment,s),g(fa.$$.fragment,s),g(ba.$$.fragment,s),g(ga.$$.fragment,s),g(ya.$$.fragment,s),g(wa.$$.fragment,s),g(zs.$$.fragment,s),g(Ja.$$.fragment,s),g(Ns.$$.fragment,s),g(_a.$$.fragment,s),g(Ta.$$.fragment,s),g(Ma.$$.fragment,s),g(Ua.$$.fragment,s),g(Za.$$.fragment,s),g(Va.$$.fragment,s),g(Ba.$$.fragment,s),g(Ga.$$.fragment,s),g(Aa.$$.fragment,s),g(za.$$.fragment,s),g(Wa.$$.fragment,s),g(Na.$$.fragment,s),g(Ds.$$.fragment,s),g(Ss.$$.fragment,s),g(Fa.$$.fragment,s),g(Xa.$$.fragment,s),g(Qa.$$.fragment,s),g(se.$$.fragment,s),g(ee.$$.fragment,s),g(Os.$$.fragment,s),g(pe.$$.fragment,s),g(ce.$$.fragment,s),g(he.$$.fragment,s),g(Ks.$$.fragment,s),g(ue.$$.fragment,s),g(ie.$$.fragment,s),g(de.$$.fragment,s),g(be.$$.fragment,s),g(ge.$$.fragment,s),g(ye.$$.fragment,s),g(we.$$.fragment,s),g(Je.$$.fragment,s),g(_e.$$.fragment,s),g(Te.$$.fragment,s),Pp=!0)},o(s){y(v.$$.fragment,s),y(U.$$.fragment,s),y(Us.$$.fragment,s),y(ha.$$.fragment,s),y(ma.$$.fragment,s),y(ua.$$.fragment,s),y(Cs.$$.fragment,s),y(ia.$$.fragment,s),y(da.$$.fragment,s),y(ja.$$.fragment,s),y(fa.$$.fragment,s),y(ba.$$.fragment,s),y(ga.$$.fragment,s),y(ya.$$.fragment,s),y(wa.$$.fragment,s),y(zs.$$.fragment,s),y(Ja.$$.fragment,s),y(Ns.$$.fragment,s),y(_a.$$.fragment,s),y(Ta.$$.fragment,s),y(Ma.$$.fragment,s),y(Ua.$$.fragment,s),y(Za.$$.fragment,s),y(Va.$$.fragment,s),y(Ba.$$.fragment,s),y(Ga.$$.fragment,s),y(Aa.$$.fragment,s),y(za.$$.fragment,s),y(Wa.$$.fragment,s),y(Na.$$.fragment,s),y(Ds.$$.fragment,s),y(Ss.$$.fragment,s),y(Fa.$$.fragment,s),y(Xa.$$.fragment,s),y(Qa.$$.fragment,s),y(se.$$.fragment,s),y(ee.$$.fragment,s),y(Os.$$.fragment,s),y(pe.$$.fragment,s),y(ce.$$.fragment,s),y(he.$$.fragment,s),y(Ks.$$.fragment,s),y(ue.$$.fragment,s),y(ie.$$.fragment,s),y(de.$$.fragment,s),y(be.$$.fragment,s),y(ge.$$.fragment,s),y(ye.$$.fragment,s),y(we.$$.fragment,s),y(Je.$$.fragment,s),y(_e.$$.fragment,s),y(Te.$$.fragment,s),Pp=!1},d(s){e(u),s&&e(_),s&&e(d),w(v),s&&e(V),w(U,s),s&&e(z),s&&e(B),s&&e(R),s&&e(M),s&&e(Ll),w(Us,s),s&&e(Ol),s&&e(Ve),s&&e(Kl),w(ha,s),s&&e(st),s&&e(Js),w(ma),s&&e(at),w(ua,s),s&&e(et),s&&e(ns),s&&e(nt),w(Cs,s),s&&e(lt),s&&e(ls),s&&e(tt),w(ia,s),s&&e(pt),s&&e(Ae),s&&e(rt),w(da,s),s&&e(ot),s&&e(ze),s&&e(ct),s&&e(ts),s&&e(ht),s&&e(Zs),s&&e(mt),w(ja,s),s&&e(ut),s&&e(ps),s&&e(it),s&&e(Xe),s&&e(dt),w(fa,s),s&&e(jt),s&&e(_s),w(ba),s&&e(ft),s&&e(Bs),s&&e(bt),s&&e(rs),s&&e(gt),w(ga,s),s&&e(yt),s&&e(Gs),s&&e(wt),s&&e(vs),w(ya),s&&e(Jt),s&&e(Pe),s&&e(_t),s&&e(os),s&&e(vt),w(wa,s),s&&e($t),w(zs,s),s&&e(Tt),s&&e($s),w(Ja),s&&e(Mt),s&&e(Qe),s&&e(Et),s&&e(X),s&&e(It),w(Ns,s),s&&e(kt),s&&e(Ts),w(_a),s&&e(Ut),s&&e(Rs),s&&e(xt),s&&e(cs),s&&e(Ct),w(Ta,s),s&&e(Zt),s&&e(hs),s&&e(Vt),w(Ma,s),s&&e(Bt),s&&e(Se),s&&e(Gt),s&&e(ms),s&&e(At),s&&e(Fs),s&&e(zt),s&&e(Ke),s&&e(Wt),w(Ua,s),s&&e(Nt),s&&e(xa),s&&e(Yt),w(Za,s),s&&e(Rt),s&&e(P),s&&e(Ft),s&&e(qs),s&&e(qt),w(Va,s),s&&e(Xt),s&&e(us),s&&e(Pt),w(Ba,s),s&&e(Qt),s&&e(an),s&&e(Dt),w(Ga,s),s&&e(St),s&&e(en),s&&e(Ht),w(Aa,s),s&&e(Lt),s&&e(Xs),s&&e(Ot),w(za,s),s&&e(Kt),s&&e(nn),s&&e(sp),w(Wa,s),s&&e(ap),s&&e(Ms),w(Na),s&&e(ep),s&&e(Qs),s&&e(np),w(Ds,s),s&&e(lp),s&&e(is),s&&e(tp),w(Ss,s),s&&e(pp),w(Fa,s),s&&e(rp),s&&e(Hs),s&&e(op),w(Xa,s),s&&e(cp),s&&e(Pa),s&&e(hp),s&&e(Ls),s&&e(mp),w(Qa,s),s&&e(up),s&&e(Q),s&&e(ip),s&&e(pn),s&&e(dp),w(se,s),s&&e(jp),s&&e(ae),s&&e(fp),w(ee,s),s&&e(bp),w(Os,s),s&&e(gp),s&&e(ne),s&&e(yp),w(pe,s),s&&e(wp),s&&e(re),s&&e(Jp),w(ce,s),s&&e(_p),s&&e(on),s&&e(vp),w(he,s),s&&e($p),s&&e(me),s&&e(Tp),w(Ks,s),s&&e(Mp),s&&e(Es),w(ue),s&&e(Ep),s&&e(Y),s&&e(Ip),w(ie,s),s&&e(kp),s&&e(Is),w(de),s&&e(Up),s&&e(ea),s&&e(xp),s&&e(ds),s&&e(Cp),w(be,s),s&&e(Zp),s&&e(js),s&&e(Vp),w(ge,s),s&&e(Bp),s&&e(fs),s&&e(Gp),w(ye,s),s&&e(Ap),s&&e(na),s&&e(zp),w(we,s),s&&e(Wp),s&&e(la),s&&e(Np),w(Je,s),s&&e(Yp),s&&e(fn),s&&e(Rp),w(_e,s),s&&e(Fp),s&&e(ve),s&&e(qp),w(Te,s),s&&e(Xp),s&&e(bs)}}}const Ud={local:"preprocess",sections:[{local:"natural-language-processing",sections:[{local:"pad",title:"Pad"},{local:"truncation",title:"Truncation"},{local:"build-tensors",title:"Build tensors"}],title:"Natural Language Processing"},{local:"audio",title:"Audio"},{local:"computer-vision",sections:[{local:"pad",title:"Pad"}],title:"Computer vision"},{local:"multimodal",title:"Multimodal"}],title:"Preprocess"};function xd(k){return dd(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wd extends hd{constructor(u){super();md(this,u,xd,kd,ud,{})}}export{Wd as default,Ud as metadata};
