import{S as uf,i as hf,s as mf,e as r,k as u,w as d,t as n,M as ff,c as p,d as e,m as h,a as o,x as b,h as t,b as i,N as pf,G as a,g as c,y as j,q as g,o as _,B as $,v as df,L as of}from"../chunks/vendor-hf-doc-builder.js";import{T as qe}from"../chunks/Tip-hf-doc-builder.js";import{Y as bf}from"../chunks/Youtube-hf-doc-builder.js";import{I as As}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as q}from"../chunks/CodeBlock-hf-doc-builder.js";import{D as jf}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as gf,M as cf}from"../chunks/Markdown-hf-doc-builder.js";function _f(A){let m,w,f,v,k,y,E;return{c(){m=r("p"),w=r("code"),f=n("AutoProcessor"),v=u(),k=r("strong"),y=n("always"),E=n(" works and automatically chooses the correct class for the model you\u2019re using, whether you\u2019re using a tokenizer, image processor, feature extractor or processor.")},l(P){m=p(P,"P",{});var C=o(m);w=p(C,"CODE",{});var I=o(w);f=t(I,"AutoProcessor"),I.forEach(e),v=h(C),k=p(C,"STRONG",{});var z=o(k);y=t(z,"always"),z.forEach(e),E=t(C," works and automatically chooses the correct class for the model you\u2019re using, whether you\u2019re using a tokenizer, image processor, feature extractor or processor."),C.forEach(e)},m(P,C){c(P,m,C),a(m,w),a(w,f),a(m,v),a(m,k),a(k,y),a(m,E)},d(P){P&&e(m)}}}function $f(A){let m,w,f,v,k;return{c(){m=r("p"),w=n("If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referrred to as the "),f=r("em"),v=n("vocab"),k=n(") during pretraining.")},l(y){m=p(y,"P",{});var E=o(m);w=t(E,"If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referrred to as the "),f=p(E,"EM",{});var P=o(f);v=t(P,"vocab"),P.forEach(e),k=t(E,") during pretraining."),E.forEach(e)},m(y,E){c(y,m,E),a(m,w),a(m,f),a(f,v),a(m,k)},d(y){y&&e(m)}}}function vf(A){let m,w,f,v,k;return{c(){m=r("p"),w=n("Check out the "),f=r("a"),v=n("Padding and truncation"),k=n(" concept guide to learn more different padding and truncation arguments."),this.h()},l(y){m=p(y,"P",{});var E=o(m);w=t(E,"Check out the "),f=p(E,"A",{href:!0});var P=o(f);v=t(P,"Padding and truncation"),P.forEach(e),k=t(E," concept guide to learn more different padding and truncation arguments."),E.forEach(e),this.h()},h(){i(f,"href","./pad_truncation")},m(y,E){c(y,m,E),a(m,w),a(m,f),a(f,v),a(m,k)},d(y){y&&e(m)}}}function wf(A){let m,w;return m=new q({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="pt")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]])}`}}),{c(){d(m.$$.fragment)},l(f){b(m.$$.fragment,f)},m(f,v){j(m,f,v),w=!0},p:of,i(f){w||(g(m.$$.fragment,f),w=!0)},o(f){_(m.$$.fragment,f),w=!1},d(f){$(m,f)}}}function yf(A){let m,w;return m=new cf({props:{$$slots:{default:[wf]},$$scope:{ctx:A}}}),{c(){d(m.$$.fragment)},l(f){b(m.$$.fragment,f)},m(f,v){j(m,f,v),w=!0},p(f,v){const k={};v&2&&(k.$$scope={dirty:v,ctx:f}),m.$set(k)},i(f){w||(g(m.$$.fragment,f),w=!0)},o(f){_(m.$$.fragment,f),w=!1},d(f){$(m,f)}}}function kf(A){let m,w;return m=new q({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="tf")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
      dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;}`}}),{c(){d(m.$$.fragment)},l(f){b(m.$$.fragment,f)},m(f,v){j(m,f,v),w=!0},p:of,i(f){w||(g(m.$$.fragment,f),w=!0)},o(f){_(m.$$.fragment,f),w=!1},d(f){$(m,f)}}}function Ef(A){let m,w;return m=new cf({props:{$$slots:{default:[kf]},$$scope:{ctx:A}}}),{c(){d(m.$$.fragment)},l(f){b(m.$$.fragment,f)},m(f,v){j(m,f,v),w=!0},p(f,v){const k={};v&2&&(k.$$scope={dirty:v,ctx:f}),m.$set(k)},i(f){w||(g(m.$$.fragment,f),w=!0)},o(f){_(m.$$.fragment,f),w=!1},d(f){$(m,f)}}}function xf(A){let m,w,f,v,k,y,E,P,C,I,z,N,O,V,W;return{c(){m=r("p"),w=n(`Image preprocessing often follows some form of image augmentation. Both image preprocessing and image augmentation
transform image data, but they serve different purposes:`),f=u(),v=r("ul"),k=r("li"),y=n("Image augmentation alters images in a way that can help prevent overfitting and increase the robustness of the model. You can get creative in how you augment your data - adjust brightness and colors, crop, rotate, resize, zoom, etc. However, be mindful not to change the meaning of the images with your augmentations."),E=u(),P=r("li"),C=n("Image preprocessing guarantees that the images match the model\u2019s expected input format. When fine-tuning a computer vision model, images must be preprocessed exactly as when the model was initially trained."),I=u(),z=r("p"),N=n("You can use any library you like for image augmentation. For image preprocessing, use the "),O=r("code"),V=n("ImageProcessor"),W=n(" associated with the model.")},l(x){m=p(x,"P",{});var T=o(m);w=t(T,`Image preprocessing often follows some form of image augmentation. Both image preprocessing and image augmentation
transform image data, but they serve different purposes:`),T.forEach(e),f=h(x),v=p(x,"UL",{});var F=o(v);k=p(F,"LI",{});var H=o(k);y=t(H,"Image augmentation alters images in a way that can help prevent overfitting and increase the robustness of the model. You can get creative in how you augment your data - adjust brightness and colors, crop, rotate, resize, zoom, etc. However, be mindful not to change the meaning of the images with your augmentations."),H.forEach(e),E=h(F),P=p(F,"LI",{});var Z=o(P);C=t(Z,"Image preprocessing guarantees that the images match the model\u2019s expected input format. When fine-tuning a computer vision model, images must be preprocessed exactly as when the model was initially trained."),Z.forEach(e),F.forEach(e),I=h(x),z=p(x,"P",{});var R=o(z);N=t(R,"You can use any library you like for image augmentation. For image preprocessing, use the "),O=p(R,"CODE",{});var ss=o(O);V=t(ss,"ImageProcessor"),ss.forEach(e),W=t(R," associated with the model."),R.forEach(e)},m(x,T){c(x,m,T),a(m,w),c(x,f,T),c(x,v,T),a(v,k),a(k,y),a(v,E),a(v,P),a(P,C),c(x,I,T),c(x,z,T),a(z,N),a(z,O),a(O,V),a(z,W)},d(x){x&&e(m),x&&e(f),x&&e(v),x&&e(I),x&&e(z)}}}function qf(A){let m,w,f,v,k;return{c(){m=r("p"),w=n("Use \u{1F917} Datasets "),f=r("code"),v=n("split"),k=n(" parameter to only load a small sample from the training split since the dataset is quite large!")},l(y){m=p(y,"P",{});var E=o(m);w=t(E,"Use \u{1F917} Datasets "),f=p(E,"CODE",{});var P=o(f);v=t(P,"split"),P.forEach(e),k=t(E," parameter to only load a small sample from the training split since the dataset is quite large!"),E.forEach(e)},m(y,E){c(y,m,E),a(m,w),a(m,f),a(f,v),a(m,k)},d(y){y&&e(m)}}}function Pf(A){let m,w,f,v,k,y,E,P,C,I,z,N,O,V,W,x,T,F,H,Z,R,ss,K;return{c(){m=r("p"),w=n("In the example above we set "),f=r("code"),v=n("do_resize=False"),k=n(` because we have already resized the images in the image augmentation transformation,
and leveraged the `),y=r("code"),E=n("size"),P=n(" attribute from the appropriate "),C=r("code"),I=n("image_processor"),z=n(`. If you do not resize images during image augmentation,
leave this parameter out. By default, `),N=r("code"),O=n("ImageProcessor"),V=n(" will handle the resizing."),W=u(),x=r("p"),T=n("If you wish to normalize images as a part of the augmentation transformation, use the "),F=r("code"),H=n("image_processor.image_mean"),Z=n(`,
and `),R=r("code"),ss=n("image_processor.image_std"),K=n(" values.")},l(L){m=p(L,"P",{});var D=o(m);w=t(D,"In the example above we set "),f=p(D,"CODE",{});var Pe=o(f);v=t(Pe,"do_resize=False"),Pe.forEach(e),k=t(D,` because we have already resized the images in the image augmentation transformation,
and leveraged the `),y=p(D,"CODE",{});var Ae=o(y);E=t(Ae,"size"),Ae.forEach(e),P=t(D," attribute from the appropriate "),C=p(D,"CODE",{});var ze=o(C);I=t(ze,"image_processor"),ze.forEach(e),z=t(D,`. If you do not resize images during image augmentation,
leave this parameter out. By default, `),N=p(D,"CODE",{});var as=o(N);O=t(as,"ImageProcessor"),as.forEach(e),V=t(D," will handle the resizing."),D.forEach(e),W=h(L),x=p(L,"P",{});var es=o(x);T=t(es,"If you wish to normalize images as a part of the augmentation transformation, use the "),F=p(es,"CODE",{});var $s=o(F);H=t($s,"image_processor.image_mean"),$s.forEach(e),Z=t(es,`,
and `),R=p(es,"CODE",{});var Ce=o(R);ss=t(Ce,"image_processor.image_std"),Ce.forEach(e),K=t(es," values."),es.forEach(e)},m(L,D){c(L,m,D),a(m,w),a(m,f),a(f,v),a(m,k),a(m,y),a(y,E),a(m,P),a(m,C),a(C,I),a(m,z),a(m,N),a(N,O),a(m,V),c(L,W,D),c(L,x,D),a(x,T),a(x,F),a(F,H),a(x,Z),a(x,R),a(R,ss),a(x,K)},d(L){L&&e(m),L&&e(W),L&&e(x)}}}function Af(A){let m,w,f,v,k;return{c(){m=r("p"),w=n("For tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, "),f=r("code"),v=n("ImageProcessor"),k=n(`
offers post processing methods. These methods convert model\u2019s raw outputs into meaningful predictions such as bounding boxes,
or segmentation maps.`)},l(y){m=p(y,"P",{});var E=o(m);w=t(E,"For tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, "),f=p(E,"CODE",{});var P=o(f);v=t(P,"ImageProcessor"),P.forEach(e),k=t(E,`
offers post processing methods. These methods convert model\u2019s raw outputs into meaningful predictions such as bounding boxes,
or segmentation maps.`),E.forEach(e)},m(y,E){c(y,m,E),a(m,w),a(m,f),a(f,v),a(m,k)},d(y){y&&e(m)}}}function zf(A){let m,w,f,v,k,y,E,P,C,I,z,N,O,V,W,x,T,F,H,Z,R,ss,K,L,D,Pe,Ae,ze,as,es,$s,Ce,wp,yp,ca,kp,De,Ep,xp,Qt,zs,Xt,Te,qp,Zt,ua,sl,vs,Cs,Cn,ha,Pp,Dn,Ap,al,ma,el,ns,zp,Ie,Cp,Dp,Tn,Tp,Ip,nl,Ds,tl,ts,Op,Oe,Lp,Sp,In,Np,Fp,ll,ia,rl,Le,Rp,pl,fa,ol,Se,Bp,cl,ls,Ne,Fe,Wp,Hp,Mp,Re,Be,Yp,Jp,Up,We,He,Gp,Vp,ul,Ts,Kp,On,Qp,Xp,hl,da,ml,rs,Zp,Ln,so,ao,Sn,eo,no,il,Me,to,fl,ba,dl,ws,Is,Nn,ja,lo,Fn,ro,bl,Os,po,Rn,oo,co,jl,ps,uo,Bn,ho,mo,Wn,io,fo,gl,ga,_l,Ls,bo,Hn,jo,go,$l,ys,Ss,Mn,_a,_o,Yn,$o,vl,Ye,vo,wl,os,wo,Jn,yo,ko,Un,Eo,xo,yl,$a,kl,Ns,El,ks,Fs,Gn,va,qo,Vn,Po,xl,Je,Ao,ql,Y,zo,Kn,Co,Do,Qn,To,Io,Xn,Oo,Lo,Pl,Rs,Al,Es,Bs,Zn,wa,So,st,No,zl,Ws,Fo,Ue,Ro,Bo,Cl,cs,Wo,ya,Ho,Mo,ka,Yo,Jo,Dl,Ea,Tl,us,Uo,at,Go,Vo,et,Ko,Qo,Il,xa,Ol,Ge,Xo,Ll,hs,Ve,nt,Zo,sc,ac,Ke,tt,ec,nc,tc,Qe,lt,lc,rc,Sl,Hs,pc,qa,oc,cc,Nl,Xe,Pa,uc,rt,hc,mc,Fl,Aa,Rl,za,Ca,ic,pt,fc,dc,Bl,Da,Wl,J,bc,ot,jc,gc,ct,_c,$c,ut,vc,wc,Hl,Ms,yc,Ze,kc,Ec,Ml,Ta,Yl,ms,xc,ht,qc,Pc,mt,Ac,zc,Jl,Ia,Ul,sn,Cc,Gl,Oa,Vl,an,Dc,Kl,La,Ql,Ys,Tc,it,Ic,Oc,Xl,Sa,Zl,en,Lc,sr,Na,ar,xs,Js,ft,Fa,Sc,dt,Nc,er,Us,Fc,nn,Rc,Bc,nr,Gs,tr,is,Wc,Ra,Hc,Mc,Ba,Yc,Jc,lr,Vs,rr,Wa,pr,Ks,Uc,Ha,bt,Gc,Vc,or,Ma,cr,Ya,jt,Sh,ur,Qs,Kc,tn,Qc,Xc,hr,Ja,mr,U,Zc,Ua,gt,su,au,Ga,eu,nu,Va,tu,lu,ir,ln,S,ru,Ka,_t,pu,ou,Qa,$t,cu,uu,Xa,vt,hu,mu,wt,iu,fu,yt,du,bu,fr,Za,dr,se,Q,ju,rn,kt,gu,_u,Et,$u,vu,xt,wu,yu,br,ae,jr,Xs,gr,ee,ne,ku,te,qt,Eu,xu,_r,le,$r,re,pe,qu,Pt,Pu,Au,vr,oe,wr,pn,zu,yr,ce,kr,ue,At,Nh,Er,Zs,xr,qs,sa,zt,he,Cu,Ct,Du,qr,B,Tu,on,Iu,Ou,cn,Lu,Su,un,Nu,Fu,Dt,Ru,Bu,Pr,me,Ar,Ps,aa,Tt,ie,Wu,It,Hu,zr,ea,Mu,hn,Yu,Ju,Cr,fs,Uu,fe,Gu,Vu,de,Ku,Qu,Dr,be,Tr,ds,Xu,Ot,Zu,sh,Lt,ah,eh,Ir,je,Or,bs,nh,St,th,lh,Nt,rh,ph,Lr,ge,Sr,na,oh,mn,ch,uh,Nr,_e,Fr,ta,hh,fn,mh,ih,Rr,$e,Br,dn,M,fh,Ft,dh,bh,Rt,jh,gh,Bt,_h,$h,Wt,vh,wh,Wr,ve,Hr,we,ye,yh,Ht,kh,Eh,Mr,ke,Yr,js,xh,Mt,qh,Ph,Yt,Ah,zh,Jr;return y=new As({}),z=new jf({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"}]}}),zs=new qe({props:{$$slots:{default:[_f]},$$scope:{ctx:A}}}),ua=new q({props:{code:"pip install datasets",highlighted:"pip install datasets"}}),ha=new As({}),ma=new bf({props:{id:"Yffk5aydLzg"}}),Ds=new qe({props:{$$slots:{default:[$f]},$$scope:{ctx:A}}}),ia=new q({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),fa=new q({props:{code:`encoded_input = tokenizer("Do not meddle in the affairs of wizards, for they are subtle and quick to anger.")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(<span class="hljs-string">&quot;Do not meddle in the affairs of wizards, for they are subtle and quick to anger.&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2079</span>, <span class="hljs-number">2025</span>, <span class="hljs-number">19960</span>, <span class="hljs-number">10362</span>, <span class="hljs-number">1999</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">3821</span>, <span class="hljs-number">1997</span>, <span class="hljs-number">16657</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">2027</span>, <span class="hljs-number">2024</span>, <span class="hljs-number">11259</span>, <span class="hljs-number">1998</span>, <span class="hljs-number">4248</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">4963</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),da=new q({props:{code:'tokenizer.decode(encoded_input["input_ids"])',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]&#x27;</span>`}}),ba=new q({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_inputs = tokenizer(batch_sentences)
print(encoded_inputs)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_inputs = tokenizer(batch_sentences)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_inputs)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]}`}}),ja=new As({}),ga=new q({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),_a=new As({}),$a=new q({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),Ns=new qe({props:{$$slots:{default:[vf]},$$scope:{ctx:A}}}),va=new As({}),Rs=new gf({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Ef],pytorch:[yf]},$$scope:{ctx:A}}}),wa=new As({}),Ea=new q({props:{code:`from datasets import load_dataset, Audio

dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),xa=new q({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),Aa=new q({props:{code:'dataset = dataset.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),Da=new q({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">2.3443763e-05</span>,  <span class="hljs-number">2.1729663e-04</span>,  <span class="hljs-number">2.2145823e-04</span>, ...,
         <span class="hljs-number">3.8356509e-05</span>, -<span class="hljs-number">7.3497440e-06</span>, -<span class="hljs-number">2.1754686e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),Ta=new q({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)`}}),Ia=new q({props:{code:`audio_input = [dataset[0]["audio"]["array"]]
feature_extractor(audio_input, sampling_rate=16000)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>audio_input = [dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor(audio_input, sampling_rate=<span class="hljs-number">16000</span>)
{<span class="hljs-string">&#x27;input_values&#x27;</span>: [array([ <span class="hljs-number">3.8106556e-04</span>,  <span class="hljs-number">2.7506407e-03</span>,  <span class="hljs-number">2.8015103e-03</span>, ...,
        <span class="hljs-number">5.6335266e-04</span>,  <span class="hljs-number">4.6588284e-06</span>, -<span class="hljs-number">1.7142107e-04</span>], dtype=float32)]}`}}),Oa=new q({props:{code:`dataset[0]["audio"]["array"].shape

dataset[1]["audio"]["array"].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">173398</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">106496</span>,)`}}),La=new q({props:{code:`def preprocess_function(examples):
    audio_arrays = [x["array"] for x in examples["audio"]]
    inputs = feature_extractor(
        audio_arrays,
        sampling_rate=16000,
        padding=True,
        max_length=100000,
        truncation=True,
    )
    return inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays,
<span class="hljs-meta">... </span>        sampling_rate=<span class="hljs-number">16000</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">100000</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`}}),Sa=new q({props:{code:"processed_dataset = preprocess_function(dataset[:5])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset = preprocess_function(dataset[:<span class="hljs-number">5</span>])'}}),Na=new q({props:{code:`processed_dataset["input_values"][0].shape

processed_dataset["input_values"][1].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">0</span>].shape
(<span class="hljs-number">100000</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">1</span>].shape
(<span class="hljs-number">100000</span>,)`}}),Fa=new As({}),Gs=new qe({props:{$$slots:{default:[xf]},$$scope:{ctx:A}}}),Vs=new qe({props:{$$slots:{default:[qf]},$$scope:{ctx:A}}}),Wa=new q({props:{code:`from datasets import load_dataset

dataset = load_dataset("food101", split="train[:100]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;food101&quot;</span>, split=<span class="hljs-string">&quot;train[:100]&quot;</span>)`}}),Ma=new q({props:{code:'dataset[0]["image"]',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]'}}),Ja=new q({props:{code:`from transformers import AutoImageProcessor

image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoImageProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>image_processor = AutoImageProcessor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224&quot;</span>)`}}),Za=new q({props:{code:`from torchvision.transforms import RandomResizedCrop, ColorJitter, Compose

size = (
    image_processor.size["shortest_edge"]
    if "shortest_edge" in image_processor.size
    else (image_processor.size["height"], image_processor.size["width"])
)

_transforms = Compose([RandomResizedCrop(size), ColorJitter(brightness=0.5, hue=0.5)])`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> RandomResizedCrop, ColorJitter, Compose

<span class="hljs-meta">&gt;&gt;&gt; </span>size = (
<span class="hljs-meta">... </span>    image_processor.size[<span class="hljs-string">&quot;shortest_edge&quot;</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;shortest_edge&quot;</span> <span class="hljs-keyword">in</span> image_processor.size
<span class="hljs-meta">... </span>    <span class="hljs-keyword">else</span> (image_processor.size[<span class="hljs-string">&quot;height&quot;</span>], image_processor.size[<span class="hljs-string">&quot;width&quot;</span>])
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>_transforms = Compose([RandomResizedCrop(size), ColorJitter(brightness=<span class="hljs-number">0.5</span>, hue=<span class="hljs-number">0.5</span>)])`}}),ae=new q({props:{code:`def transforms(examples):
    images = [_transforms(img.convert("RGB")) for img in examples["image"]]
    examples["pixel_values"] = image_processor(images, do_resize=False, return_tensors="pt")["pixel_values"]
    return examples`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    images = [_transforms(img.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> img <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = image_processor(images, do_resize=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`}}),Xs=new qe({props:{$$slots:{default:[Pf]},$$scope:{ctx:A}}}),le=new q({props:{code:"dataset.set_transform(transforms)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_transform(transforms)'}}),oe=new q({props:{code:"dataset[0].keys()",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>].keys()'}}),ce=new q({props:{code:`import numpy as np
import matplotlib.pyplot as plt

img = dataset[0]["pixel_values"]
plt.imshow(img.permute(1, 2, 0))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt

<span class="hljs-meta">&gt;&gt;&gt; </span>img = dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>plt.imshow(img.permute(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>))`}}),Zs=new qe({props:{$$slots:{default:[Af]},$$scope:{ctx:A}}}),he=new As({}),me=new q({props:{code:`def collate_fn(batch):
    pixel_values = [item["pixel_values"] for item in batch]
    encoding = image_processor.pad_and_create_pixel_mask(pixel_values, return_tensors="pt")
    labels = [item["labels"] for item in batch]
    batch = {}
    batch["pixel_values"] = encoding["pixel_values"]
    batch["pixel_mask"] = encoding["pixel_mask"]
    batch["labels"] = labels
    return batch`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">collate_fn</span>(<span class="hljs-params">batch</span>):
<span class="hljs-meta">... </span>    pixel_values = [item[<span class="hljs-string">&quot;pixel_values&quot;</span>] <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> batch]
<span class="hljs-meta">... </span>    encoding = image_processor.pad_and_create_pixel_mask(pixel_values, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">... </span>    labels = [item[<span class="hljs-string">&quot;labels&quot;</span>] <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> batch]
<span class="hljs-meta">... </span>    batch = {}
<span class="hljs-meta">... </span>    batch[<span class="hljs-string">&quot;pixel_values&quot;</span>] = encoding[<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">... </span>    batch[<span class="hljs-string">&quot;pixel_mask&quot;</span>] = encoding[<span class="hljs-string">&quot;pixel_mask&quot;</span>]
<span class="hljs-meta">... </span>    batch[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> batch`}}),ie=new As({}),be=new q({props:{code:`from datasets import load_dataset

lj_speech = load_dataset("lj_speech", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = load_dataset(<span class="hljs-string">&quot;lj_speech&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),je=new q({props:{code:'lj_speech = lj_speech.map(remove_columns=["file", "id", "normalized_text"])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.<span class="hljs-built_in">map</span>(remove_columns=[<span class="hljs-string">&quot;file&quot;</span>, <span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;normalized_text&quot;</span>])'}}),ge=new q({props:{code:`lj_speech[0]["audio"]

lj_speech[0]["text"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">7.3242188e-04</span>, -<span class="hljs-number">7.6293945e-04</span>, -<span class="hljs-number">6.4086914e-04</span>, ...,
         <span class="hljs-number">7.3242188e-04</span>,  <span class="hljs-number">2.1362305e-04</span>,  <span class="hljs-number">6.1035156e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">22050</span>}

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]
<span class="hljs-string">&#x27;Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition&#x27;</span>`}}),_e=new q({props:{code:'lj_speech = lj_speech.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),$e=new q({props:{code:`from transformers import AutoProcessor

processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>)`}}),ve=new q({props:{code:`def prepare_dataset(example):
    audio = example["audio"]

    example.update(processor(audio=audio["array"], text=example["text"], sampling_rate=16000))

    return example`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">prepare_dataset</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    audio = example[<span class="hljs-string">&quot;audio&quot;</span>]

<span class="hljs-meta">... </span>    example.update(processor(audio=audio[<span class="hljs-string">&quot;array&quot;</span>], text=example[<span class="hljs-string">&quot;text&quot;</span>], sampling_rate=<span class="hljs-number">16000</span>))

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> example`}}),ke=new q({props:{code:"prepare_dataset(lj_speech[0])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prepare_dataset(lj_speech[<span class="hljs-number">0</span>])'}}),{c(){m=r("meta"),w=u(),f=r("h1"),v=r("a"),k=r("span"),d(y.$$.fragment),E=u(),P=r("span"),C=n("Preprocess"),I=u(),d(z.$$.fragment),N=u(),O=r("p"),V=n("Before you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. \u{1F917} Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you\u2019ll learn that for:"),W=u(),x=r("ul"),T=r("li"),F=n("Text, use a "),H=r("a"),Z=n("Tokenizer"),R=n(" to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors."),ss=u(),K=r("li"),L=n("Speech and audio, use a "),D=r("a"),Pe=n("Feature extractor"),Ae=n(" to extract sequential features from audio waveforms and convert them into tensors."),ze=u(),as=r("li"),es=n("Image inputs use a "),$s=r("a"),Ce=n("ImageProcessor"),wp=n(" to convert images into tensors."),yp=u(),ca=r("li"),kp=n("Multimodal inputs, use a "),De=r("a"),Ep=n("Processor"),xp=n(" to combine a tokenizer and a feature extractor or image processor."),Qt=u(),d(zs.$$.fragment),Xt=u(),Te=r("p"),qp=n("Before you begin, install \u{1F917} Datasets so you can load some datasets to experiment with:"),Zt=u(),d(ua.$$.fragment),sl=u(),vs=r("h2"),Cs=r("a"),Cn=r("span"),d(ha.$$.fragment),Pp=u(),Dn=r("span"),Ap=n("Natural Language Processing"),al=u(),d(ma.$$.fragment),el=u(),ns=r("p"),zp=n("The main tool for preprocessing textual data is a "),Ie=r("a"),Cp=n("tokenizer"),Dp=n(". A tokenizer splits text into "),Tn=r("em"),Tp=n("tokens"),Ip=n(" according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer."),nl=u(),d(Ds.$$.fragment),tl=u(),ts=r("p"),Op=n("Get started by loading a pretrained tokenizer with the "),Oe=r("a"),Lp=n("AutoTokenizer.from_pretrained()"),Sp=n(" method. This downloads the "),In=r("em"),Np=n("vocab"),Fp=n(" a model was pretrained with:"),ll=u(),d(ia.$$.fragment),rl=u(),Le=r("p"),Rp=n("Then pass your text to the tokenizer:"),pl=u(),d(fa.$$.fragment),ol=u(),Se=r("p"),Bp=n("The tokenizer returns a dictionary with three important items:"),cl=u(),ls=r("ul"),Ne=r("li"),Fe=r("a"),Wp=n("input_ids"),Hp=n(" are the indices corresponding to each token in the sentence."),Mp=u(),Re=r("li"),Be=r("a"),Yp=n("attention_mask"),Jp=n(" indicates whether a token should be attended to or not."),Up=u(),We=r("li"),He=r("a"),Gp=n("token_type_ids"),Vp=n(" identifies which sequence a token belongs to when there is more than one sequence."),ul=u(),Ts=r("p"),Kp=n("Return your input by decoding the "),On=r("code"),Qp=n("input_ids"),Xp=n(":"),hl=u(),d(da.$$.fragment),ml=u(),rs=r("p"),Zp=n("As you can see, the tokenizer added two special tokens - "),Ln=r("code"),so=n("CLS"),ao=n(" and "),Sn=r("code"),eo=n("SEP"),no=n(` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer automatically adds them for you.`),il=u(),Me=r("p"),to=n("If there are several sentences you want to preprocess, pass them as a list to the tokenizer:"),fl=u(),d(ba.$$.fragment),dl=u(),ws=r("h3"),Is=r("a"),Nn=r("span"),d(ja.$$.fragment),lo=u(),Fn=r("span"),ro=n("Pad"),bl=u(),Os=r("p"),po=n("Sentences aren\u2019t always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),Rn=r("em"),oo=n("padding token"),co=n(" to shorter sentences."),jl=u(),ps=r("p"),uo=n("Set the "),Bn=r("code"),ho=n("padding"),mo=n(" parameter to "),Wn=r("code"),io=n("True"),fo=n(" to pad the shorter sequences in the batch to match the longest sequence:"),gl=u(),d(ga.$$.fragment),_l=u(),Ls=r("p"),bo=n("The first and third sentences are now padded with "),Hn=r("code"),jo=n("0"),go=n("\u2019s because they are shorter."),$l=u(),ys=r("h3"),Ss=r("a"),Mn=r("span"),d(_a.$$.fragment),_o=u(),Yn=r("span"),$o=n("Truncation"),vl=u(),Ye=r("p"),vo=n("On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you\u2019ll need to truncate the sequence to a shorter length."),wl=u(),os=r("p"),wo=n("Set the "),Jn=r("code"),yo=n("truncation"),ko=n(" parameter to "),Un=r("code"),Eo=n("True"),xo=n(" to truncate a sequence to the maximum length accepted by the model:"),yl=u(),d($a.$$.fragment),kl=u(),d(Ns.$$.fragment),El=u(),ks=r("h3"),Fs=r("a"),Gn=r("span"),d(va.$$.fragment),qo=u(),Vn=r("span"),Po=n("Build tensors"),xl=u(),Je=r("p"),Ao=n("Finally, you want the tokenizer to return the actual tensors that get fed to the model."),ql=u(),Y=r("p"),zo=n("Set the "),Kn=r("code"),Co=n("return_tensors"),Do=n(" parameter to either "),Qn=r("code"),To=n("pt"),Io=n(" for PyTorch, or "),Xn=r("code"),Oo=n("tf"),Lo=n(" for TensorFlow:"),Pl=u(),d(Rs.$$.fragment),Al=u(),Es=r("h2"),Bs=r("a"),Zn=r("span"),d(wa.$$.fragment),So=u(),st=r("span"),No=n("Audio"),zl=u(),Ws=r("p"),Fo=n("For audio tasks, you\u2019ll need a "),Ue=r("a"),Ro=n("feature extractor"),Bo=n(" to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors."),Cl=u(),cs=r("p"),Wo=n("Load the "),ya=r("a"),Ho=n("MInDS-14"),Mo=n(" dataset (see the \u{1F917} "),ka=r("a"),Yo=n("Datasets tutorial"),Jo=n(" for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:"),Dl=u(),d(Ea.$$.fragment),Tl=u(),us=r("p"),Uo=n("Access the first element of the "),at=r("code"),Go=n("audio"),Vo=n(" column to take a look at the input. Calling the "),et=r("code"),Ko=n("audio"),Qo=n(" column automatically loads and resamples the audio file:"),Il=u(),d(xa.$$.fragment),Ol=u(),Ge=r("p"),Xo=n("This returns three items:"),Ll=u(),hs=r("ul"),Ve=r("li"),nt=r("code"),Zo=n("array"),sc=n(" is the speech signal loaded - and potentially resampled - as a 1D array."),ac=u(),Ke=r("li"),tt=r("code"),ec=n("path"),nc=n(" points to the location of the audio file."),tc=u(),Qe=r("li"),lt=r("code"),lc=n("sampling_rate"),rc=n(" refers to how many data points in the speech signal are measured per second."),Sl=u(),Hs=r("p"),pc=n("For this tutorial, you\u2019ll use the "),qa=r("a"),oc=n("Wav2Vec2"),cc=n(" model. Take a look at the model card, and you\u2019ll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your data."),Nl=u(),Xe=r("ol"),Pa=r("li"),uc=n("Use \u{1F917} Datasets\u2019 "),rt=r("code"),hc=n("cast_column"),mc=n(" method to upsample the sampling rate to 16kHz:"),Fl=u(),d(Aa.$$.fragment),Rl=u(),za=r("ol"),Ca=r("li"),ic=n("Call the "),pt=r("code"),fc=n("audio"),dc=n(" column again to resample the audio file:"),Bl=u(),d(Da.$$.fragment),Wl=u(),J=r("p"),bc=n("Next, load a feature extractor to normalize and pad the input. When padding textual data, a "),ot=r("code"),jc=n("0"),gc=n(" is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a "),ct=r("code"),_c=n("0"),$c=n(" - interpreted as silence - to "),ut=r("code"),vc=n("array"),wc=n("."),Hl=u(),Ms=r("p"),yc=n("Load the feature extractor with "),Ze=r("a"),kc=n("AutoFeatureExtractor.from_pretrained()"),Ec=n(":"),Ml=u(),d(Ta.$$.fragment),Yl=u(),ms=r("p"),xc=n("Pass the audio "),ht=r("code"),qc=n("array"),Pc=n(" to the feature extractor. We also recommend adding the "),mt=r("code"),Ac=n("sampling_rate"),zc=n(" argument in the feature extractor in order to better debug any silent errors that may occur."),Jl=u(),d(Ia.$$.fragment),Ul=u(),sn=r("p"),Cc=n("Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),Gl=u(),d(Oa.$$.fragment),Vl=u(),an=r("p"),Dc=n("Create a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),Kl=u(),d(La.$$.fragment),Ql=u(),Ys=r("p"),Tc=n("Apply the "),it=r("code"),Ic=n("preprocess_function"),Oc=n(" to the the first few examples in the dataset:"),Xl=u(),d(Sa.$$.fragment),Zl=u(),en=r("p"),Lc=n("The sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!"),sr=u(),d(Na.$$.fragment),ar=u(),xs=r("h2"),Js=r("a"),ft=r("span"),d(Fa.$$.fragment),Sc=u(),dt=r("span"),Nc=n("Computer vision"),er=u(),Us=r("p"),Fc=n("For computer vision tasks, you\u2019ll need an "),nn=r("a"),Rc=n("image processor"),Bc=n(` to prepare your dataset for the model.
Image preprocessing consists of several steps that convert images into the input expected by the model. These steps
include but are not limited to resizing, normalizing, color channel correction, and converting images to tensors.`),nr=u(),d(Gs.$$.fragment),tr=u(),is=r("p"),Wc=n("Load the "),Ra=r("a"),Hc=n("food101"),Mc=n(" dataset (see the \u{1F917} "),Ba=r("a"),Yc=n("Datasets tutorial"),Jc=n(" for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:"),lr=u(),d(Vs.$$.fragment),rr=u(),d(Wa.$$.fragment),pr=u(),Ks=r("p"),Uc=n("Next, take a look at the image with \u{1F917} Datasets "),Ha=r("a"),bt=r("code"),Gc=n("Image"),Vc=n(" feature:"),or=u(),d(Ma.$$.fragment),cr=u(),Ya=r("div"),jt=r("img"),ur=u(),Qs=r("p"),Kc=n("Load the image processor with "),tn=r("a"),Qc=n("AutoImageProcessor.from_pretrained()"),Xc=n(":"),hr=u(),d(Ja.$$.fragment),mr=u(),U=r("p"),Zc=n("First, let\u2019s add some image augmentation. You can use any library you prefer, but in this tutorial, we\u2019ll use torchvision\u2019s "),Ua=r("a"),gt=r("code"),su=n("transforms"),au=n(" module. If you\u2019re interested in using another data augmentation library, learn how in the "),Ga=r("a"),eu=n("Albumentations"),nu=n(" or "),Va=r("a"),tu=n("Kornia notebooks"),lu=n("."),ir=u(),ln=r("ol"),S=r("li"),ru=n("Here we use "),Ka=r("a"),_t=r("code"),pu=n("Compose"),ou=n(` to chain together a couple of
transforms - `),Qa=r("a"),$t=r("code"),cu=n("RandomResizedCrop"),uu=n(" and "),Xa=r("a"),vt=r("code"),hu=n("ColorJitter"),mu=n(`.
Note that for resizing, we can get the image size requirements from the `),wt=r("code"),iu=n("image_processor"),fu=n(`. For some models, an exact height and
width are expected, for others only the `),yt=r("code"),du=n("shortest_edge"),bu=n(" is defined."),fr=u(),d(Za.$$.fragment),dr=u(),se=r("ol"),Q=r("li"),ju=n("The model accepts "),rn=r("a"),kt=r("code"),gu=n("pixel_values"),_u=n(`
as its input. `),Et=r("code"),$u=n("ImageProcessor"),vu=n(` can take care of normalizing the images, and generating appropriate tensors.
Create a function that combines image augmentation and image preprocessing for a batch of images and generates `),xt=r("code"),wu=n("pixel_values"),yu=n(":"),br=u(),d(ae.$$.fragment),jr=u(),d(Xs.$$.fragment),gr=u(),ee=r("ol"),ne=r("li"),ku=n("Then use \u{1F917} Datasets "),te=r("a"),qt=r("code"),Eu=n("set_transform"),xu=n(" to apply the transforms on the fly:"),_r=u(),d(le.$$.fragment),$r=u(),re=r("ol"),pe=r("li"),qu=n("Now when you access the image, you\u2019ll notice the image processor has added "),Pt=r("code"),Pu=n("pixel_values"),Au=n(". You can pass your processed dataset to the model now!"),vr=u(),d(oe.$$.fragment),wr=u(),pn=r("p"),zu=n("Here is what the image looks like after the transforms are applied. The image has been randomly cropped and it\u2019s color properties are different."),yr=u(),d(ce.$$.fragment),kr=u(),ue=r("div"),At=r("img"),Er=u(),d(Zs.$$.fragment),xr=u(),qs=r("h3"),sa=r("a"),zt=r("span"),d(he.$$.fragment),Cu=u(),Ct=r("span"),Du=n("Pad"),qr=u(),B=r("p"),Tu=n("In some cases, for instance, when fine-tuning "),on=r("a"),Iu=n("DETR"),Ou=n(`, the model applies scale augmentation at training
time. This may cause images to be different sizes in a batch. You can use `),cn=r("a"),Lu=n("DetrImageProcessor.pad_and_create_pixel_mask()"),Su=n(`
from `),un=r("a"),Nu=n("DetrImageProcessor"),Fu=n(" and define a custom "),Dt=r("code"),Ru=n("collate_fn"),Bu=n(" to batch images together."),Pr=u(),d(me.$$.fragment),Ar=u(),Ps=r("h2"),aa=r("a"),Tt=r("span"),d(ie.$$.fragment),Wu=u(),It=r("span"),Hu=n("Multimodal"),zr=u(),ea=r("p"),Mu=n("For tasks involving multimodal inputs, you\u2019ll need a "),hn=r("a"),Yu=n("processor"),Ju=n(" to prepare your dataset for the model. A processor couples together two processing objects such as as tokenizer and feature extractor."),Cr=u(),fs=r("p"),Uu=n("Load the "),fe=r("a"),Gu=n("LJ Speech"),Vu=n(" dataset (see the \u{1F917} "),de=r("a"),Ku=n("Datasets tutorial"),Qu=n(" for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):"),Dr=u(),d(be.$$.fragment),Tr=u(),ds=r("p"),Xu=n("For ASR, you\u2019re mainly focused on "),Ot=r("code"),Zu=n("audio"),sh=n(" and "),Lt=r("code"),ah=n("text"),eh=n(" so you can remove the other columns:"),Ir=u(),d(je.$$.fragment),Or=u(),bs=r("p"),nh=n("Now take a look at the "),St=r("code"),th=n("audio"),lh=n(" and "),Nt=r("code"),rh=n("text"),ph=n(" columns:"),Lr=u(),d(ge.$$.fragment),Sr=u(),na=r("p"),oh=n("Remember you should always "),mn=r("a"),ch=n("resample"),uh=n(" your audio dataset\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model!"),Nr=u(),d(_e.$$.fragment),Fr=u(),ta=r("p"),hh=n("Load a processor with "),fn=r("a"),mh=n("AutoProcessor.from_pretrained()"),ih=n(":"),Rr=u(),d($e.$$.fragment),Br=u(),dn=r("ol"),M=r("li"),fh=n("Create a function to process the audio data contained in "),Ft=r("code"),dh=n("array"),bh=n(" to "),Rt=r("code"),jh=n("input_values"),gh=n(", and tokenize "),Bt=r("code"),_h=n("text"),$h=n(" to "),Wt=r("code"),vh=n("labels"),wh=n(". These are the inputs to the model:"),Wr=u(),d(ve.$$.fragment),Hr=u(),we=r("ol"),ye=r("li"),yh=n("Apply the "),Ht=r("code"),kh=n("prepare_dataset"),Eh=n(" function to a sample:"),Mr=u(),d(ke.$$.fragment),Yr=u(),js=r("p"),xh=n("The processor has now added "),Mt=r("code"),qh=n("input_values"),Ph=n(" and "),Yt=r("code"),Ah=n("labels"),zh=n(", and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!"),this.h()},l(s){const l=ff('[data-svelte="svelte-1phssyn"]',document.head);m=p(l,"META",{name:!0,content:!0}),l.forEach(e),w=h(s),f=p(s,"H1",{class:!0});var Ee=o(f);v=p(Ee,"A",{id:!0,class:!0,href:!0});var Jt=o(v);k=p(Jt,"SPAN",{});var Ut=o(k);b(y.$$.fragment,Ut),Ut.forEach(e),Jt.forEach(e),E=h(Ee),P=p(Ee,"SPAN",{});var Gt=o(P);C=t(Gt,"Preprocess"),Gt.forEach(e),Ee.forEach(e),I=h(s),b(z.$$.fragment,s),N=h(s),O=p(s,"P",{});var Vt=o(O);V=t(Vt,"Before you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. \u{1F917} Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you\u2019ll learn that for:"),Vt.forEach(e),W=h(s),x=p(s,"UL",{});var X=o(x);T=p(X,"LI",{});var xe=o(T);F=t(xe,"Text, use a "),H=p(xe,"A",{href:!0});var Kt=o(H);Z=t(Kt,"Tokenizer"),Kt.forEach(e),R=t(xe," to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors."),xe.forEach(e),ss=h(X),K=p(X,"LI",{});var Ur=o(K);L=t(Ur,"Speech and audio, use a "),D=p(Ur,"A",{href:!0});var Fh=o(D);Pe=t(Fh,"Feature extractor"),Fh.forEach(e),Ae=t(Ur," to extract sequential features from audio waveforms and convert them into tensors."),Ur.forEach(e),ze=h(X),as=p(X,"LI",{});var Gr=o(as);es=t(Gr,"Image inputs use a "),$s=p(Gr,"A",{href:!0});var Rh=o($s);Ce=t(Rh,"ImageProcessor"),Rh.forEach(e),wp=t(Gr," to convert images into tensors."),Gr.forEach(e),yp=h(X),ca=p(X,"LI",{});var Vr=o(ca);kp=t(Vr,"Multimodal inputs, use a "),De=p(Vr,"A",{href:!0});var Bh=o(De);Ep=t(Bh,"Processor"),Bh.forEach(e),xp=t(Vr," to combine a tokenizer and a feature extractor or image processor."),Vr.forEach(e),X.forEach(e),Qt=h(s),b(zs.$$.fragment,s),Xt=h(s),Te=p(s,"P",{});var Wh=o(Te);qp=t(Wh,"Before you begin, install \u{1F917} Datasets so you can load some datasets to experiment with:"),Wh.forEach(e),Zt=h(s),b(ua.$$.fragment,s),sl=h(s),vs=p(s,"H2",{class:!0});var Kr=o(vs);Cs=p(Kr,"A",{id:!0,class:!0,href:!0});var Hh=o(Cs);Cn=p(Hh,"SPAN",{});var Mh=o(Cn);b(ha.$$.fragment,Mh),Mh.forEach(e),Hh.forEach(e),Pp=h(Kr),Dn=p(Kr,"SPAN",{});var Yh=o(Dn);Ap=t(Yh,"Natural Language Processing"),Yh.forEach(e),Kr.forEach(e),al=h(s),b(ma.$$.fragment,s),el=h(s),ns=p(s,"P",{});var bn=o(ns);zp=t(bn,"The main tool for preprocessing textual data is a "),Ie=p(bn,"A",{href:!0});var Jh=o(Ie);Cp=t(Jh,"tokenizer"),Jh.forEach(e),Dp=t(bn,". A tokenizer splits text into "),Tn=p(bn,"EM",{});var Uh=o(Tn);Tp=t(Uh,"tokens"),Uh.forEach(e),Ip=t(bn," according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer."),bn.forEach(e),nl=h(s),b(Ds.$$.fragment,s),tl=h(s),ts=p(s,"P",{});var jn=o(ts);Op=t(jn,"Get started by loading a pretrained tokenizer with the "),Oe=p(jn,"A",{href:!0});var Gh=o(Oe);Lp=t(Gh,"AutoTokenizer.from_pretrained()"),Gh.forEach(e),Sp=t(jn," method. This downloads the "),In=p(jn,"EM",{});var Vh=o(In);Np=t(Vh,"vocab"),Vh.forEach(e),Fp=t(jn," a model was pretrained with:"),jn.forEach(e),ll=h(s),b(ia.$$.fragment,s),rl=h(s),Le=p(s,"P",{});var Kh=o(Le);Rp=t(Kh,"Then pass your text to the tokenizer:"),Kh.forEach(e),pl=h(s),b(fa.$$.fragment,s),ol=h(s),Se=p(s,"P",{});var Qh=o(Se);Bp=t(Qh,"The tokenizer returns a dictionary with three important items:"),Qh.forEach(e),cl=h(s),ls=p(s,"UL",{});var gn=o(ls);Ne=p(gn,"LI",{});var Ch=o(Ne);Fe=p(Ch,"A",{href:!0});var Xh=o(Fe);Wp=t(Xh,"input_ids"),Xh.forEach(e),Hp=t(Ch," are the indices corresponding to each token in the sentence."),Ch.forEach(e),Mp=h(gn),Re=p(gn,"LI",{});var Dh=o(Re);Be=p(Dh,"A",{href:!0});var Zh=o(Be);Yp=t(Zh,"attention_mask"),Zh.forEach(e),Jp=t(Dh," indicates whether a token should be attended to or not."),Dh.forEach(e),Up=h(gn),We=p(gn,"LI",{});var Th=o(We);He=p(Th,"A",{href:!0});var sm=o(He);Gp=t(sm,"token_type_ids"),sm.forEach(e),Vp=t(Th," identifies which sequence a token belongs to when there is more than one sequence."),Th.forEach(e),gn.forEach(e),ul=h(s),Ts=p(s,"P",{});var Qr=o(Ts);Kp=t(Qr,"Return your input by decoding the "),On=p(Qr,"CODE",{});var am=o(On);Qp=t(am,"input_ids"),am.forEach(e),Xp=t(Qr,":"),Qr.forEach(e),hl=h(s),b(da.$$.fragment,s),ml=h(s),rs=p(s,"P",{});var _n=o(rs);Zp=t(_n,"As you can see, the tokenizer added two special tokens - "),Ln=p(_n,"CODE",{});var em=o(Ln);so=t(em,"CLS"),em.forEach(e),ao=t(_n," and "),Sn=p(_n,"CODE",{});var nm=o(Sn);eo=t(nm,"SEP"),nm.forEach(e),no=t(_n,` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer automatically adds them for you.`),_n.forEach(e),il=h(s),Me=p(s,"P",{});var tm=o(Me);to=t(tm,"If there are several sentences you want to preprocess, pass them as a list to the tokenizer:"),tm.forEach(e),fl=h(s),b(ba.$$.fragment,s),dl=h(s),ws=p(s,"H3",{class:!0});var Xr=o(ws);Is=p(Xr,"A",{id:!0,class:!0,href:!0});var lm=o(Is);Nn=p(lm,"SPAN",{});var rm=o(Nn);b(ja.$$.fragment,rm),rm.forEach(e),lm.forEach(e),lo=h(Xr),Fn=p(Xr,"SPAN",{});var pm=o(Fn);ro=t(pm,"Pad"),pm.forEach(e),Xr.forEach(e),bl=h(s),Os=p(s,"P",{});var Zr=o(Os);po=t(Zr,"Sentences aren\u2019t always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),Rn=p(Zr,"EM",{});var om=o(Rn);oo=t(om,"padding token"),om.forEach(e),co=t(Zr," to shorter sentences."),Zr.forEach(e),jl=h(s),ps=p(s,"P",{});var $n=o(ps);uo=t($n,"Set the "),Bn=p($n,"CODE",{});var cm=o(Bn);ho=t(cm,"padding"),cm.forEach(e),mo=t($n," parameter to "),Wn=p($n,"CODE",{});var um=o(Wn);io=t(um,"True"),um.forEach(e),fo=t($n," to pad the shorter sequences in the batch to match the longest sequence:"),$n.forEach(e),gl=h(s),b(ga.$$.fragment,s),_l=h(s),Ls=p(s,"P",{});var sp=o(Ls);bo=t(sp,"The first and third sentences are now padded with "),Hn=p(sp,"CODE",{});var hm=o(Hn);jo=t(hm,"0"),hm.forEach(e),go=t(sp,"\u2019s because they are shorter."),sp.forEach(e),$l=h(s),ys=p(s,"H3",{class:!0});var ap=o(ys);Ss=p(ap,"A",{id:!0,class:!0,href:!0});var mm=o(Ss);Mn=p(mm,"SPAN",{});var im=o(Mn);b(_a.$$.fragment,im),im.forEach(e),mm.forEach(e),_o=h(ap),Yn=p(ap,"SPAN",{});var fm=o(Yn);$o=t(fm,"Truncation"),fm.forEach(e),ap.forEach(e),vl=h(s),Ye=p(s,"P",{});var dm=o(Ye);vo=t(dm,"On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you\u2019ll need to truncate the sequence to a shorter length."),dm.forEach(e),wl=h(s),os=p(s,"P",{});var vn=o(os);wo=t(vn,"Set the "),Jn=p(vn,"CODE",{});var bm=o(Jn);yo=t(bm,"truncation"),bm.forEach(e),ko=t(vn," parameter to "),Un=p(vn,"CODE",{});var jm=o(Un);Eo=t(jm,"True"),jm.forEach(e),xo=t(vn," to truncate a sequence to the maximum length accepted by the model:"),vn.forEach(e),yl=h(s),b($a.$$.fragment,s),kl=h(s),b(Ns.$$.fragment,s),El=h(s),ks=p(s,"H3",{class:!0});var ep=o(ks);Fs=p(ep,"A",{id:!0,class:!0,href:!0});var gm=o(Fs);Gn=p(gm,"SPAN",{});var _m=o(Gn);b(va.$$.fragment,_m),_m.forEach(e),gm.forEach(e),qo=h(ep),Vn=p(ep,"SPAN",{});var $m=o(Vn);Po=t($m,"Build tensors"),$m.forEach(e),ep.forEach(e),xl=h(s),Je=p(s,"P",{});var vm=o(Je);Ao=t(vm,"Finally, you want the tokenizer to return the actual tensors that get fed to the model."),vm.forEach(e),ql=h(s),Y=p(s,"P",{});var la=o(Y);zo=t(la,"Set the "),Kn=p(la,"CODE",{});var wm=o(Kn);Co=t(wm,"return_tensors"),wm.forEach(e),Do=t(la," parameter to either "),Qn=p(la,"CODE",{});var ym=o(Qn);To=t(ym,"pt"),ym.forEach(e),Io=t(la," for PyTorch, or "),Xn=p(la,"CODE",{});var km=o(Xn);Oo=t(km,"tf"),km.forEach(e),Lo=t(la," for TensorFlow:"),la.forEach(e),Pl=h(s),b(Rs.$$.fragment,s),Al=h(s),Es=p(s,"H2",{class:!0});var np=o(Es);Bs=p(np,"A",{id:!0,class:!0,href:!0});var Em=o(Bs);Zn=p(Em,"SPAN",{});var xm=o(Zn);b(wa.$$.fragment,xm),xm.forEach(e),Em.forEach(e),So=h(np),st=p(np,"SPAN",{});var qm=o(st);No=t(qm,"Audio"),qm.forEach(e),np.forEach(e),zl=h(s),Ws=p(s,"P",{});var tp=o(Ws);Fo=t(tp,"For audio tasks, you\u2019ll need a "),Ue=p(tp,"A",{href:!0});var Pm=o(Ue);Ro=t(Pm,"feature extractor"),Pm.forEach(e),Bo=t(tp," to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors."),tp.forEach(e),Cl=h(s),cs=p(s,"P",{});var wn=o(cs);Wo=t(wn,"Load the "),ya=p(wn,"A",{href:!0,rel:!0});var Am=o(ya);Ho=t(Am,"MInDS-14"),Am.forEach(e),Mo=t(wn," dataset (see the \u{1F917} "),ka=p(wn,"A",{href:!0,rel:!0});var zm=o(ka);Yo=t(zm,"Datasets tutorial"),zm.forEach(e),Jo=t(wn," for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:"),wn.forEach(e),Dl=h(s),b(Ea.$$.fragment,s),Tl=h(s),us=p(s,"P",{});var yn=o(us);Uo=t(yn,"Access the first element of the "),at=p(yn,"CODE",{});var Cm=o(at);Go=t(Cm,"audio"),Cm.forEach(e),Vo=t(yn," column to take a look at the input. Calling the "),et=p(yn,"CODE",{});var Dm=o(et);Ko=t(Dm,"audio"),Dm.forEach(e),Qo=t(yn," column automatically loads and resamples the audio file:"),yn.forEach(e),Il=h(s),b(xa.$$.fragment,s),Ol=h(s),Ge=p(s,"P",{});var Tm=o(Ge);Xo=t(Tm,"This returns three items:"),Tm.forEach(e),Ll=h(s),hs=p(s,"UL",{});var kn=o(hs);Ve=p(kn,"LI",{});var Ih=o(Ve);nt=p(Ih,"CODE",{});var Im=o(nt);Zo=t(Im,"array"),Im.forEach(e),sc=t(Ih," is the speech signal loaded - and potentially resampled - as a 1D array."),Ih.forEach(e),ac=h(kn),Ke=p(kn,"LI",{});var Oh=o(Ke);tt=p(Oh,"CODE",{});var Om=o(tt);ec=t(Om,"path"),Om.forEach(e),nc=t(Oh," points to the location of the audio file."),Oh.forEach(e),tc=h(kn),Qe=p(kn,"LI",{});var Lh=o(Qe);lt=p(Lh,"CODE",{});var Lm=o(lt);lc=t(Lm,"sampling_rate"),Lm.forEach(e),rc=t(Lh," refers to how many data points in the speech signal are measured per second."),Lh.forEach(e),kn.forEach(e),Sl=h(s),Hs=p(s,"P",{});var lp=o(Hs);pc=t(lp,"For this tutorial, you\u2019ll use the "),qa=p(lp,"A",{href:!0,rel:!0});var Sm=o(qa);oc=t(Sm,"Wav2Vec2"),Sm.forEach(e),cc=t(lp," model. Take a look at the model card, and you\u2019ll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your data."),lp.forEach(e),Nl=h(s),Xe=p(s,"OL",{});var Nm=o(Xe);Pa=p(Nm,"LI",{});var rp=o(Pa);uc=t(rp,"Use \u{1F917} Datasets\u2019 "),rt=p(rp,"CODE",{});var Fm=o(rt);hc=t(Fm,"cast_column"),Fm.forEach(e),mc=t(rp," method to upsample the sampling rate to 16kHz:"),rp.forEach(e),Nm.forEach(e),Fl=h(s),b(Aa.$$.fragment,s),Rl=h(s),za=p(s,"OL",{start:!0});var Rm=o(za);Ca=p(Rm,"LI",{});var pp=o(Ca);ic=t(pp,"Call the "),pt=p(pp,"CODE",{});var Bm=o(pt);fc=t(Bm,"audio"),Bm.forEach(e),dc=t(pp," column again to resample the audio file:"),pp.forEach(e),Rm.forEach(e),Bl=h(s),b(Da.$$.fragment,s),Wl=h(s),J=p(s,"P",{});var ra=o(J);bc=t(ra,"Next, load a feature extractor to normalize and pad the input. When padding textual data, a "),ot=p(ra,"CODE",{});var Wm=o(ot);jc=t(Wm,"0"),Wm.forEach(e),gc=t(ra," is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a "),ct=p(ra,"CODE",{});var Hm=o(ct);_c=t(Hm,"0"),Hm.forEach(e),$c=t(ra," - interpreted as silence - to "),ut=p(ra,"CODE",{});var Mm=o(ut);vc=t(Mm,"array"),Mm.forEach(e),wc=t(ra,"."),ra.forEach(e),Hl=h(s),Ms=p(s,"P",{});var op=o(Ms);yc=t(op,"Load the feature extractor with "),Ze=p(op,"A",{href:!0});var Ym=o(Ze);kc=t(Ym,"AutoFeatureExtractor.from_pretrained()"),Ym.forEach(e),Ec=t(op,":"),op.forEach(e),Ml=h(s),b(Ta.$$.fragment,s),Yl=h(s),ms=p(s,"P",{});var En=o(ms);xc=t(En,"Pass the audio "),ht=p(En,"CODE",{});var Jm=o(ht);qc=t(Jm,"array"),Jm.forEach(e),Pc=t(En," to the feature extractor. We also recommend adding the "),mt=p(En,"CODE",{});var Um=o(mt);Ac=t(Um,"sampling_rate"),Um.forEach(e),zc=t(En," argument in the feature extractor in order to better debug any silent errors that may occur."),En.forEach(e),Jl=h(s),b(Ia.$$.fragment,s),Ul=h(s),sn=p(s,"P",{});var Gm=o(sn);Cc=t(Gm,"Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),Gm.forEach(e),Gl=h(s),b(Oa.$$.fragment,s),Vl=h(s),an=p(s,"P",{});var Vm=o(an);Dc=t(Vm,"Create a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),Vm.forEach(e),Kl=h(s),b(La.$$.fragment,s),Ql=h(s),Ys=p(s,"P",{});var cp=o(Ys);Tc=t(cp,"Apply the "),it=p(cp,"CODE",{});var Km=o(it);Ic=t(Km,"preprocess_function"),Km.forEach(e),Oc=t(cp," to the the first few examples in the dataset:"),cp.forEach(e),Xl=h(s),b(Sa.$$.fragment,s),Zl=h(s),en=p(s,"P",{});var Qm=o(en);Lc=t(Qm,"The sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!"),Qm.forEach(e),sr=h(s),b(Na.$$.fragment,s),ar=h(s),xs=p(s,"H2",{class:!0});var up=o(xs);Js=p(up,"A",{id:!0,class:!0,href:!0});var Xm=o(Js);ft=p(Xm,"SPAN",{});var Zm=o(ft);b(Fa.$$.fragment,Zm),Zm.forEach(e),Xm.forEach(e),Sc=h(up),dt=p(up,"SPAN",{});var si=o(dt);Nc=t(si,"Computer vision"),si.forEach(e),up.forEach(e),er=h(s),Us=p(s,"P",{});var hp=o(Us);Fc=t(hp,"For computer vision tasks, you\u2019ll need an "),nn=p(hp,"A",{href:!0});var ai=o(nn);Rc=t(ai,"image processor"),ai.forEach(e),Bc=t(hp,` to prepare your dataset for the model.
Image preprocessing consists of several steps that convert images into the input expected by the model. These steps
include but are not limited to resizing, normalizing, color channel correction, and converting images to tensors.`),hp.forEach(e),nr=h(s),b(Gs.$$.fragment,s),tr=h(s),is=p(s,"P",{});var xn=o(is);Wc=t(xn,"Load the "),Ra=p(xn,"A",{href:!0,rel:!0});var ei=o(Ra);Hc=t(ei,"food101"),ei.forEach(e),Mc=t(xn," dataset (see the \u{1F917} "),Ba=p(xn,"A",{href:!0,rel:!0});var ni=o(Ba);Yc=t(ni,"Datasets tutorial"),ni.forEach(e),Jc=t(xn," for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:"),xn.forEach(e),lr=h(s),b(Vs.$$.fragment,s),rr=h(s),b(Wa.$$.fragment,s),pr=h(s),Ks=p(s,"P",{});var mp=o(Ks);Uc=t(mp,"Next, take a look at the image with \u{1F917} Datasets "),Ha=p(mp,"A",{href:!0,rel:!0});var ti=o(Ha);bt=p(ti,"CODE",{});var li=o(bt);Gc=t(li,"Image"),li.forEach(e),ti.forEach(e),Vc=t(mp," feature:"),mp.forEach(e),or=h(s),b(Ma.$$.fragment,s),cr=h(s),Ya=p(s,"DIV",{class:!0});var ri=o(Ya);jt=p(ri,"IMG",{src:!0}),ri.forEach(e),ur=h(s),Qs=p(s,"P",{});var ip=o(Qs);Kc=t(ip,"Load the image processor with "),tn=p(ip,"A",{href:!0});var pi=o(tn);Qc=t(pi,"AutoImageProcessor.from_pretrained()"),pi.forEach(e),Xc=t(ip,":"),ip.forEach(e),hr=h(s),b(Ja.$$.fragment,s),mr=h(s),U=p(s,"P",{});var pa=o(U);Zc=t(pa,"First, let\u2019s add some image augmentation. You can use any library you prefer, but in this tutorial, we\u2019ll use torchvision\u2019s "),Ua=p(pa,"A",{href:!0,rel:!0});var oi=o(Ua);gt=p(oi,"CODE",{});var ci=o(gt);su=t(ci,"transforms"),ci.forEach(e),oi.forEach(e),au=t(pa," module. If you\u2019re interested in using another data augmentation library, learn how in the "),Ga=p(pa,"A",{href:!0,rel:!0});var ui=o(Ga);eu=t(ui,"Albumentations"),ui.forEach(e),nu=t(pa," or "),Va=p(pa,"A",{href:!0,rel:!0});var hi=o(Va);tu=t(hi,"Kornia notebooks"),hi.forEach(e),lu=t(pa,"."),pa.forEach(e),ir=h(s),ln=p(s,"OL",{});var mi=o(ln);S=p(mi,"LI",{});var G=o(S);ru=t(G,"Here we use "),Ka=p(G,"A",{href:!0,rel:!0});var ii=o(Ka);_t=p(ii,"CODE",{});var fi=o(_t);pu=t(fi,"Compose"),fi.forEach(e),ii.forEach(e),ou=t(G,` to chain together a couple of
transforms - `),Qa=p(G,"A",{href:!0,rel:!0});var di=o(Qa);$t=p(di,"CODE",{});var bi=o($t);cu=t(bi,"RandomResizedCrop"),bi.forEach(e),di.forEach(e),uu=t(G," and "),Xa=p(G,"A",{href:!0,rel:!0});var ji=o(Xa);vt=p(ji,"CODE",{});var gi=o(vt);hu=t(gi,"ColorJitter"),gi.forEach(e),ji.forEach(e),mu=t(G,`.
Note that for resizing, we can get the image size requirements from the `),wt=p(G,"CODE",{});var _i=o(wt);iu=t(_i,"image_processor"),_i.forEach(e),fu=t(G,`. For some models, an exact height and
width are expected, for others only the `),yt=p(G,"CODE",{});var $i=o(yt);du=t($i,"shortest_edge"),$i.forEach(e),bu=t(G," is defined."),G.forEach(e),mi.forEach(e),fr=h(s),b(Za.$$.fragment,s),dr=h(s),se=p(s,"OL",{start:!0});var vi=o(se);Q=p(vi,"LI",{});var oa=o(Q);ju=t(oa,"The model accepts "),rn=p(oa,"A",{href:!0});var wi=o(rn);kt=p(wi,"CODE",{});var yi=o(kt);gu=t(yi,"pixel_values"),yi.forEach(e),wi.forEach(e),_u=t(oa,`
as its input. `),Et=p(oa,"CODE",{});var ki=o(Et);$u=t(ki,"ImageProcessor"),ki.forEach(e),vu=t(oa,` can take care of normalizing the images, and generating appropriate tensors.
Create a function that combines image augmentation and image preprocessing for a batch of images and generates `),xt=p(oa,"CODE",{});var Ei=o(xt);wu=t(Ei,"pixel_values"),Ei.forEach(e),yu=t(oa,":"),oa.forEach(e),vi.forEach(e),br=h(s),b(ae.$$.fragment,s),jr=h(s),b(Xs.$$.fragment,s),gr=h(s),ee=p(s,"OL",{start:!0});var xi=o(ee);ne=p(xi,"LI",{});var fp=o(ne);ku=t(fp,"Then use \u{1F917} Datasets "),te=p(fp,"A",{href:!0,rel:!0});var qi=o(te);qt=p(qi,"CODE",{});var Pi=o(qt);Eu=t(Pi,"set_transform"),Pi.forEach(e),qi.forEach(e),xu=t(fp," to apply the transforms on the fly:"),fp.forEach(e),xi.forEach(e),_r=h(s),b(le.$$.fragment,s),$r=h(s),re=p(s,"OL",{start:!0});var Ai=o(re);pe=p(Ai,"LI",{});var dp=o(pe);qu=t(dp,"Now when you access the image, you\u2019ll notice the image processor has added "),Pt=p(dp,"CODE",{});var zi=o(Pt);Pu=t(zi,"pixel_values"),zi.forEach(e),Au=t(dp,". You can pass your processed dataset to the model now!"),dp.forEach(e),Ai.forEach(e),vr=h(s),b(oe.$$.fragment,s),wr=h(s),pn=p(s,"P",{});var Ci=o(pn);zu=t(Ci,"Here is what the image looks like after the transforms are applied. The image has been randomly cropped and it\u2019s color properties are different."),Ci.forEach(e),yr=h(s),b(ce.$$.fragment,s),kr=h(s),ue=p(s,"DIV",{class:!0});var Di=o(ue);At=p(Di,"IMG",{src:!0}),Di.forEach(e),Er=h(s),b(Zs.$$.fragment,s),xr=h(s),qs=p(s,"H3",{class:!0});var bp=o(qs);sa=p(bp,"A",{id:!0,class:!0,href:!0});var Ti=o(sa);zt=p(Ti,"SPAN",{});var Ii=o(zt);b(he.$$.fragment,Ii),Ii.forEach(e),Ti.forEach(e),Cu=h(bp),Ct=p(bp,"SPAN",{});var Oi=o(Ct);Du=t(Oi,"Pad"),Oi.forEach(e),bp.forEach(e),qr=h(s),B=p(s,"P",{});var gs=o(B);Tu=t(gs,"In some cases, for instance, when fine-tuning "),on=p(gs,"A",{href:!0});var Li=o(on);Iu=t(Li,"DETR"),Li.forEach(e),Ou=t(gs,`, the model applies scale augmentation at training
time. This may cause images to be different sizes in a batch. You can use `),cn=p(gs,"A",{href:!0});var Si=o(cn);Lu=t(Si,"DetrImageProcessor.pad_and_create_pixel_mask()"),Si.forEach(e),Su=t(gs,`
from `),un=p(gs,"A",{href:!0});var Ni=o(un);Nu=t(Ni,"DetrImageProcessor"),Ni.forEach(e),Fu=t(gs," and define a custom "),Dt=p(gs,"CODE",{});var Fi=o(Dt);Ru=t(Fi,"collate_fn"),Fi.forEach(e),Bu=t(gs," to batch images together."),gs.forEach(e),Pr=h(s),b(me.$$.fragment,s),Ar=h(s),Ps=p(s,"H2",{class:!0});var jp=o(Ps);aa=p(jp,"A",{id:!0,class:!0,href:!0});var Ri=o(aa);Tt=p(Ri,"SPAN",{});var Bi=o(Tt);b(ie.$$.fragment,Bi),Bi.forEach(e),Ri.forEach(e),Wu=h(jp),It=p(jp,"SPAN",{});var Wi=o(It);Hu=t(Wi,"Multimodal"),Wi.forEach(e),jp.forEach(e),zr=h(s),ea=p(s,"P",{});var gp=o(ea);Mu=t(gp,"For tasks involving multimodal inputs, you\u2019ll need a "),hn=p(gp,"A",{href:!0});var Hi=o(hn);Yu=t(Hi,"processor"),Hi.forEach(e),Ju=t(gp," to prepare your dataset for the model. A processor couples together two processing objects such as as tokenizer and feature extractor."),gp.forEach(e),Cr=h(s),fs=p(s,"P",{});var qn=o(fs);Uu=t(qn,"Load the "),fe=p(qn,"A",{href:!0,rel:!0});var Mi=o(fe);Gu=t(Mi,"LJ Speech"),Mi.forEach(e),Vu=t(qn," dataset (see the \u{1F917} "),de=p(qn,"A",{href:!0,rel:!0});var Yi=o(de);Ku=t(Yi,"Datasets tutorial"),Yi.forEach(e),Qu=t(qn," for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):"),qn.forEach(e),Dr=h(s),b(be.$$.fragment,s),Tr=h(s),ds=p(s,"P",{});var Pn=o(ds);Xu=t(Pn,"For ASR, you\u2019re mainly focused on "),Ot=p(Pn,"CODE",{});var Ji=o(Ot);Zu=t(Ji,"audio"),Ji.forEach(e),sh=t(Pn," and "),Lt=p(Pn,"CODE",{});var Ui=o(Lt);ah=t(Ui,"text"),Ui.forEach(e),eh=t(Pn," so you can remove the other columns:"),Pn.forEach(e),Ir=h(s),b(je.$$.fragment,s),Or=h(s),bs=p(s,"P",{});var An=o(bs);nh=t(An,"Now take a look at the "),St=p(An,"CODE",{});var Gi=o(St);th=t(Gi,"audio"),Gi.forEach(e),lh=t(An," and "),Nt=p(An,"CODE",{});var Vi=o(Nt);rh=t(Vi,"text"),Vi.forEach(e),ph=t(An," columns:"),An.forEach(e),Lr=h(s),b(ge.$$.fragment,s),Sr=h(s),na=p(s,"P",{});var _p=o(na);oh=t(_p,"Remember you should always "),mn=p(_p,"A",{href:!0});var Ki=o(mn);ch=t(Ki,"resample"),Ki.forEach(e),uh=t(_p," your audio dataset\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model!"),_p.forEach(e),Nr=h(s),b(_e.$$.fragment,s),Fr=h(s),ta=p(s,"P",{});var $p=o(ta);hh=t($p,"Load a processor with "),fn=p($p,"A",{href:!0});var Qi=o(fn);mh=t(Qi,"AutoProcessor.from_pretrained()"),Qi.forEach(e),ih=t($p,":"),$p.forEach(e),Rr=h(s),b($e.$$.fragment,s),Br=h(s),dn=p(s,"OL",{});var Xi=o(dn);M=p(Xi,"LI",{});var _s=o(M);fh=t(_s,"Create a function to process the audio data contained in "),Ft=p(_s,"CODE",{});var Zi=o(Ft);dh=t(Zi,"array"),Zi.forEach(e),bh=t(_s," to "),Rt=p(_s,"CODE",{});var sf=o(Rt);jh=t(sf,"input_values"),sf.forEach(e),gh=t(_s,", and tokenize "),Bt=p(_s,"CODE",{});var af=o(Bt);_h=t(af,"text"),af.forEach(e),$h=t(_s," to "),Wt=p(_s,"CODE",{});var ef=o(Wt);vh=t(ef,"labels"),ef.forEach(e),wh=t(_s,". These are the inputs to the model:"),_s.forEach(e),Xi.forEach(e),Wr=h(s),b(ve.$$.fragment,s),Hr=h(s),we=p(s,"OL",{start:!0});var nf=o(we);ye=p(nf,"LI",{});var vp=o(ye);yh=t(vp,"Apply the "),Ht=p(vp,"CODE",{});var tf=o(Ht);kh=t(tf,"prepare_dataset"),tf.forEach(e),Eh=t(vp," function to a sample:"),vp.forEach(e),nf.forEach(e),Mr=h(s),b(ke.$$.fragment,s),Yr=h(s),js=p(s,"P",{});var zn=o(js);xh=t(zn,"The processor has now added "),Mt=p(zn,"CODE",{});var lf=o(Mt);qh=t(lf,"input_values"),lf.forEach(e),Ph=t(zn," and "),Yt=p(zn,"CODE",{});var rf=o(Yt);Ah=t(rf,"labels"),rf.forEach(e),zh=t(zn,", and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!"),zn.forEach(e),this.h()},h(){i(m,"name","hf:doc:metadata"),i(m,"content",JSON.stringify(Cf)),i(v,"id","preprocess"),i(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(v,"href","#preprocess"),i(f,"class","relative group"),i(H,"href","./main_classes/tokenizer"),i(D,"href","./main_classes/feature_extractor"),i($s,"href","./main_classes/image"),i(De,"href","./main_classes/processors"),i(Cs,"id","natural-language-processing"),i(Cs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(Cs,"href","#natural-language-processing"),i(vs,"class","relative group"),i(Ie,"href","main_classes/tokenizer"),i(Oe,"href","/docs/transformers/v4.27.1/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained"),i(Fe,"href","glossary#input-ids"),i(Be,"href","glossary#attention-mask"),i(He,"href","glossary#token-type-ids"),i(Is,"id","pad"),i(Is,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(Is,"href","#pad"),i(ws,"class","relative group"),i(Ss,"id","truncation"),i(Ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(Ss,"href","#truncation"),i(ys,"class","relative group"),i(Fs,"id","build-tensors"),i(Fs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(Fs,"href","#build-tensors"),i(ks,"class","relative group"),i(Bs,"id","audio"),i(Bs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(Bs,"href","#audio"),i(Es,"class","relative group"),i(Ue,"href","main_classes/feature_extractor"),i(ya,"href","https://huggingface.co/datasets/PolyAI/minds14"),i(ya,"rel","nofollow"),i(ka,"href","https://huggingface.co/docs/datasets/load_hub.html"),i(ka,"rel","nofollow"),i(qa,"href","https://huggingface.co/facebook/wav2vec2-base"),i(qa,"rel","nofollow"),i(za,"start","2"),i(Ze,"href","/docs/transformers/v4.27.1/en/model_doc/auto#transformers.AutoFeatureExtractor.from_pretrained"),i(Js,"id","computer-vision"),i(Js,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(Js,"href","#computer-vision"),i(xs,"class","relative group"),i(nn,"href","main_classes/image_processor"),i(Ra,"href","https://huggingface.co/datasets/food101"),i(Ra,"rel","nofollow"),i(Ba,"href","https://huggingface.co/docs/datasets/load_hub.html"),i(Ba,"rel","nofollow"),i(Ha,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=image#datasets.Image"),i(Ha,"rel","nofollow"),pf(jt.src,Sh="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png")||i(jt,"src",Sh),i(Ya,"class","flex justify-center"),i(tn,"href","/docs/transformers/v4.27.1/en/model_doc/auto#transformers.AutoImageProcessor.from_pretrained"),i(Ua,"href","https://pytorch.org/vision/stable/transforms.html"),i(Ua,"rel","nofollow"),i(Ga,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb"),i(Ga,"rel","nofollow"),i(Va,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb"),i(Va,"rel","nofollow"),i(Ka,"href","https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html"),i(Ka,"rel","nofollow"),i(Qa,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html"),i(Qa,"rel","nofollow"),i(Xa,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html"),i(Xa,"rel","nofollow"),i(rn,"href","model_doc/visionencoderdecoder#transformers.VisionEncoderDecoderModel.forward.pixel_values"),i(se,"start","2"),i(te,"href","https://huggingface.co/docs/datasets/process.html#format-transform"),i(te,"rel","nofollow"),i(ee,"start","3"),i(re,"start","4"),pf(At.src,Nh="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png")||i(At,"src",Nh),i(ue,"class","flex justify-center"),i(sa,"id","pad"),i(sa,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(sa,"href","#pad"),i(qs,"class","relative group"),i(on,"href","./model_doc/detr"),i(cn,"href","/docs/transformers/v4.27.1/en/model_doc/detr#transformers.DetrFeatureExtractor.pad_and_create_pixel_mask"),i(un,"href","/docs/transformers/v4.27.1/en/model_doc/detr#transformers.DetrImageProcessor"),i(aa,"id","multimodal"),i(aa,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(aa,"href","#multimodal"),i(Ps,"class","relative group"),i(hn,"href","main_classes/processors"),i(fe,"href","https://huggingface.co/datasets/lj_speech"),i(fe,"rel","nofollow"),i(de,"href","https://huggingface.co/docs/datasets/load_hub.html"),i(de,"rel","nofollow"),i(mn,"href","preprocessing#audio"),i(fn,"href","/docs/transformers/v4.27.1/en/model_doc/auto#transformers.AutoProcessor.from_pretrained"),i(we,"start","2")},m(s,l){a(document.head,m),c(s,w,l),c(s,f,l),a(f,v),a(v,k),j(y,k,null),a(f,E),a(f,P),a(P,C),c(s,I,l),j(z,s,l),c(s,N,l),c(s,O,l),a(O,V),c(s,W,l),c(s,x,l),a(x,T),a(T,F),a(T,H),a(H,Z),a(T,R),a(x,ss),a(x,K),a(K,L),a(K,D),a(D,Pe),a(K,Ae),a(x,ze),a(x,as),a(as,es),a(as,$s),a($s,Ce),a(as,wp),a(x,yp),a(x,ca),a(ca,kp),a(ca,De),a(De,Ep),a(ca,xp),c(s,Qt,l),j(zs,s,l),c(s,Xt,l),c(s,Te,l),a(Te,qp),c(s,Zt,l),j(ua,s,l),c(s,sl,l),c(s,vs,l),a(vs,Cs),a(Cs,Cn),j(ha,Cn,null),a(vs,Pp),a(vs,Dn),a(Dn,Ap),c(s,al,l),j(ma,s,l),c(s,el,l),c(s,ns,l),a(ns,zp),a(ns,Ie),a(Ie,Cp),a(ns,Dp),a(ns,Tn),a(Tn,Tp),a(ns,Ip),c(s,nl,l),j(Ds,s,l),c(s,tl,l),c(s,ts,l),a(ts,Op),a(ts,Oe),a(Oe,Lp),a(ts,Sp),a(ts,In),a(In,Np),a(ts,Fp),c(s,ll,l),j(ia,s,l),c(s,rl,l),c(s,Le,l),a(Le,Rp),c(s,pl,l),j(fa,s,l),c(s,ol,l),c(s,Se,l),a(Se,Bp),c(s,cl,l),c(s,ls,l),a(ls,Ne),a(Ne,Fe),a(Fe,Wp),a(Ne,Hp),a(ls,Mp),a(ls,Re),a(Re,Be),a(Be,Yp),a(Re,Jp),a(ls,Up),a(ls,We),a(We,He),a(He,Gp),a(We,Vp),c(s,ul,l),c(s,Ts,l),a(Ts,Kp),a(Ts,On),a(On,Qp),a(Ts,Xp),c(s,hl,l),j(da,s,l),c(s,ml,l),c(s,rs,l),a(rs,Zp),a(rs,Ln),a(Ln,so),a(rs,ao),a(rs,Sn),a(Sn,eo),a(rs,no),c(s,il,l),c(s,Me,l),a(Me,to),c(s,fl,l),j(ba,s,l),c(s,dl,l),c(s,ws,l),a(ws,Is),a(Is,Nn),j(ja,Nn,null),a(ws,lo),a(ws,Fn),a(Fn,ro),c(s,bl,l),c(s,Os,l),a(Os,po),a(Os,Rn),a(Rn,oo),a(Os,co),c(s,jl,l),c(s,ps,l),a(ps,uo),a(ps,Bn),a(Bn,ho),a(ps,mo),a(ps,Wn),a(Wn,io),a(ps,fo),c(s,gl,l),j(ga,s,l),c(s,_l,l),c(s,Ls,l),a(Ls,bo),a(Ls,Hn),a(Hn,jo),a(Ls,go),c(s,$l,l),c(s,ys,l),a(ys,Ss),a(Ss,Mn),j(_a,Mn,null),a(ys,_o),a(ys,Yn),a(Yn,$o),c(s,vl,l),c(s,Ye,l),a(Ye,vo),c(s,wl,l),c(s,os,l),a(os,wo),a(os,Jn),a(Jn,yo),a(os,ko),a(os,Un),a(Un,Eo),a(os,xo),c(s,yl,l),j($a,s,l),c(s,kl,l),j(Ns,s,l),c(s,El,l),c(s,ks,l),a(ks,Fs),a(Fs,Gn),j(va,Gn,null),a(ks,qo),a(ks,Vn),a(Vn,Po),c(s,xl,l),c(s,Je,l),a(Je,Ao),c(s,ql,l),c(s,Y,l),a(Y,zo),a(Y,Kn),a(Kn,Co),a(Y,Do),a(Y,Qn),a(Qn,To),a(Y,Io),a(Y,Xn),a(Xn,Oo),a(Y,Lo),c(s,Pl,l),j(Rs,s,l),c(s,Al,l),c(s,Es,l),a(Es,Bs),a(Bs,Zn),j(wa,Zn,null),a(Es,So),a(Es,st),a(st,No),c(s,zl,l),c(s,Ws,l),a(Ws,Fo),a(Ws,Ue),a(Ue,Ro),a(Ws,Bo),c(s,Cl,l),c(s,cs,l),a(cs,Wo),a(cs,ya),a(ya,Ho),a(cs,Mo),a(cs,ka),a(ka,Yo),a(cs,Jo),c(s,Dl,l),j(Ea,s,l),c(s,Tl,l),c(s,us,l),a(us,Uo),a(us,at),a(at,Go),a(us,Vo),a(us,et),a(et,Ko),a(us,Qo),c(s,Il,l),j(xa,s,l),c(s,Ol,l),c(s,Ge,l),a(Ge,Xo),c(s,Ll,l),c(s,hs,l),a(hs,Ve),a(Ve,nt),a(nt,Zo),a(Ve,sc),a(hs,ac),a(hs,Ke),a(Ke,tt),a(tt,ec),a(Ke,nc),a(hs,tc),a(hs,Qe),a(Qe,lt),a(lt,lc),a(Qe,rc),c(s,Sl,l),c(s,Hs,l),a(Hs,pc),a(Hs,qa),a(qa,oc),a(Hs,cc),c(s,Nl,l),c(s,Xe,l),a(Xe,Pa),a(Pa,uc),a(Pa,rt),a(rt,hc),a(Pa,mc),c(s,Fl,l),j(Aa,s,l),c(s,Rl,l),c(s,za,l),a(za,Ca),a(Ca,ic),a(Ca,pt),a(pt,fc),a(Ca,dc),c(s,Bl,l),j(Da,s,l),c(s,Wl,l),c(s,J,l),a(J,bc),a(J,ot),a(ot,jc),a(J,gc),a(J,ct),a(ct,_c),a(J,$c),a(J,ut),a(ut,vc),a(J,wc),c(s,Hl,l),c(s,Ms,l),a(Ms,yc),a(Ms,Ze),a(Ze,kc),a(Ms,Ec),c(s,Ml,l),j(Ta,s,l),c(s,Yl,l),c(s,ms,l),a(ms,xc),a(ms,ht),a(ht,qc),a(ms,Pc),a(ms,mt),a(mt,Ac),a(ms,zc),c(s,Jl,l),j(Ia,s,l),c(s,Ul,l),c(s,sn,l),a(sn,Cc),c(s,Gl,l),j(Oa,s,l),c(s,Vl,l),c(s,an,l),a(an,Dc),c(s,Kl,l),j(La,s,l),c(s,Ql,l),c(s,Ys,l),a(Ys,Tc),a(Ys,it),a(it,Ic),a(Ys,Oc),c(s,Xl,l),j(Sa,s,l),c(s,Zl,l),c(s,en,l),a(en,Lc),c(s,sr,l),j(Na,s,l),c(s,ar,l),c(s,xs,l),a(xs,Js),a(Js,ft),j(Fa,ft,null),a(xs,Sc),a(xs,dt),a(dt,Nc),c(s,er,l),c(s,Us,l),a(Us,Fc),a(Us,nn),a(nn,Rc),a(Us,Bc),c(s,nr,l),j(Gs,s,l),c(s,tr,l),c(s,is,l),a(is,Wc),a(is,Ra),a(Ra,Hc),a(is,Mc),a(is,Ba),a(Ba,Yc),a(is,Jc),c(s,lr,l),j(Vs,s,l),c(s,rr,l),j(Wa,s,l),c(s,pr,l),c(s,Ks,l),a(Ks,Uc),a(Ks,Ha),a(Ha,bt),a(bt,Gc),a(Ks,Vc),c(s,or,l),j(Ma,s,l),c(s,cr,l),c(s,Ya,l),a(Ya,jt),c(s,ur,l),c(s,Qs,l),a(Qs,Kc),a(Qs,tn),a(tn,Qc),a(Qs,Xc),c(s,hr,l),j(Ja,s,l),c(s,mr,l),c(s,U,l),a(U,Zc),a(U,Ua),a(Ua,gt),a(gt,su),a(U,au),a(U,Ga),a(Ga,eu),a(U,nu),a(U,Va),a(Va,tu),a(U,lu),c(s,ir,l),c(s,ln,l),a(ln,S),a(S,ru),a(S,Ka),a(Ka,_t),a(_t,pu),a(S,ou),a(S,Qa),a(Qa,$t),a($t,cu),a(S,uu),a(S,Xa),a(Xa,vt),a(vt,hu),a(S,mu),a(S,wt),a(wt,iu),a(S,fu),a(S,yt),a(yt,du),a(S,bu),c(s,fr,l),j(Za,s,l),c(s,dr,l),c(s,se,l),a(se,Q),a(Q,ju),a(Q,rn),a(rn,kt),a(kt,gu),a(Q,_u),a(Q,Et),a(Et,$u),a(Q,vu),a(Q,xt),a(xt,wu),a(Q,yu),c(s,br,l),j(ae,s,l),c(s,jr,l),j(Xs,s,l),c(s,gr,l),c(s,ee,l),a(ee,ne),a(ne,ku),a(ne,te),a(te,qt),a(qt,Eu),a(ne,xu),c(s,_r,l),j(le,s,l),c(s,$r,l),c(s,re,l),a(re,pe),a(pe,qu),a(pe,Pt),a(Pt,Pu),a(pe,Au),c(s,vr,l),j(oe,s,l),c(s,wr,l),c(s,pn,l),a(pn,zu),c(s,yr,l),j(ce,s,l),c(s,kr,l),c(s,ue,l),a(ue,At),c(s,Er,l),j(Zs,s,l),c(s,xr,l),c(s,qs,l),a(qs,sa),a(sa,zt),j(he,zt,null),a(qs,Cu),a(qs,Ct),a(Ct,Du),c(s,qr,l),c(s,B,l),a(B,Tu),a(B,on),a(on,Iu),a(B,Ou),a(B,cn),a(cn,Lu),a(B,Su),a(B,un),a(un,Nu),a(B,Fu),a(B,Dt),a(Dt,Ru),a(B,Bu),c(s,Pr,l),j(me,s,l),c(s,Ar,l),c(s,Ps,l),a(Ps,aa),a(aa,Tt),j(ie,Tt,null),a(Ps,Wu),a(Ps,It),a(It,Hu),c(s,zr,l),c(s,ea,l),a(ea,Mu),a(ea,hn),a(hn,Yu),a(ea,Ju),c(s,Cr,l),c(s,fs,l),a(fs,Uu),a(fs,fe),a(fe,Gu),a(fs,Vu),a(fs,de),a(de,Ku),a(fs,Qu),c(s,Dr,l),j(be,s,l),c(s,Tr,l),c(s,ds,l),a(ds,Xu),a(ds,Ot),a(Ot,Zu),a(ds,sh),a(ds,Lt),a(Lt,ah),a(ds,eh),c(s,Ir,l),j(je,s,l),c(s,Or,l),c(s,bs,l),a(bs,nh),a(bs,St),a(St,th),a(bs,lh),a(bs,Nt),a(Nt,rh),a(bs,ph),c(s,Lr,l),j(ge,s,l),c(s,Sr,l),c(s,na,l),a(na,oh),a(na,mn),a(mn,ch),a(na,uh),c(s,Nr,l),j(_e,s,l),c(s,Fr,l),c(s,ta,l),a(ta,hh),a(ta,fn),a(fn,mh),a(ta,ih),c(s,Rr,l),j($e,s,l),c(s,Br,l),c(s,dn,l),a(dn,M),a(M,fh),a(M,Ft),a(Ft,dh),a(M,bh),a(M,Rt),a(Rt,jh),a(M,gh),a(M,Bt),a(Bt,_h),a(M,$h),a(M,Wt),a(Wt,vh),a(M,wh),c(s,Wr,l),j(ve,s,l),c(s,Hr,l),c(s,we,l),a(we,ye),a(ye,yh),a(ye,Ht),a(Ht,kh),a(ye,Eh),c(s,Mr,l),j(ke,s,l),c(s,Yr,l),c(s,js,l),a(js,xh),a(js,Mt),a(Mt,qh),a(js,Ph),a(js,Yt),a(Yt,Ah),a(js,zh),Jr=!0},p(s,[l]){const Ee={};l&2&&(Ee.$$scope={dirty:l,ctx:s}),zs.$set(Ee);const Jt={};l&2&&(Jt.$$scope={dirty:l,ctx:s}),Ds.$set(Jt);const Ut={};l&2&&(Ut.$$scope={dirty:l,ctx:s}),Ns.$set(Ut);const Gt={};l&2&&(Gt.$$scope={dirty:l,ctx:s}),Rs.$set(Gt);const Vt={};l&2&&(Vt.$$scope={dirty:l,ctx:s}),Gs.$set(Vt);const X={};l&2&&(X.$$scope={dirty:l,ctx:s}),Vs.$set(X);const xe={};l&2&&(xe.$$scope={dirty:l,ctx:s}),Xs.$set(xe);const Kt={};l&2&&(Kt.$$scope={dirty:l,ctx:s}),Zs.$set(Kt)},i(s){Jr||(g(y.$$.fragment,s),g(z.$$.fragment,s),g(zs.$$.fragment,s),g(ua.$$.fragment,s),g(ha.$$.fragment,s),g(ma.$$.fragment,s),g(Ds.$$.fragment,s),g(ia.$$.fragment,s),g(fa.$$.fragment,s),g(da.$$.fragment,s),g(ba.$$.fragment,s),g(ja.$$.fragment,s),g(ga.$$.fragment,s),g(_a.$$.fragment,s),g($a.$$.fragment,s),g(Ns.$$.fragment,s),g(va.$$.fragment,s),g(Rs.$$.fragment,s),g(wa.$$.fragment,s),g(Ea.$$.fragment,s),g(xa.$$.fragment,s),g(Aa.$$.fragment,s),g(Da.$$.fragment,s),g(Ta.$$.fragment,s),g(Ia.$$.fragment,s),g(Oa.$$.fragment,s),g(La.$$.fragment,s),g(Sa.$$.fragment,s),g(Na.$$.fragment,s),g(Fa.$$.fragment,s),g(Gs.$$.fragment,s),g(Vs.$$.fragment,s),g(Wa.$$.fragment,s),g(Ma.$$.fragment,s),g(Ja.$$.fragment,s),g(Za.$$.fragment,s),g(ae.$$.fragment,s),g(Xs.$$.fragment,s),g(le.$$.fragment,s),g(oe.$$.fragment,s),g(ce.$$.fragment,s),g(Zs.$$.fragment,s),g(he.$$.fragment,s),g(me.$$.fragment,s),g(ie.$$.fragment,s),g(be.$$.fragment,s),g(je.$$.fragment,s),g(ge.$$.fragment,s),g(_e.$$.fragment,s),g($e.$$.fragment,s),g(ve.$$.fragment,s),g(ke.$$.fragment,s),Jr=!0)},o(s){_(y.$$.fragment,s),_(z.$$.fragment,s),_(zs.$$.fragment,s),_(ua.$$.fragment,s),_(ha.$$.fragment,s),_(ma.$$.fragment,s),_(Ds.$$.fragment,s),_(ia.$$.fragment,s),_(fa.$$.fragment,s),_(da.$$.fragment,s),_(ba.$$.fragment,s),_(ja.$$.fragment,s),_(ga.$$.fragment,s),_(_a.$$.fragment,s),_($a.$$.fragment,s),_(Ns.$$.fragment,s),_(va.$$.fragment,s),_(Rs.$$.fragment,s),_(wa.$$.fragment,s),_(Ea.$$.fragment,s),_(xa.$$.fragment,s),_(Aa.$$.fragment,s),_(Da.$$.fragment,s),_(Ta.$$.fragment,s),_(Ia.$$.fragment,s),_(Oa.$$.fragment,s),_(La.$$.fragment,s),_(Sa.$$.fragment,s),_(Na.$$.fragment,s),_(Fa.$$.fragment,s),_(Gs.$$.fragment,s),_(Vs.$$.fragment,s),_(Wa.$$.fragment,s),_(Ma.$$.fragment,s),_(Ja.$$.fragment,s),_(Za.$$.fragment,s),_(ae.$$.fragment,s),_(Xs.$$.fragment,s),_(le.$$.fragment,s),_(oe.$$.fragment,s),_(ce.$$.fragment,s),_(Zs.$$.fragment,s),_(he.$$.fragment,s),_(me.$$.fragment,s),_(ie.$$.fragment,s),_(be.$$.fragment,s),_(je.$$.fragment,s),_(ge.$$.fragment,s),_(_e.$$.fragment,s),_($e.$$.fragment,s),_(ve.$$.fragment,s),_(ke.$$.fragment,s),Jr=!1},d(s){e(m),s&&e(w),s&&e(f),$(y),s&&e(I),$(z,s),s&&e(N),s&&e(O),s&&e(W),s&&e(x),s&&e(Qt),$(zs,s),s&&e(Xt),s&&e(Te),s&&e(Zt),$(ua,s),s&&e(sl),s&&e(vs),$(ha),s&&e(al),$(ma,s),s&&e(el),s&&e(ns),s&&e(nl),$(Ds,s),s&&e(tl),s&&e(ts),s&&e(ll),$(ia,s),s&&e(rl),s&&e(Le),s&&e(pl),$(fa,s),s&&e(ol),s&&e(Se),s&&e(cl),s&&e(ls),s&&e(ul),s&&e(Ts),s&&e(hl),$(da,s),s&&e(ml),s&&e(rs),s&&e(il),s&&e(Me),s&&e(fl),$(ba,s),s&&e(dl),s&&e(ws),$(ja),s&&e(bl),s&&e(Os),s&&e(jl),s&&e(ps),s&&e(gl),$(ga,s),s&&e(_l),s&&e(Ls),s&&e($l),s&&e(ys),$(_a),s&&e(vl),s&&e(Ye),s&&e(wl),s&&e(os),s&&e(yl),$($a,s),s&&e(kl),$(Ns,s),s&&e(El),s&&e(ks),$(va),s&&e(xl),s&&e(Je),s&&e(ql),s&&e(Y),s&&e(Pl),$(Rs,s),s&&e(Al),s&&e(Es),$(wa),s&&e(zl),s&&e(Ws),s&&e(Cl),s&&e(cs),s&&e(Dl),$(Ea,s),s&&e(Tl),s&&e(us),s&&e(Il),$(xa,s),s&&e(Ol),s&&e(Ge),s&&e(Ll),s&&e(hs),s&&e(Sl),s&&e(Hs),s&&e(Nl),s&&e(Xe),s&&e(Fl),$(Aa,s),s&&e(Rl),s&&e(za),s&&e(Bl),$(Da,s),s&&e(Wl),s&&e(J),s&&e(Hl),s&&e(Ms),s&&e(Ml),$(Ta,s),s&&e(Yl),s&&e(ms),s&&e(Jl),$(Ia,s),s&&e(Ul),s&&e(sn),s&&e(Gl),$(Oa,s),s&&e(Vl),s&&e(an),s&&e(Kl),$(La,s),s&&e(Ql),s&&e(Ys),s&&e(Xl),$(Sa,s),s&&e(Zl),s&&e(en),s&&e(sr),$(Na,s),s&&e(ar),s&&e(xs),$(Fa),s&&e(er),s&&e(Us),s&&e(nr),$(Gs,s),s&&e(tr),s&&e(is),s&&e(lr),$(Vs,s),s&&e(rr),$(Wa,s),s&&e(pr),s&&e(Ks),s&&e(or),$(Ma,s),s&&e(cr),s&&e(Ya),s&&e(ur),s&&e(Qs),s&&e(hr),$(Ja,s),s&&e(mr),s&&e(U),s&&e(ir),s&&e(ln),s&&e(fr),$(Za,s),s&&e(dr),s&&e(se),s&&e(br),$(ae,s),s&&e(jr),$(Xs,s),s&&e(gr),s&&e(ee),s&&e(_r),$(le,s),s&&e($r),s&&e(re),s&&e(vr),$(oe,s),s&&e(wr),s&&e(pn),s&&e(yr),$(ce,s),s&&e(kr),s&&e(ue),s&&e(Er),$(Zs,s),s&&e(xr),s&&e(qs),$(he),s&&e(qr),s&&e(B),s&&e(Pr),$(me,s),s&&e(Ar),s&&e(Ps),$(ie),s&&e(zr),s&&e(ea),s&&e(Cr),s&&e(fs),s&&e(Dr),$(be,s),s&&e(Tr),s&&e(ds),s&&e(Ir),$(je,s),s&&e(Or),s&&e(bs),s&&e(Lr),$(ge,s),s&&e(Sr),s&&e(na),s&&e(Nr),$(_e,s),s&&e(Fr),s&&e(ta),s&&e(Rr),$($e,s),s&&e(Br),s&&e(dn),s&&e(Wr),$(ve,s),s&&e(Hr),s&&e(we),s&&e(Mr),$(ke,s),s&&e(Yr),s&&e(js)}}}const Cf={local:"preprocess",sections:[{local:"natural-language-processing",sections:[{local:"pad",title:"Pad"},{local:"truncation",title:"Truncation"},{local:"build-tensors",title:"Build tensors"}],title:"Natural Language Processing"},{local:"audio",title:"Audio"},{local:"computer-vision",sections:[{local:"pad",title:"Pad"}],title:"Computer vision"},{local:"multimodal",title:"Multimodal"}],title:"Preprocess"};function Df(A){return df(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Rf extends uf{constructor(m){super();hf(this,m,Df,zf,mf,{})}}export{Rf as default,Cf as metadata};
