import{S as ki,i as $i,s as Fi,e as r,k as c,w as T,t as n,M as Ei,c as a,d as t,m,a as s,x as b,h as i,b as d,N as zi,G as e,g as f,y,q as x,o as w,B as M,v as Ci,L as Mr}from"../../chunks/vendor-hf-doc-builder.js";import{T as mn}from"../../chunks/Tip-hf-doc-builder.js";import{D as A}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Pr}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as ne}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as wr}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function ji(k){let p,P,_,u,v;return u=new Pr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFR2bHRDb25maWclMkMlMjBUdmx0TW9kZWwlMEElMEElMjMlMjAlMjMlMjBJbml0aWFsaXppbmclMjBhJTIwVFZMVCUyMFppbmVuZ1RhbmclMkZ0dmx0LWJhc2UlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwVHZsdENvbmZpZygpJTBBJTBBJTIzJTIwJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMG1vZGVsJTIwKHdpdGglMjByYW5kb20lMjB3ZWlnaHRzKSUyMGZyb20lMjB0aGUlMjBaaW5lbmdUYW5nJTJGdHZsdC1iYXNlJTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBbW9kZWwlMjAlM0QlMjBUdmx0TW9kZWwoY29uZmlndXJhdGlvbiklMEElMEElMjMlMjBBY2Nlc3NpbmclMjB0aGUlMjBtb2RlbCUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwbW9kZWwuY29uZmln",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TvltConfig, TvltModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># # Initializing a TVLT ZinengTang/tvlt-base style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = TvltConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># # Initializing a model (with random weights) from the ZinengTang/tvlt-base style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = TvltModel(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`}}),{c(){p=r("p"),P=n("Example:"),_=c(),T(u.$$.fragment)},l(l){p=a(l,"P",{});var g=s(p);P=i(g,"Example:"),g.forEach(t),_=m(l),b(u.$$.fragment,l)},m(l,g){f(l,p,g),e(p,P),f(l,_,g),y(u,l,g),v=!0},p:Mr,i(l){v||(x(u.$$.fragment,l),v=!0)},o(l){w(u.$$.fragment,l),v=!1},d(l){l&&t(p),l&&t(_),M(u,l)}}}function Ii(k){let p,P,_,u,v;return{c(){p=r("p"),P=n("Although the recipe for forward pass needs to be defined within this function, one should call the "),_=r("code"),u=n("Module"),v=n(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(l){p=a(l,"P",{});var g=s(p);P=i(g,"Although the recipe for forward pass needs to be defined within this function, one should call the "),_=a(g,"CODE",{});var $=s(_);u=i($,"Module"),$.forEach(t),v=i(g,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),g.forEach(t)},m(l,g){f(l,p,g),e(p,P),e(p,_),e(_,u),e(p,v)},d(l){l&&t(p)}}}function Ai(k){let p,P,_,u,v;return u=new Pr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFR2bHRQcm9jZXNzb3IlMkMlMjBUdmx0TW9kZWwlMEFpbXBvcnQlMjBudW1weSUyMGFzJTIwbnAlMEFpbXBvcnQlMjB0b3JjaCUwQSUwQW51bV9mcmFtZXMlMjAlM0QlMjA4JTBBaW1hZ2VzJTIwJTNEJTIwbGlzdChucC5yYW5kb20ucmFuZG4obnVtX2ZyYW1lcyUyQyUyMDMlMkMlMjAyMjQlMkMlMjAyMjQpKSUwQWF1ZGlvJTIwJTNEJTIwbGlzdChucC5yYW5kb20ucmFuZG4oMTAwMDApKSUwQSUwQXByb2Nlc3NvciUyMCUzRCUyMFR2bHRQcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKCUyMlppbmVuZ1RhbmclMkZ0dmx0LWJhc2UlMjIpJTBBbW9kZWwlMjAlM0QlMjBUdmx0TW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMlppbmVuZ1RhbmclMkZ0dmx0LWJhc2UlMjIpJTBBJTBBaW5wdXRfZGljdCUyMCUzRCUyMHByb2Nlc3NvcihpbWFnZXMlMkMlMjBhdWRpbyUyQyUyMHNhbXBsaW5nX3JhdGUlM0Q0NDEwMCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRfZGljdCklMEFsb3NzJTIwJTNEJTIwb3V0cHV0cy5sb3Nz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TvltProcessor, TvltModel
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>num_frames = <span class="hljs-number">8</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>images = <span class="hljs-built_in">list</span>(np.random.randn(num_frames, <span class="hljs-number">3</span>, <span class="hljs-number">224</span>, <span class="hljs-number">224</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>audio = <span class="hljs-built_in">list</span>(np.random.randn(<span class="hljs-number">10000</span>))

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = TvltProcessor.from_pretrained(<span class="hljs-string">&quot;ZinengTang/tvlt-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = TvltModel.from_pretrained(<span class="hljs-string">&quot;ZinengTang/tvlt-base&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_dict = processor(images, audio, sampling_rate=<span class="hljs-number">44100</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(**input_dict)
<span class="hljs-meta">&gt;&gt;&gt; </span>loss = outputs.loss`}}),{c(){p=r("p"),P=n("Examples:"),_=c(),T(u.$$.fragment)},l(l){p=a(l,"P",{});var g=s(p);P=i(g,"Examples:"),g.forEach(t),_=m(l),b(u.$$.fragment,l)},m(l,g){f(l,p,g),e(p,P),f(l,_,g),y(u,l,g),v=!0},p:Mr,i(l){v||(x(u.$$.fragment,l),v=!0)},o(l){w(u.$$.fragment,l),v=!1},d(l){l&&t(p),l&&t(_),M(u,l)}}}function Vi(k){let p,P,_,u,v;return{c(){p=r("p"),P=n("Although the recipe for forward pass needs to be defined within this function, one should call the "),_=r("code"),u=n("Module"),v=n(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(l){p=a(l,"P",{});var g=s(p);P=i(g,"Although the recipe for forward pass needs to be defined within this function, one should call the "),_=a(g,"CODE",{});var $=s(_);u=i($,"Module"),$.forEach(t),v=i(g,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),g.forEach(t)},m(l,g){f(l,p,g),e(p,P),e(p,_),e(_,u),e(p,v)},d(l){l&&t(p)}}}function Ni(k){let p,P,_,u,v;return u=new Pr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFR2bHRQcm9jZXNzb3IlMkMlMjBUdmx0Rm9yUHJlVHJhaW5pbmclMEFpbXBvcnQlMjBudW1weSUyMGFzJTIwbnAlMEFpbXBvcnQlMjB0b3JjaCUwQSUwQW51bV9mcmFtZXMlMjAlM0QlMjA4JTBBaW1hZ2VzJTIwJTNEJTIwbGlzdChucC5yYW5kb20ucmFuZG4obnVtX2ZyYW1lcyUyQyUyMDMlMkMlMjAyMjQlMkMlMjAyMjQpKSUwQWltYWdlc19taXhlZCUyMCUzRCUyMGxpc3QobnAucmFuZG9tLnJhbmRuKG51bV9mcmFtZXMlMkMlMjAzJTJDJTIwMjI0JTJDJTIwMjI0KSklMEFhdWRpbyUyMCUzRCUyMGxpc3QobnAucmFuZG9tLnJhbmRuKDEwMDAwKSklMEFwcm9jZXNzb3IlMjAlM0QlMjBUdmx0UHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJaaW5lbmdUYW5nJTJGdHZsdC1iYXNlJTIyKSUwQW1vZGVsJTIwJTNEJTIwVHZsdEZvclByZVRyYWluaW5nLmZyb21fcHJldHJhaW5lZCglMjJaaW5lbmdUYW5nJTJGdHZsdC1iYXNlJTIyKSUwQWlucHV0X2RpY3QlMjAlM0QlMjBwcm9jZXNzb3IoJTBBJTIwJTIwJTIwJTIwaW1hZ2VzJTJDJTIwYXVkaW8lMkMlMjBpbWFnZXNfbWl4ZWQlMkMlMjBzYW1wbGluZ19yYXRlJTNENDQxMDAlMkMlMjBtYXNrX3BpeGVsJTNEVHJ1ZSUyQyUyMG1hc2tfYXVkaW8lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiUwQSklMEElMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dF9kaWN0KSUwQWxvc3MlMjAlM0QlMjBvdXRwdXRzLmxvc3M=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TvltProcessor, TvltForPreTraining
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>num_frames = <span class="hljs-number">8</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>images = <span class="hljs-built_in">list</span>(np.random.randn(num_frames, <span class="hljs-number">3</span>, <span class="hljs-number">224</span>, <span class="hljs-number">224</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>images_mixed = <span class="hljs-built_in">list</span>(np.random.randn(num_frames, <span class="hljs-number">3</span>, <span class="hljs-number">224</span>, <span class="hljs-number">224</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>audio = <span class="hljs-built_in">list</span>(np.random.randn(<span class="hljs-number">10000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>processor = TvltProcessor.from_pretrained(<span class="hljs-string">&quot;ZinengTang/tvlt-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = TvltForPreTraining.from_pretrained(<span class="hljs-string">&quot;ZinengTang/tvlt-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_dict = processor(
<span class="hljs-meta">... </span>    images, audio, images_mixed, sampling_rate=<span class="hljs-number">44100</span>, mask_pixel=<span class="hljs-literal">True</span>, mask_audio=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(**input_dict)
<span class="hljs-meta">&gt;&gt;&gt; </span>loss = outputs.loss`}}),{c(){p=r("p"),P=n("Examples:"),_=c(),T(u.$$.fragment)},l(l){p=a(l,"P",{});var g=s(p);P=i(g,"Examples:"),g.forEach(t),_=m(l),b(u.$$.fragment,l)},m(l,g){f(l,p,g),e(p,P),f(l,_,g),y(u,l,g),v=!0},p:Mr,i(l){v||(x(u.$$.fragment,l),v=!0)},o(l){w(u.$$.fragment,l),v=!1},d(l){l&&t(p),l&&t(_),M(u,l)}}}function Wi(k){let p,P,_,u,v;return{c(){p=r("p"),P=n("Although the recipe for forward pass needs to be defined within this function, one should call the "),_=r("code"),u=n("Module"),v=n(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(l){p=a(l,"P",{});var g=s(p);P=i(g,"Although the recipe for forward pass needs to be defined within this function, one should call the "),_=a(g,"CODE",{});var $=s(_);u=i($,"Module"),$.forEach(t),v=i(g,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),g.forEach(t)},m(l,g){f(l,p,g),e(p,P),e(p,_),e(_,u),e(p,v)},d(l){l&&t(p)}}}function Ji(k){let p,P,_,u,v;return u=new Pr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFR2bHRQcm9jZXNzb3IlMkMlMjBUdmx0Rm9yQXVkaW9WaXN1YWxDbGFzc2lmaWNhdGlvbiUwQWltcG9ydCUyMG51bXB5JTIwYXMlMjBucCUwQWltcG9ydCUyMHRvcmNoJTBBJTBBbnVtX2ZyYW1lcyUyMCUzRCUyMDglMEFpbWFnZXMlMjAlM0QlMjBsaXN0KG5wLnJhbmRvbS5yYW5kbihudW1fZnJhbWVzJTJDJTIwMyUyQyUyMDIyNCUyQyUyMDIyNCkpJTBBYXVkaW8lMjAlM0QlMjBsaXN0KG5wLnJhbmRvbS5yYW5kbigxMDAwMCkpJTBBcHJvY2Vzc29yJTIwJTNEJTIwVHZsdFByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIyWmluZW5nVGFuZyUyRnR2bHQtYmFzZSUyMiklMEFtb2RlbCUyMCUzRCUyMFR2bHRGb3JBdWRpb1Zpc3VhbENsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMjJaaW5lbmdUYW5nJTJGdHZsdC1iYXNlJTIyKSUwQWlucHV0X2RpY3QlMjAlM0QlMjBwcm9jZXNzb3IoaW1hZ2VzJTJDJTIwYXVkaW8lMkMlMjBzYW1wbGluZ19yYXRlJTNENDQxMDAlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0X2RpY3QpJTBBbG9zcyUyMCUzRCUyMG91dHB1dHMubG9zcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TvltProcessor, TvltForAudioVisualClassification
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>num_frames = <span class="hljs-number">8</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>images = <span class="hljs-built_in">list</span>(np.random.randn(num_frames, <span class="hljs-number">3</span>, <span class="hljs-number">224</span>, <span class="hljs-number">224</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>audio = <span class="hljs-built_in">list</span>(np.random.randn(<span class="hljs-number">10000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>processor = TvltProcessor.from_pretrained(<span class="hljs-string">&quot;ZinengTang/tvlt-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = TvltForAudioVisualClassification.from_pretrained(<span class="hljs-string">&quot;ZinengTang/tvlt-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_dict = processor(images, audio, sampling_rate=<span class="hljs-number">44100</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(**input_dict)
<span class="hljs-meta">&gt;&gt;&gt; </span>loss = outputs.loss`}}),{c(){p=r("p"),P=n("Examples:"),_=c(),T(u.$$.fragment)},l(l){p=a(l,"P",{});var g=s(p);P=i(g,"Examples:"),g.forEach(t),_=m(l),b(u.$$.fragment,l)},m(l,g){f(l,p,g),e(p,P),f(l,_,g),y(u,l,g),v=!0},p:Mr,i(l){v||(x(u.$$.fragment,l),v=!0)},o(l){w(u.$$.fragment,l),v=!1},d(l){l&&t(p),l&&t(_),M(u,l)}}}function Li(k){let p,P,_,u,v,l,g,$,kr,Nt,D,ie,Ho,We,$r,Xo,Fr,Wt,le,Er,Je,zr,Cr,Jt,wo,jr,Lt,Mo,Qo,Ir,Ut,Po,Ar,St,V,U,Vr,Yo,Nr,Wr,Ko,Jr,Lr,ko,Ur,Sr,Zr,q,Rr,et,Br,Dr,ot,qr,Or,Gr,Le,Hr,$o,Xr,Qr,Yr,tt,Kr,Zt,Ue,Se,hn,Rt,de,ea,Fo,oa,ta,Bt,R,ra,Ze,aa,sa,Re,na,ia,Dt,O,ce,rt,Be,la,at,da,qt,F,De,ca,G,ma,Eo,ha,pa,qe,fa,ua,ga,H,_a,zo,va,Ta,Co,ba,ya,xa,me,Ot,X,he,st,Oe,wa,nt,Ma,Gt,E,Ge,Pa,it,ka,$a,N,jo,Fa,Ea,Io,za,Ca,Ao,ja,Ia,He,lt,Aa,Va,Na,Wa,pe,Xe,Ja,z,La,dt,Ua,Sa,Vo,Za,Ra,ct,Ba,Da,Qe,mt,qa,Oa,Ga,Ht,Q,fe,ht,Ye,Ha,pt,Xa,Xt,C,Ke,Qa,ft,Ya,Ka,ut,es,os,ue,eo,ts,gt,rs,Qt,Y,ge,_t,oo,as,vt,ss,Yt,j,to,ns,Tt,is,ls,ro,ds,No,cs,ms,hs,_e,ao,ps,bt,fs,Kt,K,ve,yt,so,us,xt,gs,er,S,no,_s,io,vs,lo,Ts,bs,ys,W,co,xs,ee,ws,Wo,Ms,Ps,wt,ks,$s,Fs,Te,Es,be,or,oe,ye,Mt,mo,zs,Pt,Cs,tr,Z,ho,js,po,Is,fo,As,Vs,Ns,J,uo,Ws,te,Js,Jo,Ls,Us,kt,Ss,Zs,Rs,xe,Bs,we,rr,re,Me,$t,go,Ds,Ft,qs,ar,I,_o,Os,Et,Gs,Hs,vo,Xs,To,Qs,Ys,Ks,L,bo,en,ae,on,Lo,tn,rn,zt,an,sn,nn,Pe,ln,ke,sr;return l=new ne({}),We=new ne({}),Be=new ne({}),De=new A({props:{name:"class transformers.TvltConfig",anchor:"transformers.TvltConfig",parameters:[{name:"image_size",val:" = 224"},{name:"spectrogram_length",val:" = 2048"},{name:"frequency_length",val:" = 128"},{name:"image_patch_size",val:" = [16, 16]"},{name:"audio_patch_size",val:" = [16, 16]"},{name:"num_image_channels",val:" = 3"},{name:"num_audio_channels",val:" = 1"},{name:"num_frames",val:" = 8"},{name:"hidden_size",val:" = 768"},{name:"num_hidden_layers",val:" = 12"},{name:"num_attention_heads",val:" = 12"},{name:"intermediate_size",val:" = 3072"},{name:"hidden_act",val:" = 'gelu'"},{name:"hidden_dropout_prob",val:" = 0.0"},{name:"attention_probs_dropout_prob",val:" = 0.0"},{name:"initializer_range",val:" = 0.02"},{name:"layer_norm_eps",val:" = 1e-06"},{name:"qkv_bias",val:" = True"},{name:"use_mean_pooling",val:" = False"},{name:"decoder_num_attention_heads",val:" = 16"},{name:"decoder_hidden_size",val:" = 512"},{name:"decoder_num_hidden_layers",val:" = 8"},{name:"decoder_intermediate_size",val:" = 2048"},{name:"pixel_mask_ratio",val:" = 0.75"},{name:"audio_mask_ratio",val:" = 0.15"},{name:"audio_mask_type",val:" = 'frame-level'"},{name:"task_matching",val:" = True"},{name:"task_mae",val:" = True"},{name:"loss_type",val:" = 'classification'"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.TvltConfig.image_size",description:`<strong>image_size</strong> (<code>int</code>, <em>optional</em>, defaults to 224) &#x2014;
The size (resolution) of each image.`,name:"image_size"},{anchor:"transformers.TvltConfig.spectrogram_length",description:`<strong>spectrogram_length</strong> (<code>int</code>, <em>optional</em>, defaults to 2048) &#x2014;
The time length of each audio spectrogram.`,name:"spectrogram_length"},{anchor:"transformers.TvltConfig.frequency_length",description:`<strong>frequency_length</strong> (<code>int</code>, <em>optional</em>, defaults to 128) &#x2014;
The frequency length of audio spectrogram.`,name:"frequency_length"},{anchor:"transformers.TvltConfig.image_patch_size",description:`<strong>image_patch_size</strong> (<code>List[int]</code>, <em>optional</em>, defaults to <code>[16, 16]</code>) &#x2014;
The size (resolution) of each image patch.`,name:"image_patch_size"},{anchor:"transformers.TvltConfig.audio_patch_size",description:`<strong>audio_patch_size</strong> (<code>List[int]</code>, <em>optional</em>, defaults to <code>[16, 16]</code>) &#x2014;
The size (resolution) of each audio patch.`,name:"audio_patch_size"},{anchor:"transformers.TvltConfig.num_image_channels",description:`<strong>num_image_channels</strong> (<code>int</code>, <em>optional</em>, defaults to 3) &#x2014;
The number of input image channels.`,name:"num_image_channels"},{anchor:"transformers.TvltConfig.num_audio_channels",description:`<strong>num_audio_channels</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of input audio channels.`,name:"num_audio_channels"},{anchor:"transformers.TvltConfig.num_frames",description:`<strong>num_frames</strong> (<code>int</code>, <em>optional</em>, defaults to 8) &#x2014;
The maximum number of frames for an input video.`,name:"num_frames"},{anchor:"transformers.TvltConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 768) &#x2014;
Dimensionality of the encoder layers and the pooler layer.`,name:"hidden_size"},{anchor:"transformers.TvltConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 12) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.TvltConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 12) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.TvltConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 3072) &#x2014;
Dimensionality of the &#x201C;intermediate&#x201D; (i.e., feed-forward) layer in the Transformer encoder.`,name:"intermediate_size"},{anchor:"transformers.TvltConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>function</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the encoder and pooler. If string, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;selu&quot;</code> and <code>&quot;gelu_new&quot;</code> are supported.`,name:"hidden_act"},{anchor:"transformers.TvltConfig.hidden_dropout_prob",description:`<strong>hidden_dropout_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.`,name:"hidden_dropout_prob"},{anchor:"transformers.TvltConfig.attention_probs_dropout_prob",description:`<strong>attention_probs_dropout_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_probs_dropout_prob"},{anchor:"transformers.TvltConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 0.02) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.TvltConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-6) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.TvltConfig.qkv_bias",description:`<strong>qkv_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to add a bias to the queries, keys and values.`,name:"qkv_bias"},{anchor:"transformers.TvltConfig.use_mean_pooling",description:`<strong>use_mean_pooling</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to mean pool the final hidden states instead of using the final hidden state of the [CLS] token.`,name:"use_mean_pooling"},{anchor:"transformers.TvltConfig.decoder_num_attention_heads",description:`<strong>decoder_num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 16) &#x2014;
Number of attention heads for each attention layer in the decoder.`,name:"decoder_num_attention_heads"},{anchor:"transformers.TvltConfig.decoder_hidden_size",description:`<strong>decoder_hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 512) &#x2014;
Dimensionality of the decoder.`,name:"decoder_hidden_size"},{anchor:"transformers.TvltConfig.decoder_num_hidden_layers",description:`<strong>decoder_num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 8) &#x2014;
Number of hidden layers in the decoder.`,name:"decoder_num_hidden_layers"},{anchor:"transformers.TvltConfig.decoder_intermediate_size",description:`<strong>decoder_intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 2048) &#x2014;
Dimensionality of the &#x201C;intermediate&#x201D; (i.e., feed-forward) layer in the decoder.`,name:"decoder_intermediate_size"},{anchor:"transformers.TvltConfig.pixel_mask_ratio",description:`<strong>pixel_mask_ratio</strong> (<code>float</code>, <em>optional</em>, defaults to 0.75) &#x2014;
Image patch masking ratio.`,name:"pixel_mask_ratio"},{anchor:"transformers.TvltConfig.audio_mask_ratio",description:`<strong>audio_mask_ratio</strong> (<code>float</code>, <em>optional</em>, defaults to 0.15) &#x2014;
Audio patch masking ratio.`,name:"audio_mask_ratio"},{anchor:"transformers.TvltConfig.audio_mask_type",description:`<strong>audio_mask_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;frame-level&quot;</code>) &#x2014;
Audio patch masking type, choose between &#x201C;frame-level&#x201D; and &#x201C;patch-level&#x201D;.`,name:"audio_mask_type"},{anchor:"transformers.TvltConfig.task_matching",description:`<strong>task_matching</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to use vision audio matching task in pretraining.`,name:"task_matching"},{anchor:"transformers.TvltConfig.task_mae",description:`<strong>task_mae</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to use the masked auto-encoder (MAE) in pretraining.`,name:"task_mae"},{anchor:"transformers.TvltConfig.loss_type",description:`<strong>loss_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;classification&quot;</code>) &#x2014;
Loss types including regression and classification.`,name:"loss_type"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/configuration_tvlt.py#L28"}}),me=new wr({props:{anchor:"transformers.TvltConfig.example",$$slots:{default:[ji]},$$scope:{ctx:k}}}),Oe=new ne({}),Ge=new A({props:{name:"class transformers.TvltProcessor",anchor:"transformers.TvltProcessor",parameters:[{name:"image_processor",val:""},{name:"feature_extractor",val:""}],parametersDescription:[{anchor:"transformers.TvltProcessor.image_processor",description:`<strong>image_processor</strong> (<code>TvltImageProcessor</code>) &#x2014;
An instance of <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltImageProcessor">TvltImageProcessor</a>. The image processor is a required input.`,name:"image_processor"},{anchor:"transformers.TvltProcessor.feature_extractor",description:`<strong>feature_extractor</strong> (<code>TvltFeatureExtractor</code>) &#x2014;
An instance of <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltFeatureExtractor">TvltFeatureExtractor</a>. The feature extractor is a required input.`,name:"feature_extractor"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/processing_tvlt.py#L22"}}),Xe=new A({props:{name:"__call__",anchor:"transformers.TvltProcessor.__call__",parameters:[{name:"images",val:" = None"},{name:"audio",val:" = None"},{name:"images_mixed",val:" = None"},{name:"sampling_rate",val:" = None"},{name:"mask_audio",val:" = False"},{name:"mask_pixel",val:" = False"},{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/processing_tvlt.py#L45"}}),Ye=new ne({}),Ke=new A({props:{name:"class transformers.TvltImageProcessor",anchor:"transformers.TvltImageProcessor",parameters:[{name:"do_resize",val:": bool = True"},{name:"size",val:": typing.Dict[str, int] = None"},{name:"patch_size",val:": typing.List[int] = [16, 16]"},{name:"num_frames",val:": int = 8"},{name:"resample",val:": Resampling = <Resampling.BILINEAR: 2>"},{name:"do_center_crop",val:": bool = True"},{name:"crop_size",val:": typing.Dict[str, int] = None"},{name:"do_rescale",val:": bool = True"},{name:"rescale_factor",val:": typing.Union[int, float] = 0.00392156862745098"},{name:"do_normalize",val:": bool = True"},{name:"image_mean",val:": typing.Union[float, typing.List[float], NoneType] = [0.5, 0.5, 0.5]"},{name:"image_std",val:": typing.Union[float, typing.List[float], NoneType] = [0.5, 0.5, 0.5]"},{name:"init_mask_generator",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.TvltImageProcessor.do_resize",description:`<strong>do_resize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to resize the image&#x2019;s (height, width) dimensions to the specified <code>size</code>. Can be overridden by the
<code>do_resize</code> parameter in the <code>preprocess</code> method.`,name:"do_resize"},{anchor:"transformers.TvltImageProcessor.size",description:`<strong>size</strong> (<code>Dict[str, int]</code> <em>optional</em>, defaults to <code>{&quot;shortest_edge&quot; -- 224}</code>):
Size of the output image after resizing. The shortest edge of the image will be resized to
<code>size[&quot;shortest_edge&quot;]</code> while maintaining the aspect ratio of the original image. Can be overriden by
<code>size</code> in the <code>preprocess</code> method.`,name:"size"},{anchor:"transformers.TvltImageProcessor.patch_size",description:`<strong>patch_size</strong> (<code>List[int]</code> <em>optional</em>, defaults to [16,16]) &#x2014;
The patch size of image patch embedding.`,name:"patch_size"},{anchor:"transformers.TvltImageProcessor.num_frames",description:`<strong>num_frames</strong> (<code>int</code> <em>optional</em>, defaults to 8) &#x2014;
The maximum number of video frames.`,name:"num_frames"},{anchor:"transformers.TvltImageProcessor.resample",description:`<strong>resample</strong> (<code>PILImageResampling</code>, <em>optional</em>, defaults to <code>PILImageResampling.BILINEAR</code>) &#x2014;
Resampling filter to use if resizing the image. Can be overridden by the <code>resample</code> parameter in the
<code>preprocess</code> method.`,name:"resample"},{anchor:"transformers.TvltImageProcessor.do_center_crop",description:`<strong>do_center_crop</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to center crop the image to the specified <code>crop_size</code>. Can be overridden by the <code>do_center_crop</code>
parameter in the <code>preprocess</code> method.`,name:"do_center_crop"},{anchor:"transformers.TvltImageProcessor.crop_size",description:`<strong>crop_size</strong> (<code>Dict[str, int]</code>, <em>optional</em>, defaults to <code>{&quot;height&quot; -- 224, &quot;width&quot;: 224}</code>):
Size of the image after applying the center crop. Can be overridden by the <code>crop_size</code> parameter in the
<code>preprocess</code> method.`,name:"crop_size"},{anchor:"transformers.TvltImageProcessor.do_rescale",description:`<strong>do_rescale</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to rescale the image by the specified scale <code>rescale_factor</code>. Can be overridden by the <code>do_rescale</code>
parameter in the <code>preprocess</code> method.`,name:"do_rescale"},{anchor:"transformers.TvltImageProcessor.rescale_factor",description:`<strong>rescale_factor</strong> (<code>int</code> or <code>float</code>, <em>optional</em>, defaults to 1/255) &#x2014;
Defines the scale factor to use if rescaling the image. Can be overridden by the <code>rescale_factor</code> parameter
in the <code>preprocess</code> method.`,name:"rescale_factor"},{anchor:"transformers.TvltImageProcessor.do_normalize",description:`<strong>do_normalize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to normalize the image. Can be overridden by the <code>do_normalize</code> parameter in the <code>preprocess</code>
method.`,name:"do_normalize"},{anchor:"transformers.TvltImageProcessor.image_mean",description:`<strong>image_mean</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>IMAGENET_STANDARD_MEAN</code>) &#x2014;
Mean to use if normalizing the image. This is a float or list of floats the length of the number of
channels in the image. Can be overridden by the <code>image_mean</code> parameter in the <code>preprocess</code> method.`,name:"image_mean"},{anchor:"transformers.TvltImageProcessor.image_std",description:`<strong>image_std</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>IMAGENET_STANDARD_STD</code>) &#x2014;
Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
number of channels in the image. Can be overridden by the <code>image_std</code> parameter in the <code>preprocess</code> method.`,name:"image_std"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/image_processing_tvlt.py#L67"}}),eo=new A({props:{name:"preprocess",anchor:"transformers.TvltImageProcessor.preprocess",parameters:[{name:"videos",val:": typing.Union[ForwardRef('PIL.Image.Image'), numpy.ndarray, ForwardRef('torch.Tensor'), typing.List[ForwardRef('PIL.Image.Image')], typing.List[numpy.ndarray], typing.List[ForwardRef('torch.Tensor')]]"},{name:"do_resize",val:": bool = None"},{name:"size",val:": typing.Dict[str, int] = None"},{name:"patch_size",val:": typing.List[int] = None"},{name:"num_frames",val:": int = None"},{name:"resample",val:": Resampling = None"},{name:"do_center_crop",val:": bool = None"},{name:"crop_size",val:": typing.Dict[str, int] = None"},{name:"do_rescale",val:": bool = None"},{name:"rescale_factor",val:": float = None"},{name:"do_normalize",val:": bool = None"},{name:"image_mean",val:": typing.Union[float, typing.List[float], NoneType] = None"},{name:"image_std",val:": typing.Union[float, typing.List[float], NoneType] = None"},{name:"is_mixed",val:": bool = False"},{name:"return_tensors",val:": typing.Union[str, transformers.utils.generic.TensorType, NoneType] = None"},{name:"data_format",val:": ChannelDimension = <ChannelDimension.FIRST: 'channels_first'>"},{name:"input_data_format",val:": typing.Union[str, transformers.image_utils.ChannelDimension, NoneType] = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.TvltImageProcessor.preprocess.videos",description:`<strong>videos</strong> (<code>ImageInput</code>) &#x2014;
Images or videos to preprocess. Expects a single or batch of frames with pixel values ranging from 0 to
255. If passing in frames with pixel values between 0 and 1, set <code>do_rescale=False</code>.`,name:"videos"},{anchor:"transformers.TvltImageProcessor.preprocess.do_resize",description:`<strong>do_resize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_resize</code>) &#x2014;
Whether to resize the image.`,name:"do_resize"},{anchor:"transformers.TvltImageProcessor.preprocess.size",description:`<strong>size</strong> (<code>Dict[str, int]</code>, <em>optional</em>, defaults to <code>self.size</code>) &#x2014;
Size of the image after applying resize.`,name:"size"},{anchor:"transformers.TvltImageProcessor.preprocess.patch_size",description:`<strong>patch_size</strong> (<code>List[int]</code> <em>optional</em>, defaults to self.patch_size) &#x2014;
The patch size of image patch embedding.`,name:"patch_size"},{anchor:"transformers.TvltImageProcessor.preprocess.num_frames",description:`<strong>num_frames</strong> (<code>int</code> <em>optional</em>, defaults to self.num_frames) &#x2014;
The maximum number of video frames.`,name:"num_frames"},{anchor:"transformers.TvltImageProcessor.preprocess.resample",description:`<strong>resample</strong> (<code>PILImageResampling</code>, <em>optional</em>, defaults to <code>self.resample</code>) &#x2014;
Resampling filter to use if resizing the image. This can be one of the enum <code>PILImageResampling</code>, Only
has an effect if <code>do_resize</code> is set to <code>True</code>.`,name:"resample"},{anchor:"transformers.TvltImageProcessor.preprocess.do_center_crop",description:`<strong>do_center_crop</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_centre_crop</code>) &#x2014;
Whether to centre crop the image.`,name:"do_center_crop"},{anchor:"transformers.TvltImageProcessor.preprocess.crop_size",description:`<strong>crop_size</strong> (<code>Dict[str, int]</code>, <em>optional</em>, defaults to <code>self.crop_size</code>) &#x2014;
Size of the image after applying the centre crop.`,name:"crop_size"},{anchor:"transformers.TvltImageProcessor.preprocess.do_rescale",description:`<strong>do_rescale</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_rescale</code>) &#x2014;
Whether to rescale the image values between [0 - 1].`,name:"do_rescale"},{anchor:"transformers.TvltImageProcessor.preprocess.rescale_factor",description:`<strong>rescale_factor</strong> (<code>float</code>, <em>optional</em>, defaults to <code>self.rescale_factor</code>) &#x2014;
Rescale factor to rescale the image by if <code>do_rescale</code> is set to <code>True</code>.`,name:"rescale_factor"},{anchor:"transformers.TvltImageProcessor.preprocess.do_normalize",description:`<strong>do_normalize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_normalize</code>) &#x2014;
Whether to normalize the image.`,name:"do_normalize"},{anchor:"transformers.TvltImageProcessor.preprocess.image_mean",description:`<strong>image_mean</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>self.image_mean</code>) &#x2014;
Image mean.`,name:"image_mean"},{anchor:"transformers.TvltImageProcessor.preprocess.image_std",description:`<strong>image_std</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>self.image_std</code>) &#x2014;
Image standard deviation.`,name:"image_std"},{anchor:"transformers.TvltImageProcessor.preprocess.is_mixed",description:`<strong>is_mixed</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If the input video has negative samples.`,name:"is_mixed"},{anchor:"transformers.TvltImageProcessor.preprocess.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <code>TensorType</code>, <em>optional</em>) &#x2014;
The type of tensors to return. Can be one of:<ul>
<li>Unset: Return a list of <code>np.ndarray</code>.</li>
<li><code>TensorType.TENSORFLOW</code> or <code>&apos;tf&apos;</code>: Return a batch of type <code>tf.Tensor</code>.</li>
<li><code>TensorType.PYTORCH</code> or <code>&apos;pt&apos;</code>: Return a batch of type <code>torch.Tensor</code>.</li>
<li><code>TensorType.NUMPY</code> or <code>&apos;np&apos;</code>: Return a batch of type <code>np.ndarray</code>.</li>
<li><code>TensorType.JAX</code> or <code>&apos;jax&apos;</code>: Return a batch of type <code>jax.numpy.ndarray</code>.</li>
</ul>`,name:"return_tensors"},{anchor:"transformers.TvltImageProcessor.preprocess.data_format",description:`<strong>data_format</strong> (<code>ChannelDimension</code> or <code>str</code>, <em>optional</em>, defaults to <code>ChannelDimension.FIRST</code>) &#x2014;
The channel dimension format for the output image. Can be one of:<ul>
<li><code>ChannelDimension.FIRST</code>: image in (num_channels, height, width) format.</li>
<li><code>ChannelDimension.LAST</code>: image in (height, width, num_channels) format.</li>
<li>Unset: Use the inferred channel dimension format of the input image.</li>
</ul>`,name:"data_format"},{anchor:"transformers.TvltImageProcessor.preprocess.input_data_format",description:`<strong>input_data_format</strong> (<code>ChannelDimension</code> or <code>str</code>, <em>optional</em>) &#x2014;
The channel dimension format for the input image. If unset, the channel dimension format is inferred
from the input image. Can be one of:<ul>
<li><code>&quot;channels_first&quot;</code> or <code>ChannelDimension.FIRST</code>: image in (num_channels, height, width) format.</li>
<li><code>&quot;channels_last&quot;</code> or <code>ChannelDimension.LAST</code>: image in (height, width, num_channels) format.</li>
<li><code>&quot;none&quot;</code> or <code>ChannelDimension.NONE</code>: image in (height, width) format.</li>
</ul>`,name:"input_data_format"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/image_processing_tvlt.py#L253",returnDescription:`
<p>A <a
  href="/docs/transformers/main/en/main_classes/feature_extractor#transformers.BatchFeature"
>BatchFeature</a> with the following fields:</p>
<ul>
<li>
<p><strong>pixel_values</strong> \u2014 Pixel values to be fed to a model, of shape (batch_size, num_channels, height,
width).</p>
</li>
<li>
<p><strong>pixel_mask</strong> \u2014 Pixel masks to be fed to a model, of shape (batch_size, num_pixel_patches).</p>
</li>
<li>
<p><strong>pixel_values_mixed</strong> \u2014 Pixel values with both postive or negative to be fed to a model, of shape
(batch_size, num_channels, height, width).</p>
</li>
<li>
<p><strong>pixel_mask_mixed</strong> \u2014 Pixel masks with both postive or negative to be fed to a model, of shape
(batch_size, num_pixel_patches).</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/main/en/main_classes/feature_extractor#transformers.BatchFeature"
>BatchFeature</a></p>
`}}),oo=new ne({}),to=new A({props:{name:"class transformers.TvltFeatureExtractor",anchor:"transformers.TvltFeatureExtractor",parameters:[{name:"spectrogram_length",val:" = 2048"},{name:"num_channels",val:" = 1"},{name:"patch_size",val:" = [16, 16]"},{name:"feature_size",val:" = 128"},{name:"sampling_rate",val:" = 44100"},{name:"hop_length_to_sampling_rate",val:" = 86"},{name:"n_fft",val:" = 2048"},{name:"padding_value",val:" = 0.0"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.TvltFeatureExtractor.spectrogram_length",description:`<strong>spectrogram_length</strong> (<code>Dict[str, int]</code> <em>optional</em>, defaults to 2048) &#x2014;
The time length of each audio spectrogram.`,name:"spectrogram_length"},{anchor:"transformers.TvltFeatureExtractor.num_channels",description:`<strong>num_channels</strong> (<code>int</code> <em>optional</em>, defaults to 1) &#x2014;
Number of audio channels.`,name:"num_channels"},{anchor:"transformers.TvltFeatureExtractor.patch_size",description:`<strong>patch_size</strong> (<code>List[int]</code> <em>optional</em>, defaults to <code>[16, 16]</code>) &#x2014;
The patch size of audio patch embedding.`,name:"patch_size"},{anchor:"transformers.TvltFeatureExtractor.feature_size",description:`<strong>feature_size</strong> (<code>int</code>, defaults to 128) &#x2014;
The frequency length of audio spectrogram.`,name:"feature_size"},{anchor:"transformers.TvltFeatureExtractor.sampling_rate",description:`<strong>sampling_rate</strong> (<code>int</code>, defaults to 44100) &#x2014;
The sampling rate at which the audio files should be digitalized expressed in Hertz (Hz).`,name:"sampling_rate"},{anchor:"transformers.TvltFeatureExtractor.hop_length_to_sampling_rate",description:`<strong>hop_length_to_sampling_rate</strong> (<code>int</code>, defaults to 86) &#x2014;
Hop length is length of the overlaping windows for the STFT used to obtain the Mel Frequency coefficients.
For example, with sampling rate 44100, the hop length is 512, with 44100 / 512 = 86`,name:"hop_length_to_sampling_rate"},{anchor:"transformers.TvltFeatureExtractor.n_fft",description:`<strong>n_fft</strong> (<code>int</code>, defaults to 2048) &#x2014;
Size of the Fourier transform.`,name:"n_fft"},{anchor:"transformers.TvltFeatureExtractor.padding_value",description:`<strong>padding_value</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
Padding value used to pad the audio. Should correspond to silences.`,name:"padding_value"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/feature_extraction_tvlt.py#L30"}}),ao=new A({props:{name:"__call__",anchor:"transformers.TvltFeatureExtractor.__call__",parameters:[{name:"raw_speech",val:": typing.Union[numpy.ndarray, typing.List[float], typing.List[numpy.ndarray], typing.List[typing.List[float]]]"},{name:"return_tensors",val:": typing.Union[str, transformers.utils.generic.TensorType, NoneType] = None"},{name:"return_attention_mask",val:": typing.Optional[bool] = True"},{name:"sampling_rate",val:": typing.Optional[int] = None"},{name:"resample",val:": bool = False"},{name:"mask_audio",val:": bool = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.TvltFeatureExtractor.__call__.raw_speech",description:`<strong>raw_speech</strong> (<code>np.ndarray</code>, <code>List[float]</code>, <code>List[np.ndarray]</code>, <code>List[List[float]]</code>) &#x2014;
The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
stereo, i.e. single float per timestep.`,name:"raw_speech"},{anchor:"transformers.TvltFeatureExtractor.__call__.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <a href="/docs/transformers/main/en/internal/file_utils#transformers.TensorType">TensorType</a>, <em>optional</em>) &#x2014;
If set, will return tensors instead of list of python integers. Acceptable values are:</p>
<ul>
<li><code>&apos;pt&apos;</code>: Return PyTorch <code>torch.Tensor</code> objects.</li>
<li><code>&apos;np&apos;</code>: Return Numpy <code>np.ndarray</code> objects.</li>
</ul>`,name:"return_tensors"},{anchor:"transformers.TvltFeatureExtractor.__call__.return_attention_mask",description:`<strong>return_attention_mask</strong> (<code>bool</code>, <em>optional</em>, default to <code>True</code>) &#x2014;
Whether to return the attention mask. If left to the default, will return the attention mask according
to the specific feature_extractor&#x2019;s default. <a href="../glossary#attention-mask">What are attention masks?</a></p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>For TvltTransformer models, <code>attention_mask</code> should alwys be passed for batched inference, to avoid
subtle bugs.</p>

					</div>`,name:"return_attention_mask"},{anchor:"transformers.TvltFeatureExtractor.__call__.sampling_rate",description:`<strong>sampling_rate</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The sampling rate at which the <code>raw_speech</code> input was sampled. It is strongly recommended to pass
<code>sampling_rate</code> at the forward call to prevent silent errors and allow automatic speech recognition
pipeline. Current model supports sampling rate 16000 and 44100.`,name:"sampling_rate"},{anchor:"transformers.TvltFeatureExtractor.__call__.resample",description:`<strong>resample</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
If the sampling rate is not matched, resample the input audio to match.`,name:"resample"},{anchor:"transformers.TvltFeatureExtractor.__call__.mask_audio",description:`<strong>mask_audio</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to mask input audio for MAE task.`,name:"mask_audio"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/feature_extraction_tvlt.py#L116",returnDescription:`
<p>A <a
  href="/docs/transformers/main/en/main_classes/feature_extractor#transformers.BatchFeature"
>BatchFeature</a> with the following fields:</p>
<ul>
<li>
<p><strong>audio_values</strong> \u2014 Audio values to be fed to a model, of shape (batch_size, num_channels, height,
width).</p>
</li>
<li>
<p><strong>audio_mask</strong> \u2014 Audio masks to be fed to a model, of shape (batch_size, num_audio_patches).</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/main/en/main_classes/feature_extractor#transformers.BatchFeature"
>BatchFeature</a></p>
`}}),so=new ne({}),no=new A({props:{name:"class transformers.TvltModel",anchor:"transformers.TvltModel",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.TvltModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltConfig">TvltConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/modeling_tvlt.py#L684"}}),co=new A({props:{name:"forward",anchor:"transformers.TvltModel.forward",parameters:[{name:"pixel_values",val:": FloatTensor"},{name:"audio_values",val:": FloatTensor"},{name:"pixel_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"audio_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"mask_pixel",val:": bool = False"},{name:"mask_audio",val:": bool = False"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.TvltModel.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_frames, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"pixel_values"},{anchor:"transformers.TvltModel.forward.audio_values",description:`<strong>audio_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Audio values. Audio values can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"audio_values"},{anchor:"transformers.TvltModel.forward.pixel_mask",description:`<strong>pixel_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_pixel_patches)</code>) &#x2014;
Pixel masks. Pixel masks can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"pixel_mask"},{anchor:"transformers.TvltModel.forward.audio_mask",description:`<strong>audio_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_audio_patches)</code>) &#x2014;
Audio masks. Audio masks can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"audio_mask"},{anchor:"transformers.TvltModel.forward.pixel_values_mixed",description:`<strong>pixel_values_mixed</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_frames, num_channels, height, width)</code>) &#x2014;
Pixel values that mix positive and negative samples in Tvlt vision-audio matching. Pixel values mixed can
be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for details.`,name:"pixel_values_mixed"},{anchor:"transformers.TvltModel.forward.pixel_mask_mixed",description:`<strong>pixel_mask_mixed</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel masks of pixel_values_mixed. Pixel masks mixed can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See
<a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for details.`,name:"pixel_mask_mixed"},{anchor:"transformers.TvltModel.forward.mask_pixel",description:`<strong>mask_pixel</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to mask pixel for MAE tasks. Only set to True in TvltForPreTraining.`,name:"mask_pixel"},{anchor:"transformers.TvltModel.forward.mask_audio",description:`<strong>mask_audio</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to mask audio for MAE tasks. Only set to True in TvltForPreTraining.`,name:"mask_audio"},{anchor:"transformers.TvltModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.TvltModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.TvltModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/modeling_tvlt.py#L714",returnDescription:`
<p>A <code>transformers.models.tvlt.modeling_tvlt.TvltModelOutput</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltConfig"
>TvltConfig</a>) and inputs.</p>
<ul>
<li><strong>last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) \u2014 Sequence of hidden-states at the output of the last layer of the model.</li>
<li><strong>last_pixel_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, pixel_sequence_length, hidden_size)</code>) \u2014 Pixel sequence of hidden-states at the output of the last layer of the model.</li>
<li><strong>last_audio_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, audio_sequence_length, hidden_size)</code>) \u2014 Audio sequence of hidden-states at the output of the last layer of the model.</li>
<li><strong>pixel_label_masks</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, pixel_patch_length)</code>) \u2014 Tensor indicating which pixel patches are masked (1) and which are not (0).</li>
<li><strong>audio_label_masks</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, audio_patch_length)</code>) \u2014 Tensor indicating which audio patches are masked (1) and which are not (0).</li>
<li><strong>pixel_ids_restore</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, pixel_patch_length)</code>) \u2014 Tensor containing the ids permutation of pixel masking.</li>
<li><strong>audio_ids_restore</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, audio_patch_length)</code>) \u2014 Tensor containing the ids permutation of audio masking.</li>
<li><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings and one for the output of each layer) of
shape <code>(batch_size, sequence_length, hidden_size)</code>. Hidden-states of the model at the output of each layer
plus the initial embedding outputs.</li>
<li><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>. Attentions weights after the attention softmax, used to compute the weighted average in
the self-attention heads.</li>
</ul>
`,returnType:`
<p><code>transformers.models.tvlt.modeling_tvlt.TvltModelOutput</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),Te=new mn({props:{$$slots:{default:[Ii]},$$scope:{ctx:k}}}),be=new wr({props:{anchor:"transformers.TvltModel.forward.example",$$slots:{default:[Ai]},$$scope:{ctx:k}}}),mo=new ne({}),ho=new A({props:{name:"class transformers.TvltForPreTraining",anchor:"transformers.TvltForPreTraining",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.TvltForPreTraining.config",description:`<strong>config</strong> (<a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltConfig">TvltConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/modeling_tvlt.py#L915"}}),uo=new A({props:{name:"forward",anchor:"transformers.TvltForPreTraining.forward",parameters:[{name:"pixel_values",val:": FloatTensor"},{name:"audio_values",val:": FloatTensor"},{name:"pixel_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"audio_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"pixel_values_mixed",val:": typing.Optional[torch.FloatTensor] = None"},{name:"pixel_mask_mixed",val:": typing.Optional[torch.FloatTensor] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.TvltForPreTraining.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_frames, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"pixel_values"},{anchor:"transformers.TvltForPreTraining.forward.audio_values",description:`<strong>audio_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Audio values. Audio values can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"audio_values"},{anchor:"transformers.TvltForPreTraining.forward.pixel_mask",description:`<strong>pixel_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_pixel_patches)</code>) &#x2014;
Pixel masks. Pixel masks can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"pixel_mask"},{anchor:"transformers.TvltForPreTraining.forward.audio_mask",description:`<strong>audio_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_audio_patches)</code>) &#x2014;
Audio masks. Audio masks can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"audio_mask"},{anchor:"transformers.TvltForPreTraining.forward.pixel_values_mixed",description:`<strong>pixel_values_mixed</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_frames, num_channels, height, width)</code>) &#x2014;
Pixel values that mix positive and negative samples in Tvlt vision-audio matching. Pixel values mixed can
be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for details.`,name:"pixel_values_mixed"},{anchor:"transformers.TvltForPreTraining.forward.pixel_mask_mixed",description:`<strong>pixel_mask_mixed</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel masks of pixel_values_mixed. Pixel masks mixed can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See
<a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for details.`,name:"pixel_mask_mixed"},{anchor:"transformers.TvltForPreTraining.forward.mask_pixel",description:`<strong>mask_pixel</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to mask pixel for MAE tasks. Only set to True in TvltForPreTraining.`,name:"mask_pixel"},{anchor:"transformers.TvltForPreTraining.forward.mask_audio",description:`<strong>mask_audio</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to mask audio for MAE tasks. Only set to True in TvltForPreTraining.`,name:"mask_audio"},{anchor:"transformers.TvltForPreTraining.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.TvltForPreTraining.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.TvltForPreTraining.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.TvltForPreTraining.forward.pixel_values_mixed",description:`<strong>pixel_values_mixed</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_frames, num_channels, height, width)</code>) &#x2014;
Pixel values that mix positive and negative samples in Tvlt vision-audio matching. Audio values can be
obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for details.`,name:"pixel_values_mixed"},{anchor:"transformers.TvltForPreTraining.forward.pixel_mask_mixed",description:`<strong>pixel_mask_mixed</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel masks of pixel_values_mixed. Pixel values mixed can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See
<a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for details.`,name:"pixel_mask_mixed"},{anchor:"transformers.TvltForPreTraining.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, num_labels)</code>, <em>optional</em>) &#x2014;
Labels for computing the vision audio matching loss. Indices should be in <code>[0, 1]</code>. num_labels has to be 1.`,name:"labels"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/modeling_tvlt.py#L1048",returnDescription:`
<p>A <code>transformers.models.tvlt.modeling_tvlt.TvltForPreTrainingOutput</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltConfig"
>TvltConfig</a>) and inputs.</p>
<ul>
<li><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>) \u2014 Pixel reconstruction loss.</li>
<li><strong>matching_logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, 1)</code>) \u2014 Matching objective logits.</li>
<li><strong>pixel_logits</strong> (<code>torch.FloatTensor</code> of shape
<code>(batch_size, pixel_patch_length, image_patch_size ** 3 * pixel_num_channels)</code>): Pixel reconstruction
logits.</li>
<li><strong>audio_logits</strong> (<code>torch.FloatTensor</code> of shape
<code>(batch_size, audio_patch_length, image_patch_size[0] * image_patch_size[1])</code>): Audio reconstruction
logits.</li>
<li><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings and one for the output of each layer) of
shape <code>(batch_size, sequence_length, hidden_size)</code>. Hidden-states of the model at the output of each layer
plus the initial embedding outputs.</li>
<li><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>. Attentions weights after the attention softmax, used to compute the weighted average in
the self-attention heads.</li>
</ul>
`,returnType:`
<p><code>transformers.models.tvlt.modeling_tvlt.TvltForPreTrainingOutput</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),xe=new mn({props:{$$slots:{default:[Vi]},$$scope:{ctx:k}}}),we=new wr({props:{anchor:"transformers.TvltForPreTraining.forward.example",$$slots:{default:[Ni]},$$scope:{ctx:k}}}),go=new ne({}),_o=new A({props:{name:"class transformers.TvltForAudioVisualClassification",anchor:"transformers.TvltForAudioVisualClassification",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.TvltForAudioVisualClassification.config",description:`<strong>config</strong> (<a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltConfig">TvltConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/modeling_tvlt.py#L1231"}}),bo=new A({props:{name:"forward",anchor:"transformers.TvltForAudioVisualClassification.forward",parameters:[{name:"pixel_values",val:": FloatTensor"},{name:"audio_values",val:": FloatTensor"},{name:"pixel_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"audio_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"}],parametersDescription:[{anchor:"transformers.TvltForAudioVisualClassification.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_frames, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"pixel_values"},{anchor:"transformers.TvltForAudioVisualClassification.forward.audio_values",description:`<strong>audio_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Audio values. Audio values can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"audio_values"},{anchor:"transformers.TvltForAudioVisualClassification.forward.pixel_mask",description:`<strong>pixel_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_pixel_patches)</code>) &#x2014;
Pixel masks. Pixel masks can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"pixel_mask"},{anchor:"transformers.TvltForAudioVisualClassification.forward.audio_mask",description:`<strong>audio_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_audio_patches)</code>) &#x2014;
Audio masks. Audio masks can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for
details.`,name:"audio_mask"},{anchor:"transformers.TvltForAudioVisualClassification.forward.pixel_values_mixed",description:`<strong>pixel_values_mixed</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_frames, num_channels, height, width)</code>) &#x2014;
Pixel values that mix positive and negative samples in Tvlt vision-audio matching. Pixel values mixed can
be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for details.`,name:"pixel_values_mixed"},{anchor:"transformers.TvltForAudioVisualClassification.forward.pixel_mask_mixed",description:`<strong>pixel_mask_mixed</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel masks of pixel_values_mixed. Pixel masks mixed can be obtained using <a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor">TvltProcessor</a>. See
<a href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__">TvltProcessor.<strong>call</strong>()</a> for details.`,name:"pixel_mask_mixed"},{anchor:"transformers.TvltForAudioVisualClassification.forward.mask_pixel",description:`<strong>mask_pixel</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to mask pixel for MAE tasks. Only set to True in TvltForPreTraining.`,name:"mask_pixel"},{anchor:"transformers.TvltForAudioVisualClassification.forward.mask_audio",description:`<strong>mask_audio</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether to mask audio for MAE tasks. Only set to True in TvltForPreTraining.`,name:"mask_audio"},{anchor:"transformers.TvltForAudioVisualClassification.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.TvltForAudioVisualClassification.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.TvltForAudioVisualClassification.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.TvltForAudioVisualClassification.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, num_labels)</code>, <em>optional</em>) &#x2014;
Labels for computing the audiovisual loss. Indices should be in <code>[0, ..., num_classes-1]</code> where num_classes
refers to the number of classes in audiovisual tasks.`,name:"labels"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/tvlt/modeling_tvlt.py#L1249",returnDescription:`
<p>A <a
  href="/docs/transformers/main/en/main_classes/output#transformers.modeling_outputs.SequenceClassifierOutput"
>transformers.modeling_outputs.SequenceClassifierOutput</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/main/en/model_doc/tvlt#transformers.TvltConfig"
>TvltConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) \u2014 Classification (or regression if config.num_labels==1) loss.</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, config.num_labels)</code>) \u2014 Classification (or regression if config.num_labels==1) scores (before SoftMax).</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/main/en/main_classes/output#transformers.modeling_outputs.SequenceClassifierOutput"
>transformers.modeling_outputs.SequenceClassifierOutput</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),Pe=new mn({props:{$$slots:{default:[Wi]},$$scope:{ctx:k}}}),ke=new wr({props:{anchor:"transformers.TvltForAudioVisualClassification.forward.example",$$slots:{default:[Ji]},$$scope:{ctx:k}}}),{c(){p=r("meta"),P=c(),_=r("h1"),u=r("a"),v=r("span"),T(l.$$.fragment),g=c(),$=r("span"),kr=n("TVLT"),Nt=c(),D=r("h2"),ie=r("a"),Ho=r("span"),T(We.$$.fragment),$r=c(),Xo=r("span"),Fr=n("Overview"),Wt=c(),le=r("p"),Er=n("The TVLT model was proposed in "),Je=r("a"),zr=n("TVLT: Textless Vision-Language Transformer"),Cr=n(`
by Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal (the first three authors contributed equally). The Textless Vision-Language Transformer (TVLT) is a model that uses raw visual and audio inputs for vision-and-language representation learning, without using text-specific modules such as tokenization or automatic speech recognition (ASR). It can perform various audiovisual and vision-language tasks like retrieval, question answering, etc.`),Jt=c(),wo=r("p"),jr=n("The abstract from the paper is the following:"),Lt=c(),Mo=r("p"),Qo=r("em"),Ir=n("In this work, we present the Textless Vision-Language Transformer (TVLT), where homogeneous transformer blocks take raw visual and audio inputs for vision-and-language representation learning with minimal modality-specific design, and do not use text-specific modules such as tokenization or automatic speech recognition (ASR). TVLT is trained by reconstructing masked patches of continuous video frames and audio spectrograms (masked autoencoding) and contrastive modeling to align video and audio. TVLT attains performance comparable to its text-based counterpart on various multimodal tasks, such as visual question answering, image retrieval, video retrieval, and multimodal sentiment analysis, with 28x faster inference speed and only 1/3 of the parameters. Our findings suggest the possibility of learning compact and efficient visual-linguistic representations from low-level visual and audio signals without assuming the prior existence of text."),Ut=c(),Po=r("p"),Ar=n("Tips:"),St=c(),V=r("ul"),U=r("li"),Vr=n("TVLT is a model that takes both "),Yo=r("code"),Nr=n("pixel_values"),Wr=n(" and "),Ko=r("code"),Jr=n("audio_values"),Lr=n(" as input. One can use "),ko=r("a"),Ur=n("TvltProcessor"),Sr=n(` to prepare data for the model.
This processor wraps an image processor (for the image/video modality) and an audio feature extractor (for the audio modality) into one.`),Zr=c(),q=r("li"),Rr=n("TVLT is trained with images/videos and audios of various sizes: the authors resize and crop the input images/videos to 224 and limit the length of audio spectrogram to 2048. To make batching of videos and audios possible, the authors use a "),et=r("code"),Br=n("pixel_mask"),Dr=n(" that indicates which pixels are real/padding and "),ot=r("code"),qr=n("audio_mask"),Or=n(" that indicates which audio values are real/padding."),Gr=c(),Le=r("li"),Hr=n("The design of TVLT is very similar to that of a standard Vision Transformer (ViT) and masked autoencoder (MAE) as in "),$o=r("a"),Xr=n("ViTMAE"),Qr=n(". The difference is that the model includes embedding layers for the audio modality."),Yr=c(),tt=r("li"),Kr=n("The PyTorch version of this model is only available in torch 1.10 and higher."),Zt=c(),Ue=r("p"),Se=r("img"),Rt=c(),de=r("small"),ea=n("TVLT architecture. Taken from the "),Fo=r("a"),oa=n("original paper"),ta=n("."),Bt=c(),R=r("p"),ra=n("The original code can be found "),Ze=r("a"),aa=n("here"),sa=n(". This model was contributed by "),Re=r("a"),na=n("Zineng Tang"),ia=n("."),Dt=c(),O=r("h2"),ce=r("a"),rt=r("span"),T(Be.$$.fragment),la=c(),at=r("span"),da=n("TvltConfig"),qt=c(),F=r("div"),T(De.$$.fragment),ca=c(),G=r("p"),ma=n("This is the configuration class to store the configuration of a "),Eo=r("a"),ha=n("TvltModel"),pa=n(`. It is used to instantiate a TVLT
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the TVLT
`),qe=r("a"),fa=n("ZinengTang/tvlt-base"),ua=n(" architecture."),ga=c(),H=r("p"),_a=n("Configuration objects inherit from "),zo=r("a"),va=n("PretrainedConfig"),Ta=n(` and can be used to control the model outputs. Read the
documentation from `),Co=r("a"),ba=n("PretrainedConfig"),ya=n(" for more information."),xa=c(),T(me.$$.fragment),Ot=c(),X=r("h2"),he=r("a"),st=r("span"),T(Oe.$$.fragment),wa=c(),nt=r("span"),Ma=n("TvltProcessor"),Gt=c(),E=r("div"),T(Ge.$$.fragment),Pa=c(),it=r("p"),ka=n("Constructs a TVLT processor which wraps a TVLT image processor and TVLT feature extractor into a single processor."),$a=c(),N=r("p"),jo=r("a"),Fa=n("TvltProcessor"),Ea=n(" offers all the functionalities of "),Io=r("a"),za=n("TvltImageProcessor"),Ca=n(" and "),Ao=r("a"),ja=n("TvltFeatureExtractor"),Ia=n(`. See the
docstring of `),He=r("a"),lt=r("strong"),Aa=n("call"),Va=n("()"),Na=n(" for more information."),Wa=c(),pe=r("div"),T(Xe.$$.fragment),Ja=c(),z=r("p"),La=n("Forwards the "),dt=r("code"),Ua=n("images"),Sa=n(" argument to TvltImageProcessor\u2019s "),Vo=r("a"),Za=n("preprocess()"),Ra=n(" and the "),ct=r("code"),Ba=n("audio"),Da=n(`
argument to TvltFeatureExtractor\u2019s `),Qe=r("a"),mt=r("strong"),qa=n("call"),Oa=n("()"),Ga=n(`. Please refer to the docstring of the
above two methods for more information.`),Ht=c(),Q=r("h2"),fe=r("a"),ht=r("span"),T(Ye.$$.fragment),Ha=c(),pt=r("span"),Xa=n("TvltImageProcessor"),Xt=c(),C=r("div"),T(Ke.$$.fragment),Qa=c(),ft=r("p"),Ya=n("Constructs a TVLT image processor."),Ka=c(),ut=r("p"),es=n("This processor can be used to prepare either videos or images for the model by converting images to 1-frame videos."),os=c(),ue=r("div"),T(eo.$$.fragment),ts=c(),gt=r("p"),rs=n("Preprocess an videos or image or batch of videos or images."),Qt=c(),Y=r("h2"),ge=r("a"),_t=r("span"),T(oo.$$.fragment),as=c(),vt=r("span"),ss=n("TvltFeatureExtractor"),Yt=c(),j=r("div"),T(to.$$.fragment),ns=c(),Tt=r("p"),is=n("Constructs a TVLT audio feature extractor. This feature extractor can be used to prepare audios for the model."),ls=c(),ro=r("p"),ds=n("This feature extractor inherits from "),No=r("a"),cs=n("FeatureExtractionMixin"),ms=n(` which contains most of the main methods. Users
should refer to this superclass for more information regarding those methods.`),hs=c(),_e=r("div"),T(ao.$$.fragment),ps=c(),bt=r("p"),fs=n("Main method to prepare one or several audio(s) for the model."),Kt=c(),K=r("h2"),ve=r("a"),yt=r("span"),T(so.$$.fragment),us=c(),xt=r("span"),gs=n("TvltModel"),er=c(),S=r("div"),T(no.$$.fragment),_s=c(),io=r("p"),vs=n(`The bare TVLT Model transformer outputting raw hidden-states without any specific head on top.
This model is a PyTorch `),lo=r("a"),Ts=n("torch.nn.Module"),bs=n(` subclass. Use it
as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),ys=c(),W=r("div"),T(co.$$.fragment),xs=c(),ee=r("p"),ws=n("The "),Wo=r("a"),Ms=n("TvltModel"),Ps=n(" forward method, overrides the "),wt=r("code"),ks=n("__call__"),$s=n(" special method."),Fs=c(),T(Te.$$.fragment),Es=c(),T(be.$$.fragment),or=c(),oe=r("h2"),ye=r("a"),Mt=r("span"),T(mo.$$.fragment),zs=c(),Pt=r("span"),Cs=n("TvltForPreTraining"),tr=c(),Z=r("div"),T(ho.$$.fragment),js=c(),po=r("p"),Is=n(`The TVLT Model transformer with the decoder on top for self-supervised pre-training.
This model is a PyTorch `),fo=r("a"),As=n("torch.nn.Module"),Vs=n(` subclass. Use it
as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),Ns=c(),J=r("div"),T(uo.$$.fragment),Ws=c(),te=r("p"),Js=n("The "),Jo=r("a"),Ls=n("TvltForPreTraining"),Us=n(" forward method, overrides the "),kt=r("code"),Ss=n("__call__"),Zs=n(" special method."),Rs=c(),T(xe.$$.fragment),Bs=c(),T(we.$$.fragment),rr=c(),re=r("h2"),Me=r("a"),$t=r("span"),T(go.$$.fragment),Ds=c(),Ft=r("span"),qs=n("TvltForAudioVisualClassification"),ar=c(),I=r("div"),T(_o.$$.fragment),Os=c(),Et=r("p"),Gs=n(`Tvlt Model transformer with a classifier head on top (an MLP on top of the final hidden state of the [CLS] token)
for audiovisual classification tasks, e.g. CMU-MOSEI Sentiment Analysis and Audio to Video Retrieval.`),Hs=c(),vo=r("p"),Xs=n("This model is a PyTorch "),To=r("a"),Qs=n("torch.nn.Module"),Ys=n(` subclass. Use it
as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),Ks=c(),L=r("div"),T(bo.$$.fragment),en=c(),ae=r("p"),on=n("The "),Lo=r("a"),tn=n("TvltForAudioVisualClassification"),rn=n(" forward method, overrides the "),zt=r("code"),an=n("__call__"),sn=n(" special method."),nn=c(),T(Pe.$$.fragment),ln=c(),T(ke.$$.fragment),this.h()},l(o){const h=Ei('[data-svelte="svelte-1phssyn"]',document.head);p=a(h,"META",{name:!0,content:!0}),h.forEach(t),P=m(o),_=a(o,"H1",{class:!0});var yo=s(_);u=a(yo,"A",{id:!0,class:!0,href:!0});var Ct=s(u);v=a(Ct,"SPAN",{});var jt=s(v);b(l.$$.fragment,jt),jt.forEach(t),Ct.forEach(t),g=m(yo),$=a(yo,"SPAN",{});var It=s($);kr=i(It,"TVLT"),It.forEach(t),yo.forEach(t),Nt=m(o),D=a(o,"H2",{class:!0});var xo=s(D);ie=a(xo,"A",{id:!0,class:!0,href:!0});var At=s(ie);Ho=a(At,"SPAN",{});var Vt=s(Ho);b(We.$$.fragment,Vt),Vt.forEach(t),At.forEach(t),$r=m(xo),Xo=a(xo,"SPAN",{});var pn=s(Xo);Fr=i(pn,"Overview"),pn.forEach(t),xo.forEach(t),Wt=m(o),le=a(o,"P",{});var nr=s(le);Er=i(nr,"The TVLT model was proposed in "),Je=a(nr,"A",{href:!0,rel:!0});var fn=s(Je);zr=i(fn,"TVLT: Textless Vision-Language Transformer"),fn.forEach(t),Cr=i(nr,`
by Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal (the first three authors contributed equally). The Textless Vision-Language Transformer (TVLT) is a model that uses raw visual and audio inputs for vision-and-language representation learning, without using text-specific modules such as tokenization or automatic speech recognition (ASR). It can perform various audiovisual and vision-language tasks like retrieval, question answering, etc.`),nr.forEach(t),Jt=m(o),wo=a(o,"P",{});var un=s(wo);jr=i(un,"The abstract from the paper is the following:"),un.forEach(t),Lt=m(o),Mo=a(o,"P",{});var gn=s(Mo);Qo=a(gn,"EM",{});var _n=s(Qo);Ir=i(_n,"In this work, we present the Textless Vision-Language Transformer (TVLT), where homogeneous transformer blocks take raw visual and audio inputs for vision-and-language representation learning with minimal modality-specific design, and do not use text-specific modules such as tokenization or automatic speech recognition (ASR). TVLT is trained by reconstructing masked patches of continuous video frames and audio spectrograms (masked autoencoding) and contrastive modeling to align video and audio. TVLT attains performance comparable to its text-based counterpart on various multimodal tasks, such as visual question answering, image retrieval, video retrieval, and multimodal sentiment analysis, with 28x faster inference speed and only 1/3 of the parameters. Our findings suggest the possibility of learning compact and efficient visual-linguistic representations from low-level visual and audio signals without assuming the prior existence of text."),_n.forEach(t),gn.forEach(t),Ut=m(o),Po=a(o,"P",{});var vn=s(Po);Ar=i(vn,"Tips:"),vn.forEach(t),St=m(o),V=a(o,"UL",{});var $e=s(V);U=a($e,"LI",{});var Fe=s(U);Vr=i(Fe,"TVLT is a model that takes both "),Yo=a(Fe,"CODE",{});var Tn=s(Yo);Nr=i(Tn,"pixel_values"),Tn.forEach(t),Wr=i(Fe," and "),Ko=a(Fe,"CODE",{});var bn=s(Ko);Jr=i(bn,"audio_values"),bn.forEach(t),Lr=i(Fe," as input. One can use "),ko=a(Fe,"A",{href:!0});var yn=s(ko);Ur=i(yn,"TvltProcessor"),yn.forEach(t),Sr=i(Fe,` to prepare data for the model.
This processor wraps an image processor (for the image/video modality) and an audio feature extractor (for the audio modality) into one.`),Fe.forEach(t),Zr=m($e),q=a($e,"LI",{});var Uo=s(q);Rr=i(Uo,"TVLT is trained with images/videos and audios of various sizes: the authors resize and crop the input images/videos to 224 and limit the length of audio spectrogram to 2048. To make batching of videos and audios possible, the authors use a "),et=a(Uo,"CODE",{});var xn=s(et);Br=i(xn,"pixel_mask"),xn.forEach(t),Dr=i(Uo," that indicates which pixels are real/padding and "),ot=a(Uo,"CODE",{});var wn=s(ot);qr=i(wn,"audio_mask"),wn.forEach(t),Or=i(Uo," that indicates which audio values are real/padding."),Uo.forEach(t),Gr=m($e),Le=a($e,"LI",{});var ir=s(Le);Hr=i(ir,"The design of TVLT is very similar to that of a standard Vision Transformer (ViT) and masked autoencoder (MAE) as in "),$o=a(ir,"A",{href:!0});var Mn=s($o);Xr=i(Mn,"ViTMAE"),Mn.forEach(t),Qr=i(ir,". The difference is that the model includes embedding layers for the audio modality."),ir.forEach(t),Yr=m($e),tt=a($e,"LI",{});var Pn=s(tt);Kr=i(Pn,"The PyTorch version of this model is only available in torch 1.10 and higher."),Pn.forEach(t),$e.forEach(t),Zt=m(o),Ue=a(o,"P",{align:!0});var kn=s(Ue);Se=a(kn,"IMG",{src:!0,alt:!0,width:!0}),kn.forEach(t),Rt=m(o),de=a(o,"SMALL",{});var lr=s(de);ea=i(lr,"TVLT architecture. Taken from the "),Fo=a(lr,"A",{href:!0});var $n=s(Fo);oa=i($n,"original paper"),$n.forEach(t),ta=i(lr,"."),lr.forEach(t),Bt=m(o),R=a(o,"P",{});var So=s(R);ra=i(So,"The original code can be found "),Ze=a(So,"A",{href:!0,rel:!0});var Fn=s(Ze);aa=i(Fn,"here"),Fn.forEach(t),sa=i(So,". This model was contributed by "),Re=a(So,"A",{href:!0,rel:!0});var En=s(Re);na=i(En,"Zineng Tang"),En.forEach(t),ia=i(So,"."),So.forEach(t),Dt=m(o),O=a(o,"H2",{class:!0});var dr=s(O);ce=a(dr,"A",{id:!0,class:!0,href:!0});var zn=s(ce);rt=a(zn,"SPAN",{});var Cn=s(rt);b(Be.$$.fragment,Cn),Cn.forEach(t),zn.forEach(t),la=m(dr),at=a(dr,"SPAN",{});var jn=s(at);da=i(jn,"TvltConfig"),jn.forEach(t),dr.forEach(t),qt=m(o),F=a(o,"DIV",{class:!0});var Ee=s(F);b(De.$$.fragment,Ee),ca=m(Ee),G=a(Ee,"P",{});var Zo=s(G);ma=i(Zo,"This is the configuration class to store the configuration of a "),Eo=a(Zo,"A",{href:!0});var In=s(Eo);ha=i(In,"TvltModel"),In.forEach(t),pa=i(Zo,`. It is used to instantiate a TVLT
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the TVLT
`),qe=a(Zo,"A",{href:!0,rel:!0});var An=s(qe);fa=i(An,"ZinengTang/tvlt-base"),An.forEach(t),ua=i(Zo," architecture."),Zo.forEach(t),ga=m(Ee),H=a(Ee,"P",{});var Ro=s(H);_a=i(Ro,"Configuration objects inherit from "),zo=a(Ro,"A",{href:!0});var Vn=s(zo);va=i(Vn,"PretrainedConfig"),Vn.forEach(t),Ta=i(Ro,` and can be used to control the model outputs. Read the
documentation from `),Co=a(Ro,"A",{href:!0});var Nn=s(Co);ba=i(Nn,"PretrainedConfig"),Nn.forEach(t),ya=i(Ro," for more information."),Ro.forEach(t),xa=m(Ee),b(me.$$.fragment,Ee),Ee.forEach(t),Ot=m(o),X=a(o,"H2",{class:!0});var cr=s(X);he=a(cr,"A",{id:!0,class:!0,href:!0});var Wn=s(he);st=a(Wn,"SPAN",{});var Jn=s(st);b(Oe.$$.fragment,Jn),Jn.forEach(t),Wn.forEach(t),wa=m(cr),nt=a(cr,"SPAN",{});var Ln=s(nt);Ma=i(Ln,"TvltProcessor"),Ln.forEach(t),cr.forEach(t),Gt=m(o),E=a(o,"DIV",{class:!0});var ze=s(E);b(Ge.$$.fragment,ze),Pa=m(ze),it=a(ze,"P",{});var Un=s(it);ka=i(Un,"Constructs a TVLT processor which wraps a TVLT image processor and TVLT feature extractor into a single processor."),Un.forEach(t),$a=m(ze),N=a(ze,"P",{});var se=s(N);jo=a(se,"A",{href:!0});var Sn=s(jo);Fa=i(Sn,"TvltProcessor"),Sn.forEach(t),Ea=i(se," offers all the functionalities of "),Io=a(se,"A",{href:!0});var Zn=s(Io);za=i(Zn,"TvltImageProcessor"),Zn.forEach(t),Ca=i(se," and "),Ao=a(se,"A",{href:!0});var Rn=s(Ao);ja=i(Rn,"TvltFeatureExtractor"),Rn.forEach(t),Ia=i(se,`. See the
docstring of `),He=a(se,"A",{href:!0});var dn=s(He);lt=a(dn,"STRONG",{});var Bn=s(lt);Aa=i(Bn,"call"),Bn.forEach(t),Va=i(dn,"()"),dn.forEach(t),Na=i(se," for more information."),se.forEach(t),Wa=m(ze),pe=a(ze,"DIV",{class:!0});var mr=s(pe);b(Xe.$$.fragment,mr),Ja=m(mr),z=a(mr,"P",{});var B=s(z);La=i(B,"Forwards the "),dt=a(B,"CODE",{});var Dn=s(dt);Ua=i(Dn,"images"),Dn.forEach(t),Sa=i(B," argument to TvltImageProcessor\u2019s "),Vo=a(B,"A",{href:!0});var qn=s(Vo);Za=i(qn,"preprocess()"),qn.forEach(t),Ra=i(B," and the "),ct=a(B,"CODE",{});var On=s(ct);Ba=i(On,"audio"),On.forEach(t),Da=i(B,`
argument to TvltFeatureExtractor\u2019s `),Qe=a(B,"A",{href:!0});var cn=s(Qe);mt=a(cn,"STRONG",{});var Gn=s(mt);qa=i(Gn,"call"),Gn.forEach(t),Oa=i(cn,"()"),cn.forEach(t),Ga=i(B,`. Please refer to the docstring of the
above two methods for more information.`),B.forEach(t),mr.forEach(t),ze.forEach(t),Ht=m(o),Q=a(o,"H2",{class:!0});var hr=s(Q);fe=a(hr,"A",{id:!0,class:!0,href:!0});var Hn=s(fe);ht=a(Hn,"SPAN",{});var Xn=s(ht);b(Ye.$$.fragment,Xn),Xn.forEach(t),Hn.forEach(t),Ha=m(hr),pt=a(hr,"SPAN",{});var Qn=s(pt);Xa=i(Qn,"TvltImageProcessor"),Qn.forEach(t),hr.forEach(t),Xt=m(o),C=a(o,"DIV",{class:!0});var Ce=s(C);b(Ke.$$.fragment,Ce),Qa=m(Ce),ft=a(Ce,"P",{});var Yn=s(ft);Ya=i(Yn,"Constructs a TVLT image processor."),Yn.forEach(t),Ka=m(Ce),ut=a(Ce,"P",{});var Kn=s(ut);es=i(Kn,"This processor can be used to prepare either videos or images for the model by converting images to 1-frame videos."),Kn.forEach(t),os=m(Ce),ue=a(Ce,"DIV",{class:!0});var pr=s(ue);b(eo.$$.fragment,pr),ts=m(pr),gt=a(pr,"P",{});var ei=s(gt);rs=i(ei,"Preprocess an videos or image or batch of videos or images."),ei.forEach(t),pr.forEach(t),Ce.forEach(t),Qt=m(o),Y=a(o,"H2",{class:!0});var fr=s(Y);ge=a(fr,"A",{id:!0,class:!0,href:!0});var oi=s(ge);_t=a(oi,"SPAN",{});var ti=s(_t);b(oo.$$.fragment,ti),ti.forEach(t),oi.forEach(t),as=m(fr),vt=a(fr,"SPAN",{});var ri=s(vt);ss=i(ri,"TvltFeatureExtractor"),ri.forEach(t),fr.forEach(t),Yt=m(o),j=a(o,"DIV",{class:!0});var je=s(j);b(to.$$.fragment,je),ns=m(je),Tt=a(je,"P",{});var ai=s(Tt);is=i(ai,"Constructs a TVLT audio feature extractor. This feature extractor can be used to prepare audios for the model."),ai.forEach(t),ls=m(je),ro=a(je,"P",{});var ur=s(ro);ds=i(ur,"This feature extractor inherits from "),No=a(ur,"A",{href:!0});var si=s(No);cs=i(si,"FeatureExtractionMixin"),si.forEach(t),ms=i(ur,` which contains most of the main methods. Users
should refer to this superclass for more information regarding those methods.`),ur.forEach(t),hs=m(je),_e=a(je,"DIV",{class:!0});var gr=s(_e);b(ao.$$.fragment,gr),ps=m(gr),bt=a(gr,"P",{});var ni=s(bt);fs=i(ni,"Main method to prepare one or several audio(s) for the model."),ni.forEach(t),gr.forEach(t),je.forEach(t),Kt=m(o),K=a(o,"H2",{class:!0});var _r=s(K);ve=a(_r,"A",{id:!0,class:!0,href:!0});var ii=s(ve);yt=a(ii,"SPAN",{});var li=s(yt);b(so.$$.fragment,li),li.forEach(t),ii.forEach(t),us=m(_r),xt=a(_r,"SPAN",{});var di=s(xt);gs=i(di,"TvltModel"),di.forEach(t),_r.forEach(t),er=m(o),S=a(o,"DIV",{class:!0});var Bo=s(S);b(no.$$.fragment,Bo),_s=m(Bo),io=a(Bo,"P",{});var vr=s(io);vs=i(vr,`The bare TVLT Model transformer outputting raw hidden-states without any specific head on top.
This model is a PyTorch `),lo=a(vr,"A",{href:!0,rel:!0});var ci=s(lo);Ts=i(ci,"torch.nn.Module"),ci.forEach(t),bs=i(vr,` subclass. Use it
as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),vr.forEach(t),ys=m(Bo),W=a(Bo,"DIV",{class:!0});var Ie=s(W);b(co.$$.fragment,Ie),xs=m(Ie),ee=a(Ie,"P",{});var Do=s(ee);ws=i(Do,"The "),Wo=a(Do,"A",{href:!0});var mi=s(Wo);Ms=i(mi,"TvltModel"),mi.forEach(t),Ps=i(Do," forward method, overrides the "),wt=a(Do,"CODE",{});var hi=s(wt);ks=i(hi,"__call__"),hi.forEach(t),$s=i(Do," special method."),Do.forEach(t),Fs=m(Ie),b(Te.$$.fragment,Ie),Es=m(Ie),b(be.$$.fragment,Ie),Ie.forEach(t),Bo.forEach(t),or=m(o),oe=a(o,"H2",{class:!0});var Tr=s(oe);ye=a(Tr,"A",{id:!0,class:!0,href:!0});var pi=s(ye);Mt=a(pi,"SPAN",{});var fi=s(Mt);b(mo.$$.fragment,fi),fi.forEach(t),pi.forEach(t),zs=m(Tr),Pt=a(Tr,"SPAN",{});var ui=s(Pt);Cs=i(ui,"TvltForPreTraining"),ui.forEach(t),Tr.forEach(t),tr=m(o),Z=a(o,"DIV",{class:!0});var qo=s(Z);b(ho.$$.fragment,qo),js=m(qo),po=a(qo,"P",{});var br=s(po);Is=i(br,`The TVLT Model transformer with the decoder on top for self-supervised pre-training.
This model is a PyTorch `),fo=a(br,"A",{href:!0,rel:!0});var gi=s(fo);As=i(gi,"torch.nn.Module"),gi.forEach(t),Vs=i(br,` subclass. Use it
as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),br.forEach(t),Ns=m(qo),J=a(qo,"DIV",{class:!0});var Ae=s(J);b(uo.$$.fragment,Ae),Ws=m(Ae),te=a(Ae,"P",{});var Oo=s(te);Js=i(Oo,"The "),Jo=a(Oo,"A",{href:!0});var _i=s(Jo);Ls=i(_i,"TvltForPreTraining"),_i.forEach(t),Us=i(Oo," forward method, overrides the "),kt=a(Oo,"CODE",{});var vi=s(kt);Ss=i(vi,"__call__"),vi.forEach(t),Zs=i(Oo," special method."),Oo.forEach(t),Rs=m(Ae),b(xe.$$.fragment,Ae),Bs=m(Ae),b(we.$$.fragment,Ae),Ae.forEach(t),qo.forEach(t),rr=m(o),re=a(o,"H2",{class:!0});var yr=s(re);Me=a(yr,"A",{id:!0,class:!0,href:!0});var Ti=s(Me);$t=a(Ti,"SPAN",{});var bi=s($t);b(go.$$.fragment,bi),bi.forEach(t),Ti.forEach(t),Ds=m(yr),Ft=a(yr,"SPAN",{});var yi=s(Ft);qs=i(yi,"TvltForAudioVisualClassification"),yi.forEach(t),yr.forEach(t),ar=m(o),I=a(o,"DIV",{class:!0});var Ve=s(I);b(_o.$$.fragment,Ve),Os=m(Ve),Et=a(Ve,"P",{});var xi=s(Et);Gs=i(xi,`Tvlt Model transformer with a classifier head on top (an MLP on top of the final hidden state of the [CLS] token)
for audiovisual classification tasks, e.g. CMU-MOSEI Sentiment Analysis and Audio to Video Retrieval.`),xi.forEach(t),Hs=m(Ve),vo=a(Ve,"P",{});var xr=s(vo);Xs=i(xr,"This model is a PyTorch "),To=a(xr,"A",{href:!0,rel:!0});var wi=s(To);Qs=i(wi,"torch.nn.Module"),wi.forEach(t),Ys=i(xr,` subclass. Use it
as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),xr.forEach(t),Ks=m(Ve),L=a(Ve,"DIV",{class:!0});var Ne=s(L);b(bo.$$.fragment,Ne),en=m(Ne),ae=a(Ne,"P",{});var Go=s(ae);on=i(Go,"The "),Lo=a(Go,"A",{href:!0});var Mi=s(Lo);tn=i(Mi,"TvltForAudioVisualClassification"),Mi.forEach(t),rn=i(Go," forward method, overrides the "),zt=a(Go,"CODE",{});var Pi=s(zt);an=i(Pi,"__call__"),Pi.forEach(t),sn=i(Go," special method."),Go.forEach(t),nn=m(Ne),b(Pe.$$.fragment,Ne),ln=m(Ne),b(ke.$$.fragment,Ne),Ne.forEach(t),Ve.forEach(t),this.h()},h(){d(p,"name","hf:doc:metadata"),d(p,"content",JSON.stringify(Ui)),d(u,"id","tvlt"),d(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(u,"href","#tvlt"),d(_,"class","relative group"),d(ie,"id","overview"),d(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ie,"href","#overview"),d(D,"class","relative group"),d(Je,"href","https://arxiv.org/abs/2209.14156"),d(Je,"rel","nofollow"),d(ko,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor"),d($o,"href","vitmae"),zi(Se.src,hn="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/tvlt_architecture.png")||d(Se,"src",hn),d(Se,"alt","drawing"),d(Se,"width","600"),d(Ue,"align","center"),d(Fo,"href","[https://arxiv.org/abs/2102.03334](https://arxiv.org/abs/2209.14156)"),d(Ze,"href","https://github.com/zinengtang/TVLT"),d(Ze,"rel","nofollow"),d(Re,"href","https://huggingface.co/ZinengTang"),d(Re,"rel","nofollow"),d(ce,"id","transformers.TvltConfig"),d(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ce,"href","#transformers.TvltConfig"),d(O,"class","relative group"),d(Eo,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltModel"),d(qe,"href","https://huggingface.co/ZinengTang/tvlt-base"),d(qe,"rel","nofollow"),d(zo,"href","/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig"),d(Co,"href","/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig"),d(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(he,"id","transformers.TvltProcessor"),d(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(he,"href","#transformers.TvltProcessor"),d(X,"class","relative group"),d(jo,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor"),d(Io,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltImageProcessor"),d(Ao,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltFeatureExtractor"),d(He,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltProcessor.__call__"),d(Vo,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltImageProcessor.preprocess"),d(Qe,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltFeatureExtractor.__call__"),d(pe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(fe,"id","transformers.TvltImageProcessor"),d(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(fe,"href","#transformers.TvltImageProcessor"),d(Q,"class","relative group"),d(ue,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(ge,"id","transformers.TvltFeatureExtractor"),d(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ge,"href","#transformers.TvltFeatureExtractor"),d(Y,"class","relative group"),d(No,"href","/docs/transformers/main/en/main_classes/feature_extractor#transformers.FeatureExtractionMixin"),d(_e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(ve,"id","transformers.TvltModel"),d(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ve,"href","#transformers.TvltModel"),d(K,"class","relative group"),d(lo,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),d(lo,"rel","nofollow"),d(Wo,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltModel"),d(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(ye,"id","transformers.TvltForPreTraining"),d(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ye,"href","#transformers.TvltForPreTraining"),d(oe,"class","relative group"),d(fo,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),d(fo,"rel","nofollow"),d(Jo,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltForPreTraining"),d(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(Me,"id","transformers.TvltForAudioVisualClassification"),d(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Me,"href","#transformers.TvltForAudioVisualClassification"),d(re,"class","relative group"),d(To,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),d(To,"rel","nofollow"),d(Lo,"href","/docs/transformers/main/en/model_doc/tvlt#transformers.TvltForAudioVisualClassification"),d(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),d(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(o,h){e(document.head,p),f(o,P,h),f(o,_,h),e(_,u),e(u,v),y(l,v,null),e(_,g),e(_,$),e($,kr),f(o,Nt,h),f(o,D,h),e(D,ie),e(ie,Ho),y(We,Ho,null),e(D,$r),e(D,Xo),e(Xo,Fr),f(o,Wt,h),f(o,le,h),e(le,Er),e(le,Je),e(Je,zr),e(le,Cr),f(o,Jt,h),f(o,wo,h),e(wo,jr),f(o,Lt,h),f(o,Mo,h),e(Mo,Qo),e(Qo,Ir),f(o,Ut,h),f(o,Po,h),e(Po,Ar),f(o,St,h),f(o,V,h),e(V,U),e(U,Vr),e(U,Yo),e(Yo,Nr),e(U,Wr),e(U,Ko),e(Ko,Jr),e(U,Lr),e(U,ko),e(ko,Ur),e(U,Sr),e(V,Zr),e(V,q),e(q,Rr),e(q,et),e(et,Br),e(q,Dr),e(q,ot),e(ot,qr),e(q,Or),e(V,Gr),e(V,Le),e(Le,Hr),e(Le,$o),e($o,Xr),e(Le,Qr),e(V,Yr),e(V,tt),e(tt,Kr),f(o,Zt,h),f(o,Ue,h),e(Ue,Se),f(o,Rt,h),f(o,de,h),e(de,ea),e(de,Fo),e(Fo,oa),e(de,ta),f(o,Bt,h),f(o,R,h),e(R,ra),e(R,Ze),e(Ze,aa),e(R,sa),e(R,Re),e(Re,na),e(R,ia),f(o,Dt,h),f(o,O,h),e(O,ce),e(ce,rt),y(Be,rt,null),e(O,la),e(O,at),e(at,da),f(o,qt,h),f(o,F,h),y(De,F,null),e(F,ca),e(F,G),e(G,ma),e(G,Eo),e(Eo,ha),e(G,pa),e(G,qe),e(qe,fa),e(G,ua),e(F,ga),e(F,H),e(H,_a),e(H,zo),e(zo,va),e(H,Ta),e(H,Co),e(Co,ba),e(H,ya),e(F,xa),y(me,F,null),f(o,Ot,h),f(o,X,h),e(X,he),e(he,st),y(Oe,st,null),e(X,wa),e(X,nt),e(nt,Ma),f(o,Gt,h),f(o,E,h),y(Ge,E,null),e(E,Pa),e(E,it),e(it,ka),e(E,$a),e(E,N),e(N,jo),e(jo,Fa),e(N,Ea),e(N,Io),e(Io,za),e(N,Ca),e(N,Ao),e(Ao,ja),e(N,Ia),e(N,He),e(He,lt),e(lt,Aa),e(He,Va),e(N,Na),e(E,Wa),e(E,pe),y(Xe,pe,null),e(pe,Ja),e(pe,z),e(z,La),e(z,dt),e(dt,Ua),e(z,Sa),e(z,Vo),e(Vo,Za),e(z,Ra),e(z,ct),e(ct,Ba),e(z,Da),e(z,Qe),e(Qe,mt),e(mt,qa),e(Qe,Oa),e(z,Ga),f(o,Ht,h),f(o,Q,h),e(Q,fe),e(fe,ht),y(Ye,ht,null),e(Q,Ha),e(Q,pt),e(pt,Xa),f(o,Xt,h),f(o,C,h),y(Ke,C,null),e(C,Qa),e(C,ft),e(ft,Ya),e(C,Ka),e(C,ut),e(ut,es),e(C,os),e(C,ue),y(eo,ue,null),e(ue,ts),e(ue,gt),e(gt,rs),f(o,Qt,h),f(o,Y,h),e(Y,ge),e(ge,_t),y(oo,_t,null),e(Y,as),e(Y,vt),e(vt,ss),f(o,Yt,h),f(o,j,h),y(to,j,null),e(j,ns),e(j,Tt),e(Tt,is),e(j,ls),e(j,ro),e(ro,ds),e(ro,No),e(No,cs),e(ro,ms),e(j,hs),e(j,_e),y(ao,_e,null),e(_e,ps),e(_e,bt),e(bt,fs),f(o,Kt,h),f(o,K,h),e(K,ve),e(ve,yt),y(so,yt,null),e(K,us),e(K,xt),e(xt,gs),f(o,er,h),f(o,S,h),y(no,S,null),e(S,_s),e(S,io),e(io,vs),e(io,lo),e(lo,Ts),e(io,bs),e(S,ys),e(S,W),y(co,W,null),e(W,xs),e(W,ee),e(ee,ws),e(ee,Wo),e(Wo,Ms),e(ee,Ps),e(ee,wt),e(wt,ks),e(ee,$s),e(W,Fs),y(Te,W,null),e(W,Es),y(be,W,null),f(o,or,h),f(o,oe,h),e(oe,ye),e(ye,Mt),y(mo,Mt,null),e(oe,zs),e(oe,Pt),e(Pt,Cs),f(o,tr,h),f(o,Z,h),y(ho,Z,null),e(Z,js),e(Z,po),e(po,Is),e(po,fo),e(fo,As),e(po,Vs),e(Z,Ns),e(Z,J),y(uo,J,null),e(J,Ws),e(J,te),e(te,Js),e(te,Jo),e(Jo,Ls),e(te,Us),e(te,kt),e(kt,Ss),e(te,Zs),e(J,Rs),y(xe,J,null),e(J,Bs),y(we,J,null),f(o,rr,h),f(o,re,h),e(re,Me),e(Me,$t),y(go,$t,null),e(re,Ds),e(re,Ft),e(Ft,qs),f(o,ar,h),f(o,I,h),y(_o,I,null),e(I,Os),e(I,Et),e(Et,Gs),e(I,Hs),e(I,vo),e(vo,Xs),e(vo,To),e(To,Qs),e(vo,Ys),e(I,Ks),e(I,L),y(bo,L,null),e(L,en),e(L,ae),e(ae,on),e(ae,Lo),e(Lo,tn),e(ae,rn),e(ae,zt),e(zt,an),e(ae,sn),e(L,nn),y(Pe,L,null),e(L,ln),y(ke,L,null),sr=!0},p(o,[h]){const yo={};h&2&&(yo.$$scope={dirty:h,ctx:o}),me.$set(yo);const Ct={};h&2&&(Ct.$$scope={dirty:h,ctx:o}),Te.$set(Ct);const jt={};h&2&&(jt.$$scope={dirty:h,ctx:o}),be.$set(jt);const It={};h&2&&(It.$$scope={dirty:h,ctx:o}),xe.$set(It);const xo={};h&2&&(xo.$$scope={dirty:h,ctx:o}),we.$set(xo);const At={};h&2&&(At.$$scope={dirty:h,ctx:o}),Pe.$set(At);const Vt={};h&2&&(Vt.$$scope={dirty:h,ctx:o}),ke.$set(Vt)},i(o){sr||(x(l.$$.fragment,o),x(We.$$.fragment,o),x(Be.$$.fragment,o),x(De.$$.fragment,o),x(me.$$.fragment,o),x(Oe.$$.fragment,o),x(Ge.$$.fragment,o),x(Xe.$$.fragment,o),x(Ye.$$.fragment,o),x(Ke.$$.fragment,o),x(eo.$$.fragment,o),x(oo.$$.fragment,o),x(to.$$.fragment,o),x(ao.$$.fragment,o),x(so.$$.fragment,o),x(no.$$.fragment,o),x(co.$$.fragment,o),x(Te.$$.fragment,o),x(be.$$.fragment,o),x(mo.$$.fragment,o),x(ho.$$.fragment,o),x(uo.$$.fragment,o),x(xe.$$.fragment,o),x(we.$$.fragment,o),x(go.$$.fragment,o),x(_o.$$.fragment,o),x(bo.$$.fragment,o),x(Pe.$$.fragment,o),x(ke.$$.fragment,o),sr=!0)},o(o){w(l.$$.fragment,o),w(We.$$.fragment,o),w(Be.$$.fragment,o),w(De.$$.fragment,o),w(me.$$.fragment,o),w(Oe.$$.fragment,o),w(Ge.$$.fragment,o),w(Xe.$$.fragment,o),w(Ye.$$.fragment,o),w(Ke.$$.fragment,o),w(eo.$$.fragment,o),w(oo.$$.fragment,o),w(to.$$.fragment,o),w(ao.$$.fragment,o),w(so.$$.fragment,o),w(no.$$.fragment,o),w(co.$$.fragment,o),w(Te.$$.fragment,o),w(be.$$.fragment,o),w(mo.$$.fragment,o),w(ho.$$.fragment,o),w(uo.$$.fragment,o),w(xe.$$.fragment,o),w(we.$$.fragment,o),w(go.$$.fragment,o),w(_o.$$.fragment,o),w(bo.$$.fragment,o),w(Pe.$$.fragment,o),w(ke.$$.fragment,o),sr=!1},d(o){t(p),o&&t(P),o&&t(_),M(l),o&&t(Nt),o&&t(D),M(We),o&&t(Wt),o&&t(le),o&&t(Jt),o&&t(wo),o&&t(Lt),o&&t(Mo),o&&t(Ut),o&&t(Po),o&&t(St),o&&t(V),o&&t(Zt),o&&t(Ue),o&&t(Rt),o&&t(de),o&&t(Bt),o&&t(R),o&&t(Dt),o&&t(O),M(Be),o&&t(qt),o&&t(F),M(De),M(me),o&&t(Ot),o&&t(X),M(Oe),o&&t(Gt),o&&t(E),M(Ge),M(Xe),o&&t(Ht),o&&t(Q),M(Ye),o&&t(Xt),o&&t(C),M(Ke),M(eo),o&&t(Qt),o&&t(Y),M(oo),o&&t(Yt),o&&t(j),M(to),M(ao),o&&t(Kt),o&&t(K),M(so),o&&t(er),o&&t(S),M(no),M(co),M(Te),M(be),o&&t(or),o&&t(oe),M(mo),o&&t(tr),o&&t(Z),M(ho),M(uo),M(xe),M(we),o&&t(rr),o&&t(re),M(go),o&&t(ar),o&&t(I),M(_o),M(bo),M(Pe),M(ke)}}}const Ui={local:"tvlt",sections:[{local:"overview",title:"Overview"},{local:"transformers.TvltConfig",title:"TvltConfig"},{local:"transformers.TvltProcessor",title:"TvltProcessor"},{local:"transformers.TvltImageProcessor",title:"TvltImageProcessor"},{local:"transformers.TvltFeatureExtractor",title:"TvltFeatureExtractor"},{local:"transformers.TvltModel",title:"TvltModel"},{local:"transformers.TvltForPreTraining",title:"TvltForPreTraining"},{local:"transformers.TvltForAudioVisualClassification",title:"TvltForAudioVisualClassification"}],title:"TVLT"};function Si(k){return Ci(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Gi extends ki{constructor(p){super();$i(this,p,Si,Li,Fi,{})}}export{Gi as default,Ui as metadata};
