import{S as kp,i as xp,s as $p,e as o,k as h,w as d,t as l,M as Vp,c as r,d as s,m as c,a as i,x as u,h as n,b as m,N as Zp,G as a,g as p,y as f,L as Ip,q as g,o as y,B as w,v as Gp}from"../../chunks/vendor-hf-doc-builder.js";import{I as Oe}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as b}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Wp}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Cp(Br){let Z,Xa,I,F,Ps,pe,Gl,Ss,Wl,Fa,he,Ra,es,Cl,Na,ss,Al,qa,M,Ds,Bl,Ql,Ks,Xl,Fl,Os,Rl,Nl,ea,ql,za,as,zl,Ya,T,G,Yl,ts,Hl,Ll,R,sa,Pl,Sl,Dl,Kl,aa,Ol,en,ta,sn,Ha,W,N,la,ce,an,na,tn,La,U,ln,oa,nn,on,ra,rn,pn,Pa,ls,hn,Sa,ns,cn,Da,me,Ka,os,mn,Oa,de,et,rs,dn,st,ue,at,C,q,ia,fe,un,pa,fn,tt,E,gn,ha,yn,wn,ge,bn,jn,lt,J,vn,z,ca,Mn,Jn,_n,ye,Tn,Un,we,En,kn,nt,is,xn,ot,be,rt,ps,$n,it,je,pt,hs,Vn,ht,k,cs,ma,Zn,In,Gn,ms,da,Wn,Cn,An,ds,ua,Bn,Qn,ct,us,Xn,mt,ve,dt,x,Fn,fa,Rn,Nn,ga,qn,zn,ut,fs,Yn,ft,Me,gt,Je,gs,Qr,yt,ys,Hn,wt,Y,Ln,ya,Pn,Sn,bt,ws,Dn,jt,_e,vt,bs,Kn,Mt,Te,Jt,A,H,wa,Ue,On,ba,eo,_t,L,so,js,ao,to,Tt,Ee,Ut,j,lo,vs,no,oo,Ms,ro,io,ja,po,ho,va,co,mo,Ma,uo,fo,Js,go,yo,Ja,wo,bo,_a,jo,vo,Et,P,Mo,Ta,Jo,_o,kt,S,To,Ua,Uo,Eo,xt,ke,$t,_,ko,Ea,xo,$o,ka,Vo,Zo,xa,Io,Go,Vt,xe,Zt,D,Wo,_s,Co,Ao,It,$e,Gt,B,K,$a,Ve,Bo,Va,Qo,Wt,O,Xo,Ts,Fo,Ro,Ct,Ze,At,Us,No,Bt,Es,Ie,qo,ks,zo,Yo,Qt,Ge,Xt,We,Ce,Ho,xs,Lo,Po,Ft,Ae,Rt,Be,Qe,So,$s,Do,Ko,Nt,Xe,qt,ee,Oo,Vs,er,sr,zt,Fe,Yt,Q,se,Za,Re,ar,Ia,tr,Ht,ae,lr,Zs,nr,or,Lt,Ne,Pt,Is,rr,St,qe,Dt,Gs,ir,Kt,Ws,pr,Ot,$,Ga,hr,cr,Wa,mr,dr,ze,ur,Ca,fr,gr,el,Ye,sl,X,te,Aa,He,yr,Ba,wr,al,V,br,Cs,jr,vr,Le,Mr,Jr,tl,le,_r,As,Tr,Ur,ll,Pe,nl,Bs,Er,ol,Se,rl,ne,kr,Qa,xr,$r,il,De,pl,Qs,Vr,hl,Ke,cl,Xs,Zr,ml;return pe=new Oe({}),he=new Wp({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/visual_question_answering.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/visual_question_answering.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/visual_question_answering.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/visual_question_answering.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/visual_question_answering.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/visual_question_answering.ipynb"}]}}),ce=new Oe({}),me=new b({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1xJTIwdHJhbnNmb3JtZXJzJTIwZGF0YXNldHM=",highlighted:"pip install -q transformers datasets"}}),de=new b({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_login()`}}),ue=new b({props:{code:"bW9kZWxfY2hlY2twb2ludCUyMCUzRCUyMCUyMmRhbmRlbGluJTJGdmlsdC1iMzItbWxtJTIy",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model_checkpoint = <span class="hljs-string">&quot;dandelin/vilt-b32-mlm&quot;</span>'}}),fe=new Oe({}),be=new b({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJHcmFwaGNvcmUlMkZ2cWElMjIlMkMlMjBzcGxpdCUzRCUyMnZhbGlkYXRpb24lNUIlM0EyMDAlNUQlMjIpJTBBZGF0YXNldA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;Graphcore/vqa&quot;</span>, split=<span class="hljs-string">&quot;validation[:200]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset
Dataset({
    features: [<span class="hljs-string">&#x27;question&#x27;</span>, <span class="hljs-string">&#x27;question_type&#x27;</span>, <span class="hljs-string">&#x27;question_id&#x27;</span>, <span class="hljs-string">&#x27;image_id&#x27;</span>, <span class="hljs-string">&#x27;answer_type&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>],
    num_rows: <span class="hljs-number">200</span>
})`}}),je=new b({props:{code:"ZGF0YXNldCU1QjAlNUQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;question&#x27;</span>: <span class="hljs-string">&#x27;Where is he looking?&#x27;</span>,
 <span class="hljs-string">&#x27;question_type&#x27;</span>: <span class="hljs-string">&#x27;none of the above&#x27;</span>,
 <span class="hljs-string">&#x27;question_id&#x27;</span>: <span class="hljs-number">262148000</span>,
 <span class="hljs-string">&#x27;image_id&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/ca733e0e000fb2d7a09fbcc94dbfe7b5a30750681d0e965f8e0a23b1c2f98c75/val2014/COCO_val2014_000000262148.jpg&#x27;</span>,
 <span class="hljs-string">&#x27;answer_type&#x27;</span>: <span class="hljs-string">&#x27;other&#x27;</span>,
 <span class="hljs-string">&#x27;label&#x27;</span>: {<span class="hljs-string">&#x27;ids&#x27;</span>: [<span class="hljs-string">&#x27;at table&#x27;</span>, <span class="hljs-string">&#x27;down&#x27;</span>, <span class="hljs-string">&#x27;skateboard&#x27;</span>, <span class="hljs-string">&#x27;table&#x27;</span>],
  <span class="hljs-string">&#x27;weights&#x27;</span>: [<span class="hljs-number">0.30000001192092896</span>,
   <span class="hljs-number">1.0</span>,
   <span class="hljs-number">0.30000001192092896</span>,
   <span class="hljs-number">0.30000001192092896</span>]}}`}}),ve=new b({props:{code:"ZGF0YXNldCUyMCUzRCUyMGRhdGFzZXQucmVtb3ZlX2NvbHVtbnMoJTVCJ3F1ZXN0aW9uX3R5cGUnJTJDJTIwJ3F1ZXN0aW9uX2lkJyUyQyUyMCdhbnN3ZXJfdHlwZSclNUQp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.remove_columns([<span class="hljs-string">&#x27;question_type&#x27;</span>, <span class="hljs-string">&#x27;question_id&#x27;</span>, <span class="hljs-string">&#x27;answer_type&#x27;</span>])'}}),Me=new b({props:{code:"ZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKGRhdGFzZXQlNUIwJTVEJTVCJ2ltYWdlX2lkJyU1RCklMEFpbWFnZQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;image_id&#x27;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>image`}}),_e=new b({props:{code:"aW1wb3J0JTIwaXRlcnRvb2xzJTBBJTBBbGFiZWxzJTIwJTNEJTIwJTVCaXRlbSU1QidpZHMnJTVEJTIwZm9yJTIwaXRlbSUyMGluJTIwZGF0YXNldCU1QidsYWJlbCclNUQlNUQlMEFmbGF0dGVuZWRfbGFiZWxzJTIwJTNEJTIwbGlzdChpdGVydG9vbHMuY2hhaW4oKmxhYmVscykpJTBBdW5pcXVlX2xhYmVscyUyMCUzRCUyMGxpc3Qoc2V0KGZsYXR0ZW5lZF9sYWJlbHMpKSUwQSUwQWxhYmVsMmlkJTIwJTNEJTIwJTdCbGFiZWwlM0ElMjBpZHglMjBmb3IlMjBpZHglMkMlMjBsYWJlbCUyMGluJTIwZW51bWVyYXRlKHVuaXF1ZV9sYWJlbHMpJTdEJTBBaWQybGFiZWwlMjAlM0QlMjAlN0JpZHglM0ElMjBsYWJlbCUyMGZvciUyMGxhYmVsJTJDJTIwaWR4JTIwaW4lMjBsYWJlbDJpZC5pdGVtcygpJTdEJTIw",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> itertools

<span class="hljs-meta">&gt;&gt;&gt; </span>labels = [item[<span class="hljs-string">&#x27;ids&#x27;</span>] <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> dataset[<span class="hljs-string">&#x27;label&#x27;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>flattened_labels = <span class="hljs-built_in">list</span>(itertools.chain(*labels))
<span class="hljs-meta">&gt;&gt;&gt; </span>unique_labels = <span class="hljs-built_in">list</span>(<span class="hljs-built_in">set</span>(flattened_labels))

<span class="hljs-meta">&gt;&gt;&gt; </span>label2id = {label: idx <span class="hljs-keyword">for</span> idx, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(unique_labels)}
<span class="hljs-meta">&gt;&gt;&gt; </span>id2label = {idx: label <span class="hljs-keyword">for</span> label, idx <span class="hljs-keyword">in</span> label2id.items()} `}}),Te=new b({props:{code:"ZGVmJTIwcmVwbGFjZV9pZHMoaW5wdXRzKSUzQSUwQSUyMCUyMGlucHV0cyU1QiUyMmxhYmVsJTIyJTVEJTVCJTIyaWRzJTIyJTVEJTIwJTNEJTIwJTVCbGFiZWwyaWQlNUJ4JTVEJTIwZm9yJTIweCUyMGluJTIwaW5wdXRzJTVCJTIybGFiZWwlMjIlNUQlNUIlMjJpZHMlMjIlNUQlNUQlMEElMjAlMjByZXR1cm4lMjBpbnB1dHMlMEElMEElMEFkYXRhc2V0JTIwJTNEJTIwZGF0YXNldC5tYXAocmVwbGFjZV9pZHMpJTBBZmxhdF9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldC5mbGF0dGVuKCklMEFmbGF0X2RhdGFzZXQuZmVhdHVyZXM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">replace_ids</span>(<span class="hljs-params">inputs</span>):
<span class="hljs-meta">... </span>  inputs[<span class="hljs-string">&quot;label&quot;</span>][<span class="hljs-string">&quot;ids&quot;</span>] = [label2id[x] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> inputs[<span class="hljs-string">&quot;label&quot;</span>][<span class="hljs-string">&quot;ids&quot;</span>]]
<span class="hljs-meta">... </span>  <span class="hljs-keyword">return</span> inputs


<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.<span class="hljs-built_in">map</span>(replace_ids)
<span class="hljs-meta">&gt;&gt;&gt; </span>flat_dataset = dataset.flatten()
<span class="hljs-meta">&gt;&gt;&gt; </span>flat_dataset.features
{<span class="hljs-string">&#x27;question&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;image_id&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;label.ids&#x27;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&#x27;int64&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;label.weights&#x27;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&#x27;float64&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}`}}),Ue=new Oe({}),Ee=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFZpbHRQcm9jZXNzb3IlMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBWaWx0UHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZChtb2RlbF9jaGVja3BvaW50KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> ViltProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = ViltProcessor.from_pretrained(model_checkpoint)`}}),ke=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFkZWYlMjBwcmVwcm9jZXNzX2RhdGEoZXhhbXBsZXMpJTNBJTBBJTIwJTIwJTIwJTIwaW1hZ2VfcGF0aHMlMjAlM0QlMjBleGFtcGxlcyU1QidpbWFnZV9pZCclNUQlMEElMjAlMjAlMjAlMjBpbWFnZXMlMjAlM0QlMjAlNUJJbWFnZS5vcGVuKGltYWdlX3BhdGgpJTIwZm9yJTIwaW1hZ2VfcGF0aCUyMGluJTIwaW1hZ2VfcGF0aHMlNUQlMEElMjAlMjAlMjAlMjB0ZXh0cyUyMCUzRCUyMGV4YW1wbGVzJTVCJ3F1ZXN0aW9uJyU1RCUyMCUyMCUyMCUyMCUwQSUwQSUyMCUyMCUyMCUyMGVuY29kaW5nJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlcyUyQyUyMHRleHRzJTJDJTIwcGFkZGluZyUzRCUyMm1heF9sZW5ndGglMjIlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBJTIwJTIwJTIwJTIwZm9yJTIwayUyQyUyMHYlMjBpbiUyMGVuY29kaW5nLml0ZW1zKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGluZyU1QmslNUQlMjAlM0QlMjB2LnNxdWVlemUoKSUwQSUyMCUyMCUyMCUyMHRhcmdldHMlMjAlM0QlMjAlNUIlNUQlMEElMEElMjAlMjAlMjAlMjBmb3IlMjBsYWJlbHMlMkMlMjBzY29yZXMlMjBpbiUyMHppcChleGFtcGxlcyU1QidsYWJlbC5pZHMnJTVEJTJDJTIwZXhhbXBsZXMlNUInbGFiZWwud2VpZ2h0cyclNUQpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdGFyZ2V0JTIwJTNEJTIwdG9yY2guemVyb3MobGVuKGlkMmxhYmVsKSklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBsYWJlbCUyQyUyMHNjb3JlJTIwaW4lMjB6aXAobGFiZWxzJTJDJTIwc2NvcmVzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRhcmdldCU1QmxhYmVsJTVEJTIwJTNEJTIwc2NvcmUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0YXJnZXRzLmFwcGVuZCh0YXJnZXQpJTBBJTBBJTIwJTIwJTIwJTIwZW5jb2RpbmclNUIlMjJsYWJlbHMlMjIlNUQlMjAlM0QlMjB0YXJnZXRzJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwZW5jb2Rpbmc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_data</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    image_paths = examples[<span class="hljs-string">&#x27;image_id&#x27;</span>]
<span class="hljs-meta">... </span>    images = [Image.<span class="hljs-built_in">open</span>(image_path) <span class="hljs-keyword">for</span> image_path <span class="hljs-keyword">in</span> image_paths]
<span class="hljs-meta">... </span>    texts = examples[<span class="hljs-string">&#x27;question&#x27;</span>]    

<span class="hljs-meta">... </span>    encoding = processor(images, texts, padding=<span class="hljs-string">&quot;max_length&quot;</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoding.items():
<span class="hljs-meta">... </span>          encoding[k] = v.squeeze()
    
<span class="hljs-meta">... </span>    targets = []

<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> labels, scores <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(examples[<span class="hljs-string">&#x27;label.ids&#x27;</span>], examples[<span class="hljs-string">&#x27;label.weights&#x27;</span>]):
<span class="hljs-meta">... </span>        target = torch.zeros(<span class="hljs-built_in">len</span>(id2label))

<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> label, score <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(labels, scores):
<span class="hljs-meta">... </span>            target[label] = score
      
<span class="hljs-meta">... </span>        targets.append(target)

<span class="hljs-meta">... </span>    encoding[<span class="hljs-string">&quot;labels&quot;</span>] = targets
    
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> encoding`}}),xe=new b({props:{code:"cHJvY2Vzc2VkX2RhdGFzZXQlMjAlM0QlMjBmbGF0X2RhdGFzZXQubWFwKHByZXByb2Nlc3NfZGF0YSUyQyUyMGJhdGNoZWQlM0RUcnVlJTJDJTIwcmVtb3ZlX2NvbHVtbnMlM0QlNUIncXVlc3Rpb24nJTJDJ3F1ZXN0aW9uX3R5cGUnJTJDJTIwJTIwJ3F1ZXN0aW9uX2lkJyUyQyUyMCdpbWFnZV9pZCclMkMlMjAnYW5zd2VyX3R5cGUnJTJDJTIwJ2xhYmVsLmlkcyclMkMlMjAnbGFiZWwud2VpZ2h0cyclNUQpJTBBcHJvY2Vzc2VkX2RhdGFzZXQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset = flat_dataset.<span class="hljs-built_in">map</span>(preprocess_data, batched=<span class="hljs-literal">True</span>, remove_columns=[<span class="hljs-string">&#x27;question&#x27;</span>,<span class="hljs-string">&#x27;question_type&#x27;</span>,  <span class="hljs-string">&#x27;question_id&#x27;</span>, <span class="hljs-string">&#x27;image_id&#x27;</span>, <span class="hljs-string">&#x27;answer_type&#x27;</span>, <span class="hljs-string">&#x27;label.ids&#x27;</span>, <span class="hljs-string">&#x27;label.weights&#x27;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset
Dataset({
    features: [<span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>, <span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;pixel_values&#x27;</span>, <span class="hljs-string">&#x27;pixel_mask&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>],
    num_rows: <span class="hljs-number">200</span>
})`}}),$e=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERlZmF1bHREYXRhQ29sbGF0b3IlMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGVmYXVsdERhdGFDb2xsYXRvcigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DefaultDataCollator

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DefaultDataCollator()`}}),Ve=new Oe({}),Ze=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFZpbHRGb3JRdWVzdGlvbkFuc3dlcmluZyUwQSUwQW1vZGVsJTIwJTNEJTIwVmlsdEZvclF1ZXN0aW9uQW5zd2VyaW5nLmZyb21fcHJldHJhaW5lZChtb2RlbF9jaGVja3BvaW50JTJDJTIwbnVtX2xhYmVscyUzRGxlbihpZDJsYWJlbCklMkMlMjBpZDJsYWJlbCUzRGlkMmxhYmVsJTJDJTIwbGFiZWwyaWQlM0RsYWJlbDJpZCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> ViltForQuestionAnswering

<span class="hljs-meta">&gt;&gt;&gt; </span>model = ViltForQuestionAnswering.from_pretrained(model_checkpoint, num_labels=<span class="hljs-built_in">len</span>(id2label), id2label=id2label, label2id=label2id)`}}),Ge=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTBBJTBBcmVwb19pZCUyMCUzRCUyMCUyMk1hcmlhSyUyRnZpbHRfZmluZXR1bmVkXzIwMCUyMiUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEcmVwb19pZCUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDQlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMjAlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3N0ZXBzJTNEMjAwJTJDJTBBJTIwJTIwJTIwJTIwbG9nZ2luZ19zdGVwcyUzRDUwJTJDJTBBJTIwJTIwJTIwJTIwbGVhcm5pbmdfcmF0ZSUzRDVlLTUlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3RvdGFsX2xpbWl0JTNEMiUyQyUwQSUyMCUyMCUyMCUyMHJlbW92ZV91bnVzZWRfY29sdW1ucyUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

<span class="hljs-meta">&gt;&gt;&gt; </span>repo_id = <span class="hljs-string">&quot;MariaK/vilt_finetuned_200&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=repo_id,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">20</span>,
<span class="hljs-meta">... </span>    save_steps=<span class="hljs-number">200</span>,
<span class="hljs-meta">... </span>    logging_steps=<span class="hljs-number">50</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">5e-5</span>,
<span class="hljs-meta">... </span>    save_total_limit=<span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    remove_unused_columns=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)`}}),Ae=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2NvbGxhdG9yJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0Rwcm9jZXNzZWRfZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHByb2Nlc3NvciUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>    train_dataset=processed_dataset,
<span class="hljs-meta">... </span>    tokenizer=processor,
<span class="hljs-meta">... </span>)`}}),Xe=new b({props:{code:"dHJhaW5lci50cmFpbigpJTIw",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train() '}}),Fe=new b({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()'}}),Re=new Oe({}),Ne=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBcGlwZSUyMCUzRCUyMHBpcGVsaW5lKCUyMnZpc3VhbC1xdWVzdGlvbi1hbnN3ZXJpbmclMjIlMkMlMjBtb2RlbCUzRCUyMk1hcmlhSyUyRnZpbHRfZmluZXR1bmVkXzIwMCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>pipe = pipeline(<span class="hljs-string">&quot;visual-question-answering&quot;</span>, model=<span class="hljs-string">&quot;MariaK/vilt_finetuned_200&quot;</span>)`}}),qe=new b({props:{code:"ZXhhbXBsZSUyMCUzRCUyMGRhdGFzZXQlNUIwJTVEJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKGV4YW1wbGUlNUInaW1hZ2VfaWQnJTVEKSUwQXF1ZXN0aW9uJTIwJTNEJTIwZXhhbXBsZSU1QidxdWVzdGlvbiclNUQlMEFwcmludChxdWVzdGlvbiklMEFwaXBlKGltYWdlJTJDJTIwcXVlc3Rpb24lMkMlMjB0b3BfayUzRDEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>example = dataset[<span class="hljs-number">0</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(example[<span class="hljs-string">&#x27;image_id&#x27;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>question = example[<span class="hljs-string">&#x27;question&#x27;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(question)
<span class="hljs-meta">&gt;&gt;&gt; </span>pipe(image, question, top_k=<span class="hljs-number">1</span>)
<span class="hljs-string">&quot;Where is he looking?&quot;</span>
[{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.5498199462890625</span>, <span class="hljs-string">&#x27;answer&#x27;</span>: <span class="hljs-string">&#x27;down&#x27;</span>}]`}}),Ye=new b({props:{code:"cHJvY2Vzc29yJTIwJTNEJTIwVmlsdFByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIyTWFyaWFLJTJGdmlsdF9maW5ldHVuZWRfMjAwJTIyKSUwQSUwQWltYWdlJTIwJTNEJTIwSW1hZ2Uub3BlbihleGFtcGxlJTVCJ2ltYWdlX2lkJyU1RCklMEFxdWVzdGlvbiUyMCUzRCUyMGV4YW1wbGUlNUIncXVlc3Rpb24nJTVEJTBBJTBBJTIzJTIwcHJlcGFyZSUyMGlucHV0cyUwQWlucHV0cyUyMCUzRCUyMHByb2Nlc3NvcihpbWFnZSUyQyUyMHF1ZXN0aW9uJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEFtb2RlbCUyMCUzRCUyMFZpbHRGb3JRdWVzdGlvbkFuc3dlcmluZy5mcm9tX3ByZXRyYWluZWQoJTIyTWFyaWFLJTJGdmlsdF9maW5ldHVuZWRfMjAwJTIyKSUwQSUwQSUyMyUyMGZvcndhcmQlMjBwYXNzJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyklMEElMEFsb2dpdHMlMjAlM0QlMjBvdXRwdXRzLmxvZ2l0cyUwQWlkeCUyMCUzRCUyMGxvZ2l0cy5hcmdtYXgoLTEpLml0ZW0oKSUwQXByaW50KCUyMlByZWRpY3RlZCUyMGFuc3dlciUzQSUyMiUyQyUyMG1vZGVsLmNvbmZpZy5pZDJsYWJlbCU1QmlkeCU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processor = ViltProcessor.from_pretrained(<span class="hljs-string">&quot;MariaK/vilt_finetuned_200&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(example[<span class="hljs-string">&#x27;image_id&#x27;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>question = example[<span class="hljs-string">&#x27;question&#x27;</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># prepare inputs</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor(image, question, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = ViltForQuestionAnswering.from_pretrained(<span class="hljs-string">&quot;MariaK/vilt_finetuned_200&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># forward pass</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    outputs = model(**inputs)

<span class="hljs-meta">&gt;&gt;&gt; </span>logits = outputs.logits
<span class="hljs-meta">&gt;&gt;&gt; </span>idx = logits.argmax(-<span class="hljs-number">1</span>).item()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Predicted answer:&quot;</span>, model.config.id2label[idx])
Predicted answer: down`}}),He=new Oe({}),Pe=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMkMlMjBCbGlwMkZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQWltcG9ydCUyMHRvcmNoJTBBJTBBcHJvY2Vzc29yJTIwJTNEJTIwQXV0b1Byb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIyU2FsZXNmb3JjZSUyRmJsaXAyLW9wdC0yLjdiJTIyKSUwQW1vZGVsJTIwJTNEJTIwQmxpcDJGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMlNhbGVzZm9yY2UlMkZibGlwMi1vcHQtMi43YiUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiklMEFkZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTIyJTIwaWYlMjB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpJTIwZWxzZSUyMCUyMmNwdSUyMiUwQW1vZGVsLnRvKGRldmljZSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, Blip2ForConditionalGeneration
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;Salesforce/blip2-opt-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = Blip2ForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;Salesforce/blip2-opt-2.7b&quot;</span>, torch_dtype=torch.float16)
<span class="hljs-meta">&gt;&gt;&gt; </span>device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.to(device)`}}),Se=new b({props:{code:"ZXhhbXBsZSUyMCUzRCUyMGRhdGFzZXQlNUIwJTVEJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKGV4YW1wbGUlNUInaW1hZ2VfaWQnJTVEKSUwQXF1ZXN0aW9uJTIwJTNEJTIwZXhhbXBsZSU1QidxdWVzdGlvbiclNUQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>example = dataset[<span class="hljs-number">0</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(example[<span class="hljs-string">&#x27;image_id&#x27;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>question = example[<span class="hljs-string">&#x27;question&#x27;</span>]`}}),De=new b({props:{code:"cHJvbXB0JTIwJTNEJTIwZiUyMlF1ZXN0aW9uJTNBJTIwJTdCcXVlc3Rpb24lN0QlMjBBbnN3ZXIlM0ElMjIlMjA=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">f&quot;Question: <span class="hljs-subst">{question}</span> Answer:&quot;</span> '}}),Ke=new b({props:{code:"aW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlJTJDJTIwdGV4dCUzRHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKGRldmljZSUyQyUyMHRvcmNoLmZsb2F0MTYpJTBBJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwbWF4X25ld190b2tlbnMlM0QxMCklMEFnZW5lcmF0ZWRfdGV4dCUyMCUzRCUyMHByb2Nlc3Nvci5iYXRjaF9kZWNvZGUoZ2VuZXJhdGVkX2lkcyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQuc3RyaXAoKSUwQXByaW50KGdlbmVyYXRlZF90ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor(image, text=prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device, torch.float16)

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**inputs, max_new_tokens=<span class="hljs-number">10</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_text = processor.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>].strip()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(generated_text)
<span class="hljs-string">&quot;He is looking at the crowd&quot;</span> `}}),{c(){Z=o("meta"),Xa=h(),I=o("h1"),F=o("a"),Ps=o("span"),d(pe.$$.fragment),Gl=h(),Ss=o("span"),Wl=l("Visual Question Answering"),Fa=h(),d(he.$$.fragment),Ra=h(),es=o("p"),Cl=l(`Visual Question Answering (VQA) is the task of answering open-ended questions based on an image.
The input to models supporting this task is typically a combination of an image and a question, and the output is an
answer expressed in natural language.`),Na=h(),ss=o("p"),Al=l("Some noteworthy use case examples for VQA include:"),qa=h(),M=o("ul"),Ds=o("li"),Bl=l("Accessibility applications for visually impaired individuals."),Ql=h(),Ks=o("li"),Xl=l("Education: posing questions about visual materials presented in lectures or textbooks. VQA can also be utilized in interactive museum exhibits or historical sites."),Fl=h(),Os=o("li"),Rl=l("Customer service and e-commerce: VQA can enhance user experience by letting users ask questions about products."),Nl=h(),ea=o("li"),ql=l("Image retrieval: VQA models can be used to retrieve images with specific characteristics. For example, the user can ask \u201CIs there a dog?\u201D to find all images with dogs from a set of images."),za=h(),as=o("p"),zl=l("In this guide you\u2019ll learn how to:"),Ya=h(),T=o("ul"),G=o("li"),Yl=l("Fine-tune a classification VQA model, specifically "),ts=o("a"),Hl=l("ViLT"),Ll=l(", on the "),R=o("a"),sa=o("code"),Pl=l("Graphcore/vqa"),Sl=l(" dataset"),Dl=l("."),Kl=h(),aa=o("li"),Ol=l("Use your fine-tuned ViLT for inference."),en=h(),ta=o("li"),sn=l("Run zero-shot VQA inference with a generative model, like BLIP-2."),Ha=h(),W=o("h2"),N=o("a"),la=o("span"),d(ce.$$.fragment),an=h(),na=o("span"),tn=l("Fine-tuning ViLT"),La=h(),U=o("p"),ln=l(`ViLT model incorporates text embeddings into a Vision Transformer (ViT), allowing it to have a minimal design for
Vision-and-Language Pre-training (VLP). This model can be used for several downstream tasks. For the VQA task, a classifier
head is placed on top (a linear layer on top of the final hidden state of the `),oa=o("code"),nn=l("[CLS]"),on=l(` token) and randomly initialized.
Visual Question Answering is thus treated as a `),ra=o("strong"),rn=l("classification problem"),pn=l("."),Pa=h(),ls=o("p"),hn=l(`More recent models, such as BLIP, BLIP-2, and InstructBLIP, treat VQA as a generative task. Later in this guide we
illustrate how to use them for zero-shot VQA inference.`),Sa=h(),ns=o("p"),cn=l("Before you begin, make sure you have all the necessary libraries installed."),Da=h(),d(me.$$.fragment),Ka=h(),os=o("p"),mn=l(`We encourage you to share your model with the community. Log in to your Hugging Face account to upload it to the \u{1F917} Hub.
When prompted, enter your token to log in:`),Oa=h(),d(de.$$.fragment),et=h(),rs=o("p"),dn=l("Let\u2019s define the model checkpoint as a global variable."),st=h(),d(ue.$$.fragment),at=h(),C=o("h2"),q=o("a"),ia=o("span"),d(fe.$$.fragment),un=h(),pa=o("span"),fn=l("Load the data"),tt=h(),E=o("p"),gn=l("For illustration purposes, in this guide we use a very small sample of the annotated visual question answering "),ha=o("code"),yn=l("Graphcore/vqa"),wn=l(` dataset.
You can find the full dataset on `),ge=o("a"),bn=l("\u{1F917} Hub"),jn=l("."),lt=h(),J=o("p"),vn=l("As an alternative to the "),z=o("a"),ca=o("code"),Mn=l("Graphcore/vqa"),Jn=l(" dataset"),_n=l(`, you can download the
same data manually from the official `),ye=o("a"),Tn=l("VQA dataset page"),Un=l(`. If you prefer to follow the
tutorial with your custom data, check out how to `),we=o("a"),En=l("Create an image dataset"),kn=l(`
guide in the \u{1F917} Datasets documentation.`),nt=h(),is=o("p"),xn=l("Let\u2019s load the first 200 examples from the validation split and explore the dataset\u2019s features:"),ot=h(),d(be.$$.fragment),rt=h(),ps=o("p"),$n=l("Let\u2019s take a look at an example to understand the dataset\u2019s features:"),it=h(),d(je.$$.fragment),pt=h(),hs=o("p"),Vn=l("The features relevant to the task include:"),ht=h(),k=o("ul"),cs=o("li"),ma=o("code"),Zn=l("question"),In=l(": the question to be answered from the image"),Gn=h(),ms=o("li"),da=o("code"),Wn=l("image_id"),Cn=l(": the path to the image the question refers to"),An=h(),ds=o("li"),ua=o("code"),Bn=l("label"),Qn=l(": the annotations"),ct=h(),us=o("p"),Xn=l("We can remove the rest of the features as they won\u2019t be necessary:"),mt=h(),d(ve.$$.fragment),dt=h(),x=o("p"),Fn=l("As you can see, the "),fa=o("code"),Rn=l("label"),Nn=l(" feature contains several answers to the same question (called "),ga=o("code"),qn=l("ids"),zn=l(` here) collected by different human annotators.
This is because the answer to a question can be subjective. In this case, the question is \u201Cwhere is he looking?\u201C. Some people
annotated this with \u201Cdown\u201D, others with \u201Cat table\u201D, another one with \u201Cskateboard\u201D, etc.`),ut=h(),fs=o("p"),Yn=l("Take a look at the image and consider which answer would you give:"),ft=h(),d(Me.$$.fragment),gt=h(),Je=o("div"),gs=o("img"),yt=h(),ys=o("p"),Hn=l(`Due to the questions\u2019 and answers\u2019 ambiguity, datasets like this are treated as a multi-label classification problem (as
multiple answers are possibly valid). Moreover, rather than just creating a one-hot encoded vector, one creates a
soft encoding, based on the number of times a certain answer appeared in the annotations.`),wt=h(),Y=o("p"),Ln=l(`For instance, in the example above, because the answer \u201Cdown\u201D is selected way more often than other answers, it has a
score (called `),ya=o("code"),Pn=l("weight"),Sn=l(" in the dataset) of 1.0, and the rest of the answers have scores < 1.0."),bt=h(),ws=o("p"),Dn=l(`To later instantiate the model with an appropriate classification head, let\u2019s create two dictionaries: one that maps
the label name to an integer and vice versa:`),jt=h(),d(_e.$$.fragment),vt=h(),bs=o("p"),Kn=l("Now that we have the mappings, we can replace the string answers with their ids, and flatten the dataset for a more convenient further preprocessing."),Mt=h(),d(Te.$$.fragment),Jt=h(),A=o("h2"),H=o("a"),wa=o("span"),d(Ue.$$.fragment),On=h(),ba=o("span"),eo=l("Preprocessing data"),_t=h(),L=o("p"),so=l(`The next step is to load a ViLT processor to prepare the image and text data for the model.
`),js=o("a"),ao=l("ViltProcessor"),to=l(" wraps a BERT tokenizer and ViLT image processor into a convenient single processor:"),Tt=h(),d(Ee.$$.fragment),Ut=h(),j=o("p"),lo=l("To preprocess the data we need to encode the images and questions using the "),vs=o("a"),no=l("ViltProcessor"),oo=l(`. The processor will use
the `),Ms=o("a"),ro=l("BertTokenizerFast"),io=l(" to tokenize the text and create "),ja=o("code"),po=l("input_ids"),ho=l(", "),va=o("code"),co=l("attention_mask"),mo=l(" and "),Ma=o("code"),uo=l("token_type_ids"),fo=l(` for the text data.
As for images, the processor will leverage `),Js=o("a"),go=l("ViltImageProcessor"),yo=l(" to resize and normalize the image, and create "),Ja=o("code"),wo=l("pixel_values"),bo=l(" and "),_a=o("code"),jo=l("pixel_mask"),vo=l("."),Et=h(),P=o("p"),Mo=l("All these preprocessing steps are done under the hood, we only need to call the "),Ta=o("code"),Jo=l("processor"),_o=l(`. However, we still need to
prepare the target labels. In this representation, each element corresponds to a possible answer (label). For correct answers, the element holds
their respective score (weight), while the remaining elements are set to zero.`),kt=h(),S=o("p"),To=l("The following function applies the "),Ua=o("code"),Uo=l("processor"),Eo=l(" to the images and questions and formats the labels as described above:"),xt=h(),d(ke.$$.fragment),$t=h(),_=o("p"),ko=l("To apply the preprocessing function over the entire dataset, use \u{1F917} Datasets "),Ea=o("code"),xo=l("map"),$o=l(" function. You can speed up "),ka=o("code"),Vo=l("map"),Zo=l(` by
setting `),xa=o("code"),Io=l("batched=True"),Go=l(" to process multiple elements of the dataset at once. At this point, feel free to remove the columns you don\u2019t need."),Vt=h(),d(xe.$$.fragment),Zt=h(),D=o("p"),Wo=l("As a final step, create a batch of examples using "),_s=o("a"),Co=l("DefaultDataCollator"),Ao=l(":"),It=h(),d($e.$$.fragment),Gt=h(),B=o("h2"),K=o("a"),$a=o("span"),d(Ve.$$.fragment),Bo=h(),Va=o("span"),Qo=l("Train the model"),Wt=h(),O=o("p"),Xo=l("You\u2019re ready to start training your model now! Load ViLT with "),Ts=o("a"),Fo=l("ViltForQuestionAnswering"),Ro=l(`. Specify the number of labels
along with the label mappings:`),Ct=h(),d(Ze.$$.fragment),At=h(),Us=o("p"),No=l("At this point, only three steps remain:"),Bt=h(),Es=o("ol"),Ie=o("li"),qo=l("Define your training hyperparameters in "),ks=o("a"),zo=l("TrainingArguments"),Yo=l(":"),Qt=h(),d(Ge.$$.fragment),Xt=h(),We=o("ol"),Ce=o("li"),Ho=l("Pass the training arguments to "),xs=o("a"),Lo=l("Trainer"),Po=l(" along with the model, dataset, processor, and data collator."),Ft=h(),d(Ae.$$.fragment),Rt=h(),Be=o("ol"),Qe=o("li"),So=l("Call "),$s=o("a"),Do=l("train()"),Ko=l(" to finetune your model."),Nt=h(),d(Xe.$$.fragment),qt=h(),ee=o("p"),Oo=l("Once training is completed, share your model to the Hub with the "),Vs=o("a"),er=l("push_to_hub()"),sr=l(" method to share your final model on the \u{1F917} Hub:"),zt=h(),d(Fe.$$.fragment),Yt=h(),Q=o("h2"),se=o("a"),Za=o("span"),d(Re.$$.fragment),ar=h(),Ia=o("span"),tr=l("Inference"),Ht=h(),ae=o("p"),lr=l(`Now that you have fine-tuned a ViLT model, and uploaded it to the \u{1F917} Hub, you can use it for inference. The simplest
way to try out your fine-tuned model for inference is to use it in a `),Zs=o("a"),nr=l("Pipeline"),or=l("."),Lt=h(),d(Ne.$$.fragment),Pt=h(),Is=o("p"),rr=l(`The model in this guide has only been trained on 200 examples, so don\u2019t expect a lot from it. Let\u2019s see if it at least
learned something from the data and take the first example from the dataset to illustrate inference:`),St=h(),d(qe.$$.fragment),Dt=h(),Gs=o("p"),ir=l("Even though not very confident, the model indeed has learned something. With more examples and longer training, you\u2019ll get far better results!"),Kt=h(),Ws=o("p"),pr=l("You can also manually replicate the results of the pipeline if you\u2019d like:"),Ot=h(),$=o("ol"),Ga=o("li"),hr=l("Take an image and a question, prepare them for the model using the processor from your model."),cr=h(),Wa=o("li"),mr=l("Forward the result or preprocessing through the model."),dr=h(),ze=o("li"),ur=l("From the logits, get the most likely answer\u2019s id, and find the actual answer in the "),Ca=o("code"),fr=l("id2label"),gr=l("."),el=h(),d(Ye.$$.fragment),sl=h(),X=o("h2"),te=o("a"),Aa=o("span"),d(He.$$.fragment),yr=h(),Ba=o("span"),wr=l("Zero-shot VQA"),al=h(),V=o("p"),br=l(`The previous model treated VQA as a classification task. Some recent models, such as BLIP, BLIP-2, and InstructBLIP approach
VQA as a generative task. Let\u2019s take `),Cs=o("a"),jr=l("BLIP-2"),vr=l(` as an example. It introduced a new visual-language pre-training
paradigm in which any combination of pre-trained vision encoder and LLM can be used (learn more in the `),Le=o("a"),Mr=l("BLIP-2 blog post"),Jr=l(`).
This enables achieving state-of-the-art results on multiple visual-language tasks including visual question answering.`),tl=h(),le=o("p"),_r=l(`Let\u2019s illustrate how you can use this model for VQA. First, let\u2019s load the model. Here we\u2019ll explicitly send the model to a
GPU, if available, which we didn\u2019t need to do earlier when training, as `),As=o("a"),Tr=l("Trainer"),Ur=l(" handles this automatically:"),ll=h(),d(Pe.$$.fragment),nl=h(),Bs=o("p"),Er=l("The model takes image and text as input, so let\u2019s use the exact same image/question pair from the first example in the VQA dataset:"),ol=h(),d(Se.$$.fragment),rl=h(),ne=o("p"),kr=l("To use BLIP-2 for visual question answering task, the textual prompt has to follow a specific format: "),Qa=o("code"),xr=l("Question: {} Answer:"),$r=l("."),il=h(),d(De.$$.fragment),pl=h(),Qs=o("p"),Vr=l("Now we need to preprocess the image/prompt with the model\u2019s processor, pass the processed input through the model, and decode the output:"),hl=h(),d(Ke.$$.fragment),cl=h(),Xs=o("p"),Zr=l(`As you can see, the model recognized the crowd, and the direction of the face (looking down), however, it seems to miss
the fact the crowd is behind the skater. Still, in cases where acquiring human-annotated datasets is not feasible, this
approach can quickly produce useful results.`),this.h()},l(e){const t=Vp('[data-svelte="svelte-1phssyn"]',document.head);Z=r(t,"META",{name:!0,content:!0}),t.forEach(s),Xa=c(e),I=r(e,"H1",{class:!0});var dl=i(I);F=r(dl,"A",{id:!0,class:!0,href:!0});var Xr=i(F);Ps=r(Xr,"SPAN",{});var Fr=i(Ps);u(pe.$$.fragment,Fr),Fr.forEach(s),Xr.forEach(s),Gl=c(dl),Ss=r(dl,"SPAN",{});var Rr=i(Ss);Wl=n(Rr,"Visual Question Answering"),Rr.forEach(s),dl.forEach(s),Fa=c(e),u(he.$$.fragment,e),Ra=c(e),es=r(e,"P",{});var Nr=i(es);Cl=n(Nr,`Visual Question Answering (VQA) is the task of answering open-ended questions based on an image.
The input to models supporting this task is typically a combination of an image and a question, and the output is an
answer expressed in natural language.`),Nr.forEach(s),Na=c(e),ss=r(e,"P",{});var qr=i(ss);Al=n(qr,"Some noteworthy use case examples for VQA include:"),qr.forEach(s),qa=c(e),M=r(e,"UL",{});var oe=i(M);Ds=r(oe,"LI",{});var zr=i(Ds);Bl=n(zr,"Accessibility applications for visually impaired individuals."),zr.forEach(s),Ql=c(oe),Ks=r(oe,"LI",{});var Yr=i(Ks);Xl=n(Yr,"Education: posing questions about visual materials presented in lectures or textbooks. VQA can also be utilized in interactive museum exhibits or historical sites."),Yr.forEach(s),Fl=c(oe),Os=r(oe,"LI",{});var Hr=i(Os);Rl=n(Hr,"Customer service and e-commerce: VQA can enhance user experience by letting users ask questions about products."),Hr.forEach(s),Nl=c(oe),ea=r(oe,"LI",{});var Lr=i(ea);ql=n(Lr,"Image retrieval: VQA models can be used to retrieve images with specific characteristics. For example, the user can ask \u201CIs there a dog?\u201D to find all images with dogs from a set of images."),Lr.forEach(s),oe.forEach(s),za=c(e),as=r(e,"P",{});var Pr=i(as);zl=n(Pr,"In this guide you\u2019ll learn how to:"),Pr.forEach(s),Ya=c(e),T=r(e,"UL",{});var Fs=i(T);G=r(Fs,"LI",{});var Rs=i(G);Yl=n(Rs,"Fine-tune a classification VQA model, specifically "),ts=r(Rs,"A",{href:!0});var Sr=i(ts);Hl=n(Sr,"ViLT"),Sr.forEach(s),Ll=n(Rs,", on the "),R=r(Rs,"A",{href:!0,rel:!0});var Ir=i(R);sa=r(Ir,"CODE",{});var Dr=i(sa);Pl=n(Dr,"Graphcore/vqa"),Dr.forEach(s),Sl=n(Ir," dataset"),Ir.forEach(s),Dl=n(Rs,"."),Rs.forEach(s),Kl=c(Fs),aa=r(Fs,"LI",{});var Kr=i(aa);Ol=n(Kr,"Use your fine-tuned ViLT for inference."),Kr.forEach(s),en=c(Fs),ta=r(Fs,"LI",{});var Or=i(ta);sn=n(Or,"Run zero-shot VQA inference with a generative model, like BLIP-2."),Or.forEach(s),Fs.forEach(s),Ha=c(e),W=r(e,"H2",{class:!0});var ul=i(W);N=r(ul,"A",{id:!0,class:!0,href:!0});var ei=i(N);la=r(ei,"SPAN",{});var si=i(la);u(ce.$$.fragment,si),si.forEach(s),ei.forEach(s),an=c(ul),na=r(ul,"SPAN",{});var ai=i(na);tn=n(ai,"Fine-tuning ViLT"),ai.forEach(s),ul.forEach(s),La=c(e),U=r(e,"P",{});var Ns=i(U);ln=n(Ns,`ViLT model incorporates text embeddings into a Vision Transformer (ViT), allowing it to have a minimal design for
Vision-and-Language Pre-training (VLP). This model can be used for several downstream tasks. For the VQA task, a classifier
head is placed on top (a linear layer on top of the final hidden state of the `),oa=r(Ns,"CODE",{});var ti=i(oa);nn=n(ti,"[CLS]"),ti.forEach(s),on=n(Ns,` token) and randomly initialized.
Visual Question Answering is thus treated as a `),ra=r(Ns,"STRONG",{});var li=i(ra);rn=n(li,"classification problem"),li.forEach(s),pn=n(Ns,"."),Ns.forEach(s),Pa=c(e),ls=r(e,"P",{});var ni=i(ls);hn=n(ni,`More recent models, such as BLIP, BLIP-2, and InstructBLIP, treat VQA as a generative task. Later in this guide we
illustrate how to use them for zero-shot VQA inference.`),ni.forEach(s),Sa=c(e),ns=r(e,"P",{});var oi=i(ns);cn=n(oi,"Before you begin, make sure you have all the necessary libraries installed."),oi.forEach(s),Da=c(e),u(me.$$.fragment,e),Ka=c(e),os=r(e,"P",{});var ri=i(os);mn=n(ri,`We encourage you to share your model with the community. Log in to your Hugging Face account to upload it to the \u{1F917} Hub.
When prompted, enter your token to log in:`),ri.forEach(s),Oa=c(e),u(de.$$.fragment,e),et=c(e),rs=r(e,"P",{});var ii=i(rs);dn=n(ii,"Let\u2019s define the model checkpoint as a global variable."),ii.forEach(s),st=c(e),u(ue.$$.fragment,e),at=c(e),C=r(e,"H2",{class:!0});var fl=i(C);q=r(fl,"A",{id:!0,class:!0,href:!0});var pi=i(q);ia=r(pi,"SPAN",{});var hi=i(ia);u(fe.$$.fragment,hi),hi.forEach(s),pi.forEach(s),un=c(fl),pa=r(fl,"SPAN",{});var ci=i(pa);fn=n(ci,"Load the data"),ci.forEach(s),fl.forEach(s),tt=c(e),E=r(e,"P",{});var qs=i(E);gn=n(qs,"For illustration purposes, in this guide we use a very small sample of the annotated visual question answering "),ha=r(qs,"CODE",{});var mi=i(ha);yn=n(mi,"Graphcore/vqa"),mi.forEach(s),wn=n(qs,` dataset.
You can find the full dataset on `),ge=r(qs,"A",{href:!0,rel:!0});var di=i(ge);bn=n(di,"\u{1F917} Hub"),di.forEach(s),jn=n(qs,"."),qs.forEach(s),lt=c(e),J=r(e,"P",{});var re=i(J);vn=n(re,"As an alternative to the "),z=r(re,"A",{href:!0,rel:!0});var Gr=i(z);ca=r(Gr,"CODE",{});var ui=i(ca);Mn=n(ui,"Graphcore/vqa"),ui.forEach(s),Jn=n(Gr," dataset"),Gr.forEach(s),_n=n(re,`, you can download the
same data manually from the official `),ye=r(re,"A",{href:!0,rel:!0});var fi=i(ye);Tn=n(fi,"VQA dataset page"),fi.forEach(s),Un=n(re,`. If you prefer to follow the
tutorial with your custom data, check out how to `),we=r(re,"A",{href:!0,rel:!0});var gi=i(we);En=n(gi,"Create an image dataset"),gi.forEach(s),kn=n(re,`
guide in the \u{1F917} Datasets documentation.`),re.forEach(s),nt=c(e),is=r(e,"P",{});var yi=i(is);xn=n(yi,"Let\u2019s load the first 200 examples from the validation split and explore the dataset\u2019s features:"),yi.forEach(s),ot=c(e),u(be.$$.fragment,e),rt=c(e),ps=r(e,"P",{});var wi=i(ps);$n=n(wi,"Let\u2019s take a look at an example to understand the dataset\u2019s features:"),wi.forEach(s),it=c(e),u(je.$$.fragment,e),pt=c(e),hs=r(e,"P",{});var bi=i(hs);Vn=n(bi,"The features relevant to the task include:"),bi.forEach(s),ht=c(e),k=r(e,"UL",{});var zs=i(k);cs=r(zs,"LI",{});var Wr=i(cs);ma=r(Wr,"CODE",{});var ji=i(ma);Zn=n(ji,"question"),ji.forEach(s),In=n(Wr,": the question to be answered from the image"),Wr.forEach(s),Gn=c(zs),ms=r(zs,"LI",{});var Cr=i(ms);da=r(Cr,"CODE",{});var vi=i(da);Wn=n(vi,"image_id"),vi.forEach(s),Cn=n(Cr,": the path to the image the question refers to"),Cr.forEach(s),An=c(zs),ds=r(zs,"LI",{});var Ar=i(ds);ua=r(Ar,"CODE",{});var Mi=i(ua);Bn=n(Mi,"label"),Mi.forEach(s),Qn=n(Ar,": the annotations"),Ar.forEach(s),zs.forEach(s),ct=c(e),us=r(e,"P",{});var Ji=i(us);Xn=n(Ji,"We can remove the rest of the features as they won\u2019t be necessary:"),Ji.forEach(s),mt=c(e),u(ve.$$.fragment,e),dt=c(e),x=r(e,"P",{});var Ys=i(x);Fn=n(Ys,"As you can see, the "),fa=r(Ys,"CODE",{});var _i=i(fa);Rn=n(_i,"label"),_i.forEach(s),Nn=n(Ys," feature contains several answers to the same question (called "),ga=r(Ys,"CODE",{});var Ti=i(ga);qn=n(Ti,"ids"),Ti.forEach(s),zn=n(Ys,` here) collected by different human annotators.
This is because the answer to a question can be subjective. In this case, the question is \u201Cwhere is he looking?\u201C. Some people
annotated this with \u201Cdown\u201D, others with \u201Cat table\u201D, another one with \u201Cskateboard\u201D, etc.`),Ys.forEach(s),ut=c(e),fs=r(e,"P",{});var Ui=i(fs);Yn=n(Ui,"Take a look at the image and consider which answer would you give:"),Ui.forEach(s),ft=c(e),u(Me.$$.fragment,e),gt=c(e),Je=r(e,"DIV",{class:!0});var Ei=i(Je);gs=r(Ei,"IMG",{src:!0,alt:!0}),Ei.forEach(s),yt=c(e),ys=r(e,"P",{});var ki=i(ys);Hn=n(ki,`Due to the questions\u2019 and answers\u2019 ambiguity, datasets like this are treated as a multi-label classification problem (as
multiple answers are possibly valid). Moreover, rather than just creating a one-hot encoded vector, one creates a
soft encoding, based on the number of times a certain answer appeared in the annotations.`),ki.forEach(s),wt=c(e),Y=r(e,"P",{});var gl=i(Y);Ln=n(gl,`For instance, in the example above, because the answer \u201Cdown\u201D is selected way more often than other answers, it has a
score (called `),ya=r(gl,"CODE",{});var xi=i(ya);Pn=n(xi,"weight"),xi.forEach(s),Sn=n(gl," in the dataset) of 1.0, and the rest of the answers have scores < 1.0."),gl.forEach(s),bt=c(e),ws=r(e,"P",{});var $i=i(ws);Dn=n($i,`To later instantiate the model with an appropriate classification head, let\u2019s create two dictionaries: one that maps
the label name to an integer and vice versa:`),$i.forEach(s),jt=c(e),u(_e.$$.fragment,e),vt=c(e),bs=r(e,"P",{});var Vi=i(bs);Kn=n(Vi,"Now that we have the mappings, we can replace the string answers with their ids, and flatten the dataset for a more convenient further preprocessing."),Vi.forEach(s),Mt=c(e),u(Te.$$.fragment,e),Jt=c(e),A=r(e,"H2",{class:!0});var yl=i(A);H=r(yl,"A",{id:!0,class:!0,href:!0});var Zi=i(H);wa=r(Zi,"SPAN",{});var Ii=i(wa);u(Ue.$$.fragment,Ii),Ii.forEach(s),Zi.forEach(s),On=c(yl),ba=r(yl,"SPAN",{});var Gi=i(ba);eo=n(Gi,"Preprocessing data"),Gi.forEach(s),yl.forEach(s),_t=c(e),L=r(e,"P",{});var wl=i(L);so=n(wl,`The next step is to load a ViLT processor to prepare the image and text data for the model.
`),js=r(wl,"A",{href:!0});var Wi=i(js);ao=n(Wi,"ViltProcessor"),Wi.forEach(s),to=n(wl," wraps a BERT tokenizer and ViLT image processor into a convenient single processor:"),wl.forEach(s),Tt=c(e),u(Ee.$$.fragment,e),Ut=c(e),j=r(e,"P",{});var v=i(j);lo=n(v,"To preprocess the data we need to encode the images and questions using the "),vs=r(v,"A",{href:!0});var Ci=i(vs);no=n(Ci,"ViltProcessor"),Ci.forEach(s),oo=n(v,`. The processor will use
the `),Ms=r(v,"A",{href:!0});var Ai=i(Ms);ro=n(Ai,"BertTokenizerFast"),Ai.forEach(s),io=n(v," to tokenize the text and create "),ja=r(v,"CODE",{});var Bi=i(ja);po=n(Bi,"input_ids"),Bi.forEach(s),ho=n(v,", "),va=r(v,"CODE",{});var Qi=i(va);co=n(Qi,"attention_mask"),Qi.forEach(s),mo=n(v," and "),Ma=r(v,"CODE",{});var Xi=i(Ma);uo=n(Xi,"token_type_ids"),Xi.forEach(s),fo=n(v,` for the text data.
As for images, the processor will leverage `),Js=r(v,"A",{href:!0});var Fi=i(Js);go=n(Fi,"ViltImageProcessor"),Fi.forEach(s),yo=n(v," to resize and normalize the image, and create "),Ja=r(v,"CODE",{});var Ri=i(Ja);wo=n(Ri,"pixel_values"),Ri.forEach(s),bo=n(v," and "),_a=r(v,"CODE",{});var Ni=i(_a);jo=n(Ni,"pixel_mask"),Ni.forEach(s),vo=n(v,"."),v.forEach(s),Et=c(e),P=r(e,"P",{});var bl=i(P);Mo=n(bl,"All these preprocessing steps are done under the hood, we only need to call the "),Ta=r(bl,"CODE",{});var qi=i(Ta);Jo=n(qi,"processor"),qi.forEach(s),_o=n(bl,`. However, we still need to
prepare the target labels. In this representation, each element corresponds to a possible answer (label). For correct answers, the element holds
their respective score (weight), while the remaining elements are set to zero.`),bl.forEach(s),kt=c(e),S=r(e,"P",{});var jl=i(S);To=n(jl,"The following function applies the "),Ua=r(jl,"CODE",{});var zi=i(Ua);Uo=n(zi,"processor"),zi.forEach(s),Eo=n(jl," to the images and questions and formats the labels as described above:"),jl.forEach(s),xt=c(e),u(ke.$$.fragment,e),$t=c(e),_=r(e,"P",{});var ie=i(_);ko=n(ie,"To apply the preprocessing function over the entire dataset, use \u{1F917} Datasets "),Ea=r(ie,"CODE",{});var Yi=i(Ea);xo=n(Yi,"map"),Yi.forEach(s),$o=n(ie," function. You can speed up "),ka=r(ie,"CODE",{});var Hi=i(ka);Vo=n(Hi,"map"),Hi.forEach(s),Zo=n(ie,` by
setting `),xa=r(ie,"CODE",{});var Li=i(xa);Io=n(Li,"batched=True"),Li.forEach(s),Go=n(ie," to process multiple elements of the dataset at once. At this point, feel free to remove the columns you don\u2019t need."),ie.forEach(s),Vt=c(e),u(xe.$$.fragment,e),Zt=c(e),D=r(e,"P",{});var vl=i(D);Wo=n(vl,"As a final step, create a batch of examples using "),_s=r(vl,"A",{href:!0});var Pi=i(_s);Co=n(Pi,"DefaultDataCollator"),Pi.forEach(s),Ao=n(vl,":"),vl.forEach(s),It=c(e),u($e.$$.fragment,e),Gt=c(e),B=r(e,"H2",{class:!0});var Ml=i(B);K=r(Ml,"A",{id:!0,class:!0,href:!0});var Si=i(K);$a=r(Si,"SPAN",{});var Di=i($a);u(Ve.$$.fragment,Di),Di.forEach(s),Si.forEach(s),Bo=c(Ml),Va=r(Ml,"SPAN",{});var Ki=i(Va);Qo=n(Ki,"Train the model"),Ki.forEach(s),Ml.forEach(s),Wt=c(e),O=r(e,"P",{});var Jl=i(O);Xo=n(Jl,"You\u2019re ready to start training your model now! Load ViLT with "),Ts=r(Jl,"A",{href:!0});var Oi=i(Ts);Fo=n(Oi,"ViltForQuestionAnswering"),Oi.forEach(s),Ro=n(Jl,`. Specify the number of labels
along with the label mappings:`),Jl.forEach(s),Ct=c(e),u(Ze.$$.fragment,e),At=c(e),Us=r(e,"P",{});var ep=i(Us);No=n(ep,"At this point, only three steps remain:"),ep.forEach(s),Bt=c(e),Es=r(e,"OL",{});var sp=i(Es);Ie=r(sp,"LI",{});var _l=i(Ie);qo=n(_l,"Define your training hyperparameters in "),ks=r(_l,"A",{href:!0});var ap=i(ks);zo=n(ap,"TrainingArguments"),ap.forEach(s),Yo=n(_l,":"),_l.forEach(s),sp.forEach(s),Qt=c(e),u(Ge.$$.fragment,e),Xt=c(e),We=r(e,"OL",{start:!0});var tp=i(We);Ce=r(tp,"LI",{});var Tl=i(Ce);Ho=n(Tl,"Pass the training arguments to "),xs=r(Tl,"A",{href:!0});var lp=i(xs);Lo=n(lp,"Trainer"),lp.forEach(s),Po=n(Tl," along with the model, dataset, processor, and data collator."),Tl.forEach(s),tp.forEach(s),Ft=c(e),u(Ae.$$.fragment,e),Rt=c(e),Be=r(e,"OL",{start:!0});var np=i(Be);Qe=r(np,"LI",{});var Ul=i(Qe);So=n(Ul,"Call "),$s=r(Ul,"A",{href:!0});var op=i($s);Do=n(op,"train()"),op.forEach(s),Ko=n(Ul," to finetune your model."),Ul.forEach(s),np.forEach(s),Nt=c(e),u(Xe.$$.fragment,e),qt=c(e),ee=r(e,"P",{});var El=i(ee);Oo=n(El,"Once training is completed, share your model to the Hub with the "),Vs=r(El,"A",{href:!0});var rp=i(Vs);er=n(rp,"push_to_hub()"),rp.forEach(s),sr=n(El," method to share your final model on the \u{1F917} Hub:"),El.forEach(s),zt=c(e),u(Fe.$$.fragment,e),Yt=c(e),Q=r(e,"H2",{class:!0});var kl=i(Q);se=r(kl,"A",{id:!0,class:!0,href:!0});var ip=i(se);Za=r(ip,"SPAN",{});var pp=i(Za);u(Re.$$.fragment,pp),pp.forEach(s),ip.forEach(s),ar=c(kl),Ia=r(kl,"SPAN",{});var hp=i(Ia);tr=n(hp,"Inference"),hp.forEach(s),kl.forEach(s),Ht=c(e),ae=r(e,"P",{});var xl=i(ae);lr=n(xl,`Now that you have fine-tuned a ViLT model, and uploaded it to the \u{1F917} Hub, you can use it for inference. The simplest
way to try out your fine-tuned model for inference is to use it in a `),Zs=r(xl,"A",{href:!0});var cp=i(Zs);nr=n(cp,"Pipeline"),cp.forEach(s),or=n(xl,"."),xl.forEach(s),Lt=c(e),u(Ne.$$.fragment,e),Pt=c(e),Is=r(e,"P",{});var mp=i(Is);rr=n(mp,`The model in this guide has only been trained on 200 examples, so don\u2019t expect a lot from it. Let\u2019s see if it at least
learned something from the data and take the first example from the dataset to illustrate inference:`),mp.forEach(s),St=c(e),u(qe.$$.fragment,e),Dt=c(e),Gs=r(e,"P",{});var dp=i(Gs);ir=n(dp,"Even though not very confident, the model indeed has learned something. With more examples and longer training, you\u2019ll get far better results!"),dp.forEach(s),Kt=c(e),Ws=r(e,"P",{});var up=i(Ws);pr=n(up,"You can also manually replicate the results of the pipeline if you\u2019d like:"),up.forEach(s),Ot=c(e),$=r(e,"OL",{});var Hs=i($);Ga=r(Hs,"LI",{});var fp=i(Ga);hr=n(fp,"Take an image and a question, prepare them for the model using the processor from your model."),fp.forEach(s),cr=c(Hs),Wa=r(Hs,"LI",{});var gp=i(Wa);mr=n(gp,"Forward the result or preprocessing through the model."),gp.forEach(s),dr=c(Hs),ze=r(Hs,"LI",{});var $l=i(ze);ur=n($l,"From the logits, get the most likely answer\u2019s id, and find the actual answer in the "),Ca=r($l,"CODE",{});var yp=i(Ca);fr=n(yp,"id2label"),yp.forEach(s),gr=n($l,"."),$l.forEach(s),Hs.forEach(s),el=c(e),u(Ye.$$.fragment,e),sl=c(e),X=r(e,"H2",{class:!0});var Vl=i(X);te=r(Vl,"A",{id:!0,class:!0,href:!0});var wp=i(te);Aa=r(wp,"SPAN",{});var bp=i(Aa);u(He.$$.fragment,bp),bp.forEach(s),wp.forEach(s),yr=c(Vl),Ba=r(Vl,"SPAN",{});var jp=i(Ba);wr=n(jp,"Zero-shot VQA"),jp.forEach(s),Vl.forEach(s),al=c(e),V=r(e,"P",{});var Ls=i(V);br=n(Ls,`The previous model treated VQA as a classification task. Some recent models, such as BLIP, BLIP-2, and InstructBLIP approach
VQA as a generative task. Let\u2019s take `),Cs=r(Ls,"A",{href:!0});var vp=i(Cs);jr=n(vp,"BLIP-2"),vp.forEach(s),vr=n(Ls,` as an example. It introduced a new visual-language pre-training
paradigm in which any combination of pre-trained vision encoder and LLM can be used (learn more in the `),Le=r(Ls,"A",{href:!0,rel:!0});var Mp=i(Le);Mr=n(Mp,"BLIP-2 blog post"),Mp.forEach(s),Jr=n(Ls,`).
This enables achieving state-of-the-art results on multiple visual-language tasks including visual question answering.`),Ls.forEach(s),tl=c(e),le=r(e,"P",{});var Zl=i(le);_r=n(Zl,`Let\u2019s illustrate how you can use this model for VQA. First, let\u2019s load the model. Here we\u2019ll explicitly send the model to a
GPU, if available, which we didn\u2019t need to do earlier when training, as `),As=r(Zl,"A",{href:!0});var Jp=i(As);Tr=n(Jp,"Trainer"),Jp.forEach(s),Ur=n(Zl," handles this automatically:"),Zl.forEach(s),ll=c(e),u(Pe.$$.fragment,e),nl=c(e),Bs=r(e,"P",{});var _p=i(Bs);Er=n(_p,"The model takes image and text as input, so let\u2019s use the exact same image/question pair from the first example in the VQA dataset:"),_p.forEach(s),ol=c(e),u(Se.$$.fragment,e),rl=c(e),ne=r(e,"P",{});var Il=i(ne);kr=n(Il,"To use BLIP-2 for visual question answering task, the textual prompt has to follow a specific format: "),Qa=r(Il,"CODE",{});var Tp=i(Qa);xr=n(Tp,"Question: {} Answer:"),Tp.forEach(s),$r=n(Il,"."),Il.forEach(s),il=c(e),u(De.$$.fragment,e),pl=c(e),Qs=r(e,"P",{});var Up=i(Qs);Vr=n(Up,"Now we need to preprocess the image/prompt with the model\u2019s processor, pass the processed input through the model, and decode the output:"),Up.forEach(s),hl=c(e),u(Ke.$$.fragment,e),cl=c(e),Xs=r(e,"P",{});var Ep=i(Xs);Zr=n(Ep,`As you can see, the model recognized the crowd, and the direction of the face (looking down), however, it seems to miss
the fact the crowd is behind the skater. Still, in cases where acquiring human-annotated datasets is not feasible, this
approach can quickly produce useful results.`),Ep.forEach(s),this.h()},h(){m(Z,"name","hf:doc:metadata"),m(Z,"content",JSON.stringify(Ap)),m(F,"id","visual-question-answering"),m(F,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(F,"href","#visual-question-answering"),m(I,"class","relative group"),m(ts,"href","../model_doc/vilt"),m(R,"href","https://huggingface.co/datasets/Graphcore/vqa"),m(R,"rel","nofollow"),m(N,"id","finetuning-vilt"),m(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(N,"href","#finetuning-vilt"),m(W,"class","relative group"),m(q,"id","load-the-data"),m(q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(q,"href","#load-the-data"),m(C,"class","relative group"),m(ge,"href","https://huggingface.co/datasets/Graphcore/vqa"),m(ge,"rel","nofollow"),m(z,"href","https://huggingface.co/datasets/Graphcore/vqa"),m(z,"rel","nofollow"),m(ye,"href","https://visualqa.org/download.html"),m(ye,"rel","nofollow"),m(we,"href","https://huggingface.co/docs/datasets/image_dataset#loading-script"),m(we,"rel","nofollow"),Zp(gs.src,Qr="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/vqa-example.png")||m(gs,"src",Qr),m(gs,"alt","VQA Image Example"),m(Je,"class","flex justify-center"),m(H,"id","preprocessing-data"),m(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(H,"href","#preprocessing-data"),m(A,"class","relative group"),m(js,"href","/docs/transformers/main/en/model_doc/vilt#transformers.ViltProcessor"),m(vs,"href","/docs/transformers/main/en/model_doc/vilt#transformers.ViltProcessor"),m(Ms,"href","/docs/transformers/main/en/model_doc/bert#transformers.BertTokenizerFast"),m(Js,"href","/docs/transformers/main/en/model_doc/vilt#transformers.ViltImageProcessor"),m(_s,"href","/docs/transformers/main/en/main_classes/data_collator#transformers.DefaultDataCollator"),m(K,"id","train-the-model"),m(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(K,"href","#train-the-model"),m(B,"class","relative group"),m(Ts,"href","/docs/transformers/main/en/model_doc/vilt#transformers.ViltForQuestionAnswering"),m(ks,"href","/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments"),m(xs,"href","/docs/transformers/main/en/main_classes/trainer#transformers.Trainer"),m(We,"start","2"),m($s,"href","/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.train"),m(Be,"start","3"),m(Vs,"href","/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.push_to_hub"),m(se,"id","inference"),m(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(se,"href","#inference"),m(Q,"class","relative group"),m(Zs,"href","/docs/transformers/main/en/main_classes/pipelines#transformers.Pipeline"),m(te,"id","zeroshot-vqa"),m(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(te,"href","#zeroshot-vqa"),m(X,"class","relative group"),m(Cs,"href","../model_doc/blip-2"),m(Le,"href","https://huggingface.co/blog/blip-2"),m(Le,"rel","nofollow"),m(As,"href","/docs/transformers/main/en/main_classes/trainer#transformers.Trainer")},m(e,t){a(document.head,Z),p(e,Xa,t),p(e,I,t),a(I,F),a(F,Ps),f(pe,Ps,null),a(I,Gl),a(I,Ss),a(Ss,Wl),p(e,Fa,t),f(he,e,t),p(e,Ra,t),p(e,es,t),a(es,Cl),p(e,Na,t),p(e,ss,t),a(ss,Al),p(e,qa,t),p(e,M,t),a(M,Ds),a(Ds,Bl),a(M,Ql),a(M,Ks),a(Ks,Xl),a(M,Fl),a(M,Os),a(Os,Rl),a(M,Nl),a(M,ea),a(ea,ql),p(e,za,t),p(e,as,t),a(as,zl),p(e,Ya,t),p(e,T,t),a(T,G),a(G,Yl),a(G,ts),a(ts,Hl),a(G,Ll),a(G,R),a(R,sa),a(sa,Pl),a(R,Sl),a(G,Dl),a(T,Kl),a(T,aa),a(aa,Ol),a(T,en),a(T,ta),a(ta,sn),p(e,Ha,t),p(e,W,t),a(W,N),a(N,la),f(ce,la,null),a(W,an),a(W,na),a(na,tn),p(e,La,t),p(e,U,t),a(U,ln),a(U,oa),a(oa,nn),a(U,on),a(U,ra),a(ra,rn),a(U,pn),p(e,Pa,t),p(e,ls,t),a(ls,hn),p(e,Sa,t),p(e,ns,t),a(ns,cn),p(e,Da,t),f(me,e,t),p(e,Ka,t),p(e,os,t),a(os,mn),p(e,Oa,t),f(de,e,t),p(e,et,t),p(e,rs,t),a(rs,dn),p(e,st,t),f(ue,e,t),p(e,at,t),p(e,C,t),a(C,q),a(q,ia),f(fe,ia,null),a(C,un),a(C,pa),a(pa,fn),p(e,tt,t),p(e,E,t),a(E,gn),a(E,ha),a(ha,yn),a(E,wn),a(E,ge),a(ge,bn),a(E,jn),p(e,lt,t),p(e,J,t),a(J,vn),a(J,z),a(z,ca),a(ca,Mn),a(z,Jn),a(J,_n),a(J,ye),a(ye,Tn),a(J,Un),a(J,we),a(we,En),a(J,kn),p(e,nt,t),p(e,is,t),a(is,xn),p(e,ot,t),f(be,e,t),p(e,rt,t),p(e,ps,t),a(ps,$n),p(e,it,t),f(je,e,t),p(e,pt,t),p(e,hs,t),a(hs,Vn),p(e,ht,t),p(e,k,t),a(k,cs),a(cs,ma),a(ma,Zn),a(cs,In),a(k,Gn),a(k,ms),a(ms,da),a(da,Wn),a(ms,Cn),a(k,An),a(k,ds),a(ds,ua),a(ua,Bn),a(ds,Qn),p(e,ct,t),p(e,us,t),a(us,Xn),p(e,mt,t),f(ve,e,t),p(e,dt,t),p(e,x,t),a(x,Fn),a(x,fa),a(fa,Rn),a(x,Nn),a(x,ga),a(ga,qn),a(x,zn),p(e,ut,t),p(e,fs,t),a(fs,Yn),p(e,ft,t),f(Me,e,t),p(e,gt,t),p(e,Je,t),a(Je,gs),p(e,yt,t),p(e,ys,t),a(ys,Hn),p(e,wt,t),p(e,Y,t),a(Y,Ln),a(Y,ya),a(ya,Pn),a(Y,Sn),p(e,bt,t),p(e,ws,t),a(ws,Dn),p(e,jt,t),f(_e,e,t),p(e,vt,t),p(e,bs,t),a(bs,Kn),p(e,Mt,t),f(Te,e,t),p(e,Jt,t),p(e,A,t),a(A,H),a(H,wa),f(Ue,wa,null),a(A,On),a(A,ba),a(ba,eo),p(e,_t,t),p(e,L,t),a(L,so),a(L,js),a(js,ao),a(L,to),p(e,Tt,t),f(Ee,e,t),p(e,Ut,t),p(e,j,t),a(j,lo),a(j,vs),a(vs,no),a(j,oo),a(j,Ms),a(Ms,ro),a(j,io),a(j,ja),a(ja,po),a(j,ho),a(j,va),a(va,co),a(j,mo),a(j,Ma),a(Ma,uo),a(j,fo),a(j,Js),a(Js,go),a(j,yo),a(j,Ja),a(Ja,wo),a(j,bo),a(j,_a),a(_a,jo),a(j,vo),p(e,Et,t),p(e,P,t),a(P,Mo),a(P,Ta),a(Ta,Jo),a(P,_o),p(e,kt,t),p(e,S,t),a(S,To),a(S,Ua),a(Ua,Uo),a(S,Eo),p(e,xt,t),f(ke,e,t),p(e,$t,t),p(e,_,t),a(_,ko),a(_,Ea),a(Ea,xo),a(_,$o),a(_,ka),a(ka,Vo),a(_,Zo),a(_,xa),a(xa,Io),a(_,Go),p(e,Vt,t),f(xe,e,t),p(e,Zt,t),p(e,D,t),a(D,Wo),a(D,_s),a(_s,Co),a(D,Ao),p(e,It,t),f($e,e,t),p(e,Gt,t),p(e,B,t),a(B,K),a(K,$a),f(Ve,$a,null),a(B,Bo),a(B,Va),a(Va,Qo),p(e,Wt,t),p(e,O,t),a(O,Xo),a(O,Ts),a(Ts,Fo),a(O,Ro),p(e,Ct,t),f(Ze,e,t),p(e,At,t),p(e,Us,t),a(Us,No),p(e,Bt,t),p(e,Es,t),a(Es,Ie),a(Ie,qo),a(Ie,ks),a(ks,zo),a(Ie,Yo),p(e,Qt,t),f(Ge,e,t),p(e,Xt,t),p(e,We,t),a(We,Ce),a(Ce,Ho),a(Ce,xs),a(xs,Lo),a(Ce,Po),p(e,Ft,t),f(Ae,e,t),p(e,Rt,t),p(e,Be,t),a(Be,Qe),a(Qe,So),a(Qe,$s),a($s,Do),a(Qe,Ko),p(e,Nt,t),f(Xe,e,t),p(e,qt,t),p(e,ee,t),a(ee,Oo),a(ee,Vs),a(Vs,er),a(ee,sr),p(e,zt,t),f(Fe,e,t),p(e,Yt,t),p(e,Q,t),a(Q,se),a(se,Za),f(Re,Za,null),a(Q,ar),a(Q,Ia),a(Ia,tr),p(e,Ht,t),p(e,ae,t),a(ae,lr),a(ae,Zs),a(Zs,nr),a(ae,or),p(e,Lt,t),f(Ne,e,t),p(e,Pt,t),p(e,Is,t),a(Is,rr),p(e,St,t),f(qe,e,t),p(e,Dt,t),p(e,Gs,t),a(Gs,ir),p(e,Kt,t),p(e,Ws,t),a(Ws,pr),p(e,Ot,t),p(e,$,t),a($,Ga),a(Ga,hr),a($,cr),a($,Wa),a(Wa,mr),a($,dr),a($,ze),a(ze,ur),a(ze,Ca),a(Ca,fr),a(ze,gr),p(e,el,t),f(Ye,e,t),p(e,sl,t),p(e,X,t),a(X,te),a(te,Aa),f(He,Aa,null),a(X,yr),a(X,Ba),a(Ba,wr),p(e,al,t),p(e,V,t),a(V,br),a(V,Cs),a(Cs,jr),a(V,vr),a(V,Le),a(Le,Mr),a(V,Jr),p(e,tl,t),p(e,le,t),a(le,_r),a(le,As),a(As,Tr),a(le,Ur),p(e,ll,t),f(Pe,e,t),p(e,nl,t),p(e,Bs,t),a(Bs,Er),p(e,ol,t),f(Se,e,t),p(e,rl,t),p(e,ne,t),a(ne,kr),a(ne,Qa),a(Qa,xr),a(ne,$r),p(e,il,t),f(De,e,t),p(e,pl,t),p(e,Qs,t),a(Qs,Vr),p(e,hl,t),f(Ke,e,t),p(e,cl,t),p(e,Xs,t),a(Xs,Zr),ml=!0},p:Ip,i(e){ml||(g(pe.$$.fragment,e),g(he.$$.fragment,e),g(ce.$$.fragment,e),g(me.$$.fragment,e),g(de.$$.fragment,e),g(ue.$$.fragment,e),g(fe.$$.fragment,e),g(be.$$.fragment,e),g(je.$$.fragment,e),g(ve.$$.fragment,e),g(Me.$$.fragment,e),g(_e.$$.fragment,e),g(Te.$$.fragment,e),g(Ue.$$.fragment,e),g(Ee.$$.fragment,e),g(ke.$$.fragment,e),g(xe.$$.fragment,e),g($e.$$.fragment,e),g(Ve.$$.fragment,e),g(Ze.$$.fragment,e),g(Ge.$$.fragment,e),g(Ae.$$.fragment,e),g(Xe.$$.fragment,e),g(Fe.$$.fragment,e),g(Re.$$.fragment,e),g(Ne.$$.fragment,e),g(qe.$$.fragment,e),g(Ye.$$.fragment,e),g(He.$$.fragment,e),g(Pe.$$.fragment,e),g(Se.$$.fragment,e),g(De.$$.fragment,e),g(Ke.$$.fragment,e),ml=!0)},o(e){y(pe.$$.fragment,e),y(he.$$.fragment,e),y(ce.$$.fragment,e),y(me.$$.fragment,e),y(de.$$.fragment,e),y(ue.$$.fragment,e),y(fe.$$.fragment,e),y(be.$$.fragment,e),y(je.$$.fragment,e),y(ve.$$.fragment,e),y(Me.$$.fragment,e),y(_e.$$.fragment,e),y(Te.$$.fragment,e),y(Ue.$$.fragment,e),y(Ee.$$.fragment,e),y(ke.$$.fragment,e),y(xe.$$.fragment,e),y($e.$$.fragment,e),y(Ve.$$.fragment,e),y(Ze.$$.fragment,e),y(Ge.$$.fragment,e),y(Ae.$$.fragment,e),y(Xe.$$.fragment,e),y(Fe.$$.fragment,e),y(Re.$$.fragment,e),y(Ne.$$.fragment,e),y(qe.$$.fragment,e),y(Ye.$$.fragment,e),y(He.$$.fragment,e),y(Pe.$$.fragment,e),y(Se.$$.fragment,e),y(De.$$.fragment,e),y(Ke.$$.fragment,e),ml=!1},d(e){s(Z),e&&s(Xa),e&&s(I),w(pe),e&&s(Fa),w(he,e),e&&s(Ra),e&&s(es),e&&s(Na),e&&s(ss),e&&s(qa),e&&s(M),e&&s(za),e&&s(as),e&&s(Ya),e&&s(T),e&&s(Ha),e&&s(W),w(ce),e&&s(La),e&&s(U),e&&s(Pa),e&&s(ls),e&&s(Sa),e&&s(ns),e&&s(Da),w(me,e),e&&s(Ka),e&&s(os),e&&s(Oa),w(de,e),e&&s(et),e&&s(rs),e&&s(st),w(ue,e),e&&s(at),e&&s(C),w(fe),e&&s(tt),e&&s(E),e&&s(lt),e&&s(J),e&&s(nt),e&&s(is),e&&s(ot),w(be,e),e&&s(rt),e&&s(ps),e&&s(it),w(je,e),e&&s(pt),e&&s(hs),e&&s(ht),e&&s(k),e&&s(ct),e&&s(us),e&&s(mt),w(ve,e),e&&s(dt),e&&s(x),e&&s(ut),e&&s(fs),e&&s(ft),w(Me,e),e&&s(gt),e&&s(Je),e&&s(yt),e&&s(ys),e&&s(wt),e&&s(Y),e&&s(bt),e&&s(ws),e&&s(jt),w(_e,e),e&&s(vt),e&&s(bs),e&&s(Mt),w(Te,e),e&&s(Jt),e&&s(A),w(Ue),e&&s(_t),e&&s(L),e&&s(Tt),w(Ee,e),e&&s(Ut),e&&s(j),e&&s(Et),e&&s(P),e&&s(kt),e&&s(S),e&&s(xt),w(ke,e),e&&s($t),e&&s(_),e&&s(Vt),w(xe,e),e&&s(Zt),e&&s(D),e&&s(It),w($e,e),e&&s(Gt),e&&s(B),w(Ve),e&&s(Wt),e&&s(O),e&&s(Ct),w(Ze,e),e&&s(At),e&&s(Us),e&&s(Bt),e&&s(Es),e&&s(Qt),w(Ge,e),e&&s(Xt),e&&s(We),e&&s(Ft),w(Ae,e),e&&s(Rt),e&&s(Be),e&&s(Nt),w(Xe,e),e&&s(qt),e&&s(ee),e&&s(zt),w(Fe,e),e&&s(Yt),e&&s(Q),w(Re),e&&s(Ht),e&&s(ae),e&&s(Lt),w(Ne,e),e&&s(Pt),e&&s(Is),e&&s(St),w(qe,e),e&&s(Dt),e&&s(Gs),e&&s(Kt),e&&s(Ws),e&&s(Ot),e&&s($),e&&s(el),w(Ye,e),e&&s(sl),e&&s(X),w(He),e&&s(al),e&&s(V),e&&s(tl),e&&s(le),e&&s(ll),w(Pe,e),e&&s(nl),e&&s(Bs),e&&s(ol),w(Se,e),e&&s(rl),e&&s(ne),e&&s(il),w(De,e),e&&s(pl),e&&s(Qs),e&&s(hl),w(Ke,e),e&&s(cl),e&&s(Xs)}}}const Ap={local:"visual-question-answering",sections:[{local:"finetuning-vilt",title:"Fine-tuning ViLT"},{local:"load-the-data",title:"Load the data"},{local:"preprocessing-data",title:"Preprocessing data"},{local:"train-the-model",title:"Train the model"},{local:"inference",title:"Inference"},{local:"zeroshot-vqa",title:"Zero-shot VQA"}],title:"Visual Question Answering"};function Bp(Br){return Gp(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Np extends kp{constructor(Z){super();xp(this,Z,Bp,Cp,$p,{})}}export{Np as default,Ap as metadata};
