import{s as Oe,n as et,o as tt}from"../chunks/scheduler.9bc65507.js";import{S as nt,i as at,g as a,s as r,u as X,A as lt,h as l,f as t,c as i,j as h,v as E,y as c,k as o,o as p,a as s,w as V,d as A,t as H,x as N}from"../chunks/index.78c82d43.js";import{I as te}from"../chunks/IconCopyLink.bedaa44d.js";import{C as Ke}from"../chunks/CodeBlock.48618ec2.js";function st($e){let u,ne,D,ae,d,w,L,_,ke,S,Re="DePlot",le,m,y,Q,k,Te,Y,Pe="Overview",se,T,Ue='DePlot was proposed in the paper <a href="https://arxiv.org/abs/2212.10505" rel="nofollow">DePlot: One-shot visual language reasoning by plot-to-table translation</a> from Fangyu Liu, Julian Martin Eisenschlos, Francesco Piccinno, Syrine Krichene, Chenxi Pang, Kenton Lee, Mandar Joshi, Wenhu Chen, Nigel Collier, Yasemin Altun.',oe,x,Ze="The abstract of the paper states the following:",re,C,We="<em>Visual language such as charts and plots is ubiquitous in the human world. Comprehending plots and charts requires strong reasoning skills. Prior state-of-the-art (SOTA) models require at least tens of thousands of training examples and their reasoning capabilities are still much limited, especially on complex human-written queries. This paper presents the first one-shot solution to visual language reasoning. We decompose the challenge of visual language reasoning into two steps: (1) plot-to-text translation, and (2) reasoning over the translated text. The key in this method is a modality conversion module, named as DePlot, which translates the image of a plot or chart to a linearized table. The output of DePlot can then be directly used to prompt a pretrained large language model (LLM), exploiting the few-shot reasoning capabilities of LLMs. To obtain DePlot, we standardize the plot-to-table task by establishing unified task formats and metrics, and train DePlot end-to-end on this task. DePlot can then be used off-the-shelf together with LLMs in a plug-and-play fashion. Compared with a SOTA model finetuned on more than &gt;28k data points, DePlot+LLM with just one-shot prompting achieves a 24.0% improvement over finetuned SOTA on human-written queries from the task of chart QA.</em>",ie,f,b,q,j,xe,I,Be="Model description",pe,$,Ge=`DePlot is a model that is trained using <code>Pix2Struct</code> architecture. You can find more information about <code>Pix2Struct</code> in the <a href="https://huggingface.co/docs/transformers/main/en/model_doc/pix2struct" rel="nofollow">Pix2Struct documentation</a>.
DePlot is a Visual Question Answering subset of <code>Pix2Struct</code> architecture. It renders the input question on the image and predicts the answer.`,he,v,M,K,R,Ce,z,Xe="Usage",ce,P,Ee="Currently one checkpoint is available for DePlot:",ue,U,Ve="<li><code>google/deplot</code>: DePlot fine-tuned on ChartQA dataset</li>",de,Z,me,g,J,O,W,je,F,Ae="Fine-tuning",fe,B,He='To fine-tune DePlot, refer to the pix2struct <a href="https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_pix2struct.ipynb" rel="nofollow">fine-tuning notebook</a>. For <code>Pix2Struct</code> models, we have found out that fine-tuning the model with Adafactor and cosine learning rate scheduler leads to faster convergence:',ve,G,ge,ee,we;return _=new te({}),k=new te({}),j=new te({}),R=new te({}),Z=new Ke({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMkMlMjBQaXgyU3RydWN0Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEElMEFtb2RlbCUyMCUzRCUyMFBpeDJTdHJ1Y3RGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmRlcGxvdCUyMiklMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUlMkZkZXBsb3QlMjIpJTBBdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZyYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tJTJGdmlzLW5scCUyRkNoYXJ0UUElMkZtYWluJTJGQ2hhcnRRQSUyNTIwRGF0YXNldCUyRnZhbCUyRnBuZyUyRjUwOTAucG5nJTIyJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlcyUzRGltYWdlJTJDJTIwdGV4dCUzRCUyMkdlbmVyYXRlJTIwdW5kZXJseWluZyUyMGRhdGElMjB0YWJsZSUyMG9mJTIwdGhlJTIwZmlndXJlJTIwYmVsb3clM0ElMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQXByZWRpY3Rpb25zJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDUxMiklMEFwcmludChwcm9jZXNzb3IuZGVjb2RlKHByZWRpY3Rpb25zJTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, Pix2StructForConditionalGeneration
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

model = Pix2StructForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
url = <span class="hljs-string">&quot;https://raw.githubusercontent.com/vis-nlp/ChartQA/main/ChartQA%20Dataset/val/png/5090.png&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

inputs = processor(images=image, text=<span class="hljs-string">&quot;Generate underlying data table of the figure below:&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
predictions = model.generate(**inputs, max_new_tokens=<span class="hljs-number">512</span>)
<span class="hljs-built_in">print</span>(processor.decode(predictions[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`}}),W=new te({}),G=new Ke({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5vcHRpbWl6YXRpb24lMjBpbXBvcnQlMjBBZGFmYWN0b3IlMkMlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwJTBBJTBBb3B0aW1pemVyJTIwJTNEJTIwQWRhZmFjdG9yKHNlbGYucGFyYW1ldGVycygpJTJDJTIwc2NhbGVfcGFyYW1ldGVyJTNERmFsc2UlMkMlMjByZWxhdGl2ZV9zdGVwJTNERmFsc2UlMkMlMjBsciUzRDAuMDElMkMlMjB3ZWlnaHRfZGVjYXklM0QxZS0wNSklMEFzY2hlZHVsZXIlMjAlM0QlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwKG9wdGltaXplciUyQyUyMG51bV93YXJtdXBfc3RlcHMlM0QxMDAwJTJDJTIwbnVtX3RyYWluaW5nX3N0ZXBzJTNENDAwMDAp",highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, get_cosine_schedule_with_warmup

optimizer = Adafactor(self.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">0.01</span>, weight_decay=<span class="hljs-number">1e-05</span>)
scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=<span class="hljs-number">1000</span>, num_training_steps=<span class="hljs-number">40000</span>)`}}),{c(){u=a("meta"),ne=r(),D=a("p"),ae=r(),d=a("h1"),w=a("a"),L=a("span"),X(_.$$.fragment),ke=r(),S=a("span"),S.textContent=Re,le=r(),m=a("h2"),y=a("a"),Q=a("span"),X(k.$$.fragment),Te=r(),Y=a("span"),Y.textContent=Pe,se=r(),T=a("p"),T.innerHTML=Ue,oe=r(),x=a("p"),x.textContent=Ze,re=r(),C=a("p"),C.innerHTML=We,ie=r(),f=a("h2"),b=a("a"),q=a("span"),X(j.$$.fragment),xe=r(),I=a("span"),I.textContent=Be,pe=r(),$=a("p"),$.innerHTML=Ge,he=r(),v=a("h2"),M=a("a"),K=a("span"),X(R.$$.fragment),Ce=r(),z=a("span"),z.textContent=Xe,ce=r(),P=a("p"),P.textContent=Ee,ue=r(),U=a("ul"),U.innerHTML=Ve,de=r(),X(Z.$$.fragment),me=r(),g=a("h2"),J=a("a"),O=a("span"),X(W.$$.fragment),je=r(),F=a("span"),F.textContent=Ae,fe=r(),B=a("p"),B.innerHTML=He,ve=r(),X(G.$$.fragment),ge=r(),ee=a("p"),this.h()},l(e){const n=lt("svelte-1phssyn",document.head);u=l(n,"META",{name:!0,content:!0}),n.forEach(t),ne=i(e),D=l(e,"P",{}),h(D).forEach(t),ae=i(e),d=l(e,"H1",{class:!0});var ye=h(d);w=l(ye,"A",{id:!0,class:!0,href:!0});var Ne=h(w);L=l(Ne,"SPAN",{});var Se=h(L);E(_.$$.fragment,Se),Se.forEach(t),Ne.forEach(t),ke=i(ye),S=l(ye,"SPAN",{"data-svelte-h":!0}),c(S)!=="svelte-15u2ejx"&&(S.textContent=Re),ye.forEach(t),le=i(e),m=l(e,"H2",{class:!0});var be=h(m);y=l(be,"A",{id:!0,class:!0,href:!0});var Ye=h(y);Q=l(Ye,"SPAN",{});var Ie=h(Q);E(k.$$.fragment,Ie),Ie.forEach(t),Ye.forEach(t),Te=i(be),Y=l(be,"SPAN",{"data-svelte-h":!0}),c(Y)!=="svelte-wgb1ai"&&(Y.textContent=Pe),be.forEach(t),se=i(e),T=l(e,"P",{"data-svelte-h":!0}),c(T)!=="svelte-19mspuy"&&(T.innerHTML=Ue),oe=i(e),x=l(e,"P",{"data-svelte-h":!0}),c(x)!=="svelte-1pvthah"&&(x.textContent=Ze),re=i(e),C=l(e,"P",{"data-svelte-h":!0}),c(C)!=="svelte-tgrbdp"&&(C.innerHTML=We),ie=i(e),f=l(e,"H2",{class:!0});var Me=h(f);b=l(Me,"A",{id:!0,class:!0,href:!0});var ze=h(b);q=l(ze,"SPAN",{});var Fe=h(q);E(j.$$.fragment,Fe),Fe.forEach(t),ze.forEach(t),xe=i(Me),I=l(Me,"SPAN",{"data-svelte-h":!0}),c(I)!=="svelte-10tgin4"&&(I.textContent=Be),Me.forEach(t),pe=i(e),$=l(e,"P",{"data-svelte-h":!0}),c($)!=="svelte-1ihm3oy"&&($.innerHTML=Ge),he=i(e),v=l(e,"H2",{class:!0});var Je=h(v);M=l(Je,"A",{id:!0,class:!0,href:!0});var De=h(M);K=l(De,"SPAN",{});var Le=h(K);E(R.$$.fragment,Le),Le.forEach(t),De.forEach(t),Ce=i(Je),z=l(Je,"SPAN",{"data-svelte-h":!0}),c(z)!=="svelte-jwh4bg"&&(z.textContent=Xe),Je.forEach(t),ce=i(e),P=l(e,"P",{"data-svelte-h":!0}),c(P)!=="svelte-1010knq"&&(P.textContent=Ee),ue=i(e),U=l(e,"UL",{"data-svelte-h":!0}),c(U)!=="svelte-kycb4s"&&(U.innerHTML=Ve),de=i(e),E(Z.$$.fragment,e),me=i(e),g=l(e,"H2",{class:!0});var _e=h(g);J=l(_e,"A",{id:!0,class:!0,href:!0});var Qe=h(J);O=l(Qe,"SPAN",{});var qe=h(O);E(W.$$.fragment,qe),qe.forEach(t),Qe.forEach(t),je=i(_e),F=l(_e,"SPAN",{"data-svelte-h":!0}),c(F)!=="svelte-nl2q21"&&(F.textContent=Ae),_e.forEach(t),fe=i(e),B=l(e,"P",{"data-svelte-h":!0}),c(B)!=="svelte-11vjzlg"&&(B.innerHTML=He),ve=i(e),E(G.$$.fragment,e),ge=i(e),ee=l(e,"P",{}),h(ee).forEach(t),this.h()},h(){o(u,"name","hf:doc:metadata"),o(u,"content",JSON.stringify(ot)),o(w,"id","deplot"),o(w,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(w,"href","#deplot"),o(d,"class","relative group"),o(y,"id","overview"),o(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(y,"href","#overview"),o(m,"class","relative group"),o(b,"id","model-description"),o(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(b,"href","#model-description"),o(f,"class","relative group"),o(M,"id","usage"),o(M,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(M,"href","#usage"),o(v,"class","relative group"),o(J,"id","finetuning"),o(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(J,"href","#finetuning"),o(g,"class","relative group")},m(e,n){p(document.head,u),s(e,ne,n),s(e,D,n),s(e,ae,n),s(e,d,n),p(d,w),p(w,L),V(_,L,null),p(d,ke),p(d,S),s(e,le,n),s(e,m,n),p(m,y),p(y,Q),V(k,Q,null),p(m,Te),p(m,Y),s(e,se,n),s(e,T,n),s(e,oe,n),s(e,x,n),s(e,re,n),s(e,C,n),s(e,ie,n),s(e,f,n),p(f,b),p(b,q),V(j,q,null),p(f,xe),p(f,I),s(e,pe,n),s(e,$,n),s(e,he,n),s(e,v,n),p(v,M),p(M,K),V(R,K,null),p(v,Ce),p(v,z),s(e,ce,n),s(e,P,n),s(e,ue,n),s(e,U,n),s(e,de,n),V(Z,e,n),s(e,me,n),s(e,g,n),p(g,J),p(J,O),V(W,O,null),p(g,je),p(g,F),s(e,fe,n),s(e,B,n),s(e,ve,n),V(G,e,n),s(e,ge,n),s(e,ee,n),we=!0},p:et,i(e){we||(A(_.$$.fragment,e),A(k.$$.fragment,e),A(j.$$.fragment,e),A(R.$$.fragment,e),A(Z.$$.fragment,e),A(W.$$.fragment,e),A(G.$$.fragment,e),we=!0)},o(e){H(_.$$.fragment,e),H(k.$$.fragment,e),H(j.$$.fragment,e),H(R.$$.fragment,e),H(Z.$$.fragment,e),H(W.$$.fragment,e),H(G.$$.fragment,e),we=!1},d(e){e&&(t(ne),t(D),t(ae),t(d),t(le),t(m),t(se),t(T),t(oe),t(x),t(re),t(C),t(ie),t(f),t(pe),t($),t(he),t(v),t(ce),t(P),t(ue),t(U),t(de),t(me),t(g),t(fe),t(B),t(ve),t(ge),t(ee)),t(u),N(_),N(k),N(j),N(R),N(Z,e),N(W),N(G,e)}}}const ot={local:"deplot",sections:[{local:"overview",title:"Overview "},{local:"model-description",title:"Model description"},{local:"usage",title:"Usage"},{local:"finetuning",title:"Fine-tuning"}],title:"DePlot"};function rt($e){return tt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ut extends nt{constructor(u){super(),at(this,u,rt,st,Oe,{})}}export{ut as component};
