import{s as re,o as pe,n as ce}from"../chunks/scheduler.9bc65507.js";import{S as me,i as ue,g as r,s as l,r as _,A as de,h as p,f as n,c as i,j as le,u as k,x as m,k as ie,y as he,a as s,v as x,d as R,t as U,w as Z}from"../chunks/index.707bf1b6.js";import{T as fe}from"../chunks/Tip.c2ecdbf4.js";import{C as oe}from"../chunks/CodeBlock.54a9f38d.js";import{H as Q}from"../chunks/Heading.342b1fa6.js";function ge(B){let a,u='DePlot is a model trained using <code>Pix2Struct</code> architecture. For API reference, see <a href="pix2struct"><code>Pix2Struct</code> documentation</a>.';return{c(){a=r("p"),a.innerHTML=u},l(o){a=p(o,"P",{"data-svelte-h":!0}),m(a)!=="svelte-zgoioq"&&(a.innerHTML=u)},m(o,P){s(o,a,P)},p:ce,d(o){o&&n(a)}}}function Me(B){let a,u,o,P,d,C,h,G,f,q='DePlot was proposed in the paper <a href="https://arxiv.org/abs/2212.10505" rel="nofollow">DePlot: One-shot visual language reasoning by plot-to-table translation</a> from Fangyu Liu, Julian Martin Eisenschlos, Francesco Piccinno, Syrine Krichene, Chenxi Pang, Kenton Lee, Mandar Joshi, Wenhu Chen, Nigel Collier, Yasemin Altun.',X,g,K="The abstract of the paper states the following:",V,M,O="<em>Visual language such as charts and plots is ubiquitous in the human world. Comprehending plots and charts requires strong reasoning skills. Prior state-of-the-art (SOTA) models require at least tens of thousands of training examples and their reasoning capabilities are still much limited, especially on complex human-written queries. This paper presents the first one-shot solution to visual language reasoning. We decompose the challenge of visual language reasoning into two steps: (1) plot-to-text translation, and (2) reasoning over the translated text. The key in this method is a modality conversion module, named as DePlot, which translates the image of a plot or chart to a linearized table. The output of DePlot can then be directly used to prompt a pretrained large language model (LLM), exploiting the few-shot reasoning capabilities of LLMs. To obtain DePlot, we standardize the plot-to-table task by establishing unified task formats and metrics, and train DePlot end-to-end on this task. DePlot can then be used off-the-shelf together with LLMs in a plug-and-play fashion. Compared with a SOTA model finetuned on more than &gt;28k data points, DePlot+LLM with just one-shot prompting achieves a 24.0% improvement over finetuned SOTA on human-written queries from the task of chart QA.</em>",Y,w,ee=`DePlot is a model that is trained using <code>Pix2Struct</code> architecture. You can find more information about <code>Pix2Struct</code> in the <a href="https://huggingface.co/docs/transformers/main/en/model_doc/pix2struct" rel="nofollow">Pix2Struct documentation</a>.
DePlot is a Visual Question Answering subset of <code>Pix2Struct</code> architecture. It renders the input question on the image and predicts the answer.`,H,y,I,b,te="Currently one checkpoint is available for DePlot:",z,T,ne="<li><code>google/deplot</code>: DePlot fine-tuned on ChartQA dataset</li>",F,J,D,$,L,v,se='To fine-tune DePlot, refer to the pix2struct <a href="https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_pix2struct.ipynb" rel="nofollow">fine-tuning notebook</a>. For <code>Pix2Struct</code> models, we have found out that fine-tuning the model with Adafactor and cosine learning rate scheduler leads to faster convergence:',S,j,E,c,N,W,A;return d=new Q({props:{title:"DePlot",local:"deplot",headingTag:"h1"}}),h=new Q({props:{title:"Overview",local:"overview",headingTag:"h2"}}),y=new Q({props:{title:"Usage example",local:"usage-example",headingTag:"h2"}}),J=new oe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMkMlMjBQaXgyU3RydWN0Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEElMEFtb2RlbCUyMCUzRCUyMFBpeDJTdHJ1Y3RGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmRlcGxvdCUyMiklMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUlMkZkZXBsb3QlMjIpJTBBdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZyYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tJTJGdmlzLW5scCUyRkNoYXJ0UUElMkZtYWluJTJGQ2hhcnRRQSUyNTIwRGF0YXNldCUyRnZhbCUyRnBuZyUyRjUwOTAucG5nJTIyJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlcyUzRGltYWdlJTJDJTIwdGV4dCUzRCUyMkdlbmVyYXRlJTIwdW5kZXJseWluZyUyMGRhdGElMjB0YWJsZSUyMG9mJTIwdGhlJTIwZmlndXJlJTIwYmVsb3clM0ElMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQXByZWRpY3Rpb25zJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDUxMiklMEFwcmludChwcm9jZXNzb3IuZGVjb2RlKHByZWRpY3Rpb25zJTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, Pix2StructForConditionalGeneration
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

model = Pix2StructForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
url = <span class="hljs-string">&quot;https://raw.githubusercontent.com/vis-nlp/ChartQA/main/ChartQA%20Dataset/val/png/5090.png&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

inputs = processor(images=image, text=<span class="hljs-string">&quot;Generate underlying data table of the figure below:&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
predictions = model.generate(**inputs, max_new_tokens=<span class="hljs-number">512</span>)
<span class="hljs-built_in">print</span>(processor.decode(predictions[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),$=new Q({props:{title:"Fine-tuning",local:"fine-tuning",headingTag:"h2"}}),j=new oe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5vcHRpbWl6YXRpb24lMjBpbXBvcnQlMjBBZGFmYWN0b3IlMkMlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwJTBBJTBBb3B0aW1pemVyJTIwJTNEJTIwQWRhZmFjdG9yKHNlbGYucGFyYW1ldGVycygpJTJDJTIwc2NhbGVfcGFyYW1ldGVyJTNERmFsc2UlMkMlMjByZWxhdGl2ZV9zdGVwJTNERmFsc2UlMkMlMjBsciUzRDAuMDElMkMlMjB3ZWlnaHRfZGVjYXklM0QxZS0wNSklMEFzY2hlZHVsZXIlMjAlM0QlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwKG9wdGltaXplciUyQyUyMG51bV93YXJtdXBfc3RlcHMlM0QxMDAwJTJDJTIwbnVtX3RyYWluaW5nX3N0ZXBzJTNENDAwMDAp",highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, get_cosine_schedule_with_warmup

optimizer = Adafactor(self.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">0.01</span>, weight_decay=<span class="hljs-number">1e-05</span>)
scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=<span class="hljs-number">1000</span>, num_training_steps=<span class="hljs-number">40000</span>)`,wrap:!1}}),c=new fe({props:{$$slots:{default:[ge]},$$scope:{ctx:B}}}),{c(){a=r("meta"),u=l(),o=r("p"),P=l(),_(d.$$.fragment),C=l(),_(h.$$.fragment),G=l(),f=r("p"),f.innerHTML=q,X=l(),g=r("p"),g.textContent=K,V=l(),M=r("p"),M.innerHTML=O,Y=l(),w=r("p"),w.innerHTML=ee,H=l(),_(y.$$.fragment),I=l(),b=r("p"),b.textContent=te,z=l(),T=r("ul"),T.innerHTML=ne,F=l(),_(J.$$.fragment),D=l(),_($.$$.fragment),L=l(),v=r("p"),v.innerHTML=se,S=l(),_(j.$$.fragment),E=l(),_(c.$$.fragment),N=l(),W=r("p"),this.h()},l(e){const t=de("svelte-u9bgzb",document.head);a=p(t,"META",{name:!0,content:!0}),t.forEach(n),u=i(e),o=p(e,"P",{}),le(o).forEach(n),P=i(e),k(d.$$.fragment,e),C=i(e),k(h.$$.fragment,e),G=i(e),f=p(e,"P",{"data-svelte-h":!0}),m(f)!=="svelte-19mspuy"&&(f.innerHTML=q),X=i(e),g=p(e,"P",{"data-svelte-h":!0}),m(g)!=="svelte-1pvthah"&&(g.textContent=K),V=i(e),M=p(e,"P",{"data-svelte-h":!0}),m(M)!=="svelte-tgrbdp"&&(M.innerHTML=O),Y=i(e),w=p(e,"P",{"data-svelte-h":!0}),m(w)!=="svelte-1ihm3oy"&&(w.innerHTML=ee),H=i(e),k(y.$$.fragment,e),I=i(e),b=p(e,"P",{"data-svelte-h":!0}),m(b)!=="svelte-1010knq"&&(b.textContent=te),z=i(e),T=p(e,"UL",{"data-svelte-h":!0}),m(T)!=="svelte-kycb4s"&&(T.innerHTML=ne),F=i(e),k(J.$$.fragment,e),D=i(e),k($.$$.fragment,e),L=i(e),v=p(e,"P",{"data-svelte-h":!0}),m(v)!=="svelte-11vjzlg"&&(v.innerHTML=se),S=i(e),k(j.$$.fragment,e),E=i(e),k(c.$$.fragment,e),N=i(e),W=p(e,"P",{}),le(W).forEach(n),this.h()},h(){ie(a,"name","hf:doc:metadata"),ie(a,"content",we)},m(e,t){he(document.head,a),s(e,u,t),s(e,o,t),s(e,P,t),x(d,e,t),s(e,C,t),x(h,e,t),s(e,G,t),s(e,f,t),s(e,X,t),s(e,g,t),s(e,V,t),s(e,M,t),s(e,Y,t),s(e,w,t),s(e,H,t),x(y,e,t),s(e,I,t),s(e,b,t),s(e,z,t),s(e,T,t),s(e,F,t),x(J,e,t),s(e,D,t),x($,e,t),s(e,L,t),s(e,v,t),s(e,S,t),x(j,e,t),s(e,E,t),x(c,e,t),s(e,N,t),s(e,W,t),A=!0},p(e,[t]){const ae={};t&2&&(ae.$$scope={dirty:t,ctx:e}),c.$set(ae)},i(e){A||(R(d.$$.fragment,e),R(h.$$.fragment,e),R(y.$$.fragment,e),R(J.$$.fragment,e),R($.$$.fragment,e),R(j.$$.fragment,e),R(c.$$.fragment,e),A=!0)},o(e){U(d.$$.fragment,e),U(h.$$.fragment,e),U(y.$$.fragment,e),U(J.$$.fragment,e),U($.$$.fragment,e),U(j.$$.fragment,e),U(c.$$.fragment,e),A=!1},d(e){e&&(n(u),n(o),n(P),n(C),n(G),n(f),n(X),n(g),n(V),n(M),n(Y),n(w),n(H),n(I),n(b),n(z),n(T),n(F),n(D),n(L),n(v),n(S),n(E),n(N),n(W)),n(a),Z(d,e),Z(h,e),Z(y,e),Z(J,e),Z($,e),Z(j,e),Z(c,e)}}}const we='{"title":"DePlot","local":"deplot","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage example","local":"usage-example","sections":[],"depth":2},{"title":"Fine-tuning","local":"fine-tuning","sections":[],"depth":2}],"depth":1}';function ye(B){return pe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class je extends me{constructor(a){super(),ue(this,a,ye,Me,re,{})}}export{je as component};
