import{s as de,o as he,n as fe}from"../chunks/scheduler.01eeda35.js";import{S as ge,i as Me,g as o,s as l,r as d,A as be,h as r,f as n,c as a,j as ce,u as h,x as m,k as te,y as ye,a as s,v as f,d as g,t as M,w as b}from"../chunks/index.6dd51b66.js";import{T as we}from"../chunks/Tip.de9bae2b.js";import{C as ue}from"../chunks/CodeBlock.864da1b0.js";import{H as ee,E as Te}from"../chunks/EditOnGithub.7faefd25.js";function $e(G){let i,y='DePlot is a model trained using <code>Pix2Struct</code> architecture. For API reference, see <a href="pix2struct"><code>Pix2Struct</code> documentation</a>.';return{c(){i=o("p"),i.innerHTML=y},l(p){i=r(p,"P",{"data-svelte-h":!0}),m(i)!=="svelte-zgoioq"&&(i.innerHTML=y)},m(p,W){s(p,i,W)},p:fe,d(p){p&&n(i)}}}function Je(G){let i,y,p,W,w,X,c,ne='<img alt="PyTorch" src="https://img.shields.io/badge/PyTorch-DE3412?style=flat&amp;logo=pytorch&amp;logoColor=white"/>',V,T,H,$,se='DePlot was proposed in the paper <a href="https://arxiv.org/abs/2212.10505" rel="nofollow">DePlot: One-shot visual language reasoning by plot-to-table translation</a> from Fangyu Liu, Julian Martin Eisenschlos, Francesco Piccinno, Syrine Krichene, Chenxi Pang, Kenton Lee, Mandar Joshi, Wenhu Chen, Nigel Collier, Yasemin Altun.',Y,J,le="The abstract of the paper states the following:",I,v,ae="<em>Visual language such as charts and plots is ubiquitous in the human world. Comprehending plots and charts requires strong reasoning skills. Prior state-of-the-art (SOTA) models require at least tens of thousands of training examples and their reasoning capabilities are still much limited, especially on complex human-written queries. This paper presents the first one-shot solution to visual language reasoning. We decompose the challenge of visual language reasoning into two steps: (1) plot-to-text translation, and (2) reasoning over the translated text. The key in this method is a modality conversion module, named as DePlot, which translates the image of a plot or chart to a linearized table. The output of DePlot can then be directly used to prompt a pretrained large language model (LLM), exploiting the few-shot reasoning capabilities of LLMs. To obtain DePlot, we standardize the plot-to-table task by establishing unified task formats and metrics, and train DePlot end-to-end on this task. DePlot can then be used off-the-shelf together with LLMs in a plug-and-play fashion. Compared with a SOTA model finetuned on more than &gt;28k data points, DePlot+LLM with just one-shot prompting achieves a 24.0% improvement over finetuned SOTA on human-written queries from the task of chart QA.</em>",z,j,ie=`DePlot is a model that is trained using <code>Pix2Struct</code> architecture. You can find more information about <code>Pix2Struct</code> in the <a href="https://huggingface.co/docs/transformers/main/en/model_doc/pix2struct" rel="nofollow">Pix2Struct documentation</a>.
DePlot is a Visual Question Answering subset of <code>Pix2Struct</code> architecture. It renders the input question on the image and predicts the answer.`,D,x,E,_,oe="Currently one checkpoint is available for DePlot:",F,k,re="<li><code>google/deplot</code>: DePlot fine-tuned on ChartQA dataset</li>",L,R,S,U,N,P,pe='To fine-tune DePlot, refer to the pix2struct <a href="https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_pix2struct.ipynb" rel="nofollow">fine-tuning notebook</a>. For <code>Pix2Struct</code> models, we have found out that fine-tuning the model with Adafactor and cosine learning rate scheduler leads to faster convergence:',A,C,Q,u,q,Z,K,B,O;return w=new ee({props:{title:"DePlot",local:"deplot",headingTag:"h1"}}),T=new ee({props:{title:"Overview",local:"overview",headingTag:"h2"}}),x=new ee({props:{title:"Usage example",local:"usage-example",headingTag:"h2"}}),R=new ue({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMkMlMjBQaXgyU3RydWN0Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEElMEFtb2RlbCUyMCUzRCUyMFBpeDJTdHJ1Y3RGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmRlcGxvdCUyMiklMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUlMkZkZXBsb3QlMjIpJTBBdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZyYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tJTJGdmlzLW5scCUyRkNoYXJ0UUElMkZtYWluJTJGQ2hhcnRRQSUyNTIwRGF0YXNldCUyRnZhbCUyRnBuZyUyRjUwOTAucG5nJTIyJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlcyUzRGltYWdlJTJDJTIwdGV4dCUzRCUyMkdlbmVyYXRlJTIwdW5kZXJseWluZyUyMGRhdGElMjB0YWJsZSUyMG9mJTIwdGhlJTIwZmlndXJlJTIwYmVsb3clM0ElMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQXByZWRpY3Rpb25zJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDUxMiklMEFwcmludChwcm9jZXNzb3IuZGVjb2RlKHByZWRpY3Rpb25zJTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, Pix2StructForConditionalGeneration
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

model = Pix2StructForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
url = <span class="hljs-string">&quot;https://raw.githubusercontent.com/vis-nlp/ChartQA/main/ChartQA%20Dataset/val/png/5090.png&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

inputs = processor(images=image, text=<span class="hljs-string">&quot;Generate underlying data table of the figure below:&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
predictions = model.generate(**inputs, max_new_tokens=<span class="hljs-number">512</span>)
<span class="hljs-built_in">print</span>(processor.decode(predictions[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),U=new ee({props:{title:"Fine-tuning",local:"fine-tuning",headingTag:"h2"}}),C=new ue({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5vcHRpbWl6YXRpb24lMjBpbXBvcnQlMjBBZGFmYWN0b3IlMkMlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwJTBBJTBBb3B0aW1pemVyJTIwJTNEJTIwQWRhZmFjdG9yKHNlbGYucGFyYW1ldGVycygpJTJDJTIwc2NhbGVfcGFyYW1ldGVyJTNERmFsc2UlMkMlMjByZWxhdGl2ZV9zdGVwJTNERmFsc2UlMkMlMjBsciUzRDAuMDElMkMlMjB3ZWlnaHRfZGVjYXklM0QxZS0wNSklMEFzY2hlZHVsZXIlMjAlM0QlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwKG9wdGltaXplciUyQyUyMG51bV93YXJtdXBfc3RlcHMlM0QxMDAwJTJDJTIwbnVtX3RyYWluaW5nX3N0ZXBzJTNENDAwMDAp",highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, get_cosine_schedule_with_warmup

optimizer = Adafactor(self.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">0.01</span>, weight_decay=<span class="hljs-number">1e-05</span>)
scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=<span class="hljs-number">1000</span>, num_training_steps=<span class="hljs-number">40000</span>)`,wrap:!1}}),u=new we({props:{$$slots:{default:[$e]},$$scope:{ctx:G}}}),Z=new Te({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/deplot.md"}}),{c(){i=o("meta"),y=l(),p=o("p"),W=l(),d(w.$$.fragment),X=l(),c=o("div"),c.innerHTML=ne,V=l(),d(T.$$.fragment),H=l(),$=o("p"),$.innerHTML=se,Y=l(),J=o("p"),J.textContent=le,I=l(),v=o("p"),v.innerHTML=ae,z=l(),j=o("p"),j.innerHTML=ie,D=l(),d(x.$$.fragment),E=l(),_=o("p"),_.textContent=oe,F=l(),k=o("ul"),k.innerHTML=re,L=l(),d(R.$$.fragment),S=l(),d(U.$$.fragment),N=l(),P=o("p"),P.innerHTML=pe,A=l(),d(C.$$.fragment),Q=l(),d(u.$$.fragment),q=l(),d(Z.$$.fragment),K=l(),B=o("p"),this.h()},l(e){const t=be("svelte-u9bgzb",document.head);i=r(t,"META",{name:!0,content:!0}),t.forEach(n),y=a(e),p=r(e,"P",{}),ce(p).forEach(n),W=a(e),h(w.$$.fragment,e),X=a(e),c=r(e,"DIV",{class:!0,"data-svelte-h":!0}),m(c)!=="svelte-13t8s2t"&&(c.innerHTML=ne),V=a(e),h(T.$$.fragment,e),H=a(e),$=r(e,"P",{"data-svelte-h":!0}),m($)!=="svelte-19mspuy"&&($.innerHTML=se),Y=a(e),J=r(e,"P",{"data-svelte-h":!0}),m(J)!=="svelte-1pvthah"&&(J.textContent=le),I=a(e),v=r(e,"P",{"data-svelte-h":!0}),m(v)!=="svelte-tgrbdp"&&(v.innerHTML=ae),z=a(e),j=r(e,"P",{"data-svelte-h":!0}),m(j)!=="svelte-1ihm3oy"&&(j.innerHTML=ie),D=a(e),h(x.$$.fragment,e),E=a(e),_=r(e,"P",{"data-svelte-h":!0}),m(_)!=="svelte-1010knq"&&(_.textContent=oe),F=a(e),k=r(e,"UL",{"data-svelte-h":!0}),m(k)!=="svelte-kycb4s"&&(k.innerHTML=re),L=a(e),h(R.$$.fragment,e),S=a(e),h(U.$$.fragment,e),N=a(e),P=r(e,"P",{"data-svelte-h":!0}),m(P)!=="svelte-11vjzlg"&&(P.innerHTML=pe),A=a(e),h(C.$$.fragment,e),Q=a(e),h(u.$$.fragment,e),q=a(e),h(Z.$$.fragment,e),K=a(e),B=r(e,"P",{}),ce(B).forEach(n),this.h()},h(){te(i,"name","hf:doc:metadata"),te(i,"content",ve),te(c,"class","flex flex-wrap space-x-1")},m(e,t){ye(document.head,i),s(e,y,t),s(e,p,t),s(e,W,t),f(w,e,t),s(e,X,t),s(e,c,t),s(e,V,t),f(T,e,t),s(e,H,t),s(e,$,t),s(e,Y,t),s(e,J,t),s(e,I,t),s(e,v,t),s(e,z,t),s(e,j,t),s(e,D,t),f(x,e,t),s(e,E,t),s(e,_,t),s(e,F,t),s(e,k,t),s(e,L,t),f(R,e,t),s(e,S,t),f(U,e,t),s(e,N,t),s(e,P,t),s(e,A,t),f(C,e,t),s(e,Q,t),f(u,e,t),s(e,q,t),f(Z,e,t),s(e,K,t),s(e,B,t),O=!0},p(e,[t]){const me={};t&2&&(me.$$scope={dirty:t,ctx:e}),u.$set(me)},i(e){O||(g(w.$$.fragment,e),g(T.$$.fragment,e),g(x.$$.fragment,e),g(R.$$.fragment,e),g(U.$$.fragment,e),g(C.$$.fragment,e),g(u.$$.fragment,e),g(Z.$$.fragment,e),O=!0)},o(e){M(w.$$.fragment,e),M(T.$$.fragment,e),M(x.$$.fragment,e),M(R.$$.fragment,e),M(U.$$.fragment,e),M(C.$$.fragment,e),M(u.$$.fragment,e),M(Z.$$.fragment,e),O=!1},d(e){e&&(n(y),n(p),n(W),n(X),n(c),n(V),n(H),n($),n(Y),n(J),n(I),n(v),n(z),n(j),n(D),n(E),n(_),n(F),n(k),n(L),n(S),n(N),n(P),n(A),n(Q),n(q),n(K),n(B)),n(i),b(w,e),b(T,e),b(x,e),b(R,e),b(U,e),b(C,e),b(u,e),b(Z,e)}}}const ve='{"title":"DePlot","local":"deplot","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage example","local":"usage-example","sections":[],"depth":2},{"title":"Fine-tuning","local":"fine-tuning","sections":[],"depth":2}],"depth":1}';function je(G){return he(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pe extends ge{constructor(i){super(),Me(this,i,je,Je,de,{})}}export{Pe as component};
