import{s as Rn,f as Nn,o as Sn,n as tt}from"../chunks/scheduler.9bc65507.js";import{S as Hn,i as En,g as c,s as r,r as f,A as Xn,h as d,f as t,c as s,j as w,u as g,x as u,k as T,y as a,a as l,v as h,d as _,t as b,w as y}from"../chunks/index.707bf1b6.js";import{T as Gn}from"../chunks/Tip.c2ecdbf4.js";import{D as k}from"../chunks/Docstring.17db21ae.js";import{C as Ft}from"../chunks/CodeBlock.54a9f38d.js";import{E as kt}from"../chunks/ExampleCodeBlock.4f515aa9.js";import{H as G}from"../chunks/Heading.342b1fa6.js";function Yn(j){let i,I="Example:",p,m,v;return m=new Ft({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMCglMEElMjAlMjAlMjAlMjBJbnN0cnVjdEJsaXBWaXNpb25Db25maWclMkMlMEElMjAlMjAlMjAlMjBJbnN0cnVjdEJsaXBRRm9ybWVyQ29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwT1BUQ29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwSW5zdHJ1Y3RCbGlwQ29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwSW5zdHJ1Y3RCbGlwRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTJDJTBBKSUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBJbnN0cnVjdEJsaXBDb25maWclMjB3aXRoJTIwU2FsZXNmb3JjZSUyRmluc3RydWN0LWJsaXAtZmxhbi10NSUyMHN0eWxlJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBJbnN0cnVjdEJsaXBDb25maWcoKSUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBJbnN0cnVjdEJsaXBGb3JDb25kaXRpb25hbEdlbmVyYXRpb24lMjAod2l0aCUyMHJhbmRvbSUyMHdlaWdodHMpJTIwZnJvbSUyMHRoZSUyMFNhbGVzZm9yY2UlMkZpbnN0cnVjdC1ibGlwLWZsYW4tdDUlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFtb2RlbCUyMCUzRCUyMEluc3RydWN0QmxpcEZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbihjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWclMEElMEElMjMlMjBXZSUyMGNhbiUyMGFsc28lMjBpbml0aWFsaXplJTIwYSUyMEluc3RydWN0QmxpcENvbmZpZyUyMGZyb20lMjBhJTIwSW5zdHJ1Y3RCbGlwVmlzaW9uQ29uZmlnJTJDJTIwSW5zdHJ1Y3RCbGlwUUZvcm1lckNvbmZpZyUyMGFuZCUyMGFueSUyMFByZXRyYWluZWRDb25maWclMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBJbnN0cnVjdEJMSVAlMjB2aXNpb24lMkMlMjBJbnN0cnVjdEJMSVAlMjBRLUZvcm1lciUyMGFuZCUyMGxhbmd1YWdlJTIwbW9kZWwlMjBjb25maWd1cmF0aW9ucyUwQXZpc2lvbl9jb25maWclMjAlM0QlMjBJbnN0cnVjdEJsaXBWaXNpb25Db25maWcoKSUwQXFmb3JtZXJfY29uZmlnJTIwJTNEJTIwSW5zdHJ1Y3RCbGlwUUZvcm1lckNvbmZpZygpJTBBdGV4dF9jb25maWclMjAlM0QlMjBPUFRDb25maWcoKSUwQSUwQWNvbmZpZyUyMCUzRCUyMEluc3RydWN0QmxpcENvbmZpZy5mcm9tX3RleHRfdmlzaW9uX2NvbmZpZ3ModmlzaW9uX2NvbmZpZyUyQyUyMHFmb3JtZXJfY29uZmlnJTJDJTIwdGV4dF9jb25maWcp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> (
<span class="hljs-meta">... </span>    InstructBlipVisionConfig,
<span class="hljs-meta">... </span>    InstructBlipQFormerConfig,
<span class="hljs-meta">... </span>    OPTConfig,
<span class="hljs-meta">... </span>    InstructBlipConfig,
<span class="hljs-meta">... </span>    InstructBlipForConditionalGeneration,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipConfig with Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = InstructBlipConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipForConditionalGeneration (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipForConditionalGeneration(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># We can also initialize a InstructBlipConfig from a InstructBlipVisionConfig, InstructBlipQFormerConfig and any PretrainedConfig</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing InstructBLIP vision, InstructBLIP Q-Former and language model configurations</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>vision_config = InstructBlipVisionConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>qformer_config = InstructBlipQFormerConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>text_config = OPTConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span>config = InstructBlipConfig.from_text_vision_configs(vision_config, qformer_config, text_config)`,wrap:!1}}),{c(){i=c("p"),i.textContent=I,p=r(),f(m.$$.fragment)},l(o){i=d(o,"P",{"data-svelte-h":!0}),u(i)!=="svelte-11lpom8"&&(i.textContent=I),p=s(o),g(m.$$.fragment,o)},m(o,M){l(o,i,M),l(o,p,M),h(m,o,M),v=!0},p:tt,i(o){v||(_(m.$$.fragment,o),v=!0)},o(o){b(m.$$.fragment,o),v=!1},d(o){o&&(t(i),t(p)),y(m,o)}}}function An(j){let i,I="Example:",p,m,v;return m=new Ft({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEluc3RydWN0QmxpcFZpc2lvbkNvbmZpZyUyQyUyMEluc3RydWN0QmxpcFZpc2lvbk1vZGVsJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEluc3RydWN0QmxpcFZpc2lvbkNvbmZpZyUyMHdpdGglMjBTYWxlc2ZvcmNlJTJGaW5zdHJ1Y3QtYmxpcC1mbGFuLXQ1JTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMEluc3RydWN0QmxpcFZpc2lvbkNvbmZpZygpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEluc3RydWN0QmxpcFZpc2lvbk1vZGVsJTIwKHdpdGglMjByYW5kb20lMjB3ZWlnaHRzKSUyMGZyb20lMjB0aGUlMjBTYWxlc2ZvcmNlJTJGaW5zdHJ1Y3QtYmxpcC1mbGFuLXQ1JTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBbW9kZWwlMjAlM0QlMjBJbnN0cnVjdEJsaXBWaXNpb25Nb2RlbChjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> InstructBlipVisionConfig, InstructBlipVisionModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipVisionConfig with Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = InstructBlipVisionConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipVisionModel (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipVisionModel(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,wrap:!1}}),{c(){i=c("p"),i.textContent=I,p=r(),f(m.$$.fragment)},l(o){i=d(o,"P",{"data-svelte-h":!0}),u(i)!=="svelte-11lpom8"&&(i.textContent=I),p=s(o),g(m.$$.fragment,o)},m(o,M){l(o,i,M),l(o,p,M),h(m,o,M),v=!0},p:tt,i(o){v||(_(m.$$.fragment,o),v=!0)},o(o){b(m.$$.fragment,o),v=!1},d(o){o&&(t(i),t(p)),y(m,o)}}}function Dn(j){let i,I="Examples:",p,m,v;return m=new Ft({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEluc3RydWN0QmxpcFFGb3JtZXJDb25maWclMkMlMjBJbnN0cnVjdEJsaXBRRm9ybWVyTW9kZWwlMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwSW5zdHJ1Y3RCTElQJTIwU2FsZXNmb3JjZSUyRmluc3RydWN0LWJsaXAtZmxhbi10NSUyMHN0eWxlJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBJbnN0cnVjdEJsaXBRRm9ybWVyQ29uZmlnKCklMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwbW9kZWwlMjAod2l0aCUyMHJhbmRvbSUyMHdlaWdodHMpJTIwZnJvbSUyMHRoZSUyMFNhbGVzZm9yY2UlMkZpbnN0cnVjdC1ibGlwLWZsYW4tdDUlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFtb2RlbCUyMCUzRCUyMEluc3RydWN0QmxpcFFGb3JtZXJNb2RlbChjb25maWd1cmF0aW9uKSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> InstructBlipQFormerConfig, InstructBlipQFormerModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBLIP Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = InstructBlipQFormerConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipQFormerModel(configuration)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,wrap:!1}}),{c(){i=c("p"),i.textContent=I,p=r(),f(m.$$.fragment)},l(o){i=d(o,"P",{"data-svelte-h":!0}),u(i)!=="svelte-kvfsh7"&&(i.textContent=I),p=s(o),g(m.$$.fragment,o)},m(o,M){l(o,i,M),l(o,p,M),h(m,o,M),v=!0},p:tt,i(o){v||(_(m.$$.fragment,o),v=!0)},o(o){b(m.$$.fragment,o),v=!1},d(o){o&&(t(i),t(p)),y(m,o)}}}function On(j){let i,I=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){i=c("p"),i.innerHTML=I},l(p){i=d(p,"P",{"data-svelte-h":!0}),u(i)!=="svelte-fincs2"&&(i.innerHTML=I)},m(p,m){l(p,i,m)},p:tt,d(p){p&&t(i)}}}function Kn(j){let i,I=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){i=c("p"),i.innerHTML=I},l(p){i=d(p,"P",{"data-svelte-h":!0}),u(i)!=="svelte-fincs2"&&(i.innerHTML=I)},m(p,m){l(p,i,m)},p:tt,d(p){p&&t(i)}}}function eo(j){let i,I="Examples:",p,m,v;return m=new Ft({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEluc3RydWN0QmxpcFByb2Nlc3NvciUyQyUyMEluc3RydWN0QmxpcEZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEElMEFtb2RlbCUyMCUzRCUyMEluc3RydWN0QmxpcEZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbi5mcm9tX3ByZXRyYWluZWQoJTIyU2FsZXNmb3JjZSUyRmluc3RydWN0YmxpcC12aWN1bmEtN2IlMjIpJTBBcHJvY2Vzc29yJTIwJTNEJTIwSW5zdHJ1Y3RCbGlwUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJTYWxlc2ZvcmNlJTJGaW5zdHJ1Y3RibGlwLXZpY3VuYS03YiUyMiklMEElMEFkZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTIyJTIwaWYlMjB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpJTIwZWxzZSUyMCUyMmNwdSUyMiUwQW1vZGVsLnRvKGRldmljZSklMEF1cmwlMjAlM0QlMjAlMjJodHRwcyUzQSUyRiUyRnJhdy5naXRodWJ1c2VyY29udGVudC5jb20lMkZzYWxlc2ZvcmNlJTJGTEFWSVMlMkZtYWluJTJGZG9jcyUyRl9zdGF0aWMlMkZDb25mdXNpbmctUGljdHVyZXMuanBnJTIyJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpLmNvbnZlcnQoJTIyUkdCJTIyKSUwQXByb21wdCUyMCUzRCUyMCUyMldoYXQlMjBpcyUyMHVudXN1YWwlMjBhYm91dCUyMHRoaXMlMjBpbWFnZSUzRiUyMiUwQWlucHV0cyUyMCUzRCUyMHByb2Nlc3NvcihpbWFnZXMlM0RpbWFnZSUyQyUyMHRleHQlM0Rwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhkZXZpY2UpJTBBJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCUwQSUyMCUyMCUyMCUyMCoqaW5wdXRzJTJDJTBBJTIwJTIwJTIwJTIwZG9fc2FtcGxlJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBudW1fYmVhbXMlM0Q1JTJDJTBBJTIwJTIwJTIwJTIwbWF4X2xlbmd0aCUzRDI1NiUyQyUwQSUyMCUyMCUyMCUyMG1pbl9sZW5ndGglM0QxJTJDJTBBJTIwJTIwJTIwJTIwdG9wX3AlM0QwLjklMkMlMEElMjAlMjAlMjAlMjByZXBldGl0aW9uX3BlbmFsdHklM0QxLjUlMkMlMEElMjAlMjAlMjAlMjBsZW5ndGhfcGVuYWx0eSUzRDEuMCUyQyUwQSUyMCUyMCUyMCUyMHRlbXBlcmF0dXJlJTNEMSUyQyUwQSklMEFnZW5lcmF0ZWRfdGV4dCUyMCUzRCUyMHByb2Nlc3Nvci5iYXRjaF9kZWNvZGUob3V0cHV0cyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQuc3RyaXAoKSUwQXByaW50KGdlbmVyYXRlZF90ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> InstructBlipProcessor, InstructBlipForConditionalGeneration
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests

<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;Salesforce/instructblip-vicuna-7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor = InstructBlipProcessor.from_pretrained(<span class="hljs-string">&quot;Salesforce/instructblip-vicuna-7b&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw).convert(<span class="hljs-string">&quot;RGB&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;What is unusual about this image?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor(images=image, text=prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(
<span class="hljs-meta">... </span>    **inputs,
<span class="hljs-meta">... </span>    do_sample=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    num_beams=<span class="hljs-number">5</span>,
<span class="hljs-meta">... </span>    max_length=<span class="hljs-number">256</span>,
<span class="hljs-meta">... </span>    min_length=<span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>    top_p=<span class="hljs-number">0.9</span>,
<span class="hljs-meta">... </span>    repetition_penalty=<span class="hljs-number">1.5</span>,
<span class="hljs-meta">... </span>    length_penalty=<span class="hljs-number">1.0</span>,
<span class="hljs-meta">... </span>    temperature=<span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_text = processor.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>].strip()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(generated_text)
The unusual aspect of this image <span class="hljs-keyword">is</span> that a man <span class="hljs-keyword">is</span> ironing clothes on the back of a yellow SUV, which <span class="hljs-keyword">is</span> parked <span class="hljs-keyword">in</span> the middle of a busy city street. This <span class="hljs-keyword">is</span> an unconventional approach to ironing clothes, <span class="hljs-keyword">as</span> it requires the man to balance himself <span class="hljs-keyword">and</span> his ironing equipment on top of the vehicle <span class="hljs-keyword">while</span> navigating through traffic. Additionally, the presence of taxis <span class="hljs-keyword">and</span> other vehicles <span class="hljs-keyword">in</span> the scene further emphasizes the unusual nature of this situation.`,wrap:!1}}),{c(){i=c("p"),i.textContent=I,p=r(),f(m.$$.fragment)},l(o){i=d(o,"P",{"data-svelte-h":!0}),u(i)!=="svelte-kvfsh7"&&(i.textContent=I),p=s(o),g(m.$$.fragment,o)},m(o,M){l(o,i,M),l(o,p,M),h(m,o,M),v=!0},p:tt,i(o){v||(_(m.$$.fragment,o),v=!0)},o(o){b(m.$$.fragment,o),v=!1},d(o){o&&(t(i),t(p)),y(m,o)}}}function to(j){let i,I,p,m,v,o,M,ot,te,un=`The InstructBLIP model was proposed in <a href="https://arxiv.org/abs/2305.06500" rel="nofollow">InstructBLIP: Towards General-purpose Vision-Language Models with Instruction Tuning</a> by Wenliang Dai, Junnan Li, Dongxu Li, Anthony Meng Huat Tiong, Junqi Zhao, Weisheng Wang, Boyang Li, Pascale Fung, Steven Hoi.
InstructBLIP leverages the <a href="blip2">BLIP-2</a> architecture for visual instruction tuning.`,rt,ne,fn="The abstract from the paper is the following:",st,oe,gn="<em>General-purpose language models that can solve various language-domain tasks have emerged driven by the pre-training and instruction-tuning pipeline. However, building general-purpose vision-language models is challenging due to the increased task discrepancy introduced by the additional visual input. Although vision-language pre-training has been widely studied, vision-language instruction tuning remains relatively less explored. In this paper, we conduct a systematic and comprehensive study on vision-language instruction tuning based on the pre-trained BLIP-2 models. We gather a wide variety of 26 publicly available datasets, transform them into instruction tuning format and categorize them into two clusters for held-in instruction tuning and held-out zero-shot evaluation. Additionally, we introduce instruction-aware visual feature extraction, a crucial method that enables the model to extract informative features tailored to the given instruction. The resulting InstructBLIP models achieve state-of-the-art zero-shot performance across all 13 held-out datasets, substantially outperforming BLIP-2 and the larger Flamingo. Our models also lead to state-of-the-art performance when finetuned on individual downstream tasks (e.g., 90.7% accuracy on ScienceQA IMG). Furthermore, we qualitatively demonstrate the advantages of InstructBLIP over concurrent multimodal models.</em>",at,re,hn="Tips:",it,se,_n='<li>InstructBLIP uses the same architecture as <a href="blip2">BLIP-2</a> with a tiny but important difference: it also feeds the text prompt (instruction) to the Q-Former.</li>',lt,R,bn,ct,ae,yn='InstructBLIP architecture. Taken from the <a href="https://arxiv.org/abs/2305.06500">original paper.</a>',dt,ie,vn=`This model was contributed by <a href="https://huggingface.co/nielsr" rel="nofollow">nielsr</a>.
The original code can be found <a href="https://github.com/salesforce/LAVIS/tree/main/projects/instructblip" rel="nofollow">here</a>.`,mt,le,pt,$,ce,Jt,Fe,Mn=`<a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipConfig">InstructBlipConfig</a> is the configuration class to store the configuration of a
<a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipForConditionalGeneration">InstructBlipForConditionalGeneration</a>. It is used to instantiate a InstructBLIP model according to the specified
arguments, defining the vision model, Q-Former model and language model configs. Instantiating a configuration with
the defaults will yield a similar configuration to that of the InstructBLIP
<a href="https://huggingface.co/Salesforce/instruct-blip-flan-t5" rel="nofollow">Salesforce/instruct-blip-flan-t5</a> architecture.`,Ut,Je,In=`Configuration objects inherit from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,zt,N,Wt,S,de,Zt,Ue,Bn=`Instantiate a <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipConfig">InstructBlipConfig</a> (or a derived class) from a InstructBLIP vision model, Q-Former and
language model configurations.`,ut,me,ft,F,pe,Pt,ze,Tn=`This is the configuration class to store the configuration of a <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipVisionModel">InstructBlipVisionModel</a>. It is used to
instantiate a InstructBLIP vision encoder according to the specified arguments, defining the model architecture.
Instantiating a configuration defaults will yield a similar configuration to that of the InstructBLIP
<a href="https://huggingface.co/Salesforce/instruct-blip-flan-t5" rel="nofollow">Salesforce/instruct-blip-flan-t5</a> architecture.`,qt,We,wn=`Configuration objects inherit from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,Lt,H,gt,ue,ht,J,fe,Qt,Ze,$n=`This is the configuration class to store the configuration of a <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipQFormerModel">InstructBlipQFormerModel</a>. It is used to
instantiate a InstructBLIP Querying Transformer (Q-Former) model according to the specified arguments, defining the
model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
the InstructBLIP <a href="https://huggingface.co/Salesforce/instruct-blip-flan-t5" rel="nofollow">Salesforce/instruct-blip-flan-t5</a>
architecture. Configuration objects inherit from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs.
Read the documentation from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,Vt,Pe,Cn='Note that <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipQFormerModel">InstructBlipQFormerModel</a> is very similar to <a href="/docs/transformers/main/en/model_doc/bert#transformers.BertLMHeadModel">BertLMHeadModel</a> with interleaved cross-attention.',Gt,E,_t,ge,bt,C,he,Rt,qe,jn=`Constructs an InstructBLIP processor which wraps a BLIP image processor and a LLaMa/T5 tokenizer into a single
processor.`,Nt,Le,xn=`<a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a> offers all the functionalities of <a href="/docs/transformers/main/en/model_doc/blip#transformers.BlipImageProcessor">BlipImageProcessor</a> and <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See the
docstring of <code>__call__()</code> and <a href="/docs/transformers/main/en/model_doc/blip#transformers.BlipProcessor.decode">decode()</a> for more information.`,St,X,_e,Ht,Qe,kn=`This method forwards all its arguments to PreTrainedTokenizer’s <a href="/docs/transformers/main/en/model_doc/speecht5#transformers.SpeechT5Tokenizer.batch_decode">batch_decode()</a>. Please
refer to the docstring of this method for more information.`,Et,Y,be,Xt,Ve,Fn=`This method forwards all its arguments to PreTrainedTokenizer’s <a href="/docs/transformers/main/en/model_doc/speecht5#transformers.SpeechT5Tokenizer.decode">decode()</a>. Please refer
to the docstring of this method for more information.`,yt,ye,vt,V,ve,Yt,L,Me,At,Ge,Jn='The <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipVisionModel">InstructBlipVisionModel</a> forward method, overrides the <code>__call__</code> special method.',Dt,A,Mt,Ie,It,Z,Be,Ot,Re,Un=`Querying Transformer (Q-Former), used in InstructBLIP. Slightly modified from BLIP-2 as it also takes the
instruction as input.`,Kt,Q,Te,en,Ne,zn=`encoder_hidden_states  (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>):
Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
the model is configured as a decoder.
encoder_attention_mask (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>):
Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
the cross-attention if the model is configured as a decoder. Mask values selected in <code>[0, 1]</code>:`,tn,Se,Wn=`<li>1 for tokens that are <strong>not masked</strong>,</li> <li>0 for tokens that are <strong>masked</strong>.
past_key_values (<code>tuple(tuple(torch.FloatTensor))</code> of length <code>config.n_layers</code> with each tuple having 4 tensors of:
shape <code>(batch_size, num_heads, sequence_length - 1, embed_size_per_head)</code>): Contains precomputed key and
value hidden states of the attention blocks. Can be used to speed up decoding. If <code>past_key_values</code> are
used, the user can optionally input only the last <code>decoder_input_ids</code> (those that don’t have their past key
value states given to this model) of shape <code>(batch_size, 1)</code> instead of all <code>decoder_input_ids</code> of shape
<code>(batch_size, sequence_length)</code>.
use_cache (<code>bool</code>, <em>optional</em>):
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).</li>`,Bt,we,Tt,B,$e,nn,He,Zn=`InstructBLIP Model for generating text given an image and an optional text prompt. The model consists of a vision
encoder, Querying Transformer (Q-Former) and a language model.`,on,Ee,Pn=`One can optionally pass <code>input_ids</code> to the model, which serve as a text prompt, to make the language model continue
the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.`,rn,Xe,qn=`This model inherits from <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`,sn,Ye,Ln=`This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`,an,W,Ce,ln,Ae,Qn='The <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipForConditionalGeneration">InstructBlipForConditionalGeneration</a> forward method, overrides the <code>__call__</code> special method.',cn,D,dn,O,mn,K,je,pn,De,Vn="Overrides <code>generate</code> function to be able to use the model as a conditional generator.",wt,nt,$t;return v=new G({props:{title:"InstructBLIP",local:"instructblip",headingTag:"h1"}}),M=new G({props:{title:"Overview",local:"overview",headingTag:"h2"}}),le=new G({props:{title:"InstructBlipConfig",local:"transformers.InstructBlipConfig",headingTag:"h2"}}),ce=new k({props:{name:"class transformers.InstructBlipConfig",anchor:"transformers.InstructBlipConfig",parameters:[{name:"vision_config",val:" = None"},{name:"qformer_config",val:" = None"},{name:"text_config",val:" = None"},{name:"num_query_tokens",val:" = 32"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipConfig.vision_config",description:`<strong>vision_config</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipVisionConfig">InstructBlipVisionConfig</a>.`,name:"vision_config"},{anchor:"transformers.InstructBlipConfig.qformer_config",description:`<strong>qformer_config</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipQFormerConfig">InstructBlipQFormerConfig</a>.`,name:"qformer_config"},{anchor:"transformers.InstructBlipConfig.text_config",description:`<strong>text_config</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize any <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a>.`,name:"text_config"},{anchor:"transformers.InstructBlipConfig.num_query_tokens",description:`<strong>num_query_tokens</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
The number of query tokens passed through the Transformer.`,name:"num_query_tokens"},{anchor:"transformers.InstructBlipConfig.kwargs",description:`<strong>kwargs</strong> (<em>optional</em>) &#x2014;
Dictionary of keyword arguments.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/configuration_instructblip.py#L252"}}),N=new kt({props:{anchor:"transformers.InstructBlipConfig.example",$$slots:{default:[Yn]},$$scope:{ctx:j}}}),de=new k({props:{name:"from_vision_qformer_text_configs",anchor:"transformers.InstructBlipConfig.from_vision_qformer_text_configs",parameters:[{name:"vision_config",val:": InstructBlipVisionConfig"},{name:"qformer_config",val:": InstructBlipQFormerConfig"},{name:"text_config",val:": PretrainedConfig"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/configuration_instructblip.py#L337",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>An instance of a configuration object</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipConfig"
>InstructBlipConfig</a></p>
`}}),me=new G({props:{title:"InstructBlipVisionConfig",local:"transformers.InstructBlipVisionConfig",headingTag:"h2"}}),pe=new k({props:{name:"class transformers.InstructBlipVisionConfig",anchor:"transformers.InstructBlipVisionConfig",parameters:[{name:"hidden_size",val:" = 1408"},{name:"intermediate_size",val:" = 6144"},{name:"num_hidden_layers",val:" = 39"},{name:"num_attention_heads",val:" = 16"},{name:"image_size",val:" = 224"},{name:"patch_size",val:" = 14"},{name:"hidden_act",val:" = 'gelu'"},{name:"layer_norm_eps",val:" = 1e-06"},{name:"attention_dropout",val:" = 0.0"},{name:"initializer_range",val:" = 1e-10"},{name:"qkv_bias",val:" = True"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipVisionConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 1408) &#x2014;
Dimensionality of the encoder layers and the pooler layer.`,name:"hidden_size"},{anchor:"transformers.InstructBlipVisionConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 6144) &#x2014;
Dimensionality of the &#x201C;intermediate&#x201D; (i.e., feed-forward) layer in the Transformer encoder.`,name:"intermediate_size"},{anchor:"transformers.InstructBlipVisionConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 39) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.InstructBlipVisionConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 16) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.InstructBlipVisionConfig.image_size",description:`<strong>image_size</strong> (<code>int</code>, <em>optional</em>, defaults to 224) &#x2014;
The size (resolution) of each image.`,name:"image_size"},{anchor:"transformers.InstructBlipVisionConfig.patch_size",description:`<strong>patch_size</strong> (<code>int</code>, <em>optional</em>, defaults to 14) &#x2014;
The size (resolution) of each patch.`,name:"patch_size"},{anchor:"transformers.InstructBlipVisionConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>function</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the encoder and pooler. If string, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;selu&quot;</code> and <code>&quot;gelu_new&quot;</code> \`<code>&quot;gelu&quot;</code> are supported. to 1e-5): The epsilon used by the layer
normalization layers.`,name:"hidden_act"},{anchor:"transformers.InstructBlipVisionConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-06) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.InstructBlipVisionConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_dropout"},{anchor:"transformers.InstructBlipVisionConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-10) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.InstructBlipVisionConfig.qkv_bias",description:`<strong>qkv_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to add a bias to the queries and values in the self-attention layers.`,name:"qkv_bias"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/configuration_instructblip.py#L33"}}),H=new kt({props:{anchor:"transformers.InstructBlipVisionConfig.example",$$slots:{default:[An]},$$scope:{ctx:j}}}),ue=new G({props:{title:"InstructBlipQFormerConfig",local:"transformers.InstructBlipQFormerConfig",headingTag:"h2"}}),fe=new k({props:{name:"class transformers.InstructBlipQFormerConfig",anchor:"transformers.InstructBlipQFormerConfig",parameters:[{name:"vocab_size",val:" = 30522"},{name:"hidden_size",val:" = 768"},{name:"num_hidden_layers",val:" = 12"},{name:"num_attention_heads",val:" = 12"},{name:"intermediate_size",val:" = 3072"},{name:"hidden_act",val:" = 'gelu'"},{name:"hidden_dropout_prob",val:" = 0.1"},{name:"attention_probs_dropout_prob",val:" = 0.1"},{name:"max_position_embeddings",val:" = 512"},{name:"initializer_range",val:" = 0.02"},{name:"layer_norm_eps",val:" = 1e-12"},{name:"pad_token_id",val:" = 0"},{name:"position_embedding_type",val:" = 'absolute'"},{name:"cross_attention_frequency",val:" = 2"},{name:"encoder_hidden_size",val:" = 1408"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipQFormerConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 30522) &#x2014;
Vocabulary size of the Q-Former model. Defines the number of different tokens that can be represented by
the <code>inputs_ids</code> passed when calling the model.`,name:"vocab_size"},{anchor:"transformers.InstructBlipQFormerConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 768) &#x2014;
Dimensionality of the encoder layers and the pooler layer.`,name:"hidden_size"},{anchor:"transformers.InstructBlipQFormerConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 12) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.InstructBlipQFormerConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 12) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.InstructBlipQFormerConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 3072) &#x2014;
Dimensionality of the &#x201C;intermediate&#x201D; (often named feed-forward) layer in the Transformer encoder.`,name:"intermediate_size"},{anchor:"transformers.InstructBlipQFormerConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>Callable</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the encoder and pooler. If string, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;silu&quot;</code> and <code>&quot;gelu_new&quot;</code> are supported.`,name:"hidden_act"},{anchor:"transformers.InstructBlipQFormerConfig.hidden_dropout_prob",description:`<strong>hidden_dropout_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.`,name:"hidden_dropout_prob"},{anchor:"transformers.InstructBlipQFormerConfig.attention_probs_dropout_prob",description:`<strong>attention_probs_dropout_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_probs_dropout_prob"},{anchor:"transformers.InstructBlipQFormerConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to 512) &#x2014;
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).`,name:"max_position_embeddings"},{anchor:"transformers.InstructBlipQFormerConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 0.02) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.InstructBlipQFormerConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-12) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.InstructBlipQFormerConfig.position_embedding_type",description:`<strong>position_embedding_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;absolute&quot;</code>) &#x2014;
Type of position embedding. Choose one of <code>&quot;absolute&quot;</code>, <code>&quot;relative_key&quot;</code>, <code>&quot;relative_key_query&quot;</code>. For
positional embeddings use <code>&quot;absolute&quot;</code>. For more information on <code>&quot;relative_key&quot;</code>, please refer to
<a href="https://arxiv.org/abs/1803.02155" rel="nofollow">Self-Attention with Relative Position Representations (Shaw et al.)</a>.
For more information on <code>&quot;relative_key_query&quot;</code>, please refer to <em>Method 4</em> in <a href="https://arxiv.org/abs/2009.13658" rel="nofollow">Improve Transformer Models
with Better Relative Position Embeddings (Huang et al.)</a>.`,name:"position_embedding_type"},{anchor:"transformers.InstructBlipQFormerConfig.cross_attention_frequency",description:`<strong>cross_attention_frequency</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
The frequency of adding cross-attention to the Transformer layers.`,name:"cross_attention_frequency"},{anchor:"transformers.InstructBlipQFormerConfig.encoder_hidden_size",description:`<strong>encoder_hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 1408) &#x2014;
The hidden size of the hidden states for cross-attention.`,name:"encoder_hidden_size"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/configuration_instructblip.py#L134"}}),E=new kt({props:{anchor:"transformers.InstructBlipQFormerConfig.example",$$slots:{default:[Dn]},$$scope:{ctx:j}}}),ge=new G({props:{title:"InstructBlipProcessor",local:"transformers.InstructBlipProcessor",headingTag:"h2"}}),he=new k({props:{name:"class transformers.InstructBlipProcessor",anchor:"transformers.InstructBlipProcessor",parameters:[{name:"image_processor",val:""},{name:"tokenizer",val:""},{name:"qformer_tokenizer",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipProcessor.image_processor",description:`<strong>image_processor</strong> (<code>BlipImageProcessor</code>) &#x2014;
An instance of <a href="/docs/transformers/main/en/model_doc/blip#transformers.BlipImageProcessor">BlipImageProcessor</a>. The image processor is a required input.`,name:"image_processor"},{anchor:"transformers.InstructBlipProcessor.tokenizer",description:"<strong>tokenizer</strong> (<code>AutoTokenizer</code>) &#x2014;\nAn instance of [&#x2018;PreTrainedTokenizer`]. The tokenizer is a required input.",name:"tokenizer"},{anchor:"transformers.InstructBlipProcessor.qformer_tokenizer",description:"<strong>qformer_tokenizer</strong> (<code>AutoTokenizer</code>) &#x2014;\nAn instance of [&#x2018;PreTrainedTokenizer`]. The Q-Former tokenizer is a required input.",name:"qformer_tokenizer"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/processing_instructblip.py#L30"}}),_e=new k({props:{name:"batch_decode",anchor:"transformers.InstructBlipProcessor.batch_decode",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/processing_instructblip.py#L135"}}),be=new k({props:{name:"decode",anchor:"transformers.InstructBlipProcessor.decode",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/processing_instructblip.py#L143"}}),ye=new G({props:{title:"InstructBlipVisionModel",local:"transformers.InstructBlipVisionModel",headingTag:"h2"}}),ve=new k({props:{name:"class transformers.InstructBlipVisionModel",anchor:"transformers.InstructBlipVisionModel",parameters:[{name:"config",val:": InstructBlipVisionConfig"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/modeling_instructblip.py#L490"}}),Me=new k({props:{name:"forward",anchor:"transformers.InstructBlipVisionModel.forward",parameters:[{name:"pixel_values",val:": typing.Optional[torch.FloatTensor] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.InstructBlipVisionModel.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See
<code>InstructBlipProcessor.__call__()</code> for details.`,name:"pixel_values"},{anchor:"transformers.InstructBlipVisionModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.InstructBlipVisionModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.InstructBlipVisionModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/modeling_instructblip.py#L505",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <a
  href="/docs/transformers/main/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPooling"
>transformers.modeling_outputs.BaseModelOutputWithPooling</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<code>&lt;class 'transformers.models.instructblip.configuration_instructblip.InstructBlipVisionConfig'&gt;</code>) and inputs.</p>
<ul>
<li>
<p><strong>last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) — Sequence of hidden-states at the output of the last layer of the model.</p>
</li>
<li>
<p><strong>pooler_output</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, hidden_size)</code>) — Last layer hidden-state of the first token of the sequence (classification token) after further processing
through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns
the classification token after processing through a linear layer and a tanh activation function. The linear
layer weights are trained from the next sentence prediction (classification) objective during pretraining.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/transformers/main/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPooling"
>transformers.modeling_outputs.BaseModelOutputWithPooling</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),A=new Gn({props:{$$slots:{default:[On]},$$scope:{ctx:j}}}),Ie=new G({props:{title:"InstructBlipQFormerModel",local:"transformers.InstructBlipQFormerModel",headingTag:"h2"}}),Be=new k({props:{name:"class transformers.InstructBlipQFormerModel",anchor:"transformers.InstructBlipQFormerModel",parameters:[{name:"config",val:": InstructBlipQFormerConfig"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/modeling_instructblip.py#L1035"}}),Te=new k({props:{name:"forward",anchor:"transformers.InstructBlipQFormerModel.forward",parameters:[{name:"input_ids",val:": LongTensor"},{name:"attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"position_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"query_embeds",val:": typing.Optional[torch.Tensor] = None"},{name:"head_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"encoder_hidden_states",val:": typing.Optional[torch.FloatTensor] = None"},{name:"encoder_attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"past_key_values",val:": typing.Optional[typing.Tuple[typing.Tuple[torch.FloatTensor]]] = None"},{name:"use_cache",val:": typing.Optional[bool] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/modeling_instructblip.py#L1108"}}),we=new G({props:{title:"InstructBlipForConditionalGeneration",local:"transformers.InstructBlipForConditionalGeneration",headingTag:"h2"}}),$e=new k({props:{name:"class transformers.InstructBlipForConditionalGeneration",anchor:"transformers.InstructBlipForConditionalGeneration",parameters:[{name:"config",val:": InstructBlipConfig"}],parametersDescription:[{anchor:"transformers.InstructBlipForConditionalGeneration.config",description:`<strong>config</strong> (<a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipConfig">InstructBlipConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/modeling_instructblip.py#L1241"}}),Ce=new k({props:{name:"forward",anchor:"transformers.InstructBlipForConditionalGeneration.forward",parameters:[{name:"pixel_values",val:": FloatTensor"},{name:"qformer_input_ids",val:": FloatTensor"},{name:"qformer_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"input_ids",val:": typing.Optional[torch.FloatTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"decoder_input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"decoder_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.InstructBlipForConditionalGeneration.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See
<code>InstructBlipProcessor.__call__()</code> for details.`,name:"pixel_values"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.qformer_input_ids",description:`<strong>qformer_input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
to serve as text prompt, which the Q-Former model will encode.</p>
<p>Indices can be obtained using <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See <code>InstructBlipProcessor.__call__()</code> for
details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"qformer_input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.qformer_attention_mask",description:`<strong>qformer_attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"qformer_attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be
provided to serve as text prompt, which the language model can continue.</p>
<p>Indices can be obtained using <a href="/docs/transformers/main/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See <code>InstructBlipProcessor.__call__()</code> for
details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.decoder_input_ids",description:`<strong>decoder_input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of decoder input sequence tokens in the vocabulary of the language model. Only relevant in case an
encoder-decoder language model (like T5) is used.</p>
<p>Indices can be obtained using <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/main/en/model_doc/speecht5#transformers.SpeechT5Tokenizer.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details. <a href="../glossary#decoder-input-ids">What are decoder input IDs?</a>`,name:"decoder_input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.decoder_attention_mask",description:`<strong>decoder_attention_mask</strong> (<code>torch.BoolTensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Default behavior: generate a tensor that ignores pad tokens in <code>decoder_input_ids</code>. Causal mask will also
be used by default.</p>
<p>Only relevant in case an encoder-decoder language model (like T5) is used.`,name:"decoder_attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) &#x2014;
Labels for computing the language modeling loss. Indices should be in <code>[-100, 0, ..., config.vocab_size - 1]</code>. All labels set to <code>-100</code> are ignored (masked), the loss is only computed for labels in <code>[0, ..., config.vocab_size]</code>`,name:"labels"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/modeling_instructblip.py#L1314",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>transformers.models.instructblip.modeling_instructblip.InstructBlipForConditionalGenerationModelOutput</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<code>&lt;class 'transformers.models.instructblip.configuration_instructblip.InstructBlipVisionConfig'&gt;</code>) and inputs.</p>
<ul>
<li><strong>loss</strong> (<code>torch.FloatTensor</code>, <em>optional</em>, returned when <code>labels</code> is provided, <code>torch.FloatTensor</code> of shape <code>(1,)</code>) — Language modeling loss from the language model.</li>
<li><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head of the language model.</li>
<li><strong>vision_outputs</strong> (<code>BaseModelOutputWithPooling</code>) — Outputs of the vision encoder.</li>
<li><strong>qformer_outputs</strong> (<code>BaseModelOutputWithPoolingAndCrossAttentions</code>) — Outputs of the Q-Former (Querying Transformer).</li>
<li><strong>language_model_outputs</strong> (<code>CausalLMOutputWithPast</code> or <code>Seq2SeqLMOutput</code>) — Outputs of the language model.</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>transformers.models.instructblip.modeling_instructblip.InstructBlipForConditionalGenerationModelOutput</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),D=new Gn({props:{$$slots:{default:[Kn]},$$scope:{ctx:j}}}),O=new kt({props:{anchor:"transformers.InstructBlipForConditionalGeneration.forward.example",$$slots:{default:[eo]},$$scope:{ctx:j}}}),je=new k({props:{name:"generate",anchor:"transformers.InstructBlipForConditionalGeneration.generate",parameters:[{name:"pixel_values",val:": FloatTensor"},{name:"qformer_input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"qformer_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"**generate_kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipForConditionalGeneration.generate.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape (batch_size, num_channels, height, width)) &#x2014;
Input images to be processed.`,name:"pixel_values"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.qformer_input_ids",description:`<strong>qformer_input_ids</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
The sequence used as a prompt to be fed to the Q-Former module.`,name:"qformer_input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.qformer_attention_mask",description:`<strong>qformer_attention_mask</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices.`,name:"qformer_attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
The sequence used as a prompt for the generation.`,name:"input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices.`,name:"attention_mask"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/instructblip/modeling_instructblip.py#L1469",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of strings of length batch_size * num_captions.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>captions (list)</p>
`}}),{c(){i=c("meta"),I=r(),p=c("p"),m=r(),f(v.$$.fragment),o=r(),f(M.$$.fragment),ot=r(),te=c("p"),te.innerHTML=un,rt=r(),ne=c("p"),ne.textContent=fn,st=r(),oe=c("p"),oe.innerHTML=gn,at=r(),re=c("p"),re.textContent=hn,it=r(),se=c("ul"),se.innerHTML=_n,lt=r(),R=c("img"),ct=r(),ae=c("small"),ae.innerHTML=yn,dt=r(),ie=c("p"),ie.innerHTML=vn,mt=r(),f(le.$$.fragment),pt=r(),$=c("div"),f(ce.$$.fragment),Jt=r(),Fe=c("p"),Fe.innerHTML=Mn,Ut=r(),Je=c("p"),Je.innerHTML=In,zt=r(),f(N.$$.fragment),Wt=r(),S=c("div"),f(de.$$.fragment),Zt=r(),Ue=c("p"),Ue.innerHTML=Bn,ut=r(),f(me.$$.fragment),ft=r(),F=c("div"),f(pe.$$.fragment),Pt=r(),ze=c("p"),ze.innerHTML=Tn,qt=r(),We=c("p"),We.innerHTML=wn,Lt=r(),f(H.$$.fragment),gt=r(),f(ue.$$.fragment),ht=r(),J=c("div"),f(fe.$$.fragment),Qt=r(),Ze=c("p"),Ze.innerHTML=$n,Vt=r(),Pe=c("p"),Pe.innerHTML=Cn,Gt=r(),f(E.$$.fragment),_t=r(),f(ge.$$.fragment),bt=r(),C=c("div"),f(he.$$.fragment),Rt=r(),qe=c("p"),qe.textContent=jn,Nt=r(),Le=c("p"),Le.innerHTML=xn,St=r(),X=c("div"),f(_e.$$.fragment),Ht=r(),Qe=c("p"),Qe.innerHTML=kn,Et=r(),Y=c("div"),f(be.$$.fragment),Xt=r(),Ve=c("p"),Ve.innerHTML=Fn,yt=r(),f(ye.$$.fragment),vt=r(),V=c("div"),f(ve.$$.fragment),Yt=r(),L=c("div"),f(Me.$$.fragment),At=r(),Ge=c("p"),Ge.innerHTML=Jn,Dt=r(),f(A.$$.fragment),Mt=r(),f(Ie.$$.fragment),It=r(),Z=c("div"),f(Be.$$.fragment),Ot=r(),Re=c("p"),Re.textContent=Un,Kt=r(),Q=c("div"),f(Te.$$.fragment),en=r(),Ne=c("p"),Ne.innerHTML=zn,tn=r(),Se=c("ul"),Se.innerHTML=Wn,Bt=r(),f(we.$$.fragment),Tt=r(),B=c("div"),f($e.$$.fragment),nn=r(),He=c("p"),He.textContent=Zn,on=r(),Ee=c("p"),Ee.innerHTML=Pn,rn=r(),Xe=c("p"),Xe.innerHTML=qn,sn=r(),Ye=c("p"),Ye.innerHTML=Ln,an=r(),W=c("div"),f(Ce.$$.fragment),ln=r(),Ae=c("p"),Ae.innerHTML=Qn,cn=r(),f(D.$$.fragment),dn=r(),f(O.$$.fragment),mn=r(),K=c("div"),f(je.$$.fragment),pn=r(),De=c("p"),De.innerHTML=Vn,wt=r(),nt=c("p"),this.h()},l(e){const n=Xn("svelte-u9bgzb",document.head);i=d(n,"META",{name:!0,content:!0}),n.forEach(t),I=s(e),p=d(e,"P",{}),w(p).forEach(t),m=s(e),g(v.$$.fragment,e),o=s(e),g(M.$$.fragment,e),ot=s(e),te=d(e,"P",{"data-svelte-h":!0}),u(te)!=="svelte-15j8z3m"&&(te.innerHTML=un),rt=s(e),ne=d(e,"P",{"data-svelte-h":!0}),u(ne)!=="svelte-vfdo9a"&&(ne.textContent=fn),st=s(e),oe=d(e,"P",{"data-svelte-h":!0}),u(oe)!=="svelte-g3w4hv"&&(oe.innerHTML=gn),at=s(e),re=d(e,"P",{"data-svelte-h":!0}),u(re)!=="svelte-axv494"&&(re.textContent=hn),it=s(e),se=d(e,"UL",{"data-svelte-h":!0}),u(se)!=="svelte-15d4wpq"&&(se.innerHTML=_n),lt=s(e),R=d(e,"IMG",{src:!0,alt:!0,width:!0}),ct=s(e),ae=d(e,"SMALL",{"data-svelte-h":!0}),u(ae)!=="svelte-gvupf8"&&(ae.innerHTML=yn),dt=s(e),ie=d(e,"P",{"data-svelte-h":!0}),u(ie)!=="svelte-1b5u17p"&&(ie.innerHTML=vn),mt=s(e),g(le.$$.fragment,e),pt=s(e),$=d(e,"DIV",{class:!0});var U=w($);g(ce.$$.fragment,U),Jt=s(U),Fe=d(U,"P",{"data-svelte-h":!0}),u(Fe)!=="svelte-18qxzwu"&&(Fe.innerHTML=Mn),Ut=s(U),Je=d(U,"P",{"data-svelte-h":!0}),u(Je)!=="svelte-o55m63"&&(Je.innerHTML=In),zt=s(U),g(N.$$.fragment,U),Wt=s(U),S=d(U,"DIV",{class:!0});var xe=w(S);g(de.$$.fragment,xe),Zt=s(xe),Ue=d(xe,"P",{"data-svelte-h":!0}),u(Ue)!=="svelte-jiu0a6"&&(Ue.innerHTML=Bn),xe.forEach(t),U.forEach(t),ut=s(e),g(me.$$.fragment,e),ft=s(e),F=d(e,"DIV",{class:!0});var P=w(F);g(pe.$$.fragment,P),Pt=s(P),ze=d(P,"P",{"data-svelte-h":!0}),u(ze)!=="svelte-w1p2u2"&&(ze.innerHTML=Tn),qt=s(P),We=d(P,"P",{"data-svelte-h":!0}),u(We)!=="svelte-o55m63"&&(We.innerHTML=wn),Lt=s(P),g(H.$$.fragment,P),P.forEach(t),gt=s(e),g(ue.$$.fragment,e),ht=s(e),J=d(e,"DIV",{class:!0});var q=w(J);g(fe.$$.fragment,q),Qt=s(q),Ze=d(q,"P",{"data-svelte-h":!0}),u(Ze)!=="svelte-1rwjuiy"&&(Ze.innerHTML=$n),Vt=s(q),Pe=d(q,"P",{"data-svelte-h":!0}),u(Pe)!=="svelte-1szlo2r"&&(Pe.innerHTML=Cn),Gt=s(q),g(E.$$.fragment,q),q.forEach(t),_t=s(e),g(ge.$$.fragment,e),bt=s(e),C=d(e,"DIV",{class:!0});var z=w(C);g(he.$$.fragment,z),Rt=s(z),qe=d(z,"P",{"data-svelte-h":!0}),u(qe)!=="svelte-1o16zs2"&&(qe.textContent=jn),Nt=s(z),Le=d(z,"P",{"data-svelte-h":!0}),u(Le)!=="svelte-y9wawf"&&(Le.innerHTML=xn),St=s(z),X=d(z,"DIV",{class:!0});var ke=w(X);g(_e.$$.fragment,ke),Ht=s(ke),Qe=d(ke,"P",{"data-svelte-h":!0}),u(Qe)!=="svelte-pqmfqu"&&(Qe.innerHTML=kn),ke.forEach(t),Et=s(z),Y=d(z,"DIV",{class:!0});var Ct=w(Y);g(be.$$.fragment,Ct),Xt=s(Ct),Ve=d(Ct,"P",{"data-svelte-h":!0}),u(Ve)!=="svelte-lpo0rc"&&(Ve.innerHTML=Fn),Ct.forEach(t),z.forEach(t),yt=s(e),g(ye.$$.fragment,e),vt=s(e),V=d(e,"DIV",{class:!0});var jt=w(V);g(ve.$$.fragment,jt),Yt=s(jt),L=d(jt,"DIV",{class:!0});var Oe=w(L);g(Me.$$.fragment,Oe),At=s(Oe),Ge=d(Oe,"P",{"data-svelte-h":!0}),u(Ge)!=="svelte-nnnlds"&&(Ge.innerHTML=Jn),Dt=s(Oe),g(A.$$.fragment,Oe),Oe.forEach(t),jt.forEach(t),Mt=s(e),g(Ie.$$.fragment,e),It=s(e),Z=d(e,"DIV",{class:!0});var Ke=w(Z);g(Be.$$.fragment,Ke),Ot=s(Ke),Re=d(Ke,"P",{"data-svelte-h":!0}),u(Re)!=="svelte-1ta0rcm"&&(Re.textContent=Un),Kt=s(Ke),Q=d(Ke,"DIV",{class:!0});var et=w(Q);g(Te.$$.fragment,et),en=s(et),Ne=d(et,"P",{"data-svelte-h":!0}),u(Ne)!=="svelte-1h74cdd"&&(Ne.innerHTML=zn),tn=s(et),Se=d(et,"UL",{"data-svelte-h":!0}),u(Se)!=="svelte-1yyv1jy"&&(Se.innerHTML=Wn),et.forEach(t),Ke.forEach(t),Bt=s(e),g(we.$$.fragment,e),Tt=s(e),B=d(e,"DIV",{class:!0});var x=w(B);g($e.$$.fragment,x),nn=s(x),He=d(x,"P",{"data-svelte-h":!0}),u(He)!=="svelte-18txus8"&&(He.textContent=Zn),on=s(x),Ee=d(x,"P",{"data-svelte-h":!0}),u(Ee)!=="svelte-1ks26sg"&&(Ee.innerHTML=Pn),rn=s(x),Xe=d(x,"P",{"data-svelte-h":!0}),u(Xe)!=="svelte-6pahdo"&&(Xe.innerHTML=qn),sn=s(x),Ye=d(x,"P",{"data-svelte-h":!0}),u(Ye)!=="svelte-hswkmf"&&(Ye.innerHTML=Ln),an=s(x),W=d(x,"DIV",{class:!0});var ee=w(W);g(Ce.$$.fragment,ee),ln=s(ee),Ae=d(ee,"P",{"data-svelte-h":!0}),u(Ae)!=="svelte-11sp8yc"&&(Ae.innerHTML=Qn),cn=s(ee),g(D.$$.fragment,ee),dn=s(ee),g(O.$$.fragment,ee),ee.forEach(t),mn=s(x),K=d(x,"DIV",{class:!0});var xt=w(K);g(je.$$.fragment,xt),pn=s(xt),De=d(xt,"P",{"data-svelte-h":!0}),u(De)!=="svelte-eq620n"&&(De.innerHTML=Vn),xt.forEach(t),x.forEach(t),wt=s(e),nt=d(e,"P",{}),w(nt).forEach(t),this.h()},h(){T(i,"name","hf:doc:metadata"),T(i,"content",no),Nn(R.src,bn="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/instructblip_architecture.jpg")||T(R,"src",bn),T(R,"alt","drawing"),T(R,"width","600"),T(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,n){a(document.head,i),l(e,I,n),l(e,p,n),l(e,m,n),h(v,e,n),l(e,o,n),h(M,e,n),l(e,ot,n),l(e,te,n),l(e,rt,n),l(e,ne,n),l(e,st,n),l(e,oe,n),l(e,at,n),l(e,re,n),l(e,it,n),l(e,se,n),l(e,lt,n),l(e,R,n),l(e,ct,n),l(e,ae,n),l(e,dt,n),l(e,ie,n),l(e,mt,n),h(le,e,n),l(e,pt,n),l(e,$,n),h(ce,$,null),a($,Jt),a($,Fe),a($,Ut),a($,Je),a($,zt),h(N,$,null),a($,Wt),a($,S),h(de,S,null),a(S,Zt),a(S,Ue),l(e,ut,n),h(me,e,n),l(e,ft,n),l(e,F,n),h(pe,F,null),a(F,Pt),a(F,ze),a(F,qt),a(F,We),a(F,Lt),h(H,F,null),l(e,gt,n),h(ue,e,n),l(e,ht,n),l(e,J,n),h(fe,J,null),a(J,Qt),a(J,Ze),a(J,Vt),a(J,Pe),a(J,Gt),h(E,J,null),l(e,_t,n),h(ge,e,n),l(e,bt,n),l(e,C,n),h(he,C,null),a(C,Rt),a(C,qe),a(C,Nt),a(C,Le),a(C,St),a(C,X),h(_e,X,null),a(X,Ht),a(X,Qe),a(C,Et),a(C,Y),h(be,Y,null),a(Y,Xt),a(Y,Ve),l(e,yt,n),h(ye,e,n),l(e,vt,n),l(e,V,n),h(ve,V,null),a(V,Yt),a(V,L),h(Me,L,null),a(L,At),a(L,Ge),a(L,Dt),h(A,L,null),l(e,Mt,n),h(Ie,e,n),l(e,It,n),l(e,Z,n),h(Be,Z,null),a(Z,Ot),a(Z,Re),a(Z,Kt),a(Z,Q),h(Te,Q,null),a(Q,en),a(Q,Ne),a(Q,tn),a(Q,Se),l(e,Bt,n),h(we,e,n),l(e,Tt,n),l(e,B,n),h($e,B,null),a(B,nn),a(B,He),a(B,on),a(B,Ee),a(B,rn),a(B,Xe),a(B,sn),a(B,Ye),a(B,an),a(B,W),h(Ce,W,null),a(W,ln),a(W,Ae),a(W,cn),h(D,W,null),a(W,dn),h(O,W,null),a(B,mn),a(B,K),h(je,K,null),a(K,pn),a(K,De),l(e,wt,n),l(e,nt,n),$t=!0},p(e,[n]){const U={};n&2&&(U.$$scope={dirty:n,ctx:e}),N.$set(U);const xe={};n&2&&(xe.$$scope={dirty:n,ctx:e}),H.$set(xe);const P={};n&2&&(P.$$scope={dirty:n,ctx:e}),E.$set(P);const q={};n&2&&(q.$$scope={dirty:n,ctx:e}),A.$set(q);const z={};n&2&&(z.$$scope={dirty:n,ctx:e}),D.$set(z);const ke={};n&2&&(ke.$$scope={dirty:n,ctx:e}),O.$set(ke)},i(e){$t||(_(v.$$.fragment,e),_(M.$$.fragment,e),_(le.$$.fragment,e),_(ce.$$.fragment,e),_(N.$$.fragment,e),_(de.$$.fragment,e),_(me.$$.fragment,e),_(pe.$$.fragment,e),_(H.$$.fragment,e),_(ue.$$.fragment,e),_(fe.$$.fragment,e),_(E.$$.fragment,e),_(ge.$$.fragment,e),_(he.$$.fragment,e),_(_e.$$.fragment,e),_(be.$$.fragment,e),_(ye.$$.fragment,e),_(ve.$$.fragment,e),_(Me.$$.fragment,e),_(A.$$.fragment,e),_(Ie.$$.fragment,e),_(Be.$$.fragment,e),_(Te.$$.fragment,e),_(we.$$.fragment,e),_($e.$$.fragment,e),_(Ce.$$.fragment,e),_(D.$$.fragment,e),_(O.$$.fragment,e),_(je.$$.fragment,e),$t=!0)},o(e){b(v.$$.fragment,e),b(M.$$.fragment,e),b(le.$$.fragment,e),b(ce.$$.fragment,e),b(N.$$.fragment,e),b(de.$$.fragment,e),b(me.$$.fragment,e),b(pe.$$.fragment,e),b(H.$$.fragment,e),b(ue.$$.fragment,e),b(fe.$$.fragment,e),b(E.$$.fragment,e),b(ge.$$.fragment,e),b(he.$$.fragment,e),b(_e.$$.fragment,e),b(be.$$.fragment,e),b(ye.$$.fragment,e),b(ve.$$.fragment,e),b(Me.$$.fragment,e),b(A.$$.fragment,e),b(Ie.$$.fragment,e),b(Be.$$.fragment,e),b(Te.$$.fragment,e),b(we.$$.fragment,e),b($e.$$.fragment,e),b(Ce.$$.fragment,e),b(D.$$.fragment,e),b(O.$$.fragment,e),b(je.$$.fragment,e),$t=!1},d(e){e&&(t(I),t(p),t(m),t(o),t(ot),t(te),t(rt),t(ne),t(st),t(oe),t(at),t(re),t(it),t(se),t(lt),t(R),t(ct),t(ae),t(dt),t(ie),t(mt),t(pt),t($),t(ut),t(ft),t(F),t(gt),t(ht),t(J),t(_t),t(bt),t(C),t(yt),t(vt),t(V),t(Mt),t(It),t(Z),t(Bt),t(Tt),t(B),t(wt),t(nt)),t(i),y(v,e),y(M,e),y(le,e),y(ce),y(N),y(de),y(me,e),y(pe),y(H),y(ue,e),y(fe),y(E),y(ge,e),y(he),y(_e),y(be),y(ye,e),y(ve),y(Me),y(A),y(Ie,e),y(Be),y(Te),y(we,e),y($e),y(Ce),y(D),y(O),y(je)}}}const no='{"title":"InstructBLIP","local":"instructblip","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"InstructBlipConfig","local":"transformers.InstructBlipConfig","sections":[],"depth":2},{"title":"InstructBlipVisionConfig","local":"transformers.InstructBlipVisionConfig","sections":[],"depth":2},{"title":"InstructBlipQFormerConfig","local":"transformers.InstructBlipQFormerConfig","sections":[],"depth":2},{"title":"InstructBlipProcessor","local":"transformers.InstructBlipProcessor","sections":[],"depth":2},{"title":"InstructBlipVisionModel","local":"transformers.InstructBlipVisionModel","sections":[],"depth":2},{"title":"InstructBlipQFormerModel","local":"transformers.InstructBlipQFormerModel","sections":[],"depth":2},{"title":"InstructBlipForConditionalGeneration","local":"transformers.InstructBlipForConditionalGeneration","sections":[],"depth":2}],"depth":1}';function oo(j){return Sn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class po extends Hn{constructor(i){super(),En(this,i,oo,to,Rn,{})}}export{po as component};
