import{S as Mc,i as $c,s as Tc,e as s,k as d,w as y,t as o,M as Cc,c as a,d as n,m as p,a as i,x as I,h as r,b as l,N as kc,G as e,g as u,y as B,q as w,o as M,B as $,v as Ec,L as or}from"../../chunks/vendor-hf-doc-builder.js";import{T as wc}from"../../chunks/Tip-hf-doc-builder.js";import{D as U}from"../../chunks/Docstring-hf-doc-builder.js";import{C as rr}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as ue}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as nr}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function jc(x){let f,C,_,h,v;return h=new rr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMCglMEElMjAlMjAlMjAlMjBJbnN0cnVjdEJsaXBWaXNpb25Db25maWclMkMlMEElMjAlMjAlMjAlMjBJbnN0cnVjdEJsaXBRRm9ybWVyQ29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwT1BUQ29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwSW5zdHJ1Y3RCbGlwQ29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwSW5zdHJ1Y3RCbGlwRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTJDJTBBKSUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBJbnN0cnVjdEJsaXBDb25maWclMjB3aXRoJTIwU2FsZXNmb3JjZSUyRmluc3RydWN0LWJsaXAtZmxhbi10NSUyMHN0eWxlJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBJbnN0cnVjdEJsaXBDb25maWcoKSUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBJbnN0cnVjdEJsaXBGb3JDb25kaXRpb25hbEdlbmVyYXRpb24lMjAod2l0aCUyMHJhbmRvbSUyMHdlaWdodHMpJTIwZnJvbSUyMHRoZSUyMFNhbGVzZm9yY2UlMkZpbnN0cnVjdC1ibGlwLWZsYW4tdDUlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFtb2RlbCUyMCUzRCUyMEluc3RydWN0QmxpcEZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbihjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWclMEElMEElMjMlMjBXZSUyMGNhbiUyMGFsc28lMjBpbml0aWFsaXplJTIwYSUyMEluc3RydWN0QmxpcENvbmZpZyUyMGZyb20lMjBhJTIwSW5zdHJ1Y3RCbGlwVmlzaW9uQ29uZmlnJTJDJTIwSW5zdHJ1Y3RCbGlwUUZvcm1lckNvbmZpZyUyMGFuZCUyMGFueSUyMFByZXRyYWluZWRDb25maWclMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBJbnN0cnVjdEJMSVAlMjB2aXNpb24lMkMlMjBJbnN0cnVjdEJMSVAlMjBRLUZvcm1lciUyMGFuZCUyMGxhbmd1YWdlJTIwbW9kZWwlMjBjb25maWd1cmF0aW9ucyUwQXZpc2lvbl9jb25maWclMjAlM0QlMjBJbnN0cnVjdEJsaXBWaXNpb25Db25maWcoKSUwQXFmb3JtZXJfY29uZmlnJTIwJTNEJTIwSW5zdHJ1Y3RCbGlwUUZvcm1lckNvbmZpZygpJTBBdGV4dF9jb25maWclMjAlM0QlMjBPUFRDb25maWcoKSUwQSUwQWNvbmZpZyUyMCUzRCUyMEluc3RydWN0QmxpcENvbmZpZy5mcm9tX3RleHRfdmlzaW9uX2NvbmZpZ3ModmlzaW9uX2NvbmZpZyUyQyUyMHFmb3JtZXJfY29uZmlnJTJDJTIwdGV4dF9jb25maWcp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> (
<span class="hljs-meta">... </span>    InstructBlipVisionConfig,
<span class="hljs-meta">... </span>    InstructBlipQFormerConfig,
<span class="hljs-meta">... </span>    OPTConfig,
<span class="hljs-meta">... </span>    InstructBlipConfig,
<span class="hljs-meta">... </span>    InstructBlipForConditionalGeneration,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipConfig with Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = InstructBlipConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipForConditionalGeneration (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipForConditionalGeneration(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># We can also initialize a InstructBlipConfig from a InstructBlipVisionConfig, InstructBlipQFormerConfig and any PretrainedConfig</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing InstructBLIP vision, InstructBLIP Q-Former and language model configurations</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>vision_config = InstructBlipVisionConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>qformer_config = InstructBlipQFormerConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>text_config = OPTConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span>config = InstructBlipConfig.from_text_vision_configs(vision_config, qformer_config, text_config)`}}),{c(){f=s("p"),C=o("Example:"),_=d(),y(h.$$.fragment)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Example:"),g.forEach(n),_=p(c),I(h.$$.fragment,c)},m(c,g){u(c,f,g),e(f,C),u(c,_,g),B(h,c,g),v=!0},p:or,i(c){v||(w(h.$$.fragment,c),v=!0)},o(c){M(h.$$.fragment,c),v=!1},d(c){c&&n(f),c&&n(_),$(h,c)}}}function Fc(x){let f,C,_,h,v;return h=new rr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEluc3RydWN0QmxpcFZpc2lvbkNvbmZpZyUyQyUyMEluc3RydWN0QmxpcFZpc2lvbk1vZGVsJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEluc3RydWN0QmxpcFZpc2lvbkNvbmZpZyUyMHdpdGglMjBTYWxlc2ZvcmNlJTJGaW5zdHJ1Y3QtYmxpcC1mbGFuLXQ1JTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMEluc3RydWN0QmxpcFZpc2lvbkNvbmZpZygpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEluc3RydWN0QmxpcFZpc2lvbk1vZGVsJTIwKHdpdGglMjByYW5kb20lMjB3ZWlnaHRzKSUyMGZyb20lMjB0aGUlMjBTYWxlc2ZvcmNlJTJGaW5zdHJ1Y3QtYmxpcC1mbGFuLXQ1JTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBbW9kZWwlMjAlM0QlMjBJbnN0cnVjdEJsaXBWaXNpb25Nb2RlbChjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> InstructBlipVisionConfig, InstructBlipVisionModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipVisionConfig with Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = InstructBlipVisionConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBlipVisionModel (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipVisionModel(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`}}),{c(){f=s("p"),C=o("Example:"),_=d(),y(h.$$.fragment)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Example:"),g.forEach(n),_=p(c),I(h.$$.fragment,c)},m(c,g){u(c,f,g),e(f,C),u(c,_,g),B(h,c,g),v=!0},p:or,i(c){v||(w(h.$$.fragment,c),v=!0)},o(c){M(h.$$.fragment,c),v=!1},d(c){c&&n(f),c&&n(_),$(h,c)}}}function Pc(x){let f,C,_,h,v;return h=new rr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEluc3RydWN0QmxpcFFGb3JtZXJDb25maWclMkMlMjBJbnN0cnVjdEJsaXBRRm9ybWVyTW9kZWwlMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwSW5zdHJ1Y3RCTElQJTIwU2FsZXNmb3JjZSUyRmluc3RydWN0LWJsaXAtZmxhbi10NSUyMHN0eWxlJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBJbnN0cnVjdEJsaXBRRm9ybWVyQ29uZmlnKCklMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwbW9kZWwlMjAod2l0aCUyMHJhbmRvbSUyMHdlaWdodHMpJTIwZnJvbSUyMHRoZSUyMFNhbGVzZm9yY2UlMkZpbnN0cnVjdC1ibGlwLWZsYW4tdDUlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFtb2RlbCUyMCUzRCUyMEluc3RydWN0QmxpcFFGb3JtZXJNb2RlbChjb25maWd1cmF0aW9uKSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> InstructBlipQFormerConfig, InstructBlipQFormerModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a InstructBLIP Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = InstructBlipQFormerConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipQFormerModel(configuration)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`}}),{c(){f=s("p"),C=o("Examples:"),_=d(),y(h.$$.fragment)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Examples:"),g.forEach(n),_=p(c),I(h.$$.fragment,c)},m(c,g){u(c,f,g),e(f,C),u(c,_,g),B(h,c,g),v=!0},p:or,i(c){v||(w(h.$$.fragment,c),v=!0)},o(c){M(h.$$.fragment,c),v=!1},d(c){c&&n(f),c&&n(_),$(h,c)}}}function xc(x){let f,C,_,h,v;return{c(){f=s("p"),C=o("Although the recipe for forward pass needs to be defined within this function, one should call the "),_=s("code"),h=o("Module"),v=o(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Although the recipe for forward pass needs to be defined within this function, one should call the "),_=a(g,"CODE",{});var G=i(_);h=r(G,"Module"),G.forEach(n),v=r(g,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),g.forEach(n)},m(c,g){u(c,f,g),e(f,C),e(f,_),e(_,h),e(f,v)},d(c){c&&n(f)}}}function Jc(x){let f,C,_,h,v;return{c(){f=s("p"),C=o("Although the recipe for forward pass needs to be defined within this function, one should call the "),_=s("code"),h=o("Module"),v=o(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Although the recipe for forward pass needs to be defined within this function, one should call the "),_=a(g,"CODE",{});var G=i(_);h=r(G,"Module"),G.forEach(n),v=r(g,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),g.forEach(n)},m(c,g){u(c,f,g),e(f,C),e(f,_),e(_,h),e(f,v)},d(c){c&&n(f)}}}function zc(x){let f,C,_,h,v;return h=new rr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEluc3RydWN0QmxpcFByb2Nlc3NvciUyQyUyMEluc3RydWN0QmxpcEZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEElMEFtb2RlbCUyMCUzRCUyMEluc3RydWN0QmxpcEZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbi5mcm9tX3ByZXRyYWluZWQoJTIyU2FsZXNmb3JjZSUyRmluc3RydWN0YmxpcC12aWN1bmEtN2IlMjIpJTBBcHJvY2Vzc29yJTIwJTNEJTIwSW5zdHJ1Y3RCbGlwUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJTYWxlc2ZvcmNlJTJGaW5zdHJ1Y3RibGlwLXZpY3VuYS03YiUyMiklMEElMEFkZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTIyJTIwaWYlMjB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpJTIwZWxzZSUyMCUyMmNwdSUyMiUwQW1vZGVsLnRvKGRldmljZSklMEF1cmwlMjAlM0QlMjAlMjJodHRwcyUzQSUyRiUyRnJhdy5naXRodWJ1c2VyY29udGVudC5jb20lMkZzYWxlc2ZvcmNlJTJGTEFWSVMlMkZtYWluJTJGZG9jcyUyRl9zdGF0aWMlMkZDb25mdXNpbmctUGljdHVyZXMuanBnJTIyJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpLmNvbnZlcnQoJTIyUkdCJTIyKSUwQXByb21wdCUyMCUzRCUyMCUyMldoYXQlMjBpcyUyMHVudXN1YWwlMjBhYm91dCUyMHRoaXMlMjBpbWFnZSUzRiUyMiUwQWlucHV0cyUyMCUzRCUyMHByb2Nlc3NvcihpbWFnZXMlM0RpbWFnZSUyQyUyMHRleHQlM0Rwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhkZXZpY2UpJTBBJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCUwQSUyMCUyMCUyMCUyMCoqaW5wdXRzJTJDJTBBJTIwJTIwJTIwJTIwZG9fc2FtcGxlJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBudW1fYmVhbXMlM0Q1JTJDJTBBJTIwJTIwJTIwJTIwbWF4X2xlbmd0aCUzRDI1NiUyQyUwQSUyMCUyMCUyMCUyMG1pbl9sZW5ndGglM0QxJTJDJTBBJTIwJTIwJTIwJTIwdG9wX3AlM0QwLjklMkMlMEElMjAlMjAlMjAlMjByZXBldGl0aW9uX3BlbmFsdHklM0QxLjUlMkMlMEElMjAlMjAlMjAlMjBsZW5ndGhfcGVuYWx0eSUzRDEuMCUyQyUwQSUyMCUyMCUyMCUyMHRlbXBlcmF0dXJlJTNEMSUyQyUwQSklMEFnZW5lcmF0ZWRfdGV4dCUyMCUzRCUyMHByb2Nlc3Nvci5iYXRjaF9kZWNvZGUob3V0cHV0cyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQuc3RyaXAoKSUwQXByaW50KGdlbmVyYXRlZF90ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> InstructBlipProcessor, InstructBlipForConditionalGeneration
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests

<span class="hljs-meta">&gt;&gt;&gt; </span>model = InstructBlipForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;Salesforce/instructblip-vicuna-7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor = InstructBlipProcessor.from_pretrained(<span class="hljs-string">&quot;Salesforce/instructblip-vicuna-7b&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw).convert(<span class="hljs-string">&quot;RGB&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;What is unusual about this image?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor(images=image, text=prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(
<span class="hljs-meta">... </span>    **inputs,
<span class="hljs-meta">... </span>    do_sample=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    num_beams=<span class="hljs-number">5</span>,
<span class="hljs-meta">... </span>    max_length=<span class="hljs-number">256</span>,
<span class="hljs-meta">... </span>    min_length=<span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>    top_p=<span class="hljs-number">0.9</span>,
<span class="hljs-meta">... </span>    repetition_penalty=<span class="hljs-number">1.5</span>,
<span class="hljs-meta">... </span>    length_penalty=<span class="hljs-number">1.0</span>,
<span class="hljs-meta">... </span>    temperature=<span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_text = processor.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>].strip()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(generated_text)
The unusual aspect of this image <span class="hljs-keyword">is</span> that a man <span class="hljs-keyword">is</span> ironing clothes on the back of a yellow SUV, which <span class="hljs-keyword">is</span> parked <span class="hljs-keyword">in</span> the middle of a busy city street. This <span class="hljs-keyword">is</span> an unconventional approach to ironing clothes, <span class="hljs-keyword">as</span> it requires the man to balance himself <span class="hljs-keyword">and</span> his ironing equipment on top of the vehicle <span class="hljs-keyword">while</span> navigating through traffic. Additionally, the presence of taxis <span class="hljs-keyword">and</span> other vehicles <span class="hljs-keyword">in</span> the scene further emphasizes the unusual nature of this situation.`}}),{c(){f=s("p"),C=o("Examples:"),_=d(),y(h.$$.fragment)},l(c){f=a(c,"P",{});var g=i(f);C=r(g,"Examples:"),g.forEach(n),_=p(c),I(h.$$.fragment,c)},m(c,g){u(c,f,g),e(f,C),u(c,_,g),B(h,c,g),v=!0},p:or,i(c){v||(w(h.$$.fragment,c),v=!0)},o(c){M(h.$$.fragment,c),v=!1},d(c){c&&n(f),c&&n(_),$(h,c)}}}function Uc(x){let f,C,_,h,v,c,g,G,sr,co,Y,fe,mn,Ue,ar,un,ir,po,V,lr,We,cr,dr,$t,pr,mr,mo,Tt,ur,uo,Ct,fn,fr,fo,kt,hr,ho,Et,Ze,gr,jt,_r,vr,go,he,qi,_o,Qe,br,Ft,yr,vo,A,Ir,qe,Br,wr,Ge,Mr,$r,bo,K,ge,hn,Ne,Tr,gn,Cr,yo,F,Se,kr,R,Pt,Er,jr,xt,Fr,Pr,Ve,xr,Jr,zr,ee,Ur,Jt,Wr,Zr,zt,Qr,qr,Gr,_e,Nr,ve,Ae,Sr,Re,Vr,Ut,Ar,Rr,Io,te,be,_n,Le,Lr,vn,Dr,Bo,W,De,Xr,ne,Hr,Wt,Or,Yr,Xe,Kr,es,ts,oe,ns,Zt,os,rs,Qt,ss,as,is,ye,wo,re,Ie,bn,He,ls,yn,cs,Mo,Z,Oe,ds,Q,ps,qt,ms,us,Ye,fs,hs,Gt,gs,_s,Nt,vs,bs,ys,se,Is,St,Bs,ws,Vt,Ms,$s,Ts,Be,$o,ae,we,In,Ke,Cs,Bn,ks,To,P,et,Es,wn,js,Fs,J,At,Ps,xs,Rt,Js,zs,Lt,Us,Ws,Mn,Zs,Qs,Dt,qs,Gs,Ns,Me,tt,Ss,nt,Vs,Xt,As,Rs,Ls,$e,ot,Ds,rt,Xs,Ht,Hs,Os,Co,ie,Te,$n,st,Ys,Tn,Ks,ko,le,at,ea,L,it,ta,ce,na,Ot,oa,ra,Cn,sa,aa,ia,Ce,Eo,de,ke,kn,lt,la,En,ca,jo,N,ct,da,jn,pa,ma,D,dt,ua,k,fa,Fn,ha,ga,Pn,_a,va,xn,ba,ya,Jn,Ia,Ba,zn,wa,Ma,Un,$a,Ta,Wn,Ca,ka,Ea,pt,mt,ja,Zn,Fa,Pa,xa,b,Ja,Qn,za,Ua,qn,Wa,Za,Gn,Qa,qa,Nn,Ga,Na,Sn,Sa,Va,Vn,Aa,Ra,An,La,Da,Rn,Xa,Ha,Ln,Oa,Ya,Dn,Ka,ei,Xn,ti,ni,Hn,oi,ri,On,si,ai,Yn,ii,li,Fo,pe,Ee,Kn,ut,ci,eo,di,Po,E,ft,pi,to,mi,ui,ht,fi,no,hi,gi,_i,gt,vi,Yt,bi,yi,Ii,_t,Bi,vt,wi,Mi,$i,q,bt,Ti,me,Ci,Kt,ki,Ei,oo,ji,Fi,Pi,je,xi,Fe,Ji,Pe,yt,zi,It,Ui,ro,Wi,Zi,xo;return c=new ue({}),Ue=new ue({}),Ne=new ue({}),Se=new U({props:{name:"class transformers.InstructBlipConfig",anchor:"transformers.InstructBlipConfig",parameters:[{name:"vision_config",val:" = None"},{name:"qformer_config",val:" = None"},{name:"text_config",val:" = None"},{name:"num_query_tokens",val:" = 32"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipConfig.vision_config",description:`<strong>vision_config</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize <a href="/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipVisionConfig">InstructBlipVisionConfig</a>.`,name:"vision_config"},{anchor:"transformers.InstructBlipConfig.qformer_config",description:`<strong>qformer_config</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize <a href="/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipQFormerConfig">InstructBlipQFormerConfig</a>.`,name:"qformer_config"},{anchor:"transformers.InstructBlipConfig.text_config",description:`<strong>text_config</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize any <a href="/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a>.`,name:"text_config"},{anchor:"transformers.InstructBlipConfig.num_query_tokens",description:`<strong>num_query_tokens</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
The number of query tokens passed through the Transformer.`,name:"num_query_tokens"},{anchor:"transformers.InstructBlipConfig.kwargs",description:`<strong>kwargs</strong> (<em>optional</em>) &#x2014;
Dictionary of keyword arguments.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/configuration_instructblip.py#L252"}}),_e=new nr({props:{anchor:"transformers.InstructBlipConfig.example",$$slots:{default:[jc]},$$scope:{ctx:x}}}),Ae=new U({props:{name:"from_vision_qformer_text_configs",anchor:"transformers.InstructBlipConfig.from_vision_qformer_text_configs",parameters:[{name:"vision_config",val:": InstructBlipVisionConfig"},{name:"qformer_config",val:": InstructBlipQFormerConfig"},{name:"text_config",val:": PretrainedConfig"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/configuration_instructblip.py#L337",returnDescription:`
<p>An instance of a configuration object</p>
`,returnType:`
<p><a
  href="/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipConfig"
>InstructBlipConfig</a></p>
`}}),Le=new ue({}),De=new U({props:{name:"class transformers.InstructBlipVisionConfig",anchor:"transformers.InstructBlipVisionConfig",parameters:[{name:"hidden_size",val:" = 1408"},{name:"intermediate_size",val:" = 6144"},{name:"num_hidden_layers",val:" = 39"},{name:"num_attention_heads",val:" = 16"},{name:"image_size",val:" = 224"},{name:"patch_size",val:" = 14"},{name:"hidden_act",val:" = 'gelu'"},{name:"layer_norm_eps",val:" = 1e-06"},{name:"attention_dropout",val:" = 0.0"},{name:"initializer_range",val:" = 1e-10"},{name:"qkv_bias",val:" = True"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipVisionConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 1408) &#x2014;
Dimensionality of the encoder layers and the pooler layer.`,name:"hidden_size"},{anchor:"transformers.InstructBlipVisionConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 6144) &#x2014;
Dimensionality of the &#x201C;intermediate&#x201D; (i.e., feed-forward) layer in the Transformer encoder.`,name:"intermediate_size"},{anchor:"transformers.InstructBlipVisionConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 39) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.InstructBlipVisionConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 16) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.InstructBlipVisionConfig.image_size",description:`<strong>image_size</strong> (<code>int</code>, <em>optional</em>, defaults to 224) &#x2014;
The size (resolution) of each image.`,name:"image_size"},{anchor:"transformers.InstructBlipVisionConfig.patch_size",description:`<strong>patch_size</strong> (<code>int</code>, <em>optional</em>, defaults to 14) &#x2014;
The size (resolution) of each patch.`,name:"patch_size"},{anchor:"transformers.InstructBlipVisionConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>function</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the encoder and pooler. If string, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;selu&quot;</code> and <code>&quot;gelu_new&quot;</code> \`<code>&quot;gelu&quot;</code> are supported. to 1e-5): The epsilon used by the layer
normalization layers.`,name:"hidden_act"},{anchor:"transformers.InstructBlipVisionConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-6) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.InstructBlipVisionConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_dropout"},{anchor:"transformers.InstructBlipVisionConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-10) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.InstructBlipVisionConfig.qkv_bias",description:`<strong>qkv_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to add a bias to the queries and values in the self-attention layers.`,name:"qkv_bias"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/configuration_instructblip.py#L33"}}),ye=new nr({props:{anchor:"transformers.InstructBlipVisionConfig.example",$$slots:{default:[Fc]},$$scope:{ctx:x}}}),He=new ue({}),Oe=new U({props:{name:"class transformers.InstructBlipQFormerConfig",anchor:"transformers.InstructBlipQFormerConfig",parameters:[{name:"vocab_size",val:" = 30522"},{name:"hidden_size",val:" = 768"},{name:"num_hidden_layers",val:" = 12"},{name:"num_attention_heads",val:" = 12"},{name:"intermediate_size",val:" = 3072"},{name:"hidden_act",val:" = 'gelu'"},{name:"hidden_dropout_prob",val:" = 0.1"},{name:"attention_probs_dropout_prob",val:" = 0.1"},{name:"max_position_embeddings",val:" = 512"},{name:"initializer_range",val:" = 0.02"},{name:"layer_norm_eps",val:" = 1e-12"},{name:"pad_token_id",val:" = 0"},{name:"position_embedding_type",val:" = 'absolute'"},{name:"cross_attention_frequency",val:" = 2"},{name:"encoder_hidden_size",val:" = 1408"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipQFormerConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 30522) &#x2014;
Vocabulary size of the Q-Former model. Defines the number of different tokens that can be represented by
the <code>inputs_ids</code> passed when calling the model.`,name:"vocab_size"},{anchor:"transformers.InstructBlipQFormerConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 768) &#x2014;
Dimensionality of the encoder layers and the pooler layer.`,name:"hidden_size"},{anchor:"transformers.InstructBlipQFormerConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 12) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.InstructBlipQFormerConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 12) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.InstructBlipQFormerConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 3072) &#x2014;
Dimensionality of the &#x201C;intermediate&#x201D; (often named feed-forward) layer in the Transformer encoder.`,name:"intermediate_size"},{anchor:"transformers.InstructBlipQFormerConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>Callable</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the encoder and pooler. If string, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;silu&quot;</code> and <code>&quot;gelu_new&quot;</code> are supported.`,name:"hidden_act"},{anchor:"transformers.InstructBlipQFormerConfig.hidden_dropout_prob",description:`<strong>hidden_dropout_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.`,name:"hidden_dropout_prob"},{anchor:"transformers.InstructBlipQFormerConfig.attention_probs_dropout_prob",description:`<strong>attention_probs_dropout_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_probs_dropout_prob"},{anchor:"transformers.InstructBlipQFormerConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to 512) &#x2014;
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).`,name:"max_position_embeddings"},{anchor:"transformers.InstructBlipQFormerConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 0.02) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.InstructBlipQFormerConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-12) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.InstructBlipQFormerConfig.position_embedding_type",description:`<strong>position_embedding_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;absolute&quot;</code>) &#x2014;
Type of position embedding. Choose one of <code>&quot;absolute&quot;</code>, <code>&quot;relative_key&quot;</code>, <code>&quot;relative_key_query&quot;</code>. For
positional embeddings use <code>&quot;absolute&quot;</code>. For more information on <code>&quot;relative_key&quot;</code>, please refer to
<a href="https://arxiv.org/abs/1803.02155" rel="nofollow">Self-Attention with Relative Position Representations (Shaw et al.)</a>.
For more information on <code>&quot;relative_key_query&quot;</code>, please refer to <em>Method 4</em> in <a href="https://arxiv.org/abs/2009.13658" rel="nofollow">Improve Transformer Models
with Better Relative Position Embeddings (Huang et al.)</a>.`,name:"position_embedding_type"},{anchor:"transformers.InstructBlipQFormerConfig.cross_attention_frequency",description:`<strong>cross_attention_frequency</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
The frequency of adding cross-attention to the Transformer layers.`,name:"cross_attention_frequency"},{anchor:"transformers.InstructBlipQFormerConfig.encoder_hidden_size",description:`<strong>encoder_hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 1408) &#x2014;
The hidden size of the hidden states for cross-attention.`,name:"encoder_hidden_size"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/configuration_instructblip.py#L134"}}),Be=new nr({props:{anchor:"transformers.InstructBlipQFormerConfig.example",$$slots:{default:[Pc]},$$scope:{ctx:x}}}),Ke=new ue({}),et=new U({props:{name:"class transformers.InstructBlipProcessor",anchor:"transformers.InstructBlipProcessor",parameters:[{name:"image_processor",val:""},{name:"tokenizer",val:""},{name:"qformer_tokenizer",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipProcessor.image_processor",description:`<strong>image_processor</strong> (<code>BlipImageProcessor</code>) &#x2014;
An instance of <a href="/docs/transformers/v4.32.0/en/model_doc/blip#transformers.BlipImageProcessor">BlipImageProcessor</a>. The image processor is a required input.`,name:"image_processor"},{anchor:"transformers.InstructBlipProcessor.tokenizer",description:"<strong>tokenizer</strong> (<code>AutoTokenizer</code>) &#x2014;\nAn instance of [&#x2018;PreTrainedTokenizer`]. The tokenizer is a required input.",name:"tokenizer"},{anchor:"transformers.InstructBlipProcessor.qformer_tokenizer",description:"<strong>qformer_tokenizer</strong> (<code>AutoTokenizer</code>) &#x2014;\nAn instance of [&#x2018;PreTrainedTokenizer`]. The Q-Former tokenizer is a required input.",name:"qformer_tokenizer"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/processing_instructblip.py#L30"}}),tt=new U({props:{name:"batch_decode",anchor:"transformers.InstructBlipProcessor.batch_decode",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/processing_instructblip.py#L135"}}),ot=new U({props:{name:"decode",anchor:"transformers.InstructBlipProcessor.decode",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/processing_instructblip.py#L143"}}),st=new ue({}),at=new U({props:{name:"class transformers.InstructBlipVisionModel",anchor:"transformers.InstructBlipVisionModel",parameters:[{name:"config",val:": InstructBlipVisionConfig"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/modeling_instructblip.py#L500"}}),it=new U({props:{name:"forward",anchor:"transformers.InstructBlipVisionModel.forward",parameters:[{name:"pixel_values",val:": typing.Optional[torch.FloatTensor] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.InstructBlipVisionModel.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using <a href="/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See
<code>InstructBlipProcessor.__call__()</code> for details.`,name:"pixel_values"},{anchor:"transformers.InstructBlipVisionModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.InstructBlipVisionModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.InstructBlipVisionModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/modeling_instructblip.py#L515",returnDescription:`
<p>A <a
  href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPooling"
>transformers.modeling_outputs.BaseModelOutputWithPooling</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<code>&lt;class 'transformers.models.instructblip.configuration_instructblip.InstructBlipVisionConfig'&gt;</code>) and inputs.</p>
<ul>
<li>
<p><strong>last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) \u2014 Sequence of hidden-states at the output of the last layer of the model.</p>
</li>
<li>
<p><strong>pooler_output</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, hidden_size)</code>) \u2014 Last layer hidden-state of the first token of the sequence (classification token) after further processing
through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns
the classification token after processing through a linear layer and a tanh activation function. The linear
layer weights are trained from the next sentence prediction (classification) objective during pretraining.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPooling"
>transformers.modeling_outputs.BaseModelOutputWithPooling</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),Ce=new wc({props:{$$slots:{default:[xc]},$$scope:{ctx:x}}}),lt=new ue({}),ct=new U({props:{name:"class transformers.InstructBlipQFormerModel",anchor:"transformers.InstructBlipQFormerModel",parameters:[{name:"config",val:": InstructBlipQFormerConfig"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/modeling_instructblip.py#L1052"}}),dt=new U({props:{name:"forward",anchor:"transformers.InstructBlipQFormerModel.forward",parameters:[{name:"input_ids",val:""},{name:"attention_mask",val:" = None"},{name:"position_ids",val:" = None"},{name:"query_embeds",val:" = None"},{name:"head_mask",val:" = None"},{name:"encoder_hidden_states",val:" = None"},{name:"encoder_attention_mask",val:" = None"},{name:"past_key_values",val:" = None"},{name:"use_cache",val:" = None"},{name:"output_attentions",val:" = None"},{name:"output_hidden_states",val:" = None"},{name:"return_dict",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/modeling_instructblip.py#L1125"}}),ut=new ue({}),ft=new U({props:{name:"class transformers.InstructBlipForConditionalGeneration",anchor:"transformers.InstructBlipForConditionalGeneration",parameters:[{name:"config",val:": InstructBlipConfig"}],parametersDescription:[{anchor:"transformers.InstructBlipForConditionalGeneration.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipConfig">InstructBlipConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/modeling_instructblip.py#L1258"}}),bt=new U({props:{name:"forward",anchor:"transformers.InstructBlipForConditionalGeneration.forward",parameters:[{name:"pixel_values",val:": FloatTensor"},{name:"qformer_input_ids",val:": FloatTensor"},{name:"qformer_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"input_ids",val:": typing.Optional[torch.FloatTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"decoder_input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"decoder_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.InstructBlipForConditionalGeneration.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using <a href="/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See
<code>InstructBlipProcessor.__call__()</code> for details.`,name:"pixel_values"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.qformer_input_ids",description:`<strong>qformer_input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
to serve as text prompt, which the Q-Former model will encode.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See <code>InstructBlipProcessor.__call__()</code> for
details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"qformer_input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.qformer_attention_mask",description:`<strong>qformer_attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"qformer_attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of input sequence tokens in the vocabulary of the language model. Input tokens can optionally be
provided to serve as text prompt, which the language model can continue.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipProcessor">InstructBlipProcessor</a>. See <code>InstructBlipProcessor.__call__()</code> for
details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.decoder_input_ids",description:`<strong>decoder_input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of decoder input sequence tokens in the vocabulary of the language model. Only relevant in case an
encoder-decoder language model (like T5) is used.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.32.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/v4.32.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.32.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details. <a href="../glossary#decoder-input-ids">What are decoder input IDs?</a>`,name:"decoder_input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.decoder_attention_mask",description:`<strong>decoder_attention_mask</strong> (<code>torch.BoolTensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Default behavior: generate a tensor that ignores pad tokens in <code>decoder_input_ids</code>. Causal mask will also
be used by default.</p>
<p>Only relevant in case an encoder-decoder language model (like T5) is used.`,name:"decoder_attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.InstructBlipForConditionalGeneration.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) &#x2014;
Labels for computing the language modeling loss. Indices should be in <code>[-100, 0, ..., config.vocab_size - 1]</code>. All labels set to <code>-100</code> are ignored (masked), the loss is only computed for labels in <code>[0, ..., config.vocab_size]</code>`,name:"labels"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/modeling_instructblip.py#L1331",returnDescription:`
<p>A <code>transformers.models.instructblip.modeling_instructblip.InstructBlipForConditionalGenerationModelOutput</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<code>&lt;class 'transformers.models.instructblip.configuration_instructblip.InstructBlipVisionConfig'&gt;</code>) and inputs.</p>
<ul>
<li><strong>loss</strong> (<code>torch.FloatTensor</code>, <em>optional</em>, returned when <code>labels</code> is provided, <code>torch.FloatTensor</code> of shape <code>(1,)</code>) \u2014 Language modeling loss from the language model.</li>
<li><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) \u2014 Prediction scores of the language modeling head of the language model.</li>
<li><strong>vision_outputs</strong> (<code>BaseModelOutputWithPooling</code>) \u2014 Outputs of the vision encoder.</li>
<li><strong>qformer_outputs</strong> (<code>BaseModelOutputWithPoolingAndCrossAttentions</code>) \u2014 Outputs of the Q-Former (Querying Transformer).</li>
<li><strong>language_model_outputs</strong> (<code>CausalLMOutputWithPast</code> or <code>Seq2SeqLMOutput</code>) \u2014 Outputs of the language model.</li>
</ul>
`,returnType:`
<p><code>transformers.models.instructblip.modeling_instructblip.InstructBlipForConditionalGenerationModelOutput</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),je=new wc({props:{$$slots:{default:[Jc]},$$scope:{ctx:x}}}),Fe=new nr({props:{anchor:"transformers.InstructBlipForConditionalGeneration.forward.example",$$slots:{default:[zc]},$$scope:{ctx:x}}}),yt=new U({props:{name:"generate",anchor:"transformers.InstructBlipForConditionalGeneration.generate",parameters:[{name:"pixel_values",val:": FloatTensor"},{name:"qformer_input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"qformer_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"**generate_kwargs",val:""}],parametersDescription:[{anchor:"transformers.InstructBlipForConditionalGeneration.generate.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape (batch_size, num_channels, height, width)) &#x2014;
Input images to be processed.`,name:"pixel_values"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.qformer_input_ids",description:`<strong>qformer_input_ids</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
The sequence used as a prompt to be fed to the Q-Former module.`,name:"qformer_input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.qformer_attention_mask",description:`<strong>qformer_attention_mask</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices.`,name:"qformer_attention_mask"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
The sequence used as a prompt for the generation.`,name:"input_ids"},{anchor:"transformers.InstructBlipForConditionalGeneration.generate.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.LongTensor</code> of shape (batch_size, sequence_length), <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices.`,name:"attention_mask"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/instructblip/modeling_instructblip.py#L1486",returnDescription:`
<p>A list of strings of length batch_size * num_captions.</p>
`,returnType:`
<p>captions (list)</p>
`}}),{c(){f=s("meta"),C=d(),_=s("h1"),h=s("a"),v=s("span"),y(c.$$.fragment),g=d(),G=s("span"),sr=o("InstructBLIP"),co=d(),Y=s("h2"),fe=s("a"),mn=s("span"),y(Ue.$$.fragment),ar=d(),un=s("span"),ir=o("Overview"),po=d(),V=s("p"),lr=o("The InstructBLIP model was proposed in "),We=s("a"),cr=o("InstructBLIP: Towards General-purpose Vision-Language Models with Instruction Tuning"),dr=o(` by Wenliang Dai, Junnan Li, Dongxu Li, Anthony Meng Huat Tiong, Junqi Zhao, Weisheng Wang, Boyang Li, Pascale Fung, Steven Hoi.
InstructBLIP leverages the `),$t=s("a"),pr=o("BLIP-2"),mr=o(" architecture for visual instruction tuning."),mo=d(),Tt=s("p"),ur=o("The abstract from the paper is the following:"),uo=d(),Ct=s("p"),fn=s("em"),fr=o("General-purpose language models that can solve various language-domain tasks have emerged driven by the pre-training and instruction-tuning pipeline. However, building general-purpose vision-language models is challenging due to the increased task discrepancy introduced by the additional visual input. Although vision-language pre-training has been widely studied, vision-language instruction tuning remains relatively less explored. In this paper, we conduct a systematic and comprehensive study on vision-language instruction tuning based on the pre-trained BLIP-2 models. We gather a wide variety of 26 publicly available datasets, transform them into instruction tuning format and categorize them into two clusters for held-in instruction tuning and held-out zero-shot evaluation. Additionally, we introduce instruction-aware visual feature extraction, a crucial method that enables the model to extract informative features tailored to the given instruction. The resulting InstructBLIP models achieve state-of-the-art zero-shot performance across all 13 held-out datasets, substantially outperforming BLIP-2 and the larger Flamingo. Our models also lead to state-of-the-art performance when finetuned on individual downstream tasks (e.g., 90.7% accuracy on ScienceQA IMG). Furthermore, we qualitatively demonstrate the advantages of InstructBLIP over concurrent multimodal models."),fo=d(),kt=s("p"),hr=o("Tips:"),ho=d(),Et=s("ul"),Ze=s("li"),gr=o("InstructBLIP uses the same architecture as "),jt=s("a"),_r=o("BLIP-2"),vr=o(" with a tiny but important difference: it also feeds the text prompt (instruction) to the Q-Former."),go=d(),he=s("img"),_o=d(),Qe=s("small"),br=o("InstructBLIP architecture. Taken from the "),Ft=s("a"),yr=o("original paper."),vo=d(),A=s("p"),Ir=o("This model was contributed by "),qe=s("a"),Br=o("nielsr"),wr=o(`.
The original code can be found `),Ge=s("a"),Mr=o("here"),$r=o("."),bo=d(),K=s("h2"),ge=s("a"),hn=s("span"),y(Ne.$$.fragment),Tr=d(),gn=s("span"),Cr=o("InstructBlipConfig"),yo=d(),F=s("div"),y(Se.$$.fragment),kr=d(),R=s("p"),Pt=s("a"),Er=o("InstructBlipConfig"),jr=o(` is the configuration class to store the configuration of a
`),xt=s("a"),Fr=o("InstructBlipForConditionalGeneration"),Pr=o(`. It is used to instantiate a InstructBLIP model according to the specified
arguments, defining the vision model, Q-Former model and language model configs. Instantiating a configuration with
the defaults will yield a similar configuration to that of the InstructBLIP
`),Ve=s("a"),xr=o("Salesforce/instruct-blip-flan-t5"),Jr=o(" architecture."),zr=d(),ee=s("p"),Ur=o("Configuration objects inherit from "),Jt=s("a"),Wr=o("PretrainedConfig"),Zr=o(` and can be used to control the model outputs. Read the
documentation from `),zt=s("a"),Qr=o("PretrainedConfig"),qr=o(" for more information."),Gr=d(),y(_e.$$.fragment),Nr=d(),ve=s("div"),y(Ae.$$.fragment),Sr=d(),Re=s("p"),Vr=o("Instantiate a "),Ut=s("a"),Ar=o("InstructBlipConfig"),Rr=o(` (or a derived class) from a InstructBLIP vision model, Q-Former and
language model configurations.`),Io=d(),te=s("h2"),be=s("a"),_n=s("span"),y(Le.$$.fragment),Lr=d(),vn=s("span"),Dr=o("InstructBlipVisionConfig"),Bo=d(),W=s("div"),y(De.$$.fragment),Xr=d(),ne=s("p"),Hr=o("This is the configuration class to store the configuration of a "),Wt=s("a"),Or=o("InstructBlipVisionModel"),Yr=o(`. It is used to
instantiate a InstructBLIP vision encoder according to the specified arguments, defining the model architecture.
Instantiating a configuration defaults will yield a similar configuration to that of the InstructBLIP
`),Xe=s("a"),Kr=o("Salesforce/instruct-blip-flan-t5"),es=o(" architecture."),ts=d(),oe=s("p"),ns=o("Configuration objects inherit from "),Zt=s("a"),os=o("PretrainedConfig"),rs=o(` and can be used to control the model outputs. Read the
documentation from `),Qt=s("a"),ss=o("PretrainedConfig"),as=o(" for more information."),is=d(),y(ye.$$.fragment),wo=d(),re=s("h2"),Ie=s("a"),bn=s("span"),y(He.$$.fragment),ls=d(),yn=s("span"),cs=o("InstructBlipQFormerConfig"),Mo=d(),Z=s("div"),y(Oe.$$.fragment),ds=d(),Q=s("p"),ps=o("This is the configuration class to store the configuration of a "),qt=s("a"),ms=o("InstructBlipQFormerModel"),us=o(`. It is used to
instantiate a InstructBLIP Querying Transformer (Q-Former) model according to the specified arguments, defining the
model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
the InstructBLIP `),Ye=s("a"),fs=o("Salesforce/instruct-blip-flan-t5"),hs=o(`
architecture. Configuration objects inherit from `),Gt=s("a"),gs=o("PretrainedConfig"),_s=o(` and can be used to control the model outputs.
Read the documentation from `),Nt=s("a"),vs=o("PretrainedConfig"),bs=o(" for more information."),ys=d(),se=s("p"),Is=o("Note that "),St=s("a"),Bs=o("InstructBlipQFormerModel"),ws=o(" is very similar to "),Vt=s("a"),Ms=o("BertLMHeadModel"),$s=o(" with interleaved cross-attention."),Ts=d(),y(Be.$$.fragment),$o=d(),ae=s("h2"),we=s("a"),In=s("span"),y(Ke.$$.fragment),Cs=d(),Bn=s("span"),ks=o("InstructBlipProcessor"),To=d(),P=s("div"),y(et.$$.fragment),Es=d(),wn=s("p"),js=o(`Constructs an InstructBLIP processor which wraps a BLIP image processor and a LLaMa/T5 tokenizer into a single
processor.`),Fs=d(),J=s("p"),At=s("a"),Ps=o("InstructBlipProcessor"),xs=o(" offers all the functionalities of "),Rt=s("a"),Js=o("BlipImageProcessor"),zs=o(" and "),Lt=s("a"),Us=o("AutoTokenizer"),Ws=o(`. See the
docstring of `),Mn=s("code"),Zs=o("__call__()"),Qs=o(" and "),Dt=s("a"),qs=o("decode()"),Gs=o(" for more information."),Ns=d(),Me=s("div"),y(tt.$$.fragment),Ss=d(),nt=s("p"),Vs=o("This method forwards all its arguments to PreTrainedTokenizer\u2019s "),Xt=s("a"),As=o("batch_decode()"),Rs=o(`. Please
refer to the docstring of this method for more information.`),Ls=d(),$e=s("div"),y(ot.$$.fragment),Ds=d(),rt=s("p"),Xs=o("This method forwards all its arguments to PreTrainedTokenizer\u2019s "),Ht=s("a"),Hs=o("decode()"),Os=o(`. Please refer
to the docstring of this method for more information.`),Co=d(),ie=s("h2"),Te=s("a"),$n=s("span"),y(st.$$.fragment),Ys=d(),Tn=s("span"),Ks=o("InstructBlipVisionModel"),ko=d(),le=s("div"),y(at.$$.fragment),ea=d(),L=s("div"),y(it.$$.fragment),ta=d(),ce=s("p"),na=o("The "),Ot=s("a"),oa=o("InstructBlipVisionModel"),ra=o(" forward method, overrides the "),Cn=s("code"),sa=o("__call__"),aa=o(" special method."),ia=d(),y(Ce.$$.fragment),Eo=d(),de=s("h2"),ke=s("a"),kn=s("span"),y(lt.$$.fragment),la=d(),En=s("span"),ca=o("InstructBlipQFormerModel"),jo=d(),N=s("div"),y(ct.$$.fragment),da=d(),jn=s("p"),pa=o(`Querying Transformer (Q-Former), used in InstructBLIP. Slightly modified from BLIP-2 as it also takes the
instruction as input.`),ma=d(),D=s("div"),y(dt.$$.fragment),ua=d(),k=s("p"),fa=o("encoder_hidden_states  ("),Fn=s("code"),ha=o("torch.FloatTensor"),ga=o(" of shape "),Pn=s("code"),_a=o("(batch_size, sequence_length, hidden_size)"),va=o(", "),xn=s("em"),ba=o("optional"),ya=o(`):
Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
the model is configured as a decoder.
encoder_attention_mask (`),Jn=s("code"),Ia=o("torch.FloatTensor"),Ba=o(" of shape "),zn=s("code"),wa=o("(batch_size, sequence_length)"),Ma=o(", "),Un=s("em"),$a=o("optional"),Ta=o(`):
Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
the cross-attention if the model is configured as a decoder. Mask values selected in `),Wn=s("code"),Ca=o("[0, 1]"),ka=o(":"),Ea=d(),pt=s("ul"),mt=s("li"),ja=o("1 for tokens that are "),Zn=s("strong"),Fa=o("not masked"),Pa=o(","),xa=d(),b=s("li"),Ja=o("0 for tokens that are "),Qn=s("strong"),za=o("masked"),Ua=o(`.
past_key_values (`),qn=s("code"),Wa=o("tuple(tuple(torch.FloatTensor))"),Za=o(" of length "),Gn=s("code"),Qa=o("config.n_layers"),qa=o(` with each tuple having 4 tensors of:
shape `),Nn=s("code"),Ga=o("(batch_size, num_heads, sequence_length - 1, embed_size_per_head)"),Na=o(`): Contains precomputed key and
value hidden states of the attention blocks. Can be used to speed up decoding. If `),Sn=s("code"),Sa=o("past_key_values"),Va=o(` are
used, the user can optionally input only the last `),Vn=s("code"),Aa=o("decoder_input_ids"),Ra=o(` (those that don\u2019t have their past key
value states given to this model) of shape `),An=s("code"),La=o("(batch_size, 1)"),Da=o(" instead of all "),Rn=s("code"),Xa=o("decoder_input_ids"),Ha=o(` of shape
`),Ln=s("code"),Oa=o("(batch_size, sequence_length)"),Ya=o(`.
use_cache (`),Dn=s("code"),Ka=o("bool"),ei=o(", "),Xn=s("em"),ti=o("optional"),ni=o(`):
If set to `),Hn=s("code"),oi=o("True"),ri=o(", "),On=s("code"),si=o("past_key_values"),ai=o(` key value states are returned and can be used to speed up decoding (see
`),Yn=s("code"),ii=o("past_key_values"),li=o(")."),Fo=d(),pe=s("h2"),Ee=s("a"),Kn=s("span"),y(ut.$$.fragment),ci=d(),eo=s("span"),di=o("InstructBlipForConditionalGeneration"),Po=d(),E=s("div"),y(ft.$$.fragment),pi=d(),to=s("p"),mi=o(`InstructBLIP Model for generating text given an image and an optional text prompt. The model consists of a vision
encoder, Querying Transformer (Q-Former) and a language model.`),ui=d(),ht=s("p"),fi=o("One can optionally pass "),no=s("code"),hi=o("input_ids"),gi=o(` to the model, which serve as a text prompt, to make the language model continue
the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.`),_i=d(),gt=s("p"),vi=o("This model inherits from "),Yt=s("a"),bi=o("PreTrainedModel"),yi=o(`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),Ii=d(),_t=s("p"),Bi=o("This model is also a PyTorch "),vt=s("a"),wi=o("torch.nn.Module"),Mi=o(` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),$i=d(),q=s("div"),y(bt.$$.fragment),Ti=d(),me=s("p"),Ci=o("The "),Kt=s("a"),ki=o("InstructBlipForConditionalGeneration"),Ei=o(" forward method, overrides the "),oo=s("code"),ji=o("__call__"),Fi=o(" special method."),Pi=d(),y(je.$$.fragment),xi=d(),y(Fe.$$.fragment),Ji=d(),Pe=s("div"),y(yt.$$.fragment),zi=d(),It=s("p"),Ui=o("Overrides "),ro=s("code"),Wi=o("generate"),Zi=o(" function to be able to use the model as a conditional generator."),this.h()},l(t){const m=Cc('[data-svelte="svelte-1phssyn"]',document.head);f=a(m,"META",{name:!0,content:!0}),m.forEach(n),C=p(t),_=a(t,"H1",{class:!0});var Bt=i(_);h=a(Bt,"A",{id:!0,class:!0,href:!0});var so=i(h);v=a(so,"SPAN",{});var ao=i(v);I(c.$$.fragment,ao),ao.forEach(n),so.forEach(n),g=p(Bt),G=a(Bt,"SPAN",{});var io=i(G);sr=r(io,"InstructBLIP"),io.forEach(n),Bt.forEach(n),co=p(t),Y=a(t,"H2",{class:!0});var wt=i(Y);fe=a(wt,"A",{id:!0,class:!0,href:!0});var lo=i(fe);mn=a(lo,"SPAN",{});var Gi=i(mn);I(Ue.$$.fragment,Gi),Gi.forEach(n),lo.forEach(n),ar=p(wt),un=a(wt,"SPAN",{});var Ni=i(un);ir=r(Ni,"Overview"),Ni.forEach(n),wt.forEach(n),po=p(t),V=a(t,"P",{});var en=i(V);lr=r(en,"The InstructBLIP model was proposed in "),We=a(en,"A",{href:!0,rel:!0});var Si=i(We);cr=r(Si,"InstructBLIP: Towards General-purpose Vision-Language Models with Instruction Tuning"),Si.forEach(n),dr=r(en,` by Wenliang Dai, Junnan Li, Dongxu Li, Anthony Meng Huat Tiong, Junqi Zhao, Weisheng Wang, Boyang Li, Pascale Fung, Steven Hoi.
InstructBLIP leverages the `),$t=a(en,"A",{href:!0});var Vi=i($t);pr=r(Vi,"BLIP-2"),Vi.forEach(n),mr=r(en," architecture for visual instruction tuning."),en.forEach(n),mo=p(t),Tt=a(t,"P",{});var Ai=i(Tt);ur=r(Ai,"The abstract from the paper is the following:"),Ai.forEach(n),uo=p(t),Ct=a(t,"P",{});var Ri=i(Ct);fn=a(Ri,"EM",{});var Li=i(fn);fr=r(Li,"General-purpose language models that can solve various language-domain tasks have emerged driven by the pre-training and instruction-tuning pipeline. However, building general-purpose vision-language models is challenging due to the increased task discrepancy introduced by the additional visual input. Although vision-language pre-training has been widely studied, vision-language instruction tuning remains relatively less explored. In this paper, we conduct a systematic and comprehensive study on vision-language instruction tuning based on the pre-trained BLIP-2 models. We gather a wide variety of 26 publicly available datasets, transform them into instruction tuning format and categorize them into two clusters for held-in instruction tuning and held-out zero-shot evaluation. Additionally, we introduce instruction-aware visual feature extraction, a crucial method that enables the model to extract informative features tailored to the given instruction. The resulting InstructBLIP models achieve state-of-the-art zero-shot performance across all 13 held-out datasets, substantially outperforming BLIP-2 and the larger Flamingo. Our models also lead to state-of-the-art performance when finetuned on individual downstream tasks (e.g., 90.7% accuracy on ScienceQA IMG). Furthermore, we qualitatively demonstrate the advantages of InstructBLIP over concurrent multimodal models."),Li.forEach(n),Ri.forEach(n),fo=p(t),kt=a(t,"P",{});var Di=i(kt);hr=r(Di,"Tips:"),Di.forEach(n),ho=p(t),Et=a(t,"UL",{});var Xi=i(Et);Ze=a(Xi,"LI",{});var Jo=i(Ze);gr=r(Jo,"InstructBLIP uses the same architecture as "),jt=a(Jo,"A",{href:!0});var Hi=i(jt);_r=r(Hi,"BLIP-2"),Hi.forEach(n),vr=r(Jo," with a tiny but important difference: it also feeds the text prompt (instruction) to the Q-Former."),Jo.forEach(n),Xi.forEach(n),go=p(t),he=a(t,"IMG",{src:!0,alt:!0,width:!0}),_o=p(t),Qe=a(t,"SMALL",{});var Qi=i(Qe);br=r(Qi,"InstructBLIP architecture. Taken from the "),Ft=a(Qi,"A",{href:!0});var Oi=i(Ft);yr=r(Oi,"original paper."),Oi.forEach(n),Qi.forEach(n),vo=p(t),A=a(t,"P",{});var tn=i(A);Ir=r(tn,"This model was contributed by "),qe=a(tn,"A",{href:!0,rel:!0});var Yi=i(qe);Br=r(Yi,"nielsr"),Yi.forEach(n),wr=r(tn,`.
The original code can be found `),Ge=a(tn,"A",{href:!0,rel:!0});var Ki=i(Ge);Mr=r(Ki,"here"),Ki.forEach(n),$r=r(tn,"."),tn.forEach(n),bo=p(t),K=a(t,"H2",{class:!0});var zo=i(K);ge=a(zo,"A",{id:!0,class:!0,href:!0});var el=i(ge);hn=a(el,"SPAN",{});var tl=i(hn);I(Ne.$$.fragment,tl),tl.forEach(n),el.forEach(n),Tr=p(zo),gn=a(zo,"SPAN",{});var nl=i(gn);Cr=r(nl,"InstructBlipConfig"),nl.forEach(n),zo.forEach(n),yo=p(t),F=a(t,"DIV",{class:!0});var X=i(F);I(Se.$$.fragment,X),kr=p(X),R=a(X,"P",{});var Mt=i(R);Pt=a(Mt,"A",{href:!0});var ol=i(Pt);Er=r(ol,"InstructBlipConfig"),ol.forEach(n),jr=r(Mt,` is the configuration class to store the configuration of a
`),xt=a(Mt,"A",{href:!0});var rl=i(xt);Fr=r(rl,"InstructBlipForConditionalGeneration"),rl.forEach(n),Pr=r(Mt,`. It is used to instantiate a InstructBLIP model according to the specified
arguments, defining the vision model, Q-Former model and language model configs. Instantiating a configuration with
the defaults will yield a similar configuration to that of the InstructBLIP
`),Ve=a(Mt,"A",{href:!0,rel:!0});var sl=i(Ve);xr=r(sl,"Salesforce/instruct-blip-flan-t5"),sl.forEach(n),Jr=r(Mt," architecture."),Mt.forEach(n),zr=p(X),ee=a(X,"P",{});var nn=i(ee);Ur=r(nn,"Configuration objects inherit from "),Jt=a(nn,"A",{href:!0});var al=i(Jt);Wr=r(al,"PretrainedConfig"),al.forEach(n),Zr=r(nn,` and can be used to control the model outputs. Read the
documentation from `),zt=a(nn,"A",{href:!0});var il=i(zt);Qr=r(il,"PretrainedConfig"),il.forEach(n),qr=r(nn," for more information."),nn.forEach(n),Gr=p(X),I(_e.$$.fragment,X),Nr=p(X),ve=a(X,"DIV",{class:!0});var Uo=i(ve);I(Ae.$$.fragment,Uo),Sr=p(Uo),Re=a(Uo,"P",{});var Wo=i(Re);Vr=r(Wo,"Instantiate a "),Ut=a(Wo,"A",{href:!0});var ll=i(Ut);Ar=r(ll,"InstructBlipConfig"),ll.forEach(n),Rr=r(Wo,` (or a derived class) from a InstructBLIP vision model, Q-Former and
language model configurations.`),Wo.forEach(n),Uo.forEach(n),X.forEach(n),Io=p(t),te=a(t,"H2",{class:!0});var Zo=i(te);be=a(Zo,"A",{id:!0,class:!0,href:!0});var cl=i(be);_n=a(cl,"SPAN",{});var dl=i(_n);I(Le.$$.fragment,dl),dl.forEach(n),cl.forEach(n),Lr=p(Zo),vn=a(Zo,"SPAN",{});var pl=i(vn);Dr=r(pl,"InstructBlipVisionConfig"),pl.forEach(n),Zo.forEach(n),Bo=p(t),W=a(t,"DIV",{class:!0});var xe=i(W);I(De.$$.fragment,xe),Xr=p(xe),ne=a(xe,"P",{});var on=i(ne);Hr=r(on,"This is the configuration class to store the configuration of a "),Wt=a(on,"A",{href:!0});var ml=i(Wt);Or=r(ml,"InstructBlipVisionModel"),ml.forEach(n),Yr=r(on,`. It is used to
instantiate a InstructBLIP vision encoder according to the specified arguments, defining the model architecture.
Instantiating a configuration defaults will yield a similar configuration to that of the InstructBLIP
`),Xe=a(on,"A",{href:!0,rel:!0});var ul=i(Xe);Kr=r(ul,"Salesforce/instruct-blip-flan-t5"),ul.forEach(n),es=r(on," architecture."),on.forEach(n),ts=p(xe),oe=a(xe,"P",{});var rn=i(oe);ns=r(rn,"Configuration objects inherit from "),Zt=a(rn,"A",{href:!0});var fl=i(Zt);os=r(fl,"PretrainedConfig"),fl.forEach(n),rs=r(rn,` and can be used to control the model outputs. Read the
documentation from `),Qt=a(rn,"A",{href:!0});var hl=i(Qt);ss=r(hl,"PretrainedConfig"),hl.forEach(n),as=r(rn," for more information."),rn.forEach(n),is=p(xe),I(ye.$$.fragment,xe),xe.forEach(n),wo=p(t),re=a(t,"H2",{class:!0});var Qo=i(re);Ie=a(Qo,"A",{id:!0,class:!0,href:!0});var gl=i(Ie);bn=a(gl,"SPAN",{});var _l=i(bn);I(He.$$.fragment,_l),_l.forEach(n),gl.forEach(n),ls=p(Qo),yn=a(Qo,"SPAN",{});var vl=i(yn);cs=r(vl,"InstructBlipQFormerConfig"),vl.forEach(n),Qo.forEach(n),Mo=p(t),Z=a(t,"DIV",{class:!0});var Je=i(Z);I(Oe.$$.fragment,Je),ds=p(Je),Q=a(Je,"P",{});var H=i(Q);ps=r(H,"This is the configuration class to store the configuration of a "),qt=a(H,"A",{href:!0});var bl=i(qt);ms=r(bl,"InstructBlipQFormerModel"),bl.forEach(n),us=r(H,`. It is used to
instantiate a InstructBLIP Querying Transformer (Q-Former) model according to the specified arguments, defining the
model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
the InstructBLIP `),Ye=a(H,"A",{href:!0,rel:!0});var yl=i(Ye);fs=r(yl,"Salesforce/instruct-blip-flan-t5"),yl.forEach(n),hs=r(H,`
architecture. Configuration objects inherit from `),Gt=a(H,"A",{href:!0});var Il=i(Gt);gs=r(Il,"PretrainedConfig"),Il.forEach(n),_s=r(H,` and can be used to control the model outputs.
Read the documentation from `),Nt=a(H,"A",{href:!0});var Bl=i(Nt);vs=r(Bl,"PretrainedConfig"),Bl.forEach(n),bs=r(H," for more information."),H.forEach(n),ys=p(Je),se=a(Je,"P",{});var sn=i(se);Is=r(sn,"Note that "),St=a(sn,"A",{href:!0});var wl=i(St);Bs=r(wl,"InstructBlipQFormerModel"),wl.forEach(n),ws=r(sn," is very similar to "),Vt=a(sn,"A",{href:!0});var Ml=i(Vt);Ms=r(Ml,"BertLMHeadModel"),Ml.forEach(n),$s=r(sn," with interleaved cross-attention."),sn.forEach(n),Ts=p(Je),I(Be.$$.fragment,Je),Je.forEach(n),$o=p(t),ae=a(t,"H2",{class:!0});var qo=i(ae);we=a(qo,"A",{id:!0,class:!0,href:!0});var $l=i(we);In=a($l,"SPAN",{});var Tl=i(In);I(Ke.$$.fragment,Tl),Tl.forEach(n),$l.forEach(n),Cs=p(qo),Bn=a(qo,"SPAN",{});var Cl=i(Bn);ks=r(Cl,"InstructBlipProcessor"),Cl.forEach(n),qo.forEach(n),To=p(t),P=a(t,"DIV",{class:!0});var O=i(P);I(et.$$.fragment,O),Es=p(O),wn=a(O,"P",{});var kl=i(wn);js=r(kl,`Constructs an InstructBLIP processor which wraps a BLIP image processor and a LLaMa/T5 tokenizer into a single
processor.`),kl.forEach(n),Fs=p(O),J=a(O,"P",{});var S=i(J);At=a(S,"A",{href:!0});var El=i(At);Ps=r(El,"InstructBlipProcessor"),El.forEach(n),xs=r(S," offers all the functionalities of "),Rt=a(S,"A",{href:!0});var jl=i(Rt);Js=r(jl,"BlipImageProcessor"),jl.forEach(n),zs=r(S," and "),Lt=a(S,"A",{href:!0});var Fl=i(Lt);Us=r(Fl,"AutoTokenizer"),Fl.forEach(n),Ws=r(S,`. See the
docstring of `),Mn=a(S,"CODE",{});var Pl=i(Mn);Zs=r(Pl,"__call__()"),Pl.forEach(n),Qs=r(S," and "),Dt=a(S,"A",{href:!0});var xl=i(Dt);qs=r(xl,"decode()"),xl.forEach(n),Gs=r(S," for more information."),S.forEach(n),Ns=p(O),Me=a(O,"DIV",{class:!0});var Go=i(Me);I(tt.$$.fragment,Go),Ss=p(Go),nt=a(Go,"P",{});var No=i(nt);Vs=r(No,"This method forwards all its arguments to PreTrainedTokenizer\u2019s "),Xt=a(No,"A",{href:!0});var Jl=i(Xt);As=r(Jl,"batch_decode()"),Jl.forEach(n),Rs=r(No,`. Please
refer to the docstring of this method for more information.`),No.forEach(n),Go.forEach(n),Ls=p(O),$e=a(O,"DIV",{class:!0});var So=i($e);I(ot.$$.fragment,So),Ds=p(So),rt=a(So,"P",{});var Vo=i(rt);Xs=r(Vo,"This method forwards all its arguments to PreTrainedTokenizer\u2019s "),Ht=a(Vo,"A",{href:!0});var zl=i(Ht);Hs=r(zl,"decode()"),zl.forEach(n),Os=r(Vo,`. Please refer
to the docstring of this method for more information.`),Vo.forEach(n),So.forEach(n),O.forEach(n),Co=p(t),ie=a(t,"H2",{class:!0});var Ao=i(ie);Te=a(Ao,"A",{id:!0,class:!0,href:!0});var Ul=i(Te);$n=a(Ul,"SPAN",{});var Wl=i($n);I(st.$$.fragment,Wl),Wl.forEach(n),Ul.forEach(n),Ys=p(Ao),Tn=a(Ao,"SPAN",{});var Zl=i(Tn);Ks=r(Zl,"InstructBlipVisionModel"),Zl.forEach(n),Ao.forEach(n),ko=p(t),le=a(t,"DIV",{class:!0});var Ro=i(le);I(at.$$.fragment,Ro),ea=p(Ro),L=a(Ro,"DIV",{class:!0});var an=i(L);I(it.$$.fragment,an),ta=p(an),ce=a(an,"P",{});var ln=i(ce);na=r(ln,"The "),Ot=a(ln,"A",{href:!0});var Ql=i(Ot);oa=r(Ql,"InstructBlipVisionModel"),Ql.forEach(n),ra=r(ln," forward method, overrides the "),Cn=a(ln,"CODE",{});var ql=i(Cn);sa=r(ql,"__call__"),ql.forEach(n),aa=r(ln," special method."),ln.forEach(n),ia=p(an),I(Ce.$$.fragment,an),an.forEach(n),Ro.forEach(n),Eo=p(t),de=a(t,"H2",{class:!0});var Lo=i(de);ke=a(Lo,"A",{id:!0,class:!0,href:!0});var Gl=i(ke);kn=a(Gl,"SPAN",{});var Nl=i(kn);I(lt.$$.fragment,Nl),Nl.forEach(n),Gl.forEach(n),la=p(Lo),En=a(Lo,"SPAN",{});var Sl=i(En);ca=r(Sl,"InstructBlipQFormerModel"),Sl.forEach(n),Lo.forEach(n),jo=p(t),N=a(t,"DIV",{class:!0});var cn=i(N);I(ct.$$.fragment,cn),da=p(cn),jn=a(cn,"P",{});var Vl=i(jn);pa=r(Vl,`Querying Transformer (Q-Former), used in InstructBLIP. Slightly modified from BLIP-2 as it also takes the
instruction as input.`),Vl.forEach(n),ma=p(cn),D=a(cn,"DIV",{class:!0});var dn=i(D);I(dt.$$.fragment,dn),ua=p(dn),k=a(dn,"P",{});var j=i(k);fa=r(j,"encoder_hidden_states  ("),Fn=a(j,"CODE",{});var Al=i(Fn);ha=r(Al,"torch.FloatTensor"),Al.forEach(n),ga=r(j," of shape "),Pn=a(j,"CODE",{});var Rl=i(Pn);_a=r(Rl,"(batch_size, sequence_length, hidden_size)"),Rl.forEach(n),va=r(j,", "),xn=a(j,"EM",{});var Ll=i(xn);ba=r(Ll,"optional"),Ll.forEach(n),ya=r(j,`):
Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
the model is configured as a decoder.
encoder_attention_mask (`),Jn=a(j,"CODE",{});var Dl=i(Jn);Ia=r(Dl,"torch.FloatTensor"),Dl.forEach(n),Ba=r(j," of shape "),zn=a(j,"CODE",{});var Xl=i(zn);wa=r(Xl,"(batch_size, sequence_length)"),Xl.forEach(n),Ma=r(j,", "),Un=a(j,"EM",{});var Hl=i(Un);$a=r(Hl,"optional"),Hl.forEach(n),Ta=r(j,`):
Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
the cross-attention if the model is configured as a decoder. Mask values selected in `),Wn=a(j,"CODE",{});var Ol=i(Wn);Ca=r(Ol,"[0, 1]"),Ol.forEach(n),ka=r(j,":"),j.forEach(n),Ea=p(dn),pt=a(dn,"UL",{});var Do=i(pt);mt=a(Do,"LI",{});var Xo=i(mt);ja=r(Xo,"1 for tokens that are "),Zn=a(Xo,"STRONG",{});var Yl=i(Zn);Fa=r(Yl,"not masked"),Yl.forEach(n),Pa=r(Xo,","),Xo.forEach(n),xa=p(Do),b=a(Do,"LI",{});var T=i(b);Ja=r(T,"0 for tokens that are "),Qn=a(T,"STRONG",{});var Kl=i(Qn);za=r(Kl,"masked"),Kl.forEach(n),Ua=r(T,`.
past_key_values (`),qn=a(T,"CODE",{});var ec=i(qn);Wa=r(ec,"tuple(tuple(torch.FloatTensor))"),ec.forEach(n),Za=r(T," of length "),Gn=a(T,"CODE",{});var tc=i(Gn);Qa=r(tc,"config.n_layers"),tc.forEach(n),qa=r(T,` with each tuple having 4 tensors of:
shape `),Nn=a(T,"CODE",{});var nc=i(Nn);Ga=r(nc,"(batch_size, num_heads, sequence_length - 1, embed_size_per_head)"),nc.forEach(n),Na=r(T,`): Contains precomputed key and
value hidden states of the attention blocks. Can be used to speed up decoding. If `),Sn=a(T,"CODE",{});var oc=i(Sn);Sa=r(oc,"past_key_values"),oc.forEach(n),Va=r(T,` are
used, the user can optionally input only the last `),Vn=a(T,"CODE",{});var rc=i(Vn);Aa=r(rc,"decoder_input_ids"),rc.forEach(n),Ra=r(T,` (those that don\u2019t have their past key
value states given to this model) of shape `),An=a(T,"CODE",{});var sc=i(An);La=r(sc,"(batch_size, 1)"),sc.forEach(n),Da=r(T," instead of all "),Rn=a(T,"CODE",{});var ac=i(Rn);Xa=r(ac,"decoder_input_ids"),ac.forEach(n),Ha=r(T,` of shape
`),Ln=a(T,"CODE",{});var ic=i(Ln);Oa=r(ic,"(batch_size, sequence_length)"),ic.forEach(n),Ya=r(T,`.
use_cache (`),Dn=a(T,"CODE",{});var lc=i(Dn);Ka=r(lc,"bool"),lc.forEach(n),ei=r(T,", "),Xn=a(T,"EM",{});var cc=i(Xn);ti=r(cc,"optional"),cc.forEach(n),ni=r(T,`):
If set to `),Hn=a(T,"CODE",{});var dc=i(Hn);oi=r(dc,"True"),dc.forEach(n),ri=r(T,", "),On=a(T,"CODE",{});var pc=i(On);si=r(pc,"past_key_values"),pc.forEach(n),ai=r(T,` key value states are returned and can be used to speed up decoding (see
`),Yn=a(T,"CODE",{});var mc=i(Yn);ii=r(mc,"past_key_values"),mc.forEach(n),li=r(T,")."),T.forEach(n),Do.forEach(n),dn.forEach(n),cn.forEach(n),Fo=p(t),pe=a(t,"H2",{class:!0});var Ho=i(pe);Ee=a(Ho,"A",{id:!0,class:!0,href:!0});var uc=i(Ee);Kn=a(uc,"SPAN",{});var fc=i(Kn);I(ut.$$.fragment,fc),fc.forEach(n),uc.forEach(n),ci=p(Ho),eo=a(Ho,"SPAN",{});var hc=i(eo);di=r(hc,"InstructBlipForConditionalGeneration"),hc.forEach(n),Ho.forEach(n),Po=p(t),E=a(t,"DIV",{class:!0});var z=i(E);I(ft.$$.fragment,z),pi=p(z),to=a(z,"P",{});var gc=i(to);mi=r(gc,`InstructBLIP Model for generating text given an image and an optional text prompt. The model consists of a vision
encoder, Querying Transformer (Q-Former) and a language model.`),gc.forEach(n),ui=p(z),ht=a(z,"P",{});var Oo=i(ht);fi=r(Oo,"One can optionally pass "),no=a(Oo,"CODE",{});var _c=i(no);hi=r(_c,"input_ids"),_c.forEach(n),gi=r(Oo,` to the model, which serve as a text prompt, to make the language model continue
the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.`),Oo.forEach(n),_i=p(z),gt=a(z,"P",{});var Yo=i(gt);vi=r(Yo,"This model inherits from "),Yt=a(Yo,"A",{href:!0});var vc=i(Yt);bi=r(vc,"PreTrainedModel"),vc.forEach(n),yi=r(Yo,`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),Yo.forEach(n),Ii=p(z),_t=a(z,"P",{});var Ko=i(_t);Bi=r(Ko,"This model is also a PyTorch "),vt=a(Ko,"A",{href:!0,rel:!0});var bc=i(vt);wi=r(bc,"torch.nn.Module"),bc.forEach(n),Mi=r(Ko,` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),Ko.forEach(n),$i=p(z),q=a(z,"DIV",{class:!0});var ze=i(q);I(bt.$$.fragment,ze),Ti=p(ze),me=a(ze,"P",{});var pn=i(me);Ci=r(pn,"The "),Kt=a(pn,"A",{href:!0});var yc=i(Kt);ki=r(yc,"InstructBlipForConditionalGeneration"),yc.forEach(n),Ei=r(pn," forward method, overrides the "),oo=a(pn,"CODE",{});var Ic=i(oo);ji=r(Ic,"__call__"),Ic.forEach(n),Fi=r(pn," special method."),pn.forEach(n),Pi=p(ze),I(je.$$.fragment,ze),xi=p(ze),I(Fe.$$.fragment,ze),ze.forEach(n),Ji=p(z),Pe=a(z,"DIV",{class:!0});var er=i(Pe);I(yt.$$.fragment,er),zi=p(er),It=a(er,"P",{});var tr=i(It);Ui=r(tr,"Overrides "),ro=a(tr,"CODE",{});var Bc=i(ro);Wi=r(Bc,"generate"),Bc.forEach(n),Zi=r(tr," function to be able to use the model as a conditional generator."),tr.forEach(n),er.forEach(n),z.forEach(n),this.h()},h(){l(f,"name","hf:doc:metadata"),l(f,"content",JSON.stringify(Wc)),l(h,"id","instructblip"),l(h,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(h,"href","#instructblip"),l(_,"class","relative group"),l(fe,"id","overview"),l(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(fe,"href","#overview"),l(Y,"class","relative group"),l(We,"href","https://arxiv.org/abs/2305.06500"),l(We,"rel","nofollow"),l($t,"href","blip2"),l(jt,"href","blip2"),kc(he.src,qi="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/instructblip_architecture.jpg")||l(he,"src",qi),l(he,"alt","drawing"),l(he,"width","600"),l(Ft,"href","https://arxiv.org/abs/2305.06500"),l(qe,"href","https://huggingface.co/nielsr"),l(qe,"rel","nofollow"),l(Ge,"href","https://github.com/salesforce/LAVIS/tree/main/projects/instructblip"),l(Ge,"rel","nofollow"),l(ge,"id","transformers.InstructBlipConfig"),l(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(ge,"href","#transformers.InstructBlipConfig"),l(K,"class","relative group"),l(Pt,"href","/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipConfig"),l(xt,"href","/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipForConditionalGeneration"),l(Ve,"href","https://huggingface.co/Salesforce/instruct-blip-flan-t5"),l(Ve,"rel","nofollow"),l(Jt,"href","/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(zt,"href","/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(Ut,"href","/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipConfig"),l(ve,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(be,"id","transformers.InstructBlipVisionConfig"),l(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(be,"href","#transformers.InstructBlipVisionConfig"),l(te,"class","relative group"),l(Wt,"href","/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipVisionModel"),l(Xe,"href","https://huggingface.co/Salesforce/instruct-blip-flan-t5"),l(Xe,"rel","nofollow"),l(Zt,"href","/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(Qt,"href","/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(Ie,"id","transformers.InstructBlipQFormerConfig"),l(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(Ie,"href","#transformers.InstructBlipQFormerConfig"),l(re,"class","relative group"),l(qt,"href","/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipQFormerModel"),l(Ye,"href","https://huggingface.co/Salesforce/instruct-blip-flan-t5"),l(Ye,"rel","nofollow"),l(Gt,"href","/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(Nt,"href","/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig"),l(St,"href","/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipQFormerModel"),l(Vt,"href","/docs/transformers/v4.32.0/en/model_doc/bert#transformers.BertLMHeadModel"),l(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(we,"id","transformers.InstructBlipProcessor"),l(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(we,"href","#transformers.InstructBlipProcessor"),l(ae,"class","relative group"),l(At,"href","/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipProcessor"),l(Rt,"href","/docs/transformers/v4.32.0/en/model_doc/blip#transformers.BlipImageProcessor"),l(Lt,"href","/docs/transformers/v4.32.0/en/model_doc/auto#transformers.AutoTokenizer"),l(Dt,"href","/docs/transformers/v4.32.0/en/model_doc/blip#transformers.BlipProcessor.decode"),l(Xt,"href","/docs/transformers/v4.32.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.batch_decode"),l(Me,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(Ht,"href","/docs/transformers/v4.32.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.decode"),l($e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(Te,"id","transformers.InstructBlipVisionModel"),l(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(Te,"href","#transformers.InstructBlipVisionModel"),l(ie,"class","relative group"),l(Ot,"href","/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipVisionModel"),l(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(ke,"id","transformers.InstructBlipQFormerModel"),l(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(ke,"href","#transformers.InstructBlipQFormerModel"),l(de,"class","relative group"),l(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(Ee,"id","transformers.InstructBlipForConditionalGeneration"),l(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(Ee,"href","#transformers.InstructBlipForConditionalGeneration"),l(pe,"class","relative group"),l(Yt,"href","/docs/transformers/v4.32.0/en/main_classes/model#transformers.PreTrainedModel"),l(vt,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),l(vt,"rel","nofollow"),l(Kt,"href","/docs/transformers/v4.32.0/en/model_doc/instructblip#transformers.InstructBlipForConditionalGeneration"),l(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(Pe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),l(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,m){e(document.head,f),u(t,C,m),u(t,_,m),e(_,h),e(h,v),B(c,v,null),e(_,g),e(_,G),e(G,sr),u(t,co,m),u(t,Y,m),e(Y,fe),e(fe,mn),B(Ue,mn,null),e(Y,ar),e(Y,un),e(un,ir),u(t,po,m),u(t,V,m),e(V,lr),e(V,We),e(We,cr),e(V,dr),e(V,$t),e($t,pr),e(V,mr),u(t,mo,m),u(t,Tt,m),e(Tt,ur),u(t,uo,m),u(t,Ct,m),e(Ct,fn),e(fn,fr),u(t,fo,m),u(t,kt,m),e(kt,hr),u(t,ho,m),u(t,Et,m),e(Et,Ze),e(Ze,gr),e(Ze,jt),e(jt,_r),e(Ze,vr),u(t,go,m),u(t,he,m),u(t,_o,m),u(t,Qe,m),e(Qe,br),e(Qe,Ft),e(Ft,yr),u(t,vo,m),u(t,A,m),e(A,Ir),e(A,qe),e(qe,Br),e(A,wr),e(A,Ge),e(Ge,Mr),e(A,$r),u(t,bo,m),u(t,K,m),e(K,ge),e(ge,hn),B(Ne,hn,null),e(K,Tr),e(K,gn),e(gn,Cr),u(t,yo,m),u(t,F,m),B(Se,F,null),e(F,kr),e(F,R),e(R,Pt),e(Pt,Er),e(R,jr),e(R,xt),e(xt,Fr),e(R,Pr),e(R,Ve),e(Ve,xr),e(R,Jr),e(F,zr),e(F,ee),e(ee,Ur),e(ee,Jt),e(Jt,Wr),e(ee,Zr),e(ee,zt),e(zt,Qr),e(ee,qr),e(F,Gr),B(_e,F,null),e(F,Nr),e(F,ve),B(Ae,ve,null),e(ve,Sr),e(ve,Re),e(Re,Vr),e(Re,Ut),e(Ut,Ar),e(Re,Rr),u(t,Io,m),u(t,te,m),e(te,be),e(be,_n),B(Le,_n,null),e(te,Lr),e(te,vn),e(vn,Dr),u(t,Bo,m),u(t,W,m),B(De,W,null),e(W,Xr),e(W,ne),e(ne,Hr),e(ne,Wt),e(Wt,Or),e(ne,Yr),e(ne,Xe),e(Xe,Kr),e(ne,es),e(W,ts),e(W,oe),e(oe,ns),e(oe,Zt),e(Zt,os),e(oe,rs),e(oe,Qt),e(Qt,ss),e(oe,as),e(W,is),B(ye,W,null),u(t,wo,m),u(t,re,m),e(re,Ie),e(Ie,bn),B(He,bn,null),e(re,ls),e(re,yn),e(yn,cs),u(t,Mo,m),u(t,Z,m),B(Oe,Z,null),e(Z,ds),e(Z,Q),e(Q,ps),e(Q,qt),e(qt,ms),e(Q,us),e(Q,Ye),e(Ye,fs),e(Q,hs),e(Q,Gt),e(Gt,gs),e(Q,_s),e(Q,Nt),e(Nt,vs),e(Q,bs),e(Z,ys),e(Z,se),e(se,Is),e(se,St),e(St,Bs),e(se,ws),e(se,Vt),e(Vt,Ms),e(se,$s),e(Z,Ts),B(Be,Z,null),u(t,$o,m),u(t,ae,m),e(ae,we),e(we,In),B(Ke,In,null),e(ae,Cs),e(ae,Bn),e(Bn,ks),u(t,To,m),u(t,P,m),B(et,P,null),e(P,Es),e(P,wn),e(wn,js),e(P,Fs),e(P,J),e(J,At),e(At,Ps),e(J,xs),e(J,Rt),e(Rt,Js),e(J,zs),e(J,Lt),e(Lt,Us),e(J,Ws),e(J,Mn),e(Mn,Zs),e(J,Qs),e(J,Dt),e(Dt,qs),e(J,Gs),e(P,Ns),e(P,Me),B(tt,Me,null),e(Me,Ss),e(Me,nt),e(nt,Vs),e(nt,Xt),e(Xt,As),e(nt,Rs),e(P,Ls),e(P,$e),B(ot,$e,null),e($e,Ds),e($e,rt),e(rt,Xs),e(rt,Ht),e(Ht,Hs),e(rt,Os),u(t,Co,m),u(t,ie,m),e(ie,Te),e(Te,$n),B(st,$n,null),e(ie,Ys),e(ie,Tn),e(Tn,Ks),u(t,ko,m),u(t,le,m),B(at,le,null),e(le,ea),e(le,L),B(it,L,null),e(L,ta),e(L,ce),e(ce,na),e(ce,Ot),e(Ot,oa),e(ce,ra),e(ce,Cn),e(Cn,sa),e(ce,aa),e(L,ia),B(Ce,L,null),u(t,Eo,m),u(t,de,m),e(de,ke),e(ke,kn),B(lt,kn,null),e(de,la),e(de,En),e(En,ca),u(t,jo,m),u(t,N,m),B(ct,N,null),e(N,da),e(N,jn),e(jn,pa),e(N,ma),e(N,D),B(dt,D,null),e(D,ua),e(D,k),e(k,fa),e(k,Fn),e(Fn,ha),e(k,ga),e(k,Pn),e(Pn,_a),e(k,va),e(k,xn),e(xn,ba),e(k,ya),e(k,Jn),e(Jn,Ia),e(k,Ba),e(k,zn),e(zn,wa),e(k,Ma),e(k,Un),e(Un,$a),e(k,Ta),e(k,Wn),e(Wn,Ca),e(k,ka),e(D,Ea),e(D,pt),e(pt,mt),e(mt,ja),e(mt,Zn),e(Zn,Fa),e(mt,Pa),e(pt,xa),e(pt,b),e(b,Ja),e(b,Qn),e(Qn,za),e(b,Ua),e(b,qn),e(qn,Wa),e(b,Za),e(b,Gn),e(Gn,Qa),e(b,qa),e(b,Nn),e(Nn,Ga),e(b,Na),e(b,Sn),e(Sn,Sa),e(b,Va),e(b,Vn),e(Vn,Aa),e(b,Ra),e(b,An),e(An,La),e(b,Da),e(b,Rn),e(Rn,Xa),e(b,Ha),e(b,Ln),e(Ln,Oa),e(b,Ya),e(b,Dn),e(Dn,Ka),e(b,ei),e(b,Xn),e(Xn,ti),e(b,ni),e(b,Hn),e(Hn,oi),e(b,ri),e(b,On),e(On,si),e(b,ai),e(b,Yn),e(Yn,ii),e(b,li),u(t,Fo,m),u(t,pe,m),e(pe,Ee),e(Ee,Kn),B(ut,Kn,null),e(pe,ci),e(pe,eo),e(eo,di),u(t,Po,m),u(t,E,m),B(ft,E,null),e(E,pi),e(E,to),e(to,mi),e(E,ui),e(E,ht),e(ht,fi),e(ht,no),e(no,hi),e(ht,gi),e(E,_i),e(E,gt),e(gt,vi),e(gt,Yt),e(Yt,bi),e(gt,yi),e(E,Ii),e(E,_t),e(_t,Bi),e(_t,vt),e(vt,wi),e(_t,Mi),e(E,$i),e(E,q),B(bt,q,null),e(q,Ti),e(q,me),e(me,Ci),e(me,Kt),e(Kt,ki),e(me,Ei),e(me,oo),e(oo,ji),e(me,Fi),e(q,Pi),B(je,q,null),e(q,xi),B(Fe,q,null),e(E,Ji),e(E,Pe),B(yt,Pe,null),e(Pe,zi),e(Pe,It),e(It,Ui),e(It,ro),e(ro,Wi),e(It,Zi),xo=!0},p(t,[m]){const Bt={};m&2&&(Bt.$$scope={dirty:m,ctx:t}),_e.$set(Bt);const so={};m&2&&(so.$$scope={dirty:m,ctx:t}),ye.$set(so);const ao={};m&2&&(ao.$$scope={dirty:m,ctx:t}),Be.$set(ao);const io={};m&2&&(io.$$scope={dirty:m,ctx:t}),Ce.$set(io);const wt={};m&2&&(wt.$$scope={dirty:m,ctx:t}),je.$set(wt);const lo={};m&2&&(lo.$$scope={dirty:m,ctx:t}),Fe.$set(lo)},i(t){xo||(w(c.$$.fragment,t),w(Ue.$$.fragment,t),w(Ne.$$.fragment,t),w(Se.$$.fragment,t),w(_e.$$.fragment,t),w(Ae.$$.fragment,t),w(Le.$$.fragment,t),w(De.$$.fragment,t),w(ye.$$.fragment,t),w(He.$$.fragment,t),w(Oe.$$.fragment,t),w(Be.$$.fragment,t),w(Ke.$$.fragment,t),w(et.$$.fragment,t),w(tt.$$.fragment,t),w(ot.$$.fragment,t),w(st.$$.fragment,t),w(at.$$.fragment,t),w(it.$$.fragment,t),w(Ce.$$.fragment,t),w(lt.$$.fragment,t),w(ct.$$.fragment,t),w(dt.$$.fragment,t),w(ut.$$.fragment,t),w(ft.$$.fragment,t),w(bt.$$.fragment,t),w(je.$$.fragment,t),w(Fe.$$.fragment,t),w(yt.$$.fragment,t),xo=!0)},o(t){M(c.$$.fragment,t),M(Ue.$$.fragment,t),M(Ne.$$.fragment,t),M(Se.$$.fragment,t),M(_e.$$.fragment,t),M(Ae.$$.fragment,t),M(Le.$$.fragment,t),M(De.$$.fragment,t),M(ye.$$.fragment,t),M(He.$$.fragment,t),M(Oe.$$.fragment,t),M(Be.$$.fragment,t),M(Ke.$$.fragment,t),M(et.$$.fragment,t),M(tt.$$.fragment,t),M(ot.$$.fragment,t),M(st.$$.fragment,t),M(at.$$.fragment,t),M(it.$$.fragment,t),M(Ce.$$.fragment,t),M(lt.$$.fragment,t),M(ct.$$.fragment,t),M(dt.$$.fragment,t),M(ut.$$.fragment,t),M(ft.$$.fragment,t),M(bt.$$.fragment,t),M(je.$$.fragment,t),M(Fe.$$.fragment,t),M(yt.$$.fragment,t),xo=!1},d(t){n(f),t&&n(C),t&&n(_),$(c),t&&n(co),t&&n(Y),$(Ue),t&&n(po),t&&n(V),t&&n(mo),t&&n(Tt),t&&n(uo),t&&n(Ct),t&&n(fo),t&&n(kt),t&&n(ho),t&&n(Et),t&&n(go),t&&n(he),t&&n(_o),t&&n(Qe),t&&n(vo),t&&n(A),t&&n(bo),t&&n(K),$(Ne),t&&n(yo),t&&n(F),$(Se),$(_e),$(Ae),t&&n(Io),t&&n(te),$(Le),t&&n(Bo),t&&n(W),$(De),$(ye),t&&n(wo),t&&n(re),$(He),t&&n(Mo),t&&n(Z),$(Oe),$(Be),t&&n($o),t&&n(ae),$(Ke),t&&n(To),t&&n(P),$(et),$(tt),$(ot),t&&n(Co),t&&n(ie),$(st),t&&n(ko),t&&n(le),$(at),$(it),$(Ce),t&&n(Eo),t&&n(de),$(lt),t&&n(jo),t&&n(N),$(ct),$(dt),t&&n(Fo),t&&n(pe),$(ut),t&&n(Po),t&&n(E),$(ft),$(bt),$(je),$(Fe),$(yt)}}}const Wc={local:"instructblip",sections:[{local:"overview",title:"Overview"},{local:"transformers.InstructBlipConfig",title:"InstructBlipConfig"},{local:"transformers.InstructBlipVisionConfig",title:"InstructBlipVisionConfig"},{local:"transformers.InstructBlipQFormerConfig",title:"InstructBlipQFormerConfig"},{local:"transformers.InstructBlipProcessor",title:"InstructBlipProcessor"},{local:"transformers.InstructBlipVisionModel",title:"InstructBlipVisionModel"},{local:"transformers.InstructBlipQFormerModel",title:"InstructBlipQFormerModel"},{local:"transformers.InstructBlipForConditionalGeneration",title:"InstructBlipForConditionalGeneration"}],title:"InstructBLIP"};function Zc(x){return Ec(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ac extends Mc{constructor(f){super();$c(this,f,Zc,Uc,Tc,{})}}export{Ac as default,Wc as metadata};
