import{s as De,o as Oe,n as Fe}from"../chunks/scheduler.9bc65507.js";import{S as Ke,i as et,g as l,s as i,r as v,A as tt,h as d,f as n,c as r,j as ne,u as _,x as h,k as ae,y as m,a,v as b,d as M,t as y,w as T}from"../chunks/index.707bf1b6.js";import{T as nt}from"../chunks/Tip.c2ecdbf4.js";import{D as Ge}from"../chunks/Docstring.17db21ae.js";import{C as Te}from"../chunks/CodeBlock.54a9f38d.js";import{E as Ae}from"../chunks/ExampleCodeBlock.4f515aa9.js";import{H as ye}from"../chunks/Heading.342b1fa6.js";function at(k){let s,w="Example:",p,c,u;return c=new Te({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFZpcExsYXZhRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTJDJTIwVmlwTGxhdmFDb25maWclMkMlMjBDTElQVmlzaW9uQ29uZmlnJTJDJTIwTGxhbWFDb25maWclMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwQ0xJUC12aXNpb24lMjBjb25maWclMEF2aXNpb25fY29uZmlnJTIwJTNEJTIwQ0xJUFZpc2lvbkNvbmZpZygpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMExsYW1hJTIwY29uZmlnJTBBdGV4dF9jb25maWclMjAlM0QlMjBMbGFtYUNvbmZpZygpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMFZpcExsYXZhJTIwdmlwbGxhdmEtN2IlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwVmlwTGxhdmFDb25maWcodmlzaW9uX2NvbmZpZyUyQyUyMHRleHRfY29uZmlnKSUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBtb2RlbCUyMGZyb20lMjB0aGUlMjB2aXBsbGF2YS03YiUyMHN0eWxlJTIwY29uZmlndXJhdGlvbiUwQW1vZGVsJTIwJTNEJTIwVmlwTGxhdmFGb3JDb25kaXRpb25hbEdlbmVyYXRpb24oY29uZmlndXJhdGlvbiklMEElMEElMjMlMjBBY2Nlc3NpbmclMjB0aGUlMjBtb2RlbCUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwbW9kZWwuY29uZmln",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> VipLlavaForConditionalGeneration, VipLlavaConfig, CLIPVisionConfig, LlamaConfig

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a CLIP-vision config</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>vision_config = CLIPVisionConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a Llama config</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>text_config = LlamaConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a VipLlava vipllava-7b style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = VipLlavaConfig(vision_config, text_config)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model from the vipllava-7b style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = VipLlavaForConditionalGeneration(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,wrap:!1}}),{c(){s=l("p"),s.textContent=w,p=i(),v(c.$$.fragment)},l(o){s=d(o,"P",{"data-svelte-h":!0}),h(s)!=="svelte-11lpom8"&&(s.textContent=w),p=r(o),_(c.$$.fragment,o)},m(o,f){a(o,s,f),a(o,p,f),b(c,o,f),u=!0},p:Fe,i(o){u||(M(c.$$.fragment,o),u=!0)},o(o){y(c.$$.fragment,o),u=!1},d(o){o&&(n(s),n(p)),T(c,o)}}}function ot(k){let s,w=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){s=l("p"),s.innerHTML=w},l(p){s=d(p,"P",{"data-svelte-h":!0}),h(s)!=="svelte-fincs2"&&(s.innerHTML=w)},m(p,c){a(p,s,c)},p:Fe,d(p){p&&n(s)}}}function st(k){let s,w="Example:",p,c,u;return c=new Te({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEFpbXBvcnQlMjByZXF1ZXN0cyUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvUHJvY2Vzc29yJTJDJTIwVmlwTGxhdmFGb3JDb25kaXRpb25hbEdlbmVyYXRpb24lMEElMEFtb2RlbCUyMCUzRCUyMFZpcExsYXZhRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uLmZyb21fcHJldHJhaW5lZCglMjJsbGF2YS1oZiUyRnZpcC1sbGF2YS03Yi1oZiUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2KSUwQXByb2Nlc3NvciUyMCUzRCUyMEF1dG9Qcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKCUyMmxsYXZhLWhmJTJGdmlwLWxsYXZhLTdiLWhmJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMkElMjBjaGF0JTIwYmV0d2VlbiUyMGElMjBjdXJpb3VzJTIwaHVtYW4lMjBhbmQlMjBhbiUyMGFydGlmaWNpYWwlMjBpbnRlbGxpZ2VuY2UlMjBhc3Npc3RhbnQuJTIwVGhlJTIwYXNzaXN0YW50JTIwZ2l2ZXMlMjBoZWxwZnVsJTJDJTIwZGV0YWlsZWQlMkMlMjBhbmQlMjBwb2xpdGUlMjBhbnN3ZXJzJTIwdG8lMjB0aGUlMjBodW1hbidzJTIwcXVlc3Rpb25zLiUyMyUyMyUyM0h1bWFuJTNBJTIwJTNDaW1hZ2UlM0UlNUNuJTdCJTdEJTIzJTIzJTIzQXNzaXN0YW50JTNBJTIyJTBBcXVlc3Rpb24lMjAlM0QlMjAlMjJDYW4lMjB5b3UlMjBwbGVhc2UlMjBkZXNjcmliZSUyMHRoaXMlMjBpbWFnZSUzRiUyMiUwQXByb21wdCUyMCUzRCUyMHByb21wdC5mb3JtYXQocXVlc3Rpb24pJTBBdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGaHVnZ2luZ2ZhY2UlMkZkb2N1bWVudGF0aW9uLWltYWdlcyUyRnJlc29sdmUlMkZtYWluJTJGZGlmZnVzZXJzJTJGY29tcGVsLW5lZy5wbmclMjIlMEFpbWFnZSUyMCUzRCUyMEltYWdlLm9wZW4ocmVxdWVzdHMuZ2V0KHVybCUyQyUyMHN0cmVhbSUzRFRydWUpLnJhdyklMEElMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IodGV4dCUzRHRleHQlMkMlMjBpbWFnZXMlM0RpbWFnZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKDAlMkMlMjB0b3JjaC5mbG9hdDE2KSUwQSUwQSUyMyUyMEdlbmVyYXRlJTBBZ2VuZXJhdGVfaWRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDIwKSUwQXByb2Nlc3Nvci5kZWNvZGUoZ2VuZXJhdGVfaWRzJTVCMCU1RCU1QmxlbihpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQlNUIwJTVEKSUzQSU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, VipLlavaForConditionalGeneration

<span class="hljs-meta">&gt;&gt;&gt; </span>model = VipLlavaForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;llava-hf/vip-llava-7b-hf&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, torch_dtype=torch.float16)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;llava-hf/vip-llava-7b-hf&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human&#x27;s questions.###Human: &lt;image&gt;\\n{}###Assistant:&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>question = <span class="hljs-string">&quot;Can you please describe this image?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = prompt.<span class="hljs-built_in">format</span>(question)
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/compel-neg.png&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor(text=text, images=image, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-number">0</span>, torch.float16)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Generate</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generate_ids = model.generate(**inputs, max_new_tokens=<span class="hljs-number">20</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor.decode(generate_ids[<span class="hljs-number">0</span>][<span class="hljs-built_in">len</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>][<span class="hljs-number">0</span>]):], skip_special_tokens=<span class="hljs-literal">True</span>)
The image features a brown <span class="hljs-keyword">and</span> white cat sitting on a green surface, <span class="hljs-keyword">with</span> a red ball <span class="hljs-keyword">in</span> its`,wrap:!1}}),{c(){s=l("p"),s.textContent=w,p=i(),v(c.$$.fragment)},l(o){s=d(o,"P",{"data-svelte-h":!0}),h(s)!=="svelte-11lpom8"&&(s.textContent=w),p=r(o),_(c.$$.fragment,o)},m(o,f){a(o,s,f),a(o,p,f),b(c,o,f),u=!0},p:Fe,i(o){u||(M(c.$$.fragment,o),u=!0)},o(o){y(c.$$.fragment,o),u=!1},d(o){o&&(n(s),n(p)),T(c,o)}}}function it(k){let s,w,p,c,u,o,f,oe,G,Ze='The VipLlava model was proposed in <a href="https://arxiv.org/abs/2312.00784" rel="nofollow">Making Large Multimodal Models Understand Arbitrary Visual Prompts</a> by Mu Cai, Haotian Liu, Siva Karthik Mustikovela, Gregory P. Meyer, Yuning Chai, Dennis Park, Yong Jae Lee.',se,F,ze="VipLlava enhances the training protocol of Llava by marking images and interact with the model using natural cues like a “red bounding box” or “pointed arrow” during training.",ie,Z,We="The abstract from the paper is the following:",re,z,Ie="<em>While existing large vision-language multimodal models focus on whole image understanding, there is a prominent gap in achieving region-specific comprehension. Current approaches that use textual coordinates or spatial encodings often fail to provide a user-friendly interface for visual prompting. To address this challenge, we introduce a novel multimodal model capable of decoding arbitrary visual prompts. This allows users to intuitively mark images and interact with the model using natural cues like a “red bounding box” or “pointed arrow”. Our simple design directly overlays visual markers onto the RGB image, eliminating the need for complex region encodings, yet achieves state-of-the-art performance on region-understanding tasks like Visual7W, PointQA, and Visual Commonsense Reasoning benchmark. Furthermore, we present ViP-Bench, a comprehensive benchmark to assess the capability of models in understanding visual prompts across multiple dimensions, enabling future research in this domain. Code, data, and model are publicly available.</em>",le,W,Be="Tips:",de,I,Ne="<li><p>The architecture is similar than llava architecture except that the multi-modal projector takes a set of concatenated vision hidden states and has an additional layernorm layer on that module.</p></li> <li><p>We advise users to use <code>padding_side=&quot;left&quot;</code> when computing batched generation as it leads to more accurate results. Simply make sure to call <code>processor.tokenizer.padding_side = &quot;left&quot;</code> before generating.</p></li> <li><p>Note the model has not been explicitly trained to process multiple images in the same prompt, although this is technically possible, you may experience inaccurate results.</p></li> <li><p>For better results, we recommend users to prompt the model with the correct prompt format:</p></li>",ce,B,pe,N,He="For multiple turns conversation:",me,H,he,X,Xe='The original code can be found <a href="https://github.com/mu-cai/ViP-LLaVA" rel="nofollow">here</a>.',ue,R,Re='This model was contributed by <a href="https://huggingface.co/ybelkada" rel="nofollow">Younes Belkada</a>',fe,Y,ge,g,P,we,q,Ye=`This is the configuration class to store the configuration of a <a href="/docs/transformers/v4.37.0/en/model_doc/vipllava#transformers.VipLlavaForConditionalGeneration">VipLlavaForConditionalGeneration</a>. It is used to instantiate an
VipLlava model according to the specified arguments, defining the model architecture. Instantiating a configuration
with the defaults will yield a similar configuration to that of the VipLlava-9B.`,je,A,Pe='e.g. <a href="https://huggingface.co/ybelkada/vip-llava-7b-hf" rel="nofollow">ybelkada/vip-llava-7b-hf</a>',Ce,D,Ee=`Configuration objects inherit from <a href="/docs/transformers/v4.37.0/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/v4.37.0/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,Ue,V,ve,E,_e,j,S,Je,O,Se=`The VIPLLAVA model which consists of a vision backbone and a language model.
This model inherits from <a href="/docs/transformers/v4.37.0/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`,xe,K,Qe=`This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`,ke,U,Q,Ve,ee,qe='The <a href="/docs/transformers/v4.37.0/en/model_doc/vipllava#transformers.VipLlavaForConditionalGeneration">VipLlavaForConditionalGeneration</a> forward method, overrides the <code>__call__</code> special method.',$e,$,Le,L,be,te,Me;return u=new ye({props:{title:"VipLlava",local:"vipllava",headingTag:"h1"}}),f=new ye({props:{title:"Overview",local:"overview",headingTag:"h2"}}),B=new Te({props:{code:"QSUyMGNoYXQlMjBiZXR3ZWVuJTIwYSUyMGN1cmlvdXMlMjBodW1hbiUyMGFuZCUyMGFuJTIwYXJ0aWZpY2lhbCUyMGludGVsbGlnZW5jZSUyMGFzc2lzdGFudC4lMjBUaGUlMjBhc3Npc3RhbnQlMjBnaXZlcyUyMGhlbHBmdWwlMkMlMjBkZXRhaWxlZCUyQyUyMGFuZCUyMHBvbGl0ZSUyMGFuc3dlcnMlMjB0byUyMHRoZSUyMGh1bWFuJ3MlMjBxdWVzdGlvbnMuJTIzJTIzJTIzSHVtYW4lM0ElMjAlM0NpbWFnZSUzRSU1Q24lM0Nwcm9tcHQlM0UlMjMlMjMlMjNBc3Npc3RhbnQlM0E=",highlighted:'A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human<span class="hljs-string">&#x27;s questions.###Human: &lt;image&gt;\\n&lt;prompt&gt;###Assistant:</span>',wrap:!1}}),H=new Te({props:{code:"QSUyMGNoYXQlMjBiZXR3ZWVuJTIwYSUyMGN1cmlvdXMlMjBodW1hbiUyMGFuZCUyMGFuJTIwYXJ0aWZpY2lhbCUyMGludGVsbGlnZW5jZSUyMGFzc2lzdGFudC4lMjBUaGUlMjBhc3Npc3RhbnQlMjBnaXZlcyUyMGhlbHBmdWwlMkMlMjBkZXRhaWxlZCUyQyUyMGFuZCUyMHBvbGl0ZSUyMGFuc3dlcnMlMjB0byUyMHRoZSUyMGh1bWFuJ3MlMjBxdWVzdGlvbnMuJTIzJTIzJTIzSHVtYW4lM0ElMjAlM0NpbWFnZSUzRSU1Q24lM0Nwcm9tcHQxJTNFJTIzJTIzJTIzQXNzaXN0YW50JTNBJTIwJTNDYW5zd2VyMSUzRSUyMyUyMyUyM0h1bWFuJTNBJTIwJTNDcHJvbXB0MiUzRSUyMyUyMyUyM0Fzc2lzdGFudCUzQQ==",highlighted:'A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human<span class="hljs-string">&#x27;s questions.###Human: &lt;image&gt;\\n&lt;prompt1&gt;###Assistant: &lt;answer1&gt;###Human: &lt;prompt2&gt;###Assistant:</span>',wrap:!1}}),Y=new ye({props:{title:"VipLlavaConfig",local:"transformers.VipLlavaConfig",headingTag:"h2"}}),P=new Ge({props:{name:"class transformers.VipLlavaConfig",anchor:"transformers.VipLlavaConfig",parameters:[{name:"vision_config",val:" = None"},{name:"text_config",val:" = None"},{name:"ignore_index",val:" = -100"},{name:"image_token_index",val:" = 32000"},{name:"projector_hidden_act",val:" = 'gelu'"},{name:"projector_layernorm_eps",val:" = 1e-05"},{name:"vision_feature_layers",val:" = [-2, -5, -8, -11, 6]"},{name:"vocab_size",val:" = 32000"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.VipLlavaConfig.vision_config",description:`<strong>vision_config</strong> (<code>VipLlavaVisionConfig</code>,  <em>optional</em>) &#x2014;
Custom vision config or dict`,name:"vision_config"},{anchor:"transformers.VipLlavaConfig.text_config",description:`<strong>text_config</strong> (<code>Union[AutoConfig, dict]</code>, <em>optional</em>) &#x2014;
The config object of the text backbone. Can be any of <code>LlamaConfig</code> or <code>MistralConfig</code>.`,name:"text_config"},{anchor:"transformers.VipLlavaConfig.ignore_index",description:`<strong>ignore_index</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The ignore index for the loss function.`,name:"ignore_index"},{anchor:"transformers.VipLlavaConfig.image_token_index",description:`<strong>image_token_index</strong> (<code>int</code>, <em>optional</em>, defaults to 32000) &#x2014;
The image token index to encode the image prompt.`,name:"image_token_index"},{anchor:"transformers.VipLlavaConfig.projector_hidden_act",description:`<strong>projector_hidden_act</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The activation function used by the multimodal projector.`,name:"projector_hidden_act"},{anchor:"transformers.VipLlavaConfig.projector_layernorm_eps",description:`<strong>projector_layernorm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-05) &#x2014;
The layer norm epsilon of the projector layernorm`,name:"projector_layernorm_eps"},{anchor:"transformers.VipLlavaConfig.vision_feature_layers",description:`<strong>vision_feature_layers</strong> (<code>List[int]</code>, <em>optional</em>, defaults to <code>[-2, -5, -8, -11, 6]</code>) &#x2014;
The list of layers to select the vision features from.`,name:"vision_feature_layers"},{anchor:"transformers.VipLlavaConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 32000) &#x2014;
Vocabulary size of the VipLlava model. Defines the number of different tokens that can be represented by the
<code>inputs_ids</code> passed when calling <a href="/docs/transformers/v4.37.0/en/model_doc/vipllava#transformers.VipLlavaForConditionalGeneration">~VipLlavaForConditionalGeneration</a>`,name:"vocab_size"}],source:"https://github.com/huggingface/transformers/blob/v4.37.0/src/transformers/models/vipllava/configuration_vipllava.py#L28"}}),V=new Ae({props:{anchor:"transformers.VipLlavaConfig.example",$$slots:{default:[at]},$$scope:{ctx:k}}}),E=new ye({props:{title:"VipLlavaForConditionalGeneration",local:"transformers.VipLlavaForConditionalGeneration",headingTag:"h2"}}),S=new Ge({props:{name:"class transformers.VipLlavaForConditionalGeneration",anchor:"transformers.VipLlavaForConditionalGeneration",parameters:[{name:"config",val:": VipLlavaConfig"}],parametersDescription:[{anchor:"transformers.VipLlavaForConditionalGeneration.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.37.0/en/model_doc/vipllava#transformers.VipLlavaConfig">VipLlavaConfig</a> or <code>VipLlavaVisionConfig</code>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/v4.37.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.37.0/src/transformers/models/vipllava/modeling_vipllava.py#L240"}}),Q=new Ge({props:{name:"forward",anchor:"transformers.VipLlavaForConditionalGeneration.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"pixel_values",val:": FloatTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Optional = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"vision_feature_layers",val:": Optional = None"},{name:"labels",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"return_dict",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.VipLlavaForConditionalGeneration.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.37.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/v4.37.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.37.0/en/model_doc/fastspeech2_conformer#transformers.FastSpeech2ConformerTokenizer.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.pixel_values",description:'<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, image_size, image_size)) -- The tensors corresponding to the input images. Pixel values can be obtained using [AutoImageProcessor](/docs/transformers/v4.37.0/en/model_doc/auto#transformers.AutoImageProcessor). See [CLIPImageProcessor.__call__()](/docs/transformers/v4.37.0/en/model_doc/vilt#transformers.ViltFeatureExtractor.__call__) for details ([]</code>LlavaProcessor`] uses\n<a href="/docs/transformers/v4.37.0/en/model_doc/clip#transformers.CLIPImageProcessor">CLIPImageProcessor</a> for processing images).',name:"pixel_values"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a></p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.37.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/v4.37.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.37.0/en/model_doc/fastspeech2_conformer#transformers.FastSpeech2ConformerTokenizer.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>If you want to change padding behavior, you should read <code>modeling_opt._prepare_decoder_attention_mask</code>
and modify to your needs. See diagram 1 in <a href="https://arxiv.org/abs/1910.13461" rel="nofollow">the paper</a> for more
information on the default strategy.</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>. <a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.37.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.</p>
<p>Args &#x2014;
labels (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>):
Labels for computing the masked language modeling loss. Indices should either be in <code>[0, ..., config.vocab_size]</code> or -100 (see <code>input_ids</code> docstring). Tokens with indices set to <code>-100</code> are ignored
(masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/v4.37.0/src/transformers/models/vipllava/modeling_vipllava.py#L356",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>transformers.models.vipllava.modeling_vipllava.VipLlavaCausalLMOutputWithPast</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/v4.37.0/en/model_doc/vipllava#transformers.VipLlavaConfig"
>VipLlavaConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>)</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
<li>
<p><strong>image_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the image embeddings, <code>(batch_size, num_images, sequence_length, hidden_size)</code>.</p>
<p>image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver</p>
</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>transformers.models.vipllava.modeling_vipllava.VipLlavaCausalLMOutputWithPast</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),$=new nt({props:{$$slots:{default:[ot]},$$scope:{ctx:k}}}),L=new Ae({props:{anchor:"transformers.VipLlavaForConditionalGeneration.forward.example",$$slots:{default:[st]},$$scope:{ctx:k}}}),{c(){s=l("meta"),w=i(),p=l("p"),c=i(),v(u.$$.fragment),o=i(),v(f.$$.fragment),oe=i(),G=l("p"),G.innerHTML=Ze,se=i(),F=l("p"),F.textContent=ze,ie=i(),Z=l("p"),Z.textContent=We,re=i(),z=l("p"),z.innerHTML=Ie,le=i(),W=l("p"),W.textContent=Be,de=i(),I=l("ul"),I.innerHTML=Ne,ce=i(),v(B.$$.fragment),pe=i(),N=l("p"),N.textContent=He,me=i(),v(H.$$.fragment),he=i(),X=l("p"),X.innerHTML=Xe,ue=i(),R=l("p"),R.innerHTML=Re,fe=i(),v(Y.$$.fragment),ge=i(),g=l("div"),v(P.$$.fragment),we=i(),q=l("p"),q.innerHTML=Ye,je=i(),A=l("p"),A.innerHTML=Pe,Ce=i(),D=l("p"),D.innerHTML=Ee,Ue=i(),v(V.$$.fragment),ve=i(),v(E.$$.fragment),_e=i(),j=l("div"),v(S.$$.fragment),Je=i(),O=l("p"),O.innerHTML=Se,xe=i(),K=l("p"),K.innerHTML=Qe,ke=i(),U=l("div"),v(Q.$$.fragment),Ve=i(),ee=l("p"),ee.innerHTML=qe,$e=i(),v($.$$.fragment),Le=i(),v(L.$$.fragment),be=i(),te=l("p"),this.h()},l(e){const t=tt("svelte-u9bgzb",document.head);s=d(t,"META",{name:!0,content:!0}),t.forEach(n),w=r(e),p=d(e,"P",{}),ne(p).forEach(n),c=r(e),_(u.$$.fragment,e),o=r(e),_(f.$$.fragment,e),oe=r(e),G=d(e,"P",{"data-svelte-h":!0}),h(G)!=="svelte-7p4sji"&&(G.innerHTML=Ze),se=r(e),F=d(e,"P",{"data-svelte-h":!0}),h(F)!=="svelte-9bd7s8"&&(F.textContent=ze),ie=r(e),Z=d(e,"P",{"data-svelte-h":!0}),h(Z)!=="svelte-vfdo9a"&&(Z.textContent=We),re=r(e),z=d(e,"P",{"data-svelte-h":!0}),h(z)!=="svelte-ym7i2j"&&(z.innerHTML=Ie),le=r(e),W=d(e,"P",{"data-svelte-h":!0}),h(W)!=="svelte-axv494"&&(W.textContent=Be),de=r(e),I=d(e,"UL",{"data-svelte-h":!0}),h(I)!=="svelte-t3cjqh"&&(I.innerHTML=Ne),ce=r(e),_(B.$$.fragment,e),pe=r(e),N=d(e,"P",{"data-svelte-h":!0}),h(N)!=="svelte-mnlzle"&&(N.textContent=He),me=r(e),_(H.$$.fragment,e),he=r(e),X=d(e,"P",{"data-svelte-h":!0}),h(X)!=="svelte-14nivh0"&&(X.innerHTML=Xe),ue=r(e),R=d(e,"P",{"data-svelte-h":!0}),h(R)!=="svelte-1q33811"&&(R.innerHTML=Re),fe=r(e),_(Y.$$.fragment,e),ge=r(e),g=d(e,"DIV",{class:!0});var C=ne(g);_(P.$$.fragment,C),we=r(C),q=d(C,"P",{"data-svelte-h":!0}),h(q)!=="svelte-jxy185"&&(q.innerHTML=Ye),je=r(C),A=d(C,"P",{"data-svelte-h":!0}),h(A)!=="svelte-1cp2sn1"&&(A.innerHTML=Pe),Ce=r(C),D=d(C,"P",{"data-svelte-h":!0}),h(D)!=="svelte-1p8c3o7"&&(D.innerHTML=Ee),Ue=r(C),_(V.$$.fragment,C),C.forEach(n),ve=r(e),_(E.$$.fragment,e),_e=r(e),j=d(e,"DIV",{class:!0});var J=ne(j);_(S.$$.fragment,J),Je=r(J),O=d(J,"P",{"data-svelte-h":!0}),h(O)!=="svelte-51rntn"&&(O.innerHTML=Se),xe=r(J),K=d(J,"P",{"data-svelte-h":!0}),h(K)!=="svelte-hswkmf"&&(K.innerHTML=Qe),ke=r(J),U=d(J,"DIV",{class:!0});var x=ne(U);_(Q.$$.fragment,x),Ve=r(x),ee=d(x,"P",{"data-svelte-h":!0}),h(ee)!=="svelte-1rjmcsj"&&(ee.innerHTML=qe),$e=r(x),_($.$$.fragment,x),Le=r(x),_(L.$$.fragment,x),x.forEach(n),J.forEach(n),be=r(e),te=d(e,"P",{}),ne(te).forEach(n),this.h()},h(){ae(s,"name","hf:doc:metadata"),ae(s,"content",rt),ae(g,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),ae(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),ae(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){m(document.head,s),a(e,w,t),a(e,p,t),a(e,c,t),b(u,e,t),a(e,o,t),b(f,e,t),a(e,oe,t),a(e,G,t),a(e,se,t),a(e,F,t),a(e,ie,t),a(e,Z,t),a(e,re,t),a(e,z,t),a(e,le,t),a(e,W,t),a(e,de,t),a(e,I,t),a(e,ce,t),b(B,e,t),a(e,pe,t),a(e,N,t),a(e,me,t),b(H,e,t),a(e,he,t),a(e,X,t),a(e,ue,t),a(e,R,t),a(e,fe,t),b(Y,e,t),a(e,ge,t),a(e,g,t),b(P,g,null),m(g,we),m(g,q),m(g,je),m(g,A),m(g,Ce),m(g,D),m(g,Ue),b(V,g,null),a(e,ve,t),b(E,e,t),a(e,_e,t),a(e,j,t),b(S,j,null),m(j,Je),m(j,O),m(j,xe),m(j,K),m(j,ke),m(j,U),b(Q,U,null),m(U,Ve),m(U,ee),m(U,$e),b($,U,null),m(U,Le),b(L,U,null),a(e,be,t),a(e,te,t),Me=!0},p(e,[t]){const C={};t&2&&(C.$$scope={dirty:t,ctx:e}),V.$set(C);const J={};t&2&&(J.$$scope={dirty:t,ctx:e}),$.$set(J);const x={};t&2&&(x.$$scope={dirty:t,ctx:e}),L.$set(x)},i(e){Me||(M(u.$$.fragment,e),M(f.$$.fragment,e),M(B.$$.fragment,e),M(H.$$.fragment,e),M(Y.$$.fragment,e),M(P.$$.fragment,e),M(V.$$.fragment,e),M(E.$$.fragment,e),M(S.$$.fragment,e),M(Q.$$.fragment,e),M($.$$.fragment,e),M(L.$$.fragment,e),Me=!0)},o(e){y(u.$$.fragment,e),y(f.$$.fragment,e),y(B.$$.fragment,e),y(H.$$.fragment,e),y(Y.$$.fragment,e),y(P.$$.fragment,e),y(V.$$.fragment,e),y(E.$$.fragment,e),y(S.$$.fragment,e),y(Q.$$.fragment,e),y($.$$.fragment,e),y(L.$$.fragment,e),Me=!1},d(e){e&&(n(w),n(p),n(c),n(o),n(oe),n(G),n(se),n(F),n(ie),n(Z),n(re),n(z),n(le),n(W),n(de),n(I),n(ce),n(pe),n(N),n(me),n(he),n(X),n(ue),n(R),n(fe),n(ge),n(g),n(ve),n(_e),n(j),n(be),n(te)),n(s),T(u,e),T(f,e),T(B,e),T(H,e),T(Y,e),T(P),T(V),T(E,e),T(S),T(Q),T($),T(L)}}}const rt='{"title":"VipLlava","local":"vipllava","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"VipLlavaConfig","local":"transformers.VipLlavaConfig","sections":[],"depth":2},{"title":"VipLlavaForConditionalGeneration","local":"transformers.VipLlavaForConditionalGeneration","sections":[],"depth":2}],"depth":1}';function lt(k){return Oe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class gt extends Ke{constructor(s){super(),et(this,s,lt,it,De,{})}}export{gt as component};
