import{s as $e,o as xe,n as Ce}from"../chunks/scheduler.25b97de1.js";import{S as ze,i as Fe,g as l,s,r as g,A as Le,h as m,f as n,c as r,j as E,u as _,x as C,k as S,y as u,a,v as b,d as y,t as v,w as k}from"../chunks/index.d9030fc9.js";import{T as Je}from"../chunks/Tip.baa67368.js";import{D as ge}from"../chunks/Docstring.ffac8efa.js";import{C as Be}from"../chunks/CodeBlock.e6cd0d95.js";import{E as je}from"../chunks/ExampleCodeBlock.22dfe688.js";import{H as de,E as Ue}from"../chunks/EditOnGithub.91d95064.js";function Ie(R){let o,w=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){o=l("p"),o.innerHTML=w},l(d){o=m(d,"P",{"data-svelte-h":!0}),C(o)!=="svelte-fincs2"&&(o.innerHTML=w)},m(d,c){a(d,o,c)},p:Ce,d(d){d&&n(o)}}}function He(R){let o,w="Example:",d,c,h;return c=new Be({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBCYW1iYUZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBCYW1iYUZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjIuLi4lMjIpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyLi4uJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMkhleSUyQyUyMGFyZSUyMHlvdSUyMGNvbnNjaW91cyUzRiUyMENhbiUyMHlvdSUyMHRhbGslMjB0byUyMG1lJTNGJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBJTIzJTIwR2VuZXJhdGUlMEFnZW5lcmF0ZV9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dHMuaW5wdXRfaWRzJTJDJTIwbWF4X2xlbmd0aCUzRDMwKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUoZ2VuZXJhdGVfaWRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUlMkMlMjBjbGVhbl91cF90b2tlbml6YXRpb25fc3BhY2VzJTNERmFsc2UpJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, BambaForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = BambaForCausalLM.from_pretrained(<span class="hljs-string">&quot;...&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;...&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;Hey, are you conscious? Can you talk to me?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Generate</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generate_ids = model.generate(inputs.input_ids, max_length=<span class="hljs-number">30</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generate_ids, skip_special_tokens=<span class="hljs-literal">True</span>, clean_up_tokenization_spaces=<span class="hljs-literal">False</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;Hey, are you conscious? Can you talk to me?\\nI&#x27;m not conscious, but I can talk to you.&quot;</span>`,wrap:!1}}),{c(){o=l("p"),o.textContent=w,d=s(),g(c.$$.fragment)},l(i){o=m(i,"P",{"data-svelte-h":!0}),C(o)!=="svelte-11lpom8"&&(o.textContent=w),d=r(i),_(c.$$.fragment,i)},m(i,T){a(i,o,T),a(i,d,T),b(c,i,T),h=!0},p:Ce,i(i){h||(y(c.$$.fragment,i),h=!0)},o(i){v(c.$$.fragment,i),h=!1},d(i){i&&(n(o),n(d)),k(c,i)}}}function qe(R){let o,w,d,c,h,i,T,D,F,_e='Bamba-9B is a decoder-only language model based on the <a href="https://github.com/state-spaces/mamba" rel="nofollow">Mamba-2</a> architecture and is designed to handle a wide range of text generation tasks. It is trained from scratch using a two-stage training approach. In the first stage, the model is trained on 2 trillion tokens from the Dolma v1.7 dataset. In the second stage, it undergoes additional training on 200 billion tokens, leveraging a carefully curated blend of high-quality data to further refine its performance and enhance output quality.',O,L,be='Checkout all Bamba-9B model checkpoints <a href="https://github.com/foundation-model-stack/bamba" rel="nofollow">here</a>.',Y,J,K,j,ye="<thead><tr><th>Model</th> <th>Params</th> <th># Layers</th> <th>Hidden Dim.</th> <th>Attention Heads</th> <th>GQA</th> <th>KV Heads</th> <th>Context Length</th> <th>Tied Embeddings</th></tr></thead> <tbody><tr><td>Bamba</td> <td>9B (9.78B)</td> <td>32</td> <td>4096</td> <td>32</td> <td>Yes</td> <td>8</td> <td>4096</td> <td>True</td></tr></tbody>",ee,p,U,le,A,ve=`This is the configuration class to store the configuration of a <a href="/docs/transformers/main/en/model_doc/bamba#transformers.BambaModel">BambaModel</a>. It is used to instantiate a
BambaModel model according to the specified arguments, defining the model architecture. Instantiating a configuration
with defaults taken from <a href="https://huggingface.co/ibm-fms/Bamba-9.8b-2.2T-hf" rel="nofollow">ibm-fms/Bamba-9.8b-2.2T-hf</a>.`,me,G,ke=`The BambaModel is a hybrid <a href="https://github.com/state-spaces/mamba" rel="nofollow">mamba2</a> architecture with SwiGLU.
The checkpoints are  jointly trained by IBM, Princeton, and UIUC.`,ce,X,Te=`Configuration objects inherit from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,te,I,ne,H,oe,B,q,pe,f,W,ue,P,Me='The <a href="/docs/transformers/main/en/model_doc/bamba#transformers.BambaForCausalLM">BambaForCausalLM</a> forward method, overrides the <code>__call__</code> special method.',he,$,fe,x,ae,N,we='This HF implementation is contributed by <a href="https://github.com/ani300" rel="nofollow">ani300</a> and <a href="https://github.com/fabianlim" rel="nofollow">fabianlim</a>.',se,Z,re,Q,ie;return h=new de({props:{title:"Bamba",local:"bamba",headingTag:"h1"}}),T=new de({props:{title:"Overview",local:"overview",headingTag:"h2"}}),J=new de({props:{title:"BambaConfig",local:"transformers.BambaConfig",headingTag:"h2"}}),U=new ge({props:{name:"class transformers.BambaConfig",anchor:"transformers.BambaConfig",parameters:[{name:"vocab_size",val:" = 128000"},{name:"tie_word_embeddings",val:" = False"},{name:"hidden_size",val:" = 4096"},{name:"intermediate_size",val:" = 14336"},{name:"num_hidden_layers",val:" = 32"},{name:"num_attention_heads",val:" = 32"},{name:"num_key_value_heads",val:" = 8"},{name:"hidden_act",val:" = 'silu'"},{name:"initializer_range",val:" = 0.02"},{name:"rms_norm_eps",val:" = 1e-05"},{name:"use_cache",val:" = True"},{name:"num_logits_to_keep",val:" = 1"},{name:"pad_token_id",val:" = 0"},{name:"bos_token_id",val:" = 1"},{name:"eos_token_id",val:" = 2"},{name:"max_position_embeddings",val:" = 262144"},{name:"attention_dropout",val:" = 0.0"},{name:"attn_layer_indices",val:" = None"},{name:"mamba_n_heads",val:" = 128"},{name:"mamba_d_head",val:" = 'auto'"},{name:"mamba_n_groups",val:" = 1"},{name:"mamba_d_state",val:" = 256"},{name:"mamba_d_conv",val:" = 4"},{name:"mamba_expand",val:" = 2"},{name:"mamba_chunk_size",val:" = 256"},{name:"mamba_conv_bias",val:" = True"},{name:"mamba_proj_bias",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BambaConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 128000) &#x2014;
Vocabulary size of the Bamba model. Defines the number of different tokens that can be represented by the
<code>inputs_ids</code> passed when calling <a href="/docs/transformers/main/en/model_doc/bamba#transformers.BambaModel">BambaModel</a>`,name:"vocab_size"},{anchor:"transformers.BambaConfig.tie_word_embeddings",description:`<strong>tie_word_embeddings</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the model&#x2019;s input and output word embeddings should be tied. Note that this is only relevant if the
model has a output word embedding layer.`,name:"tie_word_embeddings"},{anchor:"transformers.BambaConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 4096) &#x2014;
Dimension of the hidden representations.`,name:"hidden_size"},{anchor:"transformers.BambaConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 14336) &#x2014;
Dimension of the MLP representations.`,name:"intermediate_size"},{anchor:"transformers.BambaConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.BambaConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.BambaConfig.num_key_value_heads",description:`<strong>num_key_value_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 8) &#x2014;
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
<code>num_key_value_heads=num_attention_heads</code>, the model will use Multi Head Attention (MHA), if
<code>num_key_value_heads=1</code> the model will use Multi Query Attention (MQA) otherwise GQA is used. When
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
by meanpooling all the original heads within that group. For more details checkout <a href="https://arxiv.org/pdf/2305.13245.pdf" rel="nofollow">this
paper</a>. If it is not specified, will default to <code>8</code>.`,name:"num_key_value_heads"},{anchor:"transformers.BambaConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>function</code>, <em>optional</em>, defaults to <code>&quot;silu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the decoder.`,name:"hidden_act"},{anchor:"transformers.BambaConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 0.02) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.BambaConfig.rms_norm_eps",description:`<strong>rms_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-05) &#x2014;
The epsilon used by the rms normalization layers.`,name:"rms_norm_eps"},{anchor:"transformers.BambaConfig.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if <code>config.is_decoder=True</code>.`,name:"use_cache"},{anchor:"transformers.BambaConfig.num_logits_to_keep",description:`<strong>num_logits_to_keep</strong> (<code>int</code> or <code>None</code>, <em>optional</em>, defaults to 1) &#x2014;
Number of prompt logits to calculate during generation. If <code>None</code>, all logits will be calculated. If an
integer value, only last <code>num_logits_to_keep</code> logits will be calculated. Default is 1 because only the
logits of the last prompt token are needed for generation. For long sequences, the logits for the entire
sequence may use a lot of memory so, setting <code>num_logits_to_keep=1</code> will reduce memory footprint
significantly.`,name:"num_logits_to_keep"},{anchor:"transformers.BambaConfig.pad_token_id",description:`<strong>pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 0) &#x2014;
The id of the padding token.`,name:"pad_token_id"},{anchor:"transformers.BambaConfig.bos_token_id",description:`<strong>bos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The id of the &#x201C;beginning-of-sequence&#x201D; token.`,name:"bos_token_id"},{anchor:"transformers.BambaConfig.eos_token_id",description:`<strong>eos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
The id of the &#x201C;end-of-sequence&#x201D; token.`,name:"eos_token_id"},{anchor:"transformers.BambaConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to 262144) &#x2014;
Max cached sequence length for the model`,name:"max_position_embeddings"},{anchor:"transformers.BambaConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_dropout"},{anchor:"transformers.BambaConfig.attn_layer_indices",description:`<strong>attn_layer_indices</strong> (<code>list</code>, <em>optional</em>) &#x2014;
Specifies the layer indices that will have full attention. Must contain values at most num_hidden_layers.`,name:"attn_layer_indices"},{anchor:"transformers.BambaConfig.mamba_n_heads",description:`<strong>mamba_n_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 128) &#x2014;
The number of mamba heads used in the v2 implementation.`,name:"mamba_n_heads"},{anchor:"transformers.BambaConfig.mamba_d_head",description:`<strong>mamba_d_head</strong> (<code>int</code>, <em>optional</em>, defaults to <code>&quot;auto&quot;</code>) &#x2014;
Head embeddding dimension size`,name:"mamba_d_head"},{anchor:"transformers.BambaConfig.mamba_n_groups",description:`<strong>mamba_n_groups</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of the mamba groups used in the v2 implementation.`,name:"mamba_n_groups"},{anchor:"transformers.BambaConfig.mamba_d_state",description:`<strong>mamba_d_state</strong> (<code>int</code>, <em>optional</em>, defaults to 256) &#x2014;
The dimension the mamba state space latents`,name:"mamba_d_state"},{anchor:"transformers.BambaConfig.mamba_d_conv",description:`<strong>mamba_d_conv</strong> (<code>int</code>, <em>optional</em>, defaults to 4) &#x2014;
The size of the mamba convolution kernel`,name:"mamba_d_conv"},{anchor:"transformers.BambaConfig.mamba_expand",description:`<strong>mamba_expand</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
Expanding factor (relative to hidden_size) used to determine the mamba intermediate size`,name:"mamba_expand"},{anchor:"transformers.BambaConfig.mamba_chunk_size",description:`<strong>mamba_chunk_size</strong> (<code>int</code>, <em>optional</em>, defaults to 256) &#x2014;
The chunks in which to break the sequence when doing prefill/training`,name:"mamba_chunk_size"},{anchor:"transformers.BambaConfig.mamba_conv_bias",description:`<strong>mamba_conv_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Flag indicating whether or not to use bias in the convolution layer of the mamba mixer block.`,name:"mamba_conv_bias"},{anchor:"transformers.BambaConfig.mamba_proj_bias",description:`<strong>mamba_proj_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Flag indicating whether or not to use bias in the input and output projections ([&#x201C;in_proj&#x201D;, &#x201C;out_proj&#x201D;]) of the mamba mixer block`,name:"mamba_proj_bias"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bamba/configuration_bamba.py#L24"}}),I=new de({props:{title:"BambaForCausalLM",local:"transformers.BambaForCausalLM",headingTag:"h2"}}),H=new Be({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmlibS1mbXMlMkZCYW1iYS05QiUyMiklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJpYm0tZm1zJTJGQmFtYmEtOUIlMjIpJTBBJTBBbWVzc2FnZSUyMCUzRCUyMCU1QiUyMk1hbWJhJTIwaXMlMjBhJTIwc25ha2UlMjB3aXRoJTIwZm9sbG93aW5nJTIwcHJvcGVydGllcyUyMCUyMCUyMiU1RCUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihtZXNzYWdlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QncHQnJTJDJTIwcmV0dXJuX3Rva2VuX3R5cGVfaWRzJTNERmFsc2UpJTBBcmVzcG9uc2UlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMG1heF9uZXdfdG9rZW5zJTNENjQpJTBBcHJpbnQodG9rZW5pemVyLmJhdGNoX2RlY29kZShyZXNwb25zZSUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;ibm-fms/Bamba-9B&quot;</span>)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;ibm-fms/Bamba-9B&quot;</span>)

message = [<span class="hljs-string">&quot;Mamba is a snake with following properties  &quot;</span>]
inputs = tokenizer(message, return_tensors=<span class="hljs-string">&#x27;pt&#x27;</span>, return_token_type_ids=<span class="hljs-literal">False</span>)
response = model.generate(**inputs, max_new_tokens=<span class="hljs-number">64</span>)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(response, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>])`,wrap:!1}}),q=new ge({props:{name:"class transformers.BambaForCausalLM",anchor:"transformers.BambaForCausalLM",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bamba/modeling_bamba.py#L1438"}}),W=new ge({props:{name:"forward",anchor:"transformers.BambaForCausalLM.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"position_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"past_key_values",val:": typing.Optional[transformers.models.bamba.modeling_bamba.HybridMambaAttentionDynamicCache] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"use_cache",val:": typing.Optional[bool] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"},{name:"cache_position",val:": typing.Optional[torch.LongTensor] = None"},{name:"num_logits_to_keep",val:": int = 0"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BambaForCausalLM.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.</p>
<p>Indices can be obtained using <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.BambaForCausalLM.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a></p>
<p>Indices can be obtained using <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>If you want to change padding behavior, you should read <code>modeling_opt._prepare_decoder_attention_mask</code>
and modify to your needs. See diagram 1 in <a href="https://arxiv.org/abs/1910.13461" rel="nofollow">the paper</a> for more
information on the default strategy.</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.BambaForCausalLM.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.BambaForCausalLM.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>HybridMambaAttentionDynamicCache</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
A HybridMambaAttentionDynamicCache object containing pre-computed hidden-states (keys and values in the
self-attention blocks and convolution and ssm states in the mamba blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.
Key and value cache tensors have shape <code>(batch_size, num_heads, seq_len, head_dim)</code>.
Convolution and ssm states tensors have shape <code>(batch_size, d_inner, d_conv)</code> and
<code>(batch_size, d_inner, d_state)</code> respectively.
See the <code>HybridMambaAttentionDynamicCache</code> class for more details.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.BambaForCausalLM.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.BambaForCausalLM.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.BambaForCausalLM.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.BambaForCausalLM.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.BambaForCausalLM.forward.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
should not be returned during inference.`,name:"output_router_logits"},{anchor:"transformers.BambaForCausalLM.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.BambaForCausalLM.forward.cache_position",description:`<strong>cache_position</strong> (<code>torch.LongTensor</code> of shape <code>(sequence_length)</code>, <em>optional</em>) &#x2014;
Indices depicting the position of the input sequence tokens in the sequence. Contrarily to <code>position_ids</code>,
this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
the complete sequence length.`,name:"cache_position"},{anchor:"transformers.BambaForCausalLM.forward.Args",description:`<strong>Args</strong> &#x2014;
labels (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>):
Labels for computing the masked language modeling loss. Indices should either be in <code>[0, ..., config.vocab_size]</code> or -100 (see <code>input_ids</code> docstring). Tokens with indices set to <code>-100</code> are ignored
(masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>.</p>
<p>num_logits_to_keep (<code>int</code> or <code>None</code>, <em>optional</em>):
Calculate logits for the last <code>num_logits_to_keep</code> tokens. If <code>None</code>, calculate logits for all
<code>input_ids</code>. Only last token logits are needed for generation, and calculating them only for that token
can save memory, which becomes pretty significant for long sequences.`,name:"Args"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bamba/modeling_bamba.py#L1469",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <a
  href="/docs/transformers/main/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithPast"
>transformers.modeling_outputs.CausalLMOutputWithPast</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/main/en/model_doc/bamba#transformers.BambaConfig"
>BambaConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>)</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><a
  href="/docs/transformers/main/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithPast"
>transformers.modeling_outputs.CausalLMOutputWithPast</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),$=new Je({props:{$$slots:{default:[Ie]},$$scope:{ctx:R}}}),x=new je({props:{anchor:"transformers.BambaForCausalLM.forward.example",$$slots:{default:[He]},$$scope:{ctx:R}}}),Z=new Ue({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/bamba.md"}}),{c(){o=l("meta"),w=s(),d=l("p"),c=s(),g(h.$$.fragment),i=s(),g(T.$$.fragment),D=s(),F=l("p"),F.innerHTML=_e,O=s(),L=l("p"),L.innerHTML=be,Y=s(),g(J.$$.fragment),K=s(),j=l("table"),j.innerHTML=ye,ee=s(),p=l("div"),g(U.$$.fragment),le=s(),A=l("p"),A.innerHTML=ve,me=s(),G=l("p"),G.innerHTML=ke,ce=s(),X=l("p"),X.innerHTML=Te,te=s(),g(I.$$.fragment),ne=s(),g(H.$$.fragment),oe=s(),B=l("div"),g(q.$$.fragment),pe=s(),f=l("div"),g(W.$$.fragment),ue=s(),P=l("p"),P.innerHTML=Me,he=s(),g($.$$.fragment),fe=s(),g(x.$$.fragment),ae=s(),N=l("p"),N.innerHTML=we,se=s(),g(Z.$$.fragment),re=s(),Q=l("p"),this.h()},l(e){const t=Le("svelte-u9bgzb",document.head);o=m(t,"META",{name:!0,content:!0}),t.forEach(n),w=r(e),d=m(e,"P",{}),E(d).forEach(n),c=r(e),_(h.$$.fragment,e),i=r(e),_(T.$$.fragment,e),D=r(e),F=m(e,"P",{"data-svelte-h":!0}),C(F)!=="svelte-1y13t4e"&&(F.innerHTML=_e),O=r(e),L=m(e,"P",{"data-svelte-h":!0}),C(L)!=="svelte-1ego9ix"&&(L.innerHTML=be),Y=r(e),_(J.$$.fragment,e),K=r(e),j=m(e,"TABLE",{"data-svelte-h":!0}),C(j)!=="svelte-1rch88z"&&(j.innerHTML=ye),ee=r(e),p=m(e,"DIV",{class:!0});var M=E(p);_(U.$$.fragment,M),le=r(M),A=m(M,"P",{"data-svelte-h":!0}),C(A)!=="svelte-sna0j2"&&(A.innerHTML=ve),me=r(M),G=m(M,"P",{"data-svelte-h":!0}),C(G)!=="svelte-5k5of"&&(G.innerHTML=ke),ce=r(M),X=m(M,"P",{"data-svelte-h":!0}),C(X)!=="svelte-o55m63"&&(X.innerHTML=Te),M.forEach(n),te=r(e),_(I.$$.fragment,e),ne=r(e),_(H.$$.fragment,e),oe=r(e),B=m(e,"DIV",{class:!0});var V=E(B);_(q.$$.fragment,V),pe=r(V),f=m(V,"DIV",{class:!0});var z=E(f);_(W.$$.fragment,z),ue=r(z),P=m(z,"P",{"data-svelte-h":!0}),C(P)!=="svelte-lvhhjq"&&(P.innerHTML=Me),he=r(z),_($.$$.fragment,z),fe=r(z),_(x.$$.fragment,z),z.forEach(n),V.forEach(n),ae=r(e),N=m(e,"P",{"data-svelte-h":!0}),C(N)!=="svelte-14g2lxt"&&(N.innerHTML=we),se=r(e),_(Z.$$.fragment,e),re=r(e),Q=m(e,"P",{}),E(Q).forEach(n),this.h()},h(){S(o,"name","hf:doc:metadata"),S(o,"content",We),S(p,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),S(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),S(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){u(document.head,o),a(e,w,t),a(e,d,t),a(e,c,t),b(h,e,t),a(e,i,t),b(T,e,t),a(e,D,t),a(e,F,t),a(e,O,t),a(e,L,t),a(e,Y,t),b(J,e,t),a(e,K,t),a(e,j,t),a(e,ee,t),a(e,p,t),b(U,p,null),u(p,le),u(p,A),u(p,me),u(p,G),u(p,ce),u(p,X),a(e,te,t),b(I,e,t),a(e,ne,t),b(H,e,t),a(e,oe,t),a(e,B,t),b(q,B,null),u(B,pe),u(B,f),b(W,f,null),u(f,ue),u(f,P),u(f,he),b($,f,null),u(f,fe),b(x,f,null),a(e,ae,t),a(e,N,t),a(e,se,t),b(Z,e,t),a(e,re,t),a(e,Q,t),ie=!0},p(e,[t]){const M={};t&2&&(M.$$scope={dirty:t,ctx:e}),$.$set(M);const V={};t&2&&(V.$$scope={dirty:t,ctx:e}),x.$set(V)},i(e){ie||(y(h.$$.fragment,e),y(T.$$.fragment,e),y(J.$$.fragment,e),y(U.$$.fragment,e),y(I.$$.fragment,e),y(H.$$.fragment,e),y(q.$$.fragment,e),y(W.$$.fragment,e),y($.$$.fragment,e),y(x.$$.fragment,e),y(Z.$$.fragment,e),ie=!0)},o(e){v(h.$$.fragment,e),v(T.$$.fragment,e),v(J.$$.fragment,e),v(U.$$.fragment,e),v(I.$$.fragment,e),v(H.$$.fragment,e),v(q.$$.fragment,e),v(W.$$.fragment,e),v($.$$.fragment,e),v(x.$$.fragment,e),v(Z.$$.fragment,e),ie=!1},d(e){e&&(n(w),n(d),n(c),n(i),n(D),n(F),n(O),n(L),n(Y),n(K),n(j),n(ee),n(p),n(te),n(ne),n(oe),n(B),n(ae),n(N),n(se),n(re),n(Q)),n(o),k(h,e),k(T,e),k(J,e),k(U),k(I,e),k(H,e),k(q),k(W),k($),k(x),k(Z,e)}}}const We='{"title":"Bamba","local":"bamba","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"BambaConfig","local":"transformers.BambaConfig","sections":[],"depth":2},{"title":"BambaForCausalLM","local":"transformers.BambaForCausalLM","sections":[],"depth":2}],"depth":1}';function Ne(R){return xe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Qe extends ze{constructor(o){super(),Fe(this,o,Ne,qe,$e,{})}}export{Qe as component};
