import{s as pn,o as mn,n as at}from"../chunks/scheduler.9bc65507.js";import{S as un,i as hn,g as r,s as a,r as m,A as fn,h as i,f as o,c as s,j as q,u,x as d,k as Z,l as gn,y as c,a as n,v as h,d as f,t as g,w as _}from"../chunks/index.707bf1b6.js";import{T as yo}from"../chunks/Tip.c2ecdbf4.js";import{D as Ge}from"../chunks/Docstring.17db21ae.js";import{C as Be}from"../chunks/CodeBlock.54a9f38d.js";import{E as cn}from"../chunks/ExampleCodeBlock.4f515aa9.js";import{H as C}from"../chunks/Heading.342b1fa6.js";function _n(z){let l,M;return l=new Be({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyME1peHRyYWxNb2RlbCUyQyUyME1peHRyYWxDb25maWclMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwTWl4dHJhbCUyMDdCJTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyME1peHRyYWxDb25maWcoKSUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBtb2RlbCUyMGZyb20lMjB0aGUlMjBNaXh0cmFsJTIwN0IlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFtb2RlbCUyMCUzRCUyME1peHRyYWxNb2RlbChjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> MixtralModel, MixtralConfig

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a Mixtral 7B style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = MixtralConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model from the Mixtral 7B style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = MixtralModel(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,wrap:!1}}),{c(){m(l.$$.fragment)},l(p){u(l.$$.fragment,p)},m(p,b){h(l,p,b),M=!0},p:at,i(p){M||(f(l.$$.fragment,p),M=!0)},o(p){g(l.$$.fragment,p),M=!1},d(p){_(l,p)}}}function Mn(z){let l,M=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){l=r("p"),l.innerHTML=M},l(p){l=i(p,"P",{"data-svelte-h":!0}),d(l)!=="svelte-fincs2"&&(l.innerHTML=M)},m(p,b){n(p,l,b)},p:at,d(p){p&&o(l)}}}function bn(z){let l,M=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){l=r("p"),l.innerHTML=M},l(p){l=i(p,"P",{"data-svelte-h":!0}),d(l)!=="svelte-fincs2"&&(l.innerHTML=M)},m(p,b){n(p,l,b)},p:at,d(p){p&&o(l)}}}function vn(z){let l,M="Example:",p,b,J;return b=new Be({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBNaXh0cmFsRm9yQ2F1c2FsTE0lMEElMEFtb2RlbCUyMCUzRCUyME1peHRyYWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIybWlzdHJhbGFpJTJGTWl4dHJhbC04eDdCLXYwLjElMjIpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWlzdHJhbGFpJTJGTWl4dHJhbC04eDdCLXYwLjElMjIpJTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIySGV5JTJDJTIwYXJlJTIweW91JTIwY29uc2Npb3VzJTNGJTIwQ2FuJTIweW91JTIwdGFsayUyMHRvJTIwbWUlM0YlMjIlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEElMjMlMjBHZW5lcmF0ZSUwQWdlbmVyYXRlX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKGlucHV0cy5pbnB1dF9pZHMlMkMlMjBtYXhfbGVuZ3RoJTNEMzApJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShnZW5lcmF0ZV9pZHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSUyQyUyMGNsZWFuX3VwX3Rva2VuaXphdGlvbl9zcGFjZXMlM0RGYWxzZSklNUIwJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, MixtralForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = MixtralForCausalLM.from_pretrained(<span class="hljs-string">&quot;mistralai/Mixtral-8x7B-v0.1&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;mistralai/Mixtral-8x7B-v0.1&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;Hey, are you conscious? Can you talk to me?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Generate</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generate_ids = model.generate(inputs.input_ids, max_length=<span class="hljs-number">30</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generate_ids, skip_special_tokens=<span class="hljs-literal">True</span>, clean_up_tokenization_spaces=<span class="hljs-literal">False</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;Hey, are you conscious? Can you talk to me?\\nI&#x27;m not conscious, but I can talk to you.&quot;</span>`,wrap:!1}}),{c(){l=r("p"),l.textContent=M,p=a(),m(b.$$.fragment)},l(v){l=i(v,"P",{"data-svelte-h":!0}),d(l)!=="svelte-11lpom8"&&(l.textContent=M),p=s(v),u(b.$$.fragment,v)},m(v,L){n(v,l,L),n(v,p,L),h(b,v,L),J=!0},p:at,i(v){J||(f(b.$$.fragment,v),J=!0)},o(v){g(b.$$.fragment,v),J=!1},d(v){v&&(o(l),o(p)),_(b,v)}}}function xn(z){let l,M=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){l=r("p"),l.innerHTML=M},l(p){l=i(p,"P",{"data-svelte-h":!0}),d(l)!=="svelte-fincs2"&&(l.innerHTML=M)},m(p,b){n(p,l,b)},p:at,d(p){p&&o(l)}}}function yn(z){let l,M,p,b,J,v,L,st,P,To="Mixtral-8x7B is Mistral AI’s second Large Language Model (LLM).",rt,V,wo='The Mixtral model was proposed by the <a href="https://mistral.ai/" rel="nofollow">Mistral AI</a> team.',it,N,ko='It was introduced in the <a href="https://mistral.ai/news/mixtral-of-experts/" rel="nofollow">Mixtral of Experts blogpost</a> with the following introduction:',lt,A,$o="<em>Today, the team is proud to release Mixtral 8x7B, a high-quality sparse mixture of experts models (SMoE) with open weights. Licensed under Apache 2.0. Mixtral outperforms Llama 2 70B on most benchmarks with 6x faster inference. It is the strongest open-weight model with a permissive license and the best model overall regarding cost/performance trade-offs. In particular, it matches or outperforms GPT3.5 on most standard benchmarks.</em>",dt,Y,Co="Tips:",ct,Q,zo='<li>The model needs to be converted using the <a href="https://github.com/huggingface/transformers/blob/main/src/transformers/models/mixtral/convert_mixtral_weights_to_hf.py" rel="nofollow">conversion script</a>.</li> <li>If the model is quantized to 4bits, a single A100 is enough to fit the entire 45B model.</li>',pt,D,Jo=`This model was contributed by <a href="https://huggingface.co/ybelkada" rel="nofollow">Younes Belkada</a> and <a href="https://huggingface.co/ArthurZ" rel="nofollow">Arthur Zucker</a> .
The original code can be found <a href="https://github.com/mistralai/mistral-src" rel="nofollow">here</a>.`,mt,O,ut,K,Fo="Mixtral-45B is a decoder-based LM with the following architectural choices:",ht,ee,Lo="<li>Mixtral is a Mixture of Expert (MOE) model with 8 experts per MLP, with a total of 45B paramateres but the compute required is the same as a 14B model. This is because even though each experts have to be loaded in RAM (70B like ram requirement) each token from the hidden states are dipatched twice (top 2 routing) and thus the compute (the operation required at each foward computation) is just 2 X sequence_length.</li>",ft,te,jo='The following implementation details are shared with Mistral AI’s first model <a href="mistral">mistral</a>:',gt,oe,Io="<li>Sliding Window Attention - Trained with 8k context length and fixed cache size, with a theoretical attention span of 128K tokens</li> <li>GQA (Grouped Query Attention) - allowing faster inference and lower cache size.</li> <li>Byte-fallback BPE tokenizer - ensures that characters are never mapped to out of vocabulary tokens.</li>",_t,ne,Uo="They also provide an instruction fine-tuned model: <code>mistralai/Mixtral-8x7B-v0.1</code> which can be used for chat-based inference.",Mt,ae,Ho='For more details please read our <a href="https://mistral.ai/news/mixtral-of-experts/" rel="nofollow">release blog post</a>',bt,se,vt,re,Wo="<code>Mixtral-8x7B</code> is released under the Apache 2.0 license.",xt,ie,yt,le,qo='<code>Mixtral-8x7B</code> can be found on the <a href="https://huggingface.co/mistralai" rel="nofollow">Huggingface Hub</a>',Tt,de,Zo="These ready-to-use checkpoints can be downloaded and used via the HuggingFace Hub:",wt,ce,kt,pe,Ro="To use the raw checkpoints with HuggingFace you can use the <code>convert_mixtral_weights_to_hf.py</code> script to convert them to the HuggingFace format:",$t,me,Ct,ue,Go="You can then load the converted model from the <code>output/path</code>:",zt,he,Jt,fe,Ft,ge,Bo="First, make sure to install the latest version of Flash Attention 2 to include the sliding window attention feature.",Lt,_e,jt,Me,So='Make also sure that you have a hardware that is compatible with Flash-Attention 2. Read more about it in the official documentation of <a href="https://github.com/Dao-AILab/flash-attention" rel="nofollow"><code>flash-attn</code></a> repository. Make also sure to load your model in half-precision (e.g. <code>torch.float16</code>)',It,be,Eo="To load and run a model using Flash Attention 2, refer to the snippet below:",Ut,ve,Ht,xe,Wt,ye,Xo="Below is a expected speedup diagram that compares pure inference time between the native implementation in transformers using <code>mistralai/Mixtral-8x7B-v0.1</code> checkpoint and the Flash Attention 2 version of the model.",qt,R,Po='<img src="https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/mixtral-7b-inference-large-seqlen.png"/>',Zt,Te,Rt,we,Vo=`The current implementation supports the sliding window attention mechanism and memory efficient cache management.
To enable sliding window attention, just make sure to have a <code>flash-attn</code> version that is compatible with sliding window attention (<code>&gt;=2.3.0</code>).`,Gt,ke,No="The Flash Attention-2 model uses also a more memory efficient cache slicing mechanism - as recommended per the official implementation of Mistral model that use rolling cache mechanism we keep the cache size fixed (<code>self.config.sliding_window</code>), support batched generation only for <code>padding_side=&quot;left&quot;</code> and use the absolute position of the current token to compute the positional embedding.",Bt,$e,St,Ce,Ao="Albert Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lélio Renard Lavaud, Lucile Saulnier, Marie-Anne Lachaux, Pierre Stock, Teven Le Scao, Thibaut Lavril, Thomas Wang, Timothée Lacroix, William El Sayed.",Et,ze,Xt,y,Je,Kt,Se,Yo=`This is the configuration class to store the configuration of a <a href="/docs/transformers/v4.37.0/en/model_doc/mixtral#transformers.MixtralModel">MixtralModel</a>. It is used to instantiate an
Mixtral model according to the specified arguments, defining the model architecture. Instantiating a configuration
with the defaults will yield a similar configuration to that of the Mixtral-7B-v0.1 or Mixtral-7B-Instruct-v0.1.`,eo,Ee,Qo='<a href="https://huggingface.co/mixtralai/Mixtral-8x7B" rel="nofollow">mixtralai/Mixtral-8x7B</a> <a href="https://huggingface.co/mixtralai/Mixtral-7B-Instruct-v0.1" rel="nofollow">mixtralai/Mixtral-7B-Instruct-v0.1</a>',to,Xe,Do=`Configuration objects inherit from <a href="/docs/transformers/v4.37.0/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/v4.37.0/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,oo,G,Pt,Fe,Vt,T,Le,no,Pe,Oo=`The bare Mixtral Model outputting raw hidden-states without any specific head on top.
This model inherits from <a href="/docs/transformers/v4.37.0/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`,ao,Ve,Ko=`This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`,so,Ne,en="Transformer decoder consisting of <em>config.num_hidden_layers</em> layers. Each layer is a <code>MixtralDecoderLayer</code>",ro,I,je,io,Ae,tn='The <a href="/docs/transformers/v4.37.0/en/model_doc/mixtral#transformers.MixtralModel">MixtralModel</a> forward method, overrides the <code>__call__</code> special method.',lo,B,Nt,Ie,At,H,Ue,co,F,He,po,Ye,on='The <a href="/docs/transformers/v4.37.0/en/model_doc/mixtral#transformers.MixtralForCausalLM">MixtralForCausalLM</a> forward method, overrides the <code>__call__</code> special method.',mo,S,uo,E,Yt,We,Qt,x,qe,ho,Qe,nn="The Mixtral Model transformer with a sequence classification head on top (linear layer).",fo,De,an=`<a href="/docs/transformers/v4.37.0/en/model_doc/mixtral#transformers.MixtralForSequenceClassification">MixtralForSequenceClassification</a> uses the last token in order to do the classification, as other causal models
(e.g. GPT-2) do.`,go,Oe,sn=`Since it does classification on the last token, it requires to know the position of the last token. If a
<code>pad_token_id</code> is defined in the configuration, it finds the last token that is not a padding token in each row. If
no <code>pad_token_id</code> is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
padding tokens when <code>inputs_embeds</code> are passed instead of <code>input_ids</code>, it does the same (take the last value in
each row of the batch).`,_o,Ke,rn=`This model inherits from <a href="/docs/transformers/v4.37.0/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`,Mo,et,ln=`This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`,bo,U,Ze,vo,tt,dn='The <a href="/docs/transformers/v4.37.0/en/model_doc/mixtral#transformers.MixtralForSequenceClassification">MixtralForSequenceClassification</a> forward method, overrides the <code>__call__</code> special method.',xo,X,Dt,nt,Ot;return J=new C({props:{title:"Mixtral",local:"mixtral",headingTag:"h1"}}),L=new C({props:{title:"Overview",local:"overview",headingTag:"h2"}}),O=new C({props:{title:"Model Details",local:"model-details",headingTag:"h3"}}),se=new C({props:{title:"License",local:"license",headingTag:"h3"}}),ie=new C({props:{title:"Usage tips",local:"usage-tips",headingTag:"h2"}}),ce=new Be({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWRldmljZSUyMCUzRCUyMCUyMmN1ZGElMjIlMjAlMjMlMjB0aGUlMjBkZXZpY2UlMjB0byUyMGxvYWQlMjB0aGUlMjBtb2RlbCUyMG9udG8lMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtaXN0cmFsYWklMkZNaXh0cmFsLTh4N0ItdjAuMSUyMiklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJtaXN0cmFsYWklMkZNaXh0cmFsLTh4N0ItdjAuMSUyMiklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjJNeSUyMGZhdm91cml0ZSUyMGNvbmRpbWVudCUyMGlzJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCU1QnByb21wdCU1RCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKGRldmljZSklMEFtb2RlbC50byhkZXZpY2UpJTBBJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqbW9kZWxfaW5wdXRzJTJDJTIwbWF4X25ld190b2tlbnMlM0QxMDAlMkMlMjBkb19zYW1wbGUlM0RUcnVlKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUoZ2VuZXJhdGVkX2lkcyklNUIwJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span>device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-comment"># the device to load the model onto</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;mistralai/Mixtral-8x7B-v0.1&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;mistralai/Mixtral-8x7B-v0.1&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;My favourite condiment is&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>model_inputs = tokenizer([prompt], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.to(device)

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**model_inputs, max_new_tokens=<span class="hljs-number">100</span>, do_sample=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generated_ids)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;The expected output&quot;</span>`,wrap:!1}}),me=new Be({props:{code:"cHl0aG9uJTIwc3JjJTJGdHJhbnNmb3JtZXJzJTJGbW9kZWxzJTJGbWl4dHJhbCUyRmNvbnZlcnRfbWl4dHJhbF93ZWlnaHRzX3RvX2hmLnB5JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1pbnB1dF9kaXIlMjAlMkZwYXRoJTJGdG8lMkZkb3dubG9hZGVkJTJGbWlzdHJhbCUyRndlaWdodHMlMjAtLW91dHB1dF9kaXIlMjAlMkZvdXRwdXQlMkZwYXRo",highlighted:`python src/transformers/models/mixtral/convert_mixtral_weights_to_hf.py \\
    --input_dir /path/to/downloaded/mistral/weights --output_dir /output/path`,wrap:!1}}),he=new Be({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyME1peHRyYWxGb3JDYXVzYWxMTSUyQyUyMExsYW1hVG9rZW5pemVyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwTGxhbWFUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMiUyRm91dHB1dCUyRnBhdGglMjIpJTBBbW9kZWwlMjAlM0QlMjBNaXh0cmFsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMiUyRm91dHB1dCUyRnBhdGglMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> MixtralForCausalLM, LlamaTokenizer

tokenizer = LlamaTokenizer.from_pretrained(<span class="hljs-string">&quot;/output/path&quot;</span>)
model = MixtralForCausalLM.from_pretrained(<span class="hljs-string">&quot;/output/path&quot;</span>)`,wrap:!1}}),fe=new C({props:{title:"Combining Mixtral and Flash Attention 2",local:"combining-mixtral-and-flash-attention-2",headingTag:"h2"}}),_e=new Be({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwZmxhc2gtYXR0biUyMC0tbm8tYnVpbGQtaXNvbGF0aW9u",highlighted:"pip install -U flash-attn --no-build-isolation",wrap:!1}}),ve=new Be({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBZGV2aWNlJTIwJTNEJTIwJTIyY3VkYSUyMiUyMCUyMyUyMHRoZSUyMGRldmljZSUyMHRvJTIwbG9hZCUyMHRoZSUyMG1vZGVsJTIwb250byUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMm1pc3RyYWxhaSUyRk1peHRyYWwtOHg3Qi12MC4xJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTIwYXR0bl9pbXBsZW1lbnRhdGlvbiUzRCUyMmZsYXNoX2F0dGVudGlvbl8yJTIyKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm1pc3RyYWxhaSUyRk1peHRyYWwtOHg3Qi12MC4xJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMk15JTIwZmF2b3VyaXRlJTIwY29uZGltZW50JTIwaXMlMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTVCcHJvbXB0JTVEJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oZGV2aWNlKSUwQW1vZGVsLnRvKGRldmljZSklMEElMEFnZW5lcmF0ZWRfaWRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKiptb2RlbF9pbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwMCUyQyUyMGRvX3NhbXBsZSUzRFRydWUpJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShnZW5lcmF0ZWRfaWRzKSU1QjAlNUQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span>device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-comment"># the device to load the model onto</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;mistralai/Mixtral-8x7B-v0.1&quot;</span>, torch_dtype=torch.float16, attn_implementation=<span class="hljs-string">&quot;flash_attention_2&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;mistralai/Mixtral-8x7B-v0.1&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;My favourite condiment is&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>model_inputs = tokenizer([prompt], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.to(device)

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**model_inputs, max_new_tokens=<span class="hljs-number">100</span>, do_sample=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(generated_ids)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;The expected output&quot;</span>`,wrap:!1}}),xe=new C({props:{title:"Expected speedups",local:"expected-speedups",headingTag:"h3"}}),Te=new C({props:{title:"Sliding window Attention",local:"sliding-window-attention",headingTag:"h3"}}),$e=new C({props:{title:"The Mistral Team",local:"the-mistral-team",headingTag:"h2"}}),ze=new C({props:{title:"MixtralConfig",local:"transformers.MixtralConfig",headingTag:"h2"}}),Je=new Ge({props:{name:"class transformers.MixtralConfig",anchor:"transformers.MixtralConfig",parameters:[{name:"vocab_size",val:" = 32000"},{name:"hidden_size",val:" = 4096"},{name:"intermediate_size",val:" = 14336"},{name:"num_hidden_layers",val:" = 32"},{name:"num_attention_heads",val:" = 32"},{name:"num_key_value_heads",val:" = 8"},{name:"hidden_act",val:" = 'silu'"},{name:"max_position_embeddings",val:" = 131072"},{name:"initializer_range",val:" = 0.02"},{name:"rms_norm_eps",val:" = 1e-05"},{name:"use_cache",val:" = True"},{name:"pad_token_id",val:" = None"},{name:"bos_token_id",val:" = 1"},{name:"eos_token_id",val:" = 2"},{name:"tie_word_embeddings",val:" = False"},{name:"rope_theta",val:" = 1000000.0"},{name:"sliding_window",val:" = None"},{name:"attention_dropout",val:" = 0.0"},{name:"num_experts_per_tok",val:" = 2"},{name:"num_local_experts",val:" = 8"},{name:"output_router_logits",val:" = False"},{name:"router_aux_loss_coef",val:" = 0.001"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.MixtralConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 32000) &#x2014;
Vocabulary size of the Mixtral model. Defines the number of different tokens that can be represented by the
<code>inputs_ids</code> passed when calling <a href="/docs/transformers/v4.37.0/en/model_doc/mixtral#transformers.MixtralModel">MixtralModel</a>`,name:"vocab_size"},{anchor:"transformers.MixtralConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 4096) &#x2014;
Dimension of the hidden representations.`,name:"hidden_size"},{anchor:"transformers.MixtralConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 14336) &#x2014;
Dimension of the MLP representations.`,name:"intermediate_size"},{anchor:"transformers.MixtralConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.MixtralConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.MixtralConfig.num_key_value_heads",description:`<strong>num_key_value_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 8) &#x2014;
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
<code>num_key_value_heads=num_attention_heads</code>, the model will use Multi Head Attention (MHA), if
<code>num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed by meanpooling all the original heads within that group. For more details checkout [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to </code>8\`.`,name:"num_key_value_heads"},{anchor:"transformers.MixtralConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>function</code>, <em>optional</em>, defaults to <code>&quot;silu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the decoder.`,name:"hidden_act"},{anchor:"transformers.MixtralConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to <code>4096*32</code>) &#x2014;
The maximum sequence length that this model might ever be used with. Mixtral&#x2019;s sliding window attention
allows sequence of up to 4096*32 tokens.`,name:"max_position_embeddings"},{anchor:"transformers.MixtralConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 0.02) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.MixtralConfig.rms_norm_eps",description:`<strong>rms_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-05) &#x2014;
The epsilon used by the rms normalization layers.`,name:"rms_norm_eps"},{anchor:"transformers.MixtralConfig.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if <code>config.is_decoder=True</code>.`,name:"use_cache"},{anchor:"transformers.MixtralConfig.pad_token_id",description:`<strong>pad_token_id</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The id of the padding token.`,name:"pad_token_id"},{anchor:"transformers.MixtralConfig.bos_token_id",description:`<strong>bos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The id of the &#x201C;beginning-of-sequence&#x201D; token.`,name:"bos_token_id"},{anchor:"transformers.MixtralConfig.eos_token_id",description:`<strong>eos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
The id of the &#x201C;end-of-sequence&#x201D; token.`,name:"eos_token_id"},{anchor:"transformers.MixtralConfig.tie_word_embeddings",description:`<strong>tie_word_embeddings</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the model&#x2019;s input and output word embeddings should be tied.`,name:"tie_word_embeddings"},{anchor:"transformers.MixtralConfig.rope_theta",description:`<strong>rope_theta</strong> (<code>float</code>, <em>optional</em>, defaults to 1000000.0) &#x2014;
The base period of the RoPE embeddings.`,name:"rope_theta"},{anchor:"transformers.MixtralConfig.sliding_window",description:`<strong>sliding_window</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Sliding window attention window size. If not specified, will default to <code>4096</code>.`,name:"sliding_window"},{anchor:"transformers.MixtralConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_dropout"},{anchor:"transformers.MixtralConfig.num_experts_per_tok",description:`<strong>num_experts_per_tok</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
The number of experts to root per-token, can be also interpreted as the <code>top-p</code> routing
parameter`,name:"num_experts_per_tok"},{anchor:"transformers.MixtralConfig.num_local_experts",description:`<strong>num_local_experts</strong> (<code>int</code>, <em>optional</em>, defaults to 8) &#x2014;
Number of experts per Sparse MLP layer.`,name:"num_local_experts"},{anchor:"transformers.MixtralConfig.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the router logits should be returned by the model. Enabeling this will also
allow the model to output the auxiliary loss. See <a href>here</a> for more details`,name:"output_router_logits"},{anchor:"transformers.MixtralConfig.router_aux_loss_coef",description:`<strong>router_aux_loss_coef</strong> (<code>float</code>, <em>optional</em>, defaults to 0.001) &#x2014;
The aux loss factor for the total loss.`,name:"router_aux_loss_coef"}],source:"https://github.com/huggingface/transformers/blob/v4.37.0/src/transformers/models/mixtral/configuration_mixtral.py#L28"}}),G=new cn({props:{anchor:"transformers.MixtralConfig.example",$$slots:{default:[_n]},$$scope:{ctx:z}}}),Fe=new C({props:{title:"MixtralModel",local:"transformers.MixtralModel",headingTag:"h2"}}),Le=new Ge({props:{name:"class transformers.MixtralModel",anchor:"transformers.MixtralModel",parameters:[{name:"config",val:": MixtralConfig"}],parametersDescription:[{anchor:"transformers.MixtralModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.37.0/en/model_doc/mixtral#transformers.MixtralConfig">MixtralConfig</a>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/v4.37.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.
config &#x2014; MixtralConfig`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.37.0/src/transformers/models/mixtral/modeling_mixtral.py#L1032"}}),je=new Ge({props:{name:"forward",anchor:"transformers.MixtralModel.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Optional = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"output_router_logits",val:": Optional = None"},{name:"return_dict",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.MixtralModel.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.37.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/v4.37.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.37.0/en/model_doc/fastspeech2_conformer#transformers.FastSpeech2ConformerTokenizer.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.MixtralModel.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a></p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.37.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/v4.37.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.37.0/en/model_doc/fastspeech2_conformer#transformers.FastSpeech2ConformerTokenizer.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>If you want to change padding behavior, you should read <code>modeling_opt._prepare_decoder_attention_mask</code>
and modify to your needs. See diagram 1 in <a href="https://arxiv.org/abs/1910.13461" rel="nofollow">the paper</a> for more
information on the default strategy.</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.MixtralModel.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.MixtralModel.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.MixtralModel.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.MixtralModel.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.MixtralModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.MixtralModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.MixtralModel.forward.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
should not be returned during inference.`,name:"output_router_logits"},{anchor:"transformers.MixtralModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.37.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/v4.37.0/src/transformers/models/mixtral/modeling_mixtral.py#L1068"}}),B=new yo({props:{$$slots:{default:[Mn]},$$scope:{ctx:z}}}),Ie=new C({props:{title:"MixtralForCausalLM",local:"transformers.MixtralForCausalLM",headingTag:"h2"}}),Ue=new Ge({props:{name:"class transformers.MixtralForCausalLM",anchor:"transformers.MixtralForCausalLM",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.37.0/src/transformers/models/mixtral/modeling_mixtral.py#L1231"}}),He=new Ge({props:{name:"forward",anchor:"transformers.MixtralForCausalLM.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Optional = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"labels",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"output_router_logits",val:": Optional = None"},{name:"return_dict",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.MixtralForCausalLM.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.37.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/v4.37.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.37.0/en/model_doc/fastspeech2_conformer#transformers.FastSpeech2ConformerTokenizer.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.MixtralForCausalLM.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a></p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.37.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/v4.37.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.37.0/en/model_doc/fastspeech2_conformer#transformers.FastSpeech2ConformerTokenizer.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>If you want to change padding behavior, you should read <code>modeling_opt._prepare_decoder_attention_mask</code>
and modify to your needs. See diagram 1 in <a href="https://arxiv.org/abs/1910.13461" rel="nofollow">the paper</a> for more
information on the default strategy.</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.MixtralForCausalLM.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.MixtralForCausalLM.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.MixtralForCausalLM.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.MixtralForCausalLM.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.MixtralForCausalLM.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.MixtralForCausalLM.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.MixtralForCausalLM.forward.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
should not be returned during inference.`,name:"output_router_logits"},{anchor:"transformers.MixtralForCausalLM.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.37.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.</p>
<p>Args &#x2014;
labels (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>):
Labels for computing the masked language modeling loss. Indices should either be in <code>[0, ..., config.vocab_size]</code> or -100 (see <code>input_ids</code> docstring). Tokens with indices set to <code>-100</code> are ignored
(masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/v4.37.0/src/transformers/models/mixtral/modeling_mixtral.py#L1263",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>transformers.modeling_outputs.MoeCausalLMOutputWithPast</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/v4.37.0/en/model_doc/mixtral#transformers.MixtralConfig"
>MixtralConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>aux_loss</strong> (<code>torch.FloatTensor</code>, <em>optional</em>, returned when <code>labels</code> is provided) — aux_loss for the sparse modules.</p>
</li>
<li>
<p><strong>router_logits</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_router_probs=True</code> and <code>config.add_router_probs=True</code> is passed or when <code>config.output_router_probs=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, sequence_length, num_experts)</code>.</p>
<p>Raw router logtis (post-softmax) that are computed by MoE routers, these terms are used to compute the auxiliary
loss for Mixture of Experts models.</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>)</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>transformers.modeling_outputs.MoeCausalLMOutputWithPast</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),S=new yo({props:{$$slots:{default:[bn]},$$scope:{ctx:z}}}),E=new cn({props:{anchor:"transformers.MixtralForCausalLM.forward.example",$$slots:{default:[vn]},$$scope:{ctx:z}}}),We=new C({props:{title:"MixtralForSequenceClassification",local:"transformers.MixtralForSequenceClassification",headingTag:"h2"}}),qe=new Ge({props:{name:"class transformers.MixtralForSequenceClassification",anchor:"transformers.MixtralForSequenceClassification",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.MixtralForSequenceClassification.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.37.0/en/model_doc/mixtral#transformers.MixtralConfig">MixtralConfig</a>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/v4.37.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.37.0/src/transformers/models/mixtral/modeling_mixtral.py#L1438"}}),Ze=new Ge({props:{name:"forward",anchor:"transformers.MixtralForSequenceClassification.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Optional = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"labels",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"return_dict",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.MixtralForSequenceClassification.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.37.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/v4.37.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.37.0/en/model_doc/fastspeech2_conformer#transformers.FastSpeech2ConformerTokenizer.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.MixtralForSequenceClassification.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a></p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.37.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/v4.37.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.37.0/en/model_doc/fastspeech2_conformer#transformers.FastSpeech2ConformerTokenizer.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>If you want to change padding behavior, you should read <code>modeling_opt._prepare_decoder_attention_mask</code>
and modify to your needs. See diagram 1 in <a href="https://arxiv.org/abs/1910.13461" rel="nofollow">the paper</a> for more
information on the default strategy.</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.MixtralForSequenceClassification.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.MixtralForSequenceClassification.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.MixtralForSequenceClassification.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.MixtralForSequenceClassification.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.MixtralForSequenceClassification.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.MixtralForSequenceClassification.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.MixtralForSequenceClassification.forward.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
should not be returned during inference.`,name:"output_router_logits"},{anchor:"transformers.MixtralForSequenceClassification.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.37.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.MixtralForSequenceClassification.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) &#x2014;
Labels for computing the sequence classification/regression loss. Indices should be in <code>[0, ..., config.num_labels - 1]</code>. If <code>config.num_labels == 1</code> a regression loss is computed (Mean-Square loss), If
<code>config.num_labels &gt; 1</code> a classification loss is computed (Cross-Entropy).`,name:"labels"}],source:"https://github.com/huggingface/transformers/blob/v4.37.0/src/transformers/models/mixtral/modeling_mixtral.py#L1470"}}),X=new yo({props:{$$slots:{default:[xn]},$$scope:{ctx:z}}}),{c(){l=r("meta"),M=a(),p=r("p"),b=a(),m(J.$$.fragment),v=a(),m(L.$$.fragment),st=a(),P=r("p"),P.textContent=To,rt=a(),V=r("p"),V.innerHTML=wo,it=a(),N=r("p"),N.innerHTML=ko,lt=a(),A=r("p"),A.innerHTML=$o,dt=a(),Y=r("p"),Y.textContent=Co,ct=a(),Q=r("ul"),Q.innerHTML=zo,pt=a(),D=r("p"),D.innerHTML=Jo,mt=a(),m(O.$$.fragment),ut=a(),K=r("p"),K.textContent=Fo,ht=a(),ee=r("ul"),ee.innerHTML=Lo,ft=a(),te=r("p"),te.innerHTML=jo,gt=a(),oe=r("ul"),oe.innerHTML=Io,_t=a(),ne=r("p"),ne.innerHTML=Uo,Mt=a(),ae=r("p"),ae.innerHTML=Ho,bt=a(),m(se.$$.fragment),vt=a(),re=r("p"),re.innerHTML=Wo,xt=a(),m(ie.$$.fragment),yt=a(),le=r("p"),le.innerHTML=qo,Tt=a(),de=r("p"),de.textContent=Zo,wt=a(),m(ce.$$.fragment),kt=a(),pe=r("p"),pe.innerHTML=Ro,$t=a(),m(me.$$.fragment),Ct=a(),ue=r("p"),ue.innerHTML=Go,zt=a(),m(he.$$.fragment),Jt=a(),m(fe.$$.fragment),Ft=a(),ge=r("p"),ge.textContent=Bo,Lt=a(),m(_e.$$.fragment),jt=a(),Me=r("p"),Me.innerHTML=So,It=a(),be=r("p"),be.textContent=Eo,Ut=a(),m(ve.$$.fragment),Ht=a(),m(xe.$$.fragment),Wt=a(),ye=r("p"),ye.innerHTML=Xo,qt=a(),R=r("div"),R.innerHTML=Po,Zt=a(),m(Te.$$.fragment),Rt=a(),we=r("p"),we.innerHTML=Vo,Gt=a(),ke=r("p"),ke.innerHTML=No,Bt=a(),m($e.$$.fragment),St=a(),Ce=r("p"),Ce.textContent=Ao,Et=a(),m(ze.$$.fragment),Xt=a(),y=r("div"),m(Je.$$.fragment),Kt=a(),Se=r("p"),Se.innerHTML=Yo,eo=a(),Ee=r("p"),Ee.innerHTML=Qo,to=a(),Xe=r("p"),Xe.innerHTML=Do,oo=a(),m(G.$$.fragment),Pt=a(),m(Fe.$$.fragment),Vt=a(),T=r("div"),m(Le.$$.fragment),no=a(),Pe=r("p"),Pe.innerHTML=Oo,ao=a(),Ve=r("p"),Ve.innerHTML=Ko,so=a(),Ne=r("p"),Ne.innerHTML=en,ro=a(),I=r("div"),m(je.$$.fragment),io=a(),Ae=r("p"),Ae.innerHTML=tn,lo=a(),m(B.$$.fragment),Nt=a(),m(Ie.$$.fragment),At=a(),H=r("div"),m(Ue.$$.fragment),co=a(),F=r("div"),m(He.$$.fragment),po=a(),Ye=r("p"),Ye.innerHTML=on,mo=a(),m(S.$$.fragment),uo=a(),m(E.$$.fragment),Yt=a(),m(We.$$.fragment),Qt=a(),x=r("div"),m(qe.$$.fragment),ho=a(),Qe=r("p"),Qe.textContent=nn,fo=a(),De=r("p"),De.innerHTML=an,go=a(),Oe=r("p"),Oe.innerHTML=sn,_o=a(),Ke=r("p"),Ke.innerHTML=rn,Mo=a(),et=r("p"),et.innerHTML=ln,bo=a(),U=r("div"),m(Ze.$$.fragment),vo=a(),tt=r("p"),tt.innerHTML=dn,xo=a(),m(X.$$.fragment),Dt=a(),nt=r("p"),this.h()},l(e){const t=fn("svelte-u9bgzb",document.head);l=i(t,"META",{name:!0,content:!0}),t.forEach(o),M=s(e),p=i(e,"P",{}),q(p).forEach(o),b=s(e),u(J.$$.fragment,e),v=s(e),u(L.$$.fragment,e),st=s(e),P=i(e,"P",{"data-svelte-h":!0}),d(P)!=="svelte-zq46db"&&(P.textContent=To),rt=s(e),V=i(e,"P",{"data-svelte-h":!0}),d(V)!=="svelte-14p3qzk"&&(V.innerHTML=wo),it=s(e),N=i(e,"P",{"data-svelte-h":!0}),d(N)!=="svelte-1ktqbtz"&&(N.innerHTML=ko),lt=s(e),A=i(e,"P",{"data-svelte-h":!0}),d(A)!=="svelte-1xjcuti"&&(A.innerHTML=$o),dt=s(e),Y=i(e,"P",{"data-svelte-h":!0}),d(Y)!=="svelte-axv494"&&(Y.textContent=Co),ct=s(e),Q=i(e,"UL",{"data-svelte-h":!0}),d(Q)!=="svelte-5f6zp0"&&(Q.innerHTML=zo),pt=s(e),D=i(e,"P",{"data-svelte-h":!0}),d(D)!=="svelte-1wxpeky"&&(D.innerHTML=Jo),mt=s(e),u(O.$$.fragment,e),ut=s(e),K=i(e,"P",{"data-svelte-h":!0}),d(K)!=="svelte-mjrhha"&&(K.textContent=Fo),ht=s(e),ee=i(e,"UL",{"data-svelte-h":!0}),d(ee)!=="svelte-1qpxs3v"&&(ee.innerHTML=Lo),ft=s(e),te=i(e,"P",{"data-svelte-h":!0}),d(te)!=="svelte-1a5n521"&&(te.innerHTML=jo),gt=s(e),oe=i(e,"UL",{"data-svelte-h":!0}),d(oe)!=="svelte-hqpplt"&&(oe.innerHTML=Io),_t=s(e),ne=i(e,"P",{"data-svelte-h":!0}),d(ne)!=="svelte-1h0nnyl"&&(ne.innerHTML=Uo),Mt=s(e),ae=i(e,"P",{"data-svelte-h":!0}),d(ae)!=="svelte-mf7428"&&(ae.innerHTML=Ho),bt=s(e),u(se.$$.fragment,e),vt=s(e),re=i(e,"P",{"data-svelte-h":!0}),d(re)!=="svelte-ne80pp"&&(re.innerHTML=Wo),xt=s(e),u(ie.$$.fragment,e),yt=s(e),le=i(e,"P",{"data-svelte-h":!0}),d(le)!=="svelte-jo1i4l"&&(le.innerHTML=qo),Tt=s(e),de=i(e,"P",{"data-svelte-h":!0}),d(de)!=="svelte-a5tlwb"&&(de.textContent=Zo),wt=s(e),u(ce.$$.fragment,e),kt=s(e),pe=i(e,"P",{"data-svelte-h":!0}),d(pe)!=="svelte-1osfv2n"&&(pe.innerHTML=Ro),$t=s(e),u(me.$$.fragment,e),Ct=s(e),ue=i(e,"P",{"data-svelte-h":!0}),d(ue)!=="svelte-1svz3xz"&&(ue.innerHTML=Go),zt=s(e),u(he.$$.fragment,e),Jt=s(e),u(fe.$$.fragment,e),Ft=s(e),ge=i(e,"P",{"data-svelte-h":!0}),d(ge)!=="svelte-o3pzzu"&&(ge.textContent=Bo),Lt=s(e),u(_e.$$.fragment,e),jt=s(e),Me=i(e,"P",{"data-svelte-h":!0}),d(Me)!=="svelte-qk7tod"&&(Me.innerHTML=So),It=s(e),be=i(e,"P",{"data-svelte-h":!0}),d(be)!=="svelte-14hchid"&&(be.textContent=Eo),Ut=s(e),u(ve.$$.fragment,e),Ht=s(e),u(xe.$$.fragment,e),Wt=s(e),ye=i(e,"P",{"data-svelte-h":!0}),d(ye)!=="svelte-1jfvy1t"&&(ye.innerHTML=Xo),qt=s(e),R=i(e,"DIV",{style:!0,"data-svelte-h":!0}),d(R)!=="svelte-lcfs6y"&&(R.innerHTML=Po),Zt=s(e),u(Te.$$.fragment,e),Rt=s(e),we=i(e,"P",{"data-svelte-h":!0}),d(we)!=="svelte-10i6fhp"&&(we.innerHTML=Vo),Gt=s(e),ke=i(e,"P",{"data-svelte-h":!0}),d(ke)!=="svelte-1bvsrfr"&&(ke.innerHTML=No),Bt=s(e),u($e.$$.fragment,e),St=s(e),Ce=i(e,"P",{"data-svelte-h":!0}),d(Ce)!=="svelte-15xllx1"&&(Ce.textContent=Ao),Et=s(e),u(ze.$$.fragment,e),Xt=s(e),y=i(e,"DIV",{class:!0});var k=q(y);u(Je.$$.fragment,k),Kt=s(k),Se=i(k,"P",{"data-svelte-h":!0}),d(Se)!=="svelte-5s66xo"&&(Se.innerHTML=Yo),eo=s(k),Ee=i(k,"P",{"data-svelte-h":!0}),d(Ee)!=="svelte-1nulibo"&&(Ee.innerHTML=Qo),to=s(k),Xe=i(k,"P",{"data-svelte-h":!0}),d(Xe)!=="svelte-1p8c3o7"&&(Xe.innerHTML=Do),oo=s(k),u(G.$$.fragment,k),k.forEach(o),Pt=s(e),u(Fe.$$.fragment,e),Vt=s(e),T=i(e,"DIV",{class:!0});var $=q(T);u(Le.$$.fragment,$),no=s($),Pe=i($,"P",{"data-svelte-h":!0}),d(Pe)!=="svelte-mts2mk"&&(Pe.innerHTML=Oo),ao=s($),Ve=i($,"P",{"data-svelte-h":!0}),d(Ve)!=="svelte-hswkmf"&&(Ve.innerHTML=Ko),so=s($),Ne=i($,"P",{"data-svelte-h":!0}),d(Ne)!=="svelte-1g9es5k"&&(Ne.innerHTML=en),ro=s($),I=i($,"DIV",{class:!0});var W=q(I);u(je.$$.fragment,W),io=s(W),Ae=i(W,"P",{"data-svelte-h":!0}),d(Ae)!=="svelte-bdb3sb"&&(Ae.innerHTML=tn),lo=s(W),u(B.$$.fragment,W),W.forEach(o),$.forEach(o),Nt=s(e),u(Ie.$$.fragment,e),At=s(e),H=i(e,"DIV",{class:!0});var Re=q(H);u(Ue.$$.fragment,Re),co=s(Re),F=i(Re,"DIV",{class:!0});var j=q(F);u(He.$$.fragment,j),po=s(j),Ye=i(j,"P",{"data-svelte-h":!0}),d(Ye)!=="svelte-fy6mrv"&&(Ye.innerHTML=on),mo=s(j),u(S.$$.fragment,j),uo=s(j),u(E.$$.fragment,j),j.forEach(o),Re.forEach(o),Yt=s(e),u(We.$$.fragment,e),Qt=s(e),x=i(e,"DIV",{class:!0});var w=q(x);u(qe.$$.fragment,w),ho=s(w),Qe=i(w,"P",{"data-svelte-h":!0}),d(Qe)!=="svelte-1ttin8p"&&(Qe.textContent=nn),fo=s(w),De=i(w,"P",{"data-svelte-h":!0}),d(De)!=="svelte-1qqir7w"&&(De.innerHTML=an),go=s(w),Oe=i(w,"P",{"data-svelte-h":!0}),d(Oe)!=="svelte-10ugs3m"&&(Oe.innerHTML=sn),_o=s(w),Ke=i(w,"P",{"data-svelte-h":!0}),d(Ke)!=="svelte-mec8jz"&&(Ke.innerHTML=rn),Mo=s(w),et=i(w,"P",{"data-svelte-h":!0}),d(et)!=="svelte-hswkmf"&&(et.innerHTML=ln),bo=s(w),U=i(w,"DIV",{class:!0});var ot=q(U);u(Ze.$$.fragment,ot),vo=s(ot),tt=i(ot,"P",{"data-svelte-h":!0}),d(tt)!=="svelte-1yxc025"&&(tt.innerHTML=dn),xo=s(ot),u(X.$$.fragment,ot),ot.forEach(o),w.forEach(o),Dt=s(e),nt=i(e,"P",{}),q(nt).forEach(o),this.h()},h(){Z(l,"name","hf:doc:metadata"),Z(l,"content",Tn),gn(R,"text-align","center"),Z(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),Z(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){c(document.head,l),n(e,M,t),n(e,p,t),n(e,b,t),h(J,e,t),n(e,v,t),h(L,e,t),n(e,st,t),n(e,P,t),n(e,rt,t),n(e,V,t),n(e,it,t),n(e,N,t),n(e,lt,t),n(e,A,t),n(e,dt,t),n(e,Y,t),n(e,ct,t),n(e,Q,t),n(e,pt,t),n(e,D,t),n(e,mt,t),h(O,e,t),n(e,ut,t),n(e,K,t),n(e,ht,t),n(e,ee,t),n(e,ft,t),n(e,te,t),n(e,gt,t),n(e,oe,t),n(e,_t,t),n(e,ne,t),n(e,Mt,t),n(e,ae,t),n(e,bt,t),h(se,e,t),n(e,vt,t),n(e,re,t),n(e,xt,t),h(ie,e,t),n(e,yt,t),n(e,le,t),n(e,Tt,t),n(e,de,t),n(e,wt,t),h(ce,e,t),n(e,kt,t),n(e,pe,t),n(e,$t,t),h(me,e,t),n(e,Ct,t),n(e,ue,t),n(e,zt,t),h(he,e,t),n(e,Jt,t),h(fe,e,t),n(e,Ft,t),n(e,ge,t),n(e,Lt,t),h(_e,e,t),n(e,jt,t),n(e,Me,t),n(e,It,t),n(e,be,t),n(e,Ut,t),h(ve,e,t),n(e,Ht,t),h(xe,e,t),n(e,Wt,t),n(e,ye,t),n(e,qt,t),n(e,R,t),n(e,Zt,t),h(Te,e,t),n(e,Rt,t),n(e,we,t),n(e,Gt,t),n(e,ke,t),n(e,Bt,t),h($e,e,t),n(e,St,t),n(e,Ce,t),n(e,Et,t),h(ze,e,t),n(e,Xt,t),n(e,y,t),h(Je,y,null),c(y,Kt),c(y,Se),c(y,eo),c(y,Ee),c(y,to),c(y,Xe),c(y,oo),h(G,y,null),n(e,Pt,t),h(Fe,e,t),n(e,Vt,t),n(e,T,t),h(Le,T,null),c(T,no),c(T,Pe),c(T,ao),c(T,Ve),c(T,so),c(T,Ne),c(T,ro),c(T,I),h(je,I,null),c(I,io),c(I,Ae),c(I,lo),h(B,I,null),n(e,Nt,t),h(Ie,e,t),n(e,At,t),n(e,H,t),h(Ue,H,null),c(H,co),c(H,F),h(He,F,null),c(F,po),c(F,Ye),c(F,mo),h(S,F,null),c(F,uo),h(E,F,null),n(e,Yt,t),h(We,e,t),n(e,Qt,t),n(e,x,t),h(qe,x,null),c(x,ho),c(x,Qe),c(x,fo),c(x,De),c(x,go),c(x,Oe),c(x,_o),c(x,Ke),c(x,Mo),c(x,et),c(x,bo),c(x,U),h(Ze,U,null),c(U,vo),c(U,tt),c(U,xo),h(X,U,null),n(e,Dt,t),n(e,nt,t),Ot=!0},p(e,[t]){const k={};t&2&&(k.$$scope={dirty:t,ctx:e}),G.$set(k);const $={};t&2&&($.$$scope={dirty:t,ctx:e}),B.$set($);const W={};t&2&&(W.$$scope={dirty:t,ctx:e}),S.$set(W);const Re={};t&2&&(Re.$$scope={dirty:t,ctx:e}),E.$set(Re);const j={};t&2&&(j.$$scope={dirty:t,ctx:e}),X.$set(j)},i(e){Ot||(f(J.$$.fragment,e),f(L.$$.fragment,e),f(O.$$.fragment,e),f(se.$$.fragment,e),f(ie.$$.fragment,e),f(ce.$$.fragment,e),f(me.$$.fragment,e),f(he.$$.fragment,e),f(fe.$$.fragment,e),f(_e.$$.fragment,e),f(ve.$$.fragment,e),f(xe.$$.fragment,e),f(Te.$$.fragment,e),f($e.$$.fragment,e),f(ze.$$.fragment,e),f(Je.$$.fragment,e),f(G.$$.fragment,e),f(Fe.$$.fragment,e),f(Le.$$.fragment,e),f(je.$$.fragment,e),f(B.$$.fragment,e),f(Ie.$$.fragment,e),f(Ue.$$.fragment,e),f(He.$$.fragment,e),f(S.$$.fragment,e),f(E.$$.fragment,e),f(We.$$.fragment,e),f(qe.$$.fragment,e),f(Ze.$$.fragment,e),f(X.$$.fragment,e),Ot=!0)},o(e){g(J.$$.fragment,e),g(L.$$.fragment,e),g(O.$$.fragment,e),g(se.$$.fragment,e),g(ie.$$.fragment,e),g(ce.$$.fragment,e),g(me.$$.fragment,e),g(he.$$.fragment,e),g(fe.$$.fragment,e),g(_e.$$.fragment,e),g(ve.$$.fragment,e),g(xe.$$.fragment,e),g(Te.$$.fragment,e),g($e.$$.fragment,e),g(ze.$$.fragment,e),g(Je.$$.fragment,e),g(G.$$.fragment,e),g(Fe.$$.fragment,e),g(Le.$$.fragment,e),g(je.$$.fragment,e),g(B.$$.fragment,e),g(Ie.$$.fragment,e),g(Ue.$$.fragment,e),g(He.$$.fragment,e),g(S.$$.fragment,e),g(E.$$.fragment,e),g(We.$$.fragment,e),g(qe.$$.fragment,e),g(Ze.$$.fragment,e),g(X.$$.fragment,e),Ot=!1},d(e){e&&(o(M),o(p),o(b),o(v),o(st),o(P),o(rt),o(V),o(it),o(N),o(lt),o(A),o(dt),o(Y),o(ct),o(Q),o(pt),o(D),o(mt),o(ut),o(K),o(ht),o(ee),o(ft),o(te),o(gt),o(oe),o(_t),o(ne),o(Mt),o(ae),o(bt),o(vt),o(re),o(xt),o(yt),o(le),o(Tt),o(de),o(wt),o(kt),o(pe),o($t),o(Ct),o(ue),o(zt),o(Jt),o(Ft),o(ge),o(Lt),o(jt),o(Me),o(It),o(be),o(Ut),o(Ht),o(Wt),o(ye),o(qt),o(R),o(Zt),o(Rt),o(we),o(Gt),o(ke),o(Bt),o(St),o(Ce),o(Et),o(Xt),o(y),o(Pt),o(Vt),o(T),o(Nt),o(At),o(H),o(Yt),o(Qt),o(x),o(Dt),o(nt)),o(l),_(J,e),_(L,e),_(O,e),_(se,e),_(ie,e),_(ce,e),_(me,e),_(he,e),_(fe,e),_(_e,e),_(ve,e),_(xe,e),_(Te,e),_($e,e),_(ze,e),_(Je),_(G),_(Fe,e),_(Le),_(je),_(B),_(Ie,e),_(Ue),_(He),_(S),_(E),_(We,e),_(qe),_(Ze),_(X)}}}const Tn='{"title":"Mixtral","local":"mixtral","sections":[{"title":"Overview","local":"overview","sections":[{"title":"Model Details","local":"model-details","sections":[],"depth":3},{"title":"License","local":"license","sections":[],"depth":3}],"depth":2},{"title":"Usage tips","local":"usage-tips","sections":[],"depth":2},{"title":"Combining Mixtral and Flash Attention 2","local":"combining-mixtral-and-flash-attention-2","sections":[{"title":"Expected speedups","local":"expected-speedups","sections":[],"depth":3},{"title":"Sliding window Attention","local":"sliding-window-attention","sections":[],"depth":3}],"depth":2},{"title":"The Mistral Team","local":"the-mistral-team","sections":[],"depth":2},{"title":"MixtralConfig","local":"transformers.MixtralConfig","sections":[],"depth":2},{"title":"MixtralModel","local":"transformers.MixtralModel","sections":[],"depth":2},{"title":"MixtralForCausalLM","local":"transformers.MixtralForCausalLM","sections":[],"depth":2},{"title":"MixtralForSequenceClassification","local":"transformers.MixtralForSequenceClassification","sections":[],"depth":2}],"depth":1}';function wn(z){return mn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jn extends un{constructor(l){super(),hn(this,l,wn,yn,pn,{})}}export{jn as component};
