import{s as ea,o as ta,n as Yt}from"../chunks/scheduler.4cd3e004.js";import{S as aa,i as na,e as i,s as n,c as p,h as oa,a as l,d as a,b as o,f as $,g as c,j as _,k as L,l as r,m as s,n as m,t as f,o as h,p as g}from"../chunks/index.b664cb2d.js";import{D as q}from"../chunks/Docstring.9ceb4334.js";import{C as Kt}from"../chunks/CodeBlock.8601e34e.js";import{E as Xt}from"../chunks/ExampleCodeBlock.4f57b575.js";import{H as E,E as ra}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.179b63f8.js";function sa(ce){let d,T="Example:",b,M,x;return M=new Kt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTSUwQWZyb20lMjBwZWZ0JTIwaW1wb3J0JTIwTG9yYU1vZGVsJTJDJTIwTG9yYUNvbmZpZyUwQSUwQWNvbmZpZyUyMCUzRCUyMExvcmFDb25maWcoJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyU0VRXzJfU0VRX0xNJTIyJTJDJTBBJTIwJTIwJTIwJTIwciUzRDglMkMlMEElMjAlMjAlMjAlMjBsb3JhX2FscGhhJTNEMzIlMkMlMEElMjAlMjAlMjAlMjB0YXJnZXRfbW9kdWxlcyUzRCU1QiUyMnElMjIlMkMlMjAlMjJ2JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwbG9yYV9kcm9wb3V0JTNEMC4wMSUyQyUwQSklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTS5mcm9tX3ByZXRyYWluZWQoJTIydDUtYmFzZSUyMiklMEFsb3JhX21vZGVsJTIwJTNEJTIwTG9yYU1vZGVsKG1vZGVsJTJDJTIwY29uZmlnJTJDJTIwJTIyZGVmYXVsdCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraModel, LoraConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>config = LoraConfig(
<span class="hljs-meta">... </span>    task_type=<span class="hljs-string">&quot;SEQ_2_SEQ_LM&quot;</span>,
<span class="hljs-meta">... </span>    r=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    lora_alpha=<span class="hljs-number">32</span>,
<span class="hljs-meta">... </span>    target_modules=[<span class="hljs-string">&quot;q&quot;</span>, <span class="hljs-string">&quot;v&quot;</span>],
<span class="hljs-meta">... </span>    lora_dropout=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;t5-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>lora_model = LoraModel(model, config, <span class="hljs-string">&quot;default&quot;</span>)`,wrap:!1}}),{c(){d=i("p"),d.textContent=T,b=n(),p(M.$$.fragment)},l(v){d=l(v,"P",{"data-svelte-h":!0}),_(d)!=="svelte-11lpom8"&&(d.textContent=T),b=o(v),c(M.$$.fragment,v)},m(v,k){s(v,d,k),s(v,b,k),m(M,v,k),x=!0},p:Yt,i(v){x||(f(M.$$.fragment,v),x=!0)},o(v){h(M.$$.fragment,v),x=!1},d(v){v&&(a(d),a(b)),g(M,v)}}}function ia(ce){let d,T;return d=new Kt({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0cmFuc2Zvcm1lcnMlMEFmcm9tJTIwcGVmdCUyMGltcG9ydCUyMExvcmFDb25maWclMkMlMjBQZWZ0TW9kZWwlMkMlMjBnZXRfcGVmdF9tb2RlbCUyQyUyMHByZXBhcmVfbW9kZWxfZm9yX2tiaXRfdHJhaW5pbmclMEElMEFyYW5rJTIwJTNEJTIwLi4uJTBBdGFyZ2V0X21vZHVsZXMlMjAlM0QlMjAlNUIlMjJxX3Byb2olMjIlMkMlMjAlMjJrX3Byb2olMjIlMkMlMjAlMjJ2X3Byb2olMjIlMkMlMjAlMjJvdXRfcHJvaiUyMiUyQyUyMCUyMmZjX2luJTIyJTJDJTIwJTIyZmNfb3V0JTIyJTJDJTIwJTIyd3RlJTIyJTVEJTBBY29uZmlnJTIwJTNEJTIwTG9yYUNvbmZpZyglMEElMjAlMjAlMjAlMjByJTNENCUyQyUyMGxvcmFfYWxwaGElM0QxNiUyQyUyMHRhcmdldF9tb2R1bGVzJTNEdGFyZ2V0X21vZHVsZXMlMkMlMjBsb3JhX2Ryb3BvdXQlM0QwLjElMkMlMjBiaWFzJTNEJTIybm9uZSUyMiUyQyUyMHRhc2tfdHlwZSUzRCUyMkNBVVNBTF9MTSUyMiUwQSklMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwdHJhbnNmb3JtZXJzLkJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzhiaXQlM0RUcnVlKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMHRyYW5zZm9ybWVycy5BdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJrYWthb2JyYWluJTJGa29ncHQlMjIlMkMlMEElMjAlMjAlMjAlMjByZXZpc2lvbiUzRCUyMktvR1BUNkItcnlhbjEuNWItZmxvYXQxNiUyMiUyQyUyMCUyMCUyMyUyMG9yJTIwZmxvYXQzMiUyMHZlcnNpb24lM0ElMjByZXZpc2lvbiUzREtvR1BUNkItcnlhbjEuNWIlMEElMjAlMjAlMjAlMjBib3NfdG9rZW4lM0QlMjIlNUJCT1MlNUQlMjIlMkMlMEElMjAlMjAlMjAlMjBlb3NfdG9rZW4lM0QlMjIlNUJFT1MlNUQlMjIlMkMlMEElMjAlMjAlMjAlMjB1bmtfdG9rZW4lM0QlMjIlNUJVTkslNUQlMjIlMkMlMEElMjAlMjAlMjAlMjBwYWRfdG9rZW4lM0QlMjIlNUJQQUQlNUQlMjIlMkMlMEElMjAlMjAlMjAlMjBtYXNrX3Rva2VuJTNEJTIyJTVCTUFTSyU1RCUyMiUyQyUwQSklMEFtb2RlbCUyMCUzRCUyMHRyYW5zZm9ybWVycy5HUFRKRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmtha2FvYnJhaW4lMkZrb2dwdCUyMiUyQyUwQSUyMCUyMCUyMCUyMHJldmlzaW9uJTNEJTIyS29HUFQ2Qi1yeWFuMS41Yi1mbG9hdDE2JTIyJTJDJTIwJTIwJTIzJTIwb3IlMjBmbG9hdDMyJTIwdmVyc2lvbiUzQSUyMHJldmlzaW9uJTNES29HUFQ2Qi1yeWFuMS41YiUwQSUyMCUyMCUyMCUyMHBhZF90b2tlbl9pZCUzRHRva2VuaXplci5lb3NfdG9rZW5faWQlMkMlMEElMjAlMjAlMjAlMjB1c2VfY2FjaGUlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlN0IlMjIlMjIlM0ElMjByYW5rJTdEJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEEpJTBBbW9kZWwlMjAlM0QlMjBwcmVwYXJlX21vZGVsX2Zvcl9rYml0X3RyYWluaW5nKG1vZGVsKSUwQWxvcmFfbW9kZWwlMjAlM0QlMjBnZXRfcGVmdF9tb2RlbChtb2RlbCUyQyUyMGNvbmZpZyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> transformers
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig, PeftModel, get_peft_model, prepare_model_for_kbit_training

<span class="hljs-meta">&gt;&gt;&gt; </span>rank = ...
<span class="hljs-meta">&gt;&gt;&gt; </span>target_modules = [<span class="hljs-string">&quot;q_proj&quot;</span>, <span class="hljs-string">&quot;k_proj&quot;</span>, <span class="hljs-string">&quot;v_proj&quot;</span>, <span class="hljs-string">&quot;out_proj&quot;</span>, <span class="hljs-string">&quot;fc_in&quot;</span>, <span class="hljs-string">&quot;fc_out&quot;</span>, <span class="hljs-string">&quot;wte&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>config = LoraConfig(
<span class="hljs-meta">... </span>    r=<span class="hljs-number">4</span>, lora_alpha=<span class="hljs-number">16</span>, target_modules=target_modules, lora_dropout=<span class="hljs-number">0.1</span>, bias=<span class="hljs-string">&quot;none&quot;</span>, task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>quantization_config = transformers.BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = transformers.AutoTokenizer.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;kakaobrain/kogpt&quot;</span>,
<span class="hljs-meta">... </span>    revision=<span class="hljs-string">&quot;KoGPT6B-ryan1.5b-float16&quot;</span>,  <span class="hljs-comment"># or float32 version: revision=KoGPT6B-ryan1.5b</span>
<span class="hljs-meta">... </span>    bos_token=<span class="hljs-string">&quot;[BOS]&quot;</span>,
<span class="hljs-meta">... </span>    eos_token=<span class="hljs-string">&quot;[EOS]&quot;</span>,
<span class="hljs-meta">... </span>    unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>,
<span class="hljs-meta">... </span>    pad_token=<span class="hljs-string">&quot;[PAD]&quot;</span>,
<span class="hljs-meta">... </span>    mask_token=<span class="hljs-string">&quot;[MASK]&quot;</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = transformers.GPTJForCausalLM.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;kakaobrain/kogpt&quot;</span>,
<span class="hljs-meta">... </span>    revision=<span class="hljs-string">&quot;KoGPT6B-ryan1.5b-float16&quot;</span>,  <span class="hljs-comment"># or float32 version: revision=KoGPT6B-ryan1.5b</span>
<span class="hljs-meta">... </span>    pad_token_id=tokenizer.eos_token_id,
<span class="hljs-meta">... </span>    use_cache=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    device_map={<span class="hljs-string">&quot;&quot;</span>: rank},
<span class="hljs-meta">... </span>    torch_dtype=torch.float16,
<span class="hljs-meta">... </span>    quantization_config=quantization_config,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = prepare_model_for_kbit_training(model)
<span class="hljs-meta">&gt;&gt;&gt; </span>lora_model = get_peft_model(model, config)`,wrap:!1}}),{c(){p(d.$$.fragment)},l(b){c(d.$$.fragment,b)},m(b,M){m(d,b,M),T=!0},p:Yt,i(b){T||(f(d.$$.fragment,b),T=!0)},o(b){h(d.$$.fragment,b),T=!1},d(b){g(d,b)}}}function la(ce){let d,T,b,M,x,v,k,jt='Low-Rank Adaptation (<a href="https://huggingface.co/papers/2309.15223" rel="nofollow">LoRA</a>) is a PEFT method that decomposes a large matrix into two smaller low-rank matrices in the attention layers. This drastically reduces the number of parameters that need to be fine-tuned.',Re,F,At="The abstract from the paper is:",Ie,Z,Ut="<em>We propose a neural language modeling system based on low-rank adaptation (LoRA) for speech recognition output rescoring. Although pretrained language models (LMs) like BERT have shown superior performance in second-pass rescoring, the high computational cost of scaling up the pretraining stage and adapting the pretrained models to specific domains limit their practical use in rescoring. Here we present a method based on low-rank decomposition to train a rescoring BERT model and adapt it to new domains using only a fraction (0.08%) of the pretrained parameters. These inserted matrices are optimized through a discriminative training objective along with a correlation-based regularization loss. The proposed low-rank adaptation Rescore-BERT (LoRB) architecture is evaluated on LibriSpeech and internal datasets with decreased training times by factors between 5.4 and 3.6.</em>.",De,P,Ne,C,B,rt,me,Jt='This is the configuration class to store the configuration of a <a href="/docs/peft/main/en/package_reference/lora#peft.LoraModel">LoraModel</a>.',st,N,W,it,fe,qt="Returns the configuration for your adapter model as a dictionary. Removes runtime configurations.",Se,Q,ze,u,O,lt,he,Et="Creates Low Rank Adapter (LoRA) model from a pretrained transformers model.",dt,ge,Rt='The method is described in detail in <a href="https://huggingface.co/papers/2106.09685" rel="nofollow">https://huggingface.co/papers/2106.09685</a>.',pt,S,ct,z,mt,ue,It="<strong>Attributes</strong>:",ft,_e,Dt='<li><strong>model</strong> (<a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel" rel="nofollow">PreTrainedModel</a>) — The model to be adapted.</li> <li><strong>peft_config</strong> (<a href="/docs/peft/main/en/package_reference/lora#peft.LoraConfig">LoraConfig</a>): The configuration of the Lora model.</li>',ht,U,G,gt,be,Nt="This method adds a new adapter by merging the given adapters with the given weights.",ut,ve,St=`When using the <code>cat</code> combination_type you should be aware that rank of the resulting adapter will be equal to
the sum of all adapters ranks. So it’s possible that the mixed adapter may become too big and result in OOM
errors.`,_t,V,H,bt,ye,zt=`This function can calculate the updates of the PiSSA/CorDA/OLoRA by comparing the parameters of the
PiSSA/CorDA/OLoRA adapter in <code>output_state_dict</code> with the initial values of PiSSA/CorDA/OLoRA in
<code>adapter_name</code>, thus converting PiSSA/CorDA/OLoRA to LoRA.`,Ve,X,Fe,Y,Ze,R,K,vt,we,Vt=`This is the sub-configuration class to store the configuration for Arrow and GenKnowSub algorithm. Arrow is a
routing algorithm to combine the trained LoRA modules to solve new tasks, proposed in
’<a href="https://arxiv.org/pdf/2405.11157&#39;" rel="nofollow">https://arxiv.org/pdf/2405.11157’</a>. GenKnowSub is a refinement on the trained modules before being combined via
Arrow, introduced in ’<a href="https://aclanthology.org/2025.acl-short.54/&#39;" rel="nofollow">https://aclanthology.org/2025.acl-short.54/’</a>`,Pe,ee,Be,w,te,yt,Me,Ft="Replace the LoRA weights of a model quantized with bitsandbytes, using the LoftQ technique.",wt,Te,Zt=`The replacement is done on the fly by loading in the non-quantized weights from a locally stored safetensors model
file and initializing the LoRA weights such that the quantization error between the original and quantized weights
is minimized.`,Mt,xe,Pt="As lazy loading is not possible with pickle, normal PyTorch checkpoint files cannot be supported.",Tt,$e,Bt="Depending on the model size, calling this function may take some time to finish.",We,ae,Qe,ne,Oe,I,oe,xt,Le,Wt=`This is the sub-configuration class to store the configuration for a data-driven initialization via EVA. EVA was
introduced in <a href="https://huggingface.co/papers/2410.07170">Explained Variance Adaptation</a>.`,Ge,re,He,j,se,$t,ke,Qt="Initialize the weights of the LoRA layers using the EVA method.",Lt,Ce,Ot=`This function initializes the weights of the LoRA layers using the EVA method. It computes the SVD for each adapter
layer and updates the weights accordingly.`,Xe,ie,Ye,A,le,kt,je,Gt="Compute the SVD for each layer in the model.",Ct,Ae,Ht=`This function computes the Singular Value Decomposition (SVD) for each layer in the model. It uses the incremental
PCA method to compute the SVD components. The function also checks for convergence of the computed components using
cosine similarity. The rank distribution for each layer is determined based on the explained variance ratio.`,Ke,de,et,Ee,tt;return x=new E({props:{title:"LoRA",local:"lora",headingTag:"h1"}}),P=new E({props:{title:"LoraConfig",local:"peft.LoraConfig",headingTag:"h2"}}),B=new q({props:{name:"class peft.LoraConfig",anchor:"peft.LoraConfig",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"r",val:": int = 8"},{name:"target_modules",val:": Optional[Union[list[str], str]] = None"},{name:"exclude_modules",val:": Optional[Union[list[str], str]] = None"},{name:"lora_alpha",val:": int = 8"},{name:"lora_dropout",val:": float = 0.0"},{name:"fan_in_fan_out",val:": bool = False"},{name:"bias",val:": Literal['none', 'all', 'lora_only'] = 'none'"},{name:"use_rslora",val:": bool = False"},{name:"modules_to_save",val:": Optional[list[str]] = None"},{name:"init_lora_weights",val:": bool | Literal['gaussian', 'eva', 'olora', 'pissa', 'pissa_niter_[number of iters]', 'corda', 'loftq', 'orthogonal'] = True"},{name:"layers_to_transform",val:": Optional[Union[list[int], int]] = None"},{name:"layers_pattern",val:": Optional[Union[list[str], str]] = None"},{name:"rank_pattern",val:": Optional[dict] = <factory>"},{name:"alpha_pattern",val:": Optional[dict] = <factory>"},{name:"megatron_config",val:": Optional[dict] = None"},{name:"megatron_core",val:": Optional[str] = 'megatron.core'"},{name:"trainable_token_indices",val:": Optional[Union[list[int], dict[str, list[int]]]] = None"},{name:"loftq_config",val:": Union[LoftQConfig, dict] = <factory>"},{name:"eva_config",val:": Optional[EvaConfig] = None"},{name:"corda_config",val:": Optional[CordaConfig] = None"},{name:"use_dora",val:": bool = False"},{name:"alora_invocation_tokens",val:": Optional[list[int]] = None"},{name:"use_qalora",val:": bool = False"},{name:"qalora_group_size",val:": int = 16"},{name:"layer_replication",val:": Optional[list[tuple[int, int]]] = None"},{name:"runtime_config",val:": LoraRuntimeConfig = <factory>"},{name:"lora_bias",val:": bool = False"},{name:"target_parameters",val:": Optional[list[str]] = None"},{name:"arrow_config",val:": Optional[ArrowConfig] = None"}],parametersDescription:[{anchor:"peft.LoraConfig.r",description:`<strong>r</strong> (<code>int</code>) &#x2014;
Lora attention dimension (the &#x201C;rank&#x201D;).`,name:"r"},{anchor:"peft.LoraConfig.target_modules",description:`<strong>target_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to apply the adapter to. If this is specified, only the modules with the specified
names will be replaced. When passing a string, a regex match will be performed. When passing a list of
strings, either an exact match will be performed or it is checked if the name of the module ends with any
of the passed strings. If this is specified as &#x2018;all-linear&#x2019;, then all linear/Conv1D modules are chosen (if
the model is a PreTrainedModel, the output layer excluded). If this is not specified, modules will be
chosen according to the model architecture. If the architecture is not known, an error will be raised &#x2014; in
this case, you should specify the target modules manually. To avoid targeting any modules (because you want
to apply <code>target_parameters</code>), set <code>target_modules=[]</code>.`,name:"target_modules"},{anchor:"peft.LoraConfig.exclude_modules",description:`<strong>exclude_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to not apply the adapter. When passing a string, a regex match will be performed.
When passing a list of strings, either an exact match will be performed or it is checked if the name of the
module ends with any of the passed strings.`,name:"exclude_modules"},{anchor:"peft.LoraConfig.lora_alpha",description:`<strong>lora_alpha</strong> (<code>int</code>) &#x2014;
The alpha parameter for Lora scaling.`,name:"lora_alpha"},{anchor:"peft.LoraConfig.lora_dropout",description:`<strong>lora_dropout</strong> (<code>float</code>) &#x2014;
The dropout probability for Lora layers.`,name:"lora_dropout"},{anchor:"peft.LoraConfig.fan_in_fan_out",description:`<strong>fan_in_fan_out</strong> (<code>bool</code>) &#x2014;
Set this to True if the layer to replace stores weight like (fan_in, fan_out). For example, gpt-2 uses
<code>Conv1D</code> which stores weights like (fan_in, fan_out) and hence this should be set to <code>True</code>.`,name:"fan_in_fan_out"},{anchor:"peft.LoraConfig.bias",description:`<strong>bias</strong> (<code>str</code>) &#x2014;
Bias type for LoRA. Can be &#x2018;none&#x2019;, &#x2018;all&#x2019; or &#x2018;lora_only&#x2019;. If &#x2018;all&#x2019; or &#x2018;lora_only&#x2019;, the corresponding biases
will be updated during training. Be aware that this means that, even when disabling the adapters, the model
will not produce the same output as the base model would have without adaptation.`,name:"bias"},{anchor:"peft.LoraConfig.use_rslora",description:`<strong>use_rslora</strong> (<code>bool</code>) &#x2014;
When set to True, uses <a href="https://huggingface.co/papers/2312.03732" rel="nofollow">Rank-Stabilized LoRA</a> which sets the
adapter scaling factor to <code>lora_alpha/math.sqrt(r)</code>, since it was proven to work better. Otherwise, it will
use the original default value of <code>lora_alpha/r</code>.`,name:"use_rslora"},{anchor:"peft.LoraConfig.modules_to_save",description:`<strong>modules_to_save</strong> (<code>List[str]</code>) &#x2014;
List of modules apart from adapter layers to be set as trainable and saved in the final checkpoint.`,name:"modules_to_save"},{anchor:"peft.LoraConfig.init_lora_weights",description:`<strong>init_lora_weights</strong> (<code>bool</code> | <code>Literal[&quot;gaussian&quot;, &quot;eva&quot;, &quot;olora&quot;, &quot;pissa&quot;, &quot;pissa_niter_[number of iters]&quot;, &quot;corda&quot;, &quot;loftq&quot;, &quot;orthogonal&quot;]</code>) &#x2014;
How to initialize the weights of the adapter layers. Passing True (default) results in the default
initialization from the reference implementation from Microsoft, with the LoRA B weight being set to 0.
This means that without further training, the LoRA adapter will be a no-op. Setting the initialization to
False leads to random initialization of LoRA A and B, meaning that LoRA is not a no-op before training;
this setting is intended for debugging purposes. Passing &#x2018;gaussian&#x2019; results in Gaussian initialization
scaled by the LoRA rank for linear and layers. Pass <code>&apos;loftq&apos;</code> to use LoftQ initialization. Passing <code>&apos;eva&apos;</code>
results in a data-driven initialization of <a href="https://huggingface.co/papers/2410.07170">Explained
Variance Adaptation</a>. EVA initializes LoRA based on the SVD of layer input activations and achieves SOTA
performance due to its ability to adapt to the finetuning data. Pass <code>&apos;olora&apos;</code> to use OLoRA initialization.
Passing <code>&apos;pissa&apos;</code> results in the initialization of <a href="&#x2019;&lt;a" rel="nofollow">https://huggingface.co/papers/2404.02948&#x2019;</a><blockquote>
<p>Principal Singular values and Singular vectors Adaptation (PiSSA), which converges more rapidly than
LoRA and ultimately achieves superior performance. Moreover, PiSSA reduces the quantization error compared
to QLoRA, leading to further enhancements. Passing <code>&apos;pissa_niter_[number of iters]&apos;</code> initiates
Fast-SVD-based PiSSA initialization, where <code>[number of iters]</code> indicates the number of subspace iterations
to perform FSVD, and must be a nonnegative integer. When <code>[number of iters]</code> is set to 16, it can complete
the initialization of a 7B model within seconds, and the training effect is approximately equivalent to
using SVD. Passing <code>&apos;corda&apos;</code> results in the initialization of <a href="https://huggingface.co/papers/2406.05223">Context-Oriented Decomposition Adaptation</a>, which
converges even more rapidly than PiSSA in Instruction-Previewed Mode, and preserves world knowledge better
than LoRA in Knowledge-Preserved Mode. Passing <code>&quot;orthogonal&quot;</code> results in LoRA A and B being intialized
orthogonally; in this, it resembles <code>&quot;olora&quot;</code>, but the base weights are left untouched (requires <code>r</code> to be
even, only supported for linear layers for now).</p>
</blockquote>`,name:"init_lora_weights"},{anchor:"peft.LoraConfig.layers_to_transform",description:`<strong>layers_to_transform</strong> (<code>Union[List[int], int]</code>) &#x2014;
The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices
that are specified in this list. If a single integer is passed, it will apply the transformations on the
layer at this index.`,name:"layers_to_transform"},{anchor:"peft.LoraConfig.layers_pattern",description:`<strong>layers_pattern</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The layer pattern name, used only if <code>layers_to_transform</code> is different from <code>None</code>. This should target the
<code>nn.ModuleList</code> of the model, which is often called <code>&apos;layers&apos;</code> or <code>&apos;h&apos;</code>.`,name:"layers_pattern"},{anchor:"peft.LoraConfig.rank_pattern",description:`<strong>rank_pattern</strong> (<code>dict</code>) &#x2014;
The mapping from layer names or regexp expression to ranks which are different from the default rank
specified by <code>r</code>. For example, <code>{&apos;^model.decoder.layers.0.encoder_attn.k_proj&apos;: 16}</code>.`,name:"rank_pattern"},{anchor:"peft.LoraConfig.alpha_pattern",description:`<strong>alpha_pattern</strong> (<code>dict</code>) &#x2014;
The mapping from layer names or regexp expression to alphas which are different from the default alpha
specified by <code>lora_alpha</code>. For example, <code>{&apos;^model.decoder.layers.0.encoder_attn.k_proj&apos;: 16}</code>.`,name:"alpha_pattern"},{anchor:"peft.LoraConfig.megatron_config",description:`<strong>megatron_config</strong> (<code>Optional[dict]</code>) &#x2014;
The TransformerConfig arguments for Megatron. It is used to create LoRA&#x2019;s parallel linear layer. You can
get it like this, <code>core_transformer_config_from_args(get_args())</code>, these two functions being from Megatron.
The arguments will be used to initialize the TransformerConfig of Megatron. You need to specify this
parameter when you want to apply LoRA to the ColumnParallelLinear and RowParallelLinear layers of megatron.`,name:"megatron_config"},{anchor:"peft.LoraConfig.megatron_core",description:`<strong>megatron_core</strong> (<code>Optional[str]</code>) &#x2014;
The core module from Megatron to use, defaults to <code>&quot;megatron.core&quot;</code>.`,name:"megatron_core"},{anchor:"peft.LoraConfig.trainable_token_indices",description:`<strong>trainable_token_indices</strong> (<code>Optional[Union[List[int], dict[str, List[int]]]]</code>) &#x2014;
Lets you specify which token indices to selectively fine-tune without requiring to re-train the whole
embedding matrix using the <code>peft.TrainableTokensModel</code> method. You can specify token indices in two ways.
Either you specify a list of indices which will then target the model&#x2019;s input embedding layer (or, if not
found, <code>embed_tokens</code>). Alternatively, you can specify a dictionary where the key is the name of the
embedding module and the values are the list of token indices, e.g. <code>{&apos;embed_tokens&apos;: [0, 1, ...]}</code>. Note
that training with FSDP requires <code>use_orig_params=True</code> to avoid issues with non-uniform <code>requires_grad</code>.`,name:"trainable_token_indices"},{anchor:"peft.LoraConfig.loftq_config",description:`<strong>loftq_config</strong> (<code>Optional[LoftQConfig]</code>) &#x2014;
The configuration of LoftQ. If this is not None, then LoftQ will be used to quantize the backbone weights
and initialize Lora layers. Also pass <code>init_lora_weights=&apos;loftq&apos;</code>. Note that you should not pass a
quantized model in this case, as LoftQ will quantize the model itself.`,name:"loftq_config"},{anchor:"peft.LoraConfig.eva_config",description:`<strong>eva_config</strong> (<code>Optional[EvaConfig]</code>) &#x2014;
The configuration of EVA. At a minimum the dataset argument needs to be set (use the same dataset as for
finetuning).`,name:"eva_config"},{anchor:"peft.LoraConfig.corda_config",description:`<strong>corda_config</strong> (<code>Optional[CordaConfig]</code>) &#x2014;
The configuration of CorDA. If this is not None, then CorDA will be used to build the adapter layers. Also
pass <code>init_lora_weights=&apos;corda&apos;</code>.`,name:"corda_config"},{anchor:"peft.LoraConfig.use_dora",description:`<strong>use_dora</strong> (<code>bool</code>) &#x2014;
Enable &#x2018;Weight-Decomposed Low-Rank Adaptation&#x2019; (DoRA). This technique decomposes the updates of the weights
into two parts, magnitude and direction. Direction is handled by normal LoRA, whereas the magnitude is
handled by a separate learnable parameter. This can improve the performance of LoRA especially at low
ranks. Right now, DoRA only supports linear and Conv2D layers. DoRA introduces a bigger overhead than pure
LoRA, so it is recommended to merge weights for inference. For more information, see
<a href="https://huggingface.co/papers/2402.09353" rel="nofollow">https://huggingface.co/papers/2402.09353</a>.`,name:"use_dora"},{anchor:"peft.LoraConfig.alora_invocation_tokens",description:`<strong>alora_invocation_tokens</strong> (<code>List[int]</code>) &#x2014;
If not None, enable <a href="https://huggingface.co/papers/2504.12397">&#x2018;Activated LoRA&#x2019; (aLoRA)</a>, with
alora_invocation_tokens being the tokenized invocation string for the adapter (must be present in all model
input strings). This technique selectively activates the adapter weights only on tokens during and after
the alora_invocation_tokens. When used in a CausalLM, this means that the KV cache prior to invocation is
interchangeable with that of the base model (and other aLoRA adapters operating this way). As a result, in
inference pipelines involving switching between base model inference and adapter inference (e.g. agentic
pipelines, see paper for examples), significant savings are realized (relative to LoRA) by saving prefill
operations. Overall adapter inference speedups of an order of magnitude or more can occur on vLLM,
depending on the length of the shared context. Note that merging is not possible due to the selective
application of the weights.`,name:"alora_invocation_tokens"},{anchor:"peft.LoraConfig.layer_replication",description:`<strong>layer_replication</strong> (<code>List[Tuple[int, int]]</code>) &#x2014;
Build a new stack of layers by stacking the original model layers according to the ranges specified. This
allows expanding (or shrinking) the model without duplicating the base model weights. The new layers will
all have separate LoRA adapters attached to them.`,name:"layer_replication"},{anchor:"peft.LoraConfig.runtime_config",description:`<strong>runtime_config</strong> (<code>LoraRuntimeConfig</code>) &#x2014;
Runtime configurations (which are not saved or restored).`,name:"runtime_config"},{anchor:"peft.LoraConfig.lora_bias",description:`<strong>lora_bias</strong> (<code>bool</code>) &#x2014;
Defaults to <code>False</code>. Whether to enable the bias term for the LoRA B parameter. Typically, this should be
disabled. The main use case for this is when the LoRA weights were extracted from fully fine-tuned
parameters so the bias of those parameters can be taken into account.`,name:"lora_bias"},{anchor:"peft.LoraConfig.target_parameters",description:`<strong>target_parameters</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of parameter names or regex expression of the parameter names to replace with LoRA. This argument
behaves similarly to <code>target_modules</code>, except that the parameter name should be passed. Generally, you
should use <code>target_modules</code> to target the module (e.g. <code>nn.Linear</code>). However, in some circumstances, this
is not possible. E.g., in many mixture of expert (MoE) layers in HF Transformers, instead of using
<code>nn.Linear</code>, an <code>nn.Parameter</code> is used. PEFT normally overwrites the <code>forward</code> method for LoRA, but for
<code>nn.Parameter</code>, there is none. Therefore, to apply LoRA to that parameter, it needs to be targeted with
<code>target_parameters</code>. As an example, for Llama4, you can pass:
<code>target_parameters=[&apos;feed_forward.experts.gate_up_proj&apos;, &apos;feed_forward.experts.down_proj]</code>. Passing a
string for regex matching is not implemented yet.`,name:"target_parameters"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py#L250"}}),W=new q({props:{name:"to_dict",anchor:"peft.LoraConfig.to_dict",parameters:[],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py#L667"}}),Q=new E({props:{title:"LoraModel",local:"peft.LoraModel",headingTag:"h2"}}),O=new q({props:{name:"class peft.LoraModel",anchor:"peft.LoraModel",parameters:[{name:"model",val:""},{name:"peft_config",val:": Union[PeftConfig, dict[str, PeftConfig]]"},{name:"adapter_name",val:": str"},{name:"low_cpu_mem_usage",val:": bool = False"},{name:"state_dict",val:": Optional[dict[str, torch.Tensor]] = None"}],parametersDescription:[{anchor:"peft.LoraModel.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014; The model to be adapted.",name:"model"},{anchor:"peft.LoraModel.config",description:'<strong>config</strong> (<a href="/docs/peft/main/en/package_reference/lora#peft.LoraConfig">LoraConfig</a>) &#x2014; The configuration of the Lora model.',name:"config"},{anchor:"peft.LoraModel.adapter_name",description:"<strong>adapter_name</strong> (<code>str</code>) &#x2014; The name of the adapter, defaults to <code>&quot;default&quot;</code>.",name:"adapter_name"},{anchor:"peft.LoraModel.low_cpu_mem_usage",description:`<strong>low_cpu_mem_usage</strong> (<code>bool</code>, <code>optional</code>, defaults to <code>False</code>) &#x2014;
Create empty adapter weights on meta device. Useful to speed up the loading process.`,name:"low_cpu_mem_usage"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/model.py#L68",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The Lora model.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.nn.Module</code></p>
`}}),S=new Xt({props:{anchor:"peft.LoraModel.example",$$slots:{default:[sa]},$$scope:{ctx:ce}}}),z=new Xt({props:{anchor:"peft.LoraModel.example-2",$$slots:{default:[ia]},$$scope:{ctx:ce}}}),G=new q({props:{name:"add_weighted_adapter",anchor:"peft.LoraModel.add_weighted_adapter",parameters:[{name:"adapters",val:": list[str]"},{name:"weights",val:": list[float]"},{name:"adapter_name",val:": str"},{name:"combination_type",val:": str = 'svd'"},{name:"svd_rank",val:": int | None = None"},{name:"svd_clamp",val:": int | None = None"},{name:"svd_full_matrices",val:": bool = True"},{name:"svd_driver",val:": str | None = None"},{name:"density",val:": float | None = None"},{name:"majority_sign_method",val:": Literal['total', 'frequency'] = 'total'"}],parametersDescription:[{anchor:"peft.LoraModel.add_weighted_adapter.adapters",description:`<strong>adapters</strong> (<code>list</code>) &#x2014;
List of adapter names to be merged.`,name:"adapters"},{anchor:"peft.LoraModel.add_weighted_adapter.weights",description:`<strong>weights</strong> (<code>list</code>) &#x2014;
List of weights for each adapter.`,name:"weights"},{anchor:"peft.LoraModel.add_weighted_adapter.adapter_name",description:`<strong>adapter_name</strong> (<code>str</code>) &#x2014;
Name of the new adapter.`,name:"adapter_name"},{anchor:"peft.LoraModel.add_weighted_adapter.combination_type",description:`<strong>combination_type</strong> (<code>str</code>) &#x2014;
The merging type can be one of [<code>svd</code>, <code>linear</code>, <code>cat</code>, <code>ties</code>, <code>ties_svd</code>, <code>dare_ties</code>, <code>dare_linear</code>,
<code>dare_ties_svd</code>, <code>dare_linear_svd</code>, <code>magnitude_prune</code>, <code>magnitude_prune_svd</code>]. When using the <code>cat</code>
combination_type, the rank of the resulting adapter is equal to the sum of all adapters ranks (the
mixed adapter may be too big and result in OOM errors).`,name:"combination_type"},{anchor:"peft.LoraModel.add_weighted_adapter.svd_rank",description:`<strong>svd_rank</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Rank of output adapter for svd. If None provided, will use max rank of merging adapters.`,name:"svd_rank"},{anchor:"peft.LoraModel.add_weighted_adapter.svd_clamp",description:`<strong>svd_clamp</strong> (<code>float</code>, <em>optional</em>) &#x2014;
A quantile threshold for clamping SVD decomposition output. If None is provided, do not perform
clamping. Defaults to None.`,name:"svd_clamp"},{anchor:"peft.LoraModel.add_weighted_adapter.svd_full_matrices",description:`<strong>svd_full_matrices</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Controls whether to compute the full or reduced SVD, and consequently, the shape of the returned
tensors U and Vh. Defaults to True.`,name:"svd_full_matrices"},{anchor:"peft.LoraModel.add_weighted_adapter.svd_driver",description:`<strong>svd_driver</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Name of the cuSOLVER method to be used. This keyword argument only works when merging on CUDA. Can be
one of [None, <code>gesvd</code>, <code>gesvdj</code>, <code>gesvda</code>]. For more info please refer to <code>torch.linalg.svd</code>
documentation. Defaults to None.`,name:"svd_driver"},{anchor:"peft.LoraModel.add_weighted_adapter.density",description:`<strong>density</strong> (<code>float</code>, <em>optional</em>) &#x2014;
Value between 0 and 1. 0 means all values are pruned and 1 means no values are pruned. Should be used
with [<code>ties</code>, <code>ties_svd</code>, <code>dare_ties</code>, <code>dare_linear</code>, <code>dare_ties_svd</code>, <code>dare_linear_svd</code>,
<code>magnintude_prune</code>, <code>magnitude_prune_svd</code>]`,name:"density"},{anchor:"peft.LoraModel.add_weighted_adapter.majority_sign_method",description:`<strong>majority_sign_method</strong> (<code>str</code>) &#x2014;
The method, should be one of [&#x201C;total&#x201D;, &#x201C;frequency&#x201D;], to use to get the magnitude of the sign values.
Should be used with [<code>ties</code>, <code>ties_svd</code>, <code>dare_ties</code>, <code>dare_ties_svd</code>]`,name:"majority_sign_method"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/model.py#L519"}}),H=new q({props:{name:"subtract_mutated_init",anchor:"peft.LoraModel.subtract_mutated_init",parameters:[{name:"output_state_dict",val:": dict[str, torch.Tensor]"},{name:"adapter_name",val:": str"},{name:"kwargs",val:" = None"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/model.py#L767"}}),X=new E({props:{title:"Utility",local:"utility",headingTag:"h2"}}),Y=new E({props:{title:"ArrowConfig",local:"peft.ArrowConfig",headingTag:"h3"}}),K=new q({props:{name:"class peft.ArrowConfig",anchor:"peft.ArrowConfig",parameters:[{name:"top_k",val:": int = 3"},{name:"router_temperature",val:": float = 1.0"},{name:"use_gks",val:": bool = False"},{name:"rng_seed",val:": Optional[int] = None"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py#L73"}}),ee=new E({props:{title:"LoftQ",local:"peft.replace_lora_weights_loftq",headingTag:"h3"}}),te=new q({props:{name:"peft.replace_lora_weights_loftq",anchor:"peft.replace_lora_weights_loftq",parameters:[{name:"peft_model",val:""},{name:"model_path",val:": Optional[str] = None"},{name:"adapter_name",val:": str = 'default'"},{name:"callback",val:": Optional[Callable[[torch.nn.Module, str], bool]] = None"}],parametersDescription:[{anchor:"peft.replace_lora_weights_loftq.peft_model",description:`<strong>peft_model</strong> (<code>PeftModel</code>) &#x2014;
The model to replace the weights of. Must be a quantized PEFT model with LoRA layers.`,name:"peft_model"},{anchor:"peft.replace_lora_weights_loftq.model_path",description:`<strong>model_path</strong> (<code>Optional[str]</code>) &#x2014;
The path to the model safetensors file. If the model is a Hugging Face model, this will be inferred from
the model&#x2019;s config. Otherwise, it must be provided.`,name:"model_path"},{anchor:"peft.replace_lora_weights_loftq.adapter_name",description:`<strong>adapter_name</strong> (<code>str</code>) &#x2014;
The name of the adapter to replace the weights of. The default adapter name is &#x201C;default&#x201D;.`,name:"adapter_name"},{anchor:"peft.replace_lora_weights_loftq.callback",description:`<strong>callback</strong> (<code>Optional[Callable[[PeftModel, str], bool]]</code>) &#x2014;
A callback function that will be called after each module is replaced. The callback function should take
the model and the name of the current module as input and return a boolean indicating whether the
replacement should be kept. If the callback returns False, the replacement will be rolled back. This can be
very useful to confirm that the LoftQ initialization actually decreases the quantization error of the
model. As an example, this callback could generate logits for given input and compare it with the logits
from the original, non-quanitzed model with the same input, and only return <code>True</code> if there is an
improvement. As this is a greedy optimization, it&#x2019;s possible that calling this function multiple times
yields incremental improvements.`,name:"callback"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/utils/loftq_utils.py#L330"}}),ae=new E({props:{title:"Eva",local:"eva",headingTag:"h3"}}),ne=new E({props:{title:"EvaConfig",local:"peft.EvaConfig",headingTag:"h4"}}),oe=new q({props:{name:"class peft.EvaConfig",anchor:"peft.EvaConfig",parameters:[{name:"rho",val:": float = 2.0"},{name:"tau",val:": float = 0.99"},{name:"use_label_mask",val:": bool = True"},{name:"label_mask_value",val:": int = -100"},{name:"whiten",val:": bool = False"},{name:"adjust_scaling_factors",val:": bool = True"}],parametersDescription:[{anchor:"peft.EvaConfig.rho",description:`<strong>rho</strong> (<code>float</code>) &#x2014;
Rho value for EVA redistribution (&gt;= 1.0). The maximum rank for a layer is lora_r * rho. Default is 2.0,
meaning the maximum rank allowed for a layer is 2r. Increasing rho will allow for a higher degree of
redistribution of ranks across layers. Some pre-trained models might be more sensitive to a rank
redistribution. It can therefore be beneficial to try rho=1.0 (no redistribution) if the performance is
lower than expected.`,name:"rho"},{anchor:"peft.EvaConfig.tau",description:`<strong>tau</strong> (<code>float</code>) &#x2014;
Cosine similarity threshold for early stopping. Compares the cosine similarity of right-singular vectors
between two consecutive SVD steps. If the cosine similarity is above this threshold, the SVD iteration is
stopped. Default is 0.99.`,name:"tau"},{anchor:"peft.EvaConfig.use_label_mask",description:`<strong>use_label_mask</strong> (<code>bool</code>) &#x2014;
Use label mask for EVA initialization. This means that positions where labels=label_mask_value are ignored
for the SVD computation. Setting use_label_mask=True is preferred in most cases and can be especially
beneficial for multi-turn conversations. The default value is True. Filtering out items based on the label
mask can sometimes lead to a small batch size and as a result instabilities in the SVD computation. For
cases where a large share of batch items would be filtered out, set use_label_mask=False.`,name:"use_label_mask"},{anchor:"peft.EvaConfig.label_mask_value",description:`<strong>label_mask_value</strong> (<code>int</code>) &#x2014;
If use_label_mask=True the value to look for to mask out ignored tokens. Default is -100.`,name:"label_mask_value"},{anchor:"peft.EvaConfig.whiten",description:`<strong>whiten</strong> (<code>bool</code>) &#x2014; Apply whitening to singular vectors. Default is False.
Whitening has been shown to be beneficial for EVA in the vision domain.`,name:"whiten"},{anchor:"peft.EvaConfig.adjust_scaling_factors",description:`<strong>adjust_scaling_factors</strong> (<code>bool</code>) &#x2014;
Adjust LoRA scaling factors after the rank redistribution. Setting this to True means the scaling factors
are adjusted so that all LoRA gradients have the same scale regardless of their rank. Default is True.`,name:"adjust_scaling_factors"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py#L123"}}),re=new E({props:{title:"initialize_lora_eva_weights",local:"peft.initialize_lora_eva_weights",headingTag:"h4"}}),se=new q({props:{name:"peft.initialize_lora_eva_weights",anchor:"peft.initialize_lora_eva_weights",parameters:[{name:"model",val:": Module"},{name:"dataloader",val:": typing.Optional[collections.abc.Iterable] = None"},{name:"eva_state_dict",val:": typing.Optional[dict] = None"},{name:"forward_fn",val:": typing.Optional[<built-in function callable>] = <function forward_fn_dict at 0x7fb8663841f0>"},{name:"prepare_model_inputs_fn",val:": typing.Optional[<built-in function callable>] = <function prepare_model_inputs_fn_language_modeling at 0x7fb8663840d0>"},{name:"prepare_layer_inputs_fn",val:": typing.Union[<built-in function callable>, dict[str, callable], NoneType] = <function prepare_layer_inputs_fn_language_modeling at 0x7fb866384160>"},{name:"adapter_name",val:": str = 'default'"},{name:"gather_distributed_inputs",val:": bool = True"},{name:"show_progress_bar",val:": bool = True"}],parametersDescription:[{anchor:"peft.initialize_lora_eva_weights.model",description:"<strong>model</strong> (PeftModel) &#x2014; The peft model to compute the SVD for.",name:"model"},{anchor:"peft.initialize_lora_eva_weights.dataloader",description:`<strong>dataloader</strong> (Optional[Iterable]) &#x2014;
The dataloader to use for the forward pass. If None, eva_state_dict needs to be provided.`,name:"dataloader"},{anchor:"peft.initialize_lora_eva_weights.eva_state_dict",description:`<strong>eva_state_dict</strong> (Optional[dict]) &#x2014;
The state_dict to load into the model. If None, a dataloader needs to be provided and the state_dict will
be computed using <code>get_eva_state_dict</code>.`,name:"eva_state_dict"},{anchor:"peft.initialize_lora_eva_weights.forward_fn",description:`<strong>forward_fn</strong> (callable) &#x2014;
The forward function to use for the forward pass. Takes two arguments: <code>model</code> and <code>inputs</code>. Default
behavior is <code>return model(**inputs)</code>`,name:"forward_fn"},{anchor:"peft.initialize_lora_eva_weights.prepare_model_inputs_fn",description:`<strong>prepare_model_inputs_fn</strong> (Optional[callable]) &#x2014;
This function receives the model inputs and the peft_config and passes the output to
<code>prepare_layer_inputs_fn</code>. Can be used to modify the input to the SVD computation based on the original
model inputs. For example for language modeling the attention mask is used to determine which indices are
padding tokens and should not be used for SVD. Any function defined here expects two arguments:
<code>model_input</code> and <code>peft_config</code>. <code>peft.tuners.lora.eva.prepare_model_inputs_fn_language_modeling</code> is used
by default.`,name:"prepare_model_inputs_fn"},{anchor:"peft.initialize_lora_eva_weights.prepare_layer_inputs_fn",description:`<strong>prepare_layer_inputs_fn</strong> (Union[callable, Dict[str, callable], None]) &#x2014;
This function receives the layer inputs, the model inputs (potentially modified by
<code>prepare_model_inputs_fn</code>) and the name of the layer and returns the inputs that should be used for SVD for
that particular layer. Any custom function defined here expects three arguments: <code>layer_input</code>,
<code>model_input</code>, and <code>layer_name</code> and should return a 2d tensor. The default logic can be found in
peft.tuners.lora.eva.prepare_layer_inputs_fn_language_modeling and works for language modeling. In this
case model_inputs is the mask used to determine which indices should be used for SVD (created by
<code>prepare_model_inputs_fn_language_modeling</code>).`,name:"prepare_layer_inputs_fn"},{anchor:"peft.initialize_lora_eva_weights.adapter_name",description:"<strong>adapter_name</strong> (str) &#x2014; The name of the adapter to initialize the weights for.",name:"adapter_name"},{anchor:"peft.initialize_lora_eva_weights.gather_distributed_inputs",description:`<strong>gather_distributed_inputs</strong> (bool) &#x2014;
Whether to gather the layer inputs from all ranks. Default is True meaning in a distributed setting the
layer inputs will be gathered from all ranks for the SVD computation. For non-distributed settings this
argument is ignored. Set to False if you are using a non-distributed dataloader in a distributed setting.`,name:"gather_distributed_inputs"},{anchor:"peft.initialize_lora_eva_weights.show_progress_bar",description:"<strong>show_progress_bar</strong> (bool) &#x2014; Whether to show a progress bar. Default is True.",name:"show_progress_bar"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/eva.py#L659",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The model with the initialized LoRA weights.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>model (torch.nn.Module)</p>
`}}),ie=new E({props:{title:"get_eva_state_dict",local:"peft.get_eva_state_dict",headingTag:"h4"}}),le=new q({props:{name:"peft.get_eva_state_dict",anchor:"peft.get_eva_state_dict",parameters:[{name:"model",val:": Module"},{name:"dataloader",val:": Iterable"},{name:"peft_config",val:": typing.Optional[peft.tuners.lora.config.LoraConfig] = None"},{name:"forward_fn",val:": typing.Optional[<built-in function callable>] = <function forward_fn_dict at 0x7fb8663841f0>"},{name:"prepare_model_inputs_fn",val:": typing.Optional[<built-in function callable>] = <function prepare_model_inputs_fn_language_modeling at 0x7fb8663840d0>"},{name:"prepare_layer_inputs_fn",val:": typing.Union[<built-in function callable>, dict[str, callable], NoneType] = <function prepare_layer_inputs_fn_language_modeling at 0x7fb866384160>"},{name:"adapter_name",val:": str = 'default'"},{name:"gather_distributed_inputs",val:": bool = True"},{name:"show_progress_bar",val:": bool = True"}],parametersDescription:[{anchor:"peft.get_eva_state_dict.model",description:"<strong>model</strong> (torch.nn.Module) &#x2014; The model to compute the SVD for. Does not need to be a PeftModel.",name:"model"},{anchor:"peft.get_eva_state_dict.dataloader",description:"<strong>dataloader</strong> (Iterable) &#x2014; The dataloader to use for the forward pass.",name:"dataloader"},{anchor:"peft.get_eva_state_dict.peft_config",description:`<strong>peft_config</strong> (Optional[LoraConfig]) &#x2014;
The configuration for the LoRA layers. Only required if <code>model</code> is not a PeftModel.`,name:"peft_config"},{anchor:"peft.get_eva_state_dict.forward_fn",description:`<strong>forward_fn</strong> (callable) &#x2014;
The forward function to use for the forward pass. Takes two arguments: <code>model</code> and <code>inputs</code>. Default
behavior is <code>return model(**inputs)</code>`,name:"forward_fn"},{anchor:"peft.get_eva_state_dict.prepare_model_inputs_fn",description:`<strong>prepare_model_inputs_fn</strong> (Optional[callable]) &#x2014;
This function receives the model inputs and the peft_config and passes the output to
<code>prepare_layer_inputs_fn</code>. Can be used to modify the input to the SVD computation based on the original
model inputs. For example for language modeling the attention mask is used to determine which indices are
padding tokens and should not be used for SVD. Any function defined here expects two arguments:
<code>model_input</code> and <code>peft_config</code>. <code>peft.tuners.lora.eva.prepare_model_inputs_fn_language_modeling</code> is used
by default.`,name:"prepare_model_inputs_fn"},{anchor:"peft.get_eva_state_dict.prepare_layer_inputs_fn",description:`<strong>prepare_layer_inputs_fn</strong> (Union[callable, Dict[str, callable], None]) &#x2014;
This function receives the layer inputs, the model inputs (potentially modified by
<code>prepare_model_inputs_fn</code>) and the name of the layer and returns the inputs that should be used for SVD for
that particular layer. Any custom function defined here expects three arguments: <code>layer_input</code>,
<code>model_input</code>, and <code>layer_name</code> and should return a 2d tensor. The default logic can be found in
peft.tuners.lora.eva.prepare_layer_inputs_fn_language_modeling and works for language modeling. In this
case model_inputs is the mask used to determine which indices should be used for SVD (created by
<code>prepare_model_inputs_fn_language_modeling</code>).`,name:"prepare_layer_inputs_fn"},{anchor:"peft.get_eva_state_dict.adapter_name",description:"<strong>adapter_name</strong> (str) &#x2014; The name of the adapter to compute the SVD for.",name:"adapter_name"},{anchor:"peft.get_eva_state_dict.gather_distributed_inputs",description:`<strong>gather_distributed_inputs</strong> (bool) &#x2014;
Whether to gather the layer inputs from all ranks. Default is True meaning in a distributed setting the
layer inputs will be gathered from all ranks for the SVD computation. For non-distributed settings this
argument is ignored. Set to False if you are using a non-distributed dataloader in a distributed setting.`,name:"gather_distributed_inputs"},{anchor:"peft.get_eva_state_dict.show_progress_bar",description:"<strong>show_progress_bar</strong> (bool) &#x2014; Whether to show a progress bar. Default is True.",name:"show_progress_bar"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/eva.py#L561",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The state dictionary containing the SVD components for each layer.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>eva_state_dict (dict)</p>
`}}),de=new ra({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/lora.md"}}),{c(){d=i("meta"),T=n(),b=i("p"),M=n(),p(x.$$.fragment),v=n(),k=i("p"),k.innerHTML=jt,Re=n(),F=i("p"),F.textContent=At,Ie=n(),Z=i("p"),Z.innerHTML=Ut,De=n(),p(P.$$.fragment),Ne=n(),C=i("div"),p(B.$$.fragment),rt=n(),me=i("p"),me.innerHTML=Jt,st=n(),N=i("div"),p(W.$$.fragment),it=n(),fe=i("p"),fe.textContent=qt,Se=n(),p(Q.$$.fragment),ze=n(),u=i("div"),p(O.$$.fragment),lt=n(),he=i("p"),he.textContent=Et,dt=n(),ge=i("p"),ge.innerHTML=Rt,pt=n(),p(S.$$.fragment),ct=n(),p(z.$$.fragment),mt=n(),ue=i("p"),ue.innerHTML=It,ft=n(),_e=i("ul"),_e.innerHTML=Dt,ht=n(),U=i("div"),p(G.$$.fragment),gt=n(),be=i("p"),be.textContent=Nt,ut=n(),ve=i("p"),ve.innerHTML=St,_t=n(),V=i("div"),p(H.$$.fragment),bt=n(),ye=i("p"),ye.innerHTML=zt,Ve=n(),p(X.$$.fragment),Fe=n(),p(Y.$$.fragment),Ze=n(),R=i("div"),p(K.$$.fragment),vt=n(),we=i("p"),we.innerHTML=Vt,Pe=n(),p(ee.$$.fragment),Be=n(),w=i("div"),p(te.$$.fragment),yt=n(),Me=i("p"),Me.textContent=Ft,wt=n(),Te=i("p"),Te.textContent=Zt,Mt=n(),xe=i("p"),xe.textContent=Pt,Tt=n(),$e=i("p"),$e.textContent=Bt,We=n(),p(ae.$$.fragment),Qe=n(),p(ne.$$.fragment),Oe=n(),I=i("div"),p(oe.$$.fragment),xt=n(),Le=i("p"),Le.innerHTML=Wt,Ge=n(),p(re.$$.fragment),He=n(),j=i("div"),p(se.$$.fragment),$t=n(),ke=i("p"),ke.textContent=Qt,Lt=n(),Ce=i("p"),Ce.textContent=Ot,Xe=n(),p(ie.$$.fragment),Ye=n(),A=i("div"),p(le.$$.fragment),kt=n(),je=i("p"),je.textContent=Gt,Ct=n(),Ae=i("p"),Ae.textContent=Ht,Ke=n(),p(de.$$.fragment),et=n(),Ee=i("p"),this.h()},l(e){const t=oa("svelte-u9bgzb",document.head);d=l(t,"META",{name:!0,content:!0}),t.forEach(a),T=o(e),b=l(e,"P",{}),$(b).forEach(a),M=o(e),c(x.$$.fragment,e),v=o(e),k=l(e,"P",{"data-svelte-h":!0}),_(k)!=="svelte-e6qy8f"&&(k.innerHTML=jt),Re=o(e),F=l(e,"P",{"data-svelte-h":!0}),_(F)!=="svelte-1cwsb16"&&(F.textContent=At),Ie=o(e),Z=l(e,"P",{"data-svelte-h":!0}),_(Z)!=="svelte-p2y1b1"&&(Z.innerHTML=Ut),De=o(e),c(P.$$.fragment,e),Ne=o(e),C=l(e,"DIV",{class:!0});var D=$(C);c(B.$$.fragment,D),rt=o(D),me=l(D,"P",{"data-svelte-h":!0}),_(me)!=="svelte-o3o25r"&&(me.innerHTML=Jt),st=o(D),N=l(D,"DIV",{class:!0});var pe=$(N);c(W.$$.fragment,pe),it=o(pe),fe=l(pe,"P",{"data-svelte-h":!0}),_(fe)!=="svelte-1nggz95"&&(fe.textContent=qt),pe.forEach(a),D.forEach(a),Se=o(e),c(Q.$$.fragment,e),ze=o(e),u=l(e,"DIV",{class:!0});var y=$(u);c(O.$$.fragment,y),lt=o(y),he=l(y,"P",{"data-svelte-h":!0}),_(he)!=="svelte-kipsu2"&&(he.textContent=Et),dt=o(y),ge=l(y,"P",{"data-svelte-h":!0}),_(ge)!=="svelte-wrg681"&&(ge.innerHTML=Rt),pt=o(y),c(S.$$.fragment,y),ct=o(y),c(z.$$.fragment,y),mt=o(y),ue=l(y,"P",{"data-svelte-h":!0}),_(ue)!=="svelte-1xx6nm4"&&(ue.innerHTML=It),ft=o(y),_e=l(y,"UL",{"data-svelte-h":!0}),_(_e)!=="svelte-3vno46"&&(_e.innerHTML=Dt),ht=o(y),U=l(y,"DIV",{class:!0});var Ue=$(U);c(G.$$.fragment,Ue),gt=o(Ue),be=l(Ue,"P",{"data-svelte-h":!0}),_(be)!=="svelte-lg9b7q"&&(be.textContent=Nt),ut=o(Ue),ve=l(Ue,"P",{"data-svelte-h":!0}),_(ve)!=="svelte-gmuli4"&&(ve.innerHTML=St),Ue.forEach(a),_t=o(y),V=l(y,"DIV",{class:!0});var at=$(V);c(H.$$.fragment,at),bt=o(at),ye=l(at,"P",{"data-svelte-h":!0}),_(ye)!=="svelte-1rwd503"&&(ye.innerHTML=zt),at.forEach(a),y.forEach(a),Ve=o(e),c(X.$$.fragment,e),Fe=o(e),c(Y.$$.fragment,e),Ze=o(e),R=l(e,"DIV",{class:!0});var nt=$(R);c(K.$$.fragment,nt),vt=o(nt),we=l(nt,"P",{"data-svelte-h":!0}),_(we)!=="svelte-17k9vq8"&&(we.innerHTML=Vt),nt.forEach(a),Pe=o(e),c(ee.$$.fragment,e),Be=o(e),w=l(e,"DIV",{class:!0});var J=$(w);c(te.$$.fragment,J),yt=o(J),Me=l(J,"P",{"data-svelte-h":!0}),_(Me)!=="svelte-3ipm59"&&(Me.textContent=Ft),wt=o(J),Te=l(J,"P",{"data-svelte-h":!0}),_(Te)!=="svelte-1c9djg"&&(Te.textContent=Zt),Mt=o(J),xe=l(J,"P",{"data-svelte-h":!0}),_(xe)!=="svelte-p9czkf"&&(xe.textContent=Pt),Tt=o(J),$e=l(J,"P",{"data-svelte-h":!0}),_($e)!=="svelte-1d7vg8p"&&($e.textContent=Bt),J.forEach(a),We=o(e),c(ae.$$.fragment,e),Qe=o(e),c(ne.$$.fragment,e),Oe=o(e),I=l(e,"DIV",{class:!0});var ot=$(I);c(oe.$$.fragment,ot),xt=o(ot),Le=l(ot,"P",{"data-svelte-h":!0}),_(Le)!=="svelte-98ps9a"&&(Le.innerHTML=Wt),ot.forEach(a),Ge=o(e),c(re.$$.fragment,e),He=o(e),j=l(e,"DIV",{class:!0});var Je=$(j);c(se.$$.fragment,Je),$t=o(Je),ke=l(Je,"P",{"data-svelte-h":!0}),_(ke)!=="svelte-1lcy19c"&&(ke.textContent=Qt),Lt=o(Je),Ce=l(Je,"P",{"data-svelte-h":!0}),_(Ce)!=="svelte-18makdi"&&(Ce.textContent=Ot),Je.forEach(a),Xe=o(e),c(ie.$$.fragment,e),Ye=o(e),A=l(e,"DIV",{class:!0});var qe=$(A);c(le.$$.fragment,qe),kt=o(qe),je=l(qe,"P",{"data-svelte-h":!0}),_(je)!=="svelte-tzfhs9"&&(je.textContent=Gt),Ct=o(qe),Ae=l(qe,"P",{"data-svelte-h":!0}),_(Ae)!=="svelte-1txgnvp"&&(Ae.textContent=Ht),qe.forEach(a),Ke=o(e),c(de.$$.fragment,e),et=o(e),Ee=l(e,"P",{}),$(Ee).forEach(a),this.h()},h(){L(d,"name","hf:doc:metadata"),L(d,"content",da),L(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){r(document.head,d),s(e,T,t),s(e,b,t),s(e,M,t),m(x,e,t),s(e,v,t),s(e,k,t),s(e,Re,t),s(e,F,t),s(e,Ie,t),s(e,Z,t),s(e,De,t),m(P,e,t),s(e,Ne,t),s(e,C,t),m(B,C,null),r(C,rt),r(C,me),r(C,st),r(C,N),m(W,N,null),r(N,it),r(N,fe),s(e,Se,t),m(Q,e,t),s(e,ze,t),s(e,u,t),m(O,u,null),r(u,lt),r(u,he),r(u,dt),r(u,ge),r(u,pt),m(S,u,null),r(u,ct),m(z,u,null),r(u,mt),r(u,ue),r(u,ft),r(u,_e),r(u,ht),r(u,U),m(G,U,null),r(U,gt),r(U,be),r(U,ut),r(U,ve),r(u,_t),r(u,V),m(H,V,null),r(V,bt),r(V,ye),s(e,Ve,t),m(X,e,t),s(e,Fe,t),m(Y,e,t),s(e,Ze,t),s(e,R,t),m(K,R,null),r(R,vt),r(R,we),s(e,Pe,t),m(ee,e,t),s(e,Be,t),s(e,w,t),m(te,w,null),r(w,yt),r(w,Me),r(w,wt),r(w,Te),r(w,Mt),r(w,xe),r(w,Tt),r(w,$e),s(e,We,t),m(ae,e,t),s(e,Qe,t),m(ne,e,t),s(e,Oe,t),s(e,I,t),m(oe,I,null),r(I,xt),r(I,Le),s(e,Ge,t),m(re,e,t),s(e,He,t),s(e,j,t),m(se,j,null),r(j,$t),r(j,ke),r(j,Lt),r(j,Ce),s(e,Xe,t),m(ie,e,t),s(e,Ye,t),s(e,A,t),m(le,A,null),r(A,kt),r(A,je),r(A,Ct),r(A,Ae),s(e,Ke,t),m(de,e,t),s(e,et,t),s(e,Ee,t),tt=!0},p(e,[t]){const D={};t&2&&(D.$$scope={dirty:t,ctx:e}),S.$set(D);const pe={};t&2&&(pe.$$scope={dirty:t,ctx:e}),z.$set(pe)},i(e){tt||(f(x.$$.fragment,e),f(P.$$.fragment,e),f(B.$$.fragment,e),f(W.$$.fragment,e),f(Q.$$.fragment,e),f(O.$$.fragment,e),f(S.$$.fragment,e),f(z.$$.fragment,e),f(G.$$.fragment,e),f(H.$$.fragment,e),f(X.$$.fragment,e),f(Y.$$.fragment,e),f(K.$$.fragment,e),f(ee.$$.fragment,e),f(te.$$.fragment,e),f(ae.$$.fragment,e),f(ne.$$.fragment,e),f(oe.$$.fragment,e),f(re.$$.fragment,e),f(se.$$.fragment,e),f(ie.$$.fragment,e),f(le.$$.fragment,e),f(de.$$.fragment,e),tt=!0)},o(e){h(x.$$.fragment,e),h(P.$$.fragment,e),h(B.$$.fragment,e),h(W.$$.fragment,e),h(Q.$$.fragment,e),h(O.$$.fragment,e),h(S.$$.fragment,e),h(z.$$.fragment,e),h(G.$$.fragment,e),h(H.$$.fragment,e),h(X.$$.fragment,e),h(Y.$$.fragment,e),h(K.$$.fragment,e),h(ee.$$.fragment,e),h(te.$$.fragment,e),h(ae.$$.fragment,e),h(ne.$$.fragment,e),h(oe.$$.fragment,e),h(re.$$.fragment,e),h(se.$$.fragment,e),h(ie.$$.fragment,e),h(le.$$.fragment,e),h(de.$$.fragment,e),tt=!1},d(e){e&&(a(T),a(b),a(M),a(v),a(k),a(Re),a(F),a(Ie),a(Z),a(De),a(Ne),a(C),a(Se),a(ze),a(u),a(Ve),a(Fe),a(Ze),a(R),a(Pe),a(Be),a(w),a(We),a(Qe),a(Oe),a(I),a(Ge),a(He),a(j),a(Xe),a(Ye),a(A),a(Ke),a(et),a(Ee)),a(d),g(x,e),g(P,e),g(B),g(W),g(Q,e),g(O),g(S),g(z),g(G),g(H),g(X,e),g(Y,e),g(K),g(ee,e),g(te),g(ae,e),g(ne,e),g(oe),g(re,e),g(se),g(ie,e),g(le),g(de,e)}}}const da='{"title":"LoRA","local":"lora","sections":[{"title":"LoraConfig","local":"peft.LoraConfig","sections":[],"depth":2},{"title":"LoraModel","local":"peft.LoraModel","sections":[],"depth":2},{"title":"Utility","local":"utility","sections":[{"title":"ArrowConfig","local":"peft.ArrowConfig","sections":[],"depth":3},{"title":"LoftQ","local":"peft.replace_lora_weights_loftq","sections":[],"depth":3},{"title":"Eva","local":"eva","sections":[{"title":"EvaConfig","local":"peft.EvaConfig","sections":[],"depth":4},{"title":"initialize_lora_eva_weights","local":"peft.initialize_lora_eva_weights","sections":[],"depth":4},{"title":"get_eva_state_dict","local":"peft.get_eva_state_dict","sections":[],"depth":4}],"depth":3}],"depth":2}],"depth":1}';function pa(ce){return ta(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class _a extends aa{constructor(d){super(),na(this,d,pa,la,ea,{})}}export{_a as component};
