import{s as me,o as fe,n as he}from"../chunks/scheduler.4cd3e004.js";import{S as ue,i as ge,e as p,s as l,c as _,h as Me,a as d,d as a,b as r,f as X,g as j,j as w,k as z,l as g,m as n,n as T,t as b,o as C,p as U}from"../chunks/index.b664cb2d.js";import{D as ce}from"../chunks/Docstring.9ceb4334.js";import{C as ye}from"../chunks/CodeBlock.8601e34e.js";import{E as _e}from"../chunks/ExampleCodeBlock.4f57b575.js";import{H as ne,E as je}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.179b63f8.js";function we(R){let s,J="Example:",h,c,m;return c=new ye({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb0hhTW9kZWwlMkMlMjBMb0hhQ29uZmlnJTBBJTBBY29uZmlnX3RlJTIwJTNEJTIwTG9IYUNvbmZpZyglMEElMjAlMjAlMjAlMjByJTNEOCUyQyUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0QzMiUyQyUwQSUyMCUyMCUyMCUyMHRhcmdldF9tb2R1bGVzJTNEJTVCJTIya19wcm9qJTIyJTJDJTIwJTIycV9wcm9qJTIyJTJDJTIwJTIydl9wcm9qJTIyJTJDJTIwJTIyb3V0X3Byb2olMjIlMkMlMjAlMjJmYzElMjIlMkMlMjAlMjJmYzIlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjByYW5rX2Ryb3BvdXQlM0QwLjAlMkMlMEElMjAlMjAlMjAlMjBtb2R1bGVfZHJvcG91dCUzRDAuMCUyQyUwQSUyMCUyMCUyMCUyMGluaXRfd2VpZ2h0cyUzRFRydWUlMkMlMEEpJTBBY29uZmlnX3VuZXQlMjAlM0QlMjBMb0hhQ29uZmlnKCUwQSUyMCUyMCUyMCUyMHIlM0Q4JTJDJTBBJTIwJTIwJTIwJTIwbG9yYV9hbHBoYSUzRDMyJTJDJTBBJTIwJTIwJTIwJTIwdGFyZ2V0X21vZHVsZXMlM0QlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwcm9qX2luJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycHJval9vdXQlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0b19rJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydG9fcSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRvX3YlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0b19vdXQuMCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmZmLm5ldC4wLnByb2olMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJmZi5uZXQuMiUyMiUyQyUwQSUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMHJhbmtfZHJvcG91dCUzRDAuMCUyQyUwQSUyMCUyMCUyMCUyMG1vZHVsZV9kcm9wb3V0JTNEMC4wJTJDJTBBJTIwJTIwJTIwJTIwaW5pdF93ZWlnaHRzJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHVzZV9lZmZlY3RpdmVfY29udjJkJTNEVHJ1ZSUyQyUwQSklMEElMEFtb2RlbCUyMCUzRCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMjJydW53YXltbCUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyMiklMEFtb2RlbC50ZXh0X2VuY29kZXIlMjAlM0QlMjBMb0hhTW9kZWwobW9kZWwudGV4dF9lbmNvZGVyJTJDJTIwY29uZmlnX3RlJTJDJTIwJTIyZGVmYXVsdCUyMiklMEFtb2RlbC51bmV0JTIwJTNEJTIwTG9IYU1vZGVsKG1vZGVsLnVuZXQlMkMlMjBjb25maWdfdW5ldCUyQyUyMCUyMmRlZmF1bHQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoHaModel, LoHaConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>config_te = LoHaConfig(
<span class="hljs-meta">... </span>    r=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    lora_alpha=<span class="hljs-number">32</span>,
<span class="hljs-meta">... </span>    target_modules=[<span class="hljs-string">&quot;k_proj&quot;</span>, <span class="hljs-string">&quot;q_proj&quot;</span>, <span class="hljs-string">&quot;v_proj&quot;</span>, <span class="hljs-string">&quot;out_proj&quot;</span>, <span class="hljs-string">&quot;fc1&quot;</span>, <span class="hljs-string">&quot;fc2&quot;</span>],
<span class="hljs-meta">... </span>    rank_dropout=<span class="hljs-number">0.0</span>,
<span class="hljs-meta">... </span>    module_dropout=<span class="hljs-number">0.0</span>,
<span class="hljs-meta">... </span>    init_weights=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>config_unet = LoHaConfig(
<span class="hljs-meta">... </span>    r=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    lora_alpha=<span class="hljs-number">32</span>,
<span class="hljs-meta">... </span>    target_modules=[
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;proj_in&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;proj_out&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_k&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_q&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_v&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_out.0&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;ff.net.0.proj&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;ff.net.2&quot;</span>,
<span class="hljs-meta">... </span>    ],
<span class="hljs-meta">... </span>    rank_dropout=<span class="hljs-number">0.0</span>,
<span class="hljs-meta">... </span>    module_dropout=<span class="hljs-number">0.0</span>,
<span class="hljs-meta">... </span>    init_weights=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    use_effective_conv2d=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = StableDiffusionPipeline.from_pretrained(<span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.text_encoder = LoHaModel(model.text_encoder, config_te, <span class="hljs-string">&quot;default&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.unet = LoHaModel(model.unet, config_unet, <span class="hljs-string">&quot;default&quot;</span>)`,wrap:!1}}),{c(){s=p("p"),s.textContent=J,h=l(),_(c.$$.fragment)},l(o){s=d(o,"P",{"data-svelte-h":!0}),w(s)!=="svelte-11lpom8"&&(s.textContent=J),h=r(o),j(c.$$.fragment,o)},m(o,f){n(o,s,f),n(o,h,f),T(c,o,f),m=!0},p:he,i(o){m||(b(c.$$.fragment,o),m=!0)},o(o){C(c.$$.fragment,o),m=!1},d(o){o&&(a(s),a(h)),U(c,o)}}}function Te(R){let s,J,h,c,m,o,f,oe='Low-Rank Hadamard Product (<a href="https://huggingface.co/papers/2108.06098" rel="nofollow">LoHa</a>), is similar to LoRA except it approximates the large weight matrix with more low-rank matrices and combines them with the Hadamard product. This method is even more parameter-efficient than LoRA and achieves comparable performance.',V,v,se="The abstract from the paper is:",N,L,le="<em>In this work, we propose a communication-efficient parameterization, FedPara, for federated learning (FL) to overcome the burdens on frequent model uploads and downloads. Our method re-parameterizes weight parameters of layers using low-rank weights followed by the Hadamard product. Compared to the conventional low-rank parameterization, our FedPara method is not restricted to low-rank constraints, and thereby it has a far larger capacity. This property enables to achieve comparable performance while requiring 3 to 10 times lower communication costs than the model with the original layers, which is not achievable by the traditional low-rank methods. The efficiency of our method can be further improved by combining with other efficient FL optimizers. In addition, we extend our method to a personalized FL application, pFedPara, which separates parameters into global and local ones. We show that pFedPara outperforms competing personalized FL methods with more than three times fewer parameters</em>.",F,H,W,M,x,Y,Z,re='This is the configuration class to store the configuration of a <a href="/docs/peft/main/en/package_reference/loha#peft.LoHaModel">LoHaModel</a>.',P,I,S,i,$,K,q,ie=`Creates Low-Rank Hadamard Product model from a pretrained model. The method is partially described in
<a href="https://huggingface.co/papers/2108.06098" rel="nofollow">https://huggingface.co/papers/2108.06098</a> Current implementation heavily borrows from
<a href="https://github.com/KohakuBlueleaf/LyCORIS/blob/eb460098187f752a5d66406d3affade6f0a07ece/lycoris/modules/loha.py" rel="nofollow">https://github.com/KohakuBlueleaf/LyCORIS/blob/eb460098187f752a5d66406d3affade6f0a07ece/lycoris/modules/loha.py</a>`,ee,y,te,E,pe="<strong>Attributes</strong>:",ae,Q,de='<li><strong>model</strong> (<code>~torch.nn.Module</code>) — The model to be adapted.</li> <li><strong>peft_config</strong> (<a href="/docs/peft/main/en/package_reference/loha#peft.LoHaConfig">LoHaConfig</a>): The configuration of the LoHa model.</li>',G,k,D,B,O;return m=new ne({props:{title:"LoHa",local:"loha",headingTag:"h1"}}),H=new ne({props:{title:"LoHaConfig",local:"peft.LoHaConfig",headingTag:"h2"}}),x=new ce({props:{name:"class peft.LoHaConfig",anchor:"peft.LoHaConfig",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"rank_pattern",val:": Optional[dict] = <factory>"},{name:"alpha_pattern",val:": Optional[dict] = <factory>"},{name:"r",val:": int = 8"},{name:"alpha",val:": int = 8"},{name:"rank_dropout",val:": float = 0.0"},{name:"module_dropout",val:": float = 0.0"},{name:"use_effective_conv2d",val:": bool = False"},{name:"target_modules",val:": Optional[Union[list[str], str]] = None"},{name:"exclude_modules",val:": Optional[Union[list[str], str]] = None"},{name:"init_weights",val:": bool = True"},{name:"layers_to_transform",val:": Optional[Union[list[int], int]] = None"},{name:"layers_pattern",val:": Optional[Union[list[str], str]] = None"},{name:"modules_to_save",val:": Optional[list[str]] = None"}],parametersDescription:[{anchor:"peft.LoHaConfig.r",description:`<strong>r</strong> (<code>int</code>) &#x2014;
LoHa rank.`,name:"r"},{anchor:"peft.LoHaConfig.alpha",description:`<strong>alpha</strong> (<code>int</code>) &#x2014;
The alpha parameter for LoHa scaling.`,name:"alpha"},{anchor:"peft.LoHaConfig.rank_dropout",description:`<strong>rank_dropout</strong> (<code>float</code>) &#x2014;
The dropout probability for rank dimension during training.`,name:"rank_dropout"},{anchor:"peft.LoHaConfig.module_dropout",description:`<strong>module_dropout</strong> (<code>float</code>) &#x2014;
The dropout probability for disabling LoHa modules during training.`,name:"module_dropout"},{anchor:"peft.LoHaConfig.use_effective_conv2d",description:`<strong>use_effective_conv2d</strong> (<code>bool</code>) &#x2014;
Use parameter effective decomposition for Conv2d (and Conv1d) with ksize &gt; 1 (&#x201C;Proposition 3&#x201D; from FedPara
paper).`,name:"use_effective_conv2d"},{anchor:"peft.LoHaConfig.target_modules",description:`<strong>target_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to apply the adapter to. If this is specified, only the modules with the specified
names will be replaced. When passing a string, a regex match will be performed. When passing a list of
strings, either an exact match will be performed or it is checked if the name of the module ends with any
of the passed strings. If this is specified as &#x2018;all-linear&#x2019;, then all linear/Conv1D modules are chosen,
excluding the output layer. If this is not specified, modules will be chosen according to the model
architecture. If the architecture is not known, an error will be raised &#x2014; in this case, you should specify
the target modules manually.`,name:"target_modules"},{anchor:"peft.LoHaConfig.exclude_modules",description:`<strong>exclude_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to not apply the adapter. When passing a string, a regex match will be performed.
When passing a list of strings, either an exact match will be performed or it is checked if the name of the
module ends with any of the passed strings.`,name:"exclude_modules"},{anchor:"peft.LoHaConfig.init_weights",description:`<strong>init_weights</strong> (<code>bool</code>) &#x2014;
Whether to perform initialization of adapter weights. This defaults to <code>True</code>, passing <code>False</code> is
discouraged.`,name:"init_weights"},{anchor:"peft.LoHaConfig.layers_to_transform",description:`<strong>layers_to_transform</strong> (<code>Union[List[int], int]</code>) &#x2014;
The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices
that are specified in this list. If a single integer is passed, it will apply the transformations on the
layer at this index.`,name:"layers_to_transform"},{anchor:"peft.LoHaConfig.layers_pattern",description:`<strong>layers_pattern</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The layer pattern name, used only if <code>layers_to_transform</code> is different from <code>None</code>. This should target the
<code>nn.ModuleList</code> of the model, which is often called <code>&apos;layers&apos;</code> or <code>&apos;h&apos;</code>.`,name:"layers_pattern"},{anchor:"peft.LoHaConfig.rank_pattern",description:`<strong>rank_pattern</strong> (<code>dict</code>) &#x2014;
The mapping from layer names or regexp expression to ranks which are different from the default rank
specified by <code>r</code>. For example, <code>{&apos;^model.decoder.layers.0.encoder_attn.k_proj&apos;: 16}</code>.`,name:"rank_pattern"},{anchor:"peft.LoHaConfig.alpha_pattern",description:`<strong>alpha_pattern</strong> (<code>dict</code>) &#x2014;
The mapping from layer names or regexp expression to alphas which are different from the default alpha
specified by <code>alpha</code>. For example, <code>{&apos;^model.decoder.layers.0.encoder_attn.k_proj&apos;: 16}</code>.`,name:"alpha_pattern"},{anchor:"peft.LoHaConfig.modules_to_save",description:`<strong>modules_to_save</strong> (<code>Optional[List[str]]</code>) &#x2014;
List of modules apart from adapter layers to be set as trainable and saved in the final checkpoint.`,name:"modules_to_save"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/loha/config.py#L24"}}),I=new ne({props:{title:"LoHaModel",local:"peft.LoHaModel",headingTag:"h2"}}),$=new ce({props:{name:"class peft.LoHaModel",anchor:"peft.LoHaModel",parameters:[{name:"model",val:""},{name:"peft_config",val:": Union[PeftConfig, dict[str, PeftConfig]]"},{name:"adapter_name",val:": str"},{name:"low_cpu_mem_usage",val:": bool = False"},{name:"state_dict",val:": Optional[dict[str, torch.Tensor]] = None"}],parametersDescription:[{anchor:"peft.LoHaModel.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014; The model to which the adapter tuner layers will be attached.",name:"model"},{anchor:"peft.LoHaModel.config",description:'<strong>config</strong> (<a href="/docs/peft/main/en/package_reference/loha#peft.LoHaConfig">LoHaConfig</a>) &#x2014; The configuration of the LoHa model.',name:"config"},{anchor:"peft.LoHaModel.adapter_name",description:"<strong>adapter_name</strong> (<code>str</code>) &#x2014; The name of the adapter, defaults to <code>&quot;default&quot;</code>.",name:"adapter_name"},{anchor:"peft.LoHaModel.low_cpu_mem_usage",description:`<strong>low_cpu_mem_usage</strong> (<code>bool</code>, <code>optional</code>, defaults to <code>False</code>) &#x2014;
Create empty adapter weights on meta device. Useful to speed up the loading process.`,name:"low_cpu_mem_usage"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/loha/model.py#L27",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The LoHa model.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.nn.Module</code></p>
`}}),y=new _e({props:{anchor:"peft.LoHaModel.example",$$slots:{default:[we]},$$scope:{ctx:R}}}),k=new je({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/loha.md"}}),{c(){s=p("meta"),J=l(),h=p("p"),c=l(),_(m.$$.fragment),o=l(),f=p("p"),f.innerHTML=oe,V=l(),v=p("p"),v.textContent=se,N=l(),L=p("p"),L.innerHTML=le,F=l(),_(H.$$.fragment),W=l(),M=p("div"),_(x.$$.fragment),Y=l(),Z=p("p"),Z.innerHTML=re,P=l(),_(I.$$.fragment),S=l(),i=p("div"),_($.$$.fragment),K=l(),q=p("p"),q.innerHTML=ie,ee=l(),_(y.$$.fragment),te=l(),E=p("p"),E.innerHTML=pe,ae=l(),Q=p("ul"),Q.innerHTML=de,G=l(),_(k.$$.fragment),D=l(),B=p("p"),this.h()},l(e){const t=Me("svelte-u9bgzb",document.head);s=d(t,"META",{name:!0,content:!0}),t.forEach(a),J=r(e),h=d(e,"P",{}),X(h).forEach(a),c=r(e),j(m.$$.fragment,e),o=r(e),f=d(e,"P",{"data-svelte-h":!0}),w(f)!=="svelte-5yesx4"&&(f.innerHTML=oe),V=r(e),v=d(e,"P",{"data-svelte-h":!0}),w(v)!=="svelte-1cwsb16"&&(v.textContent=se),N=r(e),L=d(e,"P",{"data-svelte-h":!0}),w(L)!=="svelte-8fm56j"&&(L.innerHTML=le),F=r(e),j(H.$$.fragment,e),W=r(e),M=d(e,"DIV",{class:!0});var A=X(M);j(x.$$.fragment,A),Y=r(A),Z=d(A,"P",{"data-svelte-h":!0}),w(Z)!=="svelte-5surb5"&&(Z.innerHTML=re),A.forEach(a),P=r(e),j(I.$$.fragment,e),S=r(e),i=d(e,"DIV",{class:!0});var u=X(i);j($.$$.fragment,u),K=r(u),q=d(u,"P",{"data-svelte-h":!0}),w(q)!=="svelte-1rluyyf"&&(q.innerHTML=ie),ee=r(u),j(y.$$.fragment,u),te=r(u),E=d(u,"P",{"data-svelte-h":!0}),w(E)!=="svelte-1xx6nm4"&&(E.innerHTML=pe),ae=r(u),Q=d(u,"UL",{"data-svelte-h":!0}),w(Q)!=="svelte-kq88x0"&&(Q.innerHTML=de),u.forEach(a),G=r(e),j(k.$$.fragment,e),D=r(e),B=d(e,"P",{}),X(B).forEach(a),this.h()},h(){z(s,"name","hf:doc:metadata"),z(s,"content",be),z(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(i,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){g(document.head,s),n(e,J,t),n(e,h,t),n(e,c,t),T(m,e,t),n(e,o,t),n(e,f,t),n(e,V,t),n(e,v,t),n(e,N,t),n(e,L,t),n(e,F,t),T(H,e,t),n(e,W,t),n(e,M,t),T(x,M,null),g(M,Y),g(M,Z),n(e,P,t),T(I,e,t),n(e,S,t),n(e,i,t),T($,i,null),g(i,K),g(i,q),g(i,ee),T(y,i,null),g(i,te),g(i,E),g(i,ae),g(i,Q),n(e,G,t),T(k,e,t),n(e,D,t),n(e,B,t),O=!0},p(e,[t]){const A={};t&2&&(A.$$scope={dirty:t,ctx:e}),y.$set(A)},i(e){O||(b(m.$$.fragment,e),b(H.$$.fragment,e),b(x.$$.fragment,e),b(I.$$.fragment,e),b($.$$.fragment,e),b(y.$$.fragment,e),b(k.$$.fragment,e),O=!0)},o(e){C(m.$$.fragment,e),C(H.$$.fragment,e),C(x.$$.fragment,e),C(I.$$.fragment,e),C($.$$.fragment,e),C(y.$$.fragment,e),C(k.$$.fragment,e),O=!1},d(e){e&&(a(J),a(h),a(c),a(o),a(f),a(V),a(v),a(N),a(L),a(F),a(W),a(M),a(P),a(S),a(i),a(G),a(D),a(B)),a(s),U(m,e),U(H,e),U(x),U(I,e),U($),U(y),U(k,e)}}}const be='{"title":"LoHa","local":"loha","sections":[{"title":"LoHaConfig","local":"peft.LoHaConfig","sections":[],"depth":2},{"title":"LoHaModel","local":"peft.LoHaModel","sections":[],"depth":2}],"depth":1}';function Ce(R){return fe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ie extends ue{constructor(s){super(),ge(this,s,Ce,Te,me,{})}}export{Ie as component};
