import{s as me,o as he,n as fe}from"../chunks/scheduler.4cd3e004.js";import{S as ge,i as ue,e as p,s as l,c as w,h as Me,a as d,d as a,b as i,f as P,g as b,j as T,k as z,l as u,m as n,n as _,t as j,o as A,p as v}from"../chunks/index.b664cb2d.js";import{D as ce}from"../chunks/Docstring.9ceb4334.js";import{C as ye}from"../chunks/CodeBlock.8601e34e.js";import{E as we}from"../chunks/ExampleCodeBlock.4f57b575.js";import{H as ne,E as be}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.179b63f8.js";function Te(Q){let o,J="Example:",f,c,m;return c=new ye({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBIUkFNb2RlbCUyQyUyMEhSQUNvbmZpZyUwQSUwQWNvbmZpZ190ZSUyMCUzRCUyMEhSQUNvbmZpZyglMEElMjAlMjAlMjAlMjByJTNEOCUyQyUwQSUyMCUyMCUyMCUyMHRhcmdldF9tb2R1bGVzJTNEJTVCJTIya19wcm9qJTIyJTJDJTIwJTIycV9wcm9qJTIyJTJDJTIwJTIydl9wcm9qJTIyJTJDJTIwJTIyb3V0X3Byb2olMjIlMkMlMjAlMjJmYzElMjIlMkMlMjAlMjJmYzIlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBpbml0X3dlaWdodHMlM0RUcnVlJTJDJTBBKSUwQWNvbmZpZ191bmV0JTIwJTNEJTIwSFJBQ29uZmlnKCUwQSUyMCUyMCUyMCUyMHIlM0Q4JTJDJTBBJTIwJTIwJTIwJTIwdGFyZ2V0X21vZHVsZXMlM0QlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwcm9qX2luJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycHJval9vdXQlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0b19rJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydG9fcSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRvX3YlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0b19vdXQuMCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmZmLm5ldC4wLnByb2olMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJmZi5uZXQuMiUyMiUyQyUwQSUyMCUyMCUyMCUyMCU1RCUyQyUwQSUyMCUyMCUyMCUyMGluaXRfd2VpZ2h0cyUzRFRydWUlMkMlMEEpJTBBJTBBbW9kZWwlMjAlM0QlMjBTdGFibGVEaWZmdXNpb25QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTIycnVud2F5bWwlMkZzdGFibGUtZGlmZnVzaW9uLXYxLTUlMjIpJTBBbW9kZWwudGV4dF9lbmNvZGVyJTIwJTNEJTIwSFJBTW9kZWwobW9kZWwudGV4dF9lbmNvZGVyJTJDJTIwY29uZmlnX3RlJTJDJTIwJTIyZGVmYXVsdCUyMiklMEFtb2RlbC51bmV0JTIwJTNEJTIwSFJBTW9kZWwobW9kZWwudW5ldCUyQyUyMGNvbmZpZ191bmV0JTJDJTIwJTIyZGVmYXVsdCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> HRAModel, HRAConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>config_te = HRAConfig(
<span class="hljs-meta">... </span>    r=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    target_modules=[<span class="hljs-string">&quot;k_proj&quot;</span>, <span class="hljs-string">&quot;q_proj&quot;</span>, <span class="hljs-string">&quot;v_proj&quot;</span>, <span class="hljs-string">&quot;out_proj&quot;</span>, <span class="hljs-string">&quot;fc1&quot;</span>, <span class="hljs-string">&quot;fc2&quot;</span>],
<span class="hljs-meta">... </span>    init_weights=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>config_unet = HRAConfig(
<span class="hljs-meta">... </span>    r=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    target_modules=[
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;proj_in&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;proj_out&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_k&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_q&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_v&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_out.0&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;ff.net.0.proj&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;ff.net.2&quot;</span>,
<span class="hljs-meta">... </span>    ],
<span class="hljs-meta">... </span>    init_weights=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = StableDiffusionPipeline.from_pretrained(<span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.text_encoder = HRAModel(model.text_encoder, config_te, <span class="hljs-string">&quot;default&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.unet = HRAModel(model.unet, config_unet, <span class="hljs-string">&quot;default&quot;</span>)`,wrap:!1}}),{c(){o=p("p"),o.textContent=J,f=l(),w(c.$$.fragment)},l(s){o=d(s,"P",{"data-svelte-h":!0}),T(o)!=="svelte-11lpom8"&&(o.textContent=J),f=i(s),b(c.$$.fragment,s)},m(s,h){n(s,o,h),n(s,f,h),_(c,s,h),m=!0},p:fe,i(s){m||(j(c.$$.fragment,s),m=!0)},o(s){A(c.$$.fragment,s),m=!1},d(s){s&&(a(o),a(f)),v(c,s)}}}function _e(Q){let o,J,f,c,m,s,h,se='<a href="https://huggingface.co/papers/2405.17484" rel="nofollow">HRA</a> is a simple but effective adapter-based fine-tuning method by leveraging Householder reflections. This method harnesses the advantages of both strategies, reducing parameters and computation costs while penalizing the loss of pre-training knowledge. It consistently achieves better performance with fewer trainable parameters and outperforms state-of-the-art adapters across different models, including large language models (LLMs) and conditional image generators.',W,U,oe="The abstract from the paper is:",G,C,le='<p>While following different technical routes, both low-rank and orthogonal adaptation techniques can efficiently adapt large-scale pre-training models in specific tasks or domains based on a small piece of trainable parameters. In this study, we bridge the gap between these two techniques, proposing a simple but effective adaptation method based on Householder reflections. Given a pre-trained model, our method fine-tunes its layers by multiplying each frozen weight matrix with an orthogonal matrix constructed by a chain of learnable Householder reflections (HRs). This HR-based orthogonal fine-tuning is equivalent to an adaptive low-rank adaptation. Moreover, we show that the orthogonality of the reflection planes corresponding to the HRs impacts the model capacity and regularity. The analysis motivates us to regularize the orthogonality of the HRs, leading to different implementations of the proposed Householder reflection adaptation (HRA) method. Compared with state-of-the-art methods, HRA achieves superior performance with fewer learnable parameters when adapting large language models and conditional image generators. The code is available at <a href="https://github.com/huggingface/peft/tree/main/src/peft/tuners/hra" rel="nofollow">peft</a> and <a href="https://github.com/DaShenZi721/HRA" rel="nofollow">HRA</a>.</p>',N,H,S,M,$,Y,q,ie='This is the configuration class to store the configuration of a <a href="/docs/peft/main/en/package_reference/hra#peft.HRAModel">HRAModel</a>.',V,R,O,r,I,K,Z,re=`Creates Householder reflection adaptation (HRA) model from a pretrained model. The method is described in
<a href="https://huggingface.co/papers/2405.17484" rel="nofollow">https://huggingface.co/papers/2405.17484</a>`,ee,y,te,B,pe="<strong>Attributes</strong>:",ae,E,de='<li><strong>model</strong> (<code>~torch.nn.Module</code>) — The model to be adapted.</li> <li><strong>peft_config</strong> (<a href="/docs/peft/main/en/package_reference/hra#peft.HRAConfig">HRAConfig</a>): The configuration of the HRA model.</li>',D,x,X,L,F;return m=new ne({props:{title:"Bridging The Gap between Low-rank and Orthogonal Adaptation via Householder Reflection Adaptation (HRA)",local:"bridging-the-gap-between-low-rank-and-orthogonal-adaptation-via-householder-reflection-adaptation-hra",headingTag:"h1"}}),H=new ne({props:{title:"HRAConfig",local:"peft.HRAConfig",headingTag:"h2"}}),$=new ce({props:{name:"class peft.HRAConfig",anchor:"peft.HRAConfig",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"r",val:": int = 8"},{name:"apply_GS",val:": bool = False"},{name:"target_modules",val:": Optional[Union[list[str], str]] = None"},{name:"exclude_modules",val:": Optional[Union[list[str], str]] = None"},{name:"init_weights",val:": bool = True"},{name:"layers_to_transform",val:": Optional[Union[list[int], int]] = None"},{name:"layers_pattern",val:": Optional[Union[list[str], str]] = None"},{name:"bias",val:": str = 'none'"},{name:"modules_to_save",val:": Optional[list[str]] = None"}],parametersDescription:[{anchor:"peft.HRAConfig.r",description:`<strong>r</strong> (<code>int</code>) &#x2014;
The rank of HRA across different layers. It is best to set &#x2018;r&#x2019; to an even number; otherwise, the default
initialization method will not work.`,name:"r"},{anchor:"peft.HRAConfig.apply_GS",description:`<strong>apply_GS</strong> (<code>bool</code>) &#x2014;
Whether to apply Gram-Schmidt orthogonalization.`,name:"apply_GS"},{anchor:"peft.HRAConfig.target_modules",description:`<strong>target_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to apply the adapter to. If this is specified, only the modules with the specified
names will be replaced. When passing a string, a regex match will be performed. When passing a list of
strings, either an exact match will be performed or it is checked if the name of the module ends with any
of the passed strings. If this is specified as &#x2018;all-linear&#x2019;, then all linear modules are chosen, excluding
the output layer. If this is not specified, modules will be chosen according to the model architecture. If
the architecture is not known, an error will be raised &#x2014; in this case, you should specify the target
modules manually.`,name:"target_modules"},{anchor:"peft.HRAConfig.exclude_modules",description:`<strong>exclude_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to not apply the adapter. When passing a string, a regex match will be performed.
When passing a list of strings, either an exact match will be performed or it is checked if the name of the
module ends with any of the passed strings.`,name:"exclude_modules"},{anchor:"peft.HRAConfig.init_weights",description:`<strong>init_weights</strong> (<code>bool</code>) &#x2014;
Whether to perform initialization of HRA weights.`,name:"init_weights"},{anchor:"peft.HRAConfig.layers_to_transform",description:`<strong>layers_to_transform</strong> (<code>Union[List[int], int]</code>) &#x2014;
The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices
that are specified in this list. If a single integer is passed, it will apply the transformations on the
layer at this index.`,name:"layers_to_transform"},{anchor:"peft.HRAConfig.layers_pattern",description:`<strong>layers_pattern</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The layer pattern name, used only if <code>layers_to_transform</code> is different from <code>None</code>. This should target the
<code>nn.ModuleList</code> of the model, which is often called <code>&apos;layers&apos;</code> or <code>&apos;h&apos;</code>.`,name:"layers_pattern"},{anchor:"peft.HRAConfig.modules_to_save",description:`<strong>modules_to_save</strong> (<code>List[str]</code>) &#x2014;
List of modules apart from adapter layers to be set as trainable and saved in the final checkpoint.`,name:"modules_to_save"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/hra/config.py#L25"}}),R=new ne({props:{title:"HRAModel",local:"peft.HRAModel",headingTag:"h2"}}),I=new ce({props:{name:"class peft.HRAModel",anchor:"peft.HRAModel",parameters:[{name:"model",val:""},{name:"peft_config",val:": Union[PeftConfig, dict[str, PeftConfig]]"},{name:"adapter_name",val:": str"},{name:"low_cpu_mem_usage",val:": bool = False"},{name:"state_dict",val:": Optional[dict[str, torch.Tensor]] = None"}],parametersDescription:[{anchor:"peft.HRAModel.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014; The model to which the adapter tuner layers will be attached.",name:"model"},{anchor:"peft.HRAModel.config",description:'<strong>config</strong> (<a href="/docs/peft/main/en/package_reference/hra#peft.HRAConfig">HRAConfig</a>) &#x2014; The configuration of the HRA model.',name:"config"},{anchor:"peft.HRAModel.adapter_name",description:"<strong>adapter_name</strong> (<code>str</code>) &#x2014; The name of the adapter, defaults to <code>&quot;default&quot;</code>.",name:"adapter_name"},{anchor:"peft.HRAModel.low_cpu_mem_usage",description:`<strong>low_cpu_mem_usage</strong> (<code>bool</code>, <code>optional</code>, defaults to <code>False</code>) &#x2014;
Create empty adapter weights on meta device. Useful to speed up the loading process.`,name:"low_cpu_mem_usage"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/hra/model.py#L24",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The HRA model.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.nn.Module</code></p>
`}}),y=new we({props:{anchor:"peft.HRAModel.example",$$slots:{default:[Te]},$$scope:{ctx:Q}}}),x=new be({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/hra.md"}}),{c(){o=p("meta"),J=l(),f=p("p"),c=l(),w(m.$$.fragment),s=l(),h=p("p"),h.innerHTML=se,W=l(),U=p("p"),U.textContent=oe,G=l(),C=p("blockquote"),C.innerHTML=le,N=l(),w(H.$$.fragment),S=l(),M=p("div"),w($.$$.fragment),Y=l(),q=p("p"),q.innerHTML=ie,V=l(),w(R.$$.fragment),O=l(),r=p("div"),w(I.$$.fragment),K=l(),Z=p("p"),Z.innerHTML=re,ee=l(),w(y.$$.fragment),te=l(),B=p("p"),B.innerHTML=pe,ae=l(),E=p("ul"),E.innerHTML=de,D=l(),w(x.$$.fragment),X=l(),L=p("p"),this.h()},l(e){const t=Me("svelte-u9bgzb",document.head);o=d(t,"META",{name:!0,content:!0}),t.forEach(a),J=i(e),f=d(e,"P",{}),P(f).forEach(a),c=i(e),b(m.$$.fragment,e),s=i(e),h=d(e,"P",{"data-svelte-h":!0}),T(h)!=="svelte-1wazsr6"&&(h.innerHTML=se),W=i(e),U=d(e,"P",{"data-svelte-h":!0}),T(U)!=="svelte-1cwsb16"&&(U.textContent=oe),G=i(e),C=d(e,"BLOCKQUOTE",{"data-svelte-h":!0}),T(C)!=="svelte-bkd9l0"&&(C.innerHTML=le),N=i(e),b(H.$$.fragment,e),S=i(e),M=d(e,"DIV",{class:!0});var k=P(M);b($.$$.fragment,k),Y=i(k),q=d(k,"P",{"data-svelte-h":!0}),T(q)!=="svelte-b9w9h2"&&(q.innerHTML=ie),k.forEach(a),V=i(e),b(R.$$.fragment,e),O=i(e),r=d(e,"DIV",{class:!0});var g=P(r);b(I.$$.fragment,g),K=i(g),Z=d(g,"P",{"data-svelte-h":!0}),T(Z)!=="svelte-oky5id"&&(Z.innerHTML=re),ee=i(g),b(y.$$.fragment,g),te=i(g),B=d(g,"P",{"data-svelte-h":!0}),T(B)!=="svelte-1xx6nm4"&&(B.innerHTML=pe),ae=i(g),E=d(g,"UL",{"data-svelte-h":!0}),T(E)!=="svelte-nyk3vg"&&(E.innerHTML=de),g.forEach(a),D=i(e),b(x.$$.fragment,e),X=i(e),L=d(e,"P",{}),P(L).forEach(a),this.h()},h(){z(o,"name","hf:doc:metadata"),z(o,"content",je),z(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(r,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){u(document.head,o),n(e,J,t),n(e,f,t),n(e,c,t),_(m,e,t),n(e,s,t),n(e,h,t),n(e,W,t),n(e,U,t),n(e,G,t),n(e,C,t),n(e,N,t),_(H,e,t),n(e,S,t),n(e,M,t),_($,M,null),u(M,Y),u(M,q),n(e,V,t),_(R,e,t),n(e,O,t),n(e,r,t),_(I,r,null),u(r,K),u(r,Z),u(r,ee),_(y,r,null),u(r,te),u(r,B),u(r,ae),u(r,E),n(e,D,t),_(x,e,t),n(e,X,t),n(e,L,t),F=!0},p(e,[t]){const k={};t&2&&(k.$$scope={dirty:t,ctx:e}),y.$set(k)},i(e){F||(j(m.$$.fragment,e),j(H.$$.fragment,e),j($.$$.fragment,e),j(R.$$.fragment,e),j(I.$$.fragment,e),j(y.$$.fragment,e),j(x.$$.fragment,e),F=!0)},o(e){A(m.$$.fragment,e),A(H.$$.fragment,e),A($.$$.fragment,e),A(R.$$.fragment,e),A(I.$$.fragment,e),A(y.$$.fragment,e),A(x.$$.fragment,e),F=!1},d(e){e&&(a(J),a(f),a(c),a(s),a(h),a(W),a(U),a(G),a(C),a(N),a(S),a(M),a(V),a(O),a(r),a(D),a(X),a(L)),a(o),v(m,e),v(H,e),v($),v(R,e),v(I),v(y),v(x,e)}}}const je='{"title":"Bridging The Gap between Low-rank and Orthogonal Adaptation via Householder Reflection Adaptation (HRA)","local":"bridging-the-gap-between-low-rank-and-orthogonal-adaptation-via-householder-reflection-adaptation-hra","sections":[{"title":"HRAConfig","local":"peft.HRAConfig","sections":[],"depth":2},{"title":"HRAModel","local":"peft.HRAModel","sections":[],"depth":2}],"depth":1}';function Ae(Q){return he(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Re extends ge{constructor(o){super(),ue(this,o,Ae,_e,me,{})}}export{Re as component};
