import{s as de,o as fe,n as he}from"../chunks/scheduler.4cd3e004.js";import{S as ue,i as ge,e as p,s as l,c as T,h as Me,a as c,d as n,b as i,f as X,g as w,j,k as z,l as g,m as a,n as b,t as _,o as J,p as B}from"../chunks/index.b664cb2d.js";import{D as me}from"../chunks/Docstring.9ceb4334.js";import{C as ye}from"../chunks/CodeBlock.8601e34e.js";import{E as Te}from"../chunks/ExampleCodeBlock.4f57b575.js";import{H as ae,E as we}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.179b63f8.js";function je(H){let o,v="Example:",h,m,d;return m=new ye({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBCb25lTW9kZWwlMkMlMjBCb25lQ29uZmlnJTBBJTBBY29uZmlnX3RlJTIwJTNEJTIwQm9uZUNvbmZpZyglMEElMjAlMjAlMjAlMjByJTNEOCUyQyUwQSUyMCUyMCUyMCUyMHRhcmdldF9tb2R1bGVzJTNEJTVCJTIya19wcm9qJTIyJTJDJTIwJTIycV9wcm9qJTIyJTJDJTIwJTIydl9wcm9qJTIyJTJDJTIwJTIyb3V0X3Byb2olMjIlMkMlMjAlMjJmYzElMjIlMkMlMjAlMjJmYzIlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBpbml0X3dlaWdodHMlM0RUcnVlJTJDJTBBKSUwQWNvbmZpZ191bmV0JTIwJTNEJTIwQm9uZUNvbmZpZyglMEElMjAlMjAlMjAlMjByJTNEOCUyQyUwQSUyMCUyMCUyMCUyMHRhcmdldF9tb2R1bGVzJTNEJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycHJval9pbiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnByb2pfb3V0JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydG9fayUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRvX3ElMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0b192JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydG9fb3V0LjAlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJmZi5uZXQuMC5wcm9qJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZmYubmV0LjIlMjIlMkMlMEElMjAlMjAlMjAlMjAlNUQlMkMlMEElMjAlMjAlMjAlMjBpbml0X3dlaWdodHMlM0RUcnVlJTJDJTBBKSUwQSUwQW1vZGVsJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyKSUwQW1vZGVsLnRleHRfZW5jb2RlciUyMCUzRCUyMEJvbmVNb2RlbChtb2RlbC50ZXh0X2VuY29kZXIlMkMlMjBjb25maWdfdGUlMkMlMjAlMjJkZWZhdWx0JTIyKSUwQW1vZGVsLnVuZXQlMjAlM0QlMjBCb25lTW9kZWwobW9kZWwudW5ldCUyQyUyMGNvbmZpZ191bmV0JTJDJTIwJTIyZGVmYXVsdCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> BoneModel, BoneConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>config_te = BoneConfig(
<span class="hljs-meta">... </span>    r=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    target_modules=[<span class="hljs-string">&quot;k_proj&quot;</span>, <span class="hljs-string">&quot;q_proj&quot;</span>, <span class="hljs-string">&quot;v_proj&quot;</span>, <span class="hljs-string">&quot;out_proj&quot;</span>, <span class="hljs-string">&quot;fc1&quot;</span>, <span class="hljs-string">&quot;fc2&quot;</span>],
<span class="hljs-meta">... </span>    init_weights=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>config_unet = BoneConfig(
<span class="hljs-meta">... </span>    r=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    target_modules=[
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;proj_in&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;proj_out&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_k&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_q&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_v&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;to_out.0&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;ff.net.0.proj&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;ff.net.2&quot;</span>,
<span class="hljs-meta">... </span>    ],
<span class="hljs-meta">... </span>    init_weights=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = StableDiffusionPipeline.from_pretrained(<span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.text_encoder = BoneModel(model.text_encoder, config_te, <span class="hljs-string">&quot;default&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.unet = BoneModel(model.unet, config_unet, <span class="hljs-string">&quot;default&quot;</span>)`,wrap:!1}}),{c(){o=p("p"),o.textContent=v,h=l(),T(m.$$.fragment)},l(s){o=c(s,"P",{"data-svelte-h":!0}),j(o)!=="svelte-11lpom8"&&(o.textContent=v),h=i(s),w(m.$$.fragment,s)},m(s,f){a(s,o,f),a(s,h,f),b(m,s,f),d=!0},p:he,i(s){d||(_(m.$$.fragment,s),d=!0)},o(s){J(m.$$.fragment,s),d=!1},d(s){s&&(n(o),n(h)),B(m,s)}}}function be(H){let o,v,h,m,d,s,f,se='DiSHA: Dimension-Sharding Adaptation (<a href="https://huggingface.co/papers/2409.15371" rel="nofollow">DiSHA</a>) We introduce Dimension-Sharding Adaptation (DiSHA), which expands the PEFT design space to unlock lower intrinsic ranks and faster convergence by default. Building on DiSHA, we propose an efficient algorithm called Block-Affine Adaptation (Bone) structure and a non-linear update method called Block Affine Transformation Adaptation (BAT).',N,C,oe="The abstract from the paper is:",S,I,le="Low-Rank Adaptation (LoRA) leverages the low intrinsic rank of weight updates in Large Language Models (LLMs), establishing a Parameter-Efficient Fine-Tuning (PEFT) paradigm. However, LoRA suffers from slow convergence. We introduce Dimension-Sharding Adaptation (DiSHA), which expands the PEFT design space to unlock lower intrinsic ranks and faster convergence by default. Within DiSHA’s design space, we propose Block Affine Adaptation (Bone), a computationally efficient structure that delivers both high performance and efficiency. While certain DiSHA configurations may result in colinear updates to weight shards, we address this with Block Affine Transformation Adaptation (BAT), a nonlinear variant of DiSHA. BAT introduces nonlinearity by combining trainable matrices with original weight shards in a nonlinear manner, inducing nonlinearity in matrix updates without introducing additional parameters. Empirical results show that Bone, under the DiSHA framework, consistently outperforms LoRA variants in both NLG and NLU tasks, with significantly improved computational efficiency. Further analysis demonstrates that BAT enhances model capabilities by leveraging its nonlinear design.",V,U,R,M,$,Y,Z,ie='This is the configuration class to store the configuration of a <a href="/docs/peft/main/en/package_reference/bone#peft.BoneModel">BoneModel</a>.',Q,x,P,r,A,K,E,re=`Creates Householder reflection adaptation (Bone) model from a pretrained model. The method is described in
<a href="https://huggingface.co/papers/2409.15371" rel="nofollow">https://huggingface.co/papers/2409.15371</a>`,ee,y,te,L,pe="<strong>Attributes</strong>:",ne,D,ce='<li><strong>model</strong> (<code>~torch.nn.Module</code>) — The model to be adapted.</li> <li><strong>peft_config</strong> (<a href="/docs/peft/main/en/package_reference/bone#peft.BoneConfig">BoneConfig</a>): The configuration of the Bone model.</li>',G,k,O,W,F;return d=new ae({props:{title:"Bone",local:"bone",headingTag:"h1"}}),U=new ae({props:{title:"BoneConfig",local:"peft.BoneConfig",headingTag:"h2"}}),$=new me({props:{name:"class peft.BoneConfig",anchor:"peft.BoneConfig",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"r",val:": int = 64"},{name:"target_modules",val:": Optional[Union[list[str], str]] = None"},{name:"exclude_modules",val:": Optional[Union[list[str], str]] = None"},{name:"init_weights",val:": bool | Literal['bat'] = True"},{name:"layers_to_transform",val:": Optional[Union[list[int], int]] = None"},{name:"layers_pattern",val:": Optional[str] = None"},{name:"bias",val:": str = 'none'"},{name:"modules_to_save",val:": Optional[list[str]] = None"}],parametersDescription:[{anchor:"peft.BoneConfig.r",description:`<strong>r</strong> (<code>int</code>) &#x2014;
The rank of Bone across different layers. It is best to set &#x2018;r&#x2019; to an even number; otherwise, the default
initialization method will not work.`,name:"r"},{anchor:"peft.BoneConfig.target_modules",description:`<strong>target_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to apply the adapter to. If this is specified, only the modules with the specified
names will be replaced. When passing a string, a regex match will be performed. When passing a list of
strings, either an exact match will be performed or it is checked if the name of the module ends with any
of the passed strings. If this is specified as &#x2018;all-linear&#x2019;, then all linear modules are chosen, excluding
the output layer. If this is not specified, modules will be chosen according to the model architecture. If
the architecture is not known, an error will be raised &#x2014; in this case, you should specify the target
modules manually.`,name:"target_modules"},{anchor:"peft.BoneConfig.exclude_modules",description:`<strong>exclude_modules</strong> (<code>Optional[Union[List[str], str]]</code>) &#x2014;
The names of the modules to not apply the adapter. When passing a string, a regex match will be performed.
When passing a list of strings, either an exact match will be performed or it is checked if the name of the
module ends with any of the passed strings.`,name:"exclude_modules"},{anchor:"peft.BoneConfig.init_weights",description:`<strong>init_weights</strong> (bool | Literal[&#x201C;bat&#x201D;]) &#x2014;
Different initializations correspond to different Bone variants. By default, setting True uses the Bone
structure, while &#x201C;bat&#x201D; selects the Bat structure.`,name:"init_weights"},{anchor:"peft.BoneConfig.layers_to_transform",description:`<strong>layers_to_transform</strong> (<code>Union[List[int], int]</code>) &#x2014;
The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices
that are specified in this list. If a single integer is passed, it will apply the transformations on the
layer at this index.`,name:"layers_to_transform"},{anchor:"peft.BoneConfig.layers_pattern",description:`<strong>layers_pattern</strong> (<code>str</code>) &#x2014;
The layer pattern name, used only if <code>layers_to_transform</code> is different from <code>None</code>.`,name:"layers_pattern"},{anchor:"peft.BoneConfig.modules_to_save",description:`<strong>modules_to_save</strong> (<code>List[str]</code>) &#x2014;
List of modules apart from adapter layers to be set as trainable and saved in the final checkpoint.`,name:"modules_to_save"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/bone/config.py#L26"}}),x=new ae({props:{title:"BoneModel",local:"peft.BoneModel",headingTag:"h2"}}),A=new me({props:{name:"class peft.BoneModel",anchor:"peft.BoneModel",parameters:[{name:"model",val:""},{name:"peft_config",val:": Union[PeftConfig, dict[str, PeftConfig]]"},{name:"adapter_name",val:": str"},{name:"low_cpu_mem_usage",val:": bool = False"},{name:"state_dict",val:": Optional[dict[str, torch.Tensor]] = None"}],parametersDescription:[{anchor:"peft.BoneModel.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014; The model to which the adapter tuner layers will be attached.",name:"model"},{anchor:"peft.BoneModel.config",description:'<strong>config</strong> (<a href="/docs/peft/main/en/package_reference/bone#peft.BoneConfig">BoneConfig</a>) &#x2014; The configuration of the Bone model.',name:"config"},{anchor:"peft.BoneModel.adapter_name",description:"<strong>adapter_name</strong> (<code>str</code>) &#x2014; The name of the adapter, defaults to <code>&quot;default&quot;</code>.",name:"adapter_name"},{anchor:"peft.BoneModel.low_cpu_mem_usage",description:`<strong>low_cpu_mem_usage</strong> (<code>bool</code>, <code>optional</code>, defaults to <code>False</code>) &#x2014;
Create empty adapter weights on meta device. Useful to speed up the loading process.`,name:"low_cpu_mem_usage"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/bone/model.py#L24",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The Bone model.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.nn.Module</code></p>
`}}),y=new Te({props:{anchor:"peft.BoneModel.example",$$slots:{default:[je]},$$scope:{ctx:H}}}),k=new we({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/bone.md"}}),{c(){o=p("meta"),v=l(),h=p("p"),m=l(),T(d.$$.fragment),s=l(),f=p("p"),f.innerHTML=se,N=l(),C=p("p"),C.textContent=oe,S=l(),I=p("p"),I.textContent=le,V=l(),T(U.$$.fragment),R=l(),M=p("div"),T($.$$.fragment),Y=l(),Z=p("p"),Z.innerHTML=ie,Q=l(),T(x.$$.fragment),P=l(),r=p("div"),T(A.$$.fragment),K=l(),E=p("p"),E.innerHTML=re,ee=l(),T(y.$$.fragment),te=l(),L=p("p"),L.innerHTML=pe,ne=l(),D=p("ul"),D.innerHTML=ce,G=l(),T(k.$$.fragment),O=l(),W=p("p"),this.h()},l(e){const t=Me("svelte-u9bgzb",document.head);o=c(t,"META",{name:!0,content:!0}),t.forEach(n),v=i(e),h=c(e,"P",{}),X(h).forEach(n),m=i(e),w(d.$$.fragment,e),s=i(e),f=c(e,"P",{"data-svelte-h":!0}),j(f)!=="svelte-aemu8y"&&(f.innerHTML=se),N=i(e),C=c(e,"P",{"data-svelte-h":!0}),j(C)!=="svelte-1cwsb16"&&(C.textContent=oe),S=i(e),I=c(e,"P",{"data-svelte-h":!0}),j(I)!=="svelte-18pyqgo"&&(I.textContent=le),V=i(e),w(U.$$.fragment,e),R=i(e),M=c(e,"DIV",{class:!0});var q=X(M);w($.$$.fragment,q),Y=i(q),Z=c(q,"P",{"data-svelte-h":!0}),j(Z)!=="svelte-i6y6q9"&&(Z.innerHTML=ie),q.forEach(n),Q=i(e),w(x.$$.fragment,e),P=i(e),r=c(e,"DIV",{class:!0});var u=X(r);w(A.$$.fragment,u),K=i(u),E=c(u,"P",{"data-svelte-h":!0}),j(E)!=="svelte-186pinq"&&(E.innerHTML=re),ee=i(u),w(y.$$.fragment,u),te=i(u),L=c(u,"P",{"data-svelte-h":!0}),j(L)!=="svelte-1xx6nm4"&&(L.innerHTML=pe),ne=i(u),D=c(u,"UL",{"data-svelte-h":!0}),j(D)!=="svelte-1l06rms"&&(D.innerHTML=ce),u.forEach(n),G=i(e),w(k.$$.fragment,e),O=i(e),W=c(e,"P",{}),X(W).forEach(n),this.h()},h(){z(o,"name","hf:doc:metadata"),z(o,"content",_e),z(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(r,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){g(document.head,o),a(e,v,t),a(e,h,t),a(e,m,t),b(d,e,t),a(e,s,t),a(e,f,t),a(e,N,t),a(e,C,t),a(e,S,t),a(e,I,t),a(e,V,t),b(U,e,t),a(e,R,t),a(e,M,t),b($,M,null),g(M,Y),g(M,Z),a(e,Q,t),b(x,e,t),a(e,P,t),a(e,r,t),b(A,r,null),g(r,K),g(r,E),g(r,ee),b(y,r,null),g(r,te),g(r,L),g(r,ne),g(r,D),a(e,G,t),b(k,e,t),a(e,O,t),a(e,W,t),F=!0},p(e,[t]){const q={};t&2&&(q.$$scope={dirty:t,ctx:e}),y.$set(q)},i(e){F||(_(d.$$.fragment,e),_(U.$$.fragment,e),_($.$$.fragment,e),_(x.$$.fragment,e),_(A.$$.fragment,e),_(y.$$.fragment,e),_(k.$$.fragment,e),F=!0)},o(e){J(d.$$.fragment,e),J(U.$$.fragment,e),J($.$$.fragment,e),J(x.$$.fragment,e),J(A.$$.fragment,e),J(y.$$.fragment,e),J(k.$$.fragment,e),F=!1},d(e){e&&(n(v),n(h),n(m),n(s),n(f),n(N),n(C),n(S),n(I),n(V),n(R),n(M),n(Q),n(P),n(r),n(G),n(O),n(W)),n(o),B(d,e),B(U,e),B($),B(x,e),B(A),B(y),B(k,e)}}}const _e='{"title":"Bone","local":"bone","sections":[{"title":"BoneConfig","local":"peft.BoneConfig","sections":[],"depth":2},{"title":"BoneModel","local":"peft.BoneModel","sections":[],"depth":2}],"depth":1}';function Je(H){return fe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class xe extends ue{constructor(o){super(),ge(this,o,Je,be,de,{})}}export{xe as component};
