import{s as Gt,n as Rt,o as Bt}from"../chunks/scheduler.4cd3e004.js";import{S as Kt,i as Qt,e as r,s as o,c as f,h as Jt,a as s,d as e,b as i,f as y,g as u,j as m,k as w,l as a,m as l,n as h,t as b,o as v,p as _}from"../chunks/index.b664cb2d.js";import{D as W}from"../chunks/Docstring.9ceb4334.js";import{H as Et,E as Wt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.179b63f8.js";function Xt(Ot){let C,Z,X,tt,k,et,L,Mt='<a href="https://huggingface.co/papers/2410.17222" rel="nofollow">CPT</a> combines In-Context Learning (ICL), Prompt Tuning (PT), and adversarial optimization to improve few-shot learning by refining context embeddings. CPT updates the context tokens by optimizing both the context and the training examples, encapsulating them into a novel loss design that minimizes overfitting, enables more effective optimization, and drives significant improvements in classification tasks.',nt,E,It="The abstract from the paper is:",at,O,Nt="<p>Large Language Models (LLMs) can perform few-shot learning using either optimization-based approaches or In-Context Learning (ICL). Optimization-based methods often suffer from overfitting, as they require updating a large number of parameters with limited data. In contrast, ICL avoids overfitting but typically underperforms compared to optimization-based methods and is highly sensitive to the selection, order, and format of demonstration examples. To overcome these challenges, we introduce Context-aware Prompt Tuning (CPT), a method inspired by ICL, Prompt Tuning (PT), and adversarial attacks. CPT builds on the ICL strategy of concatenating examples before the input, extending it by incorporating PT-like learning to refine the context embedding through iterative optimization, extracting deeper insights from the training examples. Our approach carefully modifies specific context tokens, considering the unique structure of the examples within the context. In addition to updating the context with PT-like optimization, CPT draws inspiration from adversarial attacks, adjusting the input based on the labels present in the context while preserving the inherent value of the user-provided data. To ensure robustness and stability during optimization, we employ a projected gradient descent algorithm, constraining token embeddings to remain close to their original values and safeguarding the quality of the context. Our method has demonstrated superior accuracy across multiple classification tasks using various LLM models, outperforming existing baselines and effectively addressing the overfitting challenge in few-shot learning.</p>",ot,M,zt='Take a look at <a href="https://github.com/huggingface/peft/blob/main/examples/cpt_finetuning/README.md" rel="nofollow">Example</a> for a step-by-step guide on how to train a model with CPT.',it,I,rt,d,N,ht,S,jt="CPT Configuration class extending PeftConfig for Context-aware Prompt Tuning (CPT).",bt,U,Dt="This class introduces additional parameters required for CPT, such as:",vt,F,Ht="<li>Token type masks</li> <li>Prompt tuning initialization</li> <li>Loss weighting</li> <li>Projection settings</li>",_t,G,qt='For more details, see the paper: <a href="https://huggingface.co/papers/2410.17222" rel="nofollow">https://huggingface.co/papers/2410.17222</a>',st,z,lt,p,j,Ct,R,At=`CPTEmbedding is a custom embedding layer designed for Context-aware Prompt Tuning (CPT) in PEFT. It initializes
embeddings, applies prompt-specific projections, and computes loss using label masks.`,xt,x,D,$t,B,Vt="Computes the loss for CPT models with optional exponential decay.",Tt,$,H,Pt,K,St="Computes the prompt embeddings and applies delta adjustments.",yt,T,q,wt,Q,Ut="Applies epsilon-based projection to the delta embeddings to control their norm.",kt,P,A,Lt,J,Ft="Sets up a backward hook to selectively update token gradients based on the CPT token type mask.",pt,V,dt,Y,mt;return k=new Et({props:{title:"Context-aware Prompt Tuning: Advancing In-Context Learning with Adversarial Methods",local:"context-aware-prompt-tuning-advancing-in-context-learning-with-adversarial-methods",headingTag:"h1"}}),I=new Et({props:{title:"CPTConfig",local:"peft.CPTConfig",headingTag:"h2"}}),N=new W({props:{name:"class peft.CPTConfig",anchor:"peft.CPTConfig",parameters:[{name:"task_type",val:": Optional[Union[str, TaskType]] = None"},{name:"peft_type",val:": Optional[Union[str, PeftType]] = None"},{name:"auto_mapping",val:": Optional[dict] = None"},{name:"peft_version",val:": Optional[str] = None"},{name:"base_model_name_or_path",val:": Optional[str] = None"},{name:"revision",val:": Optional[str] = None"},{name:"inference_mode",val:": bool = False"},{name:"num_virtual_tokens",val:": int = None"},{name:"token_dim",val:": int = None"},{name:"num_transformer_submodules",val:": Optional[int] = None"},{name:"num_attention_heads",val:": Optional[int] = None"},{name:"num_layers",val:": Optional[int] = None"},{name:"modules_to_save",val:": Optional[list[str]] = None"},{name:"cpt_token_ids",val:": typing.Optional[list[int]] = None"},{name:"cpt_mask",val:": typing.Optional[list[int]] = None"},{name:"cpt_tokens_type_mask",val:": typing.Optional[list[int]] = None"},{name:"opt_weighted_loss_type",val:": typing.Optional[typing.Literal['none', 'decay']] = 'none'"},{name:"opt_loss_decay_factor",val:": typing.Optional[float] = 1.0"},{name:"opt_projection_epsilon",val:": typing.Optional[float] = 0.1"},{name:"opt_projection_format_epsilon",val:": typing.Optional[float] = 0.1"},{name:"tokenizer_name_or_path",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/cpt/config.py#L24"}}),z=new Et({props:{title:"CPTEmbedding",local:"peft.CPTEmbedding",headingTag:"h2"}}),j=new W({props:{name:"class peft.CPTEmbedding",anchor:"peft.CPTEmbedding",parameters:[{name:"config",val:""},{name:"word_embeddings",val:""}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/cpt/model.py#L23"}}),D=new W({props:{name:"calculate_loss",anchor:"peft.CPTEmbedding.calculate_loss",parameters:[{name:"base_model_output",val:""},{name:"labels",val:""},{name:"cpt_type_mask",val:""},{name:"config",val:""}],parametersDescription:[{anchor:"peft.CPTEmbedding.calculate_loss.base_model_output",description:`<strong>base_model_output</strong> (ModelOutput) &#x2014;
Output from the base model containing logits.`,name:"base_model_output"},{anchor:"peft.CPTEmbedding.calculate_loss.labels",description:`<strong>labels</strong> (torch.Tensor) &#x2014;
Ground-truth labels for the input tokens.`,name:"labels"},{anchor:"peft.CPTEmbedding.calculate_loss.cpt_type_mask",description:`<strong>cpt_type_mask</strong> (torch.Tensor) &#x2014;
Token type mask used for filtering valid loss terms.`,name:"cpt_type_mask"},{anchor:"peft.CPTEmbedding.calculate_loss.config",description:`<strong>config</strong> (Namespace) &#x2014;
Configuration object containing loss-related hyperparameters.`,name:"config"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/cpt/model.py#L141",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The base model output with computed loss.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>ModelOutput</p>
`}}),H=new W({props:{name:"forward",anchor:"peft.CPTEmbedding.forward",parameters:[{name:"indices",val:""}],parametersDescription:[{anchor:"peft.CPTEmbedding.forward.indices",description:`<strong>indices</strong> (torch.Tensor) &#x2014;
Indices of the tokens to be embedded.`,name:"indices"}],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/cpt/model.py#L63",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Sum of prompt embeddings and delta embeddings.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>torch.Tensor</p>
`}}),q=new W({props:{name:"get_projection",anchor:"peft.CPTEmbedding.get_projection",parameters:[],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/cpt/model.py#L123"}}),A=new W({props:{name:"set_updated_tokens",anchor:"peft.CPTEmbedding.set_updated_tokens",parameters:[],source:"https://github.com/huggingface/peft/blob/main/src/peft/tuners/cpt/model.py#L84"}}),V=new Wt({props:{source:"https://github.com/huggingface/peft/blob/main/docs/source/package_reference/cpt.md"}}),{c(){C=r("meta"),Z=o(),X=r("p"),tt=o(),f(k.$$.fragment),et=o(),L=r("p"),L.innerHTML=Mt,nt=o(),E=r("p"),E.textContent=It,at=o(),O=r("blockquote"),O.innerHTML=Nt,ot=o(),M=r("p"),M.innerHTML=zt,it=o(),f(I.$$.fragment),rt=o(),d=r("div"),f(N.$$.fragment),ht=o(),S=r("p"),S.textContent=jt,bt=o(),U=r("p"),U.textContent=Dt,vt=o(),F=r("ul"),F.innerHTML=Ht,_t=o(),G=r("p"),G.innerHTML=qt,st=o(),f(z.$$.fragment),lt=o(),p=r("div"),f(j.$$.fragment),Ct=o(),R=r("p"),R.textContent=At,xt=o(),x=r("div"),f(D.$$.fragment),$t=o(),B=r("p"),B.textContent=Vt,Tt=o(),$=r("div"),f(H.$$.fragment),Pt=o(),K=r("p"),K.textContent=St,yt=o(),T=r("div"),f(q.$$.fragment),wt=o(),Q=r("p"),Q.textContent=Ut,kt=o(),P=r("div"),f(A.$$.fragment),Lt=o(),J=r("p"),J.textContent=Ft,pt=o(),f(V.$$.fragment),dt=o(),Y=r("p"),this.h()},l(t){const n=Jt("svelte-u9bgzb",document.head);C=s(n,"META",{name:!0,content:!0}),n.forEach(e),Z=i(t),X=s(t,"P",{}),y(X).forEach(e),tt=i(t),u(k.$$.fragment,t),et=i(t),L=s(t,"P",{"data-svelte-h":!0}),m(L)!=="svelte-nhd3j4"&&(L.innerHTML=Mt),nt=i(t),E=s(t,"P",{"data-svelte-h":!0}),m(E)!=="svelte-1cwsb16"&&(E.textContent=It),at=i(t),O=s(t,"BLOCKQUOTE",{"data-svelte-h":!0}),m(O)!=="svelte-166ut3c"&&(O.innerHTML=Nt),ot=i(t),M=s(t,"P",{"data-svelte-h":!0}),m(M)!=="svelte-ih8qk1"&&(M.innerHTML=zt),it=i(t),u(I.$$.fragment,t),rt=i(t),d=s(t,"DIV",{class:!0});var g=y(d);u(N.$$.fragment,g),ht=i(g),S=s(g,"P",{"data-svelte-h":!0}),m(S)!=="svelte-pvpmqk"&&(S.textContent=jt),bt=i(g),U=s(g,"P",{"data-svelte-h":!0}),m(U)!=="svelte-jwcs2b"&&(U.textContent=Dt),vt=i(g),F=s(g,"UL",{"data-svelte-h":!0}),m(F)!=="svelte-aqudvo"&&(F.innerHTML=Ht),_t=i(g),G=s(g,"P",{"data-svelte-h":!0}),m(G)!=="svelte-1rfly87"&&(G.innerHTML=qt),g.forEach(e),st=i(t),u(z.$$.fragment,t),lt=i(t),p=s(t,"DIV",{class:!0});var c=y(p);u(j.$$.fragment,c),Ct=i(c),R=s(c,"P",{"data-svelte-h":!0}),m(R)!=="svelte-1cdhmyj"&&(R.textContent=At),xt=i(c),x=s(c,"DIV",{class:!0});var ct=y(x);u(D.$$.fragment,ct),$t=i(ct),B=s(ct,"P",{"data-svelte-h":!0}),m(B)!=="svelte-9bizft"&&(B.textContent=Vt),ct.forEach(e),Tt=i(c),$=s(c,"DIV",{class:!0});var gt=y($);u(H.$$.fragment,gt),Pt=i(gt),K=s(gt,"P",{"data-svelte-h":!0}),m(K)!=="svelte-3v5t0c"&&(K.textContent=St),gt.forEach(e),yt=i(c),T=s(c,"DIV",{class:!0});var ft=y(T);u(q.$$.fragment,ft),wt=i(ft),Q=s(ft,"P",{"data-svelte-h":!0}),m(Q)!=="svelte-pg5azd"&&(Q.textContent=Ut),ft.forEach(e),kt=i(c),P=s(c,"DIV",{class:!0});var ut=y(P);u(A.$$.fragment,ut),Lt=i(ut),J=s(ut,"P",{"data-svelte-h":!0}),m(J)!=="svelte-1bcpx1v"&&(J.textContent=Ft),ut.forEach(e),c.forEach(e),pt=i(t),u(V.$$.fragment,t),dt=i(t),Y=s(t,"P",{}),y(Y).forEach(e),this.h()},h(){w(C,"name","hf:doc:metadata"),w(C,"content",Yt),w(d,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(p,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,n){a(document.head,C),l(t,Z,n),l(t,X,n),l(t,tt,n),h(k,t,n),l(t,et,n),l(t,L,n),l(t,nt,n),l(t,E,n),l(t,at,n),l(t,O,n),l(t,ot,n),l(t,M,n),l(t,it,n),h(I,t,n),l(t,rt,n),l(t,d,n),h(N,d,null),a(d,ht),a(d,S),a(d,bt),a(d,U),a(d,vt),a(d,F),a(d,_t),a(d,G),l(t,st,n),h(z,t,n),l(t,lt,n),l(t,p,n),h(j,p,null),a(p,Ct),a(p,R),a(p,xt),a(p,x),h(D,x,null),a(x,$t),a(x,B),a(p,Tt),a(p,$),h(H,$,null),a($,Pt),a($,K),a(p,yt),a(p,T),h(q,T,null),a(T,wt),a(T,Q),a(p,kt),a(p,P),h(A,P,null),a(P,Lt),a(P,J),l(t,pt,n),h(V,t,n),l(t,dt,n),l(t,Y,n),mt=!0},p:Rt,i(t){mt||(b(k.$$.fragment,t),b(I.$$.fragment,t),b(N.$$.fragment,t),b(z.$$.fragment,t),b(j.$$.fragment,t),b(D.$$.fragment,t),b(H.$$.fragment,t),b(q.$$.fragment,t),b(A.$$.fragment,t),b(V.$$.fragment,t),mt=!0)},o(t){v(k.$$.fragment,t),v(I.$$.fragment,t),v(N.$$.fragment,t),v(z.$$.fragment,t),v(j.$$.fragment,t),v(D.$$.fragment,t),v(H.$$.fragment,t),v(q.$$.fragment,t),v(A.$$.fragment,t),v(V.$$.fragment,t),mt=!1},d(t){t&&(e(Z),e(X),e(tt),e(et),e(L),e(nt),e(E),e(at),e(O),e(ot),e(M),e(it),e(rt),e(d),e(st),e(lt),e(p),e(pt),e(dt),e(Y)),e(C),_(k,t),_(I,t),_(N),_(z,t),_(j),_(D),_(H),_(q),_(A),_(V,t)}}}const Yt='{"title":"Context-aware Prompt Tuning: Advancing In-Context Learning with Adversarial Methods","local":"context-aware-prompt-tuning-advancing-in-context-learning-with-adversarial-methods","sections":[{"title":"CPTConfig","local":"peft.CPTConfig","sections":[],"depth":2},{"title":"CPTEmbedding","local":"peft.CPTEmbedding","sections":[],"depth":2}],"depth":1}';function Zt(Ot){return Bt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class oe extends Kt{constructor(C){super(),Qt(this,C,Zt,Xt,Gt,{})}}export{oe as component};
