import{s as ve,n as Me,o as Pe}from"../chunks/scheduler.d627b047.js";import{S as Le,i as Ce,g as r,s,r as i,A as qe,h as o,f as a,c as l,j as ye,u as m,x as h,k as Te,y as xe,a as n,v as p,d,t as u,w as c}from"../chunks/index.73c51727.js";import{C as pe}from"../chunks/CodeBlock.5f78c87f.js";import{H as A,E as je}from"../chunks/index.8df1db42.js";function He(de){let f,E,F,O,g,V,$,ue="Here we present an approach that uses a single base model for the entire PPO algorithm - which includes retrieving the reference logits, computing the active logits and the rewards. This feature is experimental as we did not test the convergence of the approach. We encourage the community to let us know if they potentially face issues.",X,_,I,b,ce="You just need to install <code>peft</code> and optionally install <code>bitsandbytes</code> as well if you want to go for 8bit base models, for more memory efficient finetuning.",S,w,Z,y,fe="You need to address this approach in three stages that we summarize as follows:",G,T,he=`1- Train a base model on the target domain (e.g. <a href="https://huggingface.co/datasets/stanfordnlp/imdb" rel="nofollow">IMDB dataset</a>) - this is the Supervised Fine Tuning stage - it can leverage the <code>SFTTrainer</code> from TRL.
2- Train a reward model using <code>peft</code>. This is required in order to re-use the adapter during the RL optimisation process (step 3 below). We show an example of leveraging the <code>RewardTrainer</code> from TRL in <a href="https://github.com/huggingface/trl/tree/main/examples/scripts/reward_modeling.py" rel="nofollow">this example</a>
3- Fine tune new adapters on the base model using PPO and the reward adapter. (“0 abstraction RL”)`,Y,v,ge="Make sure to use the same model (i.e. same architecture and same weights) for the stages 2 & 3.",z,M,N,P,$e=`Let us assume you have trained your reward adapter on <code>llama-7b</code> model using <code>RewardTrainer</code> and pushed the weights on the hub under <code>trl-lib/llama-7b-hh-rm-adapter</code>.
When doing PPO, before passing the model to <code>PPOTrainer</code> create your model as follows:`,Q,L,K,C,_e="Then inside your PPO training loop, call the <code>compute_reward_score</code> method by accessing the <code>model</code> attribute from <code>PPOTrainer</code>.",D,q,ee,x,te,j,ae,H,be=`If you are familiar with the <code>peft</code> library, you know that you can use multiple adapters inside the same model. What you can do is train multiple adapters on the same base model to fine-tune on different policies.
In this case, you want to be able to control the adapter name you want to activate back, after retrieving the reward. For that, simply pass the appropriate <code>adapter_name</code> to <code>ppo_adapter_name</code> argument when calling <code>compute_reward_score</code>.`,ne,k,se,R,le,B,we=`For more memory efficient fine-tuning, you can load your base model in 8-bit or 4-bit while keeping the adapters in the default precision (float32).
Just pass the appropriate arguments (i.e. <code>load_in_8bit=True</code> or <code>load_in_4bit=True</code>) to <code>AutoModelForCausalLMWithValueHead.from_pretrained</code> as follows (assuming you have installed <code>bitsandbytes</code>):`,re,J,ie,W,oe,U,me;return g=new A({props:{title:"Multi Adapter RL (MARL) - a single base model for everything",local:"multi-adapter-rl-marl---a-single-base-model-for-everything",headingTag:"h1"}}),_=new A({props:{title:"Requirements",local:"requirements",headingTag:"h2"}}),w=new A({props:{title:"Summary",local:"summary",headingTag:"h2"}}),M=new A({props:{title:"Quickstart",local:"quickstart",headingTag:"h2"}}),L=new pe({props:{code:"JTBBJTBBJTBB",highlighted:`model_name = <span class="hljs-string">&quot;huggyllama/llama-7b&quot;</span>
rm_adapter_id = <span class="hljs-string">&quot;trl-lib/llama-7b-hh-rm-adapter&quot;</span>

<span class="hljs-comment"># PPO adapter</span>
lora_config = LoraConfig(
    r=<span class="hljs-number">16</span>,
    lora_alpha=<span class="hljs-number">32</span>,
    lora_dropout=<span class="hljs-number">0.05</span>,
    bias=<span class="hljs-string">&quot;none&quot;</span>,
    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
)

model = AutoModelForCausalLMWithValueHead.from_pretrained(
    model_name,
    peft_config=lora_config,
    reward_adapter=rm_adapter_id,
)

...
trainer = PPOTrainer(
    model=model,
    ...
)

...`,wrap:!1}}),q=new pe({props:{code:"cmV3YXJkcyUyMCUzRCUyMHRyYWluZXIubW9kZWwuY29tcHV0ZV9yZXdhcmRfc2NvcmUoKippbnB1dHMp",highlighted:"rewards = trainer.model.compute_reward_score(**inputs)",wrap:!1}}),x=new A({props:{title:"Advanced usage",local:"advanced-usage",headingTag:"h2"}}),j=new A({props:{title:"Control on the adapter name",local:"control-on-the-adapter-name",headingTag:"h3"}}),k=new pe({props:{code:"YWRhcHRlcl9uYW1lX3BvbGljeV8xJTIwJTNEJTIwJTIycG9saWN5XzElMjIlMEFyZXdhcmRzJTIwJTNEJTIwdHJhaW5lci5tb2RlbC5jb21wdXRlX3Jld2FyZF9zY29yZSgqKmlucHV0cyUyQyUyMHBwb19hZGFwdGVyX25hbWUlM0RhZGFwdGVyX25hbWVfcG9saWN5XzEpJTBBLi4u",highlighted:`adapter_name_policy_1 = <span class="hljs-string">&quot;policy_1&quot;</span>
rewards = trainer.model.compute_reward_score(**inputs, ppo_adapter_name=adapter_name_policy_1)
...`,wrap:!1}}),R=new A({props:{title:"Using 4-bit and 8-bit base models",local:"using-4-bit-and-8-bit-base-models",headingTag:"h3"}}),J=new pe({props:{code:"JTBBJTBB",highlighted:`model_name = <span class="hljs-string">&quot;llama-7b&quot;</span>
rm_adapter_id = <span class="hljs-string">&quot;trl-lib/llama-7b-hh-rm-adapter&quot;</span>

<span class="hljs-comment"># PPO adapter</span>
lora_config = LoraConfig(
    r=<span class="hljs-number">16</span>,
    lora_alpha=<span class="hljs-number">32</span>,
    lora_dropout=<span class="hljs-number">0.05</span>,
    bias=<span class="hljs-string">&quot;none&quot;</span>,
    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
)

model = AutoModelForCausalLMWithValueHead.from_pretrained(
    model_name,
    peft_config=lora_config,
    reward_adapter=rm_adapter_id,
    load_in_8bit=<span class="hljs-literal">True</span>,
)

...
trainer = PPOTrainer(
    model=model,
    ...
)
...`,wrap:!1}}),W=new je({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/multi_adapter_rl.md"}}),{c(){f=r("meta"),E=s(),F=r("p"),O=s(),i(g.$$.fragment),V=s(),$=r("p"),$.textContent=ue,X=s(),i(_.$$.fragment),I=s(),b=r("p"),b.innerHTML=ce,S=s(),i(w.$$.fragment),Z=s(),y=r("p"),y.textContent=fe,G=s(),T=r("p"),T.innerHTML=he,Y=s(),v=r("p"),v.textContent=ge,z=s(),i(M.$$.fragment),N=s(),P=r("p"),P.innerHTML=$e,Q=s(),i(L.$$.fragment),K=s(),C=r("p"),C.innerHTML=_e,D=s(),i(q.$$.fragment),ee=s(),i(x.$$.fragment),te=s(),i(j.$$.fragment),ae=s(),H=r("p"),H.innerHTML=be,ne=s(),i(k.$$.fragment),se=s(),i(R.$$.fragment),le=s(),B=r("p"),B.innerHTML=we,re=s(),i(J.$$.fragment),ie=s(),i(W.$$.fragment),oe=s(),U=r("p"),this.h()},l(e){const t=qe("svelte-u9bgzb",document.head);f=o(t,"META",{name:!0,content:!0}),t.forEach(a),E=l(e),F=o(e,"P",{}),ye(F).forEach(a),O=l(e),m(g.$$.fragment,e),V=l(e),$=o(e,"P",{"data-svelte-h":!0}),h($)!=="svelte-1ni6bkb"&&($.textContent=ue),X=l(e),m(_.$$.fragment,e),I=l(e),b=o(e,"P",{"data-svelte-h":!0}),h(b)!=="svelte-1crdfuc"&&(b.innerHTML=ce),S=l(e),m(w.$$.fragment,e),Z=l(e),y=o(e,"P",{"data-svelte-h":!0}),h(y)!=="svelte-1nxlg56"&&(y.textContent=fe),G=l(e),T=o(e,"P",{"data-svelte-h":!0}),h(T)!=="svelte-1v59j6k"&&(T.innerHTML=he),Y=l(e),v=o(e,"P",{"data-svelte-h":!0}),h(v)!=="svelte-x4geit"&&(v.textContent=ge),z=l(e),m(M.$$.fragment,e),N=l(e),P=o(e,"P",{"data-svelte-h":!0}),h(P)!=="svelte-yusdbc"&&(P.innerHTML=$e),Q=l(e),m(L.$$.fragment,e),K=l(e),C=o(e,"P",{"data-svelte-h":!0}),h(C)!=="svelte-uwecsv"&&(C.innerHTML=_e),D=l(e),m(q.$$.fragment,e),ee=l(e),m(x.$$.fragment,e),te=l(e),m(j.$$.fragment,e),ae=l(e),H=o(e,"P",{"data-svelte-h":!0}),h(H)!=="svelte-3l2jwi"&&(H.innerHTML=be),ne=l(e),m(k.$$.fragment,e),se=l(e),m(R.$$.fragment,e),le=l(e),B=o(e,"P",{"data-svelte-h":!0}),h(B)!=="svelte-17i0flk"&&(B.innerHTML=we),re=l(e),m(J.$$.fragment,e),ie=l(e),m(W.$$.fragment,e),oe=l(e),U=o(e,"P",{}),ye(U).forEach(a),this.h()},h(){Te(f,"name","hf:doc:metadata"),Te(f,"content",ke)},m(e,t){xe(document.head,f),n(e,E,t),n(e,F,t),n(e,O,t),p(g,e,t),n(e,V,t),n(e,$,t),n(e,X,t),p(_,e,t),n(e,I,t),n(e,b,t),n(e,S,t),p(w,e,t),n(e,Z,t),n(e,y,t),n(e,G,t),n(e,T,t),n(e,Y,t),n(e,v,t),n(e,z,t),p(M,e,t),n(e,N,t),n(e,P,t),n(e,Q,t),p(L,e,t),n(e,K,t),n(e,C,t),n(e,D,t),p(q,e,t),n(e,ee,t),p(x,e,t),n(e,te,t),p(j,e,t),n(e,ae,t),n(e,H,t),n(e,ne,t),p(k,e,t),n(e,se,t),p(R,e,t),n(e,le,t),n(e,B,t),n(e,re,t),p(J,e,t),n(e,ie,t),p(W,e,t),n(e,oe,t),n(e,U,t),me=!0},p:Me,i(e){me||(d(g.$$.fragment,e),d(_.$$.fragment,e),d(w.$$.fragment,e),d(M.$$.fragment,e),d(L.$$.fragment,e),d(q.$$.fragment,e),d(x.$$.fragment,e),d(j.$$.fragment,e),d(k.$$.fragment,e),d(R.$$.fragment,e),d(J.$$.fragment,e),d(W.$$.fragment,e),me=!0)},o(e){u(g.$$.fragment,e),u(_.$$.fragment,e),u(w.$$.fragment,e),u(M.$$.fragment,e),u(L.$$.fragment,e),u(q.$$.fragment,e),u(x.$$.fragment,e),u(j.$$.fragment,e),u(k.$$.fragment,e),u(R.$$.fragment,e),u(J.$$.fragment,e),u(W.$$.fragment,e),me=!1},d(e){e&&(a(E),a(F),a(O),a(V),a($),a(X),a(I),a(b),a(S),a(Z),a(y),a(G),a(T),a(Y),a(v),a(z),a(N),a(P),a(Q),a(K),a(C),a(D),a(ee),a(te),a(ae),a(H),a(ne),a(se),a(le),a(B),a(re),a(ie),a(oe),a(U)),a(f),c(g,e),c(_,e),c(w,e),c(M,e),c(L,e),c(q,e),c(x,e),c(j,e),c(k,e),c(R,e),c(J,e),c(W,e)}}}const ke='{"title":"Multi Adapter RL (MARL) - a single base model for everything","local":"multi-adapter-rl-marl---a-single-base-model-for-everything","sections":[{"title":"Requirements","local":"requirements","sections":[],"depth":2},{"title":"Summary","local":"summary","sections":[],"depth":2},{"title":"Quickstart","local":"quickstart","sections":[],"depth":2},{"title":"Advanced usage","local":"advanced-usage","sections":[{"title":"Control on the adapter name","local":"control-on-the-adapter-name","sections":[],"depth":3},{"title":"Using 4-bit and 8-bit base models","local":"using-4-bit-and-8-bit-base-models","sections":[],"depth":3}],"depth":2}],"depth":1}';function Re(de){return Pe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fe extends Le{constructor(f){super(),Ce(this,f,Re,He,ve,{})}}export{Fe as component};
