import{s as ce,f as de,n as ue,o as he}from"../chunks/scheduler.d627b047.js";import{S as we,i as ye,g as o,s as a,r as m,A as Je,h as i,f as t,c as n,j as Me,u as p,x as C,k as ee,y as be,a as s,v as M,d as c,t as d,w as u}from"../chunks/index.73c51727.js";import{C as le}from"../chunks/CodeBlock.5f78c87f.js";import{H as O,E as Te}from"../chunks/index.8df1db42.js";function je(te){let r,_,v,Q,h,G,w,$,y,se="Fine-tuning a language model via PPO consists of roughly three steps:",E,J,ae="<li><strong>Rollout</strong>: The language model generates a response or continuation based on a query which could be the start of a sentence.</li> <li><strong>Evaluation</strong>: The query and response are evaluated with a function, model, human feedback, or some combination of them. The important thing is that this process should yield a scalar value for each query/response pair. The optimization will aim at maximizing this value.</li> <li><strong>Optimization</strong>: This is the most complex part. In the optimisation step the query/response pairs are used to calculate the log-probabilities of the tokens in the sequences. This is done with the model that is trained and a reference model, which is usually the pre-trained model before fine-tuning. The KL-divergence between the two outputs is used as an additional reward signal to make sure the generated responses don’t deviate too far from the reference language model. The active language model is then trained with PPO.</li>",H,b,ne="The full process is illustrated in the following figure:",R,X,oe,F,T,N,j,ie="The following code illustrates the steps above.",q,f,x,U,re="In general, you would run steps 3-6 in a for-loop and run it on many diverse queries. You can find more realistic examples in the examples section.",z,Z,S,W,me="After training a <code>AutoModelForCausalLMWithValueHead</code>, you can directly use the model in <code>transformers</code>.",A,g,L,B,pe="You can also load your model with <code>AutoModelForCausalLMWithValueHead</code> if you want to use the value head, for example to continue training.",Y,I,P,k,D,V,K;return h=new O({props:{title:"Quickstart",local:"quickstart",headingTag:"h1"}}),w=new O({props:{title:"How does it work?",local:"how-does-it-work",headingTag:"h2"}}),T=new O({props:{title:"Minimal example",local:"minimal-example",headingTag:"h2"}}),f=new le({props:{code:"JTIzJTIwMC4lMjBpbXBvcnRzJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwR1BUMlRva2VuaXplciUwQSUwQWZyb20lMjB0cmwlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTVdpdGhWYWx1ZUhlYWQlMkMlMjBQUE9Db25maWclMkMlMjBQUE9UcmFpbmVyJTBBJTBBJTBBJTIzJTIwMS4lMjBsb2FkJTIwYSUyMHByZXRyYWluZWQlMjBtb2RlbCUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE1XaXRoVmFsdWVIZWFkLmZyb21fcHJldHJhaW5lZCglMjJncHQyJTIyKSUwQXJlZl9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNV2l0aFZhbHVlSGVhZC5mcm9tX3ByZXRyYWluZWQoJTIyZ3B0MiUyMiklMEF0b2tlbml6ZXIlMjAlM0QlMjBHUFQyVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJncHQyJTIyKSUwQXRva2VuaXplci5wYWRfdG9rZW4lMjAlM0QlMjB0b2tlbml6ZXIuZW9zX3Rva2VuJTBBJTBBJTIzJTIwMi4lMjBpbml0aWFsaXplJTIwdHJhaW5lciUwQXBwb19jb25maWclMjAlM0QlMjAlN0IlMjJtaW5pX2JhdGNoX3NpemUlMjIlM0ElMjAxJTJDJTIwJTIyYmF0Y2hfc2l6ZSUyMiUzQSUyMDElN0QlMEFjb25maWclMjAlM0QlMjBQUE9Db25maWcoKipwcG9fY29uZmlnKSUwQXBwb190cmFpbmVyJTIwJTNEJTIwUFBPVHJhaW5lcihjb25maWclMkMlMjBtb2RlbCUyQyUyMHJlZl9tb2RlbCUyQyUyMHRva2VuaXplciklMEElMEElMjMlMjAzLiUyMGVuY29kZSUyMGElMjBxdWVyeSUwQXF1ZXJ5X3R4dCUyMCUzRCUyMCUyMlRoaXMlMjBtb3JuaW5nJTIwSSUyMHdlbnQlMjB0byUyMHRoZSUyMCUyMiUwQXF1ZXJ5X3RlbnNvciUyMCUzRCUyMHRva2VuaXplci5lbmNvZGUocXVlcnlfdHh0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwucHJldHJhaW5lZF9tb2RlbC5kZXZpY2UpJTBBJTBBJTIzJTIwNC4lMjBnZW5lcmF0ZSUyMG1vZGVsJTIwcmVzcG9uc2UlMEFnZW5lcmF0aW9uX2t3YXJncyUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMm1pbl9sZW5ndGglMjIlM0ElMjAtMSUyQyUwQSUyMCUyMCUyMCUyMCUyMnRvcF9rJTIyJTNBJTIwMC4wJTJDJTBBJTIwJTIwJTIwJTIwJTIydG9wX3AlMjIlM0ElMjAxLjAlMkMlMEElMjAlMjAlMjAlMjAlMjJkb19zYW1wbGUlMjIlM0ElMjBUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIycGFkX3Rva2VuX2lkJTIyJTNBJTIwdG9rZW5pemVyLmVvc190b2tlbl9pZCUyQyUwQSUyMCUyMCUyMCUyMCUyMm1heF9uZXdfdG9rZW5zJTIyJTNBJTIwMjAlMkMlMEElN0QlMEFyZXNwb25zZV90ZW5zb3IlMjAlM0QlMjBwcG9fdHJhaW5lci5nZW5lcmF0ZSglNUJpdGVtJTIwZm9yJTIwaXRlbSUyMGluJTIwcXVlcnlfdGVuc29yJTVEJTJDJTIwcmV0dXJuX3Byb21wdCUzREZhbHNlJTJDJTIwKipnZW5lcmF0aW9uX2t3YXJncyklMEFyZXNwb25zZV90eHQlMjAlM0QlMjB0b2tlbml6ZXIuZGVjb2RlKHJlc3BvbnNlX3RlbnNvciU1QjAlNUQpJTBBJTBBJTIzJTIwNS4lMjBkZWZpbmUlMjBhJTIwcmV3YXJkJTIwZm9yJTIwcmVzcG9uc2UlMEElMjMlMjAodGhpcyUyMGNvdWxkJTIwYmUlMjBhbnklMjByZXdhcmQlMjBzdWNoJTIwYXMlMjBodW1hbiUyMGZlZWRiYWNrJTIwb3IlMjBvdXRwdXQlMjBmcm9tJTIwYW5vdGhlciUyMG1vZGVsKSUwQXJld2FyZCUyMCUzRCUyMCU1QnRvcmNoLnRlbnNvcigxLjAlMkMlMjBkZXZpY2UlM0Rtb2RlbC5wcmV0cmFpbmVkX21vZGVsLmRldmljZSklNUQlMEElMEElMjMlMjA2LiUyMHRyYWluJTIwbW9kZWwlMjB3aXRoJTIwcHBvJTBBdHJhaW5fc3RhdHMlMjAlM0QlMjBwcG9fdHJhaW5lci5zdGVwKCU1QnF1ZXJ5X3RlbnNvciU1QjAlNUQlNUQlMkMlMjAlNUJyZXNwb25zZV90ZW5zb3IlNUIwJTVEJTVEJTJDJTIwcmV3YXJkKQ==",highlighted:`<span class="hljs-comment"># 0. imports</span>
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPT2Tokenizer

<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> AutoModelForCausalLMWithValueHead, PPOConfig, PPOTrainer


<span class="hljs-comment"># 1. load a pretrained model</span>
model = AutoModelForCausalLMWithValueHead.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)
ref_model = AutoModelForCausalLMWithValueHead.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)
tokenizer = GPT2Tokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)
tokenizer.pad_token = tokenizer.eos_token

<span class="hljs-comment"># 2. initialize trainer</span>
ppo_config = {<span class="hljs-string">&quot;mini_batch_size&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;batch_size&quot;</span>: <span class="hljs-number">1</span>}
config = PPOConfig(**ppo_config)
ppo_trainer = PPOTrainer(config, model, ref_model, tokenizer)

<span class="hljs-comment"># 3. encode a query</span>
query_txt = <span class="hljs-string">&quot;This morning I went to the &quot;</span>
query_tensor = tokenizer.encode(query_txt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.pretrained_model.device)

<span class="hljs-comment"># 4. generate model response</span>
generation_kwargs = {
    <span class="hljs-string">&quot;min_length&quot;</span>: -<span class="hljs-number">1</span>,
    <span class="hljs-string">&quot;top_k&quot;</span>: <span class="hljs-number">0.0</span>,
    <span class="hljs-string">&quot;top_p&quot;</span>: <span class="hljs-number">1.0</span>,
    <span class="hljs-string">&quot;do_sample&quot;</span>: <span class="hljs-literal">True</span>,
    <span class="hljs-string">&quot;pad_token_id&quot;</span>: tokenizer.eos_token_id,
    <span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">20</span>,
}
response_tensor = ppo_trainer.generate([item <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> query_tensor], return_prompt=<span class="hljs-literal">False</span>, **generation_kwargs)
response_txt = tokenizer.decode(response_tensor[<span class="hljs-number">0</span>])

<span class="hljs-comment"># 5. define a reward for response</span>
<span class="hljs-comment"># (this could be any reward such as human feedback or output from another model)</span>
reward = [torch.tensor(<span class="hljs-number">1.0</span>, device=model.pretrained_model.device)]

<span class="hljs-comment"># 6. train model with ppo</span>
train_stats = ppo_trainer.step([query_tensor[<span class="hljs-number">0</span>]], [response_tensor[<span class="hljs-number">0</span>]], reward)`,wrap:!1}}),Z=new O({props:{title:"How to use a trained model",local:"how-to-use-a-trained-model",headingTag:"h2"}}),g=new le({props:{code:"JTBBJTIzJTIwLi4lMjBMZXQncyUyMGFzc3VtZSUyMHdlJTIwaGF2ZSUyMGElMjB0cmFpbmVkJTIwbW9kZWwlMjB1c2luZyUyMCU2MFBQT1RyYWluZXIlNjAlMjBhbmQlMjAlNjBBdXRvTW9kZWxGb3JDYXVzYWxMTVdpdGhWYWx1ZUhlYWQlNjAlMEElMEElMjMlMjBwdXNoJTIwdGhlJTIwbW9kZWwlMjBvbiUyMHRoZSUyMEh1YiUwQW1vZGVsLnB1c2hfdG9faHViKCUyMm15LWZpbmUtdHVuZWQtbW9kZWwtcHBvJTIyKSUwQSUwQSUyMyUyMG9yJTIwc2F2ZSUyMGl0JTIwbG9jYWxseSUwQW1vZGVsLnNhdmVfcHJldHJhaW5lZCglMjJteS1maW5lLXR1bmVkLW1vZGVsLXBwbyUyMiklMEElMEElMjMlMjBsb2FkJTIwdGhlJTIwbW9kZWwlMjBmcm9tJTIwdGhlJTIwSHViJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIybXktZmluZS10dW5lZC1tb2RlbC1wcG8lMjIp",highlighted:`
<span class="hljs-comment"># .. Let&#x27;s assume we have a trained model using \`PPOTrainer\` and \`AutoModelForCausalLMWithValueHead\`</span>

<span class="hljs-comment"># push the model on the Hub</span>
model.push_to_hub(<span class="hljs-string">&quot;my-fine-tuned-model-ppo&quot;</span>)

<span class="hljs-comment"># or save it locally</span>
model.save_pretrained(<span class="hljs-string">&quot;my-fine-tuned-model-ppo&quot;</span>)

<span class="hljs-comment"># load the model from the Hub</span>
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;my-fine-tuned-model-ppo&quot;</span>)`,wrap:!1}}),I=new le({props:{code:"ZnJvbSUyMHRybC5tb2RlbCUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNV2l0aFZhbHVlSGVhZCUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE1XaXRoVmFsdWVIZWFkLmZyb21fcHJldHJhaW5lZCglMjJteS1maW5lLXR1bmVkLW1vZGVsLXBwbyUyMik=",highlighted:`<span class="hljs-keyword">from</span> trl.model <span class="hljs-keyword">import</span> AutoModelForCausalLMWithValueHead

model = AutoModelForCausalLMWithValueHead.from_pretrained(<span class="hljs-string">&quot;my-fine-tuned-model-ppo&quot;</span>)`,wrap:!1}}),k=new Te({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/quickstart.md"}}),{c(){r=o("meta"),_=a(),v=o("p"),Q=a(),m(h.$$.fragment),G=a(),m(w.$$.fragment),$=a(),y=o("p"),y.textContent=se,E=a(),J=o("ol"),J.innerHTML=ae,H=a(),b=o("p"),b.textContent=ne,R=a(),X=o("img"),F=a(),m(T.$$.fragment),N=a(),j=o("p"),j.textContent=ie,q=a(),m(f.$$.fragment),x=a(),U=o("p"),U.textContent=re,z=a(),m(Z.$$.fragment),S=a(),W=o("p"),W.innerHTML=me,A=a(),m(g.$$.fragment),L=a(),B=o("p"),B.innerHTML=pe,Y=a(),m(I.$$.fragment),P=a(),m(k.$$.fragment),D=a(),V=o("p"),this.h()},l(e){const l=Je("svelte-u9bgzb",document.head);r=i(l,"META",{name:!0,content:!0}),l.forEach(t),_=n(e),v=i(e,"P",{}),Me(v).forEach(t),Q=n(e),p(h.$$.fragment,e),G=n(e),p(w.$$.fragment,e),$=n(e),y=i(e,"P",{"data-svelte-h":!0}),C(y)!=="svelte-79li59"&&(y.textContent=se),E=n(e),J=i(e,"OL",{"data-svelte-h":!0}),C(J)!=="svelte-1uekaot"&&(J.innerHTML=ae),H=n(e),b=i(e,"P",{"data-svelte-h":!0}),C(b)!=="svelte-62xfzx"&&(b.textContent=ne),R=n(e),X=i(e,"IMG",{src:!0}),F=n(e),p(T.$$.fragment,e),N=n(e),j=i(e,"P",{"data-svelte-h":!0}),C(j)!=="svelte-11erqnm"&&(j.textContent=ie),q=n(e),p(f.$$.fragment,e),x=n(e),U=i(e,"P",{"data-svelte-h":!0}),C(U)!=="svelte-1rp76uw"&&(U.textContent=re),z=n(e),p(Z.$$.fragment,e),S=n(e),W=i(e,"P",{"data-svelte-h":!0}),C(W)!=="svelte-znbdcu"&&(W.innerHTML=me),A=n(e),p(g.$$.fragment,e),L=n(e),B=i(e,"P",{"data-svelte-h":!0}),C(B)!=="svelte-12cg6c8"&&(B.innerHTML=pe),Y=n(e),p(I.$$.fragment,e),P=n(e),p(k.$$.fragment,e),D=n(e),V=i(e,"P",{}),Me(V).forEach(t),this.h()},h(){ee(r,"name","hf:doc:metadata"),ee(r,"content",fe),de(X.src,oe="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/trl_overview.png")||ee(X,"src",oe)},m(e,l){be(document.head,r),s(e,_,l),s(e,v,l),s(e,Q,l),M(h,e,l),s(e,G,l),M(w,e,l),s(e,$,l),s(e,y,l),s(e,E,l),s(e,J,l),s(e,H,l),s(e,b,l),s(e,R,l),s(e,X,l),s(e,F,l),M(T,e,l),s(e,N,l),s(e,j,l),s(e,q,l),M(f,e,l),s(e,x,l),s(e,U,l),s(e,z,l),M(Z,e,l),s(e,S,l),s(e,W,l),s(e,A,l),M(g,e,l),s(e,L,l),s(e,B,l),s(e,Y,l),M(I,e,l),s(e,P,l),M(k,e,l),s(e,D,l),s(e,V,l),K=!0},p:ue,i(e){K||(c(h.$$.fragment,e),c(w.$$.fragment,e),c(T.$$.fragment,e),c(f.$$.fragment,e),c(Z.$$.fragment,e),c(g.$$.fragment,e),c(I.$$.fragment,e),c(k.$$.fragment,e),K=!0)},o(e){d(h.$$.fragment,e),d(w.$$.fragment,e),d(T.$$.fragment,e),d(f.$$.fragment,e),d(Z.$$.fragment,e),d(g.$$.fragment,e),d(I.$$.fragment,e),d(k.$$.fragment,e),K=!1},d(e){e&&(t(_),t(v),t(Q),t(G),t($),t(y),t(E),t(J),t(H),t(b),t(R),t(X),t(F),t(N),t(j),t(q),t(x),t(U),t(z),t(S),t(W),t(A),t(L),t(B),t(Y),t(P),t(D),t(V)),t(r),u(h,e),u(w,e),u(T,e),u(f,e),u(Z,e),u(g,e),u(I,e),u(k,e)}}}const fe='{"title":"Quickstart","local":"quickstart","sections":[{"title":"How does it work?","local":"how-does-it-work","sections":[],"depth":2},{"title":"Minimal example","local":"minimal-example","sections":[],"depth":2},{"title":"How to use a trained model","local":"how-to-use-a-trained-model","sections":[],"depth":2}],"depth":1}';function Ue(te){return he(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ie extends we{constructor(r){super(),ye(this,r,Ue,je,ce,{})}}export{Ie as component};
