import{s as pe,n as me,o as de}from"../chunks/scheduler.d627b047.js";import{S as ce,i as ue,g as o,s as a,r as p,A as fe,h as i,f as l,c as n,j as oe,u as m,x as v,k as ie,y as be,a as s,v as d,d as c,t as u,w as f}from"../chunks/index.73c51727.js";import{C as W}from"../chunks/CodeBlock.5f78c87f.js";import{H as re,E as ye}from"../chunks/index.8df1db42.js";function Me(O){let r,R,$,z,b,G,y,K=`Within the extras module is the <code>best-of-n</code> sampler class that serves as an alternative method of generating better model output.
As to how it fares against the RL based fine-tuning, please look in the <code>examples</code> directory for a comparison example`,C,M,x,h,ee="To get started quickly, instantiate an instance of the class with a model, a length sampler, a tokenizer and a callable that serves as a proxy reward pipeline that outputs reward scores for input queries",N,w,Q,_,te="And assuming you have a list/tensor of tokenized queries, you can generate better output by calling the <code>generate</code> method",Y,Z,F,g,le="The default sample size is 4, but you can change it at the time of instance initialization like so",H,X,E,B,se="The default output is the result of taking the top scored output for each query, but you can change it to top 2 and so on by passing the <code>n_candidates</code> argument at the time of instance initialization",I,J,S,T,ae=`There is the option of setting the generation settings (like <code>temperature</code>, <code>pad_token_id</code>) at the time of instance creation as opposed to when calling the <code>generate</code> method.
This is done by passing a <code>GenerationConfig</code> from the <code>transformers</code> library at the time of initialization`,L,k,q,U,ne="Furthermore, at the time of initialization you can set the seed to control the repeatability of the generation process and the number of samples to generate for each query",A,j,P,V,D;return b=new re({props:{title:"Best of N sampling: Alternative ways to get better model output without RL based fine-tuning",local:"best-of-n-sampling-alternative-ways-to-get-better-model-output-without-rl-based-fine-tuning",headingTag:"h1"}}),M=new re({props:{title:"Usage",local:"usage",headingTag:"h2"}}),w=new W({props:{code:"JTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTJDJTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjB0cmwlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTVdpdGhWYWx1ZUhlYWQlMEFmcm9tJTIwdHJsLmNvcmUlMjBpbXBvcnQlMjBMZW5ndGhTYW1wbGVyJTBBZnJvbSUyMHRybC5leHRyYXMlMjBpbXBvcnQlMjBCZXN0T2ZOU2FtcGxlciUwQSUwQXJlZl9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNV2l0aFZhbHVlSGVhZC5mcm9tX3ByZXRyYWluZWQocmVmX21vZGVsX25hbWUpJTBBcmV3YXJkX3BpcGUlMjAlM0QlMjBwaXBlbGluZSglMjJzZW50aW1lbnQtYW5hbHlzaXMlMjIlMkMlMjBtb2RlbCUzRHJld2FyZF9tb2RlbCUyQyUyMGRldmljZSUzRGRldmljZSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChyZWZfbW9kZWxfbmFtZSklMEF0b2tlbml6ZXIucGFkX3Rva2VuJTIwJTNEJTIwdG9rZW5pemVyLmVvc190b2tlbiUwQSUwQSUwQSUyMyUyMGNhbGxhYmxlJTIwdGhhdCUyMHRha2VzJTIwYSUyMGxpc3QlMjBvZiUyMHJhdyUyMHRleHQlMjBhbmQlMjByZXR1cm5zJTIwYSUyMGxpc3QlMjBvZiUyMGNvcnJlc3BvbmRpbmclMjByZXdhcmQlMjBzY29yZXMlMEFkZWYlMjBxdWVyaWVzX3RvX3Njb3JlcyhsaXN0X29mX3N0cmluZ3MpJTNBJTBBJTIwJTIwcmV0dXJuJTIwJTVCb3V0cHV0JTVCJTIyc2NvcmUlMjIlNUQlMjBmb3IlMjBvdXRwdXQlMjBpbiUyMHJld2FyZF9waXBlKGxpc3Rfb2Zfc3RyaW5ncyklNUQlMEElMEFiZXN0X29mX24lMjAlM0QlMjBCZXN0T2ZOU2FtcGxlcihtb2RlbCUyQyUyMHRva2VuaXplciUyQyUyMHF1ZXJpZXNfdG9fc2NvcmVzJTJDJTIwbGVuZ3RoX3NhbXBsZXIlM0RvdXRwdXRfbGVuZ3RoX3NhbXBsZXIpJTBBJTBB",highlighted:`
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline, AutoTokenizer
<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> AutoModelForCausalLMWithValueHead
<span class="hljs-keyword">from</span> trl.core <span class="hljs-keyword">import</span> LengthSampler
<span class="hljs-keyword">from</span> trl.extras <span class="hljs-keyword">import</span> BestOfNSampler

ref_model = AutoModelForCausalLMWithValueHead.from_pretrained(ref_model_name)
reward_pipe = pipeline(<span class="hljs-string">&quot;sentiment-analysis&quot;</span>, model=reward_model, device=device)
tokenizer = AutoTokenizer.from_pretrained(ref_model_name)
tokenizer.pad_token = tokenizer.eos_token


<span class="hljs-comment"># callable that takes a list of raw text and returns a list of corresponding reward scores</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">queries_to_scores</span>(<span class="hljs-params">list_of_strings</span>):
  <span class="hljs-keyword">return</span> [output[<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">for</span> output <span class="hljs-keyword">in</span> reward_pipe(list_of_strings)]

best_of_n = BestOfNSampler(model, tokenizer, queries_to_scores, length_sampler=output_length_sampler)

`,wrap:!1}}),Z=new W({props:{code:"JTBBYmVzdF9vZl9uLmdlbmVyYXRlKHF1ZXJ5X3RlbnNvcnMlMkMlMjBkZXZpY2UlM0RkZXZpY2UlMkMlMjAqKmdlbl9rd2FyZ3MpJTBB",highlighted:`
best_of_n.generate(query_tensors, device=device, **gen_kwargs)
`,wrap:!1}}),X=new W({props:{code:"JTBBYmVzdF9vZl9uJTIwJTNEJTIwQmVzdE9mTlNhbXBsZXIobW9kZWwlMkMlMjB0b2tlbml6ZXIlMkMlMjBxdWVyaWVzX3RvX3Njb3JlcyUyQyUyMGxlbmd0aF9zYW1wbGVyJTNEb3V0cHV0X2xlbmd0aF9zYW1wbGVyJTJDJTIwc2FtcGxlX3NpemUlM0Q4KSUwQQ==",highlighted:`
best_of_n = BestOfNSampler(model, tokenizer, queries_to_scores, length_sampler=output_length_sampler, sample_size=<span class="hljs-number">8</span>)
`,wrap:!1}}),J=new W({props:{code:"JTBBYmVzdF9vZl9uJTIwJTNEJTIwQmVzdE9mTlNhbXBsZXIobW9kZWwlMkMlMjB0b2tlbml6ZXIlMkMlMjBxdWVyaWVzX3RvX3Njb3JlcyUyQyUyMGxlbmd0aF9zYW1wbGVyJTNEb3V0cHV0X2xlbmd0aF9zYW1wbGVyJTJDJTIwbl9jYW5kaWRhdGVzJTNEMiklMEE=",highlighted:`
best_of_n = BestOfNSampler(model, tokenizer, queries_to_scores, length_sampler=output_length_sampler, n_candidates=<span class="hljs-number">2</span>)
`,wrap:!1}}),k=new W({props:{code:"JTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdlbmVyYXRpb25Db25maWclMEElMEFnZW5lcmF0aW9uX2NvbmZpZyUyMCUzRCUyMEdlbmVyYXRpb25Db25maWcobWluX2xlbmd0aCUzRCUyMC0xJTJDJTIwdG9wX2slM0QwLjAlMkMlMjB0b3BfcCUzRCUyMDEuMCUyQyUyMGRvX3NhbXBsZSUzRCUyMFRydWUlMkMlMjBwYWRfdG9rZW5faWQlM0R0b2tlbml6ZXIuZW9zX3Rva2VuX2lkKSUwQSUwQWJlc3Rfb2ZfbiUyMCUzRCUyMEJlc3RPZk5TYW1wbGVyKG1vZGVsJTJDJTIwdG9rZW5pemVyJTJDJTIwcXVlcmllc190b19zY29yZXMlMkMlMjBsZW5ndGhfc2FtcGxlciUzRG91dHB1dF9sZW5ndGhfc2FtcGxlciUyQyUyMGdlbmVyYXRpb25fY29uZmlnJTNEZ2VuZXJhdGlvbl9jb25maWcpJTBBJTBBYmVzdF9vZl9uLmdlbmVyYXRlKHF1ZXJ5X3RlbnNvcnMlMkMlMjBkZXZpY2UlM0RkZXZpY2UpJTBB",highlighted:`
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GenerationConfig

generation_config = GenerationConfig(min_length= -<span class="hljs-number">1</span>, top_k=<span class="hljs-number">0.0</span>, top_p= <span class="hljs-number">1.0</span>, do_sample= <span class="hljs-literal">True</span>, pad_token_id=tokenizer.eos_token_id)

best_of_n = BestOfNSampler(model, tokenizer, queries_to_scores, length_sampler=output_length_sampler, generation_config=generation_config)

best_of_n.generate(query_tensors, device=device)
`,wrap:!1}}),j=new ye({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/best_of_n.md"}}),{c(){r=o("meta"),R=a(),$=o("p"),z=a(),p(b.$$.fragment),G=a(),y=o("p"),y.innerHTML=K,C=a(),p(M.$$.fragment),x=a(),h=o("p"),h.textContent=ee,N=a(),p(w.$$.fragment),Q=a(),_=o("p"),_.innerHTML=te,Y=a(),p(Z.$$.fragment),F=a(),g=o("p"),g.textContent=le,H=a(),p(X.$$.fragment),E=a(),B=o("p"),B.innerHTML=se,I=a(),p(J.$$.fragment),S=a(),T=o("p"),T.innerHTML=ae,L=a(),p(k.$$.fragment),q=a(),U=o("p"),U.textContent=ne,A=a(),p(j.$$.fragment),P=a(),V=o("p"),this.h()},l(e){const t=fe("svelte-u9bgzb",document.head);r=i(t,"META",{name:!0,content:!0}),t.forEach(l),R=n(e),$=i(e,"P",{}),oe($).forEach(l),z=n(e),m(b.$$.fragment,e),G=n(e),y=i(e,"P",{"data-svelte-h":!0}),v(y)!=="svelte-8p1ktq"&&(y.innerHTML=K),C=n(e),m(M.$$.fragment,e),x=n(e),h=i(e,"P",{"data-svelte-h":!0}),v(h)!=="svelte-jw2eay"&&(h.textContent=ee),N=n(e),m(w.$$.fragment,e),Q=n(e),_=i(e,"P",{"data-svelte-h":!0}),v(_)!=="svelte-r9yrv0"&&(_.innerHTML=te),Y=n(e),m(Z.$$.fragment,e),F=n(e),g=i(e,"P",{"data-svelte-h":!0}),v(g)!=="svelte-p3zzho"&&(g.textContent=le),H=n(e),m(X.$$.fragment,e),E=n(e),B=i(e,"P",{"data-svelte-h":!0}),v(B)!=="svelte-11zsz5r"&&(B.innerHTML=se),I=n(e),m(J.$$.fragment,e),S=n(e),T=i(e,"P",{"data-svelte-h":!0}),v(T)!=="svelte-10dgdpu"&&(T.innerHTML=ae),L=n(e),m(k.$$.fragment,e),q=n(e),U=i(e,"P",{"data-svelte-h":!0}),v(U)!=="svelte-1avtdqm"&&(U.textContent=ne),A=n(e),m(j.$$.fragment,e),P=n(e),V=i(e,"P",{}),oe(V).forEach(l),this.h()},h(){ie(r,"name","hf:doc:metadata"),ie(r,"content",he)},m(e,t){be(document.head,r),s(e,R,t),s(e,$,t),s(e,z,t),d(b,e,t),s(e,G,t),s(e,y,t),s(e,C,t),d(M,e,t),s(e,x,t),s(e,h,t),s(e,N,t),d(w,e,t),s(e,Q,t),s(e,_,t),s(e,Y,t),d(Z,e,t),s(e,F,t),s(e,g,t),s(e,H,t),d(X,e,t),s(e,E,t),s(e,B,t),s(e,I,t),d(J,e,t),s(e,S,t),s(e,T,t),s(e,L,t),d(k,e,t),s(e,q,t),s(e,U,t),s(e,A,t),d(j,e,t),s(e,P,t),s(e,V,t),D=!0},p:me,i(e){D||(c(b.$$.fragment,e),c(M.$$.fragment,e),c(w.$$.fragment,e),c(Z.$$.fragment,e),c(X.$$.fragment,e),c(J.$$.fragment,e),c(k.$$.fragment,e),c(j.$$.fragment,e),D=!0)},o(e){u(b.$$.fragment,e),u(M.$$.fragment,e),u(w.$$.fragment,e),u(Z.$$.fragment,e),u(X.$$.fragment,e),u(J.$$.fragment,e),u(k.$$.fragment,e),u(j.$$.fragment,e),D=!1},d(e){e&&(l(R),l($),l(z),l(G),l(y),l(C),l(x),l(h),l(N),l(Q),l(_),l(Y),l(F),l(g),l(H),l(E),l(B),l(I),l(S),l(T),l(L),l(q),l(U),l(A),l(P),l(V)),l(r),f(b,e),f(M,e),f(w,e),f(Z,e),f(X,e),f(J,e),f(k,e),f(j,e)}}}const he='{"title":"Best of N sampling: Alternative ways to get better model output without RL based fine-tuning","local":"best-of-n-sampling-alternative-ways-to-get-better-model-output-without-rl-based-fine-tuning","sections":[{"title":"Usage","local":"usage","sections":[],"depth":2}],"depth":1}';function we(O){return de(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Be extends ce{constructor(r){super(),ue(this,r,we,Me,pe,{})}}export{Be as component};
