import{s as Re,n as Le,o as Ve}from"../chunks/scheduler.d627b047.js";import{S as Ye,i as Ee,g as i,s as a,r,A as Ne,h as p,f as l,c as n,j as We,u as m,x as o,k as Xe,l as ze,y as Fe,a as s,v as h,d as c,t as d,w as g}from"../chunks/index.73c51727.js";import{C as je}from"../chunks/CodeBlock.5f78c87f.js";import{H as V,E as Pe}from"../chunks/index.8df1db42.js";function qe(Je){let u,N,Y,z,M,F,w,Ue='<a href="https://huggingface.co/models?other=alignprop,trl" rel="nofollow"><img src="https://img.shields.io/badge/All_models-AlignProp-blue"/></a>',P,y,q,b,$e=`If your reward function is differentiable, directly backpropagating gradients from the reward models to the diffusion model is significantly more sample and compute efficient (25x) than doing policy gradient algorithm like DDPO.
AlignProp does full backpropagation through time, which allows updating the earlier steps of denoising via reward backpropagation.`,D,f,Ce='<img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/reward_tuning.png"/>',K,T,O,j,ke="The <code>alignprop.py</code> script is a working example of using the <code>AlignProp</code> trainer to finetune a Stable Diffusion model. This example explicitly configures a small subset of the overall parameters associated with the config object (<code>AlignPropConfig</code>).",ee,J,xe="<strong>Note:</strong> one A100 GPU is recommended to get this running. For lower memory setting, consider setting truncated_backprop_rand to False. With default settings this will do truncated backpropagation with K=1.",te,U,_e='Almost every configuration parameter has a default. There is only one commandline flag argument that is required of the user to get things up and running. The user is expected to have a <a href="https://huggingface.co/docs/hub/security-tokens" rel="nofollow">huggingface user access token</a> that will be used to upload the model post-finetuning to HuggingFace hub. The following bash command is to be entered to get things running',le,$,se,C,Ie="To obtain the documentation of <code>stable_diffusion_tuning.py</code>, please run <code>python stable_diffusion_tuning.py --help</code>",ae,k,ve="The following are things to keep in mind (The code checks this for you as well) in general while configuring the trainer (beyond the use case of using the example script)",ne,x,Ae="<li>The configurable randomized truncation range (<code>--alignprop_config.truncated_rand_backprop_minmax=(0,50)</code>) the first number should be equal and greater than 0, while the second number should equal or less to the number of diffusion timesteps (sample_num_steps)</li> <li>The configurable truncation backprop absolute step (<code>--alignprop_config.truncated_backprop_timestep=49</code>) the number should be less than the number of diffusion timesteps (sample_num_steps), it only matters when truncated_backprop_rand is set to False</li>",ie,_,pe,I,Ze="Expect the function to be given a dictionary with keys",oe,v,re,A,Be=`and <code>image</code>, <code>prompt</code>, <code>prompt_metadata</code>, <code>rewards</code>are batched.
You are free to log however you want the use of <code>wandb</code> or <code>tensorboard</code> is recommended.`,me,Z,he,B,Ge='<li><code>rewards</code> : The rewards/score is a numerical associated with the generated image and is key to steering the RL process</li> <li><code>prompt</code> : The prompt is the text that is used to generate the image</li> <li><code>prompt_metadata</code> : The prompt metadata is the metadata associated with the prompt. A situation where this will not be empty is when the reward model comprises of a <a href="https://huggingface.co/docs/transformers/model_doc/flava" rel="nofollow"><code>FLAVA</code></a> setup where questions and ground answers (linked to the generated image) are expected with the generated image (See here: <a href="https://github.com/kvablack/ddpo-pytorch/blob/main/ddpo_pytorch/rewards.py#L45" rel="nofollow">https://github.com/kvablack/ddpo-pytorch/blob/main/ddpo_pytorch/rewards.py#L45</a>)</li> <li><code>image</code> : The image generated by the Stable Diffusion model</li>',ce,G,He="Example code for logging sampled images with <code>wandb</code> is given below.",de,H,ge,Q,ue,S,Qe="Assuming you’ve done with all the epochs and have pushed up your model to the hub, you can use the finetuned model as follows",fe,W,Me,X,we,R,Se=`This work is heavily influenced by the repo <a href="https://github.com/mihirp1998/AlignProp/" rel="nofollow">here</a> and the associated paper <a href="https://huggingface.co/papers/2310.03739" rel="nofollow">Aligning Text-to-Image Diffusion Models with Reward Backpropagation
by Mihir Prabhudesai, Anirudh Goyal, Deepak Pathak, Katerina Fragkiadaki</a>.`,ye,L,be,E,Te;return M=new V({props:{title:"Aligning Text-to-Image Diffusion Models with Reward Backpropagation",local:"aligning-text-to-image-diffusion-models-with-reward-backpropagation",headingTag:"h1"}}),y=new V({props:{title:"The why",local:"the-why",headingTag:"h2"}}),T=new V({props:{title:"Getting started with examples/scripts/alignprop.py",local:"getting-started-with-examplesscriptsalignproppy",headingTag:"h2"}}),$=new je({props:{code:"cHl0aG9uJTIwYWxpZ25wcm9wLnB5JTIwLS1oZl91c2VyX2FjY2Vzc190b2tlbiUyMCUzQ3Rva2VuJTNF",highlighted:'<span class="hljs-keyword">python</span> alignprop.<span class="hljs-keyword">py</span> --hf_user_access_token <span class="hljs-symbol">&lt;token&gt;</span>',wrap:!1}}),_=new V({props:{title:"Setting up the image logging hook function",local:"setting-up-the-image-logging-hook-function",headingTag:"h2"}}),v=new je({props:{code:"JTVCJ2ltYWdlJyUyQyUyMCdwcm9tcHQnJTJDJTIwJ3Byb21wdF9tZXRhZGF0YSclMkMlMjAncmV3YXJkcyclNUQlMEE=",highlighted:`[<span class="hljs-string">&#x27;image&#x27;</span>, <span class="hljs-string">&#x27;prompt&#x27;</span>, <span class="hljs-string">&#x27;prompt_metadata&#x27;</span>, <span class="hljs-string">&#x27;rewards&#x27;</span>]
`,wrap:!1}}),Z=new V({props:{title:"Key terms",local:"key-terms",headingTag:"h3"}}),H=new je({props:{code:"JTIzJTIwZm9yJTIwbG9nZ2luZyUyMHRoZXNlJTIwaW1hZ2VzJTIwdG8lMjB3YW5kYiUwQSUwQWRlZiUyMGltYWdlX291dHB1dHNfaG9vayhpbWFnZV9kYXRhJTJDJTIwZ2xvYmFsX3N0ZXAlMkMlMjBhY2NlbGVyYXRlX2xvZ2dlciklM0ElMEElMjAlMjAlMjAlMjAlMjMlMjBGb3IlMjB0aGUlMjBzYWtlJTIwb2YlMjB0aGlzJTIwZXhhbXBsZSUyQyUyMHdlJTIwb25seSUyMGNhcmUlMjBhYm91dCUyMHRoZSUyMGxhc3QlMjBiYXRjaCUwQSUyMCUyMCUyMCUyMCUyMyUyMGhlbmNlJTIwd2UlMjBleHRyYWN0JTIwdGhlJTIwbGFzdCUyMGVsZW1lbnQlMjBvZiUyMHRoZSUyMGxpc3QlMEElMjAlMjAlMjAlMjByZXN1bHQlMjAlM0QlMjAlN0IlN0QlMEElMjAlMjAlMjAlMjBpbWFnZXMlMkMlMjBwcm9tcHRzJTJDJTIwcmV3YXJkcyUyMCUzRCUyMCU1QmltYWdlX2RhdGElNUInaW1hZ2VzJyU1RCUyQ2ltYWdlX2RhdGElNUIncHJvbXB0cyclNUQlMkNpbWFnZV9kYXRhJTVCJ3Jld2FyZHMnJTVEJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwaSUyQyUyMGltYWdlJTIwaW4lMjBlbnVtZXJhdGUoaW1hZ2VzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBpbCUyMCUzRCUyMEltYWdlLmZyb21hcnJheSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAoaW1hZ2UuY3B1KCkubnVtcHkoKS50cmFuc3Bvc2UoMSUyQyUyMDIlMkMlMjAwKSUyMColMjAyNTUpLmFzdHlwZShucC51aW50OCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGlsJTIwJTNEJTIwcGlsLnJlc2l6ZSgoMjU2JTJDJTIwMjU2KSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXN1bHQlNUJmJTIyJTdCcHJvbXB0cyU1QmklNUQlM0EuMjUlN0QlMjAlN0MlMjAlN0JyZXdhcmRzJTVCaSU1RCUzQS4yZiU3RCUyMiU1RCUyMCUzRCUyMCU1QnBpbCU1RCUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdGVfbG9nZ2VyLmxvZ19pbWFnZXMoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RlcCUzRGdsb2JhbF9zdGVwJTJDJTBBJTIwJTIwJTIwJTIwKSUwQQ==",highlighted:`<span class="hljs-comment"># for logging these images to wandb</span>

<span class="hljs-keyword">def</span> <span class="hljs-title function_">image_outputs_hook</span>(<span class="hljs-params">image_data, global_step, accelerate_logger</span>):
    <span class="hljs-comment"># For the sake of this example, we only care about the last batch</span>
    <span class="hljs-comment"># hence we extract the last element of the list</span>
    result = {}
    images, prompts, rewards = [image_data[<span class="hljs-string">&#x27;images&#x27;</span>],image_data[<span class="hljs-string">&#x27;prompts&#x27;</span>],image_data[<span class="hljs-string">&#x27;rewards&#x27;</span>]]
    <span class="hljs-keyword">for</span> i, image <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(images):
        pil = Image.fromarray(
            (image.cpu().numpy().transpose(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>) * <span class="hljs-number">255</span>).astype(np.uint8)
        )
        pil = pil.resize((<span class="hljs-number">256</span>, <span class="hljs-number">256</span>))
        result[<span class="hljs-string">f&quot;<span class="hljs-subst">{prompts[i]:<span class="hljs-number">.25</span>}</span> | <span class="hljs-subst">{rewards[i]:<span class="hljs-number">.2</span>f}</span>&quot;</span>] = [pil]
    accelerate_logger.log_images(
        result,
        step=global_step,
    )
`,wrap:!1}}),Q=new V({props:{title:"Using the finetuned model",local:"using-the-finetuned-model",headingTag:"h3"}}),W=new je({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBcGlwZWxpbmUlMjAlM0QlMjBTdGFibGVEaWZmdXNpb25QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTIycnVud2F5bWwlMkZzdGFibGUtZGlmZnVzaW9uLXYxLTUlMjIpJTBBcGlwZWxpbmUudG8oJTIyY3VkYSUyMiklMEElMEFwaXBlbGluZS5sb2FkX2xvcmFfd2VpZ2h0cygnbWloaXJwZCUyRmFsaWducHJvcC10cmwtYWVzdGhldGljcycpJTBBJTBBcHJvbXB0cyUyMCUzRCUyMCU1QiUyMnNxdWlycmVsJTIyJTJDJTIwJTIyY3JhYiUyMiUyQyUyMCUyMnN0YXJmaXNoJTIyJTJDJTIwJTIyd2hhbGUlMjIlMkMlMjJzcG9uZ2UlMjIlMkMlMjAlMjJwbGFua3RvbiUyMiU1RCUwQXJlc3VsdHMlMjAlM0QlMjBwaXBlbGluZShwcm9tcHRzKSUwQSUwQWZvciUyMHByb21wdCUyQyUyMGltYWdlJTIwaW4lMjB6aXAocHJvbXB0cyUyQ3Jlc3VsdHMuaW1hZ2VzKSUzQSUwQSUyMCUyMCUyMCUyMGltYWdlLnNhdmUoZiUyMmR1bXAlMkYlN0Jwcm9tcHQlN0QucG5nJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
pipeline = StableDiffusionPipeline.from_pretrained(<span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>)
pipeline.to(<span class="hljs-string">&quot;cuda&quot;</span>)

pipeline.load_lora_weights(<span class="hljs-string">&#x27;mihirpd/alignprop-trl-aesthetics&#x27;</span>)

prompts = [<span class="hljs-string">&quot;squirrel&quot;</span>, <span class="hljs-string">&quot;crab&quot;</span>, <span class="hljs-string">&quot;starfish&quot;</span>, <span class="hljs-string">&quot;whale&quot;</span>,<span class="hljs-string">&quot;sponge&quot;</span>, <span class="hljs-string">&quot;plankton&quot;</span>]
results = pipeline(prompts)

<span class="hljs-keyword">for</span> prompt, image <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prompts,results.images):
    image.save(<span class="hljs-string">f&quot;dump/<span class="hljs-subst">{prompt}</span>.png&quot;</span>)`,wrap:!1}}),X=new V({props:{title:"Credits",local:"credits",headingTag:"h2"}}),L=new Pe({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/alignprop_trainer.md"}}),{c(){u=i("meta"),N=a(),Y=i("p"),z=a(),r(M.$$.fragment),F=a(),w=i("p"),w.innerHTML=Ue,P=a(),r(y.$$.fragment),q=a(),b=i("p"),b.textContent=$e,D=a(),f=i("div"),f.innerHTML=Ce,K=a(),r(T.$$.fragment),O=a(),j=i("p"),j.innerHTML=ke,ee=a(),J=i("p"),J.innerHTML=xe,te=a(),U=i("p"),U.innerHTML=_e,le=a(),r($.$$.fragment),se=a(),C=i("p"),C.innerHTML=Ie,ae=a(),k=i("p"),k.textContent=ve,ne=a(),x=i("ul"),x.innerHTML=Ae,ie=a(),r(_.$$.fragment),pe=a(),I=i("p"),I.textContent=Ze,oe=a(),r(v.$$.fragment),re=a(),A=i("p"),A.innerHTML=Be,me=a(),r(Z.$$.fragment),he=a(),B=i("ul"),B.innerHTML=Ge,ce=a(),G=i("p"),G.innerHTML=He,de=a(),r(H.$$.fragment),ge=a(),r(Q.$$.fragment),ue=a(),S=i("p"),S.textContent=Qe,fe=a(),r(W.$$.fragment),Me=a(),r(X.$$.fragment),we=a(),R=i("p"),R.innerHTML=Se,ye=a(),r(L.$$.fragment),be=a(),E=i("p"),this.h()},l(e){const t=Ne("svelte-u9bgzb",document.head);u=p(t,"META",{name:!0,content:!0}),t.forEach(l),N=n(e),Y=p(e,"P",{}),We(Y).forEach(l),z=n(e),m(M.$$.fragment,e),F=n(e),w=p(e,"P",{"data-svelte-h":!0}),o(w)!=="svelte-e1hp4c"&&(w.innerHTML=Ue),P=n(e),m(y.$$.fragment,e),q=n(e),b=p(e,"P",{"data-svelte-h":!0}),o(b)!=="svelte-1ekl94b"&&(b.textContent=$e),D=n(e),f=p(e,"DIV",{style:!0,"data-svelte-h":!0}),o(f)!=="svelte-1mythgm"&&(f.innerHTML=Ce),K=n(e),m(T.$$.fragment,e),O=n(e),j=p(e,"P",{"data-svelte-h":!0}),o(j)!=="svelte-5pt88m"&&(j.innerHTML=ke),ee=n(e),J=p(e,"P",{"data-svelte-h":!0}),o(J)!=="svelte-13c8m5g"&&(J.innerHTML=xe),te=n(e),U=p(e,"P",{"data-svelte-h":!0}),o(U)!=="svelte-mg8jo0"&&(U.innerHTML=_e),le=n(e),m($.$$.fragment,e),se=n(e),C=p(e,"P",{"data-svelte-h":!0}),o(C)!=="svelte-mmabua"&&(C.innerHTML=Ie),ae=n(e),k=p(e,"P",{"data-svelte-h":!0}),o(k)!=="svelte-gn7d4"&&(k.textContent=ve),ne=n(e),x=p(e,"UL",{"data-svelte-h":!0}),o(x)!=="svelte-1q038ly"&&(x.innerHTML=Ae),ie=n(e),m(_.$$.fragment,e),pe=n(e),I=p(e,"P",{"data-svelte-h":!0}),o(I)!=="svelte-irm1pa"&&(I.textContent=Ze),oe=n(e),m(v.$$.fragment,e),re=n(e),A=p(e,"P",{"data-svelte-h":!0}),o(A)!=="svelte-1txp1uc"&&(A.innerHTML=Be),me=n(e),m(Z.$$.fragment,e),he=n(e),B=p(e,"UL",{"data-svelte-h":!0}),o(B)!=="svelte-1rki34a"&&(B.innerHTML=Ge),ce=n(e),G=p(e,"P",{"data-svelte-h":!0}),o(G)!=="svelte-1r6a60u"&&(G.innerHTML=He),de=n(e),m(H.$$.fragment,e),ge=n(e),m(Q.$$.fragment,e),ue=n(e),S=p(e,"P",{"data-svelte-h":!0}),o(S)!=="svelte-eqxomu"&&(S.textContent=Qe),fe=n(e),m(W.$$.fragment,e),Me=n(e),m(X.$$.fragment,e),we=n(e),R=p(e,"P",{"data-svelte-h":!0}),o(R)!=="svelte-8i7v5m"&&(R.innerHTML=Se),ye=n(e),m(L.$$.fragment,e),be=n(e),E=p(e,"P",{}),We(E).forEach(l),this.h()},h(){Xe(u,"name","hf:doc:metadata"),Xe(u,"content",De),ze(f,"text-align","center")},m(e,t){Fe(document.head,u),s(e,N,t),s(e,Y,t),s(e,z,t),h(M,e,t),s(e,F,t),s(e,w,t),s(e,P,t),h(y,e,t),s(e,q,t),s(e,b,t),s(e,D,t),s(e,f,t),s(e,K,t),h(T,e,t),s(e,O,t),s(e,j,t),s(e,ee,t),s(e,J,t),s(e,te,t),s(e,U,t),s(e,le,t),h($,e,t),s(e,se,t),s(e,C,t),s(e,ae,t),s(e,k,t),s(e,ne,t),s(e,x,t),s(e,ie,t),h(_,e,t),s(e,pe,t),s(e,I,t),s(e,oe,t),h(v,e,t),s(e,re,t),s(e,A,t),s(e,me,t),h(Z,e,t),s(e,he,t),s(e,B,t),s(e,ce,t),s(e,G,t),s(e,de,t),h(H,e,t),s(e,ge,t),h(Q,e,t),s(e,ue,t),s(e,S,t),s(e,fe,t),h(W,e,t),s(e,Me,t),h(X,e,t),s(e,we,t),s(e,R,t),s(e,ye,t),h(L,e,t),s(e,be,t),s(e,E,t),Te=!0},p:Le,i(e){Te||(c(M.$$.fragment,e),c(y.$$.fragment,e),c(T.$$.fragment,e),c($.$$.fragment,e),c(_.$$.fragment,e),c(v.$$.fragment,e),c(Z.$$.fragment,e),c(H.$$.fragment,e),c(Q.$$.fragment,e),c(W.$$.fragment,e),c(X.$$.fragment,e),c(L.$$.fragment,e),Te=!0)},o(e){d(M.$$.fragment,e),d(y.$$.fragment,e),d(T.$$.fragment,e),d($.$$.fragment,e),d(_.$$.fragment,e),d(v.$$.fragment,e),d(Z.$$.fragment,e),d(H.$$.fragment,e),d(Q.$$.fragment,e),d(W.$$.fragment,e),d(X.$$.fragment,e),d(L.$$.fragment,e),Te=!1},d(e){e&&(l(N),l(Y),l(z),l(F),l(w),l(P),l(q),l(b),l(D),l(f),l(K),l(O),l(j),l(ee),l(J),l(te),l(U),l(le),l(se),l(C),l(ae),l(k),l(ne),l(x),l(ie),l(pe),l(I),l(oe),l(re),l(A),l(me),l(he),l(B),l(ce),l(G),l(de),l(ge),l(ue),l(S),l(fe),l(Me),l(we),l(R),l(ye),l(be),l(E)),l(u),g(M,e),g(y,e),g(T,e),g($,e),g(_,e),g(v,e),g(Z,e),g(H,e),g(Q,e),g(W,e),g(X,e),g(L,e)}}}const De='{"title":"Aligning Text-to-Image Diffusion Models with Reward Backpropagation","local":"aligning-text-to-image-diffusion-models-with-reward-backpropagation","sections":[{"title":"The why","local":"the-why","sections":[],"depth":2},{"title":"Getting started with examples/scripts/alignprop.py","local":"getting-started-with-examplesscriptsalignproppy","sections":[],"depth":2},{"title":"Setting up the image logging hook function","local":"setting-up-the-image-logging-hook-function","sections":[{"title":"Key terms","local":"key-terms","sections":[],"depth":3},{"title":"Using the finetuned model","local":"using-the-finetuned-model","sections":[],"depth":3}],"depth":2},{"title":"Credits","local":"credits","sections":[],"depth":2}],"depth":1}';function Ke(Je){return Ve(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class st extends Ye{constructor(u){super(),Ee(this,u,Ke,qe,Re,{})}}export{st as component};
