import{s as ln,n as cn,o as dn}from"../chunks/scheduler.d627b047.js";import{S as pn,i as mn,g as r,s as o,r as d,A as gn,h as s,f as n,c as i,j as C,u as p,x as l,k as $,y as c,a,v as m,d as g,t as h,w as f}from"../chunks/index.73c51727.js";import{D as xe}from"../chunks/Docstring.0ad99a5d.js";import{C as _t}from"../chunks/CodeBlock.5f78c87f.js";import{H as y,E as hn}from"../chunks/index.8df1db42.js";function fn(Pt){let v,ke,Ce,je,k,Je,j,Ut='<a href="https://huggingface.co/models?other=ddpo,trl" rel="nofollow"><img src="https://img.shields.io/badge/All_models-DDPO-blue"/></a>',Pe,J,Ue,P,Ot='<thead><tr><th>Before</th> <th>After DDPO finetuning</th></tr></thead> <tbody><tr><td><div style="text-align: center"><img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/pre_squirrel.png"/></div></td> <td><div style="text-align: center"><img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/post_squirrel.png"/></div></td></tr> <tr><td><div style="text-align: center"><img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/pre_crab.png"/></div></td> <td><div style="text-align: center"><img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/post_crab.png"/></div></td></tr> <tr><td><div style="text-align: center"><img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/pre_starfish.png"/></div></td> <td><div style="text-align: center"><img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/post_starfish.png"/></div></td></tr></tbody>',Oe,U,Ae,O,At=`The machinery for finetuning of Stable Diffusion models with reinforcement learning makes heavy use of HuggingFace’s <code>diffusers</code>
library. A reason for stating this is that getting started requires a bit of familiarity with the <code>diffusers</code> library concepts, mainly two of them - pipelines and schedulers.
Right out of the box (<code>diffusers</code> library), there isn’t a <code>Pipeline</code> nor a <code>Scheduler</code> instance that is suitable for finetuning with reinforcement learning. Some adjustments need to be made.`,Ie,A,It=`There is a pipeline interface that is provided by this library that is required to be implemented to be used with the <code>DDPOTrainer</code>, which is the main machinery for fine-tuning Stable Diffusion with reinforcement learning. <strong>Note: Only the StableDiffusion architecture is supported at this point.</strong>
There is a default implementation of this interface that you can use out of the box. Assuming the default implementation is sufficient and/or to get things moving, refer to the training example alongside this guide.`,He,I,Ht="The point of the interface is to fuse the pipeline and the scheduler into one object which allows for minimalness in terms of having the constraints all in one place. The interface was designed in hopes of catering to pipelines and schedulers beyond the examples in this repository and elsewhere at this time of writing. Also the scheduler step is a method of this pipeline interface and this may seem redundant given that the raw scheduler is accessible via the interface but this is the only way to constrain the scheduler step output to an output type befitting of the algorithm at hand (DDPO).",Le,H,Lt='For a more detailed look into the interface and the associated default implementation, go <a href="https://github.com/lvwerra/trl/tree/main/trl/models/modeling_sd_base.py" rel="nofollow">here</a>',Ne,L,Nt="Note that the default implementation has a LoRA implementation path and a non-LoRA based implementation path. The LoRA flag enabled by default and this can be turned off by passing in the flag to do so. LORA based training is faster and the LORA associated model hyperparameters responsible for model convergence aren’t as finicky as non-LORA based training.",Ze,N,Zt="Also in addition, there is the expectation of providing a reward function and a prompt function. The reward function is used to evaluate the generated images and the prompt function is used to generate the prompts that are used to generate the images.",Be,Z,Ge,B,Bt="The <code>ddpo.py</code> script is a working example of using the <code>DDPO</code> trainer to finetune a Stable Diffusion model. This example explicitly configures a small subset of the overall parameters associated with the config object (<code>DDPOConfig</code>).",ze,G,Gt="<strong>Note:</strong> one A100 GPU is recommended to get this running. Anything below a A100 will not be able to run this example script and even if it does via relatively smaller sized parameters, the results will most likely be poor.",qe,z,zt='Almost every configuration parameter has a default. There is only one commandline flag argument that is required of the user to get things up and running. The user is expected to have a <a href="https://huggingface.co/docs/hub/security-tokens" rel="nofollow">huggingface user access token</a> that will be used to upload the model post finetuning to HuggingFace hub. The following bash command is to be entered to get things running',Se,q,Ee,S,qt="To obtain the documentation of <code>stable_diffusion_tuning.py</code>, please run <code>python stable_diffusion_tuning.py --help</code>",Re,E,St="The following are things to keep in mind (The code checks this for you as well) in general while configuring the trainer (beyond the use case of using the example script)",Xe,R,Et="<li>The configurable sample batch size (<code>--ddpo_config.sample_batch_size=6</code>) should be greater than or equal to the configurable training batch size (<code>--ddpo_config.train_batch_size=3</code>)</li> <li>The configurable sample batch size (<code>--ddpo_config.sample_batch_size=6</code>) must be divisible by the configurable train batch size (<code>--ddpo_config.train_batch_size=3</code>)</li> <li>The configurable sample batch size (<code>--ddpo_config.sample_batch_size=6</code>) must be divisible by both the configurable gradient accumulation steps (<code>--ddpo_config.train_gradient_accumulation_steps=1</code>) and the configurable accelerator processes count</li>",We,X,Ye,W,Rt="Expect the function to be given a list of lists of the form",Qe,Y,Ve,Q,Xt=`and <code>image</code>, <code>prompt</code>, <code>prompt_metadata</code>, <code>rewards</code>, <code>reward_metadata</code> are batched.
The last list in the lists of lists represents the last sample batch. You are likely to want to log this one
While you are free to log however you want the use of <code>wandb</code> or <code>tensorboard</code> is recommended.`,Fe,V,Ke,F,Wt='<li><code>rewards</code> : The rewards/score is a numerical associated with the generated image and is key to steering the RL process</li> <li><code>reward_metadata</code> : The reward metadata is the metadata associated with the reward. Think of this as extra information payload delivered alongside the reward</li> <li><code>prompt</code> : The prompt is the text that is used to generate the image</li> <li><code>prompt_metadata</code> : The prompt metadata is the metadata associated with the prompt. A situation where this will not be empty is when the reward model comprises of a <a href="https://huggingface.co/docs/transformers/model_doc/flava" rel="nofollow"><code>FLAVA</code></a> setup where questions and ground answers (linked to the generated image) are expected with the generated image (See here: <a href="https://github.com/kvablack/ddpo-pytorch/blob/main/ddpo_pytorch/rewards.py#L45" rel="nofollow">https://github.com/kvablack/ddpo-pytorch/blob/main/ddpo_pytorch/rewards.py#L45</a>)</li> <li><code>image</code> : The image generated by the Stable Diffusion model</li>',et,K,Yt="Example code for logging sampled images with <code>wandb</code> is given below.",tt,ee,nt,te,at,ne,Qt="Assuming you’ve done with all the epochs and have pushed up your model to the hub, you can use the finetuned model as follows",ot,ae,it,oe,rt,ie,Vt=`This work is heavily influenced by the repo <a href="https://github.com/kvablack/ddpo-pytorch" rel="nofollow">here</a> and the associated paper <a href="https://huggingface.co/papers/2305.13301" rel="nofollow">Training Diffusion Models
with Reinforcement Learning by Kevin Black, Michael Janner, Yilan Du, Ilya Kostrikov, Sergey Levine</a>.`,st,re,lt,u,se,bt,fe,Ft=`The DDPOTrainer uses Deep Diffusion Policy Optimization to optimise diffusion models.
Note, this trainer is heavily inspired by the work here: <a href="https://github.com/kvablack/ddpo-pytorch" rel="nofollow">https://github.com/kvablack/ddpo-pytorch</a>
As of now only Stable Diffusion based pipelines are supported`,wt,T,le,yt,ue,Kt="Calculate the loss for a batch of an unpacked sample",vt,M,ce,Tt,_e,en="Creates a draft of a model card using the information available to the <code>Trainer</code>.",Mt,_,de,Dt,be,tn="Perform a single step of training.",xt,we,nn="Side Effects:",Ct,ye,an="<li>Model weights are updated</li> <li>Logs the statistics to the accelerator trackers.</li> <li>If <code>self.image_samples_callback</code> is not None, it will be called with the prompt_image_pairs, global_step, and the accelerator tracker.</li>",$t,D,pe,kt,ve,on="Train the model for a given number of epochs",ct,me,dt,w,ge,jt,Te,rn='Configuration class for the <a href="/docs/trl/main/en/ddpo_trainer#trl.DDPOTrainer">DDPOTrainer</a>.',Jt,Me,sn=`Using <a href="https://huggingface.co/docs/transformers/main/en/internal/trainer_utils#transformers.HfArgumentParser" rel="nofollow">HfArgumentParser</a> we can turn this class into
<a href="https://docs.python.org/3/library/argparse#module-argparse" rel="nofollow">argparse</a> arguments that can be specified on the
command line.`,pt,he,mt,$e,gt;return k=new y({props:{title:"Denoising Diffusion Policy Optimization",local:"denoising-diffusion-policy-optimization",headingTag:"h1"}}),J=new y({props:{title:"The why",local:"the-why",headingTag:"h2"}}),U=new y({props:{title:"Getting started with Stable Diffusion finetuning with reinforcement learning",local:"getting-started-with-stable-diffusion-finetuning-with-reinforcement-learning",headingTag:"h2"}}),Z=new y({props:{title:"Getting started with examples/scripts/ddpo.py",local:"getting-started-with-examplesscriptsddpopy",headingTag:"h2"}}),q=new _t({props:{code:"cHl0aG9uJTIwZGRwby5weSUyMC0taGZfdXNlcl9hY2Nlc3NfdG9rZW4lMjAlM0N0b2tlbiUzRQ==",highlighted:'<span class="hljs-keyword">python</span> ddpo.<span class="hljs-keyword">py</span> --hf_user_access_token <span class="hljs-symbol">&lt;token&gt;</span>',wrap:!1}}),X=new y({props:{title:"Setting up the image logging hook function",local:"setting-up-the-image-logging-hook-function",headingTag:"h2"}}),Y=new _t({props:{code:"JTVCJTVCaW1hZ2UlMkMlMjBwcm9tcHQlMkMlMjBwcm9tcHRfbWV0YWRhdGElMkMlMjByZXdhcmRzJTJDJTIwcmV3YXJkX21ldGFkYXRhJTVEJTJDJTIwLi4uJTVEJTBB",highlighted:`[[image, prompt, prompt_metadata, rewards, reward_metadata], ...]
`,wrap:!1}}),V=new y({props:{title:"Key terms",local:"key-terms",headingTag:"h3"}}),ee=new _t({props:{code:"JTIzJTIwZm9yJTIwbG9nZ2luZyUyMHRoZXNlJTIwaW1hZ2VzJTIwdG8lMjB3YW5kYiUwQSUwQWRlZiUyMGltYWdlX291dHB1dHNfaG9vayhpbWFnZV9kYXRhJTJDJTIwZ2xvYmFsX3N0ZXAlMkMlMjBhY2NlbGVyYXRlX2xvZ2dlciklM0ElMEElMjAlMjAlMjAlMjAlMjMlMjBGb3IlMjB0aGUlMjBzYWtlJTIwb2YlMjB0aGlzJTIwZXhhbXBsZSUyQyUyMHdlJTIwb25seSUyMGNhcmUlMjBhYm91dCUyMHRoZSUyMGxhc3QlMjBiYXRjaCUwQSUyMCUyMCUyMCUyMCUyMyUyMGhlbmNlJTIwd2UlMjBleHRyYWN0JTIwdGhlJTIwbGFzdCUyMGVsZW1lbnQlMjBvZiUyMHRoZSUyMGxpc3QlMEElMjAlMjAlMjAlMjByZXN1bHQlMjAlM0QlMjAlN0IlN0QlMEElMjAlMjAlMjAlMjBpbWFnZXMlMkMlMjBwcm9tcHRzJTJDJTIwXyUyQyUyMHJld2FyZHMlMkMlMjBfJTIwJTNEJTIwaW1hZ2VfZGF0YSU1Qi0xJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwaSUyQyUyMGltYWdlJTIwaW4lMjBlbnVtZXJhdGUoaW1hZ2VzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBpbCUyMCUzRCUyMEltYWdlLmZyb21hcnJheSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAoaW1hZ2UuY3B1KCkubnVtcHkoKS50cmFuc3Bvc2UoMSUyQyUyMDIlMkMlMjAwKSUyMColMjAyNTUpLmFzdHlwZShucC51aW50OCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGlsJTIwJTNEJTIwcGlsLnJlc2l6ZSgoMjU2JTJDJTIwMjU2KSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXN1bHQlNUJmJTIyJTdCcHJvbXB0cyU1QmklNUQlM0EuMjUlN0QlMjAlN0MlMjAlN0JyZXdhcmRzJTVCaSU1RCUzQS4yZiU3RCUyMiU1RCUyMCUzRCUyMCU1QnBpbCU1RCUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdGVfbG9nZ2VyLmxvZ19pbWFnZXMoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RlcCUzRGdsb2JhbF9zdGVwJTJDJTBBJTIwJTIwJTIwJTIwKSUwQQ==",highlighted:`<span class="hljs-comment"># for logging these images to wandb</span>

<span class="hljs-keyword">def</span> <span class="hljs-title function_">image_outputs_hook</span>(<span class="hljs-params">image_data, global_step, accelerate_logger</span>):
    <span class="hljs-comment"># For the sake of this example, we only care about the last batch</span>
    <span class="hljs-comment"># hence we extract the last element of the list</span>
    result = {}
    images, prompts, _, rewards, _ = image_data[-<span class="hljs-number">1</span>]
    <span class="hljs-keyword">for</span> i, image <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(images):
        pil = Image.fromarray(
            (image.cpu().numpy().transpose(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>) * <span class="hljs-number">255</span>).astype(np.uint8)
        )
        pil = pil.resize((<span class="hljs-number">256</span>, <span class="hljs-number">256</span>))
        result[<span class="hljs-string">f&quot;<span class="hljs-subst">{prompts[i]:<span class="hljs-number">.25</span>}</span> | <span class="hljs-subst">{rewards[i]:<span class="hljs-number">.2</span>f}</span>&quot;</span>] = [pil]
    accelerate_logger.log_images(
        result,
        step=global_step,
    )
`,wrap:!1}}),te=new y({props:{title:"Using the finetuned model",local:"using-the-finetuned-model",headingTag:"h3"}}),ae=new _t({props:{code:"JTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJsJTIwaW1wb3J0JTIwRGVmYXVsdEREUE9TdGFibGVEaWZmdXNpb25QaXBlbGluZSUwQSUwQXBpcGVsaW5lJTIwJTNEJTIwRGVmYXVsdEREUE9TdGFibGVEaWZmdXNpb25QaXBlbGluZSglMjJtZXRyaWMtc3BhY2UlMkZkZHBvLWZpbmV0dW5lZC1zZC1tb2RlbCUyMiklMEElMEFkZXZpY2UlMjAlM0QlMjB0b3JjaC5kZXZpY2UoJTIyY3VkYSUyMiklMjBpZiUyMHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklMjBlbHNlJTIwdG9yY2guZGV2aWNlKCUyMmNwdSUyMiklMEElMEElMjMlMjBtZW1vcnklMjBvcHRpbWl6YXRpb24lMEFwaXBlbGluZS52YWUudG8oZGV2aWNlJTJDJTIwdG9yY2guZmxvYXQxNiklMEFwaXBlbGluZS50ZXh0X2VuY29kZXIudG8oZGV2aWNlJTJDJTIwdG9yY2guZmxvYXQxNiklMEFwaXBlbGluZS51bmV0LnRvKGRldmljZSUyQyUyMHRvcmNoLmZsb2F0MTYpJTBBJTBBcHJvbXB0cyUyMCUzRCUyMCU1QiUyMnNxdWlycmVsJTIyJTJDJTIwJTIyY3JhYiUyMiUyQyUyMCUyMnN0YXJmaXNoJTIyJTJDJTIwJTIyd2hhbGUlMjIlMkMlMjJzcG9uZ2UlMjIlMkMlMjAlMjJwbGFua3RvbiUyMiU1RCUwQXJlc3VsdHMlMjAlM0QlMjBwaXBlbGluZShwcm9tcHRzKSUwQSUwQWZvciUyMHByb21wdCUyQyUyMGltYWdlJTIwaW4lMjB6aXAocHJvbXB0cyUyQ3Jlc3VsdHMuaW1hZ2VzKSUzQSUwQSUyMCUyMCUyMCUyMGltYWdlLnNhdmUoZiUyMiU3QnByb21wdCU3RC5wbmclMjIpJTBB",highlighted:`
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> DefaultDDPOStableDiffusionPipeline

pipeline = DefaultDDPOStableDiffusionPipeline(<span class="hljs-string">&quot;metric-space/ddpo-finetuned-sd-model&quot;</span>)

device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>) <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)

<span class="hljs-comment"># memory optimization</span>
pipeline.vae.to(device, torch.float16)
pipeline.text_encoder.to(device, torch.float16)
pipeline.unet.to(device, torch.float16)

prompts = [<span class="hljs-string">&quot;squirrel&quot;</span>, <span class="hljs-string">&quot;crab&quot;</span>, <span class="hljs-string">&quot;starfish&quot;</span>, <span class="hljs-string">&quot;whale&quot;</span>,<span class="hljs-string">&quot;sponge&quot;</span>, <span class="hljs-string">&quot;plankton&quot;</span>]
results = pipeline(prompts)

<span class="hljs-keyword">for</span> prompt, image <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prompts,results.images):
    image.save(<span class="hljs-string">f&quot;<span class="hljs-subst">{prompt}</span>.png&quot;</span>)
`,wrap:!1}}),oe=new y({props:{title:"Credits",local:"credits",headingTag:"h2"}}),re=new y({props:{title:"DDPOTrainer",local:"trl.DDPOTrainer",headingTag:"h2"}}),se=new xe({props:{name:"class trl.DDPOTrainer",anchor:"trl.DDPOTrainer",parameters:[{name:"config",val:": DDPOConfig"},{name:"reward_function",val:": typing.Callable[[torch.Tensor, tuple[str], tuple[typing.Any]], torch.Tensor]"},{name:"prompt_function",val:": typing.Callable[[], tuple[str, typing.Any]]"},{name:"sd_pipeline",val:": DDPOStableDiffusionPipeline"},{name:"image_samples_hook",val:": typing.Optional[typing.Callable[[typing.Any, typing.Any, typing.Any], typing.Any]] = None"}],parametersDescription:[{anchor:"trl.DDPOTrainer.*config**",description:`*<strong>*config**</strong> (<code>DDPOConfig</code>) &#x2014; Configuration object for DDPOTrainer. Check the documentation of <code>PPOConfig</code> for more &#x2014;
details.`,name:"*config**"},{anchor:"trl.DDPOTrainer.*reward_function**",description:"*<strong>*reward_function**</strong> (Callable[[torch.Tensor, tuple[str], tuple[Any]], torch.Tensor]) &#x2014; Reward function to be used &#x2014;",name:"*reward_function**"},{anchor:"trl.DDPOTrainer.*prompt_function**",description:"*<strong>*prompt_function**</strong> (Callable[[], tuple[str, Any]]) &#x2014; Function to generate prompts to guide model &#x2014;",name:"*prompt_function**"},{anchor:"trl.DDPOTrainer.*sd_pipeline**",description:"*<strong>*sd_pipeline**</strong> (<code>DDPOStableDiffusionPipeline</code>) &#x2014; Stable Diffusion pipeline to be used for training. &#x2014;",name:"*sd_pipeline**"},{anchor:"trl.DDPOTrainer.*image_samples_hook**",description:"*<strong>*image_samples_hook**</strong> (Optional[Callable[[Any, Any, Any], Any]]) &#x2014; Hook to be called to log images &#x2014;",name:"*image_samples_hook**"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/ddpo_trainer.py#L41"}}),le=new xe({props:{name:"calculate_loss",anchor:"trl.DDPOTrainer.calculate_loss",parameters:[{name:"latents",val:""},{name:"timesteps",val:""},{name:"next_latents",val:""},{name:"log_probs",val:""},{name:"advantages",val:""},{name:"embeds",val:""}],parametersDescription:[{anchor:"trl.DDPOTrainer.calculate_loss.latents",description:`<strong>latents</strong> (torch.Tensor) &#x2014;
The latents sampled from the diffusion model, shape: [batch_size, num_channels_latents, height, width]`,name:"latents"},{anchor:"trl.DDPOTrainer.calculate_loss.timesteps",description:`<strong>timesteps</strong> (torch.Tensor) &#x2014;
The timesteps sampled from the diffusion model, shape: [batch_size]`,name:"timesteps"},{anchor:"trl.DDPOTrainer.calculate_loss.next_latents",description:`<strong>next_latents</strong> (torch.Tensor) &#x2014;
The next latents sampled from the diffusion model, shape: [batch_size, num_channels_latents, height, width]`,name:"next_latents"},{anchor:"trl.DDPOTrainer.calculate_loss.log_probs",description:`<strong>log_probs</strong> (torch.Tensor) &#x2014;
The log probabilities of the latents, shape: [batch_size]`,name:"log_probs"},{anchor:"trl.DDPOTrainer.calculate_loss.advantages",description:`<strong>advantages</strong> (torch.Tensor) &#x2014;
The advantages of the latents, shape: [batch_size]`,name:"advantages"},{anchor:"trl.DDPOTrainer.calculate_loss.embeds",description:`<strong>embeds</strong> (torch.Tensor) &#x2014;
The embeddings of the prompts, shape: [2*batch_size or batch_size, &#x2026;]
Note: the &#x201C;or&#x201D; is because if train_cfg is True, the expectation is that negative prompts are concatenated to the embeds`,name:"embeds"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/ddpo_trainer.py#L326",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>loss (torch.Tensor), approx_kl (torch.Tensor), clipfrac (torch.Tensor)
(all of these are of shape (1,))</p>
`}}),ce=new xe({props:{name:"create_model_card",anchor:"trl.DDPOTrainer.create_model_card",parameters:[{name:"model_name",val:": typing.Optional[str] = None"},{name:"dataset_name",val:": typing.Optional[str] = None"},{name:"tags",val:": typing.Union[str, list[str], NoneType] = None"}],parametersDescription:[{anchor:"trl.DDPOTrainer.create_model_card.model_name",description:`<strong>model_name</strong> (<code>str</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Name of the model.`,name:"model_name"},{anchor:"trl.DDPOTrainer.create_model_card.dataset_name",description:`<strong>dataset_name</strong> (<code>str</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Name of the dataset used for training.`,name:"dataset_name"},{anchor:"trl.DDPOTrainer.create_model_card.tags",description:`<strong>tags</strong> (<code>str</code>, <code>list[str]</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Tags to be associated with the model card.`,name:"tags"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/ddpo_trainer.py#L596"}}),de=new xe({props:{name:"step",anchor:"trl.DDPOTrainer.step",parameters:[{name:"epoch",val:": int"},{name:"global_step",val:": int"}],parametersDescription:[{anchor:"trl.DDPOTrainer.step.epoch",description:"<strong>epoch</strong> (int) &#x2014; The current epoch.",name:"epoch"},{anchor:"trl.DDPOTrainer.step.global_step",description:"<strong>global_step</strong> (int) &#x2014; The current global step.",name:"global_step"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/ddpo_trainer.py#L220",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The updated global step.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>global_step (int)</p>
`}}),pe=new xe({props:{name:"train",anchor:"trl.DDPOTrainer.train",parameters:[{name:"epochs",val:": typing.Optional[int] = None"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/ddpo_trainer.py#L582"}}),me=new y({props:{title:"DDPOConfig",local:"trl.DDPOConfig",headingTag:"h2"}}),ge=new xe({props:{name:"class trl.DDPOConfig",anchor:"trl.DDPOConfig",parameters:[{name:"exp_name",val:": str = 'doc-buil'"},{name:"run_name",val:": str = ''"},{name:"seed",val:": int = 0"},{name:"log_with",val:": typing.Optional[str] = None"},{name:"tracker_kwargs",val:": dict = <factory>"},{name:"accelerator_kwargs",val:": dict = <factory>"},{name:"project_kwargs",val:": dict = <factory>"},{name:"tracker_project_name",val:": str = 'trl'"},{name:"logdir",val:": str = 'logs'"},{name:"num_epochs",val:": int = 100"},{name:"save_freq",val:": int = 1"},{name:"num_checkpoint_limit",val:": int = 5"},{name:"mixed_precision",val:": str = 'fp16'"},{name:"allow_tf32",val:": bool = True"},{name:"resume_from",val:": str = ''"},{name:"sample_num_steps",val:": int = 50"},{name:"sample_eta",val:": float = 1.0"},{name:"sample_guidance_scale",val:": float = 5.0"},{name:"sample_batch_size",val:": int = 1"},{name:"sample_num_batches_per_epoch",val:": int = 2"},{name:"train_batch_size",val:": int = 1"},{name:"train_use_8bit_adam",val:": bool = False"},{name:"train_learning_rate",val:": float = 0.0003"},{name:"train_adam_beta1",val:": float = 0.9"},{name:"train_adam_beta2",val:": float = 0.999"},{name:"train_adam_weight_decay",val:": float = 0.0001"},{name:"train_adam_epsilon",val:": float = 1e-08"},{name:"train_gradient_accumulation_steps",val:": int = 1"},{name:"train_max_grad_norm",val:": float = 1.0"},{name:"train_num_inner_epochs",val:": int = 1"},{name:"train_cfg",val:": bool = True"},{name:"train_adv_clip_max",val:": float = 5.0"},{name:"train_clip_range",val:": float = 0.0001"},{name:"train_timestep_fraction",val:": float = 1.0"},{name:"per_prompt_stat_tracking",val:": bool = False"},{name:"per_prompt_stat_tracking_buffer_size",val:": int = 16"},{name:"per_prompt_stat_tracking_min_count",val:": int = 16"},{name:"async_reward_computation",val:": bool = False"},{name:"max_workers",val:": int = 2"},{name:"negative_prompts",val:": str = ''"},{name:"push_to_hub",val:": bool = False"}],parametersDescription:[{anchor:"trl.DDPOConfig.exp_name",description:`<strong>exp_name</strong> (<code>str</code>, <em>optional</em>, defaults to <code>os.path.basename(sys.argv[0])[ -- -len(&quot;.py&quot;)]</code>):
Name of this experiment (by default is the file name without the extension name).`,name:"exp_name"},{anchor:"trl.DDPOConfig.run_name",description:`<strong>run_name</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&quot;</code>) &#x2014;
Name of this run.`,name:"run_name"},{anchor:"trl.DDPOConfig.seed",description:`<strong>seed</strong> (<code>int</code>, <em>optional</em>, defaults to <code>0</code>) &#x2014;
Random seed.`,name:"seed"},{anchor:"trl.DDPOConfig.log_with",description:`<strong>log_with</strong> (<code>Literal[&quot;wandb&quot;, &quot;tensorboard&quot;]]</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Log with either &#x2018;wandb&#x2019; or &#x2018;tensorboard&#x2019;, check
<a href="https://huggingface.co/docs/accelerate/usage_guides/tracking" rel="nofollow">https://huggingface.co/docs/accelerate/usage_guides/tracking</a> for more details.`,name:"log_with"},{anchor:"trl.DDPOConfig.tracker_kwargs",description:`<strong>tracker_kwargs</strong> (<code>Dict</code>, <em>optional</em>, defaults to <code>{}</code>) &#x2014;
Keyword arguments for the tracker (e.g. wandb_project).`,name:"tracker_kwargs"},{anchor:"trl.DDPOConfig.accelerator_kwargs",description:`<strong>accelerator_kwargs</strong> (<code>Dict</code>, <em>optional</em>, defaults to <code>{}</code>) &#x2014;
Keyword arguments for the accelerator.`,name:"accelerator_kwargs"},{anchor:"trl.DDPOConfig.project_kwargs",description:`<strong>project_kwargs</strong> (<code>Dict</code>, <em>optional</em>, defaults to <code>{}</code>) &#x2014;
Keyword arguments for the accelerator project config (e.g. <code>logging_dir</code>).`,name:"project_kwargs"},{anchor:"trl.DDPOConfig.tracker_project_name",description:`<strong>tracker_project_name</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;trl&quot;</code>) &#x2014;
Name of project to use for tracking.`,name:"tracker_project_name"},{anchor:"trl.DDPOConfig.logdir",description:`<strong>logdir</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;logs&quot;</code>) &#x2014;
Top-level logging directory for checkpoint saving.`,name:"logdir"},{anchor:"trl.DDPOConfig.num_epochs",description:`<strong>num_epochs</strong> (<code>int</code>, <em>optional</em>, defaults to <code>100</code>) &#x2014;
Number of epochs to train.`,name:"num_epochs"},{anchor:"trl.DDPOConfig.save_freq",description:`<strong>save_freq</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1</code>) &#x2014;
Number of epochs between saving model checkpoints.`,name:"save_freq"},{anchor:"trl.DDPOConfig.num_checkpoint_limit",description:`<strong>num_checkpoint_limit</strong> (<code>int</code>, <em>optional</em>, defaults to <code>5</code>) &#x2014;
Number of checkpoints to keep before overwriting old ones.`,name:"num_checkpoint_limit"},{anchor:"trl.DDPOConfig.mixed_precision",description:`<strong>mixed_precision</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;fp16&quot;</code>) &#x2014;
Mixed precision training.`,name:"mixed_precision"},{anchor:"trl.DDPOConfig.allow_tf32",description:`<strong>allow_tf32</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Allow <code>tf32</code> on Ampere GPUs.`,name:"allow_tf32"},{anchor:"trl.DDPOConfig.resume_from",description:`<strong>resume_from</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&quot;</code>) &#x2014;
Resume training from a checkpoint.`,name:"resume_from"},{anchor:"trl.DDPOConfig.sample_num_steps",description:`<strong>sample_num_steps</strong> (<code>int</code>, <em>optional</em>, defaults to <code>50</code>) &#x2014;
Number of sampler inference steps.`,name:"sample_num_steps"},{anchor:"trl.DDPOConfig.sample_eta",description:`<strong>sample_eta</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1.0</code>) &#x2014;
Eta parameter for the DDIM sampler.`,name:"sample_eta"},{anchor:"trl.DDPOConfig.sample_guidance_scale",description:`<strong>sample_guidance_scale</strong> (<code>float</code>, <em>optional</em>, defaults to <code>5.0</code>) &#x2014;
Classifier-free guidance weight.`,name:"sample_guidance_scale"},{anchor:"trl.DDPOConfig.sample_batch_size",description:`<strong>sample_batch_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1</code>) &#x2014;
Batch size (per GPU) to use for sampling.`,name:"sample_batch_size"},{anchor:"trl.DDPOConfig.sample_num_batches_per_epoch",description:`<strong>sample_num_batches_per_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to <code>2</code>) &#x2014;
Number of batches to sample per epoch.`,name:"sample_num_batches_per_epoch"},{anchor:"trl.DDPOConfig.train_batch_size",description:`<strong>train_batch_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1</code>) &#x2014;
Batch size (per GPU) to use for training.`,name:"train_batch_size"},{anchor:"trl.DDPOConfig.train_use_8bit_adam",description:`<strong>train_use_8bit_adam</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Use 8bit Adam optimizer from bitsandbytes.`,name:"train_use_8bit_adam"},{anchor:"trl.DDPOConfig.train_learning_rate",description:`<strong>train_learning_rate</strong> (<code>float</code>, <em>optional</em>, defaults to <code>3e-4</code>) &#x2014;
Learning rate.`,name:"train_learning_rate"},{anchor:"trl.DDPOConfig.train_adam_beta1",description:`<strong>train_adam_beta1</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.9</code>) &#x2014;
Adam beta1.`,name:"train_adam_beta1"},{anchor:"trl.DDPOConfig.train_adam_beta2",description:`<strong>train_adam_beta2</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.999</code>) &#x2014;
Adam beta2.`,name:"train_adam_beta2"},{anchor:"trl.DDPOConfig.train_adam_weight_decay",description:`<strong>train_adam_weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1e-4</code>) &#x2014;
Adam weight decay.`,name:"train_adam_weight_decay"},{anchor:"trl.DDPOConfig.train_adam_epsilon",description:`<strong>train_adam_epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1e-8</code>) &#x2014;
Adam epsilon.`,name:"train_adam_epsilon"},{anchor:"trl.DDPOConfig.train_gradient_accumulation_steps",description:`<strong>train_gradient_accumulation_steps</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1</code>) &#x2014;
Number of gradient accumulation steps.`,name:"train_gradient_accumulation_steps"},{anchor:"trl.DDPOConfig.train_max_grad_norm",description:`<strong>train_max_grad_norm</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1.0</code>) &#x2014;
Maximum gradient norm for gradient clipping.`,name:"train_max_grad_norm"},{anchor:"trl.DDPOConfig.train_num_inner_epochs",description:`<strong>train_num_inner_epochs</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1</code>) &#x2014;
Number of inner epochs per outer epoch.`,name:"train_num_inner_epochs"},{anchor:"trl.DDPOConfig.train_cfg",description:`<strong>train_cfg</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to use classifier-free guidance during training.`,name:"train_cfg"},{anchor:"trl.DDPOConfig.train_adv_clip_max",description:`<strong>train_adv_clip_max</strong> (<code>float</code>, <em>optional</em>, defaults to <code>5.0</code>) &#x2014;
Clip advantages to the range.`,name:"train_adv_clip_max"},{anchor:"trl.DDPOConfig.train_clip_range",description:`<strong>train_clip_range</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1e-4</code>) &#x2014;
PPO clip range.`,name:"train_clip_range"},{anchor:"trl.DDPOConfig.train_timestep_fraction",description:`<strong>train_timestep_fraction</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1.0</code>) &#x2014;
Fraction of timesteps to train on.`,name:"train_timestep_fraction"},{anchor:"trl.DDPOConfig.per_prompt_stat_tracking",description:`<strong>per_prompt_stat_tracking</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to track statistics for each prompt separately.`,name:"per_prompt_stat_tracking"},{anchor:"trl.DDPOConfig.per_prompt_stat_tracking_buffer_size",description:`<strong>per_prompt_stat_tracking_buffer_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>16</code>) &#x2014;
Number of reward values to store in the buffer for each prompt.`,name:"per_prompt_stat_tracking_buffer_size"},{anchor:"trl.DDPOConfig.per_prompt_stat_tracking_min_count",description:`<strong>per_prompt_stat_tracking_min_count</strong> (<code>int</code>, <em>optional</em>, defaults to <code>16</code>) &#x2014;
Minimum number of reward values to store in the buffer.`,name:"per_prompt_stat_tracking_min_count"},{anchor:"trl.DDPOConfig.async_reward_computation",description:`<strong>async_reward_computation</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to compute rewards asynchronously.`,name:"async_reward_computation"},{anchor:"trl.DDPOConfig.max_workers",description:`<strong>max_workers</strong> (<code>int</code>, <em>optional</em>, defaults to <code>2</code>) &#x2014;
Maximum number of workers to use for async reward computation.`,name:"max_workers"},{anchor:"trl.DDPOConfig.negative_prompts",description:`<strong>negative_prompts</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&quot;</code>) &#x2014;
Comma-separated list of prompts to use as negative examples.`,name:"negative_prompts"},{anchor:"trl.DDPOConfig.push_to_hub",description:`<strong>push_to_hub</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to push the final model checkpoint to the Hub.`,name:"push_to_hub"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/ddpo_config.py#L25"}}),he=new hn({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/ddpo_trainer.md"}}),{c(){v=r("meta"),ke=o(),Ce=r("p"),je=o(),d(k.$$.fragment),Je=o(),j=r("p"),j.innerHTML=Ut,Pe=o(),d(J.$$.fragment),Ue=o(),P=r("table"),P.innerHTML=Ot,Oe=o(),d(U.$$.fragment),Ae=o(),O=r("p"),O.innerHTML=At,Ie=o(),A=r("p"),A.innerHTML=It,He=o(),I=r("p"),I.textContent=Ht,Le=o(),H=r("p"),H.innerHTML=Lt,Ne=o(),L=r("p"),L.textContent=Nt,Ze=o(),N=r("p"),N.textContent=Zt,Be=o(),d(Z.$$.fragment),Ge=o(),B=r("p"),B.innerHTML=Bt,ze=o(),G=r("p"),G.innerHTML=Gt,qe=o(),z=r("p"),z.innerHTML=zt,Se=o(),d(q.$$.fragment),Ee=o(),S=r("p"),S.innerHTML=qt,Re=o(),E=r("p"),E.textContent=St,Xe=o(),R=r("ul"),R.innerHTML=Et,We=o(),d(X.$$.fragment),Ye=o(),W=r("p"),W.textContent=Rt,Qe=o(),d(Y.$$.fragment),Ve=o(),Q=r("p"),Q.innerHTML=Xt,Fe=o(),d(V.$$.fragment),Ke=o(),F=r("ul"),F.innerHTML=Wt,et=o(),K=r("p"),K.innerHTML=Yt,tt=o(),d(ee.$$.fragment),nt=o(),d(te.$$.fragment),at=o(),ne=r("p"),ne.textContent=Qt,ot=o(),d(ae.$$.fragment),it=o(),d(oe.$$.fragment),rt=o(),ie=r("p"),ie.innerHTML=Vt,st=o(),d(re.$$.fragment),lt=o(),u=r("div"),d(se.$$.fragment),bt=o(),fe=r("p"),fe.innerHTML=Ft,wt=o(),T=r("div"),d(le.$$.fragment),yt=o(),ue=r("p"),ue.textContent=Kt,vt=o(),M=r("div"),d(ce.$$.fragment),Tt=o(),_e=r("p"),_e.innerHTML=en,Mt=o(),_=r("div"),d(de.$$.fragment),Dt=o(),be=r("p"),be.textContent=tn,xt=o(),we=r("p"),we.textContent=nn,Ct=o(),ye=r("ul"),ye.innerHTML=an,$t=o(),D=r("div"),d(pe.$$.fragment),kt=o(),ve=r("p"),ve.textContent=on,ct=o(),d(me.$$.fragment),dt=o(),w=r("div"),d(ge.$$.fragment),jt=o(),Te=r("p"),Te.innerHTML=rn,Jt=o(),Me=r("p"),Me.innerHTML=sn,pt=o(),d(he.$$.fragment),mt=o(),$e=r("p"),this.h()},l(e){const t=gn("svelte-u9bgzb",document.head);v=s(t,"META",{name:!0,content:!0}),t.forEach(n),ke=i(e),Ce=s(e,"P",{}),C(Ce).forEach(n),je=i(e),p(k.$$.fragment,e),Je=i(e),j=s(e,"P",{"data-svelte-h":!0}),l(j)!=="svelte-1e9ed26"&&(j.innerHTML=Ut),Pe=i(e),p(J.$$.fragment,e),Ue=i(e),P=s(e,"TABLE",{"data-svelte-h":!0}),l(P)!=="svelte-11rsbqy"&&(P.innerHTML=Ot),Oe=i(e),p(U.$$.fragment,e),Ae=i(e),O=s(e,"P",{"data-svelte-h":!0}),l(O)!=="svelte-1nki4s0"&&(O.innerHTML=At),Ie=i(e),A=s(e,"P",{"data-svelte-h":!0}),l(A)!=="svelte-xp6gwm"&&(A.innerHTML=It),He=i(e),I=s(e,"P",{"data-svelte-h":!0}),l(I)!=="svelte-a4w6o0"&&(I.textContent=Ht),Le=i(e),H=s(e,"P",{"data-svelte-h":!0}),l(H)!=="svelte-t0ve8a"&&(H.innerHTML=Lt),Ne=i(e),L=s(e,"P",{"data-svelte-h":!0}),l(L)!=="svelte-1s71aap"&&(L.textContent=Nt),Ze=i(e),N=s(e,"P",{"data-svelte-h":!0}),l(N)!=="svelte-1a6cs7n"&&(N.textContent=Zt),Be=i(e),p(Z.$$.fragment,e),Ge=i(e),B=s(e,"P",{"data-svelte-h":!0}),l(B)!=="svelte-1cfw9tr"&&(B.innerHTML=Bt),ze=i(e),G=s(e,"P",{"data-svelte-h":!0}),l(G)!=="svelte-xqnbvi"&&(G.innerHTML=Gt),qe=i(e),z=s(e,"P",{"data-svelte-h":!0}),l(z)!=="svelte-1iay88v"&&(z.innerHTML=zt),Se=i(e),p(q.$$.fragment,e),Ee=i(e),S=s(e,"P",{"data-svelte-h":!0}),l(S)!=="svelte-mmabua"&&(S.innerHTML=qt),Re=i(e),E=s(e,"P",{"data-svelte-h":!0}),l(E)!=="svelte-gn7d4"&&(E.textContent=St),Xe=i(e),R=s(e,"UL",{"data-svelte-h":!0}),l(R)!=="svelte-1wxt0fz"&&(R.innerHTML=Et),We=i(e),p(X.$$.fragment,e),Ye=i(e),W=s(e,"P",{"data-svelte-h":!0}),l(W)!=="svelte-171n2tq"&&(W.textContent=Rt),Qe=i(e),p(Y.$$.fragment,e),Ve=i(e),Q=s(e,"P",{"data-svelte-h":!0}),l(Q)!=="svelte-qc0duo"&&(Q.innerHTML=Xt),Fe=i(e),p(V.$$.fragment,e),Ke=i(e),F=s(e,"UL",{"data-svelte-h":!0}),l(F)!=="svelte-hf6txl"&&(F.innerHTML=Wt),et=i(e),K=s(e,"P",{"data-svelte-h":!0}),l(K)!=="svelte-1r6a60u"&&(K.innerHTML=Yt),tt=i(e),p(ee.$$.fragment,e),nt=i(e),p(te.$$.fragment,e),at=i(e),ne=s(e,"P",{"data-svelte-h":!0}),l(ne)!=="svelte-eqxomu"&&(ne.textContent=Qt),ot=i(e),p(ae.$$.fragment,e),it=i(e),p(oe.$$.fragment,e),rt=i(e),ie=s(e,"P",{"data-svelte-h":!0}),l(ie)!=="svelte-16vahxi"&&(ie.innerHTML=Vt),st=i(e),p(re.$$.fragment,e),lt=i(e),u=s(e,"DIV",{class:!0});var b=C(u);p(se.$$.fragment,b),bt=i(b),fe=s(b,"P",{"data-svelte-h":!0}),l(fe)!=="svelte-8q861t"&&(fe.innerHTML=Ft),wt=i(b),T=s(b,"DIV",{class:!0});var ht=C(T);p(le.$$.fragment,ht),yt=i(ht),ue=s(ht,"P",{"data-svelte-h":!0}),l(ue)!=="svelte-1ag17u3"&&(ue.textContent=Kt),ht.forEach(n),vt=i(b),M=s(b,"DIV",{class:!0});var ft=C(M);p(ce.$$.fragment,ft),Tt=i(ft),_e=s(ft,"P",{"data-svelte-h":!0}),l(_e)!=="svelte-1mh859w"&&(_e.innerHTML=en),ft.forEach(n),Mt=i(b),_=s(b,"DIV",{class:!0});var x=C(_);p(de.$$.fragment,x),Dt=i(x),be=s(x,"P",{"data-svelte-h":!0}),l(be)!=="svelte-r6y15n"&&(be.textContent=tn),xt=i(x),we=s(x,"P",{"data-svelte-h":!0}),l(we)!=="svelte-imqqfp"&&(we.textContent=nn),Ct=i(x),ye=s(x,"UL",{"data-svelte-h":!0}),l(ye)!=="svelte-105j238"&&(ye.innerHTML=an),x.forEach(n),$t=i(b),D=s(b,"DIV",{class:!0});var ut=C(D);p(pe.$$.fragment,ut),kt=i(ut),ve=s(ut,"P",{"data-svelte-h":!0}),l(ve)!=="svelte-1an944l"&&(ve.textContent=on),ut.forEach(n),b.forEach(n),ct=i(e),p(me.$$.fragment,e),dt=i(e),w=s(e,"DIV",{class:!0});var De=C(w);p(ge.$$.fragment,De),jt=i(De),Te=s(De,"P",{"data-svelte-h":!0}),l(Te)!=="svelte-59d54v"&&(Te.innerHTML=rn),Jt=i(De),Me=s(De,"P",{"data-svelte-h":!0}),l(Me)!=="svelte-ekuf1t"&&(Me.innerHTML=sn),De.forEach(n),pt=i(e),p(he.$$.fragment,e),mt=i(e),$e=s(e,"P",{}),C($e).forEach(n),this.h()},h(){$(v,"name","hf:doc:metadata"),$(v,"content",un),$(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),$(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),$(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),$(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),$(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),$(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){c(document.head,v),a(e,ke,t),a(e,Ce,t),a(e,je,t),m(k,e,t),a(e,Je,t),a(e,j,t),a(e,Pe,t),m(J,e,t),a(e,Ue,t),a(e,P,t),a(e,Oe,t),m(U,e,t),a(e,Ae,t),a(e,O,t),a(e,Ie,t),a(e,A,t),a(e,He,t),a(e,I,t),a(e,Le,t),a(e,H,t),a(e,Ne,t),a(e,L,t),a(e,Ze,t),a(e,N,t),a(e,Be,t),m(Z,e,t),a(e,Ge,t),a(e,B,t),a(e,ze,t),a(e,G,t),a(e,qe,t),a(e,z,t),a(e,Se,t),m(q,e,t),a(e,Ee,t),a(e,S,t),a(e,Re,t),a(e,E,t),a(e,Xe,t),a(e,R,t),a(e,We,t),m(X,e,t),a(e,Ye,t),a(e,W,t),a(e,Qe,t),m(Y,e,t),a(e,Ve,t),a(e,Q,t),a(e,Fe,t),m(V,e,t),a(e,Ke,t),a(e,F,t),a(e,et,t),a(e,K,t),a(e,tt,t),m(ee,e,t),a(e,nt,t),m(te,e,t),a(e,at,t),a(e,ne,t),a(e,ot,t),m(ae,e,t),a(e,it,t),m(oe,e,t),a(e,rt,t),a(e,ie,t),a(e,st,t),m(re,e,t),a(e,lt,t),a(e,u,t),m(se,u,null),c(u,bt),c(u,fe),c(u,wt),c(u,T),m(le,T,null),c(T,yt),c(T,ue),c(u,vt),c(u,M),m(ce,M,null),c(M,Tt),c(M,_e),c(u,Mt),c(u,_),m(de,_,null),c(_,Dt),c(_,be),c(_,xt),c(_,we),c(_,Ct),c(_,ye),c(u,$t),c(u,D),m(pe,D,null),c(D,kt),c(D,ve),a(e,ct,t),m(me,e,t),a(e,dt,t),a(e,w,t),m(ge,w,null),c(w,jt),c(w,Te),c(w,Jt),c(w,Me),a(e,pt,t),m(he,e,t),a(e,mt,t),a(e,$e,t),gt=!0},p:cn,i(e){gt||(g(k.$$.fragment,e),g(J.$$.fragment,e),g(U.$$.fragment,e),g(Z.$$.fragment,e),g(q.$$.fragment,e),g(X.$$.fragment,e),g(Y.$$.fragment,e),g(V.$$.fragment,e),g(ee.$$.fragment,e),g(te.$$.fragment,e),g(ae.$$.fragment,e),g(oe.$$.fragment,e),g(re.$$.fragment,e),g(se.$$.fragment,e),g(le.$$.fragment,e),g(ce.$$.fragment,e),g(de.$$.fragment,e),g(pe.$$.fragment,e),g(me.$$.fragment,e),g(ge.$$.fragment,e),g(he.$$.fragment,e),gt=!0)},o(e){h(k.$$.fragment,e),h(J.$$.fragment,e),h(U.$$.fragment,e),h(Z.$$.fragment,e),h(q.$$.fragment,e),h(X.$$.fragment,e),h(Y.$$.fragment,e),h(V.$$.fragment,e),h(ee.$$.fragment,e),h(te.$$.fragment,e),h(ae.$$.fragment,e),h(oe.$$.fragment,e),h(re.$$.fragment,e),h(se.$$.fragment,e),h(le.$$.fragment,e),h(ce.$$.fragment,e),h(de.$$.fragment,e),h(pe.$$.fragment,e),h(me.$$.fragment,e),h(ge.$$.fragment,e),h(he.$$.fragment,e),gt=!1},d(e){e&&(n(ke),n(Ce),n(je),n(Je),n(j),n(Pe),n(Ue),n(P),n(Oe),n(Ae),n(O),n(Ie),n(A),n(He),n(I),n(Le),n(H),n(Ne),n(L),n(Ze),n(N),n(Be),n(Ge),n(B),n(ze),n(G),n(qe),n(z),n(Se),n(Ee),n(S),n(Re),n(E),n(Xe),n(R),n(We),n(Ye),n(W),n(Qe),n(Ve),n(Q),n(Fe),n(Ke),n(F),n(et),n(K),n(tt),n(nt),n(at),n(ne),n(ot),n(it),n(rt),n(ie),n(st),n(lt),n(u),n(ct),n(dt),n(w),n(pt),n(mt),n($e)),n(v),f(k,e),f(J,e),f(U,e),f(Z,e),f(q,e),f(X,e),f(Y,e),f(V,e),f(ee,e),f(te,e),f(ae,e),f(oe,e),f(re,e),f(se),f(le),f(ce),f(de),f(pe),f(me,e),f(ge),f(he,e)}}}const un='{"title":"Denoising Diffusion Policy Optimization","local":"denoising-diffusion-policy-optimization","sections":[{"title":"The why","local":"the-why","sections":[],"depth":2},{"title":"Getting started with Stable Diffusion finetuning with reinforcement learning","local":"getting-started-with-stable-diffusion-finetuning-with-reinforcement-learning","sections":[],"depth":2},{"title":"Getting started with examples/scripts/ddpo.py","local":"getting-started-with-examplesscriptsddpopy","sections":[],"depth":2},{"title":"Setting up the image logging hook function","local":"setting-up-the-image-logging-hook-function","sections":[{"title":"Key terms","local":"key-terms","sections":[],"depth":3},{"title":"Using the finetuned model","local":"using-the-finetuned-model","sections":[],"depth":3}],"depth":2},{"title":"Credits","local":"credits","sections":[],"depth":2},{"title":"DDPOTrainer","local":"trl.DDPOTrainer","sections":[],"depth":2},{"title":"DDPOConfig","local":"trl.DDPOConfig","sections":[],"depth":2}],"depth":1}';function _n(Pt){return dn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Mn extends pn{constructor(v){super(),mn(this,v,_n,fn,ln,{})}}export{Mn as component};
