import{s as qr,f as Rr,o as Zr,n as Lr}from"../chunks/scheduler.d627b047.js";import{S as Wr,i as Er,g as l,s as n,r as p,A as zr,h as s,f as o,c as r,j as T,u as c,x as d,k as v,y as i,a,v as m,d as h,t as g,w as u}from"../chunks/index.73c51727.js";import{D as $}from"../chunks/Docstring.0ad99a5d.js";import{C as G}from"../chunks/CodeBlock.5f78c87f.js";import{E as Nr}from"../chunks/ExampleCodeBlock.e6081cec.js";import{H as M,E as Hr}from"../chunks/index.8df1db42.js";function Br(Pt){let y,O="Example:",C,w,x;return w=new G({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVDJUb2tlbml6ZXIlMEF0b2tlbml6ZXIlMjAlM0QlMjBHUFQyVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJncHQyJTIyKSUwQWZlYXR1cmVzJTIwJTNEJTIwJTdCJTIycHJvbXB0JTIyJTNBJTIwJTIyVGhlJTIwc2t5JTIwaXMlMjIlMkMlMjAlMjJjaG9zZW4lMjIlM0ElMjAlMjIlMjBibHVlJTIyJTJDJTIwJTIycmVqZWN0ZWQlMjIlM0ElMjAlMjIlMjBncmVlbiUyMiU3RCUwQURQT1RyYWluZXIudG9rZW5pemVfcm93KCUwQSUyMCUyMCUyMCUyMGZlYXR1cmVzJTJDJTIwdG9rZW5pemVyJTJDJTIwbWF4X3Byb21wdF9sZW5ndGglM0QzJTJDJTIwbWF4X2NvbXBsZXRpb25fbGVuZ3RoJTNEMyUyQyUyMGFkZF9zcGVjaWFsX3Rva2VucyUzREZhbHNlJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPT2Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPT2Tokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>features = {<span class="hljs-string">&quot;prompt&quot;</span>: <span class="hljs-string">&quot;The sky is&quot;</span>, <span class="hljs-string">&quot;chosen&quot;</span>: <span class="hljs-string">&quot; blue&quot;</span>, <span class="hljs-string">&quot;rejected&quot;</span>: <span class="hljs-string">&quot; green&quot;</span>}
<span class="hljs-meta">&gt;&gt;&gt; </span>DPOTrainer.tokenize_row(
<span class="hljs-meta">... </span>    features, tokenizer, max_prompt_length=<span class="hljs-number">3</span>, max_completion_length=<span class="hljs-number">3</span>, add_special_tokens=<span class="hljs-literal">False</span>
<span class="hljs-meta">... </span>)
{<span class="hljs-string">&#x27;prompt_input_ids&#x27;</span>: [<span class="hljs-number">464</span>, <span class="hljs-number">6766</span>, <span class="hljs-number">318</span>], <span class="hljs-string">&#x27;chosen_input_ids&#x27;</span>: [<span class="hljs-number">4171</span>, <span class="hljs-number">50256</span>], <span class="hljs-string">&#x27;rejected_input_ids&#x27;</span>: [<span class="hljs-number">4077</span>, <span class="hljs-number">50256</span>]}`,wrap:!1}}),{c(){y=l("p"),y.textContent=O,C=n(),p(w.$$.fragment)},l(b){y=s(b,"P",{"data-svelte-h":!0}),d(y)!=="svelte-11lpom8"&&(y.textContent=O),C=r(b),c(w.$$.fragment,b)},m(b,j){a(b,y,j),a(b,C,j),m(w,b,j),x=!0},p:Lr,i(b){x||(h(w.$$.fragment,b),x=!0)},o(b){g(w.$$.fragment,b),x=!1},d(b){b&&(o(y),o(C)),u(w,b)}}}function Qr(Pt){let y,O="Examples:",C,w,x;return w=new G({props:{code:"ZnJvbSUyMHRybCUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclByZWZlcmVuY2UlMEFjb2xsYXRvciUyMCUzRCUyMERhdGFDb2xsYXRvckZvclByZWZlcmVuY2UocGFkX3Rva2VuX2lkJTNEMCklMEFleGFtcGxlcyUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCU3QiUyMnByb21wdF9pbnB1dF9pZHMlMjIlM0ElMjAlNUIxJTJDJTIwMiUyQyUyMDMlNUQlMkMlMjAlMjJjaG9zZW5faW5wdXRfaWRzJTIyJTNBJTIwJTVCNCUyQyUyMDUlNUQlMkMlMjAlMjJyZWplY3RlZF9pbnB1dF9pZHMlMjIlM0ElMjAlNUI2JTVEJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdCJTIycHJvbXB0X2lucHV0X2lkcyUyMiUzQSUyMCU1QjclMkMlMjA4JTVEJTJDJTIwJTIyY2hvc2VuX2lucHV0X2lkcyUyMiUzQSUyMCU1QjklMkMlMjAxMCU1RCUyQyUyMCUyMnJlamVjdGVkX2lucHV0X2lkcyUyMiUzQSUyMCU1QjExJTJDJTIwMTIlMkMlMjAxMyU1RCU3RCUwQSU1RCUwQWNvbGxhdG9yKGV4YW1wbGVzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> DataCollatorForPreference
<span class="hljs-meta">&gt;&gt;&gt; </span>collator = DataCollatorForPreference(pad_token_id=<span class="hljs-number">0</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>examples = [
<span class="hljs-meta">... </span>    {<span class="hljs-string">&quot;prompt_input_ids&quot;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>], <span class="hljs-string">&quot;chosen_input_ids&quot;</span>: [<span class="hljs-number">4</span>, <span class="hljs-number">5</span>], <span class="hljs-string">&quot;rejected_input_ids&quot;</span>: [<span class="hljs-number">6</span>]},
<span class="hljs-meta">... </span>    {<span class="hljs-string">&quot;prompt_input_ids&quot;</span>: [<span class="hljs-number">7</span>, <span class="hljs-number">8</span>], <span class="hljs-string">&quot;chosen_input_ids&quot;</span>: [<span class="hljs-number">9</span>, <span class="hljs-number">10</span>], <span class="hljs-string">&quot;rejected_input_ids&quot;</span>: [<span class="hljs-number">11</span>, <span class="hljs-number">12</span>, <span class="hljs-number">13</span>]}
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>collator(examples)
{<span class="hljs-string">&#x27;prompt_input_ids&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>],
                             [<span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]),
 <span class="hljs-string">&#x27;prompt_attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                                  [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]),
 <span class="hljs-string">&#x27;chosen_input_ids&#x27;</span>: tensor([[ <span class="hljs-number">4</span>,  <span class="hljs-number">5</span>],
                             [ <span class="hljs-number">9</span>, <span class="hljs-number">10</span>]]),
 <span class="hljs-string">&#x27;chosen_attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                                  [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]),
 <span class="hljs-string">&#x27;rejected_input_ids&#x27;</span>: tensor([[ <span class="hljs-number">6</span>,  <span class="hljs-number">0</span>,  <span class="hljs-number">0</span>],
                               [<span class="hljs-number">11</span>, <span class="hljs-number">12</span>, <span class="hljs-number">13</span>]]),
 <span class="hljs-string">&#x27;rejected_attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]])
}`,wrap:!1}}),{c(){y=l("p"),y.textContent=O,C=n(),p(w.$$.fragment)},l(b){y=s(b,"P",{"data-svelte-h":!0}),d(y)!=="svelte-kvfsh7"&&(y.textContent=O),C=r(b),c(w.$$.fragment,b)},m(b,j){a(b,y,j),a(b,C,j),m(w,b,j),x=!0},p:Lr,i(b){x||(h(w.$$.fragment,b),x=!0)},o(b){g(w.$$.fragment,b),x=!1},d(b){b&&(o(y),o(C)),u(w,b)}}}function Vr(Pt){let y,O,C,w,x,b,j,Tn='<a href="https://huggingface.co/models?other=dpo,trl" rel="nofollow"><img src="https://img.shields.io/badge/All_models-DPO-blue"/></a> <a href="https://github.com/huggingface/smol-course/tree/main/2_preference_alignment" rel="nofollow"><img src="https://img.shields.io/badge/smol_course-Chapter_2-yellow"/></a>',so,A,io,X,Mn='TRL supports the DPO Trainer for training language models from preference data, as described in the paper <a href="https://huggingface.co/papers/2305.18290" rel="nofollow">Direct Preference Optimization: Your Language Model is Secretly a Reward Model</a> by <a href="https://huggingface.co/rmrafailov" rel="nofollow">Rafael Rafailov</a>, Archit Sharma, Eric Mitchell, <a href="https://huggingface.co/ermonste" rel="nofollow">Stefano Ermon</a>, <a href="https://huggingface.co/manning" rel="nofollow">Christopher D. Manning</a>, <a href="https://huggingface.co/cbfinn" rel="nofollow">Chelsea Finn</a>.',po,S,xn="The abstract from the paper is the following:",co,Y,$n="<p>While large-scale unsupervised language models (LMs) learn broad world knowledge and some reasoning skills, achieving precise control of their behavior is difficult due to the completely unsupervised nature of their training. Existing methods for gaining such steerability collect human labels of the relative quality of model generations and fine-tune the unsupervised LM to align with these preferences, often with reinforcement learning from human feedback (RLHF). However, RLHF is a complex and often unstable procedure, first fitting a reward model that reflects the human preferences, and then fine-tuning the large unsupervised LM using reinforcement learning to maximize this estimated reward without drifting too far from the original model. In this paper we introduce a new parameterization of the reward model in RLHF that enables extraction of the corresponding optimal policy in closed form, allowing us to solve the standard RLHF problem with only a simple classification loss. The resulting algorithm, which we call Direct Preference Optimization (DPO), is stable, performant, and computationally lightweight, eliminating the need for sampling from the LM during fine-tuning or performing significant hyperparameter tuning. Our experiments show that DPO can fine-tune LMs to align with human preferences as well as or better than existing methods. Notably, fine-tuning with DPO exceeds PPO-based RLHF in ability to control sentiment of generations, and matches or improves response quality in summarization and single-turn dialogue while being substantially simpler to implement and train.</p>",mo,K,Cn="The first step is to train an SFT model, to ensure the data we train on is in-distribution for the DPO algorithm.",ho,ee,jn='Then, fine-tuning a language model via DPO consists of two steps and is easier than <a href="ppo_trainer">PPO</a>:',go,te,Pn='<li><strong>Data collection</strong>: Gather a <a href="dataset_formats#preference">preference dataset</a> with positive and negative selected pairs of generation, given a prompt.</li> <li><strong>Optimization</strong>: Maximize the log-likelihood of the DPO loss directly.</li>',uo,oe,Un='This process is illustrated in the sketch below (from <a href="https://huggingface.co/papers/2305.18290" rel="nofollow">Figure 1 of the DPO paper</a>):',fo,ae,On='<img src="https://github.com/huggingface/trl/assets/49240599/9150fac6-3d88-4ca2-8ec6-2a6f3473216d"/>',_o,ne,Jn='Read more about DPO algorithm in the <a href="https://huggingface.co/papers/2305.18290" rel="nofollow">original paper</a>.',bo,re,yo,le,kn='This example demonstrates how to train a model using the DPO method. We use the <a href="https://huggingface.co/Qwen/Qwen2-0.5B-Instruct" rel="nofollow">Qwen 0.5B model</a> as the base model. We use the preference data from the <a href="https://huggingface.co/datasets/openbmb/UltraFeedback" rel="nofollow">UltraFeedback dataset</a>. You can view the data in the dataset here:',vo,J,Dn,wo,se,Fn="Below is the script to train the model:",To,ie,Mo,de,In="Execute the script using the following command:",xo,pe,$o,ce,Nn="Distributed across 8 GPUs, the training takes approximately 3 minutes. You can verify the training progress by checking the reward graph. An increasing trend in the reward margin indicates that the model is improving and generating better responses over time.",Co,me,Ln='<img src="https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/dpo-qwen2-reward-margin.png"/>',jo,he,qn='To see how the <a href="https://huggingface.co/trl-lib/Qwen2-0.5B-DPO" rel="nofollow">trained model</a> performs, you can use the <a href="https://huggingface.co/docs/transformers/quicktour#chat-with-text-generation-models" rel="nofollow">Transformers Chat CLI</a>.',Po,ge,Rn=`<code>$ transformers-cli chat --model_name_or_path trl-lib/Qwen2-0.5B-DPO
<strong><span style="color: red;">&lt;quentin_gallouedec&gt;:</span></strong>
What is the best programming language?

<strong><span style="color: blue;">&lt;trl-lib/Qwen2-0.5B-DPO&gt;:</span></strong>
The best programming language for specific applications can vary depending on the use case and knowledge level of the programmer. Here are some general factors that can be used as input to choose the best programming language:

 <strong><span style="color: green;">1</span></strong> Ease of use: Some programming languages are more user-friendly than others, such as Python, Java, or Ruby. Python is popular due to its simplicity and great scalability.
 <strong><span style="color: green;">2</span></strong> Versatility: The ability to work with a wide range of data structures and frameworks can define the language as versatile.
 <strong><span style="color: green;">3</span></strong> Ease of learning: Different programming languages have different learning curves, so users must be willing to take some time to master one.
 <strong><span style="color: green;">4</span></strong> Community support: The broader community of developers and enthusiasts in the selected programming language can provide great support and resources.
 <strong><span style="color: green;">5</span></strong> Reusability: Languages that emphasize code reuse and can be easily modifiable can be more suitable for software development.

The best programming language based on these factors is subjective and depends on what the programmer intends to accomplish.
</code>`,Uo,ue,Oo,fe,Zn='DPO requires a <a href="dataset_formats#preference">preference dataset</a>. The <a href="/docs/trl/main/en/dpo_trainer#trl.DPOTrainer">DPOTrainer</a> supports both <a href="dataset_formats#conversational">conversational</a> and <a href="dataset_formats#standard">standard</a> dataset formats. When provided with a conversational dataset, the trainer will automatically apply the chat template to the dataset.',Jo,_e,Wn='Although the <a href="/docs/trl/main/en/dpo_trainer#trl.DPOTrainer">DPOTrainer</a> supports both explicit and implicit prompts, we recommend using explicit prompts. If provided with an implicit prompt dataset, the trainer will automatically extract the prompt from the <code>&quot;chosen&quot;</code> and <code>&quot;rejected&quot;</code> columns. For more information, refer to the <a href="dataset_formats#preference">preference style</a> section.',ko,be,Do,ye,En='The <a href="/docs/trl/main/en/dpo_trainer#trl.DPOTrainer">DPOTrainer</a> supports fine-tuning vision-language models (VLMs). For these models, a vision dataset is required. To learn more about the specific format for vision datasets, refer to the <a href="dataset_formats#vision-datasets">Vision dataset format</a> section.',Fo,ve,zn="Additionally, unlike standard text-based models where a <code>tokenizer</code> is used, for VLMs, you should replace the <code>tokenizer</code> with a <code>processor</code>.",Io,we,No,Te,Hn='For a complete example of fine-tuning a vision-language model, refer to the script in <a href="https://github.com/huggingface/trl/blob/main/examples/scripts/dpo_vlm.py" rel="nofollow"><code>examples/scripts/dpo_vlm.py</code></a>.',Lo,Me,qo,xe,Bn='We provide an example script to train a model using the DPO method. The script is available in <a href="https://github.com/huggingface/trl/blob/main/trl/scripts/dpo.py" rel="nofollow"><code>trl/scripts/dpo.py</code></a>',Ro,$e,Qn='To test the DPO script with the <a href="https://huggingface.co/Qwen/Qwen2-0.5B-Instruct" rel="nofollow">Qwen2 0.5B model</a> on the <a href="https://huggingface.co/datasets/trl-lib/ultrafeedback_binarized" rel="nofollow">UltraFeedback dataset</a>, run the following command:',Zo,Ce,Wo,je,Eo,Pe,Vn="While training and evaluating we record the following reward metrics:",zo,Ue,Gn="<li><code>rewards/chosen</code>: the mean difference between the log probabilities of the policy model and the reference model for the chosen responses scaled by beta</li> <li><code>rewards/rejected</code>: the mean difference between the log probabilities of the policy model and the reference model for the rejected responses scaled by beta</li> <li><code>rewards/accuracies</code>: mean of how often the chosen rewards are &gt; than the corresponding rejected rewards</li> <li><code>rewards/margins</code>: the mean difference between the chosen and corresponding rejected rewards</li>",Ho,Oe,Bo,Je,An='The DPO algorithm supports several loss functions. The loss function can be set using the <code>loss_type</code> parameter in the <a href="/docs/trl/main/en/dpo_trainer#trl.DPOConfig">DPOConfig</a>. The following loss functions are supported:',Qo,ke,Xn='<thead><tr><th><code>loss_type=</code></th> <th>Description</th></tr></thead> <tbody><tr><td><code>&quot;sigmoid&quot;</code> (default)</td> <td>Given the preference data, we can fit a binary classifier according to the Bradley-Terry model and in fact the <a href="https://huggingface.co/papers/2305.18290" rel="nofollow">DPO</a> authors propose the sigmoid loss on the normalized likelihood via the <code>logsigmoid</code> to fit a logistic regression.</td></tr> <tr><td><code>&quot;hinge&quot;</code></td> <td>The <a href="https://huggingface.co/papers/2309.06657" rel="nofollow">RSO</a> authors propose to use a hinge loss on the normalized likelihood from the <a href="https://huggingface.co/papers/2305.10425" rel="nofollow">SLiC</a> paper. In this case, the <code>beta</code> is the reciprocal of the margin.</td></tr> <tr><td><code>&quot;ipo&quot;</code></td> <td>The <a href="https://huggingface.co/papers/2310.12036" rel="nofollow">IPO</a> authors provide a deeper theoretical understanding of the DPO algorithms and identify an issue with overfitting and propose an alternative loss. In this case, the <code>beta</code> is the reciprocal of the gap between the log-likelihood ratios of the chosen vs the rejected completion pair and thus the smaller the <code>beta</code> the larger this gaps is. As per the paper the loss is averaged over log-likelihoods of the completion (unlike DPO which is summed only).</td></tr> <tr><td><code>&quot;exo_pair&quot;</code></td> <td>The <a href="https://huggingface.co/papers/2402.00856" rel="nofollow">EXO</a> authors propose to minimize the reverse KL instead of the negative log-sigmoid loss of DPO which corresponds to forward KL. Setting non-zero <code>label_smoothing</code> (default <code>1e-3</code>) leads to a simplified version of EXO on pair-wise preferences (see Eqn. (16) of the <a href="https://huggingface.co/papers/2402.00856" rel="nofollow">EXO paper</a>). The full version of EXO uses <code>K&gt;2</code> completions generated by the SFT policy, which becomes an unbiased estimator of the PPO objective (up to a constant) when <code>K</code> is sufficiently large.</td></tr> <tr><td><code>&quot;nca_pair&quot;</code></td> <td>The <a href="https://huggingface.co/papers/2402.05369" rel="nofollow">NCA</a> authors shows that NCA optimizes the absolute likelihood for each response rather than the relative likelihood.</td></tr> <tr><td><code>&quot;robust&quot;</code></td> <td>The <a href="https://huggingface.co/papers/2403.00409" rel="nofollow">Robust DPO</a> authors propose an unbiased estimate of the DPO loss that is robust to preference noise in the data. Like in cDPO, it assumes that the preference labels are noisy with some probability. In this approach, the <code>label_smoothing</code> parameter in the <a href="/docs/trl/main/en/dpo_trainer#trl.DPOConfig">DPOConfig</a> is used to model the probability of existing label noise. To apply this conservative loss, set <code>label_smoothing</code> to a value greater than 0.0 (between 0.0 and 0.5; the default is 0.0)</td></tr> <tr><td><code>&quot;bco_pair&quot;</code></td> <td>The <a href="https://huggingface.co/papers/2404.04656" rel="nofollow">BCO</a> authors train a binary classifier whose logit serves as a reward so that the classifier maps {prompt, chosen completion} pairs to 1 and {prompt, rejected completion} pairs to 0. For unpaired data, we recommend the dedicated <a href="/docs/trl/main/en/bco_trainer#trl.BCOTrainer">BCOTrainer</a>.</td></tr> <tr><td><code>&quot;sppo_hard&quot;</code></td> <td>The <a href="https://huggingface.co/papers/2405.00675" rel="nofollow">SPPO</a> authors claim that SPPO is capable of solving the Nash equilibrium iteratively by pushing the chosen rewards to be as large as 1/2 and the rejected rewards to be as small as -1/2 and can alleviate data sparsity issues. The implementation approximates this algorithm by employing hard label probabilities, assigning 1 to the winner and 0 to the loser.</td></tr> <tr><td><code>&quot;aot&quot;</code>  or <code>loss_type=&quot;aot_pair&quot;</code></td> <td>The <a href="https://huggingface.co/papers/2406.05882" rel="nofollow">AOT</a> authors propose to use Distributional Preference Alignment Via Optimal Transport. Traditionally, the alignment algorithms use paired preferences at a sample level, which does not ensure alignment on the distributional level. AOT, on the other hand, can align LLMs on paired or unpaired preference data by making the reward distribution of the positive samples stochastically dominant in the first order on the distribution of negative samples. Specifically, <code>loss_type=&quot;aot&quot;</code> is appropriate for paired datasets, where each prompt has both chosen and rejected responses; <code>loss_type=&quot;aot_pair&quot;</code> is for unpaired datasets. In a nutshell, <code>loss_type=&quot;aot&quot;</code> ensures that the log-likelihood ratio of chosen to rejected of the aligned model has higher quantiles than that ratio for the reference model. <code>loss_type=&quot;aot_pair&quot;</code> ensures that the chosen reward is higher on all quantiles than the rejected reward. Note that in both cases quantiles are obtained via sorting. To fully leverage the advantages of the AOT algorithm, it is important to maximize the per-GPU batch size.</td></tr> <tr><td><code>&quot;apo_zero&quot;</code> or <code>loss_type=&quot;apo_down&quot;</code></td> <td>The <a href="https://huggingface.co/papers/2408.06266" rel="nofollow">APO</a> method introduces an “anchored” version of the alignment objective. There are two variants: <code>apo_zero</code> and <code>apo_down</code>. The <code>apo_zero</code> loss increases the likelihood of winning outputs while decreasing the likelihood of losing outputs, making it suitable when the model is less performant than the winning outputs. On the other hand, <code>apo_down</code> decreases the likelihood of both winning and losing outputs, but with a stronger emphasis on reducing the likelihood of losing outputs. This variant is more effective when the model is better than the winning outputs.</td></tr> <tr><td><code>&quot;discopop&quot;</code></td> <td>The <a href="https://huggingface.co/papers/2406.08414" rel="nofollow">DiscoPOP</a> paper uses LLMs to discover more efficient offline preference optimization losses. In the paper the proposed DiscoPOP loss (which is a log-ratio modulated loss) outperformed other optimization losses on different tasks (IMDb positive text generation, Reddit TLDR summarization, and Alpaca Eval 2.0).</td></tr></tbody>',Vo,De,Go,Fe,Sn='The <a href="https://ericmitchell.ai/cdpo.pdf" rel="nofollow">cDPO</a> is a tweak on the DPO loss where we assume that the preference labels are noisy with some probability. In this approach, the <code>label_smoothing</code> parameter in the <a href="/docs/trl/main/en/dpo_trainer#trl.DPOConfig">DPOConfig</a> is used to model the probability of existing label noise. To apply this conservative loss, set <code>label_smoothing</code> to a value greater than 0.0 (between 0.0 and 0.5; the default is 0.0).',Ao,Ie,Xo,Ne,Yn='The <a href="https://huggingface.co/papers/2404.09656" rel="nofollow">TR-DPO</a> paper suggests syncing the reference model weights after every <code>ref_model_sync_steps</code> steps of SGD with weight <code>ref_model_mixup_alpha</code> during DPO training. To toggle this callback use the <code>sync_ref_model=True</code> in the <a href="/docs/trl/main/en/dpo_trainer#trl.DPOConfig">DPOConfig</a>.',So,Le,Yo,qe,Kn='The <a href="https://huggingface.co/papers/2404.19733" rel="nofollow">RPO</a> paper implements an iterative preference tuning algorithm using a loss related to the RPO loss in this <a href="https://huggingface.co/papers/2405.16436" rel="nofollow">paper</a> that essentially consists of a weighted SFT loss on the chosen preferences together with the DPO loss. To use this loss, set the <code>rpo_alpha</code> in the <a href="/docs/trl/main/en/dpo_trainer#trl.DPOConfig">DPOConfig</a> to an appropriate value. The paper suggests setting this weight to <code>1.0</code>.',Ko,Re,ea,Ze,er='The <a href="https://huggingface.co/papers/2406.11827" rel="nofollow">WPO</a> paper adapts off-policy data to resemble on-policy data more closely by reweighting preference pairs according to their probability under the current policy. To use this method, set the <code>use_weighting</code> flag to <code>True</code> in the <a href="/docs/trl/main/en/dpo_trainer#trl.DPOConfig">DPOConfig</a>.',ta,We,oa,Ee,tr=`MOEs are the most efficient if the load is about equally distributed between experts.<br/>
To ensure that we train MOEs similarly during preference-tuning, it is beneficial to add the auxiliary loss from the load balancer to the final loss.`,aa,ze,or=`This option is enabled by setting <code>output_router_logits=True</code> in the model config (e.g. <a href="https://huggingface.co/docs/transformers/main/en/model_doc/mixtral#transformers.MixtralConfig" rel="nofollow">MixtralConfig</a>).<br/>
To scale how much the auxiliary loss contributes to the total loss, use the hyperparameter <code>router_aux_loss_coef=...</code> (default: <code>0.001</code>) in the model config.`,na,He,ra,Be,ar='You can further accelerate QLoRA / LoRA (2x faster, 60% less memory) using the <a href="https://github.com/unslothai/unsloth" rel="nofollow"><code>unsloth</code></a> library that is fully compatible with <code>SFTTrainer</code>. Currently <code>unsloth</code> supports only Llama (Yi, TinyLlama, Qwen, Deepseek etc) and Mistral architectures. Some benchmarks for DPO listed below:',la,Qe,nr="<thead><tr><th>GPU</th> <th>Model</th> <th>Dataset</th> <th>🤗</th> <th>🤗 + Flash Attention 2</th> <th>🦥 Unsloth</th> <th>🦥 VRAM saved</th></tr></thead> <tbody><tr><td>A100 40G</td> <td>Zephyr 7b</td> <td>Ultra Chat</td> <td>1x</td> <td>1.24x</td> <td><strong>1.88x</strong></td> <td>-11.6%</td></tr> <tr><td>Tesla T4</td> <td>Zephyr 7b</td> <td>Ultra Chat</td> <td>1x</td> <td>1.09x</td> <td><strong>1.55x</strong></td> <td>-18.6%</td></tr></tbody>",sa,Ve,rr='First install <code>unsloth</code> according to the <a href="https://github.com/unslothai/unsloth" rel="nofollow">official documentation</a>. Once installed, you can incorporate unsloth into your workflow in a very simple manner; instead of loading <code>AutoModelForCausalLM</code>, you just need to load a <code>FastLanguageModel</code> as follows:',ia,Ge,da,Ae,lr='The saved model is fully compatible with Hugging Face’s transformers library. Learn more about unsloth in their <a href="https://github.com/unslothai/unsloth" rel="nofollow">official repository</a>.',pa,Xe,ca,Se,sr="You have three main options (plus several variants) for how the reference model works when using PEFT, assuming the model that you would like to further enhance with DPO was tuned using (Q)LoRA.",ma,Ye,ir="<li>Simply create two instances of the model, each loading your adapter - works fine but is very inefficient.</li> <li>Merge the adapter into the base model, create another adapter on top, then leave the <code>ref_model</code> param null, in which case DPOTrainer will unload the adapter for reference inference - efficient, but has potential downsides discussed below.</li> <li>Load the adapter twice with different names, then use <code>set_adapter</code> during training to swap between the adapter being DPO’d and the reference adapter - slightly less efficient compared to 2 (~adapter size VRAM overhead), but avoids the pitfalls.</li>",ha,Ke,ga,et,dr='As suggested by <a href="https://medium.com/@bnjmn_marie/dont-merge-your-lora-adapter-into-a-4-bit-llm-65b6da287997" rel="nofollow">Benjamin Marie</a>, the best option for merging QLoRA adapters is to first dequantize the base model, then merge the adapter. Something similar to <a href="https://github.com/jondurbin/qlora/blob/main/qmerge.py" rel="nofollow">this script</a>.',ua,tt,pr="However, after using this approach, you will have an unquantized base model. Therefore, to use QLoRA for DPO, you will need to re-quantize the merged model or use the unquantized merge (resulting in higher memory demand).",fa,ot,_a,at,cr='To avoid the downsides with option 2, you can load your fine-tuned adapter into the model twice, with different names, and set the model/ref adapter names in <a href="/docs/trl/main/en/dpo_trainer#trl.DPOTrainer">DPOTrainer</a>.',ba,nt,mr="For example:",ya,rt,va,lt,wa,f,st,Na,Ut,hr="Initialize DPOTrainer.",La,q,it,qa,Ot,gr="Computes log probabilities of the reference model for a single padded batch of a DPO specific dataset.",Ra,k,dt,Za,Jt,ur="Run the given model on the given batch of inputs, concatenating the chosen and rejected inputs together.",Wa,kt,fr="We do this to avoid doing two forward passes, because it’s faster for FSDP.",Ea,D,pt,za,Dt,_r=`Concatenate the <code>chosen</code> and <code>rejected</code> inputs from the batch into a single tensor for both the prompt
and completion sequences.`,Ha,Ft,br=`Notes:
The completion input IDs and attention masks are padded to the maximum completion length of the chosen
or rejected sequences.`,Ba,R,ct,Qa,It,yr="Creates a draft of a model card using the information available to the <code>Trainer</code>.",Va,Z,mt,Ga,Nt,vr="Compute the DPO loss for a batch of policy and reference model log probabilities.",Aa,F,ht,Xa,Lt,wr=`Overriding built-in evaluation loop to store metrics for each batch.
Prediction/evaluation loop, shared by <code>Trainer.evaluate()</code> and <code>Trainer.predict()</code>.`,Sa,qt,Tr="Works both with or without labels.",Ya,W,gt,Ka,Rt,Mr="Generate samples from the model and reference model for the given batch of inputs.",en,E,ut,tn,Zt,xr="Compute the DPO loss and other metrics for the given batch of inputs for train or test.",on,I,ft,an,Wt,$r="Returns the evaluation <code>~torch.utils.data.DataLoader</code>.",nn,Et,Cr="Subclass of transformers.src.transformers.trainer.get_eval_dataloader to precompute <code>ref_log_probs</code>.",rn,N,_t,ln,zt,jr="Returns the training <code>~torch.utils.data.DataLoader</code>.",sn,Ht,Pr="Subclass of transformers.src.transformers.trainer.get_train_dataloader to precompute <code>ref_log_probs</code>.",dn,z,bt,pn,Bt,Ur="Log <code>logs</code> on the various objects watching training, including stored metrics.",cn,H,yt,mn,Qt,Or="Context manager for handling null reference model (that is, peft adapter manipulation).",hn,B,vt,gn,Vt,Jr="Same as <code>tokenize_row</code> but for vision models. Please refer to <code>tokenize_row</code> for more information.",un,L,wt,fn,Gt,kr="Tokenize a row of the dataset.",_n,Q,Ta,Tt,Ma,P,Mt,bn,At,Dr='Configuration class for the <a href="/docs/trl/main/en/dpo_trainer#trl.DPOTrainer">DPOTrainer</a>.',yn,Xt,Fr=`Using <a href="https://huggingface.co/docs/transformers/main/en/internal/trainer_utils#transformers.HfArgumentParser" rel="nofollow">HfArgumentParser</a> we can turn this class into
<a href="https://docs.python.org/3/library/argparse#module-argparse" rel="nofollow">argparse</a> arguments that can be specified on the
command line.`,xa,xt,$a,U,$t,vn,St,Ir=`Data collator used for preference data. Inputs are dynamically padded to the maximum length of a batch if they
are not all of the same length.`,wn,V,Ca,Ct,ja,lo,Pa;return x=new M({props:{title:"DPO Trainer",local:"dpo-trainer",headingTag:"h1"}}),A=new M({props:{title:"Overview",local:"overview",headingTag:"h2"}}),re=new M({props:{title:"Quick start",local:"quick-start",headingTag:"h2"}}),ie=new G({props:{code:"JTIzJTIwdHJhaW5fZHBvLnB5JTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMHRybCUyMGltcG9ydCUyMERQT0NvbmZpZyUyQyUyMERQT1RyYWluZXIlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyUXdlbiUyRlF3ZW4yLTAuNUItSW5zdHJ1Y3QlMjIpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyUXdlbiUyRlF3ZW4yLTAuNUItSW5zdHJ1Y3QlMjIpJTBBdHJhaW5fZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ0cmwtbGliJTJGdWx0cmFmZWVkYmFja19iaW5hcml6ZWQlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBEUE9Db25maWcob3V0cHV0X2RpciUzRCUyMlF3ZW4yLTAuNUItRFBPJTIyJTJDJTIwbG9nZ2luZ19zdGVwcyUzRDEwKSUwQXRyYWluZXIlMjAlM0QlMjBEUE9UcmFpbmVyKG1vZGVsJTNEbW9kZWwlMkMlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUyMHByb2Nlc3NpbmdfY2xhc3MlM0R0b2tlbml6ZXIlMkMlMjB0cmFpbl9kYXRhc2V0JTNEdHJhaW5fZGF0YXNldCklMEF0cmFpbmVyLnRyYWluKCk=",highlighted:`<span class="hljs-comment"># train_dpo.py</span>
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> DPOConfig, DPOTrainer
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;Qwen/Qwen2-0.5B-Instruct&quot;</span>)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;Qwen/Qwen2-0.5B-Instruct&quot;</span>)
train_dataset = load_dataset(<span class="hljs-string">&quot;trl-lib/ultrafeedback_binarized&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)

training_args = DPOConfig(output_dir=<span class="hljs-string">&quot;Qwen2-0.5B-DPO&quot;</span>, logging_steps=<span class="hljs-number">10</span>)
trainer = DPOTrainer(model=model, args=training_args, processing_class=tokenizer, train_dataset=train_dataset)
trainer.train()`,wrap:!1}}),pe=new G({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX2Rwby5weQ==",highlighted:"accelerate launch train_dpo.py",wrap:!1}}),ue=new M({props:{title:"Expected dataset type",local:"expected-dataset-type",headingTag:"h2"}}),be=new M({props:{title:"Special considerations for vision-language models",local:"special-considerations-for-vision-language-models",headingTag:"h3"}}),we=new G({props:{code:"LSUyMG1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUyQiUyMG1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yVmlzaW9uMlNlcS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBJTBBLSUyMHRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUyQiUyMHByb2Nlc3NvciUyMCUzRCUyMEF1dG9Qcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQSUyMCUyMHRyYWluZXIlMjAlM0QlMjBEUE9UcmFpbmVyKCUwQSUyMCUyMCUyMCUyMCUyMCUyMG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEdHJhaW5fZGF0YXNldCUyQyUwQS0lMjAlMjAlMjAlMjAlMjBwcm9jZXNzaW5nX2NsYXNzJTNEdG9rZW5pemVyJTJDJTBBJTJCJTIwJTIwJTIwJTIwJTIwcHJvY2Vzc2luZ19jbGFzcyUzRHByb2Nlc3NvciUyQyUwQSk=",highlighted:`<span class="hljs-deletion">- model = AutoModelForCausalLM.from_pretrained(model_id)</span>
<span class="hljs-addition">+ model = AutoModelForVision2Seq.from_pretrained(model_id)</span>

<span class="hljs-deletion">- tokenizer = AutoTokenizer.from_pretrained(model_id)</span>
<span class="hljs-addition">+ processor = AutoProcessor.from_pretrained(model_id)</span>

  trainer = DPOTrainer(
      model,
      args=training_args,
      train_dataset=train_dataset,
<span class="hljs-deletion">-     processing_class=tokenizer,</span>
<span class="hljs-addition">+     processing_class=processor,</span>
)`,wrap:!1}}),Me=new M({props:{title:"Example script",local:"example-script",headingTag:"h2"}}),Ce=new G({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRybCUyRnNjcmlwdHMlMkZkcG8ucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1vZGVsX25hbWVfb3JfcGF0aCUyMFF3ZW4lMkZRd2VuMi0wLjVCLUluc3RydWN0JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kYXRhc2V0X25hbWUlMjB0cmwtbGliJTJGdWx0cmFmZWVkYmFja19iaW5hcml6ZWQlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW51bV90cmFpbl9lcG9jaHMlMjAxJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sb2dnaW5nX3N0ZXBzJTIwMjUlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW91dHB1dF9kaXIlMjBRd2VuMi0wLjVCLURQTw==",highlighted:`accelerate launch trl/scripts/dpo.py \\
    --model_name_or_path Qwen/Qwen2-0.5B-Instruct \\
    --dataset_name trl-lib/ultrafeedback_binarized \\
    --num_train_epochs 1 \\
    --logging_steps 25 \\
    --output_dir Qwen2-0.5B-DPO`,wrap:!1}}),je=new M({props:{title:"Logged metrics",local:"logged-metrics",headingTag:"h2"}}),Oe=new M({props:{title:"Loss functions",local:"loss-functions",headingTag:"h2"}}),De=new M({props:{title:"Label smoothing",local:"label-smoothing",headingTag:"h3"}}),Ie=new M({props:{title:"Syncing the reference model",local:"syncing-the-reference-model",headingTag:"h3"}}),Le=new M({props:{title:"RPO loss",local:"rpo-loss",headingTag:"h3"}}),Re=new M({props:{title:"WPO loss",local:"wpo-loss",headingTag:"h3"}}),We=new M({props:{title:"For Mixture of Experts Models: Enabling the auxiliary loss",local:"for-mixture-of-experts-models-enabling-the-auxiliary-loss",headingTag:"h3"}}),He=new M({props:{title:"Accelerate DPO fine-tuning using unsloth",local:"accelerate-dpo-fine-tuning-using-unsloth",headingTag:"h2"}}),Ge=new G({props:{code:"JTIwJTIwZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTIwJTIwZnJvbSUyMHRybCUyMGltcG9ydCUyMERQT0NvbmZpZyUyQyUyMERQT1RyYWluZXIlMEEtJTIwZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUyQiUyMGZyb20lMjB1bnNsb3RoJTIwaW1wb3J0JTIwRmFzdExhbmd1YWdlTW9kZWwlMEElMEEtJTIwbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyUXdlbiUyRlF3ZW4yLTAuNUItSW5zdHJ1Y3QlMjIpJTBBLSUyMHRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMlF3ZW4lMkZRd2VuMi0wLjVCLUluc3RydWN0JTIyKSUwQSUyQiUyMG1vZGVsJTJDJTIwdG9rZW5pemVyJTIwJTNEJTIwRmFzdExhbmd1YWdlTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMlF3ZW4lMkZRd2VuMi0wLjVCLUluc3RydWN0JTIyKSUwQSUyQiUyMG1vZGVsJTIwJTNEJTIwRmFzdExhbmd1YWdlTW9kZWwuZ2V0X3BlZnRfbW9kZWwobW9kZWwpJTBBJTIwJTIwdHJhaW5fZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ0cmwtbGliJTJGdWx0cmFmZWVkYmFja19iaW5hcml6ZWQlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKSUwQSUwQS0lMjB0cmFpbmluZ19hcmdzJTIwJTNEJTIwRFBPQ29uZmlnKG91dHB1dF9kaXIlM0QlMjJRd2VuMi0wLjVCLURQTyUyMiUyQyUyMGxvZ2dpbmdfc3RlcHMlM0QxMCklMEElMkIlMjB0cmFpbmluZ19hcmdzJTIwJTNEJTIwRFBPQ29uZmlnKG91dHB1dF9kaXIlM0QlMjJRd2VuMi0wLjVCLURQTyUyMiUyQyUyMGxvZ2dpbmdfc3RlcHMlM0QxMCUyQyUyMGJmMTYlM0RUcnVlKSUwQSUyMCUyMHRyYWluZXIlMjAlM0QlMjBEUE9UcmFpbmVyKG1vZGVsJTNEbW9kZWwlMkMlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUyMHByb2Nlc3NpbmdfY2xhc3MlM0R0b2tlbml6ZXIlMkMlMjB0cmFpbl9kYXRhc2V0JTNEdHJhaW5fZGF0YXNldCklMEElMjAlMjB0cmFpbmVyLnRyYWluKCklMEE=",highlighted:`  from datasets import load_dataset
  from trl import DPOConfig, DPOTrainer
<span class="hljs-deletion">- from transformers import AutoModelForCausalLM, AutoTokenizer</span>
<span class="hljs-addition">+ from unsloth import FastLanguageModel</span>

<span class="hljs-deletion">- model = AutoModelForCausalLM.from_pretrained(&quot;Qwen/Qwen2-0.5B-Instruct&quot;)</span>
<span class="hljs-deletion">- tokenizer = AutoTokenizer.from_pretrained(&quot;Qwen/Qwen2-0.5B-Instruct&quot;)</span>
<span class="hljs-addition">+ model, tokenizer = FastLanguageModel.from_pretrained(&quot;Qwen/Qwen2-0.5B-Instruct&quot;)</span>
<span class="hljs-addition">+ model = FastLanguageModel.get_peft_model(model)</span>
  train_dataset = load_dataset(&quot;trl-lib/ultrafeedback_binarized&quot;, split=&quot;train&quot;)

<span class="hljs-deletion">- training_args = DPOConfig(output_dir=&quot;Qwen2-0.5B-DPO&quot;, logging_steps=10)</span>
<span class="hljs-addition">+ training_args = DPOConfig(output_dir=&quot;Qwen2-0.5B-DPO&quot;, logging_steps=10, bf16=True)</span>
  trainer = DPOTrainer(model=model, args=training_args, processing_class=tokenizer, train_dataset=train_dataset)
  trainer.train()
`,wrap:!1}}),Xe=new M({props:{title:"Reference model considerations with PEFT",local:"reference-model-considerations-with-peft",headingTag:"h2"}}),Ke=new M({props:{title:"Downsides to merging QLoRA before DPO (approach 2)",local:"downsides-to-merging-qlora-before-dpo-approach-2",headingTag:"h3"}}),ot=new M({props:{title:"Using option 3 - load the adapter twice",local:"using-option-3---load-the-adapter-twice",headingTag:"h3"}}),rt=new G({props:{code:"JTIzJTIwTG9hZCUyMHRoZSUyMGJhc2UlMjBtb2RlbC4lMEFibmJfY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBsbG1faW50OF90aHJlc2hvbGQlM0Q2LjAlMkMlMEElMjAlMjAlMjAlMjBsbG1faW50OF9oYXNfZnAxNl93ZWlnaHQlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X2NvbXB1dGVfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfcXVhbnRfdHlwZSUzRCUyMm5mNCUyMiUyQyUwQSklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJtaXN0cmFsYWklMkZtaXh0cmFsLTh4N2ItdjAuMSUyMiUyQyUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEYm5iX2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMGF0dG5faW1wbGVtZW50YXRpb24lM0QlMjJmbGFzaF9hdHRlbnRpb25fMiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEElMjAlMjAlMjAlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUwQSklMEFtb2RlbC5jb25maWcudXNlX2NhY2hlJTIwJTNEJTIwRmFsc2UlMEElMEElMjMlMjBMb2FkJTIwdGhlJTIwYWRhcHRlci4lMEFtb2RlbCUyMCUzRCUyMFBlZnRNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwbW9kZWwlMkMlMEElMjAlMjAlMjAlMjAlMjIlMkZwYXRoJTJGdG8lMkZwZWZ0JTIyJTJDJTBBJTIwJTIwJTIwJTIwaXNfdHJhaW5hYmxlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGFkYXB0ZXJfbmFtZSUzRCUyMnRyYWluJTIyJTJDJTBBKSUwQSUyMyUyMExvYWQlMjB0aGUlMjBhZGFwdGVyJTIwYSUyMHNlY29uZCUyMHRpbWUlMkMlMjB3aXRoJTIwYSUyMGRpZmZlcmVudCUyMG5hbWUlMkMlMjB3aGljaCUyMHdpbGwlMjBiZSUyMG91ciUyMHJlZmVyZW5jZSUyMG1vZGVsLiUwQW1vZGVsLmxvYWRfYWRhcHRlciglMjIlMkZwYXRoJTJGdG8lMkZwZWZ0JTIyJTJDJTIwYWRhcHRlcl9uYW1lJTNEJTIycmVmZXJlbmNlJTIyKSUwQSUwQSUyMyUyMEluaXRpYWxpemUlMjB0aGUlMjB0cmFpbmVyJTJDJTIwd2l0aG91dCUyMGElMjByZWZfbW9kZWwlMjBwYXJhbS4lMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwRFBPQ29uZmlnKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2FkYXB0ZXJfbmFtZSUzRCUyMnRyYWluJTIyJTJDJTBBJTIwJTIwJTIwJTIwcmVmX2FkYXB0ZXJfbmFtZSUzRCUyMnJlZmVyZW5jZSUyMiUyQyUwQSklMEFkcG9fdHJhaW5lciUyMCUzRCUyMERQT1RyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUwQSUyMCUyMCUyMCUyMC4uLiUwQSk=",highlighted:`<span class="hljs-comment"># Load the base model.</span>
bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    llm_int8_threshold=<span class="hljs-number">6.0</span>,
    llm_int8_has_fp16_weight=<span class="hljs-literal">False</span>,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)
model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;mistralai/mixtral-8x7b-v0.1&quot;</span>,
    load_in_4bit=<span class="hljs-literal">True</span>,
    quantization_config=bnb_config,
    attn_implementation=<span class="hljs-string">&quot;flash_attention_2&quot;</span>,
    torch_dtype=torch.bfloat16,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)
model.config.use_cache = <span class="hljs-literal">False</span>

<span class="hljs-comment"># Load the adapter.</span>
model = PeftModel.from_pretrained(
    model,
    <span class="hljs-string">&quot;/path/to/peft&quot;</span>,
    is_trainable=<span class="hljs-literal">True</span>,
    adapter_name=<span class="hljs-string">&quot;train&quot;</span>,
)
<span class="hljs-comment"># Load the adapter a second time, with a different name, which will be our reference model.</span>
model.load_adapter(<span class="hljs-string">&quot;/path/to/peft&quot;</span>, adapter_name=<span class="hljs-string">&quot;reference&quot;</span>)

<span class="hljs-comment"># Initialize the trainer, without a ref_model param.</span>
training_args = DPOConfig(
    model_adapter_name=<span class="hljs-string">&quot;train&quot;</span>,
    ref_adapter_name=<span class="hljs-string">&quot;reference&quot;</span>,
)
dpo_trainer = DPOTrainer(
    model,
    args=training_args,
    ...
)`,wrap:!1}}),lt=new M({props:{title:"DPOTrainer",local:"trl.DPOTrainer",headingTag:"h2"}}),st=new $({props:{name:"class trl.DPOTrainer",anchor:"trl.DPOTrainer",parameters:[{name:"model",val:": typing.Union[transformers.modeling_utils.PreTrainedModel, torch.nn.modules.module.Module, str, NoneType] = None"},{name:"ref_model",val:": typing.Union[transformers.modeling_utils.PreTrainedModel, torch.nn.modules.module.Module, str, NoneType] = None"},{name:"args",val:": typing.Optional[trl.trainer.dpo_config.DPOConfig] = None"},{name:"data_collator",val:": typing.Optional[transformers.data.data_collator.DataCollator] = None"},{name:"train_dataset",val:": typing.Optional[datasets.arrow_dataset.Dataset] = None"},{name:"eval_dataset",val:": typing.Union[datasets.arrow_dataset.Dataset, dict[str, datasets.arrow_dataset.Dataset], NoneType] = None"},{name:"processing_class",val:": typing.Union[transformers.tokenization_utils_base.PreTrainedTokenizerBase, transformers.image_processing_utils.BaseImageProcessor, transformers.feature_extraction_utils.FeatureExtractionMixin, transformers.processing_utils.ProcessorMixin, NoneType] = None"},{name:"model_init",val:": typing.Optional[typing.Callable[[], transformers.modeling_utils.PreTrainedModel]] = None"},{name:"compute_metrics",val:": typing.Optional[typing.Callable[[transformers.trainer_utils.EvalLoopOutput], dict]] = None"},{name:"callbacks",val:": typing.Optional[list[transformers.trainer_callback.TrainerCallback]] = None"},{name:"optimizers",val:": tuple = (None, None)"},{name:"preprocess_logits_for_metrics",val:": typing.Optional[typing.Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None"},{name:"peft_config",val:": typing.Optional[dict] = None"}],parametersDescription:[{anchor:"trl.DPOTrainer.model",description:`<strong>model</strong> (<code>transformers.PreTrainedModel</code>) &#x2014;
The model to train, preferably an <code>AutoModelForSequenceClassification</code>.`,name:"model"},{anchor:"trl.DPOTrainer.ref_model",description:`<strong>ref_model</strong> (<code>PreTrainedModelWrapper</code>) &#x2014;
Hugging Face transformer model with a casual language modelling head. Used for implicit reward computation and loss. If no
reference model is provided, the trainer will create a reference model with the same architecture as the model to be optimized.`,name:"ref_model"},{anchor:"trl.DPOTrainer.args",description:`<strong>args</strong> (<code>DPOConfig</code>) &#x2014;
The DPO config arguments to use for training.`,name:"args"},{anchor:"trl.DPOTrainer.data_collator",description:`<strong>data_collator</strong> (<code>transformers.DataCollator</code>) &#x2014;
The data collator to use for training. If None is specified, the default data collator (<code>DataCollatorForPreference</code>) will be used
which will pad the sequences to the maximum length of the sequences in the batch, given a dataset of paired sequences.`,name:"data_collator"},{anchor:"trl.DPOTrainer.train_dataset",description:`<strong>train_dataset</strong> (<code>datasets.Dataset</code>) &#x2014;
The dataset to use for training.`,name:"train_dataset"},{anchor:"trl.DPOTrainer.eval_dataset",description:`<strong>eval_dataset</strong> (<code>datasets.Dataset</code>) &#x2014;
The dataset to use for evaluation.`,name:"eval_dataset"},{anchor:"trl.DPOTrainer.processing_class",description:`<strong>processing_class</strong> (<code>PreTrainedTokenizerBase</code> or <code>BaseImageProcessor</code> or <code>FeatureExtractionMixin</code> or <code>ProcessorMixin</code>, <em>optional</em>) &#x2014;
Processing class used to process the data. If provided, will be used to automatically process the inputs
for the model, and it will be saved along the model to make it easier to rerun an interrupted training or
reuse the fine-tuned model.`,name:"processing_class"},{anchor:"trl.DPOTrainer.model_init",description:`<strong>model_init</strong> (<code>Callable[[], transformers.PreTrainedModel]</code>) &#x2014;
The model initializer to use for training. If None is specified, the default model initializer will be used.`,name:"model_init"},{anchor:"trl.DPOTrainer.compute_metrics",description:`<strong>compute_metrics</strong> (<code>Callable[[EvalPrediction], dict]</code>, <em>optional</em>) &#x2014;
The function to use to compute the metrics. Must take a <code>EvalPrediction</code> and return
a dictionary string to metric values.`,name:"compute_metrics"},{anchor:"trl.DPOTrainer.callbacks",description:`<strong>callbacks</strong> (<code>list[transformers.TrainerCallback]</code>) &#x2014;
The callbacks to use for training.`,name:"callbacks"},{anchor:"trl.DPOTrainer.optimizers",description:`<strong>optimizers</strong> (<code>tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]</code>) &#x2014;
The optimizer and scheduler to use for training.`,name:"optimizers"},{anchor:"trl.DPOTrainer.preprocess_logits_for_metrics",description:`<strong>preprocess_logits_for_metrics</strong> (<code>Callable[[torch.Tensor, torch.Tensor], torch.Tensor]</code>) &#x2014;
The function to use to preprocess the logits before computing the metrics.`,name:"preprocess_logits_for_metrics"},{anchor:"trl.DPOTrainer.peft_config",description:`<strong>peft_config</strong> (<code>dict</code>, defaults to <code>None</code>) &#x2014;
The PEFT configuration to use for training. If you pass a PEFT configuration, the model will be wrapped in a PEFT model.`,name:"peft_config"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L160"}}),it=new $({props:{name:"compute_ref_log_probs",anchor:"trl.DPOTrainer.compute_ref_log_probs",parameters:[{name:"batch",val:": dict"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L800"}}),dt=new $({props:{name:"concatenated_forward",anchor:"trl.DPOTrainer.concatenated_forward",parameters:[{name:"model",val:": Module"},{name:"batch",val:": dict"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L1069"}}),pt=new $({props:{name:"concatenated_inputs",anchor:"trl.DPOTrainer.concatenated_inputs",parameters:[{name:"batch",val:": dict"},{name:"padding_value",val:": int"}],parametersDescription:[{anchor:"trl.DPOTrainer.concatenated_inputs.batch",description:`<strong>batch</strong> (<code>dict[str, Union[list, torch.LongTensor]]</code>) &#x2014;
A batch of input data. The batch must contain the following keys:</p>
<ul>
<li><code>&quot;prompt_input_ids&quot;</code>: Tensor of shape <code>(batch_size, prompt_length)</code> representing the prompt input IDs.</li>
<li><code>&quot;chosen_input_ids&quot;</code>: Tensor of shape <code>(batch_size, chosen_length)</code> representing the chosen completion input IDs.</li>
<li><code>&quot;rejected_input_ids&quot;</code>: Tensor of shape <code>(batch_size, rejected_length)</code> representing the rejected completion input IDs.</li>
<li><code>&quot;prompt_pixel_values&quot;</code> (optional): Tensor for pixel values, if available.</li>
<li><code>&quot;prompt_pixel_attention_mask&quot;</code> (optional): Tensor for pixel attention masks, if available.</li>
</ul>`,name:"batch"},{anchor:"trl.DPOTrainer.concatenated_inputs.padding_value",description:`<strong>padding_value</strong> (<code>int</code>) &#x2014;
The padding value to use for the concatenated completion sequences (<code>chosen_input_ids</code> and
<code>rejected_input_ids</code>).`,name:"padding_value"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L812",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A dictionary containing:</p>
<ul>
<li><code>"prompt_input_ids"</code>: Concatenated prompt input IDs of shape <code>(2 * batch_size, prompt_length)</code>.</li>
<li><code>"completion_input_ids"</code>: Concatenated chosen and rejected completion input IDs of shape <code>(2 * batch_size, max_completion_length)</code>.</li>
<li><code>"prompt_attention_mask"</code>: Concatenated prompt attention masks of shape <code>(2 * batch_size, prompt_length)</code>.</li>
<li><code>"completion_attention_mask"</code>: Concatenated chosen and rejected attention masks of shape <code>(2 * batch_size, max_completion_length)</code>.</li>
<li><code>"pixel_values"</code> (optional): Concatenated pixel values if <code>"prompt_pixel_values"</code> are present.</li>
<li><code>"pixel_attention_mask"</code> (optional): Concatenated pixel attention masks if <code>"prompt_pixel_attention_mask"</code> are present.</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>dict[str, torch.LongTensor]</code></p>
`}}),ct=new $({props:{name:"create_model_card",anchor:"trl.DPOTrainer.create_model_card",parameters:[{name:"model_name",val:": typing.Optional[str] = None"},{name:"dataset_name",val:": typing.Optional[str] = None"},{name:"tags",val:": typing.Union[str, list[str], NoneType] = None"}],parametersDescription:[{anchor:"trl.DPOTrainer.create_model_card.model_name",description:`<strong>model_name</strong> (<code>str</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Name of the model.`,name:"model_name"},{anchor:"trl.DPOTrainer.create_model_card.dataset_name",description:`<strong>dataset_name</strong> (<code>str</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Name of the dataset used for training.`,name:"dataset_name"},{anchor:"trl.DPOTrainer.create_model_card.tags",description:`<strong>tags</strong> (<code>str</code>, <code>list[str]</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Tags to be associated with the model card.`,name:"tags"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L1493"}}),mt=new $({props:{name:"dpo_loss",anchor:"trl.DPOTrainer.dpo_loss",parameters:[{name:"chosen_logps",val:": FloatTensor"},{name:"rejected_logps",val:": FloatTensor"},{name:"ref_chosen_logps",val:": FloatTensor"},{name:"ref_rejected_logps",val:": FloatTensor"}],parametersDescription:[{anchor:"trl.DPOTrainer.dpo_loss.chosen_logps",description:`<strong>chosen_logps</strong> (<code>torch.FloatTensor</code>) &#x2014;
Log probabilities of the model for the chosen responses. Shape: <code>(batch_size,)</code>.`,name:"chosen_logps"},{anchor:"trl.DPOTrainer.dpo_loss.rejected_logps",description:`<strong>rejected_logps</strong> (<code>torch.FloatTensor</code>) &#x2014;
Log probabilities of the model for the rejected responses. Shape: <code>(batch_size,)</code>.`,name:"rejected_logps"},{anchor:"trl.DPOTrainer.dpo_loss.ref_chosen_logps",description:`<strong>ref_chosen_logps</strong> (<code>torch.FloatTensor</code>) &#x2014;
Log probabilities of the reference model for the chosen responses. Shape: <code>(batch_size,)</code>.`,name:"ref_chosen_logps"},{anchor:"trl.DPOTrainer.dpo_loss.ref_rejected_logps",description:`<strong>ref_rejected_logps</strong> (<code>torch.FloatTensor</code>) &#x2014;
Log probabilities of the reference model for the rejected responses. Shape: <code>(batch_size,)</code>.`,name:"ref_rejected_logps"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L882",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>(losses, chosen_rewards, rejected_rewards)</code>.
The losses tensor contains the DPO loss for each example in the batch.
The <code>chosen_rewards</code> and <code>rejected_rewards</code> tensors contain the rewards for the chosen and rejected
responses, respectively.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A tuple of three tensors</p>
`}}),ht=new $({props:{name:"evaluation_loop",anchor:"trl.DPOTrainer.evaluation_loop",parameters:[{name:"dataloader",val:": DataLoader"},{name:"description",val:": str"},{name:"prediction_loss_only",val:": typing.Optional[bool] = None"},{name:"ignore_keys",val:": typing.Optional[list[str]] = None"},{name:"metric_key_prefix",val:": str = 'eval'"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L1418"}}),gt=new $({props:{name:"generate_from_model_and_ref",anchor:"trl.DPOTrainer.generate_from_model_and_ref",parameters:[{name:"model",val:""},{name:"batch",val:": dict"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L1331"}}),ut=new $({props:{name:"get_batch_loss_metrics",anchor:"trl.DPOTrainer.get_batch_loss_metrics",parameters:[{name:"model",val:""},{name:"batch",val:": dict"},{name:"train_eval",val:": typing.Literal['train', 'eval'] = 'train'"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L1245"}}),ft=new $({props:{name:"get_eval_dataloader",anchor:"trl.DPOTrainer.get_eval_dataloader",parameters:[{name:"eval_dataset",val:": typing.Optional[datasets.arrow_dataset.Dataset] = None"}],parametersDescription:[{anchor:"trl.DPOTrainer.get_eval_dataloader.eval_dataset",description:`<strong>eval_dataset</strong> (<code>torch.utils.data.Dataset</code>, <em>optional</em>) &#x2014;
If provided, will override <code>self.eval_dataset</code>. If it is a <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset" rel="nofollow">Dataset</a>, columns not accepted
by the <code>model.forward()</code> method are automatically removed. It must implement <code>__len__</code>.`,name:"eval_dataset"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L735"}}),_t=new $({props:{name:"get_train_dataloader",anchor:"trl.DPOTrainer.get_train_dataloader",parameters:[],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L689"}}),bt=new $({props:{name:"log",anchor:"trl.DPOTrainer.log",parameters:[{name:"logs",val:": dict"},{name:"start_time",val:": typing.Optional[float] = None"}],parametersDescription:[{anchor:"trl.DPOTrainer.log.logs",description:`<strong>logs</strong> (<code>dict[str, float]</code>) &#x2014;
The values to log.`,name:"logs"},{anchor:"trl.DPOTrainer.log.start_time",description:`<strong>start_time</strong> (<code>float</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Start time of the training.`,name:"start_time"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L1471"}}),yt=new $({props:{name:"null_ref_context",anchor:"trl.DPOTrainer.null_ref_context",parameters:[],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L786"}}),vt=new $({props:{name:"process_row",anchor:"trl.DPOTrainer.process_row",parameters:[{name:"features",val:""},{name:"processing_class",val:""},{name:"max_prompt_length",val:""},{name:"max_completion_length",val:""},{name:"add_special_tokens",val:""}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L631"}}),wt=new $({props:{name:"tokenize_row",anchor:"trl.DPOTrainer.tokenize_row",parameters:[{name:"features",val:""},{name:"processing_class",val:""},{name:"max_prompt_length",val:""},{name:"max_completion_length",val:""},{name:"add_special_tokens",val:""}],parametersDescription:[{anchor:"trl.DPOTrainer.tokenize_row.features",description:`<strong>features</strong> (<code>dict[str, str]</code>) &#x2014;
Row of the dataset, should contain the keys <code>&quot;prompt&quot;</code>, <code>&quot;chosen&quot;</code>, and <code>&quot;rejected&quot;</code>.`,name:"features"},{anchor:"trl.DPOTrainer.tokenize_row.processing_class",description:`<strong>processing_class</strong> (<code>PreTrainedTokenizerBase</code>) &#x2014;
Processing class used to process the data.`,name:"processing_class"},{anchor:"trl.DPOTrainer.tokenize_row.max_prompt_length",description:`<strong>max_prompt_length</strong> (<code>int</code> or <code>None</code>) &#x2014;
Maximum length of the prompt sequence. If <code>None</code>, the prompt sequence is not truncated.`,name:"max_prompt_length"},{anchor:"trl.DPOTrainer.tokenize_row.max_completion_length",description:`<strong>max_completion_length</strong> (<code>int</code> or <code>None</code>) &#x2014;
Maximum length of the completion sequences. If <code>None</code>, the completion sequences are not truncated.`,name:"max_completion_length"},{anchor:"trl.DPOTrainer.tokenize_row.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>) &#x2014;
Whether to add special tokens to the sequences. Typically used for encoder-decoder models. If <code>True</code>,
the prompt sequence will have a bos token prepended and an eos token appended. In any case, the
completion sequences will have an eos token appended.`,name:"add_special_tokens"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L569",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Tokenized sequences with the keys <code>"prompt_input_ids"</code>, <code>"chosen_input_ids"</code>, and
\`“rejected_input_ids”.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>dict[str, list[int]]</code></p>
`}}),Q=new Nr({props:{anchor:"trl.DPOTrainer.tokenize_row.example",$$slots:{default:[Br]},$$scope:{ctx:Pt}}}),Tt=new M({props:{title:"DPOConfig",local:"trl.DPOConfig",headingTag:"h2"}}),Mt=new $({props:{name:"class trl.DPOConfig",anchor:"trl.DPOConfig",parameters:[{name:"output_dir",val:": typing.Optional[str] = None"},{name:"overwrite_output_dir",val:": bool = False"},{name:"do_train",val:": bool = False"},{name:"do_eval",val:": bool = False"},{name:"do_predict",val:": bool = False"},{name:"eval_strategy",val:": typing.Union[transformers.trainer_utils.IntervalStrategy, str] = 'no'"},{name:"prediction_loss_only",val:": bool = False"},{name:"per_device_train_batch_size",val:": int = 8"},{name:"per_device_eval_batch_size",val:": int = 8"},{name:"per_gpu_train_batch_size",val:": typing.Optional[int] = None"},{name:"per_gpu_eval_batch_size",val:": typing.Optional[int] = None"},{name:"gradient_accumulation_steps",val:": int = 1"},{name:"eval_accumulation_steps",val:": typing.Optional[int] = None"},{name:"eval_delay",val:": typing.Optional[float] = 0"},{name:"torch_empty_cache_steps",val:": typing.Optional[int] = None"},{name:"learning_rate",val:": float = 1e-06"},{name:"weight_decay",val:": float = 0.0"},{name:"adam_beta1",val:": float = 0.9"},{name:"adam_beta2",val:": float = 0.999"},{name:"adam_epsilon",val:": float = 1e-08"},{name:"max_grad_norm",val:": float = 1.0"},{name:"num_train_epochs",val:": float = 3.0"},{name:"max_steps",val:": int = -1"},{name:"lr_scheduler_type",val:": typing.Union[transformers.trainer_utils.SchedulerType, str] = 'linear'"},{name:"lr_scheduler_kwargs",val:": typing.Union[dict, str, NoneType] = <factory>"},{name:"warmup_ratio",val:": float = 0.0"},{name:"warmup_steps",val:": int = 0"},{name:"log_level",val:": typing.Optional[str] = 'passive'"},{name:"log_level_replica",val:": typing.Optional[str] = 'warning'"},{name:"log_on_each_node",val:": bool = True"},{name:"logging_dir",val:": typing.Optional[str] = None"},{name:"logging_strategy",val:": typing.Union[transformers.trainer_utils.IntervalStrategy, str] = 'steps'"},{name:"logging_first_step",val:": bool = False"},{name:"logging_steps",val:": float = 500"},{name:"logging_nan_inf_filter",val:": bool = True"},{name:"save_strategy",val:": typing.Union[transformers.trainer_utils.SaveStrategy, str] = 'steps'"},{name:"save_steps",val:": float = 500"},{name:"save_total_limit",val:": typing.Optional[int] = None"},{name:"save_safetensors",val:": typing.Optional[bool] = True"},{name:"save_on_each_node",val:": bool = False"},{name:"save_only_model",val:": bool = False"},{name:"restore_callback_states_from_checkpoint",val:": bool = False"},{name:"no_cuda",val:": bool = False"},{name:"use_cpu",val:": bool = False"},{name:"use_mps_device",val:": bool = False"},{name:"seed",val:": int = 42"},{name:"data_seed",val:": typing.Optional[int] = None"},{name:"jit_mode_eval",val:": bool = False"},{name:"use_ipex",val:": bool = False"},{name:"bf16",val:": bool = False"},{name:"fp16",val:": bool = False"},{name:"fp16_opt_level",val:": str = 'O1'"},{name:"half_precision_backend",val:": str = 'auto'"},{name:"bf16_full_eval",val:": bool = False"},{name:"fp16_full_eval",val:": bool = False"},{name:"tf32",val:": typing.Optional[bool] = None"},{name:"local_rank",val:": int = -1"},{name:"ddp_backend",val:": typing.Optional[str] = None"},{name:"tpu_num_cores",val:": typing.Optional[int] = None"},{name:"tpu_metrics_debug",val:": bool = False"},{name:"debug",val:": typing.Union[str, list[transformers.debug_utils.DebugOption]] = ''"},{name:"dataloader_drop_last",val:": bool = False"},{name:"eval_steps",val:": typing.Optional[float] = None"},{name:"dataloader_num_workers",val:": int = 0"},{name:"dataloader_prefetch_factor",val:": typing.Optional[int] = None"},{name:"past_index",val:": int = -1"},{name:"run_name",val:": typing.Optional[str] = None"},{name:"disable_tqdm",val:": typing.Optional[bool] = None"},{name:"remove_unused_columns",val:": typing.Optional[bool] = True"},{name:"label_names",val:": typing.Optional[list[str]] = None"},{name:"load_best_model_at_end",val:": typing.Optional[bool] = False"},{name:"metric_for_best_model",val:": typing.Optional[str] = None"},{name:"greater_is_better",val:": typing.Optional[bool] = None"},{name:"ignore_data_skip",val:": bool = False"},{name:"fsdp",val:": typing.Union[list[transformers.trainer_utils.FSDPOption], str, NoneType] = ''"},{name:"fsdp_min_num_params",val:": int = 0"},{name:"fsdp_config",val:": typing.Union[dict, str, NoneType] = None"},{name:"fsdp_transformer_layer_cls_to_wrap",val:": typing.Optional[str] = None"},{name:"accelerator_config",val:": typing.Union[dict, str, NoneType] = None"},{name:"deepspeed",val:": typing.Union[dict, str, NoneType] = None"},{name:"label_smoothing_factor",val:": float = 0.0"},{name:"optim",val:": typing.Union[transformers.training_args.OptimizerNames, str] = 'adamw_torch'"},{name:"optim_args",val:": typing.Optional[str] = None"},{name:"adafactor",val:": bool = False"},{name:"group_by_length",val:": bool = False"},{name:"length_column_name",val:": typing.Optional[str] = 'length'"},{name:"report_to",val:": typing.Union[NoneType, str, list[str]] = None"},{name:"ddp_find_unused_parameters",val:": typing.Optional[bool] = None"},{name:"ddp_bucket_cap_mb",val:": typing.Optional[int] = None"},{name:"ddp_broadcast_buffers",val:": typing.Optional[bool] = None"},{name:"dataloader_pin_memory",val:": bool = True"},{name:"dataloader_persistent_workers",val:": bool = False"},{name:"skip_memory_metrics",val:": bool = True"},{name:"use_legacy_prediction_loop",val:": bool = False"},{name:"push_to_hub",val:": bool = False"},{name:"resume_from_checkpoint",val:": typing.Optional[str] = None"},{name:"hub_model_id",val:": typing.Optional[str] = None"},{name:"hub_strategy",val:": typing.Union[transformers.trainer_utils.HubStrategy, str] = 'every_save'"},{name:"hub_token",val:": typing.Optional[str] = None"},{name:"hub_private_repo",val:": typing.Optional[bool] = None"},{name:"hub_always_push",val:": bool = False"},{name:"gradient_checkpointing",val:": bool = False"},{name:"gradient_checkpointing_kwargs",val:": typing.Union[dict, str, NoneType] = None"},{name:"include_inputs_for_metrics",val:": bool = False"},{name:"include_for_metrics",val:": list = <factory>"},{name:"eval_do_concat_batches",val:": bool = True"},{name:"fp16_backend",val:": str = 'auto'"},{name:"push_to_hub_model_id",val:": typing.Optional[str] = None"},{name:"push_to_hub_organization",val:": typing.Optional[str] = None"},{name:"push_to_hub_token",val:": typing.Optional[str] = None"},{name:"mp_parameters",val:": str = ''"},{name:"auto_find_batch_size",val:": bool = False"},{name:"full_determinism",val:": bool = False"},{name:"torchdynamo",val:": typing.Optional[str] = None"},{name:"ray_scope",val:": typing.Optional[str] = 'last'"},{name:"ddp_timeout",val:": typing.Optional[int] = 1800"},{name:"torch_compile",val:": bool = False"},{name:"torch_compile_backend",val:": typing.Optional[str] = None"},{name:"torch_compile_mode",val:": typing.Optional[str] = None"},{name:"include_tokens_per_second",val:": typing.Optional[bool] = False"},{name:"include_num_input_tokens_seen",val:": typing.Optional[bool] = False"},{name:"neftune_noise_alpha",val:": typing.Optional[float] = None"},{name:"optim_target_modules",val:": typing.Union[NoneType, str, list[str]] = None"},{name:"batch_eval_metrics",val:": bool = False"},{name:"eval_on_start",val:": bool = False"},{name:"use_liger_kernel",val:": typing.Optional[bool] = False"},{name:"eval_use_gather_object",val:": typing.Optional[bool] = False"},{name:"average_tokens_across_devices",val:": typing.Optional[bool] = False"},{name:"model_init_kwargs",val:": typing.Optional[dict[str, typing.Any]] = None"},{name:"ref_model_init_kwargs",val:": typing.Optional[dict[str, typing.Any]] = None"},{name:"model_adapter_name",val:": typing.Optional[str] = None"},{name:"ref_adapter_name",val:": typing.Optional[str] = None"},{name:"force_use_ref_model",val:": bool = False"},{name:"disable_dropout",val:": bool = True"},{name:"use_logits_to_keep",val:": bool = False"},{name:"dataset_num_proc",val:": typing.Optional[int] = None"},{name:"padding_value",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"max_prompt_length",val:": typing.Optional[int] = 512"},{name:"max_completion_length",val:": typing.Optional[int] = None"},{name:"max_length",val:": typing.Optional[int] = 1024"},{name:"truncation_mode",val:": str = 'keep_end'"},{name:"padding_free",val:": bool = False"},{name:"precompute_ref_log_probs",val:": bool = False"},{name:"precompute_ref_batch_size",val:": typing.Optional[int] = None"},{name:"tools",val:": typing.Optional[list[typing.Union[dict, typing.Callable]]] = None"},{name:"loss_type",val:": str = 'sigmoid'"},{name:"beta",val:": float = 0.1"},{name:"f_divergence_type",val:": FDivergenceType = <FDivergenceType.REVERSE_KL: 'reverse_kl'>"},{name:"f_alpha_divergence_coef",val:": float = 1.0"},{name:"reference_free",val:": bool = False"},{name:"label_smoothing",val:": float = 0.0"},{name:"use_weighting",val:": bool = False"},{name:"rpo_alpha",val:": typing.Optional[float] = None"},{name:"discopop_tau",val:": float = 0.05"},{name:"sync_ref_model",val:": bool = False"},{name:"ref_model_mixup_alpha",val:": float = 0.6"},{name:"ref_model_sync_steps",val:": int = 512"},{name:"generate_during_eval",val:": bool = False"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_config.py#L33",parameterGroups:[{title:"Parameters that control the model and reference model",parametersDescription:[{anchor:"trl.DPOConfig.model_init_kwargs",description:`<strong>model_init_kwargs</strong> (<code>dict[str, Any]</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Keyword arguments for <code>AutoModelForCausalLM.from_pretrained</code>, used when the <code>model</code> argument of the
<a href="/docs/trl/main/en/dpo_trainer#trl.DPOTrainer">DPOTrainer</a> is provided as a string.`,name:"model_init_kwargs"},{anchor:"trl.DPOConfig.ref_model_init_kwargs",description:`<strong>ref_model_init_kwargs</strong> (<code>dict[str, Any]</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Keyword arguments for <code>AutoModelForCausalLM.from_pretrained</code>, used when the <code>ref_model</code> argument of the
<a href="/docs/trl/main/en/dpo_trainer#trl.DPOTrainer">DPOTrainer</a> is provided as a string.`,name:"ref_model_init_kwargs"},{anchor:"trl.DPOConfig.model_adapter_name",description:`<strong>model_adapter_name</strong> (<code>str</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Name of the train target PEFT adapter, when using LoRA with multiple adapters.`,name:"model_adapter_name"},{anchor:"trl.DPOConfig.ref_adapter_name",description:`<strong>ref_adapter_name</strong> (<code>str</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Name of the reference PEFT adapter, when using LoRA with multiple adapters.`,name:"ref_adapter_name"},{anchor:"trl.DPOConfig.force_use_ref_model",description:`<strong>force_use_ref_model</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
If you provide a PEFT model as the active model and wish to use a different model for the <code>ref_model</code>, set
this flag to <code>True</code>.`,name:"force_use_ref_model"},{anchor:"trl.DPOConfig.disable_dropout",description:`<strong>disable_dropout</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to disable dropout in the model and reference model.`,name:"disable_dropout"},{anchor:"trl.DPOConfig.use_logits_to_keep",description:`<strong>use_logits_to_keep</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
If <code>True</code>, only a specified number of logits are computed in the forward pass. This can be useful for
saving memory and speeding up training by not computing the logits for all tokens, especially in
scenarios when working with very long prompts where labels are ignored (-100).`,name:"use_logits_to_keep"}]},{title:"Parameters that control the data preprocessing",parametersDescription:[{anchor:"trl.DPOConfig.dataset_num_proc",description:`<strong>dataset_num_proc</strong> (<code>int</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Number of processes to use for processing the dataset.`,name:"dataset_num_proc"},{anchor:"trl.DPOConfig.padding_value",description:`<strong>padding_value</strong> (<code>int</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Padding value to use. If <code>None</code>, the padding value of the tokenizer is used.`,name:"padding_value"},{anchor:"trl.DPOConfig.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to <code>-100</code>) &#x2014;
Padding value to use for labels.`,name:"label_pad_token_id"},{anchor:"trl.DPOConfig.max_prompt_length",description:`<strong>max_prompt_length</strong> (<code>int</code> or <code>None</code>, <em>optional</em>, defaults to <code>512</code>) &#x2014;
Maximum length of the prompt.`,name:"max_prompt_length"},{anchor:"trl.DPOConfig.max_completion_length",description:`<strong>max_completion_length</strong> (<code>int</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Maximum length of the completion.`,name:"max_completion_length"},{anchor:"trl.DPOConfig.max_length",description:`<strong>max_length</strong> (<code>int</code> or <code>None</code>, <em>optional</em>, defaults to <code>1024</code>) &#x2014;
Maximum length of the full sequence (prompt + completion).`,name:"max_length"},{anchor:"trl.DPOConfig.truncation_mode",description:`<strong>truncation_mode</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;keep_end&quot;</code>) &#x2014;
Truncation mode to use when the sequence exceeds <code>max_length</code>. Possible values are <code>&quot;keep_end&quot;</code> and
<code>&quot;keep_start&quot;</code>.`,name:"truncation_mode"},{anchor:"trl.DPOConfig.padding_free",description:`<strong>padding_free</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to perform forward passes without padding by flattening all sequences in the batch into a single
continuous sequence. This reduces memory usage by eliminating padding overhead. Currently, this is only
supported with the <code>flash_attention_2</code> attention implementation, which can efficiently handle the flattened
batch structure.`,name:"padding_free"},{anchor:"trl.DPOConfig.precompute_ref_log_probs",description:`<strong>precompute_ref_log_probs</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to precompute the log probabilities from the reference model. Setting this to <code>True</code> allows
training without needing the reference model during training, which can help reduce GPU memory usage. If
set to <code>False</code> (default), the reference model will be used during training to compute log probabilities
on-the-fly.`,name:"precompute_ref_log_probs"},{anchor:"trl.DPOConfig.precompute_ref_batch_size",description:`<strong>precompute_ref_batch_size</strong> (<code>int</code> or <code>None</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Batch size to use when precomputing reference model log probabilities. This can be set higher than the
training batch size to speed up preprocessing. If <code>None</code>, defaults to <code>per_device_train_batch_size</code> for
training and <code>per_device_eval_batch_size</code> for evaluation.`,name:"precompute_ref_batch_size"},{anchor:"trl.DPOConfig.tools",description:`<strong>tools</strong> (<code>Optional[list[Union[dict, Callable]]]</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
List of tools (callable functions) that will be accessible to the model.
If the template does not support function calling, this argument will have no effect.`,name:"tools"}]},{title:"Parameters that control the training",parametersDescription:[{anchor:"trl.DPOConfig.learning_rate",description:`<strong>learning_rate</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1e-6</code>) &#x2014;
Initial learning rate for <code>AdamW</code> optimizer. The default value replaces that of
<a href="https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments" rel="nofollow">TrainingArguments</a>.`,name:"learning_rate"},{anchor:"trl.DPOConfig.loss_type",description:`<strong>loss_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;sigmoid&quot;</code>) &#x2014;
Type of loss to use. Possible values are:</p>
<ul>
<li><code>&quot;sigmoid&quot;</code>: sigmoid loss from the original <a href="https://huggingface.co/papers/2305.18290" rel="nofollow">DPO</a> paper.</li>
<li><code>&quot;hinge&quot;</code>: hinge loss on the normalized likelihood from the <a href="https://huggingface.co/papers/2305.10425" rel="nofollow">SLiC</a> paper.</li>
<li><code>&quot;ipo&quot;</code>: IPO loss from the <a href="https://huggingface.co/papers/2310.12036" rel="nofollow">IPO</a> paper.</li>
<li><code>&quot;exo_pair&quot;</code>: pairwise EXO loss from the <a href="https://huggingface.co/papers/2402.00856" rel="nofollow">EXO</a> paper.</li>
<li><code>&quot;nca_pair&quot;</code>: pairwise NCA loss from the <a href="https://huggingface.co/papers/2402.05369" rel="nofollow">NCA</a> paper.</li>
<li><code>&quot;robust&quot;</code>: unbiased estimate of the DPO loss that is robust to preference noise from the <a href="https://huggingface.co/papers/2403.00409" rel="nofollow">Robust DPO</a> paper.</li>
<li><code>&quot;bco_pair&quot;</code>: pairwise BCO loss from the <a href="https://huggingface.co/papers/2404.04656" rel="nofollow">BCO</a> paper.</li>
<li><code>&quot;sppo_hard&quot;</code>: SPPO loss with hard label from the <a href="https://huggingface.co/papers/2405.00675" rel="nofollow">SPPO</a> paper.</li>
<li><code>&quot;aot&quot;</code>: AOT loss for paired datasets from the <a href="https://huggingface.co/papers/2406.05882" rel="nofollow">AOT</a> paper.</li>
<li><code>&quot;aot_pair&quot;</code>: AOT loss for unpaired datasets from the <a href="https://huggingface.co/papers/2406.05882" rel="nofollow">AOT</a> paper.</li>
<li><code>&quot;discopop&quot;</code>: DiscoPOP (a.k.a Log-Ratio Modulated Loss, LRML) loss from the <a href="https://huggingface.co/papers/2406.08414" rel="nofollow">DiscoPOP</a> paper.</li>
<li><code>&quot;apo_zero&quot;</code>: APO-zero loss from the <a href="https://huggingface.co/papers/2408.06266" rel="nofollow">APO</a> paper.</li>
<li><code>&quot;apo_down&quot;</code>: APO-down loss from the <a href="https://huggingface.co/papers/2408.06266" rel="nofollow">APO</a> paper.</li>
</ul>`,name:"loss_type"},{anchor:"trl.DPOConfig.beta",description:`<strong>beta</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.1</code>) &#x2014;
Parameter controlling the deviation from the reference model. Higher &#x3B2; means less deviation from the
reference model. For the IPO loss (<code>loss_type=&quot;ipo&quot;</code>), &#x3B2; is the regularization parameter denoted by &#x3C4; in
the <a href="https://huggingface.co/papers/2310.12036" rel="nofollow">paper</a>.`,name:"beta"},{anchor:"trl.DPOConfig.f_divergence_type",description:`<strong>f_divergence_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>FDivergenceType.REVERSE_KL</code>) &#x2014;
Type of f-divergence regularization function to compute divergence between policy and reference model.`,name:"f_divergence_type"},{anchor:"trl.DPOConfig.f_alpha_divergence_coef",description:`<strong>f_alpha_divergence_coef</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1.0</code>) &#x2014;
&#x3B1; coefficient in the &#x3B1;-divergence u^-&#x3B1; regularization function for DPO loss.`,name:"f_alpha_divergence_coef"},{anchor:"trl.DPOConfig.reference_free",description:`<strong>reference_free</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to ignore the provided reference model and implicitly use a reference model that assigns equal
probability to all responses.`,name:"reference_free"},{anchor:"trl.DPOConfig.label_smoothing",description:`<strong>label_smoothing</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.0</code>) &#x2014;
Robust DPO label smoothing parameter from the <a href="https://ericmitchell.ai/cdpo.pdf" rel="nofollow">cDPO</a> report and
<a href="https://huggingface.co/papers/2403.00409" rel="nofollow">Robust DPO</a> paper that should be between <code>0.0</code> and <code>0.5</code>.`,name:"label_smoothing"},{anchor:"trl.DPOConfig.use_weighting",description:`<strong>use_weighting</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to weight the loss as done in the <a href="https://huggingface.co/papers/2406.11827" rel="nofollow">WPO</a> paper.`,name:"use_weighting"},{anchor:"trl.DPOConfig.rpo_alpha",description:`<strong>rpo_alpha</strong> (<code>float</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
&#x3B1; parameter from the <a href="https://huggingface.co/papers/2404.19733" rel="nofollow">RPO</a> paper (v3), which controls the
weighting of the NLL term in the loss. If <code>None</code>, no weighting is applied and the loss is the same as the
DPO loss. The paper recommends <code>rpo_alpha=1.0</code>.`,name:"rpo_alpha"},{anchor:"trl.DPOConfig.discopop_tau",description:`<strong>discopop_tau</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.05</code>) &#x2014;
&#x3C4;/temperature parameter from the <a href="https://huggingface.co/papers/2406.08414" rel="nofollow">DiscoPOP</a> paper, which controls
the shape of log ratio modulated loss. The paper recommends the default value <code>discopop_tau=0.05</code>.`,name:"discopop_tau"},{anchor:"trl.DPOConfig.sync_ref_model",description:`<strong>sync_ref_model</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to synchronize the reference model with the active model every <code>ref_model_sync_steps</code> steps, using
the <code>ref_model_mixup_alpha</code> parameter. This synchronization originites from the
<a href="https://huggingface.co/papers/2404.09656" rel="nofollow">TR-DPO</a> paper.`,name:"sync_ref_model"},{anchor:"trl.DPOConfig.ref_model_mixup_alpha",description:`<strong>ref_model_mixup_alpha</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.6</code>) &#x2014;
&#x3B1; parameter from the <a href="https://huggingface.co/papers/2404.09656" rel="nofollow">TR-DPO</a> paper, which controls the mix
between the current policy and the previous reference policy during updates. The reference policy is
updated according to the equation: <code>&#x3C0;_ref = &#x3B1; * &#x3C0;_&#x3B8; + (1 - &#x3B1;) * &#x3C0;_ref_prev</code>. To use this parameter, you
must set <code>sync_ref_model=True</code>.`,name:"ref_model_mixup_alpha"},{anchor:"trl.DPOConfig.ref_model_sync_steps",description:`<strong>ref_model_sync_steps</strong> (<code>int</code>, <em>optional</em>, defaults to <code>512</code>) &#x2014;
&#x3C4; parameter from the <a href="https://huggingface.co/papers/2404.09656" rel="nofollow">TR-DPO</a> paper, which determines how
frequently the current policy is synchronized with the reference policy. To use this parameter, you must
set <code>sync_ref_model=True</code>.`,name:"ref_model_sync_steps"}]},{title:"Parameters that control the logging",parametersDescription:[{anchor:"trl.DPOConfig.generate_during_eval",description:`<strong>generate_during_eval</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to generate and log completions from both the model and the reference model to W&amp;B or Comet during
evaluation.`,name:"generate_during_eval"}]}]}}),xt=new M({props:{title:"DataCollatorForPreference",local:"trl.trainer.dpo_trainer.DataCollatorForPreference",headingTag:"h2"}}),$t=new $({props:{name:"class trl.trainer.dpo_trainer.DataCollatorForPreference",anchor:"trl.trainer.dpo_trainer.DataCollatorForPreference",parameters:[{name:"pad_token_id",val:": int"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"trl.trainer.dpo_trainer.DataCollatorForPreference.pad_token_id",description:`<strong>pad_token_id</strong> (<code>int</code>) &#x2014;
Token ID to use for padding.`,name:"pad_token_id"},{anchor:"trl.trainer.dpo_trainer.DataCollatorForPreference.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
Type of Tensor to return. Only <code>&quot;pt&quot;</code> is currently supported.`,name:"return_tensors"}],source:"https://github.com/huggingface/trl/blob/main/trl/trainer/dpo_trainer.py#L83"}}),V=new Nr({props:{anchor:"trl.trainer.dpo_trainer.DataCollatorForPreference.example",$$slots:{default:[Qr]},$$scope:{ctx:Pt}}}),Ct=new Hr({props:{source:"https://github.com/huggingface/trl/blob/main/docs/source/dpo_trainer.md"}}),{c(){y=l("meta"),O=n(),C=l("p"),w=n(),p(x.$$.fragment),b=n(),j=l("p"),j.innerHTML=Tn,so=n(),p(A.$$.fragment),io=n(),X=l("p"),X.innerHTML=Mn,po=n(),S=l("p"),S.textContent=xn,co=n(),Y=l("blockquote"),Y.innerHTML=$n,mo=n(),K=l("p"),K.textContent=Cn,ho=n(),ee=l("p"),ee.innerHTML=jn,go=n(),te=l("ol"),te.innerHTML=Pn,uo=n(),oe=l("p"),oe.innerHTML=Un,fo=n(),ae=l("p"),ae.innerHTML=On,_o=n(),ne=l("p"),ne.innerHTML=Jn,bo=n(),p(re.$$.fragment),yo=n(),le=l("p"),le.innerHTML=kn,vo=n(),J=l("iframe"),wo=n(),se=l("p"),se.textContent=Fn,To=n(),p(ie.$$.fragment),Mo=n(),de=l("p"),de.textContent=In,xo=n(),p(pe.$$.fragment),$o=n(),ce=l("p"),ce.textContent=Nn,Co=n(),me=l("p"),me.innerHTML=Ln,jo=n(),he=l("p"),he.innerHTML=qn,Po=n(),ge=l("pre"),ge.innerHTML=Rn,Uo=n(),p(ue.$$.fragment),Oo=n(),fe=l("p"),fe.innerHTML=Zn,Jo=n(),_e=l("p"),_e.innerHTML=Wn,ko=n(),p(be.$$.fragment),Do=n(),ye=l("p"),ye.innerHTML=En,Fo=n(),ve=l("p"),ve.innerHTML=zn,Io=n(),p(we.$$.fragment),No=n(),Te=l("p"),Te.innerHTML=Hn,Lo=n(),p(Me.$$.fragment),qo=n(),xe=l("p"),xe.innerHTML=Bn,Ro=n(),$e=l("p"),$e.innerHTML=Qn,Zo=n(),p(Ce.$$.fragment),Wo=n(),p(je.$$.fragment),Eo=n(),Pe=l("p"),Pe.textContent=Vn,zo=n(),Ue=l("ul"),Ue.innerHTML=Gn,Ho=n(),p(Oe.$$.fragment),Bo=n(),Je=l("p"),Je.innerHTML=An,Qo=n(),ke=l("table"),ke.innerHTML=Xn,Vo=n(),p(De.$$.fragment),Go=n(),Fe=l("p"),Fe.innerHTML=Sn,Ao=n(),p(Ie.$$.fragment),Xo=n(),Ne=l("p"),Ne.innerHTML=Yn,So=n(),p(Le.$$.fragment),Yo=n(),qe=l("p"),qe.innerHTML=Kn,Ko=n(),p(Re.$$.fragment),ea=n(),Ze=l("p"),Ze.innerHTML=er,ta=n(),p(We.$$.fragment),oa=n(),Ee=l("p"),Ee.innerHTML=tr,aa=n(),ze=l("p"),ze.innerHTML=or,na=n(),p(He.$$.fragment),ra=n(),Be=l("p"),Be.innerHTML=ar,la=n(),Qe=l("table"),Qe.innerHTML=nr,sa=n(),Ve=l("p"),Ve.innerHTML=rr,ia=n(),p(Ge.$$.fragment),da=n(),Ae=l("p"),Ae.innerHTML=lr,pa=n(),p(Xe.$$.fragment),ca=n(),Se=l("p"),Se.textContent=sr,ma=n(),Ye=l("ol"),Ye.innerHTML=ir,ha=n(),p(Ke.$$.fragment),ga=n(),et=l("p"),et.innerHTML=dr,ua=n(),tt=l("p"),tt.textContent=pr,fa=n(),p(ot.$$.fragment),_a=n(),at=l("p"),at.innerHTML=cr,ba=n(),nt=l("p"),nt.textContent=mr,ya=n(),p(rt.$$.fragment),va=n(),p(lt.$$.fragment),wa=n(),f=l("div"),p(st.$$.fragment),Na=n(),Ut=l("p"),Ut.textContent=hr,La=n(),q=l("div"),p(it.$$.fragment),qa=n(),Ot=l("p"),Ot.textContent=gr,Ra=n(),k=l("div"),p(dt.$$.fragment),Za=n(),Jt=l("p"),Jt.textContent=ur,Wa=n(),kt=l("p"),kt.textContent=fr,Ea=n(),D=l("div"),p(pt.$$.fragment),za=n(),Dt=l("p"),Dt.innerHTML=_r,Ha=n(),Ft=l("p"),Ft.textContent=br,Ba=n(),R=l("div"),p(ct.$$.fragment),Qa=n(),It=l("p"),It.innerHTML=yr,Va=n(),Z=l("div"),p(mt.$$.fragment),Ga=n(),Nt=l("p"),Nt.textContent=vr,Aa=n(),F=l("div"),p(ht.$$.fragment),Xa=n(),Lt=l("p"),Lt.innerHTML=wr,Sa=n(),qt=l("p"),qt.textContent=Tr,Ya=n(),W=l("div"),p(gt.$$.fragment),Ka=n(),Rt=l("p"),Rt.textContent=Mr,en=n(),E=l("div"),p(ut.$$.fragment),tn=n(),Zt=l("p"),Zt.textContent=xr,on=n(),I=l("div"),p(ft.$$.fragment),an=n(),Wt=l("p"),Wt.innerHTML=$r,nn=n(),Et=l("p"),Et.innerHTML=Cr,rn=n(),N=l("div"),p(_t.$$.fragment),ln=n(),zt=l("p"),zt.innerHTML=jr,sn=n(),Ht=l("p"),Ht.innerHTML=Pr,dn=n(),z=l("div"),p(bt.$$.fragment),pn=n(),Bt=l("p"),Bt.innerHTML=Ur,cn=n(),H=l("div"),p(yt.$$.fragment),mn=n(),Qt=l("p"),Qt.textContent=Or,hn=n(),B=l("div"),p(vt.$$.fragment),gn=n(),Vt=l("p"),Vt.innerHTML=Jr,un=n(),L=l("div"),p(wt.$$.fragment),fn=n(),Gt=l("p"),Gt.textContent=kr,_n=n(),p(Q.$$.fragment),Ta=n(),p(Tt.$$.fragment),Ma=n(),P=l("div"),p(Mt.$$.fragment),bn=n(),At=l("p"),At.innerHTML=Dr,yn=n(),Xt=l("p"),Xt.innerHTML=Fr,xa=n(),p(xt.$$.fragment),$a=n(),U=l("div"),p($t.$$.fragment),vn=n(),St=l("p"),St.textContent=Ir,wn=n(),p(V.$$.fragment),Ca=n(),p(Ct.$$.fragment),ja=n(),lo=l("p"),this.h()},l(e){const t=zr("svelte-u9bgzb",document.head);y=s(t,"META",{name:!0,content:!0}),t.forEach(o),O=r(e),C=s(e,"P",{}),T(C).forEach(o),w=r(e),c(x.$$.fragment,e),b=r(e),j=s(e,"P",{"data-svelte-h":!0}),d(j)!=="svelte-1p6l4u9"&&(j.innerHTML=Tn),so=r(e),c(A.$$.fragment,e),io=r(e),X=s(e,"P",{"data-svelte-h":!0}),d(X)!=="svelte-z3e1vo"&&(X.innerHTML=Mn),po=r(e),S=s(e,"P",{"data-svelte-h":!0}),d(S)!=="svelte-vfdo9a"&&(S.textContent=xn),co=r(e),Y=s(e,"BLOCKQUOTE",{"data-svelte-h":!0}),d(Y)!=="svelte-79s4co"&&(Y.innerHTML=$n),mo=r(e),K=s(e,"P",{"data-svelte-h":!0}),d(K)!=="svelte-nj0mw8"&&(K.textContent=Cn),ho=r(e),ee=s(e,"P",{"data-svelte-h":!0}),d(ee)!=="svelte-1shy0cc"&&(ee.innerHTML=jn),go=r(e),te=s(e,"OL",{"data-svelte-h":!0}),d(te)!=="svelte-191izpi"&&(te.innerHTML=Pn),uo=r(e),oe=s(e,"P",{"data-svelte-h":!0}),d(oe)!=="svelte-12mg2q5"&&(oe.innerHTML=Un),fo=r(e),ae=s(e,"P",{"data-svelte-h":!0}),d(ae)!=="svelte-1281kkc"&&(ae.innerHTML=On),_o=r(e),ne=s(e,"P",{"data-svelte-h":!0}),d(ne)!=="svelte-16acsjo"&&(ne.innerHTML=Jn),bo=r(e),c(re.$$.fragment,e),yo=r(e),le=s(e,"P",{"data-svelte-h":!0}),d(le)!=="svelte-hit9c8"&&(le.innerHTML=kn),vo=r(e),J=s(e,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0}),T(J).forEach(o),wo=r(e),se=s(e,"P",{"data-svelte-h":!0}),d(se)!=="svelte-uqytq6"&&(se.textContent=Fn),To=r(e),c(ie.$$.fragment,e),Mo=r(e),de=s(e,"P",{"data-svelte-h":!0}),d(de)!=="svelte-15hino8"&&(de.textContent=In),xo=r(e),c(pe.$$.fragment,e),$o=r(e),ce=s(e,"P",{"data-svelte-h":!0}),d(ce)!=="svelte-yajp6a"&&(ce.textContent=Nn),Co=r(e),me=s(e,"P",{"data-svelte-h":!0}),d(me)!=="svelte-6xl7ht"&&(me.innerHTML=Ln),jo=r(e),he=s(e,"P",{"data-svelte-h":!0}),d(he)!=="svelte-u1w0a8"&&(he.innerHTML=qn),Po=r(e),ge=s(e,"PRE",{"data-svelte-h":!0}),d(ge)!=="svelte-siqh0e"&&(ge.innerHTML=Rn),Uo=r(e),c(ue.$$.fragment,e),Oo=r(e),fe=s(e,"P",{"data-svelte-h":!0}),d(fe)!=="svelte-v9j10l"&&(fe.innerHTML=Zn),Jo=r(e),_e=s(e,"P",{"data-svelte-h":!0}),d(_e)!=="svelte-1v5f57c"&&(_e.innerHTML=Wn),ko=r(e),c(be.$$.fragment,e),Do=r(e),ye=s(e,"P",{"data-svelte-h":!0}),d(ye)!=="svelte-124rokg"&&(ye.innerHTML=En),Fo=r(e),ve=s(e,"P",{"data-svelte-h":!0}),d(ve)!=="svelte-1641qgj"&&(ve.innerHTML=zn),Io=r(e),c(we.$$.fragment,e),No=r(e),Te=s(e,"P",{"data-svelte-h":!0}),d(Te)!=="svelte-112nfxj"&&(Te.innerHTML=Hn),Lo=r(e),c(Me.$$.fragment,e),qo=r(e),xe=s(e,"P",{"data-svelte-h":!0}),d(xe)!=="svelte-9gpknh"&&(xe.innerHTML=Bn),Ro=r(e),$e=s(e,"P",{"data-svelte-h":!0}),d($e)!=="svelte-ob6zvc"&&($e.innerHTML=Qn),Zo=r(e),c(Ce.$$.fragment,e),Wo=r(e),c(je.$$.fragment,e),Eo=r(e),Pe=s(e,"P",{"data-svelte-h":!0}),d(Pe)!=="svelte-132s7j9"&&(Pe.textContent=Vn),zo=r(e),Ue=s(e,"UL",{"data-svelte-h":!0}),d(Ue)!=="svelte-1oji3mm"&&(Ue.innerHTML=Gn),Ho=r(e),c(Oe.$$.fragment,e),Bo=r(e),Je=s(e,"P",{"data-svelte-h":!0}),d(Je)!=="svelte-tmttss"&&(Je.innerHTML=An),Qo=r(e),ke=s(e,"TABLE",{"data-svelte-h":!0}),d(ke)!=="svelte-lx95ok"&&(ke.innerHTML=Xn),Vo=r(e),c(De.$$.fragment,e),Go=r(e),Fe=s(e,"P",{"data-svelte-h":!0}),d(Fe)!=="svelte-1t8obhw"&&(Fe.innerHTML=Sn),Ao=r(e),c(Ie.$$.fragment,e),Xo=r(e),Ne=s(e,"P",{"data-svelte-h":!0}),d(Ne)!=="svelte-1sifkns"&&(Ne.innerHTML=Yn),So=r(e),c(Le.$$.fragment,e),Yo=r(e),qe=s(e,"P",{"data-svelte-h":!0}),d(qe)!=="svelte-1wb7wca"&&(qe.innerHTML=Kn),Ko=r(e),c(Re.$$.fragment,e),ea=r(e),Ze=s(e,"P",{"data-svelte-h":!0}),d(Ze)!=="svelte-15e9h0w"&&(Ze.innerHTML=er),ta=r(e),c(We.$$.fragment,e),oa=r(e),Ee=s(e,"P",{"data-svelte-h":!0}),d(Ee)!=="svelte-14px1ga"&&(Ee.innerHTML=tr),aa=r(e),ze=s(e,"P",{"data-svelte-h":!0}),d(ze)!=="svelte-11it1aj"&&(ze.innerHTML=or),na=r(e),c(He.$$.fragment,e),ra=r(e),Be=s(e,"P",{"data-svelte-h":!0}),d(Be)!=="svelte-2145zu"&&(Be.innerHTML=ar),la=r(e),Qe=s(e,"TABLE",{"data-svelte-h":!0}),d(Qe)!=="svelte-435mua"&&(Qe.innerHTML=nr),sa=r(e),Ve=s(e,"P",{"data-svelte-h":!0}),d(Ve)!=="svelte-1p74c9x"&&(Ve.innerHTML=rr),ia=r(e),c(Ge.$$.fragment,e),da=r(e),Ae=s(e,"P",{"data-svelte-h":!0}),d(Ae)!=="svelte-1o18rwj"&&(Ae.innerHTML=lr),pa=r(e),c(Xe.$$.fragment,e),ca=r(e),Se=s(e,"P",{"data-svelte-h":!0}),d(Se)!=="svelte-4infz6"&&(Se.textContent=sr),ma=r(e),Ye=s(e,"OL",{"data-svelte-h":!0}),d(Ye)!=="svelte-4ytutj"&&(Ye.innerHTML=ir),ha=r(e),c(Ke.$$.fragment,e),ga=r(e),et=s(e,"P",{"data-svelte-h":!0}),d(et)!=="svelte-rqj7uy"&&(et.innerHTML=dr),ua=r(e),tt=s(e,"P",{"data-svelte-h":!0}),d(tt)!=="svelte-1xhv3st"&&(tt.textContent=pr),fa=r(e),c(ot.$$.fragment,e),_a=r(e),at=s(e,"P",{"data-svelte-h":!0}),d(at)!=="svelte-v40bvb"&&(at.innerHTML=cr),ba=r(e),nt=s(e,"P",{"data-svelte-h":!0}),d(nt)!=="svelte-1gkqha7"&&(nt.textContent=mr),ya=r(e),c(rt.$$.fragment,e),va=r(e),c(lt.$$.fragment,e),wa=r(e),f=s(e,"DIV",{class:!0});var _=T(f);c(st.$$.fragment,_),Na=r(_),Ut=s(_,"P",{"data-svelte-h":!0}),d(Ut)!=="svelte-n6cc8y"&&(Ut.textContent=hr),La=r(_),q=s(_,"DIV",{class:!0});var jt=T(q);c(it.$$.fragment,jt),qa=r(jt),Ot=s(jt,"P",{"data-svelte-h":!0}),d(Ot)!=="svelte-10t1062"&&(Ot.textContent=gr),jt.forEach(o),Ra=r(_),k=s(_,"DIV",{class:!0});var Yt=T(k);c(dt.$$.fragment,Yt),Za=r(Yt),Jt=s(Yt,"P",{"data-svelte-h":!0}),d(Jt)!=="svelte-16i1nac"&&(Jt.textContent=ur),Wa=r(Yt),kt=s(Yt,"P",{"data-svelte-h":!0}),d(kt)!=="svelte-tgti68"&&(kt.textContent=fr),Yt.forEach(o),Ea=r(_),D=s(_,"DIV",{class:!0});var Kt=T(D);c(pt.$$.fragment,Kt),za=r(Kt),Dt=s(Kt,"P",{"data-svelte-h":!0}),d(Dt)!=="svelte-1tqislr"&&(Dt.innerHTML=_r),Ha=r(Kt),Ft=s(Kt,"P",{"data-svelte-h":!0}),d(Ft)!=="svelte-1a2t2mf"&&(Ft.textContent=br),Kt.forEach(o),Ba=r(_),R=s(_,"DIV",{class:!0});var Ua=T(R);c(ct.$$.fragment,Ua),Qa=r(Ua),It=s(Ua,"P",{"data-svelte-h":!0}),d(It)!=="svelte-1mh859w"&&(It.innerHTML=yr),Ua.forEach(o),Va=r(_),Z=s(_,"DIV",{class:!0});var Oa=T(Z);c(mt.$$.fragment,Oa),Ga=r(Oa),Nt=s(Oa,"P",{"data-svelte-h":!0}),d(Nt)!=="svelte-ymgxq7"&&(Nt.textContent=vr),Oa.forEach(o),Aa=r(_),F=s(_,"DIV",{class:!0});var eo=T(F);c(ht.$$.fragment,eo),Xa=r(eo),Lt=s(eo,"P",{"data-svelte-h":!0}),d(Lt)!=="svelte-8qlty5"&&(Lt.innerHTML=wr),Sa=r(eo),qt=s(eo,"P",{"data-svelte-h":!0}),d(qt)!=="svelte-1tyo99t"&&(qt.textContent=Tr),eo.forEach(o),Ya=r(_),W=s(_,"DIV",{class:!0});var Ja=T(W);c(gt.$$.fragment,Ja),Ka=r(Ja),Rt=s(Ja,"P",{"data-svelte-h":!0}),d(Rt)!=="svelte-18uwvbi"&&(Rt.textContent=Mr),Ja.forEach(o),en=r(_),E=s(_,"DIV",{class:!0});var ka=T(E);c(ut.$$.fragment,ka),tn=r(ka),Zt=s(ka,"P",{"data-svelte-h":!0}),d(Zt)!=="svelte-11q8u7l"&&(Zt.textContent=xr),ka.forEach(o),on=r(_),I=s(_,"DIV",{class:!0});var to=T(I);c(ft.$$.fragment,to),an=r(to),Wt=s(to,"P",{"data-svelte-h":!0}),d(Wt)!=="svelte-xesobz"&&(Wt.innerHTML=$r),nn=r(to),Et=s(to,"P",{"data-svelte-h":!0}),d(Et)!=="svelte-qg43tm"&&(Et.innerHTML=Cr),to.forEach(o),rn=r(_),N=s(_,"DIV",{class:!0});var oo=T(N);c(_t.$$.fragment,oo),ln=r(oo),zt=s(oo,"P",{"data-svelte-h":!0}),d(zt)!=="svelte-dkae9b"&&(zt.innerHTML=jr),sn=r(oo),Ht=s(oo,"P",{"data-svelte-h":!0}),d(Ht)!=="svelte-hmd5cw"&&(Ht.innerHTML=Pr),oo.forEach(o),dn=r(_),z=s(_,"DIV",{class:!0});var Da=T(z);c(bt.$$.fragment,Da),pn=r(Da),Bt=s(Da,"P",{"data-svelte-h":!0}),d(Bt)!=="svelte-nqksfz"&&(Bt.innerHTML=Ur),Da.forEach(o),cn=r(_),H=s(_,"DIV",{class:!0});var Fa=T(H);c(yt.$$.fragment,Fa),mn=r(Fa),Qt=s(Fa,"P",{"data-svelte-h":!0}),d(Qt)!=="svelte-17kjsy4"&&(Qt.textContent=Or),Fa.forEach(o),hn=r(_),B=s(_,"DIV",{class:!0});var Ia=T(B);c(vt.$$.fragment,Ia),gn=r(Ia),Vt=s(Ia,"P",{"data-svelte-h":!0}),d(Vt)!=="svelte-ps14rv"&&(Vt.innerHTML=Jr),Ia.forEach(o),un=r(_),L=s(_,"DIV",{class:!0});var ao=T(L);c(wt.$$.fragment,ao),fn=r(ao),Gt=s(ao,"P",{"data-svelte-h":!0}),d(Gt)!=="svelte-dujibq"&&(Gt.textContent=kr),_n=r(ao),c(Q.$$.fragment,ao),ao.forEach(o),_.forEach(o),Ta=r(e),c(Tt.$$.fragment,e),Ma=r(e),P=s(e,"DIV",{class:!0});var no=T(P);c(Mt.$$.fragment,no),bn=r(no),At=s(no,"P",{"data-svelte-h":!0}),d(At)!=="svelte-u2mn4r"&&(At.innerHTML=Dr),yn=r(no),Xt=s(no,"P",{"data-svelte-h":!0}),d(Xt)!=="svelte-ekuf1t"&&(Xt.innerHTML=Fr),no.forEach(o),xa=r(e),c(xt.$$.fragment,e),$a=r(e),U=s(e,"DIV",{class:!0});var ro=T(U);c($t.$$.fragment,ro),vn=r(ro),St=s(ro,"P",{"data-svelte-h":!0}),d(St)!=="svelte-jiakry"&&(St.textContent=Ir),wn=r(ro),c(V.$$.fragment,ro),ro.forEach(o),Ca=r(e),c(Ct.$$.fragment,e),ja=r(e),lo=s(e,"P",{}),T(lo).forEach(o),this.h()},h(){v(y,"name","hf:doc:metadata"),v(y,"content",Gr),Rr(J.src,Dn="https://huggingface.co/datasets/trl-lib/ultrafeedback_binarized/embed/viewer/default/train?row=0")||v(J,"src",Dn),v(J,"frameborder","0"),v(J,"width","100%"),v(J,"height","560px"),v(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){i(document.head,y),a(e,O,t),a(e,C,t),a(e,w,t),m(x,e,t),a(e,b,t),a(e,j,t),a(e,so,t),m(A,e,t),a(e,io,t),a(e,X,t),a(e,po,t),a(e,S,t),a(e,co,t),a(e,Y,t),a(e,mo,t),a(e,K,t),a(e,ho,t),a(e,ee,t),a(e,go,t),a(e,te,t),a(e,uo,t),a(e,oe,t),a(e,fo,t),a(e,ae,t),a(e,_o,t),a(e,ne,t),a(e,bo,t),m(re,e,t),a(e,yo,t),a(e,le,t),a(e,vo,t),a(e,J,t),a(e,wo,t),a(e,se,t),a(e,To,t),m(ie,e,t),a(e,Mo,t),a(e,de,t),a(e,xo,t),m(pe,e,t),a(e,$o,t),a(e,ce,t),a(e,Co,t),a(e,me,t),a(e,jo,t),a(e,he,t),a(e,Po,t),a(e,ge,t),a(e,Uo,t),m(ue,e,t),a(e,Oo,t),a(e,fe,t),a(e,Jo,t),a(e,_e,t),a(e,ko,t),m(be,e,t),a(e,Do,t),a(e,ye,t),a(e,Fo,t),a(e,ve,t),a(e,Io,t),m(we,e,t),a(e,No,t),a(e,Te,t),a(e,Lo,t),m(Me,e,t),a(e,qo,t),a(e,xe,t),a(e,Ro,t),a(e,$e,t),a(e,Zo,t),m(Ce,e,t),a(e,Wo,t),m(je,e,t),a(e,Eo,t),a(e,Pe,t),a(e,zo,t),a(e,Ue,t),a(e,Ho,t),m(Oe,e,t),a(e,Bo,t),a(e,Je,t),a(e,Qo,t),a(e,ke,t),a(e,Vo,t),m(De,e,t),a(e,Go,t),a(e,Fe,t),a(e,Ao,t),m(Ie,e,t),a(e,Xo,t),a(e,Ne,t),a(e,So,t),m(Le,e,t),a(e,Yo,t),a(e,qe,t),a(e,Ko,t),m(Re,e,t),a(e,ea,t),a(e,Ze,t),a(e,ta,t),m(We,e,t),a(e,oa,t),a(e,Ee,t),a(e,aa,t),a(e,ze,t),a(e,na,t),m(He,e,t),a(e,ra,t),a(e,Be,t),a(e,la,t),a(e,Qe,t),a(e,sa,t),a(e,Ve,t),a(e,ia,t),m(Ge,e,t),a(e,da,t),a(e,Ae,t),a(e,pa,t),m(Xe,e,t),a(e,ca,t),a(e,Se,t),a(e,ma,t),a(e,Ye,t),a(e,ha,t),m(Ke,e,t),a(e,ga,t),a(e,et,t),a(e,ua,t),a(e,tt,t),a(e,fa,t),m(ot,e,t),a(e,_a,t),a(e,at,t),a(e,ba,t),a(e,nt,t),a(e,ya,t),m(rt,e,t),a(e,va,t),m(lt,e,t),a(e,wa,t),a(e,f,t),m(st,f,null),i(f,Na),i(f,Ut),i(f,La),i(f,q),m(it,q,null),i(q,qa),i(q,Ot),i(f,Ra),i(f,k),m(dt,k,null),i(k,Za),i(k,Jt),i(k,Wa),i(k,kt),i(f,Ea),i(f,D),m(pt,D,null),i(D,za),i(D,Dt),i(D,Ha),i(D,Ft),i(f,Ba),i(f,R),m(ct,R,null),i(R,Qa),i(R,It),i(f,Va),i(f,Z),m(mt,Z,null),i(Z,Ga),i(Z,Nt),i(f,Aa),i(f,F),m(ht,F,null),i(F,Xa),i(F,Lt),i(F,Sa),i(F,qt),i(f,Ya),i(f,W),m(gt,W,null),i(W,Ka),i(W,Rt),i(f,en),i(f,E),m(ut,E,null),i(E,tn),i(E,Zt),i(f,on),i(f,I),m(ft,I,null),i(I,an),i(I,Wt),i(I,nn),i(I,Et),i(f,rn),i(f,N),m(_t,N,null),i(N,ln),i(N,zt),i(N,sn),i(N,Ht),i(f,dn),i(f,z),m(bt,z,null),i(z,pn),i(z,Bt),i(f,cn),i(f,H),m(yt,H,null),i(H,mn),i(H,Qt),i(f,hn),i(f,B),m(vt,B,null),i(B,gn),i(B,Vt),i(f,un),i(f,L),m(wt,L,null),i(L,fn),i(L,Gt),i(L,_n),m(Q,L,null),a(e,Ta,t),m(Tt,e,t),a(e,Ma,t),a(e,P,t),m(Mt,P,null),i(P,bn),i(P,At),i(P,yn),i(P,Xt),a(e,xa,t),m(xt,e,t),a(e,$a,t),a(e,U,t),m($t,U,null),i(U,vn),i(U,St),i(U,wn),m(V,U,null),a(e,Ca,t),m(Ct,e,t),a(e,ja,t),a(e,lo,t),Pa=!0},p(e,[t]){const _={};t&2&&(_.$$scope={dirty:t,ctx:e}),Q.$set(_);const jt={};t&2&&(jt.$$scope={dirty:t,ctx:e}),V.$set(jt)},i(e){Pa||(h(x.$$.fragment,e),h(A.$$.fragment,e),h(re.$$.fragment,e),h(ie.$$.fragment,e),h(pe.$$.fragment,e),h(ue.$$.fragment,e),h(be.$$.fragment,e),h(we.$$.fragment,e),h(Me.$$.fragment,e),h(Ce.$$.fragment,e),h(je.$$.fragment,e),h(Oe.$$.fragment,e),h(De.$$.fragment,e),h(Ie.$$.fragment,e),h(Le.$$.fragment,e),h(Re.$$.fragment,e),h(We.$$.fragment,e),h(He.$$.fragment,e),h(Ge.$$.fragment,e),h(Xe.$$.fragment,e),h(Ke.$$.fragment,e),h(ot.$$.fragment,e),h(rt.$$.fragment,e),h(lt.$$.fragment,e),h(st.$$.fragment,e),h(it.$$.fragment,e),h(dt.$$.fragment,e),h(pt.$$.fragment,e),h(ct.$$.fragment,e),h(mt.$$.fragment,e),h(ht.$$.fragment,e),h(gt.$$.fragment,e),h(ut.$$.fragment,e),h(ft.$$.fragment,e),h(_t.$$.fragment,e),h(bt.$$.fragment,e),h(yt.$$.fragment,e),h(vt.$$.fragment,e),h(wt.$$.fragment,e),h(Q.$$.fragment,e),h(Tt.$$.fragment,e),h(Mt.$$.fragment,e),h(xt.$$.fragment,e),h($t.$$.fragment,e),h(V.$$.fragment,e),h(Ct.$$.fragment,e),Pa=!0)},o(e){g(x.$$.fragment,e),g(A.$$.fragment,e),g(re.$$.fragment,e),g(ie.$$.fragment,e),g(pe.$$.fragment,e),g(ue.$$.fragment,e),g(be.$$.fragment,e),g(we.$$.fragment,e),g(Me.$$.fragment,e),g(Ce.$$.fragment,e),g(je.$$.fragment,e),g(Oe.$$.fragment,e),g(De.$$.fragment,e),g(Ie.$$.fragment,e),g(Le.$$.fragment,e),g(Re.$$.fragment,e),g(We.$$.fragment,e),g(He.$$.fragment,e),g(Ge.$$.fragment,e),g(Xe.$$.fragment,e),g(Ke.$$.fragment,e),g(ot.$$.fragment,e),g(rt.$$.fragment,e),g(lt.$$.fragment,e),g(st.$$.fragment,e),g(it.$$.fragment,e),g(dt.$$.fragment,e),g(pt.$$.fragment,e),g(ct.$$.fragment,e),g(mt.$$.fragment,e),g(ht.$$.fragment,e),g(gt.$$.fragment,e),g(ut.$$.fragment,e),g(ft.$$.fragment,e),g(_t.$$.fragment,e),g(bt.$$.fragment,e),g(yt.$$.fragment,e),g(vt.$$.fragment,e),g(wt.$$.fragment,e),g(Q.$$.fragment,e),g(Tt.$$.fragment,e),g(Mt.$$.fragment,e),g(xt.$$.fragment,e),g($t.$$.fragment,e),g(V.$$.fragment,e),g(Ct.$$.fragment,e),Pa=!1},d(e){e&&(o(O),o(C),o(w),o(b),o(j),o(so),o(io),o(X),o(po),o(S),o(co),o(Y),o(mo),o(K),o(ho),o(ee),o(go),o(te),o(uo),o(oe),o(fo),o(ae),o(_o),o(ne),o(bo),o(yo),o(le),o(vo),o(J),o(wo),o(se),o(To),o(Mo),o(de),o(xo),o($o),o(ce),o(Co),o(me),o(jo),o(he),o(Po),o(ge),o(Uo),o(Oo),o(fe),o(Jo),o(_e),o(ko),o(Do),o(ye),o(Fo),o(ve),o(Io),o(No),o(Te),o(Lo),o(qo),o(xe),o(Ro),o($e),o(Zo),o(Wo),o(Eo),o(Pe),o(zo),o(Ue),o(Ho),o(Bo),o(Je),o(Qo),o(ke),o(Vo),o(Go),o(Fe),o(Ao),o(Xo),o(Ne),o(So),o(Yo),o(qe),o(Ko),o(ea),o(Ze),o(ta),o(oa),o(Ee),o(aa),o(ze),o(na),o(ra),o(Be),o(la),o(Qe),o(sa),o(Ve),o(ia),o(da),o(Ae),o(pa),o(ca),o(Se),o(ma),o(Ye),o(ha),o(ga),o(et),o(ua),o(tt),o(fa),o(_a),o(at),o(ba),o(nt),o(ya),o(va),o(wa),o(f),o(Ta),o(Ma),o(P),o(xa),o($a),o(U),o(Ca),o(ja),o(lo)),o(y),u(x,e),u(A,e),u(re,e),u(ie,e),u(pe,e),u(ue,e),u(be,e),u(we,e),u(Me,e),u(Ce,e),u(je,e),u(Oe,e),u(De,e),u(Ie,e),u(Le,e),u(Re,e),u(We,e),u(He,e),u(Ge,e),u(Xe,e),u(Ke,e),u(ot,e),u(rt,e),u(lt,e),u(st),u(it),u(dt),u(pt),u(ct),u(mt),u(ht),u(gt),u(ut),u(ft),u(_t),u(bt),u(yt),u(vt),u(wt),u(Q),u(Tt,e),u(Mt),u(xt,e),u($t),u(V),u(Ct,e)}}}const Gr='{"title":"DPO Trainer","local":"dpo-trainer","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Quick start","local":"quick-start","sections":[],"depth":2},{"title":"Expected dataset type","local":"expected-dataset-type","sections":[{"title":"Special considerations for vision-language models","local":"special-considerations-for-vision-language-models","sections":[],"depth":3}],"depth":2},{"title":"Example script","local":"example-script","sections":[],"depth":2},{"title":"Logged metrics","local":"logged-metrics","sections":[],"depth":2},{"title":"Loss functions","local":"loss-functions","sections":[{"title":"Label smoothing","local":"label-smoothing","sections":[],"depth":3},{"title":"Syncing the reference model","local":"syncing-the-reference-model","sections":[],"depth":3},{"title":"RPO loss","local":"rpo-loss","sections":[],"depth":3},{"title":"WPO loss","local":"wpo-loss","sections":[],"depth":3},{"title":"For Mixture of Experts Models: Enabling the auxiliary loss","local":"for-mixture-of-experts-models-enabling-the-auxiliary-loss","sections":[],"depth":3}],"depth":2},{"title":"Accelerate DPO fine-tuning using unsloth","local":"accelerate-dpo-fine-tuning-using-unsloth","sections":[],"depth":2},{"title":"Reference model considerations with PEFT","local":"reference-model-considerations-with-peft","sections":[{"title":"Downsides to merging QLoRA before DPO (approach 2)","local":"downsides-to-merging-qlora-before-dpo-approach-2","sections":[],"depth":3},{"title":"Using option 3 - load the adapter twice","local":"using-option-3---load-the-adapter-twice","sections":[],"depth":3}],"depth":2},{"title":"DPOTrainer","local":"trl.DPOTrainer","sections":[],"depth":2},{"title":"DPOConfig","local":"trl.DPOConfig","sections":[],"depth":2},{"title":"DataCollatorForPreference","local":"trl.trainer.dpo_trainer.DataCollatorForPreference","sections":[],"depth":2}],"depth":1}';function Ar(Pt){return Zr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ol extends Wr{constructor(y){super(),Er(this,y,Ar,Vr,qr,{})}}export{ol as component};
