import{s as Fn,f as Jr,o as Vn,n as Zr}from"../chunks/scheduler.9bc65507.js";import{S as Hn,i as Jn,g as a,s as o,u as f,A as Gn,h as n,f as t,c as s,j as p,v as g,y as h,k as c,o as r,a as l,w as _,d as w,t as v,x as b}from"../chunks/index.78c82d43.js";import{D as z}from"../chunks/Docstring.f3d04bd2.js";import{C as qr}from"../chunks/CodeBlock.48618ec2.js";import{I as Y}from"../chunks/IconCopyLink.bedaa44d.js";import{E as Gr}from"../chunks/ExampleCodeBlock.872b014d.js";function Zn(k){let d,M="Example:",x,u,y;return u=new qr({props:{code:"QWRhZmFjdG9yKG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMHNjYWxlX3BhcmFtZXRlciUzREZhbHNlJTJDJTIwcmVsYXRpdmVfc3RlcCUzREZhbHNlJTJDJTIwd2FybXVwX2luaXQlM0RGYWxzZSUyQyUyMGxyJTNEMWUtMyk=",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, warmup_init=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">1e-3</span>)'}}),{c(){d=a("p"),d.textContent=M,x=o(),f(u.$$.fragment)},l(m){d=n(m,"P",{"data-svelte-h":!0}),h(d)!=="svelte-11lpom8"&&(d.textContent=M),x=s(m),g(u.$$.fragment,m)},m(m,T){l(m,d,T),l(m,x,T),_(u,m,T),y=!0},p:Zr,i(m){y||(w(u.$$.fragment,m),y=!0)},o(m){v(u.$$.fragment,m),y=!1},d(m){m&&(t(d),t(x)),b(u,m)}}}function qn(k){let d,M="Others reported the following combination to work well:",x,u,y;return u=new qr({props:{code:"QWRhZmFjdG9yKG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMHNjYWxlX3BhcmFtZXRlciUzRFRydWUlMkMlMjByZWxhdGl2ZV9zdGVwJTNEVHJ1ZSUyQyUyMHdhcm11cF9pbml0JTNEVHJ1ZSUyQyUyMGxyJTNETm9uZSk=",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)'}}),{c(){d=a("p"),d.textContent=M,x=o(),f(u.$$.fragment)},l(m){d=n(m,"P",{"data-svelte-h":!0}),h(d)!=="svelte-mxeef1"&&(d.textContent=M),x=s(m),g(u.$$.fragment,m)},m(m,T){l(m,d,T),l(m,x,T),_(u,m,T),y=!0},p:Zr,i(m){y||(w(u.$$.fragment,m),y=!0)},o(m){v(u.$$.fragment,m),y=!1},d(m){m&&(t(d),t(x)),b(u,m)}}}function Bn(k){let d,M="scheduler as following:",x,u,y;return u=new qr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5vcHRpbWl6YXRpb24lMjBpbXBvcnQlMjBBZGFmYWN0b3IlMkMlMjBBZGFmYWN0b3JTY2hlZHVsZSUwQSUwQW9wdGltaXplciUyMCUzRCUyMEFkYWZhY3Rvcihtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMjBzY2FsZV9wYXJhbWV0ZXIlM0RUcnVlJTJDJTIwcmVsYXRpdmVfc3RlcCUzRFRydWUlMkMlMjB3YXJtdXBfaW5pdCUzRFRydWUlMkMlMjBsciUzRE5vbmUpJTBBbHJfc2NoZWR1bGVyJTIwJTNEJTIwQWRhZmFjdG9yU2NoZWR1bGUob3B0aW1pemVyKSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKC4uLiUyQyUyMG9wdGltaXplcnMlM0Qob3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGVyKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, AdafactorSchedule

optimizer = Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)
lr_scheduler = AdafactorSchedule(optimizer)
trainer = Trainer(..., optimizers=(optimizer, lr_scheduler))`}}),{c(){d=a("p"),d.textContent=M,x=o(),f(u.$$.fragment)},l(m){d=n(m,"P",{"data-svelte-h":!0}),h(d)!=="svelte-1eua222"&&(d.textContent=M),x=s(m),g(u.$$.fragment,m)},m(m,T){l(m,d,T),l(m,x,T),_(u,m,T),y=!0},p:Zr,i(m){y||(w(u.$$.fragment,m),y=!0)},o(m){v(u.$$.fragment,m),y=!1},d(m){m&&(t(d),t(x)),b(u,m)}}}function Xn(k){let d,M="Usage:",x,u,y;return u=new qr({props:{code:"JTIzJTIwcmVwbGFjZSUyMEFkYW1XJTIwd2l0aCUyMEFkYWZhY3RvciUwQW9wdGltaXplciUyMCUzRCUyMEFkYWZhY3RvciglMEElMjAlMjAlMjAlMjBtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMEElMjAlMjAlMjAlMjBsciUzRDFlLTMlMkMlMEElMjAlMjAlMjAlMjBlcHMlM0QoMWUtMzAlMkMlMjAxZS0zKSUyQyUwQSUyMCUyMCUyMCUyMGNsaXBfdGhyZXNob2xkJTNEMS4wJTJDJTBBJTIwJTIwJTIwJTIwZGVjYXlfcmF0ZSUzRC0wLjglMkMlMEElMjAlMjAlMjAlMjBiZXRhMSUzRE5vbmUlMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRfZGVjYXklM0QwLjAlMkMlMEElMjAlMjAlMjAlMjByZWxhdGl2ZV9zdGVwJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBzY2FsZV9wYXJhbWV0ZXIlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMHdhcm11cF9pbml0JTNERmFsc2UlMkMlMEEp",highlighted:`<span class="hljs-comment"># replace AdamW with Adafactor</span>
optimizer = Adafactor(
    model.parameters(),
    lr=<span class="hljs-number">1e-3</span>,
    eps=(<span class="hljs-number">1e-30</span>, <span class="hljs-number">1e-3</span>),
    clip_threshold=<span class="hljs-number">1.0</span>,
    decay_rate=-<span class="hljs-number">0.8</span>,
    beta1=<span class="hljs-literal">None</span>,
    weight_decay=<span class="hljs-number">0.0</span>,
    relative_step=<span class="hljs-literal">False</span>,
    scale_parameter=<span class="hljs-literal">False</span>,
    warmup_init=<span class="hljs-literal">False</span>,
)`}}),{c(){d=a("p"),d.textContent=M,x=o(),f(u.$$.fragment)},l(m){d=n(m,"P",{"data-svelte-h":!0}),h(d)!=="svelte-5wyjqd"&&(d.textContent=M),x=s(m),g(u.$$.fragment,m)},m(m,T){l(m,d,T),l(m,x,T),_(u,m,T),y=!0},p:Zr,i(m){y||(w(u.$$.fragment,m),y=!0)},o(m){v(u.$$.fragment,m),y=!1},d(m){m&&(t(d),t(x)),b(u,m)}}}function On(k){let d,M,x,u,y,m,T,fe,Br,Ye,Ra="Optimization",Zt,ge,Na="The <code>.optimization</code> module provides:",qt,_e,Sa="<li>an optimizer with weight decay fixed that can be used to fine-tuned models, and</li> <li>several schedules in the form of schedule objects that inherit from <code>_LRSchedule</code>:</li> <li>a gradient accumulation class to accumulate the gradients of multiple batches</li>",Bt,E,Q,Dt,we,Xr,Qe,Ia="AdamW (PyTorch)",Xt,W,ve,Or,Ke,Fa=`Implements Adam algorithm with weight decay fix as introduced in <a href="https://arxiv.org/abs/1711.05101" rel="nofollow">Decoupled Weight Decay
Regularization</a>.`,Yr,K,be,Qr,et,Va="Performs a single optimization step.",Ot,U,ee,Pt,ye,Kr,tt,Ha="AdaFactor (PyTorch)",Yt,$,$e,ea,rt,Ja=`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
<a href="https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py" rel="nofollow">https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py</a>`,ta,at,Ga=`Paper: <em>Adafactor: Adaptive Learning Rates with Sublinear Memory Cost</em> <a href="https://arxiv.org/abs/1804.04235" rel="nofollow">https://arxiv.org/abs/1804.04235</a> Note that
this optimizer internally adjusts the learning rate depending on the <code>scale_parameter</code>, <code>relative_step</code> and
<code>warmup_init</code> options. To use a manual (external) learning rate schedule you should set <code>scale_parameter=False</code> and
<code>relative_step=False</code>.`,ra,nt,Za="This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested.",aa,ot,qa='Recommended T5 finetuning settings (<a href="https://discuss.huggingface.co/t/t5-finetuning-tips/684/3" rel="nofollow">https://discuss.huggingface.co/t/t5-finetuning-tips/684/3</a>):',na,st,Ba='<li><p>Training without LR warmup or clip_threshold is not recommended.</p> <ul><li>use scheduled LR warm-up to fixed LR</li> <li>use clip_threshold=1.0 (<a href="https://arxiv.org/abs/1804.04235" rel="nofollow">https://arxiv.org/abs/1804.04235</a>)</li></ul></li> <li><p>Disable relative updates</p></li> <li><p>Use scale_parameter=False</p></li> <li><p>Additional optimizer operations like gradient clipping should not be used alongside Adafactor</p></li>',oa,te,sa,re,ia,it,Xa='When using <code>lr=None</code> with <a href="/docs/transformers/v4.33.3/en/main_classes/trainer#transformers.Trainer">Trainer</a> you will most likely need to use <code>AdafactorSchedule</code>',la,ae,ma,ne,ca,oe,xe,pa,lt,Oa="Performs a single optimization step",Qt,D,se,Rt,Ae,da,mt,Ya="AdamWeightDecay (TensorFlow)",Kt,C,Te,ha,ct,Qa=`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is <em>not</em> the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in <a href="https://arxiv.org/abs/1711.05101" rel="nofollow">Decoupled Weight Decay
Regularization</a>.`,ua,pt,Ka=`Instead we want to decay the weights in a manner that doesn’t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`,fa,ie,ze,ga,dt,en="Creates an optimizer from its config with WarmUp custom object.",er,P,Me,_a,ht,tn="Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay.",tr,R,le,Nt,Ce,wa,ut,rn="Schedules",rr,N,me,St,We,va,ft,an="Learning Rate Schedules (Pytorch)",ar,S,je,ba,gt,nn="An enumeration.",nr,I,Le,ya,_t,on="Unified API to get any scheduler from its name.",or,F,ke,$a,wt,sn="Create a schedule with a constant learning rate, using the learning rate set in optimizer.",sr,V,Ee,xa,vt,ln=`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`,ir,Ue,mn,lr,H,De,Aa,bt,cn=`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`,mr,Pe,pn,cr,J,Re,Ta,yt,dn=`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`,pr,Ne,hn,dr,G,Se,za,$t,un=`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`,hr,Ie,fn,ur,j,Fe,Ma,xt,gn=`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by <em>lr_end</em>, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`,Ca,At,_n=`Note: <em>power</em> defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
<a href="https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37" rel="nofollow">https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37</a>`,fr,Z,Ve,Wa,Tt,wn=`Create a schedule with an inverse square-root learning rate, from the initial lr set in the optimizer, after a
warmup period which increases lr linearly from 0 to the initial lr set in the optimizer.`,gr,q,ce,It,He,ja,zt,vn="Warmup (TensorFlow)",_r,B,Je,La,Mt,bn="Applies a warmup schedule on a given learning rate decay schedule.",wr,X,pe,Ft,Ge,ka,Ct,yn="Gradient Strategies",vr,O,de,Vt,Ze,Ea,Wt,$n="GradientAccumulator (TensorFlow)",br,L,qe,Ua,jt,xn=`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call <code>.gradients</code>, scale the gradients if required, and pass the result to <code>apply_gradients</code>.`,Da,he,Be,Pa,Lt,An="Resets the accumulated gradients on the current replica.",yr,Ht,$r;return fe=new Y({}),we=new Y({}),ve=new z({props:{name:"class transformers.AdamW",anchor:"transformers.AdamW",parameters:[{name:"params",val:": typing.Iterable[torch.nn.parameter.Parameter]"},{name:"lr",val:": float = 0.001"},{name:"betas",val:": typing.Tuple[float, float] = (0.9, 0.999)"},{name:"eps",val:": float = 1e-06"},{name:"weight_decay",val:": float = 0.0"},{name:"correct_bias",val:": bool = True"},{name:"no_deprecation_warning",val:": bool = False"}],parametersDescription:[{anchor:"transformers.AdamW.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.AdamW.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-3) &#x2014;
The learning rate to use.`,name:"lr"},{anchor:"transformers.AdamW.betas",description:`<strong>betas</strong> (<code>Tuple[float,float]</code>, <em>optional</em>, defaults to (0.9, 0.999)) &#x2014;
Adam&#x2019;s betas parameters (b1, b2).`,name:"betas"},{anchor:"transformers.AdamW.eps",description:`<strong>eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-6) &#x2014;
Adam&#x2019;s epsilon for numerical stability.`,name:"eps"},{anchor:"transformers.AdamW.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
Decoupled weight decay to apply.`,name:"weight_decay"},{anchor:"transformers.AdamW.correct_bias",description:`<strong>correct_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to correct bias in Adam (for instance, in Bert TF repository they use <code>False</code>).`,name:"correct_bias"},{anchor:"transformers.AdamW.no_deprecation_warning",description:`<strong>no_deprecation_warning</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
A flag used to disable the deprecation warning (set to <code>True</code> to disable the warning).`,name:"no_deprecation_warning"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L378"}}),be=new z({props:{name:"step",anchor:"transformers.AdamW.step",parameters:[{name:"closure",val:": typing.Callable = None"}],parametersDescription:[{anchor:"transformers.AdamW.step.closure",description:"<strong>closure</strong> (<code>Callable</code>, <em>optional</em>) &#x2014; A closure that reevaluates the model and returns the loss.",name:"closure"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L429"}}),ye=new Y({}),$e=new z({props:{name:"class transformers.Adafactor",anchor:"transformers.Adafactor",parameters:[{name:"params",val:""},{name:"lr",val:" = None"},{name:"eps",val:" = (1e-30, 0.001)"},{name:"clip_threshold",val:" = 1.0"},{name:"decay_rate",val:" = -0.8"},{name:"beta1",val:" = None"},{name:"weight_decay",val:" = 0.0"},{name:"scale_parameter",val:" = True"},{name:"relative_step",val:" = True"},{name:"warmup_init",val:" = False"}],parametersDescription:[{anchor:"transformers.Adafactor.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.Adafactor.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>) &#x2014;
The external learning rate.`,name:"lr"},{anchor:"transformers.Adafactor.eps",description:`<strong>eps</strong> (<code>Tuple[float, float]</code>, <em>optional</em>, defaults to (1e-30, 1e-3)) &#x2014;
Regularization constants for square gradient and parameter scale respectively`,name:"eps"},{anchor:"transformers.Adafactor.clip_threshold",description:`<strong>clip_threshold</strong> (<code>float</code>, <em>optional</em>, defaults 1.0) &#x2014;
Threshold of root mean square of final gradient update`,name:"clip_threshold"},{anchor:"transformers.Adafactor.decay_rate",description:`<strong>decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to -0.8) &#x2014;
Coefficient used to compute running averages of square`,name:"decay_rate"},{anchor:"transformers.Adafactor.beta1",description:`<strong>beta1</strong> (<code>float</code>, <em>optional</em>) &#x2014;
Coefficient used for computing running averages of gradient`,name:"beta1"},{anchor:"transformers.Adafactor.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
Weight decay (L2 penalty)`,name:"weight_decay"},{anchor:"transformers.Adafactor.scale_parameter",description:`<strong>scale_parameter</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, learning rate is scaled by root mean square`,name:"scale_parameter"},{anchor:"transformers.Adafactor.relative_step",description:`<strong>relative_step</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, time-dependent learning rate is computed instead of external learning rate`,name:"relative_step"},{anchor:"transformers.Adafactor.warmup_init",description:`<strong>warmup_init</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Time-dependent learning rate computation depends on whether warm-up initialization is being used`,name:"warmup_init"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L492"}}),te=new Gr({props:{anchor:"transformers.Adafactor.example",$$slots:{default:[Zn]},$$scope:{ctx:k}}}),re=new Gr({props:{anchor:"transformers.Adafactor.example-2",$$slots:{default:[qn]},$$scope:{ctx:k}}}),ae=new Gr({props:{anchor:"transformers.Adafactor.example-3",$$slots:{default:[Bn]},$$scope:{ctx:k}}}),ne=new Gr({props:{anchor:"transformers.Adafactor.example-4",$$slots:{default:[Xn]},$$scope:{ctx:k}}}),xe=new z({props:{name:"step",anchor:"transformers.Adafactor.step",parameters:[{name:"closure",val:" = None"}],parametersDescription:[{anchor:"transformers.Adafactor.step.closure",description:`<strong>closure</strong> (callable, optional) &#x2014; A closure that reevaluates the model
and returns the loss.`,name:"closure"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L638"}}),Ae=new Y({}),Te=new z({props:{name:"class transformers.AdamWeightDecay",anchor:"transformers.AdamWeightDecay",parameters:[{name:"learning_rate",val:": typing.Union[float, keras.src.optimizers.schedules.learning_rate_schedule.LearningRateSchedule] = 0.001"},{name:"beta_1",val:": float = 0.9"},{name:"beta_2",val:": float = 0.999"},{name:"epsilon",val:": float = 1e-07"},{name:"amsgrad",val:": bool = False"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"exclude_from_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"name",val:": str = 'AdamWeightDecay'"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.AdamWeightDecay.learning_rate",description:`<strong>learning_rate</strong> (<code>Union[float, tf.keras.optimizers.schedules.LearningRateSchedule]</code>, <em>optional</em>, defaults to 1e-3) &#x2014;
The learning rate to use or a schedule.`,name:"learning_rate"},{anchor:"transformers.AdamWeightDecay.beta_1",description:`<strong>beta_1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 parameter in Adam, which is the exponential decay rate for the 1st momentum estimates.`,name:"beta_1"},{anchor:"transformers.AdamWeightDecay.beta_2",description:`<strong>beta_2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 parameter in Adam, which is the exponential decay rate for the 2nd momentum estimates.`,name:"beta_2"},{anchor:"transformers.AdamWeightDecay.epsilon",description:`<strong>epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The epsilon parameter in Adam, which is a small constant for numerical stability.`,name:"epsilon"},{anchor:"transformers.AdamWeightDecay.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, <em>optional</em>, default to <code>False</code>) &#x2014;
Whether to apply AMSGrad variant of this algorithm or not, see <a href="https://arxiv.org/abs/1904.09237" rel="nofollow">On the Convergence of Adam and
Beyond</a>.`,name:"amsgrad"},{anchor:"transformers.AdamWeightDecay.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to apply.`,name:"weight_decay_rate"},{anchor:"transformers.AdamWeightDecay.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters by default (unless they are in <code>exclude_from_weight_decay</code>).`,name:"include_in_weight_decay"},{anchor:"transformers.AdamWeightDecay.exclude_from_weight_decay",description:`<strong>exclude_from_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to exclude from applying weight decay to. If a
<code>include_in_weight_decay</code> is passed, the names in it will supersede this list.`,name:"exclude_from_weight_decay"},{anchor:"transformers.AdamWeightDecay.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>, defaults to &#x2018;AdamWeightDecay&#x2019;) &#x2014;
Optional name for the operations created when applying gradients.`,name:"name"},{anchor:"transformers.AdamWeightDecay.kwargs",description:`<strong>kwargs</strong> (<code>Dict[str, Any]</code>, <em>optional</em>) &#x2014;
Keyword arguments. Allowed to be {<code>clipnorm</code>, <code>clipvalue</code>, <code>lr</code>, <code>decay</code>}. <code>clipnorm</code> is clip gradients by
norm; <code>clipvalue</code> is clip gradients by value, <code>decay</code> is included for backward compatibility to allow time
inverse decay of learning rate. <code>lr</code> is included for backward compatibility, recommended to use
<code>learning_rate</code> instead.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization_tf.py#L172"}}),ze=new z({props:{name:"from_config",anchor:"transformers.AdamWeightDecay.from_config",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization_tf.py#L229"}}),Me=new z({props:{name:"transformers.create_optimizer",anchor:"transformers.create_optimizer",parameters:[{name:"init_lr",val:": float"},{name:"num_train_steps",val:": int"},{name:"num_warmup_steps",val:": int"},{name:"min_lr_ratio",val:": float = 0.0"},{name:"adam_beta1",val:": float = 0.9"},{name:"adam_beta2",val:": float = 0.999"},{name:"adam_epsilon",val:": float = 1e-08"},{name:"adam_clipnorm",val:": typing.Optional[float] = None"},{name:"adam_global_clipnorm",val:": typing.Optional[float] = None"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"power",val:": float = 1.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"}],parametersDescription:[{anchor:"transformers.create_optimizer.init_lr",description:`<strong>init_lr</strong> (<code>float</code>) &#x2014;
The desired learning rate at the end of the warmup phase.`,name:"init_lr"},{anchor:"transformers.create_optimizer.num_train_steps",description:`<strong>num_train_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_train_steps"},{anchor:"transformers.create_optimizer.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of warmup steps.`,name:"num_warmup_steps"},{anchor:"transformers.create_optimizer.min_lr_ratio",description:`<strong>min_lr_ratio</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The final learning rate at the end of the linear decay will be <code>init_lr * min_lr_ratio</code>.`,name:"min_lr_ratio"},{anchor:"transformers.create_optimizer.adam_beta1",description:`<strong>adam_beta1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 to use in Adam.`,name:"adam_beta1"},{anchor:"transformers.create_optimizer.adam_beta2",description:`<strong>adam_beta2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 to use in Adam.`,name:"adam_beta2"},{anchor:"transformers.create_optimizer.adam_epsilon",description:`<strong>adam_epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-8) &#x2014;
The epsilon to use in Adam.`,name:"adam_epsilon"},{anchor:"transformers.create_optimizer.adam_clipnorm",description:`<strong>adam_clipnorm</strong> (<code>float</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
If not <code>None</code>, clip the gradient norm for each weight tensor to this value.`,name:"adam_clipnorm"},{anchor:"transformers.create_optimizer.adam_global_clipnorm",description:`<strong>adam_global_clipnorm</strong> (<code>float</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
If not <code>None</code>, clip gradient norm to this value. When using this argument, the norm is computed over all
weight tensors, as if they were concatenated into a single vector.`,name:"adam_global_clipnorm"},{anchor:"transformers.create_optimizer.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to use.`,name:"weight_decay_rate"},{anchor:"transformers.create_optimizer.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The power to use for PolynomialDecay.`,name:"power"},{anchor:"transformers.create_optimizer.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters except bias and layer norm parameters.`,name:"include_in_weight_decay"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization_tf.py#L88"}}),Ce=new Y({}),We=new Y({}),je=new z({props:{name:"class transformers.SchedulerType",anchor:"transformers.SchedulerType",parameters:[{name:"value",val:""},{name:"names",val:" = None"},{name:"module",val:" = None"},{name:"qualname",val:" = None"},{name:"type",val:" = None"},{name:"start",val:" = 1"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/trainer_utils.py#L360"}}),Le=new z({props:{name:"transformers.get_scheduler",anchor:"transformers.get_scheduler",parameters:[{name:"name",val:": typing.Union[str, transformers.trainer_utils.SchedulerType]"},{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": typing.Optional[int] = None"},{name:"num_training_steps",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"transformers.get_scheduler.name",description:`<strong>name</strong> (<code>str</code> or <code>SchedulerType</code>) &#x2014;
The name of the scheduler to use.`,name:"name"},{anchor:"transformers.get_scheduler.optimizer",description:`<strong>optimizer</strong> (<code>torch.optim.Optimizer</code>) &#x2014;
The optimizer that will be used during training.`,name:"optimizer"},{anchor:"transformers.get_scheduler.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The number of warmup steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_warmup_steps"},{anchor:"transformers.get_scheduler.num_training_steps",description:`<strong>num_training_steps</strong> (\`int&#x201C;, <em>optional</em>) &#x2014;
The number of training steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_training_steps"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L335"}}),ke=new z({props:{name:"transformers.get_constant_schedule",anchor:"transformers.get_constant_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L39",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Ee=new z({props:{name:"transformers.get_constant_schedule_with_warmup",anchor:"transformers.get_constant_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_constant_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L77",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),De=new z({props:{name:"transformers.get_cosine_schedule_with_warmup",anchor:"transformers.get_cosine_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": float = 0.5"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>float</code>, <em>optional</em>, defaults to 0.5) &#x2014;
The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0
following a half-cosine).`,name:"num_cycles"},{anchor:"transformers.get_cosine_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L140",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Re=new z({props:{name:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": int = 1"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of hard restarts to use.`,name:"num_cycles"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L185",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Se=new z({props:{name:"transformers.get_linear_schedule_with_warmup",anchor:"transformers.get_linear_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_linear_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_linear_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L104",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Fe=new z({props:{name:"transformers.get_polynomial_decay_schedule_with_warmup",anchor:"transformers.get_polynomial_decay_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"lr_end",val:" = 1e-07"},{name:"power",val:" = 1.0"},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.lr_end",description:`<strong>lr_end</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The end LR.`,name:"lr_end"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
Power factor.`,name:"power"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L239",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Ve=new z({props:{name:"transformers.get_inverse_sqrt_schedule",anchor:"transformers.get_inverse_sqrt_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"timescale",val:": int = None"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_inverse_sqrt_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_inverse_sqrt_schedule.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_inverse_sqrt_schedule.timescale",description:`<strong>timescale</strong> (<code>int</code>, <em>optional</em>, defaults to <code>num_warmup_steps</code>) &#x2014;
Time scale.`,name:"timescale"},{anchor:"transformers.get_inverse_sqrt_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization.py#L293",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),He=new Y({}),Je=new z({props:{name:"class transformers.WarmUp",anchor:"transformers.WarmUp",parameters:[{name:"initial_learning_rate",val:": float"},{name:"decay_schedule_fn",val:": typing.Callable"},{name:"warmup_steps",val:": int"},{name:"power",val:": float = 1.0"},{name:"name",val:": str = None"}],parametersDescription:[{anchor:"transformers.WarmUp.initial_learning_rate",description:`<strong>initial_learning_rate</strong> (<code>float</code>) &#x2014;
The initial learning rate for the schedule after the warmup (so this will be the learning rate at the end
of the warmup).`,name:"initial_learning_rate"},{anchor:"transformers.WarmUp.decay_schedule_fn",description:`<strong>decay_schedule_fn</strong> (<code>Callable</code>) &#x2014;
The schedule function to apply after the warmup for the rest of training.`,name:"decay_schedule_fn"},{anchor:"transformers.WarmUp.warmup_steps",description:`<strong>warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup part of training.`,name:"warmup_steps"},{anchor:"transformers.WarmUp.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1) &#x2014;
The power to use for the polynomial warmup (defaults is a linear warmup).`,name:"power"},{anchor:"transformers.WarmUp.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Optional name prefix for the returned tensors during the schedule.`,name:"name"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization_tf.py#L30"}}),Ge=new Y({}),Ze=new Y({}),qe=new z({props:{name:"class transformers.GradientAccumulator",anchor:"transformers.GradientAccumulator",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization_tf.py#L302"}}),Be=new z({props:{name:"reset",anchor:"transformers.GradientAccumulator.reset",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/optimization_tf.py#L364"}}),{c(){d=a("meta"),M=o(),x=a("p"),u=o(),y=a("h1"),m=a("a"),T=a("span"),f(fe.$$.fragment),Br=o(),Ye=a("span"),Ye.textContent=Ra,Zt=o(),ge=a("p"),ge.innerHTML=Na,qt=o(),_e=a("ul"),_e.innerHTML=Sa,Bt=o(),E=a("h2"),Q=a("a"),Dt=a("span"),f(we.$$.fragment),Xr=o(),Qe=a("span"),Qe.textContent=Ia,Xt=o(),W=a("div"),f(ve.$$.fragment),Or=o(),Ke=a("p"),Ke.innerHTML=Fa,Yr=o(),K=a("div"),f(be.$$.fragment),Qr=o(),et=a("p"),et.textContent=Va,Ot=o(),U=a("h2"),ee=a("a"),Pt=a("span"),f(ye.$$.fragment),Kr=o(),tt=a("span"),tt.textContent=Ha,Yt=o(),$=a("div"),f($e.$$.fragment),ea=o(),rt=a("p"),rt.innerHTML=Ja,ta=o(),at=a("p"),at.innerHTML=Ga,ra=o(),nt=a("p"),nt.textContent=Za,aa=o(),ot=a("p"),ot.innerHTML=qa,na=o(),st=a("ul"),st.innerHTML=Ba,oa=o(),f(te.$$.fragment),sa=o(),f(re.$$.fragment),ia=o(),it=a("p"),it.innerHTML=Xa,la=o(),f(ae.$$.fragment),ma=o(),f(ne.$$.fragment),ca=o(),oe=a("div"),f(xe.$$.fragment),pa=o(),lt=a("p"),lt.textContent=Oa,Qt=o(),D=a("h2"),se=a("a"),Rt=a("span"),f(Ae.$$.fragment),da=o(),mt=a("span"),mt.textContent=Ya,Kt=o(),C=a("div"),f(Te.$$.fragment),ha=o(),ct=a("p"),ct.innerHTML=Qa,ua=o(),pt=a("p"),pt.textContent=Ka,fa=o(),ie=a("div"),f(ze.$$.fragment),ga=o(),dt=a("p"),dt.textContent=en,er=o(),P=a("div"),f(Me.$$.fragment),_a=o(),ht=a("p"),ht.textContent=tn,tr=o(),R=a("h2"),le=a("a"),Nt=a("span"),f(Ce.$$.fragment),wa=o(),ut=a("span"),ut.textContent=rn,rr=o(),N=a("h3"),me=a("a"),St=a("span"),f(We.$$.fragment),va=o(),ft=a("span"),ft.textContent=an,ar=o(),S=a("div"),f(je.$$.fragment),ba=o(),gt=a("p"),gt.textContent=nn,nr=o(),I=a("div"),f(Le.$$.fragment),ya=o(),_t=a("p"),_t.textContent=on,or=o(),F=a("div"),f(ke.$$.fragment),$a=o(),wt=a("p"),wt.textContent=sn,sr=o(),V=a("div"),f(Ee.$$.fragment),xa=o(),vt=a("p"),vt.textContent=ln,ir=o(),Ue=a("img"),lr=o(),H=a("div"),f(De.$$.fragment),Aa=o(),bt=a("p"),bt.textContent=cn,mr=o(),Pe=a("img"),cr=o(),J=a("div"),f(Re.$$.fragment),Ta=o(),yt=a("p"),yt.textContent=dn,pr=o(),Ne=a("img"),dr=o(),G=a("div"),f(Se.$$.fragment),za=o(),$t=a("p"),$t.textContent=un,hr=o(),Ie=a("img"),ur=o(),j=a("div"),f(Fe.$$.fragment),Ma=o(),xt=a("p"),xt.innerHTML=gn,Ca=o(),At=a("p"),At.innerHTML=_n,fr=o(),Z=a("div"),f(Ve.$$.fragment),Wa=o(),Tt=a("p"),Tt.textContent=wn,gr=o(),q=a("h3"),ce=a("a"),It=a("span"),f(He.$$.fragment),ja=o(),zt=a("span"),zt.textContent=vn,_r=o(),B=a("div"),f(Je.$$.fragment),La=o(),Mt=a("p"),Mt.textContent=bn,wr=o(),X=a("h2"),pe=a("a"),Ft=a("span"),f(Ge.$$.fragment),ka=o(),Ct=a("span"),Ct.textContent=yn,vr=o(),O=a("h3"),de=a("a"),Vt=a("span"),f(Ze.$$.fragment),Ea=o(),Wt=a("span"),Wt.textContent=$n,br=o(),L=a("div"),f(qe.$$.fragment),Ua=o(),jt=a("p"),jt.innerHTML=xn,Da=o(),he=a("div"),f(Be.$$.fragment),Pa=o(),Lt=a("p"),Lt.textContent=An,yr=o(),Ht=a("p"),this.h()},l(e){const i=Gn("svelte-1phssyn",document.head);d=n(i,"META",{name:!0,content:!0}),i.forEach(t),M=s(e),x=n(e,"P",{}),p(x).forEach(t),u=s(e),y=n(e,"H1",{class:!0});var Xe=p(y);m=n(Xe,"A",{id:!0,class:!0,href:!0});var Jt=p(m);T=n(Jt,"SPAN",{});var Gt=p(T);g(fe.$$.fragment,Gt),Gt.forEach(t),Jt.forEach(t),Br=s(Xe),Ye=n(Xe,"SPAN",{"data-svelte-h":!0}),h(Ye)!=="svelte-1jvrjc2"&&(Ye.textContent=Ra),Xe.forEach(t),Zt=s(e),ge=n(e,"P",{"data-svelte-h":!0}),h(ge)!=="svelte-312sfx"&&(ge.innerHTML=Na),qt=s(e),_e=n(e,"UL",{"data-svelte-h":!0}),h(_e)!=="svelte-ps1e52"&&(_e.innerHTML=Sa),Bt=s(e),E=n(e,"H2",{class:!0});var Oe=p(E);Q=n(Oe,"A",{id:!0,class:!0,href:!0});var Tn=p(Q);Dt=n(Tn,"SPAN",{});var zn=p(Dt);g(we.$$.fragment,zn),zn.forEach(t),Tn.forEach(t),Xr=s(Oe),Qe=n(Oe,"SPAN",{"data-svelte-h":!0}),h(Qe)!=="svelte-1e8kjch"&&(Qe.textContent=Ia),Oe.forEach(t),Xt=s(e),W=n(e,"DIV",{class:!0});var kt=p(W);g(ve.$$.fragment,kt),Or=s(kt),Ke=n(kt,"P",{"data-svelte-h":!0}),h(Ke)!=="svelte-b86q21"&&(Ke.innerHTML=Fa),Yr=s(kt),K=n(kt,"DIV",{class:!0});var xr=p(K);g(be.$$.fragment,xr),Qr=s(xr),et=n(xr,"P",{"data-svelte-h":!0}),h(et)!=="svelte-1gr03w4"&&(et.textContent=Va),xr.forEach(t),kt.forEach(t),Ot=s(e),U=n(e,"H2",{class:!0});var Ar=p(U);ee=n(Ar,"A",{id:!0,class:!0,href:!0});var Mn=p(ee);Pt=n(Mn,"SPAN",{});var Cn=p(Pt);g(ye.$$.fragment,Cn),Cn.forEach(t),Mn.forEach(t),Kr=s(Ar),tt=n(Ar,"SPAN",{"data-svelte-h":!0}),h(tt)!=="svelte-3dh1pa"&&(tt.textContent=Ha),Ar.forEach(t),Yt=s(e),$=n(e,"DIV",{class:!0});var A=p($);g($e.$$.fragment,A),ea=s(A),rt=n(A,"P",{"data-svelte-h":!0}),h(rt)!=="svelte-114ll6r"&&(rt.innerHTML=Ja),ta=s(A),at=n(A,"P",{"data-svelte-h":!0}),h(at)!=="svelte-9bvrpy"&&(at.innerHTML=Ga),ra=s(A),nt=n(A,"P",{"data-svelte-h":!0}),h(nt)!=="svelte-1dxk71o"&&(nt.textContent=Za),aa=s(A),ot=n(A,"P",{"data-svelte-h":!0}),h(ot)!=="svelte-kc6644"&&(ot.innerHTML=qa),na=s(A),st=n(A,"UL",{"data-svelte-h":!0}),h(st)!=="svelte-p81jc"&&(st.innerHTML=Ba),oa=s(A),g(te.$$.fragment,A),sa=s(A),g(re.$$.fragment,A),ia=s(A),it=n(A,"P",{"data-svelte-h":!0}),h(it)!=="svelte-1ys599r"&&(it.innerHTML=Xa),la=s(A),g(ae.$$.fragment,A),ma=s(A),g(ne.$$.fragment,A),ca=s(A),oe=n(A,"DIV",{class:!0});var Tr=p(oe);g(xe.$$.fragment,Tr),pa=s(Tr),lt=n(Tr,"P",{"data-svelte-h":!0}),h(lt)!=="svelte-6lkj7q"&&(lt.textContent=Oa),Tr.forEach(t),A.forEach(t),Qt=s(e),D=n(e,"H2",{class:!0});var zr=p(D);se=n(zr,"A",{id:!0,class:!0,href:!0});var Wn=p(se);Rt=n(Wn,"SPAN",{});var jn=p(Rt);g(Ae.$$.fragment,jn),jn.forEach(t),Wn.forEach(t),da=s(zr),mt=n(zr,"SPAN",{"data-svelte-h":!0}),h(mt)!=="svelte-155xbpi"&&(mt.textContent=Ya),zr.forEach(t),Kt=s(e),C=n(e,"DIV",{class:!0});var ue=p(C);g(Te.$$.fragment,ue),ha=s(ue),ct=n(ue,"P",{"data-svelte-h":!0}),h(ct)!=="svelte-towids"&&(ct.innerHTML=Qa),ua=s(ue),pt=n(ue,"P",{"data-svelte-h":!0}),h(pt)!=="svelte-id029h"&&(pt.textContent=Ka),fa=s(ue),ie=n(ue,"DIV",{class:!0});var Mr=p(ie);g(ze.$$.fragment,Mr),ga=s(Mr),dt=n(Mr,"P",{"data-svelte-h":!0}),h(dt)!=="svelte-3iict9"&&(dt.textContent=en),Mr.forEach(t),ue.forEach(t),er=s(e),P=n(e,"DIV",{class:!0});var Cr=p(P);g(Me.$$.fragment,Cr),_a=s(Cr),ht=n(Cr,"P",{"data-svelte-h":!0}),h(ht)!=="svelte-16qojcu"&&(ht.textContent=tn),Cr.forEach(t),tr=s(e),R=n(e,"H2",{class:!0});var Wr=p(R);le=n(Wr,"A",{id:!0,class:!0,href:!0});var Ln=p(le);Nt=n(Ln,"SPAN",{});var kn=p(Nt);g(Ce.$$.fragment,kn),kn.forEach(t),Ln.forEach(t),wa=s(Wr),ut=n(Wr,"SPAN",{"data-svelte-h":!0}),h(ut)!=="svelte-bkt8ul"&&(ut.textContent=rn),Wr.forEach(t),rr=s(e),N=n(e,"H3",{class:!0});var jr=p(N);me=n(jr,"A",{id:!0,class:!0,href:!0});var En=p(me);St=n(En,"SPAN",{});var Un=p(St);g(We.$$.fragment,Un),Un.forEach(t),En.forEach(t),va=s(jr),ft=n(jr,"SPAN",{"data-svelte-h":!0}),h(ft)!=="svelte-szjh8n"&&(ft.textContent=an),jr.forEach(t),ar=s(e),S=n(e,"DIV",{class:!0});var Lr=p(S);g(je.$$.fragment,Lr),ba=s(Lr),gt=n(Lr,"P",{"data-svelte-h":!0}),h(gt)!=="svelte-ofujqi"&&(gt.textContent=nn),Lr.forEach(t),nr=s(e),I=n(e,"DIV",{class:!0});var kr=p(I);g(Le.$$.fragment,kr),ya=s(kr),_t=n(kr,"P",{"data-svelte-h":!0}),h(_t)!=="svelte-1oh310h"&&(_t.textContent=on),kr.forEach(t),or=s(e),F=n(e,"DIV",{class:!0});var Er=p(F);g(ke.$$.fragment,Er),$a=s(Er),wt=n(Er,"P",{"data-svelte-h":!0}),h(wt)!=="svelte-cadr4a"&&(wt.textContent=sn),Er.forEach(t),sr=s(e),V=n(e,"DIV",{class:!0});var Ur=p(V);g(Ee.$$.fragment,Ur),xa=s(Ur),vt=n(Ur,"P",{"data-svelte-h":!0}),h(vt)!=="svelte-1lj8vcj"&&(vt.textContent=ln),Ur.forEach(t),ir=s(e),Ue=n(e,"IMG",{alt:!0,src:!0}),lr=s(e),H=n(e,"DIV",{class:!0});var Dr=p(H);g(De.$$.fragment,Dr),Aa=s(Dr),bt=n(Dr,"P",{"data-svelte-h":!0}),h(bt)!=="svelte-1lhkhz0"&&(bt.textContent=cn),Dr.forEach(t),mr=s(e),Pe=n(e,"IMG",{alt:!0,src:!0}),cr=s(e),J=n(e,"DIV",{class:!0});var Pr=p(J);g(Re.$$.fragment,Pr),Ta=s(Pr),yt=n(Pr,"P",{"data-svelte-h":!0}),h(yt)!=="svelte-1pclsvt"&&(yt.textContent=dn),Pr.forEach(t),pr=s(e),Ne=n(e,"IMG",{alt:!0,src:!0}),dr=s(e),G=n(e,"DIV",{class:!0});var Rr=p(G);g(Se.$$.fragment,Rr),za=s(Rr),$t=n(Rr,"P",{"data-svelte-h":!0}),h($t)!=="svelte-17q4ov5"&&($t.textContent=un),Rr.forEach(t),hr=s(e),Ie=n(e,"IMG",{alt:!0,src:!0}),ur=s(e),j=n(e,"DIV",{class:!0});var Et=p(j);g(Fe.$$.fragment,Et),Ma=s(Et),xt=n(Et,"P",{"data-svelte-h":!0}),h(xt)!=="svelte-1y55gjq"&&(xt.innerHTML=gn),Ca=s(Et),At=n(Et,"P",{"data-svelte-h":!0}),h(At)!=="svelte-1tn0l2l"&&(At.innerHTML=_n),Et.forEach(t),fr=s(e),Z=n(e,"DIV",{class:!0});var Nr=p(Z);g(Ve.$$.fragment,Nr),Wa=s(Nr),Tt=n(Nr,"P",{"data-svelte-h":!0}),h(Tt)!=="svelte-yp2yn6"&&(Tt.textContent=wn),Nr.forEach(t),gr=s(e),q=n(e,"H3",{class:!0});var Sr=p(q);ce=n(Sr,"A",{id:!0,class:!0,href:!0});var Dn=p(ce);It=n(Dn,"SPAN",{});var Pn=p(It);g(He.$$.fragment,Pn),Pn.forEach(t),Dn.forEach(t),ja=s(Sr),zt=n(Sr,"SPAN",{"data-svelte-h":!0}),h(zt)!=="svelte-1fpf4gj"&&(zt.textContent=vn),Sr.forEach(t),_r=s(e),B=n(e,"DIV",{class:!0});var Ir=p(B);g(Je.$$.fragment,Ir),La=s(Ir),Mt=n(Ir,"P",{"data-svelte-h":!0}),h(Mt)!=="svelte-1mfn66a"&&(Mt.textContent=bn),Ir.forEach(t),wr=s(e),X=n(e,"H2",{class:!0});var Fr=p(X);pe=n(Fr,"A",{id:!0,class:!0,href:!0});var Rn=p(pe);Ft=n(Rn,"SPAN",{});var Nn=p(Ft);g(Ge.$$.fragment,Nn),Nn.forEach(t),Rn.forEach(t),ka=s(Fr),Ct=n(Fr,"SPAN",{"data-svelte-h":!0}),h(Ct)!=="svelte-13r2x6y"&&(Ct.textContent=yn),Fr.forEach(t),vr=s(e),O=n(e,"H3",{class:!0});var Vr=p(O);de=n(Vr,"A",{id:!0,class:!0,href:!0});var Sn=p(de);Vt=n(Sn,"SPAN",{});var In=p(Vt);g(Ze.$$.fragment,In),In.forEach(t),Sn.forEach(t),Ea=s(Vr),Wt=n(Vr,"SPAN",{"data-svelte-h":!0}),h(Wt)!=="svelte-eif24r"&&(Wt.textContent=$n),Vr.forEach(t),br=s(e),L=n(e,"DIV",{class:!0});var Ut=p(L);g(qe.$$.fragment,Ut),Ua=s(Ut),jt=n(Ut,"P",{"data-svelte-h":!0}),h(jt)!=="svelte-8tkiw"&&(jt.innerHTML=xn),Da=s(Ut),he=n(Ut,"DIV",{class:!0});var Hr=p(he);g(Be.$$.fragment,Hr),Pa=s(Hr),Lt=n(Hr,"P",{"data-svelte-h":!0}),h(Lt)!=="svelte-v0y289"&&(Lt.textContent=An),Hr.forEach(t),Ut.forEach(t),yr=s(e),Ht=n(e,"P",{}),p(Ht).forEach(t),this.h()},h(){c(d,"name","hf:doc:metadata"),c(d,"content",JSON.stringify(Yn)),c(m,"id","optimization"),c(m,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(m,"href","#optimization"),c(y,"class","relative group"),c(Q,"id","transformers.AdamW"),c(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Q,"href","#transformers.AdamW"),c(E,"class","relative group"),c(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(ee,"id","transformers.Adafactor"),c(ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ee,"href","#transformers.Adafactor"),c(U,"class","relative group"),c(oe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(se,"id","transformers.AdamWeightDecay"),c(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(se,"href","#transformers.AdamWeightDecay"),c(D,"class","relative group"),c(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(le,"id","schedules"),c(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(le,"href","#schedules"),c(R,"class","relative group"),c(me,"id","transformers.SchedulerType"),c(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(me,"href","#transformers.SchedulerType"),c(N,"class","relative group"),c(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Ue,"alt",""),Jr(Ue.src,mn="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_constant_schedule.png")||c(Ue,"src",mn),c(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Pe,"alt",""),Jr(Pe.src,pn="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_schedule.png")||c(Pe,"src",pn),c(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Ne,"alt",""),Jr(Ne.src,hn="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_hard_restarts_schedule.png")||c(Ne,"src",hn),c(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Ie,"alt",""),Jr(Ie.src,fn="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_linear_schedule.png")||c(Ie,"src",fn),c(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(ce,"id","transformers.WarmUp"),c(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ce,"href","#transformers.WarmUp"),c(q,"class","relative group"),c(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(pe,"id","gradient-strategies"),c(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(pe,"href","#gradient-strategies"),c(X,"class","relative group"),c(de,"id","transformers.GradientAccumulator"),c(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(de,"href","#transformers.GradientAccumulator"),c(O,"class","relative group"),c(he,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,i){r(document.head,d),l(e,M,i),l(e,x,i),l(e,u,i),l(e,y,i),r(y,m),r(m,T),_(fe,T,null),r(y,Br),r(y,Ye),l(e,Zt,i),l(e,ge,i),l(e,qt,i),l(e,_e,i),l(e,Bt,i),l(e,E,i),r(E,Q),r(Q,Dt),_(we,Dt,null),r(E,Xr),r(E,Qe),l(e,Xt,i),l(e,W,i),_(ve,W,null),r(W,Or),r(W,Ke),r(W,Yr),r(W,K),_(be,K,null),r(K,Qr),r(K,et),l(e,Ot,i),l(e,U,i),r(U,ee),r(ee,Pt),_(ye,Pt,null),r(U,Kr),r(U,tt),l(e,Yt,i),l(e,$,i),_($e,$,null),r($,ea),r($,rt),r($,ta),r($,at),r($,ra),r($,nt),r($,aa),r($,ot),r($,na),r($,st),r($,oa),_(te,$,null),r($,sa),_(re,$,null),r($,ia),r($,it),r($,la),_(ae,$,null),r($,ma),_(ne,$,null),r($,ca),r($,oe),_(xe,oe,null),r(oe,pa),r(oe,lt),l(e,Qt,i),l(e,D,i),r(D,se),r(se,Rt),_(Ae,Rt,null),r(D,da),r(D,mt),l(e,Kt,i),l(e,C,i),_(Te,C,null),r(C,ha),r(C,ct),r(C,ua),r(C,pt),r(C,fa),r(C,ie),_(ze,ie,null),r(ie,ga),r(ie,dt),l(e,er,i),l(e,P,i),_(Me,P,null),r(P,_a),r(P,ht),l(e,tr,i),l(e,R,i),r(R,le),r(le,Nt),_(Ce,Nt,null),r(R,wa),r(R,ut),l(e,rr,i),l(e,N,i),r(N,me),r(me,St),_(We,St,null),r(N,va),r(N,ft),l(e,ar,i),l(e,S,i),_(je,S,null),r(S,ba),r(S,gt),l(e,nr,i),l(e,I,i),_(Le,I,null),r(I,ya),r(I,_t),l(e,or,i),l(e,F,i),_(ke,F,null),r(F,$a),r(F,wt),l(e,sr,i),l(e,V,i),_(Ee,V,null),r(V,xa),r(V,vt),l(e,ir,i),l(e,Ue,i),l(e,lr,i),l(e,H,i),_(De,H,null),r(H,Aa),r(H,bt),l(e,mr,i),l(e,Pe,i),l(e,cr,i),l(e,J,i),_(Re,J,null),r(J,Ta),r(J,yt),l(e,pr,i),l(e,Ne,i),l(e,dr,i),l(e,G,i),_(Se,G,null),r(G,za),r(G,$t),l(e,hr,i),l(e,Ie,i),l(e,ur,i),l(e,j,i),_(Fe,j,null),r(j,Ma),r(j,xt),r(j,Ca),r(j,At),l(e,fr,i),l(e,Z,i),_(Ve,Z,null),r(Z,Wa),r(Z,Tt),l(e,gr,i),l(e,q,i),r(q,ce),r(ce,It),_(He,It,null),r(q,ja),r(q,zt),l(e,_r,i),l(e,B,i),_(Je,B,null),r(B,La),r(B,Mt),l(e,wr,i),l(e,X,i),r(X,pe),r(pe,Ft),_(Ge,Ft,null),r(X,ka),r(X,Ct),l(e,vr,i),l(e,O,i),r(O,de),r(de,Vt),_(Ze,Vt,null),r(O,Ea),r(O,Wt),l(e,br,i),l(e,L,i),_(qe,L,null),r(L,Ua),r(L,jt),r(L,Da),r(L,he),_(Be,he,null),r(he,Pa),r(he,Lt),l(e,yr,i),l(e,Ht,i),$r=!0},p(e,[i]){const Xe={};i&2&&(Xe.$$scope={dirty:i,ctx:e}),te.$set(Xe);const Jt={};i&2&&(Jt.$$scope={dirty:i,ctx:e}),re.$set(Jt);const Gt={};i&2&&(Gt.$$scope={dirty:i,ctx:e}),ae.$set(Gt);const Oe={};i&2&&(Oe.$$scope={dirty:i,ctx:e}),ne.$set(Oe)},i(e){$r||(w(fe.$$.fragment,e),w(we.$$.fragment,e),w(ve.$$.fragment,e),w(be.$$.fragment,e),w(ye.$$.fragment,e),w($e.$$.fragment,e),w(te.$$.fragment,e),w(re.$$.fragment,e),w(ae.$$.fragment,e),w(ne.$$.fragment,e),w(xe.$$.fragment,e),w(Ae.$$.fragment,e),w(Te.$$.fragment,e),w(ze.$$.fragment,e),w(Me.$$.fragment,e),w(Ce.$$.fragment,e),w(We.$$.fragment,e),w(je.$$.fragment,e),w(Le.$$.fragment,e),w(ke.$$.fragment,e),w(Ee.$$.fragment,e),w(De.$$.fragment,e),w(Re.$$.fragment,e),w(Se.$$.fragment,e),w(Fe.$$.fragment,e),w(Ve.$$.fragment,e),w(He.$$.fragment,e),w(Je.$$.fragment,e),w(Ge.$$.fragment,e),w(Ze.$$.fragment,e),w(qe.$$.fragment,e),w(Be.$$.fragment,e),$r=!0)},o(e){v(fe.$$.fragment,e),v(we.$$.fragment,e),v(ve.$$.fragment,e),v(be.$$.fragment,e),v(ye.$$.fragment,e),v($e.$$.fragment,e),v(te.$$.fragment,e),v(re.$$.fragment,e),v(ae.$$.fragment,e),v(ne.$$.fragment,e),v(xe.$$.fragment,e),v(Ae.$$.fragment,e),v(Te.$$.fragment,e),v(ze.$$.fragment,e),v(Me.$$.fragment,e),v(Ce.$$.fragment,e),v(We.$$.fragment,e),v(je.$$.fragment,e),v(Le.$$.fragment,e),v(ke.$$.fragment,e),v(Ee.$$.fragment,e),v(De.$$.fragment,e),v(Re.$$.fragment,e),v(Se.$$.fragment,e),v(Fe.$$.fragment,e),v(Ve.$$.fragment,e),v(He.$$.fragment,e),v(Je.$$.fragment,e),v(Ge.$$.fragment,e),v(Ze.$$.fragment,e),v(qe.$$.fragment,e),v(Be.$$.fragment,e),$r=!1},d(e){e&&(t(M),t(x),t(u),t(y),t(Zt),t(ge),t(qt),t(_e),t(Bt),t(E),t(Xt),t(W),t(Ot),t(U),t(Yt),t($),t(Qt),t(D),t(Kt),t(C),t(er),t(P),t(tr),t(R),t(rr),t(N),t(ar),t(S),t(nr),t(I),t(or),t(F),t(sr),t(V),t(ir),t(Ue),t(lr),t(H),t(mr),t(Pe),t(cr),t(J),t(pr),t(Ne),t(dr),t(G),t(hr),t(Ie),t(ur),t(j),t(fr),t(Z),t(gr),t(q),t(_r),t(B),t(wr),t(X),t(vr),t(O),t(br),t(L),t(yr),t(Ht)),t(d),b(fe),b(we),b(ve),b(be),b(ye),b($e),b(te),b(re),b(ae),b(ne),b(xe),b(Ae),b(Te),b(ze),b(Me),b(Ce),b(We),b(je),b(Le),b(ke),b(Ee),b(De),b(Re),b(Se),b(Fe),b(Ve),b(He),b(Je),b(Ge),b(Ze),b(qe),b(Be)}}}const Yn={local:"optimization",sections:[{local:"transformers.AdamW",title:"AdamW (PyTorch)"},{local:"transformers.Adafactor",title:"AdaFactor (PyTorch)"},{local:"transformers.AdamWeightDecay",title:"AdamWeightDecay (TensorFlow)"},{local:"schedules",sections:[{local:"transformers.SchedulerType",title:"Learning Rate Schedules (Pytorch)"},{local:"transformers.WarmUp",title:"Warmup (TensorFlow)"}],title:"Schedules"},{local:"gradient-strategies",sections:[{local:"transformers.GradientAccumulator",title:"GradientAccumulator (TensorFlow)"}],title:"Gradient Strategies"}],title:"Optimization"};function Qn(k){return Vn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class oo extends Hn{constructor(d){super(),Jn(this,d,Qn,On,Fn,{})}}export{oo as component};
