import{S as Mi,i as Hi,s as Bi,e as a,k as l,w as g,t as s,M as Ji,c as n,d as r,m,a as o,x as _,h as i,b as c,N as ja,G as t,g as h,y as w,q as v,o as b,B as $,v as Ki,L as Ga}from"../../chunks/vendor-hf-doc-builder.js";import{D as L}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Va}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as oe}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as Ua}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function Qi(C){let f,T,x,u,z;return u=new Va({props:{code:"Adafactor(model.parameters(), scale_parameter=False, relative_step=False, warmup_init=False, lr=1e-3)",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, warmup_init=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">1e-3</span>)'}}),{c(){f=a("p"),T=s("Example:"),x=l(),g(u.$$.fragment)},l(d){f=n(d,"P",{});var y=o(f);T=i(y,"Example:"),y.forEach(r),x=m(d),_(u.$$.fragment,d)},m(d,y){h(d,f,y),t(f,T),h(d,x,y),w(u,d,y),z=!0},p:Ga,i(d){z||(v(u.$$.fragment,d),z=!0)},o(d){b(u.$$.fragment,d),z=!1},d(d){d&&r(f),d&&r(x),$(u,d)}}}function Xi(C){let f,T,x,u,z;return u=new Va({props:{code:"Adafactor(model.parameters(), scale_parameter=True, relative_step=True, warmup_init=True, lr=None)",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)'}}),{c(){f=a("p"),T=s("Others reported the following combination to work well:"),x=l(),g(u.$$.fragment)},l(d){f=n(d,"P",{});var y=o(f);T=i(y,"Others reported the following combination to work well:"),y.forEach(r),x=m(d),_(u.$$.fragment,d)},m(d,y){h(d,f,y),t(f,T),h(d,x,y),w(u,d,y),z=!0},p:Ga,i(d){z||(v(u.$$.fragment,d),z=!0)},o(d){b(u.$$.fragment,d),z=!1},d(d){d&&r(f),d&&r(x),$(u,d)}}}function Yi(C){let f,T,x,u,z;return u=new Va({props:{code:`from transformers.optimization import Adafactor, AdafactorSchedule

optimizer = Adafactor(model.parameters(), scale_parameter=True, relative_step=True, warmup_init=True, lr=None)
lr_scheduler = AdafactorSchedule(optimizer)
trainer = Trainer(..., optimizers=(optimizer, lr_scheduler))`,highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, AdafactorSchedule

optimizer = Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)
lr_scheduler = AdafactorSchedule(optimizer)
trainer = Trainer(..., optimizers=(optimizer, lr_scheduler))`}}),{c(){f=a("p"),T=s("scheduler as following:"),x=l(),g(u.$$.fragment)},l(d){f=n(d,"P",{});var y=o(f);T=i(y,"scheduler as following:"),y.forEach(r),x=m(d),_(u.$$.fragment,d)},m(d,y){h(d,f,y),t(f,T),h(d,x,y),w(u,d,y),z=!0},p:Ga,i(d){z||(v(u.$$.fragment,d),z=!0)},o(d){b(u.$$.fragment,d),z=!1},d(d){d&&r(f),d&&r(x),$(u,d)}}}function Zi(C){let f,T,x,u,z;return u=new Va({props:{code:`# replace AdamW with Adafactor
optimizer = Adafactor(
    model.parameters(),
    lr=1e-3,
    eps=(1e-30, 1e-3),
    clip_threshold=1.0,
    decay_rate=-0.8,
    beta1=None,
    weight_decay=0.0,
    relative_step=False,
    scale_parameter=False,
    warmup_init=False,
)`,highlighted:`<span class="hljs-comment"># replace AdamW with Adafactor</span>
optimizer = Adafactor(
    model.parameters(),
    lr=<span class="hljs-number">1e-3</span>,
    eps=(<span class="hljs-number">1e-30</span>, <span class="hljs-number">1e-3</span>),
    clip_threshold=<span class="hljs-number">1.0</span>,
    decay_rate=-<span class="hljs-number">0.8</span>,
    beta1=<span class="hljs-literal">None</span>,
    weight_decay=<span class="hljs-number">0.0</span>,
    relative_step=<span class="hljs-literal">False</span>,
    scale_parameter=<span class="hljs-literal">False</span>,
    warmup_init=<span class="hljs-literal">False</span>,
)`}}),{c(){f=a("p"),T=s("Usage:"),x=l(),g(u.$$.fragment)},l(d){f=n(d,"P",{});var y=o(f);T=i(y,"Usage:"),y.forEach(r),x=m(d),_(u.$$.fragment,d)},m(d,y){h(d,f,y),t(f,T),h(d,x,y),w(u,d,y),z=!0},p:Ga,i(d){z||(v(u.$$.fragment,d),z=!0)},o(d){b(u.$$.fragment,d),z=!1},d(d){d&&r(f),d&&r(x),$(u,d)}}}function el(C){let f,T,x,u,z,d,y,Tt,Ma,Ir,se,Ha,Dt,Ba,Ja,Cr,F,Lt,Ka,Qa,Ee,Xa,Pt,Ya,Za,en,Wt,tn,Fr,q,ie,kt,Te,rn,Nt,an,Or,k,De,nn,Le,on,Pe,sn,ln,mn,le,We,cn,St,pn,qr,R,me,It,ke,dn,Ct,hn,Rr,A,Ne,un,vt,fn,Se,gn,_n,D,wn,Ft,vn,bn,Ie,$n,yn,Ot,xn,An,qt,zn,En,Rt,Tn,Dn,jt,Ln,Pn,Ut,Wn,kn,Nn,Gt,Sn,In,Ce,Cn,Fe,Fn,On,qn,N,Oe,Vt,Rn,jn,qe,Mt,Un,Gn,Re,Vn,je,Mn,Hn,Bn,Ht,Bt,Jn,Kn,Jt,Kt,Qn,Xn,Qt,Xt,Yn,Zn,ce,eo,pe,to,O,ro,Yt,ao,no,bt,oo,so,Zt,io,lo,de,mo,he,co,ue,Ue,po,er,ho,jr,j,fe,tr,Ge,uo,rr,fo,Ur,W,Ve,go,U,_o,ar,wo,vo,Me,bo,$o,yo,nr,xo,Ao,ge,He,zo,or,Eo,Gr,G,Be,To,sr,Do,Vr,V,_e,ir,Je,Lo,lr,Po,Mr,M,we,mr,Ke,Wo,cr,ko,Hr,H,Qe,No,pr,So,Br,B,Xe,Io,dr,Co,Jr,J,Ye,Fo,hr,Oo,Kr,K,Ze,qo,ur,Ro,Qr,et,ys,Xr,Q,tt,jo,fr,Uo,Yr,rt,xs,Zr,X,at,Go,gr,Vo,ea,nt,As,ta,Y,ot,Mo,_r,Ho,ra,st,zs,aa,S,it,Bo,lt,Jo,wr,Ko,Qo,Xo,ve,Yo,vr,Zo,es,mt,ts,na,Z,ct,rs,br,as,oa,ee,be,$r,pt,ns,yr,os,sa,te,dt,ss,xr,is,ia,re,$e,Ar,ht,ls,zr,ms,la,ae,ye,Er,ut,cs,Tr,ps,ma,I,ft,ds,ne,hs,Dr,us,fs,Lr,gs,_s,ws,xe,gt,vs,Pr,bs,ca;return d=new oe({}),Te=new oe({}),De=new L({props:{name:"class transformers.AdamW",anchor:"transformers.AdamW",parameters:[{name:"params",val:": typing.Iterable[torch.nn.parameter.Parameter]"},{name:"lr",val:": float = 0.001"},{name:"betas",val:": typing.Tuple[float, float] = (0.9, 0.999)"},{name:"eps",val:": float = 1e-06"},{name:"weight_decay",val:": float = 0.0"},{name:"correct_bias",val:": bool = True"},{name:"no_deprecation_warning",val:": bool = False"}],parametersDescription:[{anchor:"transformers.AdamW.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.AdamW.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-3) &#x2014;
The learning rate to use.`,name:"lr"},{anchor:"transformers.AdamW.betas",description:`<strong>betas</strong> (<code>Tuple[float,float]</code>, <em>optional</em>, defaults to (0.9, 0.999)) &#x2014;
Adam&#x2019;s betas parameters (b1, b2).`,name:"betas"},{anchor:"transformers.AdamW.eps",description:`<strong>eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-6) &#x2014;
Adam&#x2019;s epsilon for numerical stability.`,name:"eps"},{anchor:"transformers.AdamW.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
Decoupled weight decay to apply.`,name:"weight_decay"},{anchor:"transformers.AdamW.correct_bias",description:`<strong>correct_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to correct bias in Adam (for instance, in Bert TF repository they use <code>False</code>).`,name:"correct_bias"},{anchor:"transformers.AdamW.no_deprecation_warning",description:`<strong>no_deprecation_warning</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
A flag used to disable the deprecation warning (set to <code>True</code> to disable the warning).`,name:"no_deprecation_warning"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L358"}}),We=new L({props:{name:"step",anchor:"transformers.AdamW.step",parameters:[{name:"closure",val:": typing.Callable = None"}],parametersDescription:[{anchor:"transformers.AdamW.step.closure",description:"<strong>closure</strong> (<code>Callable</code>, <em>optional</em>) &#x2014; A closure that reevaluates the model and returns the loss.",name:"closure"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L409"}}),ke=new oe({}),Ne=new L({props:{name:"class transformers.Adafactor",anchor:"transformers.Adafactor",parameters:[{name:"params",val:""},{name:"lr",val:" = None"},{name:"eps",val:" = (1e-30, 0.001)"},{name:"clip_threshold",val:" = 1.0"},{name:"decay_rate",val:" = -0.8"},{name:"beta1",val:" = None"},{name:"weight_decay",val:" = 0.0"},{name:"scale_parameter",val:" = True"},{name:"relative_step",val:" = True"},{name:"warmup_init",val:" = False"}],parametersDescription:[{anchor:"transformers.Adafactor.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.Adafactor.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>) &#x2014;
The external learning rate.`,name:"lr"},{anchor:"transformers.Adafactor.eps",description:`<strong>eps</strong> (<code>Tuple[float, float]</code>, <em>optional</em>, defaults to (1e-30, 1e-3)) &#x2014;
Regularization constants for square gradient and parameter scale respectively`,name:"eps"},{anchor:"transformers.Adafactor.clip_threshold",description:`<strong>clip_threshold</strong> (<code>float</code>, <em>optional</em>, defaults 1.0) &#x2014;
Threshold of root mean square of final gradient update`,name:"clip_threshold"},{anchor:"transformers.Adafactor.decay_rate",description:`<strong>decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to -0.8) &#x2014;
Coefficient used to compute running averages of square`,name:"decay_rate"},{anchor:"transformers.Adafactor.beta1",description:`<strong>beta1</strong> (<code>float</code>, <em>optional</em>) &#x2014;
Coefficient used for computing running averages of gradient`,name:"beta1"},{anchor:"transformers.Adafactor.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
Weight decay (L2 penalty)`,name:"weight_decay"},{anchor:"transformers.Adafactor.scale_parameter",description:`<strong>scale_parameter</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, learning rate is scaled by root mean square`,name:"scale_parameter"},{anchor:"transformers.Adafactor.relative_step",description:`<strong>relative_step</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, time-dependent learning rate is computed instead of external learning rate`,name:"relative_step"},{anchor:"transformers.Adafactor.warmup_init",description:`<strong>warmup_init</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Time-dependent learning rate computation depends on whether warm-up initialization is being used`,name:"warmup_init"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L471"}}),ce=new Ua({props:{anchor:"transformers.Adafactor.example",$$slots:{default:[Qi]},$$scope:{ctx:C}}}),pe=new Ua({props:{anchor:"transformers.Adafactor.example-2",$$slots:{default:[Xi]},$$scope:{ctx:C}}}),de=new Ua({props:{anchor:"transformers.Adafactor.example-3",$$slots:{default:[Yi]},$$scope:{ctx:C}}}),he=new Ua({props:{anchor:"transformers.Adafactor.example-4",$$slots:{default:[Zi]},$$scope:{ctx:C}}}),Ue=new L({props:{name:"step",anchor:"transformers.Adafactor.step",parameters:[{name:"closure",val:" = None"}],parametersDescription:[{anchor:"transformers.Adafactor.step.closure",description:`<strong>closure</strong> (callable, optional) &#x2014; A closure that reevaluates the model
and returns the loss.`,name:"closure"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L617"}}),Ge=new oe({}),Ve=new L({props:{name:"class transformers.AdamWeightDecay",anchor:"transformers.AdamWeightDecay",parameters:[{name:"learning_rate",val:": typing.Union[float, keras.optimizers.schedules.learning_rate_schedule.LearningRateSchedule] = 0.001"},{name:"beta_1",val:": float = 0.9"},{name:"beta_2",val:": float = 0.999"},{name:"epsilon",val:": float = 1e-07"},{name:"amsgrad",val:": bool = False"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"exclude_from_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"name",val:": str = 'AdamWeightDecay'"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.AdamWeightDecay.learning_rate",description:`<strong>learning_rate</strong> (<code>Union[float, tf.keras.optimizers.schedules.LearningRateSchedule]</code>, <em>optional</em>, defaults to 1e-3) &#x2014;
The learning rate to use or a schedule.`,name:"learning_rate"},{anchor:"transformers.AdamWeightDecay.beta_1",description:`<strong>beta_1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 parameter in Adam, which is the exponential decay rate for the 1st momentum estimates.`,name:"beta_1"},{anchor:"transformers.AdamWeightDecay.beta_2",description:`<strong>beta_2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 parameter in Adam, which is the exponential decay rate for the 2nd momentum estimates.`,name:"beta_2"},{anchor:"transformers.AdamWeightDecay.epsilon",description:`<strong>epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The epsilon parameter in Adam, which is a small constant for numerical stability.`,name:"epsilon"},{anchor:"transformers.AdamWeightDecay.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, <em>optional</em>, default to <code>False</code>) &#x2014;
Whether to apply AMSGrad variant of this algorithm or not, see <a href="https://arxiv.org/abs/1904.09237" rel="nofollow">On the Convergence of Adam and
Beyond</a>.`,name:"amsgrad"},{anchor:"transformers.AdamWeightDecay.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to apply.`,name:"weight_decay_rate"},{anchor:"transformers.AdamWeightDecay.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters by default (unless they are in <code>exclude_from_weight_decay</code>).`,name:"include_in_weight_decay"},{anchor:"transformers.AdamWeightDecay.exclude_from_weight_decay",description:`<strong>exclude_from_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to exclude from applying weight decay to. If a
<code>include_in_weight_decay</code> is passed, the names in it will supersede this list.`,name:"exclude_from_weight_decay"},{anchor:"transformers.AdamWeightDecay.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>, defaults to &#x2018;AdamWeightDecay&#x2019;) &#x2014;
Optional name for the operations created when applying gradients.
kwargs &#x2014;
Keyword arguments. Allowed to be {<code>clipnorm</code>, <code>clipvalue</code>, <code>lr</code>, <code>decay</code>}. <code>clipnorm</code> is clip gradients by
norm; <code>clipvalue</code> is clip gradients by value, <code>decay</code> is included for backward compatibility to allow time
inverse decay of learning rate. <code>lr</code> is included for backward compatibility, recommended to use
<code>learning_rate</code> instead.`,name:"name"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization_tf.py#L172"}}),He=new L({props:{name:"from_config",anchor:"transformers.AdamWeightDecay.from_config",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization_tf.py#L229"}}),Be=new L({props:{name:"transformers.create_optimizer",anchor:"transformers.create_optimizer",parameters:[{name:"init_lr",val:": float"},{name:"num_train_steps",val:": int"},{name:"num_warmup_steps",val:": int"},{name:"min_lr_ratio",val:": float = 0.0"},{name:"adam_beta1",val:": float = 0.9"},{name:"adam_beta2",val:": float = 0.999"},{name:"adam_epsilon",val:": float = 1e-08"},{name:"adam_clipnorm",val:": typing.Optional[float] = None"},{name:"adam_global_clipnorm",val:": typing.Optional[float] = None"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"power",val:": float = 1.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"}],parametersDescription:[{anchor:"transformers.create_optimizer.init_lr",description:`<strong>init_lr</strong> (<code>float</code>) &#x2014;
The desired learning rate at the end of the warmup phase.`,name:"init_lr"},{anchor:"transformers.create_optimizer.num_train_steps",description:`<strong>num_train_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_train_steps"},{anchor:"transformers.create_optimizer.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of warmup steps.`,name:"num_warmup_steps"},{anchor:"transformers.create_optimizer.min_lr_ratio",description:`<strong>min_lr_ratio</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The final learning rate at the end of the linear decay will be <code>init_lr * min_lr_ratio</code>.`,name:"min_lr_ratio"},{anchor:"transformers.create_optimizer.adam_beta1",description:`<strong>adam_beta1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 to use in Adam.`,name:"adam_beta1"},{anchor:"transformers.create_optimizer.adam_beta2",description:`<strong>adam_beta2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 to use in Adam.`,name:"adam_beta2"},{anchor:"transformers.create_optimizer.adam_epsilon",description:`<strong>adam_epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-8) &#x2014;
The epsilon to use in Adam.
adam_clipnorm &#x2014; (<code>float</code>, <em>optional</em>, defaults to <code>None</code>):
If not <code>None</code>, clip the gradient norm for each weight tensor to this value.
adam_global_clipnorm &#x2014; (<code>float</code>, <em>optional</em>, defaults to <code>None</code>)
If not <code>None</code>, clip gradient norm to this value. When using this argument, the norm is computed over all
weight tensors, as if they were concatenated into a single vector.`,name:"adam_epsilon"},{anchor:"transformers.create_optimizer.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to use.`,name:"weight_decay_rate"},{anchor:"transformers.create_optimizer.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The power to use for PolynomialDecay.`,name:"power"},{anchor:"transformers.create_optimizer.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters except bias and layer norm parameters.`,name:"include_in_weight_decay"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization_tf.py#L88"}}),Je=new oe({}),Ke=new oe({}),Qe=new L({props:{name:"class transformers.SchedulerType",anchor:"transformers.SchedulerType",parameters:[{name:"value",val:""},{name:"names",val:" = None"},{name:"module",val:" = None"},{name:"qualname",val:" = None"},{name:"type",val:" = None"},{name:"start",val:" = 1"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/trainer_utils.py#L362"}}),Xe=new L({props:{name:"transformers.get_scheduler",anchor:"transformers.get_scheduler",parameters:[{name:"name",val:": typing.Union[str, transformers.trainer_utils.SchedulerType]"},{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": typing.Optional[int] = None"},{name:"num_training_steps",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"transformers.get_scheduler.name",description:`<strong>name</strong> (<code>str</code> or <code>SchedulerType</code>) &#x2014;
The name of the scheduler to use.`,name:"name"},{anchor:"transformers.get_scheduler.optimizer",description:`<strong>optimizer</strong> (<code>torch.optim.Optimizer</code>) &#x2014;
The optimizer that will be used during training.`,name:"optimizer"},{anchor:"transformers.get_scheduler.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The number of warmup steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_warmup_steps"},{anchor:"transformers.get_scheduler.num_training_steps",description:`<strong>num_training_steps</strong> (\`int&#x201C;, <em>optional</em>) &#x2014;
The number of training steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_training_steps"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L315"}}),Ye=new L({props:{name:"transformers.get_constant_schedule",anchor:"transformers.get_constant_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L35",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Ze=new L({props:{name:"transformers.get_constant_schedule_with_warmup",anchor:"transformers.get_constant_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_constant_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L58",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),tt=new L({props:{name:"transformers.get_cosine_schedule_with_warmup",anchor:"transformers.get_cosine_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": float = 0.5"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>float</code>, <em>optional</em>, defaults to 0.5) &#x2014;
The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0
following a half-cosine).`,name:"num_cycles"},{anchor:"transformers.get_cosine_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L121",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),at=new L({props:{name:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": int = 1"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of hard restarts to use.`,name:"num_cycles"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L166",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),ot=new L({props:{name:"transformers.get_linear_schedule_with_warmup",anchor:"transformers.get_linear_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_linear_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_linear_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L85",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),it=new L({props:{name:"transformers.get_polynomial_decay_schedule_with_warmup",anchor:"transformers.get_polynomial_decay_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"lr_end",val:" = 1e-07"},{name:"power",val:" = 1.0"},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.lr_end",description:`<strong>lr_end</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The end LR.`,name:"lr_end"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
Power factor.`,name:"power"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L220",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),ct=new L({props:{name:"transformers.get_inverse_sqrt_schedule",anchor:"transformers.get_inverse_sqrt_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"timescale",val:": int = None"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_inverse_sqrt_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_inverse_sqrt_schedule.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_inverse_sqrt_schedule.timescale",description:`<strong>timescale</strong> (<code>int</code>, <em>optional</em>, defaults to <code>num_warmup_steps</code>) &#x2014;
Time scale.`,name:"timescale"},{anchor:"transformers.get_inverse_sqrt_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization.py#L274",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),pt=new oe({}),dt=new L({props:{name:"class transformers.WarmUp",anchor:"transformers.WarmUp",parameters:[{name:"initial_learning_rate",val:": float"},{name:"decay_schedule_fn",val:": typing.Callable"},{name:"warmup_steps",val:": int"},{name:"power",val:": float = 1.0"},{name:"name",val:": str = None"}],parametersDescription:[{anchor:"transformers.WarmUp.initial_learning_rate",description:`<strong>initial_learning_rate</strong> (<code>float</code>) &#x2014;
The initial learning rate for the schedule after the warmup (so this will be the learning rate at the end
of the warmup).`,name:"initial_learning_rate"},{anchor:"transformers.WarmUp.decay_schedule_fn",description:`<strong>decay_schedule_fn</strong> (<code>Callable</code>) &#x2014;
The schedule function to apply after the warmup for the rest of training.`,name:"decay_schedule_fn"},{anchor:"transformers.WarmUp.warmup_steps",description:`<strong>warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup part of training.`,name:"warmup_steps"},{anchor:"transformers.WarmUp.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1) &#x2014;
The power to use for the polynomial warmup (defaults is a linear warmup).`,name:"power"},{anchor:"transformers.WarmUp.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Optional name prefix for the returned tensors during the schedule.`,name:"name"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization_tf.py#L30"}}),ht=new oe({}),ut=new oe({}),ft=new L({props:{name:"class transformers.GradientAccumulator",anchor:"transformers.GradientAccumulator",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization_tf.py#L302"}}),gt=new L({props:{name:"reset",anchor:"transformers.GradientAccumulator.reset",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/optimization_tf.py#L364"}}),{c(){f=a("meta"),T=l(),x=a("h1"),u=a("a"),z=a("span"),g(d.$$.fragment),y=l(),Tt=a("span"),Ma=s("Optimization"),Ir=l(),se=a("p"),Ha=s("The "),Dt=a("code"),Ba=s(".optimization"),Ja=s(" module provides:"),Cr=l(),F=a("ul"),Lt=a("li"),Ka=s("an optimizer with weight decay fixed that can be used to fine-tuned models, and"),Qa=l(),Ee=a("li"),Xa=s("several schedules in the form of schedule objects that inherit from "),Pt=a("code"),Ya=s("_LRSchedule"),Za=s(":"),en=l(),Wt=a("li"),tn=s("a gradient accumulation class to accumulate the gradients of multiple batches"),Fr=l(),q=a("h2"),ie=a("a"),kt=a("span"),g(Te.$$.fragment),rn=l(),Nt=a("span"),an=s("AdamW (PyTorch)"),Or=l(),k=a("div"),g(De.$$.fragment),nn=l(),Le=a("p"),on=s("Implements Adam algorithm with weight decay fix as introduced in "),Pe=a("a"),sn=s(`Decoupled Weight Decay
Regularization`),ln=s("."),mn=l(),le=a("div"),g(We.$$.fragment),cn=l(),St=a("p"),pn=s("Performs a single optimization step."),qr=l(),R=a("h2"),me=a("a"),It=a("span"),g(ke.$$.fragment),dn=l(),Ct=a("span"),hn=s("AdaFactor (PyTorch)"),Rr=l(),A=a("div"),g(Ne.$$.fragment),un=l(),vt=a("p"),fn=s(`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
`),Se=a("a"),gn=s("https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),_n=l(),D=a("p"),wn=s("Paper: "),Ft=a("em"),vn=s("Adafactor: Adaptive Learning Rates with Sublinear Memory Cost"),bn=l(),Ie=a("a"),$n=s("https://arxiv.org/abs/1804.04235"),yn=s(` Note that
this optimizer internally adjusts the learning rate depending on the `),Ot=a("code"),xn=s("scale_parameter"),An=s(", "),qt=a("code"),zn=s("relative_step"),En=s(` and
`),Rt=a("code"),Tn=s("warmup_init"),Dn=s(" options. To use a manual (external) learning rate schedule you should set "),jt=a("code"),Ln=s("scale_parameter=False"),Pn=s(` and
`),Ut=a("code"),Wn=s("relative_step=False"),kn=s("."),Nn=l(),Gt=a("p"),Sn=s("This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested."),In=l(),Ce=a("p"),Cn=s("Recommended T5 finetuning settings ("),Fe=a("a"),Fn=s("https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),On=s("):"),qn=l(),N=a("ul"),Oe=a("li"),Vt=a("p"),Rn=s("Training without LR warmup or clip_threshold is not recommended."),jn=l(),qe=a("ul"),Mt=a("li"),Un=s("use scheduled LR warm-up to fixed LR"),Gn=l(),Re=a("li"),Vn=s("use clip_threshold=1.0 ("),je=a("a"),Mn=s("https://arxiv.org/abs/1804.04235"),Hn=s(")"),Bn=l(),Ht=a("li"),Bt=a("p"),Jn=s("Disable relative updates"),Kn=l(),Jt=a("li"),Kt=a("p"),Qn=s("Use scale_parameter=False"),Xn=l(),Qt=a("li"),Xt=a("p"),Yn=s("Additional optimizer operations like gradient clipping should not be used alongside Adafactor"),Zn=l(),g(ce.$$.fragment),eo=l(),g(pe.$$.fragment),to=l(),O=a("p"),ro=s("When using "),Yt=a("code"),ao=s("lr=None"),no=s(" with "),bt=a("a"),oo=s("Trainer"),so=s(" you will most likely need to use "),Zt=a("code"),io=s("AdafactorSchedule"),lo=l(),g(de.$$.fragment),mo=l(),g(he.$$.fragment),co=l(),ue=a("div"),g(Ue.$$.fragment),po=l(),er=a("p"),ho=s("Performs a single optimization step"),jr=l(),j=a("h2"),fe=a("a"),tr=a("span"),g(Ge.$$.fragment),uo=l(),rr=a("span"),fo=s("AdamWeightDecay (TensorFlow)"),Ur=l(),W=a("div"),g(Ve.$$.fragment),go=l(),U=a("p"),_o=s(`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is `),ar=a("em"),wo=s("not"),vo=s(` the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in `),Me=a("a"),bo=s(`Decoupled Weight Decay
Regularization`),$o=s("."),yo=l(),nr=a("p"),xo=s(`Instead we want to decay the weights in a manner that doesn\u2019t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`),Ao=l(),ge=a("div"),g(He.$$.fragment),zo=l(),or=a("p"),Eo=s("Creates an optimizer from its config with WarmUp custom object."),Gr=l(),G=a("div"),g(Be.$$.fragment),To=l(),sr=a("p"),Do=s("Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay."),Vr=l(),V=a("h2"),_e=a("a"),ir=a("span"),g(Je.$$.fragment),Lo=l(),lr=a("span"),Po=s("Schedules"),Mr=l(),M=a("h3"),we=a("a"),mr=a("span"),g(Ke.$$.fragment),Wo=l(),cr=a("span"),ko=s("Learning Rate Schedules (Pytorch)"),Hr=l(),H=a("div"),g(Qe.$$.fragment),No=l(),pr=a("p"),So=s("An enumeration."),Br=l(),B=a("div"),g(Xe.$$.fragment),Io=l(),dr=a("p"),Co=s("Unified API to get any scheduler from its name."),Jr=l(),J=a("div"),g(Ye.$$.fragment),Fo=l(),hr=a("p"),Oo=s("Create a schedule with a constant learning rate, using the learning rate set in optimizer."),Kr=l(),K=a("div"),g(Ze.$$.fragment),qo=l(),ur=a("p"),Ro=s(`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`),Qr=l(),et=a("img"),Xr=l(),Q=a("div"),g(tt.$$.fragment),jo=l(),fr=a("p"),Uo=s(`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`),Yr=l(),rt=a("img"),Zr=l(),X=a("div"),g(at.$$.fragment),Go=l(),gr=a("p"),Vo=s(`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`),ea=l(),nt=a("img"),ta=l(),Y=a("div"),g(ot.$$.fragment),Mo=l(),_r=a("p"),Ho=s(`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`),ra=l(),st=a("img"),aa=l(),S=a("div"),g(it.$$.fragment),Bo=l(),lt=a("p"),Jo=s(`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by `),wr=a("em"),Ko=s("lr_end"),Qo=s(`, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`),Xo=l(),ve=a("p"),Yo=s("Note: "),vr=a("em"),Zo=s("power"),es=s(` defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
`),mt=a("a"),ts=s("https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),na=l(),Z=a("div"),g(ct.$$.fragment),rs=l(),br=a("p"),as=s(`Create a schedule with an inverse square-root learning rate, from the initial lr set in the optimizer, after a
warmup period which increases lr linearly from 0 to the initial lr set in the optimizer.`),oa=l(),ee=a("h3"),be=a("a"),$r=a("span"),g(pt.$$.fragment),ns=l(),yr=a("span"),os=s("Warmup (TensorFlow)"),sa=l(),te=a("div"),g(dt.$$.fragment),ss=l(),xr=a("p"),is=s("Applies a warmup schedule on a given learning rate decay schedule."),ia=l(),re=a("h2"),$e=a("a"),Ar=a("span"),g(ht.$$.fragment),ls=l(),zr=a("span"),ms=s("Gradient Strategies"),la=l(),ae=a("h3"),ye=a("a"),Er=a("span"),g(ut.$$.fragment),cs=l(),Tr=a("span"),ps=s("GradientAccumulator (TensorFlow)"),ma=l(),I=a("div"),g(ft.$$.fragment),ds=l(),ne=a("p"),hs=s(`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call `),Dr=a("code"),us=s(".gradients"),fs=s(", scale the gradients if required, and pass the result to "),Lr=a("code"),gs=s("apply_gradients"),_s=s("."),ws=l(),xe=a("div"),g(gt.$$.fragment),vs=l(),Pr=a("p"),bs=s("Resets the accumulated gradients on the current replica."),this.h()},l(e){const p=Ji('[data-svelte="svelte-1phssyn"]',document.head);f=n(p,"META",{name:!0,content:!0}),p.forEach(r),T=m(e),x=n(e,"H1",{class:!0});var _t=o(x);u=n(_t,"A",{id:!0,class:!0,href:!0});var Wr=o(u);z=n(Wr,"SPAN",{});var kr=o(z);_(d.$$.fragment,kr),kr.forEach(r),Wr.forEach(r),y=m(_t),Tt=n(_t,"SPAN",{});var Nr=o(Tt);Ma=i(Nr,"Optimization"),Nr.forEach(r),_t.forEach(r),Ir=m(e),se=n(e,"P",{});var pa=o(se);Ha=i(pa,"The "),Dt=n(pa,"CODE",{});var Es=o(Dt);Ba=i(Es,".optimization"),Es.forEach(r),Ja=i(pa," module provides:"),pa.forEach(r),Cr=m(e),F=n(e,"UL",{});var $t=o(F);Lt=n($t,"LI",{});var Ts=o(Lt);Ka=i(Ts,"an optimizer with weight decay fixed that can be used to fine-tuned models, and"),Ts.forEach(r),Qa=m($t),Ee=n($t,"LI",{});var da=o(Ee);Xa=i(da,"several schedules in the form of schedule objects that inherit from "),Pt=n(da,"CODE",{});var Ds=o(Pt);Ya=i(Ds,"_LRSchedule"),Ds.forEach(r),Za=i(da,":"),da.forEach(r),en=m($t),Wt=n($t,"LI",{});var Ls=o(Wt);tn=i(Ls,"a gradient accumulation class to accumulate the gradients of multiple batches"),Ls.forEach(r),$t.forEach(r),Fr=m(e),q=n(e,"H2",{class:!0});var ha=o(q);ie=n(ha,"A",{id:!0,class:!0,href:!0});var Ps=o(ie);kt=n(Ps,"SPAN",{});var Ws=o(kt);_(Te.$$.fragment,Ws),Ws.forEach(r),Ps.forEach(r),rn=m(ha),Nt=n(ha,"SPAN",{});var ks=o(Nt);an=i(ks,"AdamW (PyTorch)"),ks.forEach(r),ha.forEach(r),Or=m(e),k=n(e,"DIV",{class:!0});var yt=o(k);_(De.$$.fragment,yt),nn=m(yt),Le=n(yt,"P",{});var ua=o(Le);on=i(ua,"Implements Adam algorithm with weight decay fix as introduced in "),Pe=n(ua,"A",{href:!0,rel:!0});var Ns=o(Pe);sn=i(Ns,`Decoupled Weight Decay
Regularization`),Ns.forEach(r),ln=i(ua,"."),ua.forEach(r),mn=m(yt),le=n(yt,"DIV",{class:!0});var fa=o(le);_(We.$$.fragment,fa),cn=m(fa),St=n(fa,"P",{});var Ss=o(St);pn=i(Ss,"Performs a single optimization step."),Ss.forEach(r),fa.forEach(r),yt.forEach(r),qr=m(e),R=n(e,"H2",{class:!0});var ga=o(R);me=n(ga,"A",{id:!0,class:!0,href:!0});var Is=o(me);It=n(Is,"SPAN",{});var Cs=o(It);_(ke.$$.fragment,Cs),Cs.forEach(r),Is.forEach(r),dn=m(ga),Ct=n(ga,"SPAN",{});var Fs=o(Ct);hn=i(Fs,"AdaFactor (PyTorch)"),Fs.forEach(r),ga.forEach(r),Rr=m(e),A=n(e,"DIV",{class:!0});var E=o(A);_(Ne.$$.fragment,E),un=m(E),vt=n(E,"P",{});var $s=o(vt);fn=i($s,`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
`),Se=n($s,"A",{href:!0,rel:!0});var Os=o(Se);gn=i(Os,"https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),Os.forEach(r),$s.forEach(r),_n=m(E),D=n(E,"P",{});var P=o(D);wn=i(P,"Paper: "),Ft=n(P,"EM",{});var qs=o(Ft);vn=i(qs,"Adafactor: Adaptive Learning Rates with Sublinear Memory Cost"),qs.forEach(r),bn=m(P),Ie=n(P,"A",{href:!0,rel:!0});var Rs=o(Ie);$n=i(Rs,"https://arxiv.org/abs/1804.04235"),Rs.forEach(r),yn=i(P,` Note that
this optimizer internally adjusts the learning rate depending on the `),Ot=n(P,"CODE",{});var js=o(Ot);xn=i(js,"scale_parameter"),js.forEach(r),An=i(P,", "),qt=n(P,"CODE",{});var Us=o(qt);zn=i(Us,"relative_step"),Us.forEach(r),En=i(P,` and
`),Rt=n(P,"CODE",{});var Gs=o(Rt);Tn=i(Gs,"warmup_init"),Gs.forEach(r),Dn=i(P," options. To use a manual (external) learning rate schedule you should set "),jt=n(P,"CODE",{});var Vs=o(jt);Ln=i(Vs,"scale_parameter=False"),Vs.forEach(r),Pn=i(P,` and
`),Ut=n(P,"CODE",{});var Ms=o(Ut);Wn=i(Ms,"relative_step=False"),Ms.forEach(r),kn=i(P,"."),P.forEach(r),Nn=m(E),Gt=n(E,"P",{});var Hs=o(Gt);Sn=i(Hs,"This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested."),Hs.forEach(r),In=m(E),Ce=n(E,"P",{});var _a=o(Ce);Cn=i(_a,"Recommended T5 finetuning settings ("),Fe=n(_a,"A",{href:!0,rel:!0});var Bs=o(Fe);Fn=i(Bs,"https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),Bs.forEach(r),On=i(_a,"):"),_a.forEach(r),qn=m(E),N=n(E,"UL",{});var Ae=o(N);Oe=n(Ae,"LI",{});var wa=o(Oe);Vt=n(wa,"P",{});var Js=o(Vt);Rn=i(Js,"Training without LR warmup or clip_threshold is not recommended."),Js.forEach(r),jn=m(wa),qe=n(wa,"UL",{});var va=o(qe);Mt=n(va,"LI",{});var Ks=o(Mt);Un=i(Ks,"use scheduled LR warm-up to fixed LR"),Ks.forEach(r),Gn=m(va),Re=n(va,"LI",{});var ba=o(Re);Vn=i(ba,"use clip_threshold=1.0 ("),je=n(ba,"A",{href:!0,rel:!0});var Qs=o(je);Mn=i(Qs,"https://arxiv.org/abs/1804.04235"),Qs.forEach(r),Hn=i(ba,")"),ba.forEach(r),va.forEach(r),wa.forEach(r),Bn=m(Ae),Ht=n(Ae,"LI",{});var Xs=o(Ht);Bt=n(Xs,"P",{});var Ys=o(Bt);Jn=i(Ys,"Disable relative updates"),Ys.forEach(r),Xs.forEach(r),Kn=m(Ae),Jt=n(Ae,"LI",{});var Zs=o(Jt);Kt=n(Zs,"P",{});var ei=o(Kt);Qn=i(ei,"Use scale_parameter=False"),ei.forEach(r),Zs.forEach(r),Xn=m(Ae),Qt=n(Ae,"LI",{});var ti=o(Qt);Xt=n(ti,"P",{});var ri=o(Xt);Yn=i(ri,"Additional optimizer operations like gradient clipping should not be used alongside Adafactor"),ri.forEach(r),ti.forEach(r),Ae.forEach(r),Zn=m(E),_(ce.$$.fragment,E),eo=m(E),_(pe.$$.fragment,E),to=m(E),O=n(E,"P",{});var wt=o(O);ro=i(wt,"When using "),Yt=n(wt,"CODE",{});var ai=o(Yt);ao=i(ai,"lr=None"),ai.forEach(r),no=i(wt," with "),bt=n(wt,"A",{href:!0});var ni=o(bt);oo=i(ni,"Trainer"),ni.forEach(r),so=i(wt," you will most likely need to use "),Zt=n(wt,"CODE",{});var oi=o(Zt);io=i(oi,"AdafactorSchedule"),oi.forEach(r),wt.forEach(r),lo=m(E),_(de.$$.fragment,E),mo=m(E),_(he.$$.fragment,E),co=m(E),ue=n(E,"DIV",{class:!0});var $a=o(ue);_(Ue.$$.fragment,$a),po=m($a),er=n($a,"P",{});var si=o(er);ho=i(si,"Performs a single optimization step"),si.forEach(r),$a.forEach(r),E.forEach(r),jr=m(e),j=n(e,"H2",{class:!0});var ya=o(j);fe=n(ya,"A",{id:!0,class:!0,href:!0});var ii=o(fe);tr=n(ii,"SPAN",{});var li=o(tr);_(Ge.$$.fragment,li),li.forEach(r),ii.forEach(r),uo=m(ya),rr=n(ya,"SPAN",{});var mi=o(rr);fo=i(mi,"AdamWeightDecay (TensorFlow)"),mi.forEach(r),ya.forEach(r),Ur=m(e),W=n(e,"DIV",{class:!0});var ze=o(W);_(Ve.$$.fragment,ze),go=m(ze),U=n(ze,"P",{});var xt=o(U);_o=i(xt,`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is `),ar=n(xt,"EM",{});var ci=o(ar);wo=i(ci,"not"),ci.forEach(r),vo=i(xt,` the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in `),Me=n(xt,"A",{href:!0,rel:!0});var pi=o(Me);bo=i(pi,`Decoupled Weight Decay
Regularization`),pi.forEach(r),$o=i(xt,"."),xt.forEach(r),yo=m(ze),nr=n(ze,"P",{});var di=o(nr);xo=i(di,`Instead we want to decay the weights in a manner that doesn\u2019t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`),di.forEach(r),Ao=m(ze),ge=n(ze,"DIV",{class:!0});var xa=o(ge);_(He.$$.fragment,xa),zo=m(xa),or=n(xa,"P",{});var hi=o(or);Eo=i(hi,"Creates an optimizer from its config with WarmUp custom object."),hi.forEach(r),xa.forEach(r),ze.forEach(r),Gr=m(e),G=n(e,"DIV",{class:!0});var Aa=o(G);_(Be.$$.fragment,Aa),To=m(Aa),sr=n(Aa,"P",{});var ui=o(sr);Do=i(ui,"Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay."),ui.forEach(r),Aa.forEach(r),Vr=m(e),V=n(e,"H2",{class:!0});var za=o(V);_e=n(za,"A",{id:!0,class:!0,href:!0});var fi=o(_e);ir=n(fi,"SPAN",{});var gi=o(ir);_(Je.$$.fragment,gi),gi.forEach(r),fi.forEach(r),Lo=m(za),lr=n(za,"SPAN",{});var _i=o(lr);Po=i(_i,"Schedules"),_i.forEach(r),za.forEach(r),Mr=m(e),M=n(e,"H3",{class:!0});var Ea=o(M);we=n(Ea,"A",{id:!0,class:!0,href:!0});var wi=o(we);mr=n(wi,"SPAN",{});var vi=o(mr);_(Ke.$$.fragment,vi),vi.forEach(r),wi.forEach(r),Wo=m(Ea),cr=n(Ea,"SPAN",{});var bi=o(cr);ko=i(bi,"Learning Rate Schedules (Pytorch)"),bi.forEach(r),Ea.forEach(r),Hr=m(e),H=n(e,"DIV",{class:!0});var Ta=o(H);_(Qe.$$.fragment,Ta),No=m(Ta),pr=n(Ta,"P",{});var $i=o(pr);So=i($i,"An enumeration."),$i.forEach(r),Ta.forEach(r),Br=m(e),B=n(e,"DIV",{class:!0});var Da=o(B);_(Xe.$$.fragment,Da),Io=m(Da),dr=n(Da,"P",{});var yi=o(dr);Co=i(yi,"Unified API to get any scheduler from its name."),yi.forEach(r),Da.forEach(r),Jr=m(e),J=n(e,"DIV",{class:!0});var La=o(J);_(Ye.$$.fragment,La),Fo=m(La),hr=n(La,"P",{});var xi=o(hr);Oo=i(xi,"Create a schedule with a constant learning rate, using the learning rate set in optimizer."),xi.forEach(r),La.forEach(r),Kr=m(e),K=n(e,"DIV",{class:!0});var Pa=o(K);_(Ze.$$.fragment,Pa),qo=m(Pa),ur=n(Pa,"P",{});var Ai=o(ur);Ro=i(Ai,`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`),Ai.forEach(r),Pa.forEach(r),Qr=m(e),et=n(e,"IMG",{alt:!0,src:!0}),Xr=m(e),Q=n(e,"DIV",{class:!0});var Wa=o(Q);_(tt.$$.fragment,Wa),jo=m(Wa),fr=n(Wa,"P",{});var zi=o(fr);Uo=i(zi,`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`),zi.forEach(r),Wa.forEach(r),Yr=m(e),rt=n(e,"IMG",{alt:!0,src:!0}),Zr=m(e),X=n(e,"DIV",{class:!0});var ka=o(X);_(at.$$.fragment,ka),Go=m(ka),gr=n(ka,"P",{});var Ei=o(gr);Vo=i(Ei,`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`),Ei.forEach(r),ka.forEach(r),ea=m(e),nt=n(e,"IMG",{alt:!0,src:!0}),ta=m(e),Y=n(e,"DIV",{class:!0});var Na=o(Y);_(ot.$$.fragment,Na),Mo=m(Na),_r=n(Na,"P",{});var Ti=o(_r);Ho=i(Ti,`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`),Ti.forEach(r),Na.forEach(r),ra=m(e),st=n(e,"IMG",{alt:!0,src:!0}),aa=m(e),S=n(e,"DIV",{class:!0});var At=o(S);_(it.$$.fragment,At),Bo=m(At),lt=n(At,"P",{});var Sa=o(lt);Jo=i(Sa,`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by `),wr=n(Sa,"EM",{});var Di=o(wr);Ko=i(Di,"lr_end"),Di.forEach(r),Qo=i(Sa,`, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`),Sa.forEach(r),Xo=m(At),ve=n(At,"P",{});var Sr=o(ve);Yo=i(Sr,"Note: "),vr=n(Sr,"EM",{});var Li=o(vr);Zo=i(Li,"power"),Li.forEach(r),es=i(Sr,` defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
`),mt=n(Sr,"A",{href:!0,rel:!0});var Pi=o(mt);ts=i(Pi,"https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),Pi.forEach(r),Sr.forEach(r),At.forEach(r),na=m(e),Z=n(e,"DIV",{class:!0});var Ia=o(Z);_(ct.$$.fragment,Ia),rs=m(Ia),br=n(Ia,"P",{});var Wi=o(br);as=i(Wi,`Create a schedule with an inverse square-root learning rate, from the initial lr set in the optimizer, after a
warmup period which increases lr linearly from 0 to the initial lr set in the optimizer.`),Wi.forEach(r),Ia.forEach(r),oa=m(e),ee=n(e,"H3",{class:!0});var Ca=o(ee);be=n(Ca,"A",{id:!0,class:!0,href:!0});var ki=o(be);$r=n(ki,"SPAN",{});var Ni=o($r);_(pt.$$.fragment,Ni),Ni.forEach(r),ki.forEach(r),ns=m(Ca),yr=n(Ca,"SPAN",{});var Si=o(yr);os=i(Si,"Warmup (TensorFlow)"),Si.forEach(r),Ca.forEach(r),sa=m(e),te=n(e,"DIV",{class:!0});var Fa=o(te);_(dt.$$.fragment,Fa),ss=m(Fa),xr=n(Fa,"P",{});var Ii=o(xr);is=i(Ii,"Applies a warmup schedule on a given learning rate decay schedule."),Ii.forEach(r),Fa.forEach(r),ia=m(e),re=n(e,"H2",{class:!0});var Oa=o(re);$e=n(Oa,"A",{id:!0,class:!0,href:!0});var Ci=o($e);Ar=n(Ci,"SPAN",{});var Fi=o(Ar);_(ht.$$.fragment,Fi),Fi.forEach(r),Ci.forEach(r),ls=m(Oa),zr=n(Oa,"SPAN",{});var Oi=o(zr);ms=i(Oi,"Gradient Strategies"),Oi.forEach(r),Oa.forEach(r),la=m(e),ae=n(e,"H3",{class:!0});var qa=o(ae);ye=n(qa,"A",{id:!0,class:!0,href:!0});var qi=o(ye);Er=n(qi,"SPAN",{});var Ri=o(Er);_(ut.$$.fragment,Ri),Ri.forEach(r),qi.forEach(r),cs=m(qa),Tr=n(qa,"SPAN",{});var ji=o(Tr);ps=i(ji,"GradientAccumulator (TensorFlow)"),ji.forEach(r),qa.forEach(r),ma=m(e),I=n(e,"DIV",{class:!0});var zt=o(I);_(ft.$$.fragment,zt),ds=m(zt),ne=n(zt,"P",{});var Et=o(ne);hs=i(Et,`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call `),Dr=n(Et,"CODE",{});var Ui=o(Dr);us=i(Ui,".gradients"),Ui.forEach(r),fs=i(Et,", scale the gradients if required, and pass the result to "),Lr=n(Et,"CODE",{});var Gi=o(Lr);gs=i(Gi,"apply_gradients"),Gi.forEach(r),_s=i(Et,"."),Et.forEach(r),ws=m(zt),xe=n(zt,"DIV",{class:!0});var Ra=o(xe);_(gt.$$.fragment,Ra),vs=m(Ra),Pr=n(Ra,"P",{});var Vi=o(Pr);bs=i(Vi,"Resets the accumulated gradients on the current replica."),Vi.forEach(r),Ra.forEach(r),zt.forEach(r),this.h()},h(){c(f,"name","hf:doc:metadata"),c(f,"content",JSON.stringify(tl)),c(u,"id","optimization"),c(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(u,"href","#optimization"),c(x,"class","relative group"),c(ie,"id","transformers.AdamW"),c(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ie,"href","#transformers.AdamW"),c(q,"class","relative group"),c(Pe,"href","https://arxiv.org/abs/1711.05101"),c(Pe,"rel","nofollow"),c(le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(me,"id","transformers.Adafactor"),c(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(me,"href","#transformers.Adafactor"),c(R,"class","relative group"),c(Se,"href","https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),c(Se,"rel","nofollow"),c(Ie,"href","https://arxiv.org/abs/1804.04235"),c(Ie,"rel","nofollow"),c(Fe,"href","https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),c(Fe,"rel","nofollow"),c(je,"href","https://arxiv.org/abs/1804.04235"),c(je,"rel","nofollow"),c(bt,"href","/docs/transformers/v4.27.1/en/main_classes/trainer#transformers.Trainer"),c(ue,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(fe,"id","transformers.AdamWeightDecay"),c(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(fe,"href","#transformers.AdamWeightDecay"),c(j,"class","relative group"),c(Me,"href","https://arxiv.org/abs/1711.05101"),c(Me,"rel","nofollow"),c(ge,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(_e,"id","schedules"),c(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(_e,"href","#schedules"),c(V,"class","relative group"),c(we,"id","transformers.SchedulerType"),c(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(we,"href","#transformers.SchedulerType"),c(M,"class","relative group"),c(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(et,"alt",""),ja(et.src,ys="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_constant_schedule.png")||c(et,"src",ys),c(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(rt,"alt",""),ja(rt.src,xs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_schedule.png")||c(rt,"src",xs),c(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(nt,"alt",""),ja(nt.src,As="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_hard_restarts_schedule.png")||c(nt,"src",As),c(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(st,"alt",""),ja(st.src,zs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_linear_schedule.png")||c(st,"src",zs),c(mt,"href","https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),c(mt,"rel","nofollow"),c(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(be,"id","transformers.WarmUp"),c(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(be,"href","#transformers.WarmUp"),c(ee,"class","relative group"),c(te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c($e,"id","gradient-strategies"),c($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c($e,"href","#gradient-strategies"),c(re,"class","relative group"),c(ye,"id","transformers.GradientAccumulator"),c(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ye,"href","#transformers.GradientAccumulator"),c(ae,"class","relative group"),c(xe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,p){t(document.head,f),h(e,T,p),h(e,x,p),t(x,u),t(u,z),w(d,z,null),t(x,y),t(x,Tt),t(Tt,Ma),h(e,Ir,p),h(e,se,p),t(se,Ha),t(se,Dt),t(Dt,Ba),t(se,Ja),h(e,Cr,p),h(e,F,p),t(F,Lt),t(Lt,Ka),t(F,Qa),t(F,Ee),t(Ee,Xa),t(Ee,Pt),t(Pt,Ya),t(Ee,Za),t(F,en),t(F,Wt),t(Wt,tn),h(e,Fr,p),h(e,q,p),t(q,ie),t(ie,kt),w(Te,kt,null),t(q,rn),t(q,Nt),t(Nt,an),h(e,Or,p),h(e,k,p),w(De,k,null),t(k,nn),t(k,Le),t(Le,on),t(Le,Pe),t(Pe,sn),t(Le,ln),t(k,mn),t(k,le),w(We,le,null),t(le,cn),t(le,St),t(St,pn),h(e,qr,p),h(e,R,p),t(R,me),t(me,It),w(ke,It,null),t(R,dn),t(R,Ct),t(Ct,hn),h(e,Rr,p),h(e,A,p),w(Ne,A,null),t(A,un),t(A,vt),t(vt,fn),t(vt,Se),t(Se,gn),t(A,_n),t(A,D),t(D,wn),t(D,Ft),t(Ft,vn),t(D,bn),t(D,Ie),t(Ie,$n),t(D,yn),t(D,Ot),t(Ot,xn),t(D,An),t(D,qt),t(qt,zn),t(D,En),t(D,Rt),t(Rt,Tn),t(D,Dn),t(D,jt),t(jt,Ln),t(D,Pn),t(D,Ut),t(Ut,Wn),t(D,kn),t(A,Nn),t(A,Gt),t(Gt,Sn),t(A,In),t(A,Ce),t(Ce,Cn),t(Ce,Fe),t(Fe,Fn),t(Ce,On),t(A,qn),t(A,N),t(N,Oe),t(Oe,Vt),t(Vt,Rn),t(Oe,jn),t(Oe,qe),t(qe,Mt),t(Mt,Un),t(qe,Gn),t(qe,Re),t(Re,Vn),t(Re,je),t(je,Mn),t(Re,Hn),t(N,Bn),t(N,Ht),t(Ht,Bt),t(Bt,Jn),t(N,Kn),t(N,Jt),t(Jt,Kt),t(Kt,Qn),t(N,Xn),t(N,Qt),t(Qt,Xt),t(Xt,Yn),t(A,Zn),w(ce,A,null),t(A,eo),w(pe,A,null),t(A,to),t(A,O),t(O,ro),t(O,Yt),t(Yt,ao),t(O,no),t(O,bt),t(bt,oo),t(O,so),t(O,Zt),t(Zt,io),t(A,lo),w(de,A,null),t(A,mo),w(he,A,null),t(A,co),t(A,ue),w(Ue,ue,null),t(ue,po),t(ue,er),t(er,ho),h(e,jr,p),h(e,j,p),t(j,fe),t(fe,tr),w(Ge,tr,null),t(j,uo),t(j,rr),t(rr,fo),h(e,Ur,p),h(e,W,p),w(Ve,W,null),t(W,go),t(W,U),t(U,_o),t(U,ar),t(ar,wo),t(U,vo),t(U,Me),t(Me,bo),t(U,$o),t(W,yo),t(W,nr),t(nr,xo),t(W,Ao),t(W,ge),w(He,ge,null),t(ge,zo),t(ge,or),t(or,Eo),h(e,Gr,p),h(e,G,p),w(Be,G,null),t(G,To),t(G,sr),t(sr,Do),h(e,Vr,p),h(e,V,p),t(V,_e),t(_e,ir),w(Je,ir,null),t(V,Lo),t(V,lr),t(lr,Po),h(e,Mr,p),h(e,M,p),t(M,we),t(we,mr),w(Ke,mr,null),t(M,Wo),t(M,cr),t(cr,ko),h(e,Hr,p),h(e,H,p),w(Qe,H,null),t(H,No),t(H,pr),t(pr,So),h(e,Br,p),h(e,B,p),w(Xe,B,null),t(B,Io),t(B,dr),t(dr,Co),h(e,Jr,p),h(e,J,p),w(Ye,J,null),t(J,Fo),t(J,hr),t(hr,Oo),h(e,Kr,p),h(e,K,p),w(Ze,K,null),t(K,qo),t(K,ur),t(ur,Ro),h(e,Qr,p),h(e,et,p),h(e,Xr,p),h(e,Q,p),w(tt,Q,null),t(Q,jo),t(Q,fr),t(fr,Uo),h(e,Yr,p),h(e,rt,p),h(e,Zr,p),h(e,X,p),w(at,X,null),t(X,Go),t(X,gr),t(gr,Vo),h(e,ea,p),h(e,nt,p),h(e,ta,p),h(e,Y,p),w(ot,Y,null),t(Y,Mo),t(Y,_r),t(_r,Ho),h(e,ra,p),h(e,st,p),h(e,aa,p),h(e,S,p),w(it,S,null),t(S,Bo),t(S,lt),t(lt,Jo),t(lt,wr),t(wr,Ko),t(lt,Qo),t(S,Xo),t(S,ve),t(ve,Yo),t(ve,vr),t(vr,Zo),t(ve,es),t(ve,mt),t(mt,ts),h(e,na,p),h(e,Z,p),w(ct,Z,null),t(Z,rs),t(Z,br),t(br,as),h(e,oa,p),h(e,ee,p),t(ee,be),t(be,$r),w(pt,$r,null),t(ee,ns),t(ee,yr),t(yr,os),h(e,sa,p),h(e,te,p),w(dt,te,null),t(te,ss),t(te,xr),t(xr,is),h(e,ia,p),h(e,re,p),t(re,$e),t($e,Ar),w(ht,Ar,null),t(re,ls),t(re,zr),t(zr,ms),h(e,la,p),h(e,ae,p),t(ae,ye),t(ye,Er),w(ut,Er,null),t(ae,cs),t(ae,Tr),t(Tr,ps),h(e,ma,p),h(e,I,p),w(ft,I,null),t(I,ds),t(I,ne),t(ne,hs),t(ne,Dr),t(Dr,us),t(ne,fs),t(ne,Lr),t(Lr,gs),t(ne,_s),t(I,ws),t(I,xe),w(gt,xe,null),t(xe,vs),t(xe,Pr),t(Pr,bs),ca=!0},p(e,[p]){const _t={};p&2&&(_t.$$scope={dirty:p,ctx:e}),ce.$set(_t);const Wr={};p&2&&(Wr.$$scope={dirty:p,ctx:e}),pe.$set(Wr);const kr={};p&2&&(kr.$$scope={dirty:p,ctx:e}),de.$set(kr);const Nr={};p&2&&(Nr.$$scope={dirty:p,ctx:e}),he.$set(Nr)},i(e){ca||(v(d.$$.fragment,e),v(Te.$$.fragment,e),v(De.$$.fragment,e),v(We.$$.fragment,e),v(ke.$$.fragment,e),v(Ne.$$.fragment,e),v(ce.$$.fragment,e),v(pe.$$.fragment,e),v(de.$$.fragment,e),v(he.$$.fragment,e),v(Ue.$$.fragment,e),v(Ge.$$.fragment,e),v(Ve.$$.fragment,e),v(He.$$.fragment,e),v(Be.$$.fragment,e),v(Je.$$.fragment,e),v(Ke.$$.fragment,e),v(Qe.$$.fragment,e),v(Xe.$$.fragment,e),v(Ye.$$.fragment,e),v(Ze.$$.fragment,e),v(tt.$$.fragment,e),v(at.$$.fragment,e),v(ot.$$.fragment,e),v(it.$$.fragment,e),v(ct.$$.fragment,e),v(pt.$$.fragment,e),v(dt.$$.fragment,e),v(ht.$$.fragment,e),v(ut.$$.fragment,e),v(ft.$$.fragment,e),v(gt.$$.fragment,e),ca=!0)},o(e){b(d.$$.fragment,e),b(Te.$$.fragment,e),b(De.$$.fragment,e),b(We.$$.fragment,e),b(ke.$$.fragment,e),b(Ne.$$.fragment,e),b(ce.$$.fragment,e),b(pe.$$.fragment,e),b(de.$$.fragment,e),b(he.$$.fragment,e),b(Ue.$$.fragment,e),b(Ge.$$.fragment,e),b(Ve.$$.fragment,e),b(He.$$.fragment,e),b(Be.$$.fragment,e),b(Je.$$.fragment,e),b(Ke.$$.fragment,e),b(Qe.$$.fragment,e),b(Xe.$$.fragment,e),b(Ye.$$.fragment,e),b(Ze.$$.fragment,e),b(tt.$$.fragment,e),b(at.$$.fragment,e),b(ot.$$.fragment,e),b(it.$$.fragment,e),b(ct.$$.fragment,e),b(pt.$$.fragment,e),b(dt.$$.fragment,e),b(ht.$$.fragment,e),b(ut.$$.fragment,e),b(ft.$$.fragment,e),b(gt.$$.fragment,e),ca=!1},d(e){r(f),e&&r(T),e&&r(x),$(d),e&&r(Ir),e&&r(se),e&&r(Cr),e&&r(F),e&&r(Fr),e&&r(q),$(Te),e&&r(Or),e&&r(k),$(De),$(We),e&&r(qr),e&&r(R),$(ke),e&&r(Rr),e&&r(A),$(Ne),$(ce),$(pe),$(de),$(he),$(Ue),e&&r(jr),e&&r(j),$(Ge),e&&r(Ur),e&&r(W),$(Ve),$(He),e&&r(Gr),e&&r(G),$(Be),e&&r(Vr),e&&r(V),$(Je),e&&r(Mr),e&&r(M),$(Ke),e&&r(Hr),e&&r(H),$(Qe),e&&r(Br),e&&r(B),$(Xe),e&&r(Jr),e&&r(J),$(Ye),e&&r(Kr),e&&r(K),$(Ze),e&&r(Qr),e&&r(et),e&&r(Xr),e&&r(Q),$(tt),e&&r(Yr),e&&r(rt),e&&r(Zr),e&&r(X),$(at),e&&r(ea),e&&r(nt),e&&r(ta),e&&r(Y),$(ot),e&&r(ra),e&&r(st),e&&r(aa),e&&r(S),$(it),e&&r(na),e&&r(Z),$(ct),e&&r(oa),e&&r(ee),$(pt),e&&r(sa),e&&r(te),$(dt),e&&r(ia),e&&r(re),$(ht),e&&r(la),e&&r(ae),$(ut),e&&r(ma),e&&r(I),$(ft),$(gt)}}}const tl={local:"optimization",sections:[{local:"transformers.AdamW",title:"AdamW (PyTorch)"},{local:"transformers.Adafactor",title:"AdaFactor (PyTorch)"},{local:"transformers.AdamWeightDecay",title:"AdamWeightDecay (TensorFlow)"},{local:"schedules",sections:[{local:"transformers.SchedulerType",title:"Learning Rate Schedules (Pytorch)"},{local:"transformers.WarmUp",title:"Warmup (TensorFlow)"}],title:"Schedules"},{local:"gradient-strategies",sections:[{local:"transformers.GradientAccumulator",title:"GradientAccumulator (TensorFlow)"}],title:"Gradient Strategies"}],title:"Optimization"};function rl(C){return Ki(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ll extends Mi{constructor(f){super();Hi(this,f,rl,el,Bi,{})}}export{ll as default,tl as metadata};
