import{S as Zi,i as Oi,s as qi,e as a,k as l,w as g,t as s,M as Bi,c as n,d as r,m,a as o,x as _,h as i,b as c,N as Fa,G as t,g as h,y as w,q as v,o as b,B as y,v as Xi,L as Ja}from"../../chunks/vendor-hf-doc-builder.js";import{D as W}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Va}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as oe}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as Ga}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function Yi(R){let f,T,x,u,z;return u=new Va({props:{code:"QWRhZmFjdG9yKG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMHNjYWxlX3BhcmFtZXRlciUzREZhbHNlJTJDJTIwcmVsYXRpdmVfc3RlcCUzREZhbHNlJTJDJTIwd2FybXVwX2luaXQlM0RGYWxzZSUyQyUyMGxyJTNEMWUtMyk=",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, warmup_init=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">1e-3</span>)'}}),{c(){f=a("p"),T=s("Example:"),x=l(),g(u.$$.fragment)},l(d){f=n(d,"P",{});var $=o(f);T=i($,"Example:"),$.forEach(r),x=m(d),_(u.$$.fragment,d)},m(d,$){h(d,f,$),t(f,T),h(d,x,$),w(u,d,$),z=!0},p:Ja,i(d){z||(v(u.$$.fragment,d),z=!0)},o(d){b(u.$$.fragment,d),z=!1},d(d){d&&r(f),d&&r(x),y(u,d)}}}function Hi(R){let f,T,x,u,z;return u=new Va({props:{code:"QWRhZmFjdG9yKG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMHNjYWxlX3BhcmFtZXRlciUzRFRydWUlMkMlMjByZWxhdGl2ZV9zdGVwJTNEVHJ1ZSUyQyUyMHdhcm11cF9pbml0JTNEVHJ1ZSUyQyUyMGxyJTNETm9uZSk=",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)'}}),{c(){f=a("p"),T=s("Others reported the following combination to work well:"),x=l(),g(u.$$.fragment)},l(d){f=n(d,"P",{});var $=o(f);T=i($,"Others reported the following combination to work well:"),$.forEach(r),x=m(d),_(u.$$.fragment,d)},m(d,$){h(d,f,$),t(f,T),h(d,x,$),w(u,d,$),z=!0},p:Ja,i(d){z||(v(u.$$.fragment,d),z=!0)},o(d){b(u.$$.fragment,d),z=!1},d(d){d&&r(f),d&&r(x),y(u,d)}}}function Qi(R){let f,T,x,u,z;return u=new Va({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5vcHRpbWl6YXRpb24lMjBpbXBvcnQlMjBBZGFmYWN0b3IlMkMlMjBBZGFmYWN0b3JTY2hlZHVsZSUwQSUwQW9wdGltaXplciUyMCUzRCUyMEFkYWZhY3Rvcihtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMjBzY2FsZV9wYXJhbWV0ZXIlM0RUcnVlJTJDJTIwcmVsYXRpdmVfc3RlcCUzRFRydWUlMkMlMjB3YXJtdXBfaW5pdCUzRFRydWUlMkMlMjBsciUzRE5vbmUpJTBBbHJfc2NoZWR1bGVyJTIwJTNEJTIwQWRhZmFjdG9yU2NoZWR1bGUob3B0aW1pemVyKSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKC4uLiUyQyUyMG9wdGltaXplcnMlM0Qob3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGVyKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, AdafactorSchedule

optimizer = Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)
lr_scheduler = AdafactorSchedule(optimizer)
trainer = Trainer(..., optimizers=(optimizer, lr_scheduler))`}}),{c(){f=a("p"),T=s("scheduler as following:"),x=l(),g(u.$$.fragment)},l(d){f=n(d,"P",{});var $=o(f);T=i($,"scheduler as following:"),$.forEach(r),x=m(d),_(u.$$.fragment,d)},m(d,$){h(d,f,$),t(f,T),h(d,x,$),w(u,d,$),z=!0},p:Ja,i(d){z||(v(u.$$.fragment,d),z=!0)},o(d){b(u.$$.fragment,d),z=!1},d(d){d&&r(f),d&&r(x),y(u,d)}}}function Ki(R){let f,T,x,u,z;return u=new Va({props:{code:"JTIzJTIwcmVwbGFjZSUyMEFkYW1XJTIwd2l0aCUyMEFkYWZhY3RvciUwQW9wdGltaXplciUyMCUzRCUyMEFkYWZhY3RvciglMEElMjAlMjAlMjAlMjBtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMEElMjAlMjAlMjAlMjBsciUzRDFlLTMlMkMlMEElMjAlMjAlMjAlMjBlcHMlM0QoMWUtMzAlMkMlMjAxZS0zKSUyQyUwQSUyMCUyMCUyMCUyMGNsaXBfdGhyZXNob2xkJTNEMS4wJTJDJTBBJTIwJTIwJTIwJTIwZGVjYXlfcmF0ZSUzRC0wLjglMkMlMEElMjAlMjAlMjAlMjBiZXRhMSUzRE5vbmUlMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRfZGVjYXklM0QwLjAlMkMlMEElMjAlMjAlMjAlMjByZWxhdGl2ZV9zdGVwJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBzY2FsZV9wYXJhbWV0ZXIlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMHdhcm11cF9pbml0JTNERmFsc2UlMkMlMEEp",highlighted:`<span class="hljs-comment"># replace AdamW with Adafactor</span>
optimizer = Adafactor(
    model.parameters(),
    lr=<span class="hljs-number">1e-3</span>,
    eps=(<span class="hljs-number">1e-30</span>, <span class="hljs-number">1e-3</span>),
    clip_threshold=<span class="hljs-number">1.0</span>,
    decay_rate=-<span class="hljs-number">0.8</span>,
    beta1=<span class="hljs-literal">None</span>,
    weight_decay=<span class="hljs-number">0.0</span>,
    relative_step=<span class="hljs-literal">False</span>,
    scale_parameter=<span class="hljs-literal">False</span>,
    warmup_init=<span class="hljs-literal">False</span>,
)`}}),{c(){f=a("p"),T=s("Usage:"),x=l(),g(u.$$.fragment)},l(d){f=n(d,"P",{});var $=o(f);T=i($,"Usage:"),$.forEach(r),x=m(d),_(u.$$.fragment,d)},m(d,$){h(d,f,$),t(f,T),h(d,x,$),w(u,d,$),z=!0},p:Ja,i(d){z||(v(u.$$.fragment,d),z=!0)},o(d){b(u.$$.fragment,d),z=!1},d(d){d&&r(f),d&&r(x),y(u,d)}}}function el(R){let f,T,x,u,z,d,$,Tt,Za,Pr,se,Oa,Mt,qa,Ba,Rr,S,Wt,Xa,Ya,Ee,Ha,Dt,Qa,Ka,en,Lt,tn,Sr,N,ie,Ut,Te,rn,kt,an,Ir,U,Me,nn,We,on,De,sn,ln,mn,le,Le,cn,jt,pn,Nr,C,me,Pt,Ue,dn,Rt,hn,Cr,A,ke,un,vt,fn,je,gn,_n,M,wn,St,vn,bn,Pe,yn,$n,It,xn,An,Nt,zn,En,Ct,Tn,Mn,Ft,Wn,Dn,Gt,Ln,Un,kn,Jt,jn,Pn,Re,Rn,Se,Sn,In,Nn,k,Ie,Vt,Cn,Fn,Ne,Zt,Gn,Jn,Ce,Vn,Fe,Zn,On,qn,Ot,qt,Bn,Xn,Bt,Xt,Yn,Hn,Yt,Ht,Qn,Kn,ce,eo,pe,to,I,ro,Qt,ao,no,bt,oo,so,Kt,io,lo,de,mo,he,co,ue,Ge,po,er,ho,Fr,F,fe,tr,Je,uo,rr,fo,Gr,L,Ve,go,G,_o,ar,wo,vo,Ze,bo,yo,$o,nr,xo,Ao,ge,Oe,zo,or,Eo,Jr,J,qe,To,sr,Mo,Vr,V,_e,ir,Be,Wo,lr,Do,Zr,Z,we,mr,Xe,Lo,cr,Uo,Or,O,Ye,ko,pr,jo,qr,q,He,Po,dr,Ro,Br,B,Qe,So,hr,Io,Xr,X,Ke,No,ur,Co,Yr,et,$s,Hr,Y,tt,Fo,fr,Go,Qr,rt,xs,Kr,H,at,Jo,gr,Vo,ea,nt,As,ta,Q,ot,Zo,_r,Oo,ra,st,zs,aa,j,it,qo,lt,Bo,wr,Xo,Yo,Ho,ve,Qo,vr,Ko,es,mt,ts,na,K,ct,rs,br,as,oa,ee,be,yr,pt,ns,$r,os,sa,te,dt,ss,xr,is,ia,re,ye,Ar,ht,ls,zr,ms,la,ae,$e,Er,ut,cs,Tr,ps,ma,P,ft,ds,ne,hs,Mr,us,fs,Wr,gs,_s,ws,xe,gt,vs,Dr,bs,ca;return d=new oe({}),Te=new oe({}),Me=new W({props:{name:"class transformers.AdamW",anchor:"transformers.AdamW",parameters:[{name:"params",val:": typing.Iterable[torch.nn.parameter.Parameter]"},{name:"lr",val:": float = 0.001"},{name:"betas",val:": typing.Tuple[float, float] = (0.9, 0.999)"},{name:"eps",val:": float = 1e-06"},{name:"weight_decay",val:": float = 0.0"},{name:"correct_bias",val:": bool = True"},{name:"no_deprecation_warning",val:": bool = False"}],parametersDescription:[{anchor:"transformers.AdamW.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.AdamW.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-3) &#x2014;
The learning rate to use.`,name:"lr"},{anchor:"transformers.AdamW.betas",description:`<strong>betas</strong> (<code>Tuple[float,float]</code>, <em>optional</em>, defaults to (0.9, 0.999)) &#x2014;
Adam&#x2019;s betas parameters (b1, b2).`,name:"betas"},{anchor:"transformers.AdamW.eps",description:`<strong>eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-6) &#x2014;
Adam&#x2019;s epsilon for numerical stability.`,name:"eps"},{anchor:"transformers.AdamW.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
Decoupled weight decay to apply.`,name:"weight_decay"},{anchor:"transformers.AdamW.correct_bias",description:`<strong>correct_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to correct bias in Adam (for instance, in Bert TF repository they use <code>False</code>).`,name:"correct_bias"},{anchor:"transformers.AdamW.no_deprecation_warning",description:`<strong>no_deprecation_warning</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
A flag used to disable the deprecation warning (set to <code>True</code> to disable the warning).`,name:"no_deprecation_warning"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L378"}}),Le=new W({props:{name:"step",anchor:"transformers.AdamW.step",parameters:[{name:"closure",val:": typing.Callable = None"}],parametersDescription:[{anchor:"transformers.AdamW.step.closure",description:"<strong>closure</strong> (<code>Callable</code>, <em>optional</em>) &#x2014; A closure that reevaluates the model and returns the loss.",name:"closure"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L429"}}),Ue=new oe({}),ke=new W({props:{name:"class transformers.Adafactor",anchor:"transformers.Adafactor",parameters:[{name:"params",val:""},{name:"lr",val:" = None"},{name:"eps",val:" = (1e-30, 0.001)"},{name:"clip_threshold",val:" = 1.0"},{name:"decay_rate",val:" = -0.8"},{name:"beta1",val:" = None"},{name:"weight_decay",val:" = 0.0"},{name:"scale_parameter",val:" = True"},{name:"relative_step",val:" = True"},{name:"warmup_init",val:" = False"}],parametersDescription:[{anchor:"transformers.Adafactor.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.Adafactor.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>) &#x2014;
The external learning rate.`,name:"lr"},{anchor:"transformers.Adafactor.eps",description:`<strong>eps</strong> (<code>Tuple[float, float]</code>, <em>optional</em>, defaults to (1e-30, 1e-3)) &#x2014;
Regularization constants for square gradient and parameter scale respectively`,name:"eps"},{anchor:"transformers.Adafactor.clip_threshold",description:`<strong>clip_threshold</strong> (<code>float</code>, <em>optional</em>, defaults 1.0) &#x2014;
Threshold of root mean square of final gradient update`,name:"clip_threshold"},{anchor:"transformers.Adafactor.decay_rate",description:`<strong>decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to -0.8) &#x2014;
Coefficient used to compute running averages of square`,name:"decay_rate"},{anchor:"transformers.Adafactor.beta1",description:`<strong>beta1</strong> (<code>float</code>, <em>optional</em>) &#x2014;
Coefficient used for computing running averages of gradient`,name:"beta1"},{anchor:"transformers.Adafactor.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
Weight decay (L2 penalty)`,name:"weight_decay"},{anchor:"transformers.Adafactor.scale_parameter",description:`<strong>scale_parameter</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, learning rate is scaled by root mean square`,name:"scale_parameter"},{anchor:"transformers.Adafactor.relative_step",description:`<strong>relative_step</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, time-dependent learning rate is computed instead of external learning rate`,name:"relative_step"},{anchor:"transformers.Adafactor.warmup_init",description:`<strong>warmup_init</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Time-dependent learning rate computation depends on whether warm-up initialization is being used`,name:"warmup_init"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L492"}}),ce=new Ga({props:{anchor:"transformers.Adafactor.example",$$slots:{default:[Yi]},$$scope:{ctx:R}}}),pe=new Ga({props:{anchor:"transformers.Adafactor.example-2",$$slots:{default:[Hi]},$$scope:{ctx:R}}}),de=new Ga({props:{anchor:"transformers.Adafactor.example-3",$$slots:{default:[Qi]},$$scope:{ctx:R}}}),he=new Ga({props:{anchor:"transformers.Adafactor.example-4",$$slots:{default:[Ki]},$$scope:{ctx:R}}}),Ge=new W({props:{name:"step",anchor:"transformers.Adafactor.step",parameters:[{name:"closure",val:" = None"}],parametersDescription:[{anchor:"transformers.Adafactor.step.closure",description:`<strong>closure</strong> (callable, optional) &#x2014; A closure that reevaluates the model
and returns the loss.`,name:"closure"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L638"}}),Je=new oe({}),Ve=new W({props:{name:"class transformers.AdamWeightDecay",anchor:"transformers.AdamWeightDecay",parameters:[{name:"learning_rate",val:": typing.Union[float, keras.optimizers.schedules.learning_rate_schedule.LearningRateSchedule] = 0.001"},{name:"beta_1",val:": float = 0.9"},{name:"beta_2",val:": float = 0.999"},{name:"epsilon",val:": float = 1e-07"},{name:"amsgrad",val:": bool = False"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"exclude_from_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"name",val:": str = 'AdamWeightDecay'"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.AdamWeightDecay.learning_rate",description:`<strong>learning_rate</strong> (<code>Union[float, tf.keras.optimizers.schedules.LearningRateSchedule]</code>, <em>optional</em>, defaults to 1e-3) &#x2014;
The learning rate to use or a schedule.`,name:"learning_rate"},{anchor:"transformers.AdamWeightDecay.beta_1",description:`<strong>beta_1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 parameter in Adam, which is the exponential decay rate for the 1st momentum estimates.`,name:"beta_1"},{anchor:"transformers.AdamWeightDecay.beta_2",description:`<strong>beta_2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 parameter in Adam, which is the exponential decay rate for the 2nd momentum estimates.`,name:"beta_2"},{anchor:"transformers.AdamWeightDecay.epsilon",description:`<strong>epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The epsilon parameter in Adam, which is a small constant for numerical stability.`,name:"epsilon"},{anchor:"transformers.AdamWeightDecay.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, <em>optional</em>, default to <code>False</code>) &#x2014;
Whether to apply AMSGrad variant of this algorithm or not, see <a href="https://arxiv.org/abs/1904.09237" rel="nofollow">On the Convergence of Adam and
Beyond</a>.`,name:"amsgrad"},{anchor:"transformers.AdamWeightDecay.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to apply.`,name:"weight_decay_rate"},{anchor:"transformers.AdamWeightDecay.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters by default (unless they are in <code>exclude_from_weight_decay</code>).`,name:"include_in_weight_decay"},{anchor:"transformers.AdamWeightDecay.exclude_from_weight_decay",description:`<strong>exclude_from_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to exclude from applying weight decay to. If a
<code>include_in_weight_decay</code> is passed, the names in it will supersede this list.`,name:"exclude_from_weight_decay"},{anchor:"transformers.AdamWeightDecay.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>, defaults to &#x2018;AdamWeightDecay&#x2019;) &#x2014;
Optional name for the operations created when applying gradients.
kwargs &#x2014;
Keyword arguments. Allowed to be {<code>clipnorm</code>, <code>clipvalue</code>, <code>lr</code>, <code>decay</code>}. <code>clipnorm</code> is clip gradients by
norm; <code>clipvalue</code> is clip gradients by value, <code>decay</code> is included for backward compatibility to allow time
inverse decay of learning rate. <code>lr</code> is included for backward compatibility, recommended to use
<code>learning_rate</code> instead.`,name:"name"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization_tf.py#L172"}}),Oe=new W({props:{name:"from_config",anchor:"transformers.AdamWeightDecay.from_config",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization_tf.py#L229"}}),qe=new W({props:{name:"transformers.create_optimizer",anchor:"transformers.create_optimizer",parameters:[{name:"init_lr",val:": float"},{name:"num_train_steps",val:": int"},{name:"num_warmup_steps",val:": int"},{name:"min_lr_ratio",val:": float = 0.0"},{name:"adam_beta1",val:": float = 0.9"},{name:"adam_beta2",val:": float = 0.999"},{name:"adam_epsilon",val:": float = 1e-08"},{name:"adam_clipnorm",val:": typing.Optional[float] = None"},{name:"adam_global_clipnorm",val:": typing.Optional[float] = None"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"power",val:": float = 1.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"}],parametersDescription:[{anchor:"transformers.create_optimizer.init_lr",description:`<strong>init_lr</strong> (<code>float</code>) &#x2014;
The desired learning rate at the end of the warmup phase.`,name:"init_lr"},{anchor:"transformers.create_optimizer.num_train_steps",description:`<strong>num_train_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_train_steps"},{anchor:"transformers.create_optimizer.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of warmup steps.`,name:"num_warmup_steps"},{anchor:"transformers.create_optimizer.min_lr_ratio",description:`<strong>min_lr_ratio</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The final learning rate at the end of the linear decay will be <code>init_lr * min_lr_ratio</code>.`,name:"min_lr_ratio"},{anchor:"transformers.create_optimizer.adam_beta1",description:`<strong>adam_beta1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 to use in Adam.`,name:"adam_beta1"},{anchor:"transformers.create_optimizer.adam_beta2",description:`<strong>adam_beta2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 to use in Adam.`,name:"adam_beta2"},{anchor:"transformers.create_optimizer.adam_epsilon",description:`<strong>adam_epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-8) &#x2014;
The epsilon to use in Adam.`,name:"adam_epsilon"},{anchor:"transformers.create_optimizer.adam_clipnorm",description:`<strong>adam_clipnorm</strong> (<code>float</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
If not <code>None</code>, clip the gradient norm for each weight tensor to this value.`,name:"adam_clipnorm"},{anchor:"transformers.create_optimizer.adam_global_clipnorm",description:`<strong>adam_global_clipnorm</strong> (<code>float</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
If not <code>None</code>, clip gradient norm to this value. When using this argument, the norm is computed over all
weight tensors, as if they were concatenated into a single vector.`,name:"adam_global_clipnorm"},{anchor:"transformers.create_optimizer.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to use.`,name:"weight_decay_rate"},{anchor:"transformers.create_optimizer.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The power to use for PolynomialDecay.`,name:"power"},{anchor:"transformers.create_optimizer.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters except bias and layer norm parameters.`,name:"include_in_weight_decay"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization_tf.py#L88"}}),Be=new oe({}),Xe=new oe({}),Ye=new W({props:{name:"class transformers.SchedulerType",anchor:"transformers.SchedulerType",parameters:[{name:"value",val:""},{name:"names",val:" = None"},{name:"module",val:" = None"},{name:"qualname",val:" = None"},{name:"type",val:" = None"},{name:"start",val:" = 1"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/trainer_utils.py#L364"}}),He=new W({props:{name:"transformers.get_scheduler",anchor:"transformers.get_scheduler",parameters:[{name:"name",val:": typing.Union[str, transformers.trainer_utils.SchedulerType]"},{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": typing.Optional[int] = None"},{name:"num_training_steps",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"transformers.get_scheduler.name",description:`<strong>name</strong> (<code>str</code> or <code>SchedulerType</code>) &#x2014;
The name of the scheduler to use.`,name:"name"},{anchor:"transformers.get_scheduler.optimizer",description:`<strong>optimizer</strong> (<code>torch.optim.Optimizer</code>) &#x2014;
The optimizer that will be used during training.`,name:"optimizer"},{anchor:"transformers.get_scheduler.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The number of warmup steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_warmup_steps"},{anchor:"transformers.get_scheduler.num_training_steps",description:`<strong>num_training_steps</strong> (\`int&#x201C;, <em>optional</em>) &#x2014;
The number of training steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_training_steps"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L335"}}),Qe=new W({props:{name:"transformers.get_constant_schedule",anchor:"transformers.get_constant_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L39",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Ke=new W({props:{name:"transformers.get_constant_schedule_with_warmup",anchor:"transformers.get_constant_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_constant_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L77",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),tt=new W({props:{name:"transformers.get_cosine_schedule_with_warmup",anchor:"transformers.get_cosine_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": float = 0.5"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>float</code>, <em>optional</em>, defaults to 0.5) &#x2014;
The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0
following a half-cosine).`,name:"num_cycles"},{anchor:"transformers.get_cosine_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L140",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),at=new W({props:{name:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": int = 1"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of hard restarts to use.`,name:"num_cycles"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L185",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),ot=new W({props:{name:"transformers.get_linear_schedule_with_warmup",anchor:"transformers.get_linear_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_linear_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_linear_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L104",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),it=new W({props:{name:"transformers.get_polynomial_decay_schedule_with_warmup",anchor:"transformers.get_polynomial_decay_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"lr_end",val:" = 1e-07"},{name:"power",val:" = 1.0"},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.lr_end",description:`<strong>lr_end</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The end LR.`,name:"lr_end"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
Power factor.`,name:"power"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L239",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),ct=new W({props:{name:"transformers.get_inverse_sqrt_schedule",anchor:"transformers.get_inverse_sqrt_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"timescale",val:": int = None"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_inverse_sqrt_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_inverse_sqrt_schedule.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_inverse_sqrt_schedule.timescale",description:`<strong>timescale</strong> (<code>int</code>, <em>optional</em>, defaults to <code>num_warmup_steps</code>) &#x2014;
Time scale.`,name:"timescale"},{anchor:"transformers.get_inverse_sqrt_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization.py#L293",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),pt=new oe({}),dt=new W({props:{name:"class transformers.WarmUp",anchor:"transformers.WarmUp",parameters:[{name:"initial_learning_rate",val:": float"},{name:"decay_schedule_fn",val:": typing.Callable"},{name:"warmup_steps",val:": int"},{name:"power",val:": float = 1.0"},{name:"name",val:": str = None"}],parametersDescription:[{anchor:"transformers.WarmUp.initial_learning_rate",description:`<strong>initial_learning_rate</strong> (<code>float</code>) &#x2014;
The initial learning rate for the schedule after the warmup (so this will be the learning rate at the end
of the warmup).`,name:"initial_learning_rate"},{anchor:"transformers.WarmUp.decay_schedule_fn",description:`<strong>decay_schedule_fn</strong> (<code>Callable</code>) &#x2014;
The schedule function to apply after the warmup for the rest of training.`,name:"decay_schedule_fn"},{anchor:"transformers.WarmUp.warmup_steps",description:`<strong>warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup part of training.`,name:"warmup_steps"},{anchor:"transformers.WarmUp.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1) &#x2014;
The power to use for the polynomial warmup (defaults is a linear warmup).`,name:"power"},{anchor:"transformers.WarmUp.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Optional name prefix for the returned tensors during the schedule.`,name:"name"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization_tf.py#L30"}}),ht=new oe({}),ut=new oe({}),ft=new W({props:{name:"class transformers.GradientAccumulator",anchor:"transformers.GradientAccumulator",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization_tf.py#L302"}}),gt=new W({props:{name:"reset",anchor:"transformers.GradientAccumulator.reset",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/optimization_tf.py#L364"}}),{c(){f=a("meta"),T=l(),x=a("h1"),u=a("a"),z=a("span"),g(d.$$.fragment),$=l(),Tt=a("span"),Za=s("Optimization"),Pr=l(),se=a("p"),Oa=s("The "),Mt=a("code"),qa=s(".optimization"),Ba=s(" module provides:"),Rr=l(),S=a("ul"),Wt=a("li"),Xa=s("an optimizer with weight decay fixed that can be used to fine-tuned models, and"),Ya=l(),Ee=a("li"),Ha=s("several schedules in the form of schedule objects that inherit from "),Dt=a("code"),Qa=s("_LRSchedule"),Ka=s(":"),en=l(),Lt=a("li"),tn=s("a gradient accumulation class to accumulate the gradients of multiple batches"),Sr=l(),N=a("h2"),ie=a("a"),Ut=a("span"),g(Te.$$.fragment),rn=l(),kt=a("span"),an=s("AdamW (PyTorch)"),Ir=l(),U=a("div"),g(Me.$$.fragment),nn=l(),We=a("p"),on=s("Implements Adam algorithm with weight decay fix as introduced in "),De=a("a"),sn=s(`Decoupled Weight Decay
Regularization`),ln=s("."),mn=l(),le=a("div"),g(Le.$$.fragment),cn=l(),jt=a("p"),pn=s("Performs a single optimization step."),Nr=l(),C=a("h2"),me=a("a"),Pt=a("span"),g(Ue.$$.fragment),dn=l(),Rt=a("span"),hn=s("AdaFactor (PyTorch)"),Cr=l(),A=a("div"),g(ke.$$.fragment),un=l(),vt=a("p"),fn=s(`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
`),je=a("a"),gn=s("https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),_n=l(),M=a("p"),wn=s("Paper: "),St=a("em"),vn=s("Adafactor: Adaptive Learning Rates with Sublinear Memory Cost"),bn=l(),Pe=a("a"),yn=s("https://arxiv.org/abs/1804.04235"),$n=s(` Note that
this optimizer internally adjusts the learning rate depending on the `),It=a("code"),xn=s("scale_parameter"),An=s(", "),Nt=a("code"),zn=s("relative_step"),En=s(` and
`),Ct=a("code"),Tn=s("warmup_init"),Mn=s(" options. To use a manual (external) learning rate schedule you should set "),Ft=a("code"),Wn=s("scale_parameter=False"),Dn=s(` and
`),Gt=a("code"),Ln=s("relative_step=False"),Un=s("."),kn=l(),Jt=a("p"),jn=s("This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested."),Pn=l(),Re=a("p"),Rn=s("Recommended T5 finetuning settings ("),Se=a("a"),Sn=s("https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),In=s("):"),Nn=l(),k=a("ul"),Ie=a("li"),Vt=a("p"),Cn=s("Training without LR warmup or clip_threshold is not recommended."),Fn=l(),Ne=a("ul"),Zt=a("li"),Gn=s("use scheduled LR warm-up to fixed LR"),Jn=l(),Ce=a("li"),Vn=s("use clip_threshold=1.0 ("),Fe=a("a"),Zn=s("https://arxiv.org/abs/1804.04235"),On=s(")"),qn=l(),Ot=a("li"),qt=a("p"),Bn=s("Disable relative updates"),Xn=l(),Bt=a("li"),Xt=a("p"),Yn=s("Use scale_parameter=False"),Hn=l(),Yt=a("li"),Ht=a("p"),Qn=s("Additional optimizer operations like gradient clipping should not be used alongside Adafactor"),Kn=l(),g(ce.$$.fragment),eo=l(),g(pe.$$.fragment),to=l(),I=a("p"),ro=s("When using "),Qt=a("code"),ao=s("lr=None"),no=s(" with "),bt=a("a"),oo=s("Trainer"),so=s(" you will most likely need to use "),Kt=a("code"),io=s("AdafactorSchedule"),lo=l(),g(de.$$.fragment),mo=l(),g(he.$$.fragment),co=l(),ue=a("div"),g(Ge.$$.fragment),po=l(),er=a("p"),ho=s("Performs a single optimization step"),Fr=l(),F=a("h2"),fe=a("a"),tr=a("span"),g(Je.$$.fragment),uo=l(),rr=a("span"),fo=s("AdamWeightDecay (TensorFlow)"),Gr=l(),L=a("div"),g(Ve.$$.fragment),go=l(),G=a("p"),_o=s(`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is `),ar=a("em"),wo=s("not"),vo=s(` the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in `),Ze=a("a"),bo=s(`Decoupled Weight Decay
Regularization`),yo=s("."),$o=l(),nr=a("p"),xo=s(`Instead we want to decay the weights in a manner that doesn\u2019t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`),Ao=l(),ge=a("div"),g(Oe.$$.fragment),zo=l(),or=a("p"),Eo=s("Creates an optimizer from its config with WarmUp custom object."),Jr=l(),J=a("div"),g(qe.$$.fragment),To=l(),sr=a("p"),Mo=s("Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay."),Vr=l(),V=a("h2"),_e=a("a"),ir=a("span"),g(Be.$$.fragment),Wo=l(),lr=a("span"),Do=s("Schedules"),Zr=l(),Z=a("h3"),we=a("a"),mr=a("span"),g(Xe.$$.fragment),Lo=l(),cr=a("span"),Uo=s("Learning Rate Schedules (Pytorch)"),Or=l(),O=a("div"),g(Ye.$$.fragment),ko=l(),pr=a("p"),jo=s("An enumeration."),qr=l(),q=a("div"),g(He.$$.fragment),Po=l(),dr=a("p"),Ro=s("Unified API to get any scheduler from its name."),Br=l(),B=a("div"),g(Qe.$$.fragment),So=l(),hr=a("p"),Io=s("Create a schedule with a constant learning rate, using the learning rate set in optimizer."),Xr=l(),X=a("div"),g(Ke.$$.fragment),No=l(),ur=a("p"),Co=s(`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`),Yr=l(),et=a("img"),Hr=l(),Y=a("div"),g(tt.$$.fragment),Fo=l(),fr=a("p"),Go=s(`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`),Qr=l(),rt=a("img"),Kr=l(),H=a("div"),g(at.$$.fragment),Jo=l(),gr=a("p"),Vo=s(`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`),ea=l(),nt=a("img"),ta=l(),Q=a("div"),g(ot.$$.fragment),Zo=l(),_r=a("p"),Oo=s(`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`),ra=l(),st=a("img"),aa=l(),j=a("div"),g(it.$$.fragment),qo=l(),lt=a("p"),Bo=s(`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by `),wr=a("em"),Xo=s("lr_end"),Yo=s(`, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`),Ho=l(),ve=a("p"),Qo=s("Note: "),vr=a("em"),Ko=s("power"),es=s(` defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
`),mt=a("a"),ts=s("https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),na=l(),K=a("div"),g(ct.$$.fragment),rs=l(),br=a("p"),as=s(`Create a schedule with an inverse square-root learning rate, from the initial lr set in the optimizer, after a
warmup period which increases lr linearly from 0 to the initial lr set in the optimizer.`),oa=l(),ee=a("h3"),be=a("a"),yr=a("span"),g(pt.$$.fragment),ns=l(),$r=a("span"),os=s("Warmup (TensorFlow)"),sa=l(),te=a("div"),g(dt.$$.fragment),ss=l(),xr=a("p"),is=s("Applies a warmup schedule on a given learning rate decay schedule."),ia=l(),re=a("h2"),ye=a("a"),Ar=a("span"),g(ht.$$.fragment),ls=l(),zr=a("span"),ms=s("Gradient Strategies"),la=l(),ae=a("h3"),$e=a("a"),Er=a("span"),g(ut.$$.fragment),cs=l(),Tr=a("span"),ps=s("GradientAccumulator (TensorFlow)"),ma=l(),P=a("div"),g(ft.$$.fragment),ds=l(),ne=a("p"),hs=s(`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call `),Mr=a("code"),us=s(".gradients"),fs=s(", scale the gradients if required, and pass the result to "),Wr=a("code"),gs=s("apply_gradients"),_s=s("."),ws=l(),xe=a("div"),g(gt.$$.fragment),vs=l(),Dr=a("p"),bs=s("Resets the accumulated gradients on the current replica."),this.h()},l(e){const p=Bi('[data-svelte="svelte-1phssyn"]',document.head);f=n(p,"META",{name:!0,content:!0}),p.forEach(r),T=m(e),x=n(e,"H1",{class:!0});var _t=o(x);u=n(_t,"A",{id:!0,class:!0,href:!0});var Lr=o(u);z=n(Lr,"SPAN",{});var Ur=o(z);_(d.$$.fragment,Ur),Ur.forEach(r),Lr.forEach(r),$=m(_t),Tt=n(_t,"SPAN",{});var kr=o(Tt);Za=i(kr,"Optimization"),kr.forEach(r),_t.forEach(r),Pr=m(e),se=n(e,"P",{});var pa=o(se);Oa=i(pa,"The "),Mt=n(pa,"CODE",{});var Es=o(Mt);qa=i(Es,".optimization"),Es.forEach(r),Ba=i(pa," module provides:"),pa.forEach(r),Rr=m(e),S=n(e,"UL",{});var yt=o(S);Wt=n(yt,"LI",{});var Ts=o(Wt);Xa=i(Ts,"an optimizer with weight decay fixed that can be used to fine-tuned models, and"),Ts.forEach(r),Ya=m(yt),Ee=n(yt,"LI",{});var da=o(Ee);Ha=i(da,"several schedules in the form of schedule objects that inherit from "),Dt=n(da,"CODE",{});var Ms=o(Dt);Qa=i(Ms,"_LRSchedule"),Ms.forEach(r),Ka=i(da,":"),da.forEach(r),en=m(yt),Lt=n(yt,"LI",{});var Ws=o(Lt);tn=i(Ws,"a gradient accumulation class to accumulate the gradients of multiple batches"),Ws.forEach(r),yt.forEach(r),Sr=m(e),N=n(e,"H2",{class:!0});var ha=o(N);ie=n(ha,"A",{id:!0,class:!0,href:!0});var Ds=o(ie);Ut=n(Ds,"SPAN",{});var Ls=o(Ut);_(Te.$$.fragment,Ls),Ls.forEach(r),Ds.forEach(r),rn=m(ha),kt=n(ha,"SPAN",{});var Us=o(kt);an=i(Us,"AdamW (PyTorch)"),Us.forEach(r),ha.forEach(r),Ir=m(e),U=n(e,"DIV",{class:!0});var $t=o(U);_(Me.$$.fragment,$t),nn=m($t),We=n($t,"P",{});var ua=o(We);on=i(ua,"Implements Adam algorithm with weight decay fix as introduced in "),De=n(ua,"A",{href:!0,rel:!0});var ks=o(De);sn=i(ks,`Decoupled Weight Decay
Regularization`),ks.forEach(r),ln=i(ua,"."),ua.forEach(r),mn=m($t),le=n($t,"DIV",{class:!0});var fa=o(le);_(Le.$$.fragment,fa),cn=m(fa),jt=n(fa,"P",{});var js=o(jt);pn=i(js,"Performs a single optimization step."),js.forEach(r),fa.forEach(r),$t.forEach(r),Nr=m(e),C=n(e,"H2",{class:!0});var ga=o(C);me=n(ga,"A",{id:!0,class:!0,href:!0});var Ps=o(me);Pt=n(Ps,"SPAN",{});var Rs=o(Pt);_(Ue.$$.fragment,Rs),Rs.forEach(r),Ps.forEach(r),dn=m(ga),Rt=n(ga,"SPAN",{});var Ss=o(Rt);hn=i(Ss,"AdaFactor (PyTorch)"),Ss.forEach(r),ga.forEach(r),Cr=m(e),A=n(e,"DIV",{class:!0});var E=o(A);_(ke.$$.fragment,E),un=m(E),vt=n(E,"P",{});var ys=o(vt);fn=i(ys,`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
`),je=n(ys,"A",{href:!0,rel:!0});var Is=o(je);gn=i(Is,"https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),Is.forEach(r),ys.forEach(r),_n=m(E),M=n(E,"P",{});var D=o(M);wn=i(D,"Paper: "),St=n(D,"EM",{});var Ns=o(St);vn=i(Ns,"Adafactor: Adaptive Learning Rates with Sublinear Memory Cost"),Ns.forEach(r),bn=m(D),Pe=n(D,"A",{href:!0,rel:!0});var Cs=o(Pe);yn=i(Cs,"https://arxiv.org/abs/1804.04235"),Cs.forEach(r),$n=i(D,` Note that
this optimizer internally adjusts the learning rate depending on the `),It=n(D,"CODE",{});var Fs=o(It);xn=i(Fs,"scale_parameter"),Fs.forEach(r),An=i(D,", "),Nt=n(D,"CODE",{});var Gs=o(Nt);zn=i(Gs,"relative_step"),Gs.forEach(r),En=i(D,` and
`),Ct=n(D,"CODE",{});var Js=o(Ct);Tn=i(Js,"warmup_init"),Js.forEach(r),Mn=i(D," options. To use a manual (external) learning rate schedule you should set "),Ft=n(D,"CODE",{});var Vs=o(Ft);Wn=i(Vs,"scale_parameter=False"),Vs.forEach(r),Dn=i(D,` and
`),Gt=n(D,"CODE",{});var Zs=o(Gt);Ln=i(Zs,"relative_step=False"),Zs.forEach(r),Un=i(D,"."),D.forEach(r),kn=m(E),Jt=n(E,"P",{});var Os=o(Jt);jn=i(Os,"This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested."),Os.forEach(r),Pn=m(E),Re=n(E,"P",{});var _a=o(Re);Rn=i(_a,"Recommended T5 finetuning settings ("),Se=n(_a,"A",{href:!0,rel:!0});var qs=o(Se);Sn=i(qs,"https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),qs.forEach(r),In=i(_a,"):"),_a.forEach(r),Nn=m(E),k=n(E,"UL",{});var Ae=o(k);Ie=n(Ae,"LI",{});var wa=o(Ie);Vt=n(wa,"P",{});var Bs=o(Vt);Cn=i(Bs,"Training without LR warmup or clip_threshold is not recommended."),Bs.forEach(r),Fn=m(wa),Ne=n(wa,"UL",{});var va=o(Ne);Zt=n(va,"LI",{});var Xs=o(Zt);Gn=i(Xs,"use scheduled LR warm-up to fixed LR"),Xs.forEach(r),Jn=m(va),Ce=n(va,"LI",{});var ba=o(Ce);Vn=i(ba,"use clip_threshold=1.0 ("),Fe=n(ba,"A",{href:!0,rel:!0});var Ys=o(Fe);Zn=i(Ys,"https://arxiv.org/abs/1804.04235"),Ys.forEach(r),On=i(ba,")"),ba.forEach(r),va.forEach(r),wa.forEach(r),qn=m(Ae),Ot=n(Ae,"LI",{});var Hs=o(Ot);qt=n(Hs,"P",{});var Qs=o(qt);Bn=i(Qs,"Disable relative updates"),Qs.forEach(r),Hs.forEach(r),Xn=m(Ae),Bt=n(Ae,"LI",{});var Ks=o(Bt);Xt=n(Ks,"P",{});var ei=o(Xt);Yn=i(ei,"Use scale_parameter=False"),ei.forEach(r),Ks.forEach(r),Hn=m(Ae),Yt=n(Ae,"LI",{});var ti=o(Yt);Ht=n(ti,"P",{});var ri=o(Ht);Qn=i(ri,"Additional optimizer operations like gradient clipping should not be used alongside Adafactor"),ri.forEach(r),ti.forEach(r),Ae.forEach(r),Kn=m(E),_(ce.$$.fragment,E),eo=m(E),_(pe.$$.fragment,E),to=m(E),I=n(E,"P",{});var wt=o(I);ro=i(wt,"When using "),Qt=n(wt,"CODE",{});var ai=o(Qt);ao=i(ai,"lr=None"),ai.forEach(r),no=i(wt," with "),bt=n(wt,"A",{href:!0});var ni=o(bt);oo=i(ni,"Trainer"),ni.forEach(r),so=i(wt," you will most likely need to use "),Kt=n(wt,"CODE",{});var oi=o(Kt);io=i(oi,"AdafactorSchedule"),oi.forEach(r),wt.forEach(r),lo=m(E),_(de.$$.fragment,E),mo=m(E),_(he.$$.fragment,E),co=m(E),ue=n(E,"DIV",{class:!0});var ya=o(ue);_(Ge.$$.fragment,ya),po=m(ya),er=n(ya,"P",{});var si=o(er);ho=i(si,"Performs a single optimization step"),si.forEach(r),ya.forEach(r),E.forEach(r),Fr=m(e),F=n(e,"H2",{class:!0});var $a=o(F);fe=n($a,"A",{id:!0,class:!0,href:!0});var ii=o(fe);tr=n(ii,"SPAN",{});var li=o(tr);_(Je.$$.fragment,li),li.forEach(r),ii.forEach(r),uo=m($a),rr=n($a,"SPAN",{});var mi=o(rr);fo=i(mi,"AdamWeightDecay (TensorFlow)"),mi.forEach(r),$a.forEach(r),Gr=m(e),L=n(e,"DIV",{class:!0});var ze=o(L);_(Ve.$$.fragment,ze),go=m(ze),G=n(ze,"P",{});var xt=o(G);_o=i(xt,`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is `),ar=n(xt,"EM",{});var ci=o(ar);wo=i(ci,"not"),ci.forEach(r),vo=i(xt,` the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in `),Ze=n(xt,"A",{href:!0,rel:!0});var pi=o(Ze);bo=i(pi,`Decoupled Weight Decay
Regularization`),pi.forEach(r),yo=i(xt,"."),xt.forEach(r),$o=m(ze),nr=n(ze,"P",{});var di=o(nr);xo=i(di,`Instead we want to decay the weights in a manner that doesn\u2019t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`),di.forEach(r),Ao=m(ze),ge=n(ze,"DIV",{class:!0});var xa=o(ge);_(Oe.$$.fragment,xa),zo=m(xa),or=n(xa,"P",{});var hi=o(or);Eo=i(hi,"Creates an optimizer from its config with WarmUp custom object."),hi.forEach(r),xa.forEach(r),ze.forEach(r),Jr=m(e),J=n(e,"DIV",{class:!0});var Aa=o(J);_(qe.$$.fragment,Aa),To=m(Aa),sr=n(Aa,"P",{});var ui=o(sr);Mo=i(ui,"Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay."),ui.forEach(r),Aa.forEach(r),Vr=m(e),V=n(e,"H2",{class:!0});var za=o(V);_e=n(za,"A",{id:!0,class:!0,href:!0});var fi=o(_e);ir=n(fi,"SPAN",{});var gi=o(ir);_(Be.$$.fragment,gi),gi.forEach(r),fi.forEach(r),Wo=m(za),lr=n(za,"SPAN",{});var _i=o(lr);Do=i(_i,"Schedules"),_i.forEach(r),za.forEach(r),Zr=m(e),Z=n(e,"H3",{class:!0});var Ea=o(Z);we=n(Ea,"A",{id:!0,class:!0,href:!0});var wi=o(we);mr=n(wi,"SPAN",{});var vi=o(mr);_(Xe.$$.fragment,vi),vi.forEach(r),wi.forEach(r),Lo=m(Ea),cr=n(Ea,"SPAN",{});var bi=o(cr);Uo=i(bi,"Learning Rate Schedules (Pytorch)"),bi.forEach(r),Ea.forEach(r),Or=m(e),O=n(e,"DIV",{class:!0});var Ta=o(O);_(Ye.$$.fragment,Ta),ko=m(Ta),pr=n(Ta,"P",{});var yi=o(pr);jo=i(yi,"An enumeration."),yi.forEach(r),Ta.forEach(r),qr=m(e),q=n(e,"DIV",{class:!0});var Ma=o(q);_(He.$$.fragment,Ma),Po=m(Ma),dr=n(Ma,"P",{});var $i=o(dr);Ro=i($i,"Unified API to get any scheduler from its name."),$i.forEach(r),Ma.forEach(r),Br=m(e),B=n(e,"DIV",{class:!0});var Wa=o(B);_(Qe.$$.fragment,Wa),So=m(Wa),hr=n(Wa,"P",{});var xi=o(hr);Io=i(xi,"Create a schedule with a constant learning rate, using the learning rate set in optimizer."),xi.forEach(r),Wa.forEach(r),Xr=m(e),X=n(e,"DIV",{class:!0});var Da=o(X);_(Ke.$$.fragment,Da),No=m(Da),ur=n(Da,"P",{});var Ai=o(ur);Co=i(Ai,`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`),Ai.forEach(r),Da.forEach(r),Yr=m(e),et=n(e,"IMG",{alt:!0,src:!0}),Hr=m(e),Y=n(e,"DIV",{class:!0});var La=o(Y);_(tt.$$.fragment,La),Fo=m(La),fr=n(La,"P",{});var zi=o(fr);Go=i(zi,`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`),zi.forEach(r),La.forEach(r),Qr=m(e),rt=n(e,"IMG",{alt:!0,src:!0}),Kr=m(e),H=n(e,"DIV",{class:!0});var Ua=o(H);_(at.$$.fragment,Ua),Jo=m(Ua),gr=n(Ua,"P",{});var Ei=o(gr);Vo=i(Ei,`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`),Ei.forEach(r),Ua.forEach(r),ea=m(e),nt=n(e,"IMG",{alt:!0,src:!0}),ta=m(e),Q=n(e,"DIV",{class:!0});var ka=o(Q);_(ot.$$.fragment,ka),Zo=m(ka),_r=n(ka,"P",{});var Ti=o(_r);Oo=i(Ti,`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`),Ti.forEach(r),ka.forEach(r),ra=m(e),st=n(e,"IMG",{alt:!0,src:!0}),aa=m(e),j=n(e,"DIV",{class:!0});var At=o(j);_(it.$$.fragment,At),qo=m(At),lt=n(At,"P",{});var ja=o(lt);Bo=i(ja,`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by `),wr=n(ja,"EM",{});var Mi=o(wr);Xo=i(Mi,"lr_end"),Mi.forEach(r),Yo=i(ja,`, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`),ja.forEach(r),Ho=m(At),ve=n(At,"P",{});var jr=o(ve);Qo=i(jr,"Note: "),vr=n(jr,"EM",{});var Wi=o(vr);Ko=i(Wi,"power"),Wi.forEach(r),es=i(jr,` defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
`),mt=n(jr,"A",{href:!0,rel:!0});var Di=o(mt);ts=i(Di,"https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),Di.forEach(r),jr.forEach(r),At.forEach(r),na=m(e),K=n(e,"DIV",{class:!0});var Pa=o(K);_(ct.$$.fragment,Pa),rs=m(Pa),br=n(Pa,"P",{});var Li=o(br);as=i(Li,`Create a schedule with an inverse square-root learning rate, from the initial lr set in the optimizer, after a
warmup period which increases lr linearly from 0 to the initial lr set in the optimizer.`),Li.forEach(r),Pa.forEach(r),oa=m(e),ee=n(e,"H3",{class:!0});var Ra=o(ee);be=n(Ra,"A",{id:!0,class:!0,href:!0});var Ui=o(be);yr=n(Ui,"SPAN",{});var ki=o(yr);_(pt.$$.fragment,ki),ki.forEach(r),Ui.forEach(r),ns=m(Ra),$r=n(Ra,"SPAN",{});var ji=o($r);os=i(ji,"Warmup (TensorFlow)"),ji.forEach(r),Ra.forEach(r),sa=m(e),te=n(e,"DIV",{class:!0});var Sa=o(te);_(dt.$$.fragment,Sa),ss=m(Sa),xr=n(Sa,"P",{});var Pi=o(xr);is=i(Pi,"Applies a warmup schedule on a given learning rate decay schedule."),Pi.forEach(r),Sa.forEach(r),ia=m(e),re=n(e,"H2",{class:!0});var Ia=o(re);ye=n(Ia,"A",{id:!0,class:!0,href:!0});var Ri=o(ye);Ar=n(Ri,"SPAN",{});var Si=o(Ar);_(ht.$$.fragment,Si),Si.forEach(r),Ri.forEach(r),ls=m(Ia),zr=n(Ia,"SPAN",{});var Ii=o(zr);ms=i(Ii,"Gradient Strategies"),Ii.forEach(r),Ia.forEach(r),la=m(e),ae=n(e,"H3",{class:!0});var Na=o(ae);$e=n(Na,"A",{id:!0,class:!0,href:!0});var Ni=o($e);Er=n(Ni,"SPAN",{});var Ci=o(Er);_(ut.$$.fragment,Ci),Ci.forEach(r),Ni.forEach(r),cs=m(Na),Tr=n(Na,"SPAN",{});var Fi=o(Tr);ps=i(Fi,"GradientAccumulator (TensorFlow)"),Fi.forEach(r),Na.forEach(r),ma=m(e),P=n(e,"DIV",{class:!0});var zt=o(P);_(ft.$$.fragment,zt),ds=m(zt),ne=n(zt,"P",{});var Et=o(ne);hs=i(Et,`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call `),Mr=n(Et,"CODE",{});var Gi=o(Mr);us=i(Gi,".gradients"),Gi.forEach(r),fs=i(Et,", scale the gradients if required, and pass the result to "),Wr=n(Et,"CODE",{});var Ji=o(Wr);gs=i(Ji,"apply_gradients"),Ji.forEach(r),_s=i(Et,"."),Et.forEach(r),ws=m(zt),xe=n(zt,"DIV",{class:!0});var Ca=o(xe);_(gt.$$.fragment,Ca),vs=m(Ca),Dr=n(Ca,"P",{});var Vi=o(Dr);bs=i(Vi,"Resets the accumulated gradients on the current replica."),Vi.forEach(r),Ca.forEach(r),zt.forEach(r),this.h()},h(){c(f,"name","hf:doc:metadata"),c(f,"content",JSON.stringify(tl)),c(u,"id","optimization"),c(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(u,"href","#optimization"),c(x,"class","relative group"),c(ie,"id","transformers.AdamW"),c(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ie,"href","#transformers.AdamW"),c(N,"class","relative group"),c(De,"href","https://arxiv.org/abs/1711.05101"),c(De,"rel","nofollow"),c(le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(me,"id","transformers.Adafactor"),c(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(me,"href","#transformers.Adafactor"),c(C,"class","relative group"),c(je,"href","https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),c(je,"rel","nofollow"),c(Pe,"href","https://arxiv.org/abs/1804.04235"),c(Pe,"rel","nofollow"),c(Se,"href","https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),c(Se,"rel","nofollow"),c(Fe,"href","https://arxiv.org/abs/1804.04235"),c(Fe,"rel","nofollow"),c(bt,"href","/docs/transformers/v4.30.0/en/main_classes/trainer#transformers.Trainer"),c(ue,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(fe,"id","transformers.AdamWeightDecay"),c(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(fe,"href","#transformers.AdamWeightDecay"),c(F,"class","relative group"),c(Ze,"href","https://arxiv.org/abs/1711.05101"),c(Ze,"rel","nofollow"),c(ge,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(_e,"id","schedules"),c(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(_e,"href","#schedules"),c(V,"class","relative group"),c(we,"id","transformers.SchedulerType"),c(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(we,"href","#transformers.SchedulerType"),c(Z,"class","relative group"),c(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(et,"alt",""),Fa(et.src,$s="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_constant_schedule.png")||c(et,"src",$s),c(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(rt,"alt",""),Fa(rt.src,xs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_schedule.png")||c(rt,"src",xs),c(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(nt,"alt",""),Fa(nt.src,As="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_hard_restarts_schedule.png")||c(nt,"src",As),c(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(st,"alt",""),Fa(st.src,zs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_linear_schedule.png")||c(st,"src",zs),c(mt,"href","https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),c(mt,"rel","nofollow"),c(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(be,"id","transformers.WarmUp"),c(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(be,"href","#transformers.WarmUp"),c(ee,"class","relative group"),c(te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(ye,"id","gradient-strategies"),c(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ye,"href","#gradient-strategies"),c(re,"class","relative group"),c($e,"id","transformers.GradientAccumulator"),c($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c($e,"href","#transformers.GradientAccumulator"),c(ae,"class","relative group"),c(xe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,p){t(document.head,f),h(e,T,p),h(e,x,p),t(x,u),t(u,z),w(d,z,null),t(x,$),t(x,Tt),t(Tt,Za),h(e,Pr,p),h(e,se,p),t(se,Oa),t(se,Mt),t(Mt,qa),t(se,Ba),h(e,Rr,p),h(e,S,p),t(S,Wt),t(Wt,Xa),t(S,Ya),t(S,Ee),t(Ee,Ha),t(Ee,Dt),t(Dt,Qa),t(Ee,Ka),t(S,en),t(S,Lt),t(Lt,tn),h(e,Sr,p),h(e,N,p),t(N,ie),t(ie,Ut),w(Te,Ut,null),t(N,rn),t(N,kt),t(kt,an),h(e,Ir,p),h(e,U,p),w(Me,U,null),t(U,nn),t(U,We),t(We,on),t(We,De),t(De,sn),t(We,ln),t(U,mn),t(U,le),w(Le,le,null),t(le,cn),t(le,jt),t(jt,pn),h(e,Nr,p),h(e,C,p),t(C,me),t(me,Pt),w(Ue,Pt,null),t(C,dn),t(C,Rt),t(Rt,hn),h(e,Cr,p),h(e,A,p),w(ke,A,null),t(A,un),t(A,vt),t(vt,fn),t(vt,je),t(je,gn),t(A,_n),t(A,M),t(M,wn),t(M,St),t(St,vn),t(M,bn),t(M,Pe),t(Pe,yn),t(M,$n),t(M,It),t(It,xn),t(M,An),t(M,Nt),t(Nt,zn),t(M,En),t(M,Ct),t(Ct,Tn),t(M,Mn),t(M,Ft),t(Ft,Wn),t(M,Dn),t(M,Gt),t(Gt,Ln),t(M,Un),t(A,kn),t(A,Jt),t(Jt,jn),t(A,Pn),t(A,Re),t(Re,Rn),t(Re,Se),t(Se,Sn),t(Re,In),t(A,Nn),t(A,k),t(k,Ie),t(Ie,Vt),t(Vt,Cn),t(Ie,Fn),t(Ie,Ne),t(Ne,Zt),t(Zt,Gn),t(Ne,Jn),t(Ne,Ce),t(Ce,Vn),t(Ce,Fe),t(Fe,Zn),t(Ce,On),t(k,qn),t(k,Ot),t(Ot,qt),t(qt,Bn),t(k,Xn),t(k,Bt),t(Bt,Xt),t(Xt,Yn),t(k,Hn),t(k,Yt),t(Yt,Ht),t(Ht,Qn),t(A,Kn),w(ce,A,null),t(A,eo),w(pe,A,null),t(A,to),t(A,I),t(I,ro),t(I,Qt),t(Qt,ao),t(I,no),t(I,bt),t(bt,oo),t(I,so),t(I,Kt),t(Kt,io),t(A,lo),w(de,A,null),t(A,mo),w(he,A,null),t(A,co),t(A,ue),w(Ge,ue,null),t(ue,po),t(ue,er),t(er,ho),h(e,Fr,p),h(e,F,p),t(F,fe),t(fe,tr),w(Je,tr,null),t(F,uo),t(F,rr),t(rr,fo),h(e,Gr,p),h(e,L,p),w(Ve,L,null),t(L,go),t(L,G),t(G,_o),t(G,ar),t(ar,wo),t(G,vo),t(G,Ze),t(Ze,bo),t(G,yo),t(L,$o),t(L,nr),t(nr,xo),t(L,Ao),t(L,ge),w(Oe,ge,null),t(ge,zo),t(ge,or),t(or,Eo),h(e,Jr,p),h(e,J,p),w(qe,J,null),t(J,To),t(J,sr),t(sr,Mo),h(e,Vr,p),h(e,V,p),t(V,_e),t(_e,ir),w(Be,ir,null),t(V,Wo),t(V,lr),t(lr,Do),h(e,Zr,p),h(e,Z,p),t(Z,we),t(we,mr),w(Xe,mr,null),t(Z,Lo),t(Z,cr),t(cr,Uo),h(e,Or,p),h(e,O,p),w(Ye,O,null),t(O,ko),t(O,pr),t(pr,jo),h(e,qr,p),h(e,q,p),w(He,q,null),t(q,Po),t(q,dr),t(dr,Ro),h(e,Br,p),h(e,B,p),w(Qe,B,null),t(B,So),t(B,hr),t(hr,Io),h(e,Xr,p),h(e,X,p),w(Ke,X,null),t(X,No),t(X,ur),t(ur,Co),h(e,Yr,p),h(e,et,p),h(e,Hr,p),h(e,Y,p),w(tt,Y,null),t(Y,Fo),t(Y,fr),t(fr,Go),h(e,Qr,p),h(e,rt,p),h(e,Kr,p),h(e,H,p),w(at,H,null),t(H,Jo),t(H,gr),t(gr,Vo),h(e,ea,p),h(e,nt,p),h(e,ta,p),h(e,Q,p),w(ot,Q,null),t(Q,Zo),t(Q,_r),t(_r,Oo),h(e,ra,p),h(e,st,p),h(e,aa,p),h(e,j,p),w(it,j,null),t(j,qo),t(j,lt),t(lt,Bo),t(lt,wr),t(wr,Xo),t(lt,Yo),t(j,Ho),t(j,ve),t(ve,Qo),t(ve,vr),t(vr,Ko),t(ve,es),t(ve,mt),t(mt,ts),h(e,na,p),h(e,K,p),w(ct,K,null),t(K,rs),t(K,br),t(br,as),h(e,oa,p),h(e,ee,p),t(ee,be),t(be,yr),w(pt,yr,null),t(ee,ns),t(ee,$r),t($r,os),h(e,sa,p),h(e,te,p),w(dt,te,null),t(te,ss),t(te,xr),t(xr,is),h(e,ia,p),h(e,re,p),t(re,ye),t(ye,Ar),w(ht,Ar,null),t(re,ls),t(re,zr),t(zr,ms),h(e,la,p),h(e,ae,p),t(ae,$e),t($e,Er),w(ut,Er,null),t(ae,cs),t(ae,Tr),t(Tr,ps),h(e,ma,p),h(e,P,p),w(ft,P,null),t(P,ds),t(P,ne),t(ne,hs),t(ne,Mr),t(Mr,us),t(ne,fs),t(ne,Wr),t(Wr,gs),t(ne,_s),t(P,ws),t(P,xe),w(gt,xe,null),t(xe,vs),t(xe,Dr),t(Dr,bs),ca=!0},p(e,[p]){const _t={};p&2&&(_t.$$scope={dirty:p,ctx:e}),ce.$set(_t);const Lr={};p&2&&(Lr.$$scope={dirty:p,ctx:e}),pe.$set(Lr);const Ur={};p&2&&(Ur.$$scope={dirty:p,ctx:e}),de.$set(Ur);const kr={};p&2&&(kr.$$scope={dirty:p,ctx:e}),he.$set(kr)},i(e){ca||(v(d.$$.fragment,e),v(Te.$$.fragment,e),v(Me.$$.fragment,e),v(Le.$$.fragment,e),v(Ue.$$.fragment,e),v(ke.$$.fragment,e),v(ce.$$.fragment,e),v(pe.$$.fragment,e),v(de.$$.fragment,e),v(he.$$.fragment,e),v(Ge.$$.fragment,e),v(Je.$$.fragment,e),v(Ve.$$.fragment,e),v(Oe.$$.fragment,e),v(qe.$$.fragment,e),v(Be.$$.fragment,e),v(Xe.$$.fragment,e),v(Ye.$$.fragment,e),v(He.$$.fragment,e),v(Qe.$$.fragment,e),v(Ke.$$.fragment,e),v(tt.$$.fragment,e),v(at.$$.fragment,e),v(ot.$$.fragment,e),v(it.$$.fragment,e),v(ct.$$.fragment,e),v(pt.$$.fragment,e),v(dt.$$.fragment,e),v(ht.$$.fragment,e),v(ut.$$.fragment,e),v(ft.$$.fragment,e),v(gt.$$.fragment,e),ca=!0)},o(e){b(d.$$.fragment,e),b(Te.$$.fragment,e),b(Me.$$.fragment,e),b(Le.$$.fragment,e),b(Ue.$$.fragment,e),b(ke.$$.fragment,e),b(ce.$$.fragment,e),b(pe.$$.fragment,e),b(de.$$.fragment,e),b(he.$$.fragment,e),b(Ge.$$.fragment,e),b(Je.$$.fragment,e),b(Ve.$$.fragment,e),b(Oe.$$.fragment,e),b(qe.$$.fragment,e),b(Be.$$.fragment,e),b(Xe.$$.fragment,e),b(Ye.$$.fragment,e),b(He.$$.fragment,e),b(Qe.$$.fragment,e),b(Ke.$$.fragment,e),b(tt.$$.fragment,e),b(at.$$.fragment,e),b(ot.$$.fragment,e),b(it.$$.fragment,e),b(ct.$$.fragment,e),b(pt.$$.fragment,e),b(dt.$$.fragment,e),b(ht.$$.fragment,e),b(ut.$$.fragment,e),b(ft.$$.fragment,e),b(gt.$$.fragment,e),ca=!1},d(e){r(f),e&&r(T),e&&r(x),y(d),e&&r(Pr),e&&r(se),e&&r(Rr),e&&r(S),e&&r(Sr),e&&r(N),y(Te),e&&r(Ir),e&&r(U),y(Me),y(Le),e&&r(Nr),e&&r(C),y(Ue),e&&r(Cr),e&&r(A),y(ke),y(ce),y(pe),y(de),y(he),y(Ge),e&&r(Fr),e&&r(F),y(Je),e&&r(Gr),e&&r(L),y(Ve),y(Oe),e&&r(Jr),e&&r(J),y(qe),e&&r(Vr),e&&r(V),y(Be),e&&r(Zr),e&&r(Z),y(Xe),e&&r(Or),e&&r(O),y(Ye),e&&r(qr),e&&r(q),y(He),e&&r(Br),e&&r(B),y(Qe),e&&r(Xr),e&&r(X),y(Ke),e&&r(Yr),e&&r(et),e&&r(Hr),e&&r(Y),y(tt),e&&r(Qr),e&&r(rt),e&&r(Kr),e&&r(H),y(at),e&&r(ea),e&&r(nt),e&&r(ta),e&&r(Q),y(ot),e&&r(ra),e&&r(st),e&&r(aa),e&&r(j),y(it),e&&r(na),e&&r(K),y(ct),e&&r(oa),e&&r(ee),y(pt),e&&r(sa),e&&r(te),y(dt),e&&r(ia),e&&r(re),y(ht),e&&r(la),e&&r(ae),y(ut),e&&r(ma),e&&r(P),y(ft),y(gt)}}}const tl={local:"optimization",sections:[{local:"transformers.AdamW",title:"AdamW (PyTorch)"},{local:"transformers.Adafactor",title:"AdaFactor (PyTorch)"},{local:"transformers.AdamWeightDecay",title:"AdamWeightDecay (TensorFlow)"},{local:"schedules",sections:[{local:"transformers.SchedulerType",title:"Learning Rate Schedules (Pytorch)"},{local:"transformers.WarmUp",title:"Warmup (TensorFlow)"}],title:"Schedules"},{local:"gradient-strategies",sections:[{local:"transformers.GradientAccumulator",title:"GradientAccumulator (TensorFlow)"}],title:"Gradient Strategies"}],title:"Optimization"};function rl(R){return Xi(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ll extends Zi{constructor(f){super();Oi(this,f,rl,el,qi,{})}}export{ll as default,tl as metadata};
