transformers/pr_37082/zh/_app/immutable/nodes/43.4e52aec3.js

import{s as ra,f as Nt,o as aa,n as Pt}from"../chunks/scheduler.25b97de1.js";import{S as na,i as sa,g as l,s as a,r as d,A as oa,h as m,f as t,c as n,j as z,u as h,x as w,k as y,y as o,a as i,v as u,d as g,t as _,w as f}from"../chunks/index.d9030fc9.js";import{D as M}from"../chunks/Docstring.ffac8efa.js";import{C as Ht}from"../chunks/CodeBlock.e6cd0d95.js";import{E as St}from"../chunks/ExampleCodeBlock.22dfe688.js";import{H as O,E as ia}from"../chunks/EditOnGithub.91d95064.js";function la(D){let c,A="Example:",x,p,v;return p=new Ht({props:{code:"QWRhZmFjdG9yKG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMHNjYWxlX3BhcmFtZXRlciUzREZhbHNlJTJDJTIwcmVsYXRpdmVfc3RlcCUzREZhbHNlJTJDJTIwd2FybXVwX2luaXQlM0RGYWxzZSUyQyUyMGxyJTNEMWUtMyk=",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, warmup_init=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">1e-3</span>)',wrap:!1}}),{c(){c=l("p"),c.textContent=A,x=a(),d(p.$$.fragment)},l(s){c=m(s,"P",{"data-svelte-h":!0}),w(c)!=="svelte-11lpom8"&&(c.textContent=A),x=n(s),h(p.$$.fragment,s)},m(s,T){i(s,c,T),i(s,x,T),u(p,s,T),v=!0},p:Pt,i(s){v||(g(p.$$.fragment,s),v=!0)},o(s){_(p.$$.fragment,s),v=!1},d(s){s&&(t(c),t(x)),f(p,s)}}}function ma(D){let c,A="Others reported the following combination to work well:",x,p,v;return p=new Ht({props:{code:"QWRhZmFjdG9yKG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMHNjYWxlX3BhcmFtZXRlciUzRFRydWUlMkMlMjByZWxhdGl2ZV9zdGVwJTNEVHJ1ZSUyQyUyMHdhcm11cF9pbml0JTNEVHJ1ZSUyQyUyMGxyJTNETm9uZSk=",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)',wrap:!1}}),{c(){c=l("p"),c.textContent=A,x=a(),d(p.$$.fragment)},l(s){c=m(s,"P",{"data-svelte-h":!0}),w(c)!=="svelte-mxeef1"&&(c.textContent=A),x=n(s),h(p.$$.fragment,s)},m(s,T){i(s,c,T),i(s,x,T),u(p,s,T),v=!0},p:Pt,i(s){v||(g(p.$$.fragment,s),v=!0)},o(s){_(p.$$.fragment,s),v=!1},d(s){s&&(t(c),t(x)),f(p,s)}}}function ca(D){let c,A="scheduler as following:",x,p,v;return p=new Ht({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5vcHRpbWl6YXRpb24lMjBpbXBvcnQlMjBBZGFmYWN0b3IlMkMlMjBBZGFmYWN0b3JTY2hlZHVsZSUwQSUwQW9wdGltaXplciUyMCUzRCUyMEFkYWZhY3Rvcihtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMjBzY2FsZV9wYXJhbWV0ZXIlM0RUcnVlJTJDJTIwcmVsYXRpdmVfc3RlcCUzRFRydWUlMkMlMjB3YXJtdXBfaW5pdCUzRFRydWUlMkMlMjBsciUzRE5vbmUpJTBBbHJfc2NoZWR1bGVyJTIwJTNEJTIwQWRhZmFjdG9yU2NoZWR1bGUob3B0aW1pemVyKSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKC4uLiUyQyUyMG9wdGltaXplcnMlM0Qob3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGVyKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, AdafactorSchedule

optimizer = Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)
lr_scheduler = AdafactorSchedule(optimizer)
trainer = Trainer(..., optimizers=(optimizer, lr_scheduler))`,wrap:!1}}),{c(){c=l("p"),c.textContent=A,x=a(),d(p.$$.fragment)},l(s){c=m(s,"P",{"data-svelte-h":!0}),w(c)!=="svelte-1eua222"&&(c.textContent=A),x=n(s),h(p.$$.fragment,s)},m(s,T){i(s,c,T),i(s,x,T),u(p,s,T),v=!0},p:Pt,i(s){v||(g(p.$$.fragment,s),v=!0)},o(s){_(p.$$.fragment,s),v=!1},d(s){s&&(t(c),t(x)),f(p,s)}}}function pa(D){let c,A="Usage:",x,p,v;return p=new Ht({props:{code:"JTIzJTIwcmVwbGFjZSUyMEFkYW1XJTIwd2l0aCUyMEFkYWZhY3RvciUwQW9wdGltaXplciUyMCUzRCUyMEFkYWZhY3RvciglMEElMjAlMjAlMjAlMjBtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMEElMjAlMjAlMjAlMjBsciUzRDFlLTMlMkMlMEElMjAlMjAlMjAlMjBlcHMlM0QoMWUtMzAlMkMlMjAxZS0zKSUyQyUwQSUyMCUyMCUyMCUyMGNsaXBfdGhyZXNob2xkJTNEMS4wJTJDJTBBJTIwJTIwJTIwJTIwZGVjYXlfcmF0ZSUzRC0wLjglMkMlMEElMjAlMjAlMjAlMjBiZXRhMSUzRE5vbmUlMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRfZGVjYXklM0QwLjAlMkMlMEElMjAlMjAlMjAlMjByZWxhdGl2ZV9zdGVwJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBzY2FsZV9wYXJhbWV0ZXIlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMHdhcm11cF9pbml0JTNERmFsc2UlMkMlMEEp",highlighted:`<span class="hljs-comment"># replace AdamW with Adafactor</span>
optimizer = Adafactor(
    model.parameters(),
    lr=<span class="hljs-number">1e-3</span>,
    eps=(<span class="hljs-number">1e-30</span>, <span class="hljs-number">1e-3</span>),
    clip_threshold=<span class="hljs-number">1.0</span>,
    decay_rate=-<span class="hljs-number">0.8</span>,
    beta1=<span class="hljs-literal">None</span>,
    weight_decay=<span class="hljs-number">0.0</span>,
    relative_step=<span class="hljs-literal">False</span>,
    scale_parameter=<span class="hljs-literal">False</span>,
    warmup_init=<span class="hljs-literal">False</span>,
)`,wrap:!1}}),{c(){c=l("p"),c.textContent=A,x=a(),d(p.$$.fragment)},l(s){c=m(s,"P",{"data-svelte-h":!0}),w(c)!=="svelte-5wyjqd"&&(c.textContent=A),x=n(s),h(p.$$.fragment,s)},m(s,T){i(s,c,T),i(s,x,T),u(p,s,T),v=!0},p:Pt,i(s){v||(g(p.$$.fragment,s),v=!0)},o(s){_(p.$$.fragment,s),v=!1},d(s){s&&(t(c),t(x)),f(p,s)}}}function da(D){let c,A,x,p,v,s,T,zr="<code>.optimization</code> 模块提供了：",nt,Y,Mr="<li>一个带有固定权重衰减的优化器，可用于微调模型</li> <li>继承自 <code>_LRSchedule</code> 多个调度器：</li> <li>一个梯度累积类，用于累积多个批次的梯度</li>",st,Q,ot,b,K,qt,Le,Ar=`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
<a href="https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py" rel="nofollow">https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py</a>`,Bt,je,Cr=`Paper: <em>Adafactor: Adaptive Learning Rates with Sublinear Memory Cost</em> <a href="https://arxiv.org/abs/1804.04235" rel="nofollow">https://arxiv.org/abs/1804.04235</a> Note that
this optimizer internally adjusts the learning rate depending on the <code>scale_parameter</code>, <code>relative_step</code> and
<code>warmup_init</code> options. To use a manual (external) learning rate schedule you should set <code>scale_parameter=False</code> and
<code>relative_step=False</code>.`,Xt,Ue,Lr="This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested.",Ot,We,jr='Recommended T5 finetuning settings (<a href="https://discuss.huggingface.co/t/t5-finetuning-tips/684/3" rel="nofollow">https://discuss.huggingface.co/t/t5-finetuning-tips/684/3</a>):',Yt,De,Ur='<li><p>Training without LR warmup or clip_threshold is not recommended.</p> <ul><li>use scheduled LR warm-up to fixed LR</li> <li>use clip_threshold=1.0 (<a href="https://arxiv.org/abs/1804.04235" rel="nofollow">https://arxiv.org/abs/1804.04235</a>)</li></ul></li> <li><p>Disable relative updates</p></li> <li><p>Use scale_parameter=False</p></li> <li><p>Additional optimizer operations like gradient clipping should not be used alongside Adafactor</p></li>',Qt,N,Kt,S,er,Re,Wr="When using <code>lr=None</code> with <code>Trainer</code> you will most likely need to use <code>AdafactorSchedule</code>",tr,P,rr,H,ar,q,ee,nr,Ee,Dr="Performs a single optimization step",it,te,lt,C,re,sr,ke,Rr=`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is <em>not</em> the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in <a href="https://arxiv.org/abs/1711.05101" rel="nofollow">Decoupled Weight Decay
Regularization</a>.`,or,Fe,Er=`Instead we want to decay the weights in a manner that doesn’t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`,ir,B,ae,lr,Ie,kr="Creates an optimizer from its config with WarmUp custom object.",mt,R,ne,mr,Ge,Fr="Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay.",ct,se,pt,oe,dt,L,ie,cr,Je,Ir=`Scheduler names for the parameter <code>lr_scheduler_type</code> in <code>TrainingArguments</code>.
By default, it uses “linear”. Internally, this retrieves <code>get_linear_schedule_with_warmup</code> scheduler from <code>Trainer</code>.
Scheduler types:`,pr,Ve,Gr="<li>“linear” = get_linear_schedule_with_warmup</li> <li>“cosine” = get_cosine_schedule_with_warmup</li> <li>“cosine_with_restarts” = get_cosine_with_hard_restarts_schedule_with_warmup</li> <li>“polynomial” = get_polynomial_decay_schedule_with_warmup</li> <li>“constant” =  get_constant_schedule</li> <li>“constant_with_warmup” = get_constant_schedule_with_warmup</li> <li>“inverse_sqrt” = get_inverse_sqrt_schedule</li> <li>“reduce_lr_on_plateau” = get_reduce_on_plateau_schedule</li> <li>“cosine_with_min_lr” = get_cosine_with_min_lr_schedule_with_warmup</li> <li>“warmup_stable_decay” = get_wsd_schedule</li>",ht,E,le,dr,Ze,Jr="Unified API to get any scheduler from its name.",ut,k,me,hr,Ne,Vr="Create a schedule with a constant learning rate, using the learning rate set in optimizer.",gt,F,ce,ur,Se,Zr=`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`,_t,pe,Nr,ft,I,de,gr,Pe,Sr=`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`,wt,he,Pr,bt,G,ue,_r,He,Hr=`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`,vt,ge,qr,$t,J,_e,fr,qe,Br=`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`,yt,fe,Xr,xt,j,we,wr,Be,Or=`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by <em>lr_end</em>, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`,br,Xe,Yr=`Note: <em>power</em> defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
<a href="https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37" rel="nofollow">https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37</a>`,Tt,V,be,vr,Oe,Qr=`Create a schedule with an inverse square-root learning rate, from the initial lr set in the optimizer, after a
warmup period which increases lr linearly from 0 to the initial lr set in the optimizer.`,zt,ve,Mt,Z,$e,$r,Ye,Kr="Applies a warmup schedule on a given learning rate decay schedule.",At,ye,Ct,xe,Lt,U,Te,yr,Qe,ea=`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call <code>.gradients</code>, scale the gradients if required, and pass the result to <code>apply_gradients</code>.`,xr,X,ze,Tr,Ke,ta="Resets the accumulated gradients on the current replica.",jt,Me,Ut,at,Wt;return v=new O({props:{title:"Optimization",local:"optimization",headingTag:"h1"}}),Q=new O({props:{title:"AdaFactor (PyTorch)",local:"transformers.Adafactor",headingTag:"h2"}}),K=new M({props:{name:"class transformers.Adafactor",anchor:"transformers.Adafactor",parameters:[{name:"params",val:""},{name:"lr",val:" = None"},{name:"eps",val:" = (1e-30, 0.001)"},{name:"clip_threshold",val:" = 1.0"},{name:"decay_rate",val:" = -0.8"},{name:"beta1",val:" = None"},{name:"weight_decay",val:" = 0.0"},{name:"scale_parameter",val:" = True"},{name:"relative_step",val:" = True"},{name:"warmup_init",val:" = False"}],parametersDescription:[{anchor:"transformers.Adafactor.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.Adafactor.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>) &#x2014;
The external learning rate.`,name:"lr"},{anchor:"transformers.Adafactor.eps",description:`<strong>eps</strong> (<code>Tuple[float, float]</code>, <em>optional</em>, defaults to <code>(1e-30, 0.001)</code>) &#x2014;
Regularization constants for square gradient and parameter scale respectively`,name:"eps"},{anchor:"transformers.Adafactor.clip_threshold",description:`<strong>clip_threshold</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
Threshold of root mean square of final gradient update`,name:"clip_threshold"},{anchor:"transformers.Adafactor.decay_rate",description:`<strong>decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to -0.8) &#x2014;
Coefficient used to compute running averages of square`,name:"decay_rate"},{anchor:"transformers.Adafactor.beta1",description:`<strong>beta1</strong> (<code>float</code>, <em>optional</em>) &#x2014;
Coefficient used for computing running averages of gradient`,name:"beta1"},{anchor:"transformers.Adafactor.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
Weight decay (L2 penalty)`,name:"weight_decay"},{anchor:"transformers.Adafactor.scale_parameter",description:`<strong>scale_parameter</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, learning rate is scaled by root mean square`,name:"scale_parameter"},{anchor:"transformers.Adafactor.relative_step",description:`<strong>relative_step</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, time-dependent learning rate is computed instead of external learning rate`,name:"relative_step"},{anchor:"transformers.Adafactor.warmup_init",description:`<strong>warmup_init</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Time-dependent learning rate computation depends on whether warm-up initialization is being used`,name:"warmup_init"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization.py#L606"}}),N=new St({props:{anchor:"transformers.Adafactor.example",$$slots:{default:[la]},$$scope:{ctx:D}}}),S=new St({props:{anchor:"transformers.Adafactor.example-2",$$slots:{default:[ma]},$$scope:{ctx:D}}}),P=new St({props:{anchor:"transformers.Adafactor.example-3",$$slots:{default:[ca]},$$scope:{ctx:D}}}),H=new St({props:{anchor:"transformers.Adafactor.example-4",$$slots:{default:[pa]},$$scope:{ctx:D}}}),ee=new M({props:{name:"step",anchor:"transformers.Adafactor.step",parameters:[{name:"closure",val:" = None"}],parametersDescription:[{anchor:"transformers.Adafactor.step.closure",description:`<strong>closure</strong> (callable, optional) &#x2014; A closure that reevaluates the model
and returns the loss.`,name:"closure"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization.py#L752"}}),te=new O({props:{title:"AdamWeightDecay (TensorFlow)",local:"transformers.AdamWeightDecay",headingTag:"h2"}}),re=new M({props:{name:"class transformers.AdamWeightDecay",anchor:"transformers.AdamWeightDecay",parameters:[{name:"learning_rate",val:": typing.Union[float, tf_keras.src.optimizers.schedules.learning_rate_schedule.LearningRateSchedule] = 0.001"},{name:"beta_1",val:": float = 0.9"},{name:"beta_2",val:": float = 0.999"},{name:"epsilon",val:": float = 1e-07"},{name:"amsgrad",val:": bool = False"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"include_in_weight_decay",val:": typing.Optional[list[str]] = None"},{name:"exclude_from_weight_decay",val:": typing.Optional[list[str]] = None"},{name:"name",val:": str = 'AdamWeightDecay'"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.AdamWeightDecay.learning_rate",description:`<strong>learning_rate</strong> (<code>Union[float, LearningRateSchedule]</code>, <em>optional</em>, defaults to 0.001) &#x2014;
The learning rate to use or a schedule.`,name:"learning_rate"},{anchor:"transformers.AdamWeightDecay.beta_1",description:`<strong>beta_1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 parameter in Adam, which is the exponential decay rate for the 1st momentum estimates.`,name:"beta_1"},{anchor:"transformers.AdamWeightDecay.beta_2",description:`<strong>beta_2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 parameter in Adam, which is the exponential decay rate for the 2nd momentum estimates.`,name:"beta_2"},{anchor:"transformers.AdamWeightDecay.epsilon",description:`<strong>epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-07) &#x2014;
The epsilon parameter in Adam, which is a small constant for numerical stability.`,name:"epsilon"},{anchor:"transformers.AdamWeightDecay.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to apply AMSGrad variant of this algorithm or not, see <a href="https://arxiv.org/abs/1904.09237" rel="nofollow">On the Convergence of Adam and
Beyond</a>.`,name:"amsgrad"},{anchor:"transformers.AdamWeightDecay.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The weight decay to apply.`,name:"weight_decay_rate"},{anchor:"transformers.AdamWeightDecay.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters by default (unless they are in <code>exclude_from_weight_decay</code>).`,name:"include_in_weight_decay"},{anchor:"transformers.AdamWeightDecay.exclude_from_weight_decay",description:`<strong>exclude_from_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to exclude from applying weight decay to. If a
<code>include_in_weight_decay</code> is passed, the names in it will supersede this list.`,name:"exclude_from_weight_decay"},{anchor:"transformers.AdamWeightDecay.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;AdamWeightDecay&quot;</code>) &#x2014;
Optional name for the operations created when applying gradients.`,name:"name"},{anchor:"transformers.AdamWeightDecay.kwargs",description:`<strong>kwargs</strong> (<code>Dict[str, Any]</code>, <em>optional</em>) &#x2014;
Keyword arguments. Allowed to be {<code>clipnorm</code>, <code>clipvalue</code>, <code>lr</code>, <code>decay</code>}. <code>clipnorm</code> is clip gradients by
norm; <code>clipvalue</code> is clip gradients by value, <code>decay</code> is included for backward compatibility to allow time
inverse decay of learning rate. <code>lr</code> is included for backward compatibility, recommended to use
<code>learning_rate</code> instead.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization_tf.py#L180"}}),ae=new M({props:{name:"from_config",anchor:"transformers.AdamWeightDecay.from_config",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization_tf.py#L237"}}),ne=new M({props:{name:"transformers.create_optimizer",anchor:"transformers.create_optimizer",parameters:[{name:"init_lr",val:": float"},{name:"num_train_steps",val:": int"},{name:"num_warmup_steps",val:": int"},{name:"min_lr_ratio",val:": float = 0.0"},{name:"adam_beta1",val:": float = 0.9"},{name:"adam_beta2",val:": float = 0.999"},{name:"adam_epsilon",val:": float = 1e-08"},{name:"adam_clipnorm",val:": typing.Optional[float] = None"},{name:"adam_global_clipnorm",val:": typing.Optional[float] = None"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"power",val:": float = 1.0"},{name:"include_in_weight_decay",val:": typing.Optional[list[str]] = None"}],parametersDescription:[{anchor:"transformers.create_optimizer.init_lr",description:`<strong>init_lr</strong> (<code>float</code>) &#x2014;
The desired learning rate at the end of the warmup phase.`,name:"init_lr"},{anchor:"transformers.create_optimizer.num_train_steps",description:`<strong>num_train_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_train_steps"},{anchor:"transformers.create_optimizer.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of warmup steps.`,name:"num_warmup_steps"},{anchor:"transformers.create_optimizer.min_lr_ratio",description:`<strong>min_lr_ratio</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The final learning rate at the end of the linear decay will be <code>init_lr * min_lr_ratio</code>.`,name:"min_lr_ratio"},{anchor:"transformers.create_optimizer.adam_beta1",description:`<strong>adam_beta1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 to use in Adam.`,name:"adam_beta1"},{anchor:"transformers.create_optimizer.adam_beta2",description:`<strong>adam_beta2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 to use in Adam.`,name:"adam_beta2"},{anchor:"transformers.create_optimizer.adam_epsilon",description:`<strong>adam_epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-8) &#x2014;
The epsilon to use in Adam.`,name:"adam_epsilon"},{anchor:"transformers.create_optimizer.adam_clipnorm",description:`<strong>adam_clipnorm</strong> (<code>float</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
If not <code>None</code>, clip the gradient norm for each weight tensor to this value.`,name:"adam_clipnorm"},{anchor:"transformers.create_optimizer.adam_global_clipnorm",description:`<strong>adam_global_clipnorm</strong> (<code>float</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
If not <code>None</code>, clip gradient norm to this value. When using this argument, the norm is computed over all
weight tensors, as if they were concatenated into a single vector.`,name:"adam_global_clipnorm"},{anchor:"transformers.create_optimizer.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to use.`,name:"weight_decay_rate"},{anchor:"transformers.create_optimizer.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The power to use for PolynomialDecay.`,name:"power"},{anchor:"transformers.create_optimizer.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters except bias and layer norm parameters.`,name:"include_in_weight_decay"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization_tf.py#L96"}}),se=new O({props:{title:"Schedules",local:"schedules",headingTag:"h2"}}),oe=new O({props:{title:"Learning Rate Schedules (Pytorch)",local:"transformers.SchedulerType",headingTag:"h3"}}),ie=new M({props:{name:"class transformers.SchedulerType",anchor:"transformers.SchedulerType",parameters:[{name:"value",val:""},{name:"names",val:" = None"},{name:"module",val:" = None"},{name:"qualname",val:" = None"},{name:"type",val:" = None"},{name:"start",val:" = 1"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/trainer_utils.py#L420"}}),le=new M({props:{name:"transformers.get_scheduler",anchor:"transformers.get_scheduler",parameters:[{name:"name",val:": typing.Union[str, transformers.trainer_utils.SchedulerType]"},{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": typing.Optional[int] = None"},{name:"num_training_steps",val:": typing.Optional[int] = None"},{name:"scheduler_specific_kwargs",val:": typing.Optional[dict] = None"}],parametersDescription:[{anchor:"transformers.get_scheduler.name",description:`<strong>name</strong> (<code>str</code> or <code>SchedulerType</code>) &#x2014;
The name of the scheduler to use.`,name:"name"},{anchor:"transformers.get_scheduler.optimizer",description:`<strong>optimizer</strong> (<code>torch.optim.Optimizer</code>) &#x2014;
The optimizer that will be used during training.`,name:"optimizer"},{anchor:"transformers.get_scheduler.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The number of warmup steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_warmup_steps"},{anchor:"transformers.get_scheduler.num_training_steps",description:`<strong>num_training_steps</strong> (\`int&#x201C;, <em>optional</em>) &#x2014;
The number of training steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_training_steps"},{anchor:"transformers.get_scheduler.scheduler_specific_kwargs",description:`<strong>scheduler_specific_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Extra parameters for schedulers such as cosine with restarts. Mismatched scheduler types and scheduler
parameters will cause the scheduler function to raise a TypeError.`,name:"scheduler_specific_kwargs"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization.py#L513"}}),me=new M({props:{name:"transformers.get_constant_schedule",anchor:"transformers.get_constant_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization.py#L38",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),ce=new M({props:{name:"transformers.get_constant_schedule_with_warmup",anchor:"transformers.get_constant_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_constant_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization.py#L79",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),de=new M({props:{name:"transformers.get_cosine_schedule_with_warmup",anchor:"transformers.get_cosine_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": float = 0.5"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>float</code>, <em>optional</em>, defaults to 0.5) &#x2014;
The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0
following a half-cosine).`,name:"num_cycles"},{anchor:"transformers.get_cosine_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization.py#L142",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),ue=new M({props:{name:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": int = 1"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of hard restarts to use.`,name:"num_cycles"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization.py#L187",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),_e=new M({props:{name:"transformers.get_linear_schedule_with_warmup",anchor:"transformers.get_linear_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_linear_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_linear_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization.py#L106",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),we=new M({props:{name:"transformers.get_polynomial_decay_schedule_with_warmup",anchor:"transformers.get_polynomial_decay_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"lr_end",val:" = 1e-07"},{name:"power",val:" = 1.0"},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.lr_end",description:`<strong>lr_end</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The end LR.`,name:"lr_end"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
Power factor.`,name:"power"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization.py#L241",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),be=new M({props:{name:"transformers.get_inverse_sqrt_schedule",anchor:"transformers.get_inverse_sqrt_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"timescale",val:": typing.Optional[int] = None"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_inverse_sqrt_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_inverse_sqrt_schedule.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_inverse_sqrt_schedule.timescale",description:`<strong>timescale</strong> (<code>int</code>, <em>optional</em>, defaults to <code>num_warmup_steps</code>) &#x2014;
Time scale.`,name:"timescale"},{anchor:"transformers.get_inverse_sqrt_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization.py#L295",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),ve=new O({props:{title:"Warmup (TensorFlow)",local:"transformers.WarmUp",headingTag:"h3"}}),$e=new M({props:{name:"class transformers.WarmUp",anchor:"transformers.WarmUp",parameters:[{name:"initial_learning_rate",val:": float"},{name:"decay_schedule_fn",val:": typing.Callable"},{name:"warmup_steps",val:": int"},{name:"power",val:": float = 1.0"},{name:"name",val:": typing.Optional[str] = None"}],parametersDescription:[{anchor:"transformers.WarmUp.initial_learning_rate",description:`<strong>initial_learning_rate</strong> (<code>float</code>) &#x2014;
The initial learning rate for the schedule after the warmup (so this will be the learning rate at the end
of the warmup).`,name:"initial_learning_rate"},{anchor:"transformers.WarmUp.decay_schedule_fn",description:`<strong>decay_schedule_fn</strong> (<code>Callable</code>) &#x2014;
The schedule function to apply after the warmup for the rest of training.`,name:"decay_schedule_fn"},{anchor:"transformers.WarmUp.warmup_steps",description:`<strong>warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup part of training.`,name:"warmup_steps"},{anchor:"transformers.WarmUp.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The power to use for the polynomial warmup (defaults is a linear warmup).`,name:"power"},{anchor:"transformers.WarmUp.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Optional name prefix for the returned tensors during the schedule.`,name:"name"}],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization_tf.py#L38"}}),ye=new O({props:{title:"Gradient Strategies",local:"gradient-strategies",headingTag:"h2"}}),xe=new O({props:{title:"GradientAccumulator (TensorFlow)",local:"transformers.GradientAccumulator",headingTag:"h3"}}),Te=new M({props:{name:"class transformers.GradientAccumulator",anchor:"transformers.GradientAccumulator",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization_tf.py#L310"}}),ze=new M({props:{name:"reset",anchor:"transformers.GradientAccumulator.reset",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_37082/src/transformers/optimization_tf.py#L372"}}),Me=new ia({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/zh/main_classes/optimizer_schedules.md"}}),{c(){c=l("meta"),A=a(),x=l("p"),p=a(),d(v.$$.fragment),s=a(),T=l("p"),T.innerHTML=zr,nt=a(),Y=l("ul"),Y.innerHTML=Mr,st=a(),d(Q.$$.fragment),ot=a(),b=l("div"),d(K.$$.fragment),qt=a(),Le=l("p"),Le.innerHTML=Ar,Bt=a(),je=l("p"),je.innerHTML=Cr,Xt=a(),Ue=l("p"),Ue.textContent=Lr,Ot=a(),We=l("p"),We.innerHTML=jr,Yt=a(),De=l("ul"),De.innerHTML=Ur,Qt=a(),d(N.$$.fragment),Kt=a(),d(S.$$.fragment),er=a(),Re=l("p"),Re.innerHTML=Wr,tr=a(),d(P.$$.fragment),rr=a(),d(H.$$.fragment),ar=a(),q=l("div"),d(ee.$$.fragment),nr=a(),Ee=l("p"),Ee.textContent=Dr,it=a(),d(te.$$.fragment),lt=a(),C=l("div"),d(re.$$.fragment),sr=a(),ke=l("p"),ke.innerHTML=Rr,or=a(),Fe=l("p"),Fe.textContent=Er,ir=a(),B=l("div"),d(ae.$$.fragment),lr=a(),Ie=l("p"),Ie.textContent=kr,mt=a(),R=l("div"),d(ne.$$.fragment),mr=a(),Ge=l("p"),Ge.textContent=Fr,ct=a(),d(se.$$.fragment),pt=a(),d(oe.$$.fragment),dt=a(),L=l("div"),d(ie.$$.fragment),cr=a(),Je=l("p"),Je.innerHTML=Ir,pr=a(),Ve=l("ul"),Ve.innerHTML=Gr,ht=a(),E=l("div"),d(le.$$.fragment),dr=a(),Ze=l("p"),Ze.textContent=Jr,ut=a(),k=l("div"),d(me.$$.fragment),hr=a(),Ne=l("p"),Ne.textContent=Vr,gt=a(),F=l("div"),d(ce.$$.fragment),ur=a(),Se=l("p"),Se.textContent=Zr,_t=a(),pe=l("img"),ft=a(),I=l("div"),d(de.$$.fragment),gr=a(),Pe=l("p"),Pe.textContent=Sr,wt=a(),he=l("img"),bt=a(),G=l("div"),d(ue.$$.fragment),_r=a(),He=l("p"),He.textContent=Hr,vt=a(),ge=l("img"),$t=a(),J=l("div"),d(_e.$$.fragment),fr=a(),qe=l("p"),qe.textContent=Br,yt=a(),fe=l("img"),xt=a(),j=l("div"),d(we.$$.fragment),wr=a(),Be=l("p"),Be.innerHTML=Or,br=a(),Xe=l("p"),Xe.innerHTML=Yr,Tt=a(),V=l("div"),d(be.$$.fragment),vr=a(),Oe=l("p"),Oe.textContent=Qr,zt=a(),d(ve.$$.fragment),Mt=a(),Z=l("div"),d($e.$$.fragment),$r=a(),Ye=l("p"),Ye.textContent=Kr,At=a(),d(ye.$$.fragment),Ct=a(),d(xe.$$.fragment),Lt=a(),U=l("div"),d(Te.$$.fragment),yr=a(),Qe=l("p"),Qe.innerHTML=ea,xr=a(),X=l("div"),d(ze.$$.fragment),Tr=a(),Ke=l("p"),Ke.textContent=ta,jt=a(),d(Me.$$.fragment),Ut=a(),at=l("p"),this.h()},l(e){const r=oa("svelte-u9bgzb",document.head);c=m(r,"META",{name:!0,content:!0}),r.forEach(t),A=n(e),x=m(e,"P",{}),z(x).forEach(t),p=n(e),h(v.$$.fragment,e),s=n(e),T=m(e,"P",{"data-svelte-h":!0}),w(T)!=="svelte-1p2ld9z"&&(T.innerHTML=zr),nt=n(e),Y=m(e,"UL",{"data-svelte-h":!0}),w(Y)!=="svelte-p818uy"&&(Y.innerHTML=Mr),st=n(e),h(Q.$$.fragment,e),ot=n(e),b=m(e,"DIV",{class:!0});var $=z(b);h(K.$$.fragment,$),qt=n($),Le=m($,"P",{"data-svelte-h":!0}),w(Le)!=="svelte-114ll6r"&&(Le.innerHTML=Ar),Bt=n($),je=m($,"P",{"data-svelte-h":!0}),w(je)!=="svelte-9bvrpy"&&(je.innerHTML=Cr),Xt=n($),Ue=m($,"P",{"data-svelte-h":!0}),w(Ue)!=="svelte-1dxk71o"&&(Ue.textContent=Lr),Ot=n($),We=m($,"P",{"data-svelte-h":!0}),w(We)!=="svelte-kc6644"&&(We.innerHTML=jr),Yt=n($),De=m($,"UL",{"data-svelte-h":!0}),w(De)!=="svelte-p81jc"&&(De.innerHTML=Ur),Qt=n($),h(N.$$.fragment,$),Kt=n($),h(S.$$.fragment,$),er=n($),Re=m($,"P",{"data-svelte-h":!0}),w(Re)!=="svelte-w4e00h"&&(Re.innerHTML=Wr),tr=n($),h(P.$$.fragment,$),rr=n($),h(H.$$.fragment,$),ar=n($),q=m($,"DIV",{class:!0});var Ae=z(q);h(ee.$$.fragment,Ae),nr=n(Ae),Ee=m(Ae,"P",{"data-svelte-h":!0}),w(Ee)!=="svelte-6lkj7q"&&(Ee.textContent=Dr),Ae.forEach(t),$.forEach(t),it=n(e),h(te.$$.fragment,e),lt=n(e),C=m(e,"DIV",{class:!0});var W=z(C);h(re.$$.fragment,W),sr=n(W),ke=m(W,"P",{"data-svelte-h":!0}),w(ke)!=="svelte-towids"&&(ke.innerHTML=Rr),or=n(W),Fe=m(W,"P",{"data-svelte-h":!0}),w(Fe)!=="svelte-id029h"&&(Fe.textContent=Er),ir=n(W),B=m(W,"DIV",{class:!0});var Ce=z(B);h(ae.$$.fragment,Ce),lr=n(Ce),Ie=m(Ce,"P",{"data-svelte-h":!0}),w(Ie)!=="svelte-3iict9"&&(Ie.textContent=kr),Ce.forEach(t),W.forEach(t),mt=n(e),R=m(e,"DIV",{class:!0});var Dt=z(R);h(ne.$$.fragment,Dt),mr=n(Dt),Ge=m(Dt,"P",{"data-svelte-h":!0}),w(Ge)!=="svelte-16qojcu"&&(Ge.textContent=Fr),Dt.forEach(t),ct=n(e),h(se.$$.fragment,e),pt=n(e),h(oe.$$.fragment,e),dt=n(e),L=m(e,"DIV",{class:!0});var et=z(L);h(ie.$$.fragment,et),cr=n(et),Je=m(et,"P",{"data-svelte-h":!0}),w(Je)!=="svelte-1u50d1g"&&(Je.innerHTML=Ir),pr=n(et),Ve=m(et,"UL",{"data-svelte-h":!0}),w(Ve)!=="svelte-1i9daf3"&&(Ve.innerHTML=Gr),et.forEach(t),ht=n(e),E=m(e,"DIV",{class:!0});var Rt=z(E);h(le.$$.fragment,Rt),dr=n(Rt),Ze=m(Rt,"P",{"data-svelte-h":!0}),w(Ze)!=="svelte-1oh310h"&&(Ze.textContent=Jr),Rt.forEach(t),ut=n(e),k=m(e,"DIV",{class:!0});var Et=z(k);h(me.$$.fragment,Et),hr=n(Et),Ne=m(Et,"P",{"data-svelte-h":!0}),w(Ne)!=="svelte-cadr4a"&&(Ne.textContent=Vr),Et.forEach(t),gt=n(e),F=m(e,"DIV",{class:!0});var kt=z(F);h(ce.$$.fragment,kt),ur=n(kt),Se=m(kt,"P",{"data-svelte-h":!0}),w(Se)!=="svelte-1lj8vcj"&&(Se.textContent=Zr),kt.forEach(t),_t=n(e),pe=m(e,"IMG",{alt:!0,src:!0}),ft=n(e),I=m(e,"DIV",{class:!0});var Ft=z(I);h(de.$$.fragment,Ft),gr=n(Ft),Pe=m(Ft,"P",{"data-svelte-h":!0}),w(Pe)!=="svelte-1lhkhz0"&&(Pe.textContent=Sr),Ft.forEach(t),wt=n(e),he=m(e,"IMG",{alt:!0,src:!0}),bt=n(e),G=m(e,"DIV",{class:!0});var It=z(G);h(ue.$$.fragment,It),_r=n(It),He=m(It,"P",{"data-svelte-h":!0}),w(He)!=="svelte-1pclsvt"&&(He.textContent=Hr),It.forEach(t),vt=n(e),ge=m(e,"IMG",{alt:!0,src:!0}),$t=n(e),J=m(e,"DIV",{class:!0});var Gt=z(J);h(_e.$$.fragment,Gt),fr=n(Gt),qe=m(Gt,"P",{"data-svelte-h":!0}),w(qe)!=="svelte-17q4ov5"&&(qe.textContent=Br),Gt.forEach(t),yt=n(e),fe=m(e,"IMG",{alt:!0,src:!0}),xt=n(e),j=m(e,"DIV",{class:!0});var tt=z(j);h(we.$$.fragment,tt),wr=n(tt),Be=m(tt,"P",{"data-svelte-h":!0}),w(Be)!=="svelte-1y55gjq"&&(Be.innerHTML=Or),br=n(tt),Xe=m(tt,"P",{"data-svelte-h":!0}),w(Xe)!=="svelte-1tn0l2l"&&(Xe.innerHTML=Yr),tt.forEach(t),Tt=n(e),V=m(e,"DIV",{class:!0});var Jt=z(V);h(be.$$.fragment,Jt),vr=n(Jt),Oe=m(Jt,"P",{"data-svelte-h":!0}),w(Oe)!=="svelte-yp2yn6"&&(Oe.textContent=Qr),Jt.forEach(t),zt=n(e),h(ve.$$.fragment,e),Mt=n(e),Z=m(e,"DIV",{class:!0});var Vt=z(Z);h($e.$$.fragment,Vt),$r=n(Vt),Ye=m(Vt,"P",{"data-svelte-h":!0}),w(Ye)!=="svelte-1mfn66a"&&(Ye.textContent=Kr),Vt.forEach(t),At=n(e),h(ye.$$.fragment,e),Ct=n(e),h(xe.$$.fragment,e),Lt=n(e),U=m(e,"DIV",{class:!0});var rt=z(U);h(Te.$$.fragment,rt),yr=n(rt),Qe=m(rt,"P",{"data-svelte-h":!0}),w(Qe)!=="svelte-8tkiw"&&(Qe.innerHTML=ea),xr=n(rt),X=m(rt,"DIV",{class:!0});var Zt=z(X);h(ze.$$.fragment,Zt),Tr=n(Zt),Ke=m(Zt,"P",{"data-svelte-h":!0}),w(Ke)!=="svelte-v0y289"&&(Ke.textContent=ta),Zt.forEach(t),rt.forEach(t),jt=n(e),h(Me.$$.fragment,e),Ut=n(e),at=m(e,"P",{}),z(at).forEach(t),this.h()},h(){y(c,"name","hf:doc:metadata"),y(c,"content",ha),y(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(pe,"alt",""),Nt(pe.src,Nr="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_constant_schedule.png")||y(pe,"src",Nr),y(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(he,"alt",""),Nt(he.src,Pr="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_schedule.png")||y(he,"src",Pr),y(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(ge,"alt",""),Nt(ge.src,qr="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_hard_restarts_schedule.png")||y(ge,"src",qr),y(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(fe,"alt",""),Nt(fe.src,Xr="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_linear_schedule.png")||y(fe,"src",Xr),y(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),y(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,r){o(document.head,c),i(e,A,r),i(e,x,r),i(e,p,r),u(v,e,r),i(e,s,r),i(e,T,r),i(e,nt,r),i(e,Y,r),i(e,st,r),u(Q,e,r),i(e,ot,r),i(e,b,r),u(K,b,null),o(b,qt),o(b,Le),o(b,Bt),o(b,je),o(b,Xt),o(b,Ue),o(b,Ot),o(b,We),o(b,Yt),o(b,De),o(b,Qt),u(N,b,null),o(b,Kt),u(S,b,null),o(b,er),o(b,Re),o(b,tr),u(P,b,null),o(b,rr),u(H,b,null),o(b,ar),o(b,q),u(ee,q,null),o(q,nr),o(q,Ee),i(e,it,r),u(te,e,r),i(e,lt,r),i(e,C,r),u(re,C,null),o(C,sr),o(C,ke),o(C,or),o(C,Fe),o(C,ir),o(C,B),u(ae,B,null),o(B,lr),o(B,Ie),i(e,mt,r),i(e,R,r),u(ne,R,null),o(R,mr),o(R,Ge),i(e,ct,r),u(se,e,r),i(e,pt,r),u(oe,e,r),i(e,dt,r),i(e,L,r),u(ie,L,null),o(L,cr),o(L,Je),o(L,pr),o(L,Ve),i(e,ht,r),i(e,E,r),u(le,E,null),o(E,dr),o(E,Ze),i(e,ut,r),i(e,k,r),u(me,k,null),o(k,hr),o(k,Ne),i(e,gt,r),i(e,F,r),u(ce,F,null),o(F,ur),o(F,Se),i(e,_t,r),i(e,pe,r),i(e,ft,r),i(e,I,r),u(de,I,null),o(I,gr),o(I,Pe),i(e,wt,r),i(e,he,r),i(e,bt,r),i(e,G,r),u(ue,G,null),o(G,_r),o(G,He),i(e,vt,r),i(e,ge,r),i(e,$t,r),i(e,J,r),u(_e,J,null),o(J,fr),o(J,qe),i(e,yt,r),i(e,fe,r),i(e,xt,r),i(e,j,r),u(we,j,null),o(j,wr),o(j,Be),o(j,br),o(j,Xe),i(e,Tt,r),i(e,V,r),u(be,V,null),o(V,vr),o(V,Oe),i(e,zt,r),u(ve,e,r),i(e,Mt,r),i(e,Z,r),u($e,Z,null),o(Z,$r),o(Z,Ye),i(e,At,r),u(ye,e,r),i(e,Ct,r),u(xe,e,r),i(e,Lt,r),i(e,U,r),u(Te,U,null),o(U,yr),o(U,Qe),o(U,xr),o(U,X),u(ze,X,null),o(X,Tr),o(X,Ke),i(e,jt,r),u(Me,e,r),i(e,Ut,r),i(e,at,r),Wt=!0},p(e,[r]){const $={};r&2&&($.$$scope={dirty:r,ctx:e}),N.$set($);const Ae={};r&2&&(Ae.$$scope={dirty:r,ctx:e}),S.$set(Ae);const W={};r&2&&(W.$$scope={dirty:r,ctx:e}),P.$set(W);const Ce={};r&2&&(Ce.$$scope={dirty:r,ctx:e}),H.$set(Ce)},i(e){Wt||(g(v.$$.fragment,e),g(Q.$$.fragment,e),g(K.$$.fragment,e),g(N.$$.fragment,e),g(S.$$.fragment,e),g(P.$$.fragment,e),g(H.$$.fragment,e),g(ee.$$.fragment,e),g(te.$$.fragment,e),g(re.$$.fragment,e),g(ae.$$.fragment,e),g(ne.$$.fragment,e),g(se.$$.fragment,e),g(oe.$$.fragment,e),g(ie.$$.fragment,e),g(le.$$.fragment,e),g(me.$$.fragment,e),g(ce.$$.fragment,e),g(de.$$.fragment,e),g(ue.$$.fragment,e),g(_e.$$.fragment,e),g(we.$$.fragment,e),g(be.$$.fragment,e),g(ve.$$.fragment,e),g($e.$$.fragment,e),g(ye.$$.fragment,e),g(xe.$$.fragment,e),g(Te.$$.fragment,e),g(ze.$$.fragment,e),g(Me.$$.fragment,e),Wt=!0)},o(e){_(v.$$.fragment,e),_(Q.$$.fragment,e),_(K.$$.fragment,e),_(N.$$.fragment,e),_(S.$$.fragment,e),_(P.$$.fragment,e),_(H.$$.fragment,e),_(ee.$$.fragment,e),_(te.$$.fragment,e),_(re.$$.fragment,e),_(ae.$$.fragment,e),_(ne.$$.fragment,e),_(se.$$.fragment,e),_(oe.$$.fragment,e),_(ie.$$.fragment,e),_(le.$$.fragment,e),_(me.$$.fragment,e),_(ce.$$.fragment,e),_(de.$$.fragment,e),_(ue.$$.fragment,e),_(_e.$$.fragment,e),_(we.$$.fragment,e),_(be.$$.fragment,e),_(ve.$$.fragment,e),_($e.$$.fragment,e),_(ye.$$.fragment,e),_(xe.$$.fragment,e),_(Te.$$.fragment,e),_(ze.$$.fragment,e),_(Me.$$.fragment,e),Wt=!1},d(e){e&&(t(A),t(x),t(p),t(s),t(T),t(nt),t(Y),t(st),t(ot),t(b),t(it),t(lt),t(C),t(mt),t(R),t(ct),t(pt),t(dt),t(L),t(ht),t(E),t(ut),t(k),t(gt),t(F),t(_t),t(pe),t(ft),t(I),t(wt),t(he),t(bt),t(G),t(vt),t(ge),t($t),t(J),t(yt),t(fe),t(xt),t(j),t(Tt),t(V),t(zt),t(Mt),t(Z),t(At),t(Ct),t(Lt),t(U),t(jt),t(Ut),t(at)),t(c),f(v,e),f(Q,e),f(K),f(N),f(S),f(P),f(H),f(ee),f(te,e),f(re),f(ae),f(ne),f(se,e),f(oe,e),f(ie),f(le),f(me),f(ce),f(de),f(ue),f(_e),f(we),f(be),f(ve,e),f($e),f(ye,e),f(xe,e),f(Te),f(ze),f(Me,e)}}}const ha='{"title":"Optimization","local":"optimization","sections":[{"title":"AdaFactor (PyTorch)","local":"transformers.Adafactor","sections":[],"depth":2},{"title":"AdamWeightDecay (TensorFlow)","local":"transformers.AdamWeightDecay","sections":[],"depth":2},{"title":"Schedules","local":"schedules","sections":[{"title":"Learning Rate Schedules (Pytorch)","local":"transformers.SchedulerType","sections":[],"depth":3},{"title":"Warmup (TensorFlow)","local":"transformers.WarmUp","sections":[],"depth":3}],"depth":2},{"title":"Gradient Strategies","local":"gradient-strategies","sections":[{"title":"GradientAccumulator (TensorFlow)","local":"transformers.GradientAccumulator","sections":[],"depth":3}],"depth":2}],"depth":1}';function ua(D){return aa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $a extends na{constructor(c){super(),sa(this,c,ua,da,ra,{})}}export{$a as component};