accelerate/pr_4064/en/_app/immutable/nodes/32.cca7ad4e.js

import{s as Xe,n as Ye,o as Ze}from"../chunks/scheduler.b9285784.js";import{S as et,i as tt,e as l,s as o,c as s,h as at,a as i,d as a,b as r,f as _,g as c,j as P,k as f,l as u,m as n,n as d,t as m,o as g,p}from"../chunks/index.26bc89a1.js";import{C as nt,H as w,E as ot}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.d17575b1.js";import{D as T}from"../chunks/Docstring.76cedc42.js";function rt(Fe){let h,oe,ae,re,D,le,C,se,S,ie,v,k,Oe,J,We=`Plugin for Megatron-LM to enable tensor, pipeline, sequence and data parallelism. Also to enable selective
activation recomputation and optimized fused kernels.`,ce,z,de,M,O,qe,K,je=`Dummy scheduler presents model parameters or param groups, this is primarily used to follow conventional training
loop when scheduler config is specified in the deepspeed config file.`,me,q,ge,b,E,Ee,Q,He="Dummy dataloader presents model parameters or param groups, this is primarily used to follow conventional training",pe,I,ue,$,A,Ie,X,Re="Abstract class for batching, forward pass and loss handler.",_e,G,fe,y,B,Ae,Y,Ue="GPT train step class.",he,V,ve,x,F,Ge,Z,Je="Bert train step class.",Me,W,be,N,j,Be,ee,Ke="T5 train step class.",$e,H,ye,L,R,Ve,te,Qe="Average losses across data parallel group.",xe,U,Ne,ne,Le;return D=new nt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),C=new w({props:{title:"Megatron-LM utilities",local:"megatron-lm-utilities",headingTag:"h1"}}),S=new w({props:{title:"MegatronLMPlugin",local:"accelerate.utils.MegatronLMPlugin",headingTag:"h2"}}),k=new T({props:{name:"class accelerate.utils.MegatronLMPlugin",anchor:"accelerate.utils.MegatronLMPlugin",parameters:[{name:"tp_degree",val:": int = None"},{name:"pp_degree",val:": int = None"},{name:"use_custom_fsdp",val:": bool = None"},{name:"overlap_cpu_optimizer_d2h_h2d",val:": bool = None"},{name:"no_load_optim",val:": bool = None"},{name:"eod_mask_loss",val:": bool = None"},{name:"no_save_optim",val:": bool = None"},{name:"optimizer_cpu_offload",val:": bool = None"},{name:"use_precision_aware_optimizer",val:": bool = None"},{name:"decoder_last_pipeline_num_layers",val:": int = None"},{name:"recompute_granularity",val:": str = None"},{name:"recompute_method",val:": str = None"},{name:"recompute_num_layers",val:": int = None"},{name:"attention_backend",val:": bool = None"},{name:"expert_model_parallel_size",val:": int = None"},{name:"context_parallel_size",val:": int = None"},{name:"attention_dropout",val:": float = None"},{name:"hidden_dropout",val:": float = None"},{name:"attention_softmax_in_fp32",val:": bool = None"},{name:"expert_tensor_parallel_size",val:": int = None"},{name:"calculate_per_token_loss",val:": bool = None"},{name:"use_rotary_position_embeddings",val:": bool = None"},{name:"num_micro_batches",val:": int = None"},{name:"gradient_clipping",val:": float = None"},{name:"sequence_parallelism",val:": bool = None"},{name:"recompute_activations",val:": bool = None"},{name:"use_distributed_optimizer",val:": bool = None"},{name:"pipeline_model_parallel_split_rank",val:": int = None"},{name:"num_layers_per_virtual_pipeline_stage",val:": int = None"},{name:"is_train_batch_min",val:": str = True"},{name:"train_iters",val:": int = None"},{name:"train_samples",val:": int = None"},{name:"weight_decay_incr_style",val:": str = 'constant'"},{name:"start_weight_decay",val:": float = None"},{name:"end_weight_decay",val:": float = None"},{name:"lr_decay_style",val:": str = 'linear'"},{name:"lr_decay_iters",val:": int = None"},{name:"lr_decay_samples",val:": int = None"},{name:"lr_warmup_iters",val:": int = None"},{name:"lr_warmup_samples",val:": int = None"},{name:"lr_warmup_fraction",val:": float = None"},{name:"min_lr",val:": float = 0"},{name:"consumed_samples",val:": list = None"},{name:"no_wd_decay_cond",val:": typing.Optional[typing.Callable] = None"},{name:"scale_lr_cond",val:": typing.Optional[typing.Callable] = None"},{name:"lr_mult",val:": float = 1.0"},{name:"megatron_dataset_flag",val:": bool = False"},{name:"seq_length",val:": int = None"},{name:"encoder_seq_length",val:": int = None"},{name:"decoder_seq_length",val:": int = None"},{name:"tensorboard_dir",val:": str = None"},{name:"set_all_logging_options",val:": bool = False"},{name:"eval_iters",val:": int = 100"},{name:"eval_interval",val:": int = 1000"},{name:"return_logits",val:": bool = False"},{name:"custom_train_step_class",val:": typing.Optional[typing.Any] = None"},{name:"custom_train_step_kwargs",val:": typing.Optional[dict[str, typing.Any]] = None"},{name:"custom_model_provider_function",val:": typing.Optional[typing.Callable] = None"},{name:"custom_prepare_model_function",val:": typing.Optional[typing.Callable] = None"},{name:"custom_megatron_datasets_provider_function",val:": typing.Optional[typing.Callable] = None"},{name:"custom_get_batch_function",val:": typing.Optional[typing.Callable] = None"},{name:"custom_loss_function",val:": typing.Optional[typing.Callable] = None"},{name:"other_megatron_args",val:": typing.Optional[dict[str, typing.Any]] = None"}],parametersDescription:[{anchor:"accelerate.utils.MegatronLMPlugin.tp_degree",description:`<strong>tp_degree</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Tensor parallelism degree.`,name:"tp_degree"},{anchor:"accelerate.utils.MegatronLMPlugin.pp_degree",description:`<strong>pp_degree</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Pipeline parallelism degree.`,name:"pp_degree"},{anchor:"accelerate.utils.MegatronLMPlugin.num_micro_batches",description:`<strong>num_micro_batches</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of micro-batches.`,name:"num_micro_batches"},{anchor:"accelerate.utils.MegatronLMPlugin.gradient_clipping",description:`<strong>gradient_clipping</strong> (<code>float</code>, defaults to <code>None</code>) &#x2014;
Gradient clipping value based on global L2 Norm (0 to disable).`,name:"gradient_clipping"},{anchor:"accelerate.utils.MegatronLMPlugin.sequence_parallelism",description:`<strong>sequence_parallelism</strong> (<code>bool</code>, defaults to <code>None</code>) &#x2014;
Enable sequence parallelism.`,name:"sequence_parallelism"},{anchor:"accelerate.utils.MegatronLMPlugin.recompute_activations",description:`<strong>recompute_activations</strong> (<code>bool</code>, defaults to <code>None</code>) &#x2014;
Enable selective activation recomputation.`,name:"recompute_activations"},{anchor:"accelerate.utils.MegatronLMPlugin.use_distributed_optimizr",description:`<strong>use_distributed_optimizr</strong> (<code>bool</code>, defaults to <code>None</code>) &#x2014;
Enable distributed optimizer.`,name:"use_distributed_optimizr"},{anchor:"accelerate.utils.MegatronLMPlugin.pipeline_model_parallel_split_rank",description:`<strong>pipeline_model_parallel_split_rank</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Rank where encoder and decoder should be split.`,name:"pipeline_model_parallel_split_rank"},{anchor:"accelerate.utils.MegatronLMPlugin.num_layers_per_virtual_pipeline_stage",description:`<strong>num_layers_per_virtual_pipeline_stage</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of layers per virtual pipeline stage.`,name:"num_layers_per_virtual_pipeline_stage"},{anchor:"accelerate.utils.MegatronLMPlugin.is_train_batch_min",description:`<strong>is_train_batch_min</strong> (<code>str</code>, defaults to <code>True</code>) &#x2014;
If both tran &amp; eval dataloaders are specified, this will decide the <code>micro_batch_size</code>.`,name:"is_train_batch_min"},{anchor:"accelerate.utils.MegatronLMPlugin.train_iters",description:`<strong>train_iters</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Total number of samples to train over all training runs. Note that either train-iters or train-samples
should be provided when using <code>MegatronLMDummyScheduler</code>.`,name:"train_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.train_samples",description:`<strong>train_samples</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Total number of samples to train over all training runs. Note that either train-iters or train-samples
should be provided when using <code>MegatronLMDummyScheduler</code>.`,name:"train_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.weight_decay_incr_style",description:`<strong>weight_decay_incr_style</strong> (<code>str</code>, defaults to <code>&apos;constant&apos;</code>) &#x2014;
Weight decay increment function. choices=[&#x201C;constant&#x201D;, &#x201C;linear&#x201D;, &#x201C;cosine&#x201D;].`,name:"weight_decay_incr_style"},{anchor:"accelerate.utils.MegatronLMPlugin.start_weight_decay",description:`<strong>start_weight_decay</strong> (<code>float</code>, defaults to <code>None</code>) &#x2014;
Initial weight decay coefficient for L2 regularization.`,name:"start_weight_decay"},{anchor:"accelerate.utils.MegatronLMPlugin.end_weight_decay",description:`<strong>end_weight_decay</strong> (<code>float</code>, defaults to <code>None</code>) &#x2014;
End of run weight decay coefficient for L2 regularization.`,name:"end_weight_decay"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_decay_style",description:`<strong>lr_decay_style</strong> (<code>str</code>, defaults to <code>&apos;linear&apos;</code>) &#x2014;
Learning rate decay function. choices=[&#x2018;constant&#x2019;, &#x2018;linear&#x2019;, &#x2018;cosine&#x2019;].`,name:"lr_decay_style"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_decay_iters",description:`<strong>lr_decay_iters</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of iterations for learning rate decay. If None defaults to <code>train_iters</code>.`,name:"lr_decay_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_decay_samples",description:`<strong>lr_decay_samples</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of samples for learning rate decay. If None defaults to <code>train_samples</code>.`,name:"lr_decay_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_warmup_iters",description:`<strong>lr_warmup_iters</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of iterations to linearly warmup learning rate over.`,name:"lr_warmup_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_warmup_samples",description:`<strong>lr_warmup_samples</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of samples to linearly warmup learning rate over.`,name:"lr_warmup_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_warmup_fraction",description:`<strong>lr_warmup_fraction</strong> (<code>float</code>, defaults to <code>None</code>) &#x2014;
Fraction of lr-warmup-(iters/samples) to linearly warmup learning rate over.`,name:"lr_warmup_fraction"},{anchor:"accelerate.utils.MegatronLMPlugin.min_lr",description:`<strong>min_lr</strong> (<code>float</code>, defaults to <code>0</code>) &#x2014;
Minimum value for learning rate. The scheduler clip values below this threshold.`,name:"min_lr"},{anchor:"accelerate.utils.MegatronLMPlugin.consumed_samples",description:`<strong>consumed_samples</strong> (<code>List</code>, defaults to <code>None</code>) &#x2014;
Number of samples consumed in the same order as the dataloaders to <code>accelerator.prepare</code> call.`,name:"consumed_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.no_wd_decay_cond",description:`<strong>no_wd_decay_cond</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Condition to disable weight decay.`,name:"no_wd_decay_cond"},{anchor:"accelerate.utils.MegatronLMPlugin.scale_lr_cond",description:`<strong>scale_lr_cond</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Condition to scale learning rate.`,name:"scale_lr_cond"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_mult",description:`<strong>lr_mult</strong> (<code>float</code>, defaults to <code>1.0</code>) &#x2014;
Learning rate multiplier.`,name:"lr_mult"},{anchor:"accelerate.utils.MegatronLMPlugin.megatron_dataset_flag",description:`<strong>megatron_dataset_flag</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the format of dataset follows Megatron-LM Indexed/Cached/MemoryMapped format.`,name:"megatron_dataset_flag"},{anchor:"accelerate.utils.MegatronLMPlugin.seq_length",description:`<strong>seq_length</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Maximum sequence length to process.`,name:"seq_length"},{anchor:"accelerate.utils.MegatronLMPlugin.encoder_seq_length",description:`<strong>encoder_seq_length</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Maximum sequence length to process for the encoder.`,name:"encoder_seq_length"},{anchor:"accelerate.utils.MegatronLMPlugin.decoder_seq_length",description:`<strong>decoder_seq_length</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Maximum sequence length to process for the decoder.`,name:"decoder_seq_length"},{anchor:"accelerate.utils.MegatronLMPlugin.tensorboard_dir",description:`<strong>tensorboard_dir</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Path to save tensorboard logs.`,name:"tensorboard_dir"},{anchor:"accelerate.utils.MegatronLMPlugin.set_all_logging_options",description:`<strong>set_all_logging_options</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to set all logging options.`,name:"set_all_logging_options"},{anchor:"accelerate.utils.MegatronLMPlugin.eval_iters",description:`<strong>eval_iters</strong> (<code>int</code>, defaults to <code>100</code>) &#x2014;
Number of iterations to run for evaluation validation/test for.`,name:"eval_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.eval_interval",description:`<strong>eval_interval</strong> (<code>int</code>, defaults to <code>1000</code>) &#x2014;
Interval between running evaluation on validation set.`,name:"eval_interval"},{anchor:"accelerate.utils.MegatronLMPlugin.return_logits",description:`<strong>return_logits</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to return logits from the model.`,name:"return_logits"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_train_step_class",description:`<strong>custom_train_step_class</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom train step class.`,name:"custom_train_step_class"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_train_step_kwargs",description:`<strong>custom_train_step_kwargs</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom train step kwargs.`,name:"custom_train_step_kwargs"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_model_provider_function",description:`<strong>custom_model_provider_function</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom model provider function.`,name:"custom_model_provider_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_prepare_model_function",description:`<strong>custom_prepare_model_function</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom prepare model function.`,name:"custom_prepare_model_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_megatron_datasets_provider_function",description:`<strong>custom_megatron_datasets_provider_function</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom megatron train_valid_test datasets provider function.`,name:"custom_megatron_datasets_provider_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_get_batch_function",description:`<strong>custom_get_batch_function</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom get batch function.`,name:"custom_get_batch_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_loss_function",description:`<strong>custom_loss_function</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom loss function.`,name:"custom_loss_function"},{anchor:"accelerate.utils.MegatronLMPlugin.other_megatron_args",description:`<strong>other_megatron_args</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Other Megatron-LM arguments. Please refer Megatron-LM.`,name:"other_megatron_args"}],source:"https://github.com/huggingface/accelerate/blob/vr_4064/src/accelerate/utils/dataclasses.py#L2316"}}),z=new w({props:{title:"MegatronLMDummyScheduler",local:"accelerate.utils.MegatronLMDummyScheduler",headingTag:"h2"}}),O=new T({props:{name:"class accelerate.utils.MegatronLMDummyScheduler",anchor:"accelerate.utils.MegatronLMDummyScheduler",parameters:[{name:"optimizer",val:""},{name:"total_num_steps",val:" = None"},{name:"warmup_num_steps",val:" = 0"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"accelerate.utils.MegatronLMDummyScheduler.optimizer",description:`<strong>optimizer</strong> (<code>torch.optim.optimizer.Optimizer</code>) &#x2014;
The optimizer to wrap.`,name:"optimizer"},{anchor:"accelerate.utils.MegatronLMDummyScheduler.total_num_steps",description:`<strong>total_num_steps</strong> (int) &#x2014;
Total number of steps.`,name:"total_num_steps"},{anchor:"accelerate.utils.MegatronLMDummyScheduler.warmup_num_steps",description:`<strong>warmup_num_steps</strong> (int) &#x2014;
Number of steps for warmup.`,name:"warmup_num_steps"},{anchor:"accelerate.utils.MegatronLMDummyScheduler.*kwargs",description:`*<strong>*kwargs</strong> (additional keyword arguments, <em>optional</em>) &#x2014;
Other arguments.`,name:"*kwargs"}],source:"https://github.com/huggingface/accelerate/blob/vr_4064/src/accelerate/utils/megatron_lm.py#L378"}}),q=new w({props:{title:"MegatronLMDummyDataLoader",local:"accelerate.utils.MegatronLMDummyDataLoader",headingTag:"h2"}}),E=new T({props:{name:"class accelerate.utils.MegatronLMDummyDataLoader",anchor:"accelerate.utils.MegatronLMDummyDataLoader",parameters:[{name:"**dataset_kwargs",val:""}],parametersDescription:[{anchor:"accelerate.utils.MegatronLMDummyDataLoader.*dataset_kwargs",description:"*<strong>*dataset_kwargs</strong> &#x2014; Megatron data arguments.",name:"*dataset_kwargs"}],source:"https://github.com/huggingface/accelerate/blob/vr_4064/src/accelerate/utils/megatron_lm.py#L162"}}),I=new w({props:{title:"AbstractTrainStep",local:"accelerate.utils.AbstractTrainStep",headingTag:"h2"}}),A=new T({props:{name:"class accelerate.utils.AbstractTrainStep",anchor:"accelerate.utils.AbstractTrainStep",parameters:[{name:"name",val:""}],source:"https://github.com/huggingface/accelerate/blob/vr_4064/src/accelerate/utils/megatron_lm.py#L415"}}),G=new w({props:{title:"GPTTrainStep",local:"accelerate.utils.GPTTrainStep",headingTag:"h2"}}),B=new T({props:{name:"class accelerate.utils.GPTTrainStep",anchor:"accelerate.utils.GPTTrainStep",parameters:[{name:"accelerator",val:""},{name:"args",val:""}],parametersDescription:[{anchor:"accelerate.utils.GPTTrainStep.args",description:"<strong>args</strong> (<code>argparse.Namespace</code>) &#x2014; Megatron-LM arguments.",name:"args"}],source:"https://github.com/huggingface/accelerate/blob/vr_4064/src/accelerate/utils/megatron_lm.py#L574"}}),V=new w({props:{title:"BertTrainStep",local:"accelerate.utils.BertTrainStep",headingTag:"h2"}}),F=new T({props:{name:"class accelerate.utils.BertTrainStep",anchor:"accelerate.utils.BertTrainStep",parameters:[{name:"accelerator",val:""},{name:"args",val:""}],parametersDescription:[{anchor:"accelerate.utils.BertTrainStep.args",description:"<strong>args</strong> (<code>argparse.Namespace</code>) &#x2014; Megatron-LM arguments.",name:"args"}],source:"https://github.com/huggingface/accelerate/blob/vr_4064/src/accelerate/utils/megatron_lm.py#L432"}}),W=new w({props:{title:"T5TrainStep",local:"accelerate.utils.T5TrainStep",headingTag:"h2"}}),j=new T({props:{name:"class accelerate.utils.T5TrainStep",anchor:"accelerate.utils.T5TrainStep",parameters:[{name:"accelerator",val:""},{name:"args",val:""}],parametersDescription:[{anchor:"accelerate.utils.T5TrainStep.args",description:"<strong>args</strong> (<code>argparse.Namespace</code>) &#x2014; Megatron-LM arguments.",name:"args"}],source:"https://github.com/huggingface/accelerate/blob/vr_4064/src/accelerate/utils/megatron_lm.py#L718"}}),H=new w({props:{title:"avg_losses_across_data_parallel_group",local:"accelerate.utils.avg_losses_across_data_parallel_group",headingTag:"h2"}}),R=new T({props:{name:"accelerate.utils.avg_losses_across_data_parallel_group",anchor:"accelerate.utils.avg_losses_across_data_parallel_group",parameters:[{name:"losses",val:""}],parametersDescription:[{anchor:"accelerate.utils.avg_losses_across_data_parallel_group.losses",description:"<strong>losses</strong> (List[Tensor]) &#x2014; List of losses to average across data parallel group.",name:"losses"}],source:"https://github.com/huggingface/accelerate/blob/vr_4064/src/accelerate/utils/megatron_lm.py#L1217"}}),U=new ot({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/package_reference/megatron_lm.md"}}),{c(){h=l("meta"),oe=o(),ae=l("p"),re=o(),s(D.$$.fragment),le=o(),s(C.$$.fragment),se=o(),s(S.$$.fragment),ie=o(),v=l("div"),s(k.$$.fragment),Oe=o(),J=l("p"),J.textContent=We,ce=o(),s(z.$$.fragment),de=o(),M=l("div"),s(O.$$.fragment),qe=o(),K=l("p"),K.textContent=je,me=o(),s(q.$$.fragment),ge=o(),b=l("div"),s(E.$$.fragment),Ee=o(),Q=l("p"),Q.textContent=He,pe=o(),s(I.$$.fragment),ue=o(),$=l("div"),s(A.$$.fragment),Ie=o(),X=l("p"),X.textContent=Re,_e=o(),s(G.$$.fragment),fe=o(),y=l("div"),s(B.$$.fragment),Ae=o(),Y=l("p"),Y.textContent=Ue,he=o(),s(V.$$.fragment),ve=o(),x=l("div"),s(F.$$.fragment),Ge=o(),Z=l("p"),Z.textContent=Je,Me=o(),s(W.$$.fragment),be=o(),N=l("div"),s(j.$$.fragment),Be=o(),ee=l("p"),ee.textContent=Ke,$e=o(),s(H.$$.fragment),ye=o(),L=l("div"),s(R.$$.fragment),Ve=o(),te=l("p"),te.textContent=Qe,xe=o(),s(U.$$.fragment),Ne=o(),ne=l("p"),this.h()},l(e){const t=at("svelte-u9bgzb",document.head);h=i(t,"META",{name:!0,content:!0}),t.forEach(a),oe=r(e),ae=i(e,"P",{}),_(ae).forEach(a),re=r(e),c(D.$$.fragment,e),le=r(e),c(C.$$.fragment,e),se=r(e),c(S.$$.fragment,e),ie=r(e),v=i(e,"DIV",{class:!0});var we=_(v);c(k.$$.fragment,we),Oe=r(we),J=i(we,"P",{"data-svelte-h":!0}),P(J)!=="svelte-51ubws"&&(J.textContent=We),we.forEach(a),ce=r(e),c(z.$$.fragment,e),de=r(e),M=i(e,"DIV",{class:!0});var Pe=_(M);c(O.$$.fragment,Pe),qe=r(Pe),K=i(Pe,"P",{"data-svelte-h":!0}),P(K)!=="svelte-mxz6g6"&&(K.textContent=je),Pe.forEach(a),me=r(e),c(q.$$.fragment,e),ge=r(e),b=i(e,"DIV",{class:!0});var Te=_(b);c(E.$$.fragment,Te),Ee=r(Te),Q=i(Te,"P",{"data-svelte-h":!0}),P(Q)!=="svelte-1r57fhg"&&(Q.textContent=He),Te.forEach(a),pe=r(e),c(I.$$.fragment,e),ue=r(e),$=i(e,"DIV",{class:!0});var De=_($);c(A.$$.fragment,De),Ie=r(De),X=i(De,"P",{"data-svelte-h":!0}),P(X)!=="svelte-7ith7r"&&(X.textContent=Re),De.forEach(a),_e=r(e),c(G.$$.fragment,e),fe=r(e),y=i(e,"DIV",{class:!0});var Ce=_(y);c(B.$$.fragment,Ce),Ae=r(Ce),Y=i(Ce,"P",{"data-svelte-h":!0}),P(Y)!=="svelte-8o1bfn"&&(Y.textContent=Ue),Ce.forEach(a),he=r(e),c(V.$$.fragment,e),ve=r(e),x=i(e,"DIV",{class:!0});var Se=_(x);c(F.$$.fragment,Se),Ge=r(Se),Z=i(Se,"P",{"data-svelte-h":!0}),P(Z)!=="svelte-1af56j7"&&(Z.textContent=Je),Se.forEach(a),Me=r(e),c(W.$$.fragment,e),be=r(e),N=i(e,"DIV",{class:!0});var ke=_(N);c(j.$$.fragment,ke),Be=r(ke),ee=i(ke,"P",{"data-svelte-h":!0}),P(ee)!=="svelte-1w9r8gv"&&(ee.textContent=Ke),ke.forEach(a),$e=r(e),c(H.$$.fragment,e),ye=r(e),L=i(e,"DIV",{class:!0});var ze=_(L);c(R.$$.fragment,ze),Ve=r(ze),te=i(ze,"P",{"data-svelte-h":!0}),P(te)!=="svelte-39hqbl"&&(te.textContent=Qe),ze.forEach(a),xe=r(e),c(U.$$.fragment,e),Ne=r(e),ne=i(e,"P",{}),_(ne).forEach(a),this.h()},h(){f(h,"name","hf:doc:metadata"),f(h,"content",lt),f(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){u(document.head,h),n(e,oe,t),n(e,ae,t),n(e,re,t),d(D,e,t),n(e,le,t),d(C,e,t),n(e,se,t),d(S,e,t),n(e,ie,t),n(e,v,t),d(k,v,null),u(v,Oe),u(v,J),n(e,ce,t),d(z,e,t),n(e,de,t),n(e,M,t),d(O,M,null),u(M,qe),u(M,K),n(e,me,t),d(q,e,t),n(e,ge,t),n(e,b,t),d(E,b,null),u(b,Ee),u(b,Q),n(e,pe,t),d(I,e,t),n(e,ue,t),n(e,$,t),d(A,$,null),u($,Ie),u($,X),n(e,_e,t),d(G,e,t),n(e,fe,t),n(e,y,t),d(B,y,null),u(y,Ae),u(y,Y),n(e,he,t),d(V,e,t),n(e,ve,t),n(e,x,t),d(F,x,null),u(x,Ge),u(x,Z),n(e,Me,t),d(W,e,t),n(e,be,t),n(e,N,t),d(j,N,null),u(N,Be),u(N,ee),n(e,$e,t),d(H,e,t),n(e,ye,t),n(e,L,t),d(R,L,null),u(L,Ve),u(L,te),n(e,xe,t),d(U,e,t),n(e,Ne,t),n(e,ne,t),Le=!0},p:Ye,i(e){Le||(m(D.$$.fragment,e),m(C.$$.fragment,e),m(S.$$.fragment,e),m(k.$$.fragment,e),m(z.$$.fragment,e),m(O.$$.fragment,e),m(q.$$.fragment,e),m(E.$$.fragment,e),m(I.$$.fragment,e),m(A.$$.fragment,e),m(G.$$.fragment,e),m(B.$$.fragment,e),m(V.$$.fragment,e),m(F.$$.fragment,e),m(W.$$.fragment,e),m(j.$$.fragment,e),m(H.$$.fragment,e),m(R.$$.fragment,e),m(U.$$.fragment,e),Le=!0)},o(e){g(D.$$.fragment,e),g(C.$$.fragment,e),g(S.$$.fragment,e),g(k.$$.fragment,e),g(z.$$.fragment,e),g(O.$$.fragment,e),g(q.$$.fragment,e),g(E.$$.fragment,e),g(I.$$.fragment,e),g(A.$$.fragment,e),g(G.$$.fragment,e),g(B.$$.fragment,e),g(V.$$.fragment,e),g(F.$$.fragment,e),g(W.$$.fragment,e),g(j.$$.fragment,e),g(H.$$.fragment,e),g(R.$$.fragment,e),g(U.$$.fragment,e),Le=!1},d(e){e&&(a(oe),a(ae),a(re),a(le),a(se),a(ie),a(v),a(ce),a(de),a(M),a(me),a(ge),a(b),a(pe),a(ue),a($),a(_e),a(fe),a(y),a(he),a(ve),a(x),a(Me),a(be),a(N),a($e),a(ye),a(L),a(xe),a(Ne),a(ne)),a(h),p(D,e),p(C,e),p(S,e),p(k),p(z,e),p(O),p(q,e),p(E),p(I,e),p(A),p(G,e),p(B),p(V,e),p(F),p(W,e),p(j),p(H,e),p(R),p(U,e)}}}const lt='{"title":"Megatron-LM utilities","local":"megatron-lm-utilities","sections":[{"title":"MegatronLMPlugin","local":"accelerate.utils.MegatronLMPlugin","sections":[],"depth":2},{"title":"MegatronLMDummyScheduler","local":"accelerate.utils.MegatronLMDummyScheduler","sections":[],"depth":2},{"title":"MegatronLMDummyDataLoader","local":"accelerate.utils.MegatronLMDummyDataLoader","sections":[],"depth":2},{"title":"AbstractTrainStep","local":"accelerate.utils.AbstractTrainStep","sections":[],"depth":2},{"title":"GPTTrainStep","local":"accelerate.utils.GPTTrainStep","sections":[],"depth":2},{"title":"BertTrainStep","local":"accelerate.utils.BertTrainStep","sections":[],"depth":2},{"title":"T5TrainStep","local":"accelerate.utils.T5TrainStep","sections":[],"depth":2},{"title":"avg_losses_across_data_parallel_group","local":"accelerate.utils.avg_losses_across_data_parallel_group","sections":[],"depth":2}],"depth":1}';function st(Fe){return Ze(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class gt extends et{constructor(h){super(),tt(this,h,st,rt,Xe,{})}}export{gt as component};