Buckets:
| {"model": {"vocab_size": 50304, "d_large": 2048, "d_parallel": 1024, "n_large_heads": 8, "n_parallel_heads": 4, "n_parallel_layers": 14, "n_paths": 4, "path_top_k": 2, "n_projection_experts": 4, "projection_top_k": 2, "aux_loss_weight": 0.01, "dtype": "bfloat16", "init_method": "pythia", "init_std": 0.02, "init_seed": 0, "_CLASS_": "olmo_core.nn.transformer.config.MoEPPythiaConfig"}, "dataset": {"tokenizer": {"vocab_size": 50280, "eos_token_id": 0, "pad_token_id": 1, "identifier": "EleutherAI/pythia-410m", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "PythiaPileStandard", "mix_base_dir": "/pfs/lustrep2/scratch/project_462001445/pythia_data/pythia_deduped_olmo", "expand_glob": false, "include_instance_metadata": true, "work_dir": "/pfs/lustrep2/scratch/project_462001445/checkpoints/MoEP_Pythia_1B", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "max_target_sequence_length": 2048, "generate_doc_lengths": false, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 2097152, "seed": 1234, "num_workers": 2, "ignore_fingerprint_mismatch": false, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig"}, "train_module": {"rank_microbatch_size": 8192, "max_sequence_length": 2048, "optim": {"group_overrides": [{"params": ["embeddings.weight"], "opts": {"weight_decay": 0.0}, "_CLASS_": "olmo_core.optim.config.OptimGroupOverride"}], "compile": false, "fixed_fields": ["initial_lr"], "lr": 0.00025, "betas": [0.9, 0.95], "eps": 1e-08, "weight_decay": 0.1, "foreach": true, "step_increment_bugfix": true, "rolling_interval_length": 128, "sigma_factor": 6, "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig"}, "max_grad_norm": 1.0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup": 1430, "alpha_f": 0.1, "warmup_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.CosWithWarmup"}, "compile_model": true, "dp_config": {"name": "hsdp", "param_dtype": "bfloat16", "reduce_dtype": "float32", "wrapping_strategy": "blocks", "prefetch_factor": 0, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerDataParallelConfig"}, "z_loss_multiplier": 0.0, "label_ignore_index": -100, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "/pfs/lustrep2/scratch/project_462001445/checkpoints/MoEP_Pythia_1B", "load_strategy": "if_available", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": false, "max_duration": {"value": 143000, "unit": "steps", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 50, "hard_stop": {"value": 143000, "unit": "steps", "_CLASS_": "olmo_core.train.common.Duration"}, "metrics_collect_interval": 50, "callbacks": {"checkpointer": {"save_interval": 2500, "save_async": false, "remove": "ephemeral_only", "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "wandb": {"enabled": true, "name": "MoEP_Pythia_1B_300b", "project": "MoEP_Pythia", "entity": "tapaninaho-joonas-university-of-oulu", "tags": ["MoEP", "Pythia", "1B", "official"], "cancel_tags": [], "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}, "monkey_patcher": {"_CLASS_": "olmo_core.train.callbacks.monkey_patcher.MonkeyPatcherCallback"}, "gpu_memory": {"_CLASS_": "olmo_core.train.callbacks.gpu_memory_monitor.GPUMemoryMonitorCallback"}, "config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}}, "bookkeeping_soft_timeout": 30, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 12536, "_CLASS_": "olmo_core.script_utils.ExperimentConfig"} |
Xet Storage Details
- Size:
- 3.83 kB
- Xet hash:
- 1bebb285ba3ea65da61f940caac2b4e6d158fe8b455bf3a5ab0ba8ff7b33aab6
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.