Buckets:
| {"model": {"d_model": 384, "vocab_size": 16384, "n_layers": 18, "block": {"sequence_mixer": {"name": "default", "n_heads": 8, "n_kv_heads": 2, "head_dim": 32, "bias": false, "gate": {"granularity": "headwise", "full_precision": true, "_CLASS_": "olmo_core.nn.attention.GateConfig"}, "rope": {"name": "default", "theta": 10000, "full_precision": true, "no_global_rope": false, "hope": {"training_sequence_length": 2048, "_CLASS_": "olmo_core.nn.rope.HoPEConfig"}, "partial_rotary_factor": 1.0, "_CLASS_": "olmo_core.nn.rope.RoPEConfig"}, "qk_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "backend": "flex", "dtype": "float32", "sliding_window": {"pattern": [512, 512, -1], "force_full_attention_on_first_layer": false, "force_full_attention_on_last_layer": true, "_CLASS_": "olmo_core.nn.attention.SlidingWindowAttentionConfig"}, "use_head_qk_norm": false, "use_attention_sink": true, "_CLASS_": "olmo_core.nn.attention.AttentionConfig", "type": "attention"}, "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "feed_forward": {"hidden_size": 2560, "name": "default", "bias": false, "dtype": "float32", "activation": "silu", "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig"}, "name": "default", "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig"}, "lm_head": {"name": "default", "layer_norm": {"name": "rms", "eps": 1e-06, "bias": false, "dtype": "float32", "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig"}, "bias": false, "dtype": "float32", "loss_implementation": "default", "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig"}, "name": "default", "dtype": "float32", "residual_dtype": "bfloat16", "init_method": "normal", "init_seed": 0, "init_std": 0.02, "tie_word_embeddings": true, "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig"}, "dataset": {"tokenizer": {"vocab_size": 16384, "eos_token_id": 16001, "pad_token_id": 16002, "identifier": "glossia/Rambley-Tokenizer-v1-16k", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "rambley_16k/pretrain_v11", "mix_base_dir": "./work/rambley-v2-350m/data", "mix_override_roots": true, "expand_glob": false, "include_instance_metadata": true, "instance_filter_config": {"repetition_max_period": 13, "repetition_min_period": 1, "repetition_max_count": 32, "_CLASS_": "olmo_core.data.numpy_dataset.InstanceFilterConfig"}, "work_dir": "./work/rambley-v2-350m", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "max_target_sequence_length": 2048, "generate_doc_lengths": false, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyFSLDatasetConfig"}, "data_loader": {"global_batch_size": 524288, "seed": 34521, "num_workers": 8, "prefetch_factor": 2048, "ignore_fingerprint_mismatch": false, "_CLASS_": "olmo_core.data.data_loader.NumpyDataLoaderConfig", "type": "numpy"}, "train_module": {"rank_microbatch_size": 131072, "max_sequence_length": 2048, "optim": {"compile": false, "fixed_fields": ["initial_lr"], "lr": 0.02, "aux_lr": 0.002, "mu": 0.95, "betas": [0.9, 0.95], "weight_decay": 0.0, "embedding_weight_decay": 0.0, "weight_decay_mode": "hyperball", "adam_weight_decay_mode": "none", "ns_steps": 5, "ns_method": "gram", "epsilon": 1e-08, "nesterov": false, "adjust_lr": "rms_norm", "_CLASS_": "olmo_core.optim.muon.MuonConfig", "type": "muon"}, "max_grad_norm": 1.0, "sfplus_diagnostics_interval": 0, "sfplus_diagnostics_prediction_tokens": 0, "scheduler": {"lr_field": "lr", "initial_lr_field": "initial_lr", "units": "steps", "warmup_fraction": 0.05, "decay_fraction": 0.2, "warmup_min_lr": 0.0, "decay_min_lr": 0.0, "_CLASS_": "olmo_core.optim.scheduler.WSD", "type": "wsd"}, "patch": {"size": 6, "patch_steps": 3250, "token_steps": 4878, "batch_size": 524288, "_CLASS_": "olmo_core.train.train_module.transformer.patch.PatchTrainingConfig"}, "compile_model": true, "compile_mode": "default", "float8_config": {"enabled": false, "_CLASS_": "olmo_core.float8.Float8Config"}, "z_loss_multiplier": 1e-05, "autocast_precision": "bfloat16", "label_ignore_index": -100, "_CLASS_": "olmo_core.train.train_module.transformer.config.TransformerTrainModuleConfig"}, "trainer": {"save_folder": "hf://buckets/glossia/checkpoints/50m-patch6", "work_dir": "./work/rambley-v2-350m", "load_strategy": "never", "checkpointer": {"pre_download": false, "throttle_uploads": false, "_CLASS_": "olmo_core.train.checkpoint.CheckpointerConfig"}, "save_overwrite": true, "max_duration": {"value": 8128, "unit": "steps", "_CLASS_": "olmo_core.train.common.Duration"}, "cancel_check_interval": 10, "metrics_collect_interval": 10, "callbacks": {"config_saver": {"fname": "config.json", "_CLASS_": "olmo_core.train.callbacks.config_saver.ConfigSaverCallback"}, "profiler": {"skip_first": 30, "wait": 1, "warmup": 2, "active": 3, "repeat": 1, "with_stack": false, "profile_memory": false, "enable_cuda_sync_events": false, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.profiler.ProfilerCallback"}, "checkpointer": {"save_interval": 2500, "pre_train_checkpoint": false, "save_async": true, "remove": "ephemeral_only", "max_checkpoints": 3, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.checkpointer.CheckpointerCallback"}, "comet": {"enabled": false, "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "notifications": "none", "failure_tag": "failed", "auto_resume": false, "_CLASS_": "olmo_core.train.callbacks.comet.CometCallback"}, "wandb": {"enabled": true, "name": "50m-patch6", "project": "rambley-v2-350m", "cancel_tags": ["cancel", "canceled", "cancelled"], "cancel_check_interval": 10, "_CLASS_": "olmo_core.train.callbacks.wandb.WandBCallback"}, "lm_evaluator": {"eval_dataset": {"tokenizer": {"vocab_size": 16384, "eos_token_id": 16001, "pad_token_id": 16002, "identifier": "glossia/Rambley-Tokenizer-v1-16k", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "mix": "wildai-ppl-eval", "mix_base_dir": "./work/rambley-v2-350m/eval-data", "mix_override_roots": false, "expand_glob": false, "include_instance_metadata": true, "work_dir": "./work/rambley-v2-350m", "ignore_fingerprint_mismatch": false, "sequence_length": 2048, "max_instances": 10000, "_CLASS_": "olmo_core.data.numpy_dataset.NumpyPaddedFSLDatasetConfig"}, "eval_interval": 500, "eval_on_startup": false, "eval_on_finish": false, "cancel_after_first_eval": false, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "log_interval": 5, "deterministic": true, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.LMEvaluatorCallbackConfig"}, "downstream_evaluator": {"tasks": ["arc_challenge_test_bpb_5shot", "arc_challenge_test_mc_5shot_fast", "arc_easy_test_bpb_5shot", "arc_easy_test_mc_5shot_fast", "basic_skills_arithmetic_rc_5shot", "basic_skills_coding_rc_5shot", "basic_skills_common_knowledge_rc_5shot", "basic_skills_logical_reasoning_rc_5shot", "basic_skills_pattern_rc_5shot", "basic_skills_string_operations_rc_5shot", "codex_humaneval_gold_bpb_3shot", "codex_mbpp_gold_bpb_3shot", "copycolors_10way_fast", "hellaswag_bpb_5shot", "minerva_math_500_gold_bpb_0shot", "mmlu_humanities_test_bpb_5shot", "mmlu_humanities_test_mc_5shot_fast", "mmlu_other_test_bpb_5shot", "mmlu_other_test_mc_5shot_fast", "mmlu_social_sciences_test_bpb_5shot", "mmlu_social_sciences_test_mc_5shot_fast", "mmlu_stem_test_bpb_5shot", "mmlu_stem_test_mc_5shot_fast", "mt_mbpp_cpp_gold_bpb_3shot", "mt_mbpp_java_gold_bpb_3shot", "mt_mbpp_rust_gold_bpb_3shot"], "tokenizer": {"vocab_size": 16384, "eos_token_id": 16001, "pad_token_id": 16002, "identifier": "glossia/Rambley-Tokenizer-v1-16k", "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig"}, "eval_interval": 1000, "eval_duration": {"value": 1, "unit": "epochs", "_CLASS_": "olmo_core.train.common.Duration"}, "eval_on_startup": false, "eval_on_finish": false, "cancel_after_first_eval": false, "log_interval": 5, "lazy": false, "enabled": true, "_CLASS_": "olmo_core.train.callbacks.evaluator_callback.DownstreamEvaluatorCallbackConfig"}, "runtime_report": {"expected_parameter_dtype": "torch.float32", "_CLASS_": "src.scripts.fizz.fizz_training.reporting.RuntimeReportCallback"}, "patch_training": {"transitioned": false, "_CLASS_": "olmo_core.train.callbacks.patch_training.PatchTrainingCallback"}}, "bookkeeping_soft_timeout": 30, "no_checkpoints": false, "no_evals": false, "_CLASS_": "olmo_core.train.config.TrainerConfig"}, "init_seed": 12536, "recipe": {"model_shape": "50m-v2", "attention": "default_swa", "positional_encoding": "hope", "precision": "bf16-mixed", "parallelism": "auto", "load_mode": "continue", "context": {"scaling": "none", "beta_fast": 32, "beta_slow": 1, "full_attn_layers_only": true, "_CLASS_": "src.scripts.fizz.fizz_training.recipe.ContextRecipe"}, "data": {"mix": "rambley_16k_pretrain_v1.1", "root": "https://olmo-data.org", "mode": "chunk", "_CLASS_": "src.scripts.fizz.fizz_training.recipe.DataRecipe"}, "download": {"train": true, "eval": true, "workers": 16, "batch_size": 256, "_CLASS_": "src.scripts.fizz.fizz_training.recipe.DownloadRecipe"}, "eval": {"ppl": true, "downstream": true, "_CLASS_": "src.scripts.fizz.fizz_training.recipe.EvalRecipe"}, "engram": {"_CLASS_": "src.scripts.fizz.fizz_training.recipe.EngramRecipe"}, "patch": {"size": 6, "frac": 0.8, "_CLASS_": "src.scripts.fizz.fizz_training.recipe.PatchRecipe"}, "_CLASS_": "src.scripts.fizz.fizz_training.recipe.Recipe"}, "_CLASS_": "src.scripts.fizz.fizz_training.config.FizzConfig"} |
Xet Storage Details
- Size:
- 9.57 kB
- Xet hash:
- e12bf3d3118f56228e6328a4652f9716a93c7f6814ff338b5662fe9dd62693e5
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.