b8c6e6efd9ab7a32a40ea8aa2a9aca807b546bf2873e10d07437ba9d8a8852d0

Pretrain_language_model/1B/1BL2/smoe_sigmoid/post_validate/args.json

@@ -0,0 +1,172 @@
+{
+    "profile": null,
+    "name": "post_validate",
+    "save_dir": "/cm/archive/namnv78_new/revise_checkpoints/pretrain_final/1BL2/smoe_sigmoid",
+    "reset": 1,
+    "log": "tb",
+    "save_interval": "20000",
+    "wandb_save_interval": "None",
+    "seed": "none",
+    "gpu": "auto",
+    "keep_alive": 0,
+    "sweep_id_for_grid_search": 0,
+    "restore": "/cm/archive/namnv78_new/revise_checkpoints/pretrain_final/1BL2/smoe_sigmoid/slimpajama_moe_no_attmoe_1B/checkpoint/model-200000.pth",
+    "wandb_bug_workaround": 0,
+    "wandb_sync_checkpoints": 0,
+    "batch_size": 10,
+    "lr": 0.00025,
+    "min_lr_multiplier": 0.1,
+    "wd": 0.01,
+    "lr_warmup": 4000,
+    "test_interval": 1000,
+    "n_microbatch": "None",
+    "per_device_batch_size": "16",
+    "lr_sched.steps": "",
+    "lr_sched.gamma": 0.1,
+    "lr_sched.type": "cos",
+    "length_bucketed_sampling": 0,
+    "grad_clip": "0.25",
+    "test_batch_size": "None",
+    "val_log_details": 0,
+    "reg_scales": "",
+    "reg_lin_decay": "",
+    "reg": 1.0,
+    "optimizer": "adamw",
+    "adam.betas": "0.9,0.999",
+    "adam.eps": 1e-08,
+    "stop_after": "200000",
+    "amp": 1,
+    "bfloat16": 1,
+    "nan_detect": 0,
+    "max_length_per_batch": "none",
+    "log_grad_norms": 0,
+    "speedtest": "none",
+    "dump_logs": 0,
+    "debug_plot_interval": "none",
+    "lm.trafo.context_blocks": 0,
+    "lm.trafo.test_context_blocks": "none",
+    "lm.trafo.same_length_eval": 0,
+    "lm.trafo.same_length": 0,
+    "lm.trafo.last_layer_context": 0,
+    "lm.trafo.xl_init": 0,
+    "lm.trafo.norm_input": 0,
+    "rope.rotate_fraction": 0.5,
+    "rope.base": 10000.0,
+    "pkm.n_heads": 8,
+    "moe.n_experts": 24,
+    "moe.expert_size": 512,
+    "moe_name": "smoe_sigmoid",
+    "moe.selection_mode": "gate",
+    "moe.perplexity_reg": 0.01,
+    "moe.perplexity_reg_mode": "step",
+    "moe.att.perplexity_reg_mode": "none",
+    "moe.activation_after_topk": 0,
+    "moe.att.expert_size": 256,
+    "moe.topk": 2,
+    "moe.bias": 0,
+    "moe.sel_bias": 0,
+    "moe.dropout_factor": 1.0,
+    "moe.drop_expert": 0.0,
+    "moe.sync_distributed": 1,
+    "moe.init_scale": 1.0,
+    "moe.att.n_experts": 4,
+    "moe.att.enable": 0,
+    "moe.att.q_expert": 1,
+    "moe.att.k_expert": 1,
+    "moe.att.v_expert": 1,
+    "moe.att.o_expert": 1,
+    "moe.att.k": 2,
+    "moe.att.v_size": "none",
+    "moe.att.same_sel": 0,
+    "moe.att.expert_dropout": "none",
+    "moe.att.selection_mode": "sigmoid",
+    "moe.att.perplexity_reg": "none",
+    "moe.att.drop_expert": "none",
+    "moe.att.separate_kq_sel": 0,
+    "moe.att.norm_init": 0,
+    "moe.att.dropout": 0.0,
+    "moe.att.selection_dropout": 0.0,
+    "moe.nonorm": 0,
+    "in_topk": 0,
+    "balance_affinity": 1,
+    "is_cosine": 0,
+    "is_norm_weight": 0,
+    "norm_softmax": 0,
+    "norm_sigmoid": 0,
+    "moa.cvloss": 0.0,
+    "moa.switchloss": 0.0,
+    "moa.zloss": 0.0,
+    "balance_loss_coef": 0.01,
+    "balance_loss_coef_comp": 0.001,
+    "router_z_loss_coef": 0.001,
+    "router_loss_coef": 0.01,
+    "max_compete_in_iter": 3,
+    "warm_up": 0.05,
+    "rate_flip": 0.07,
+    "router_theta": 0.1,
+    "scale_weight": 1.0,
+    "hybrid": 0,
+    "tribrid": 0,
+    "moa.miloss": 0.0,
+    "sut.sample_topk": 0,
+    "sut.max_relative_positions": 64,
+    "sut.drop_gate": 0.0,
+    "moe.selection_dropout": 0.0,
+    "moe.layer_std_constant": 2.0,
+    "transformer.universal.group_size": 24,
+    "transformer.universal.group_type": "abab",
+    "transformer.embedding_scale": "none",
+    "transformer.topk_value": 32,
+    "transformer.activation": "relu",
+    "transformer.p_drop_layer": 0.0,
+    "transformer.head_projection_size": "128",
+    "transformer.act_loss": 0.0,
+    "transformer.plot_head_details": 0,
+    "lm.trafo.force_out_norm": 0,
+    "plot.n_steps": -128,
+    "dump_validation_plots": "",
+    "details_log_interval": "500",
+    "lm.state_drop_probability": 0.0,
+    "lm.unroll": 1024,
+    "lm.unroll_eval": "none",
+    "lm.example_context": 100,
+    "lm.example_window": 40,
+    "lm.eval.blimp.batch_mul": 16,
+    "lm.eval.enabled": 1,
+    "lm.eval.lambada.enabled": 1,
+    "lm.eval.cbt.batch_mul": 1,
+    "lm.eval.cbt.length_limit": "none",
+    "lm.eval.cbt.enabled": 1,
+    "lm.eval.cbt.end_only": 0,
+    "lm.eval.blimp.enabled": 1,
+    "lm.eval.hellaswag.enabled": 1,
+    "lm.eval.hellaswag.batch_mul": 16,
+    "lm.eval.piqa.enabled": 1,
+    "lm.eval.piqa.batch_mul": 16,
+    "lm.eval.ai2arc.enabled": 1,
+    "lm.eval.ai2arc.batch_mul": 4,
+    "lm.eval.mmlu.enabled": 0,
+    "lm.eval.openbookqa.enabled": 0,
+    "lm.eval.race.enabled": 0,
+    "lm.eval.siqa.enabled": 0,
+    "lm.eval.winogrande.enabled": 0,
+    "lm.eval.commonsenseqa.enabled": 0,
+    "sentencepiece.n_pieces": 8000,
+    "lmds.valid_ratio": 0.005,
+    "thestack.languages": "python,html,c++,rust,javascript,haskell,scala,assembly",
+    "state_size": 1024,
+    "task": "slimpajama_transformer",
+    "dropout": 0.0,
+    "embedding_size": "none",
+    "transformer.n_heads": 32,
+    "transformer.variant": "preln_moe",
+    "transformer.ff_multiplier": 2.0,
+    "transformer.encoder_n_layers": 24,
+    "transformer.attention_dropout": 0.0,
+    "load_pretrained_model": null,
+    "test_pretrained": 1,
+    "train_baseline": 0,
+    "test_only": 1,
+    "save_name_logs": "results",
+    "fs_cache_pattern": "*"
+}

Pretrain_language_model/1B/1BL2/smoe_sigmoid/post_validate/startup_log.txt

@@ -0,0 +1,2 @@
+2025-08-23 09:21:21.099864 SPP00018465: main.py --name post_validate --restore /cm/archive/namnv78_new/revise_checkpoints/pretrain_final/1BL2/smoe_sigmoid/slimpajama_moe_no_attmoe_1B/checkpoint/model-200000.pth --test_only 1 -reset 1 -lm.eval.enabled 1 -lm.eval.lambada.enabled 1 -lm.eval.cbt.enabled 1 -lm.eval.hellaswag.enabled 1 -lm.eval.piqa.enabled 1 -lm.eval.blimp.enabled 1 -lm.eval.ai2arc.enabled 1 --keep_alive 0 --batch_size 20
+2025-08-23 11:04:41.934938 SPP00018465: main.py --name post_validate --restore /cm/archive/namnv78_new/revise_checkpoints/pretrain_final/1BL2/smoe_sigmoid/slimpajama_moe_no_attmoe_1B/checkpoint/model-200000.pth --test_only 1 -reset 1 -lm.eval.enabled 1 -lm.eval.lambada.enabled 1 -lm.eval.cbt.enabled 1 -lm.eval.hellaswag.enabled 1 -lm.eval.piqa.enabled 1 -lm.eval.blimp.enabled 1 -lm.eval.ai2arc.enabled 1 --keep_alive 0 --batch_size 10

Pretrain_language_model/1B/1BL2/smoe_sigmoid/post_validate/tensorboard/events.out.tfevents.1755921881.SPP00018465.925002.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3869cd86db2cb15ffecf4dbb6099647c38e979c83a5a95d151b68712e571dc74
+size 282