data-ladder: dl_smollm2_135m_refinedweb

data-ladder-runs/dl_smollm2_135m_refinedweb/run_config.json

@@ -0,0 +1,119 @@
+{
+  "config_json": "configs/hf/smollm2_135m_v2.json",
+  "arm": "dl_smollm2_135m_refinedweb",
+  "data_dir": "/workspace/datasets/refinedweb_smollm2",
+  "neutral_eval_dir": "/workspace/datasets/wiki_eval_union",
+  "hf_tokenizer": "HuggingFaceTB/SmolLM2-135M",
+  "token_budget": 8870000000,
+  "n_epochs": 1,
+  "out_dir": "/workspace/runs/dl_smollm2_135m_refinedweb",
+  "global_batch_tokens": 524288,
+  "device_batch_size": 32,
+  "block_size": 1024,
+  "peak_lr": 0.003,
+  "schedule": "wsd",
+  "warmup_frac": 0.01,
+  "wsd_decay_frac": 0.2,
+  "min_lr_ratio": 0.0,
+  "weight_decay": 0.01,
+  "betas": [
+    0.9,
+    0.95
+  ],
+  "grad_clip": 1.0,
+  "n_checkpoints": 25,
+  "first_ckpt_frac": 0.001,
+  "warmup_timed_steps": 10,
+  "max_steps": 0,
+  "eval_device_batch_size": 8,
+  "seed": 1234,
+  "save_checkpoints": 1,
+  "keep_checkpoints": 3,
+  "arch": "llama",
+  "hf_config": {
+    "transformers_version": "5.15.1",
+    "architectures": null,
+    "output_hidden_states": false,
+    "return_dict": true,
+    "dtype": null,
+    "chunk_size_feed_forward": 0,
+    "is_encoder_decoder": false,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "problem_type": null,
+    "vocab_size": 49152,
+    "hidden_size": 576,
+    "intermediate_size": 1536,
+    "num_hidden_layers": 30,
+    "num_attention_heads": 9,
+    "num_key_value_heads": 3,
+    "hidden_act": "silu",
+    "max_position_embeddings": 8192,
+    "initializer_range": 0.041666666666666664,
+    "rms_norm_eps": 1e-05,
+    "use_cache": true,
+    "pad_token_id": null,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "pretraining_tp": 1,
+    "tie_word_embeddings": true,
+    "rope_parameters": {
+      "rope_theta": 100000,
+      "rope_type": "default"
+    },
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "mlp_bias": false,
+    "head_dim": 64,
+    "_name_or_path": "",
+    "model_type": "llama",
+    "output_attentions": false
+  },
+  "world_size": 8,
+  "total_steps": 16919,
+  "tokens_per_step": 524288,
+  "grad_accum": 2,
+  "n_params": 134515008,
+  "device": "cuda:0",
+  "gpu_name": "NVIDIA H100 80GB HBM3",
+  "torch_version": "2.10.0+cu128",
+  "algorithm": "hf_llama",
+  "ckpt_steps": [
+    17,
+    23,
+    30,
+    40,
+    54,
+    71,
+    95,
+    127,
+    169,
+    226,
+    301,
+    401,
+    535,
+    713,
+    951,
+    1269,
+    1692,
+    2256,
+    3009,
+    4012,
+    5350,
+    7135,
+    9514,
+    12687,
+    15227,
+    15565,
+    15904,
+    16242,
+    16581,
+    16919
+  ]
+}