Training in progress, epoch 1

config.json

@@ -5,6 +5,7 @@
   "architectures": [
     "LLaDAModelLM"
   ],
+  "attention_bias": false,
   "attention_dropout": 0.0,
   "attention_layer_norm": false,
   "attention_layer_norm_with_affine": true,
@@ -16,21 +17,27 @@
   "bias_for_layer_norm": false,
   "block_group_size": 1,
   "block_type": "llama",
+  "bos_token_id": 1,
   "d_model": 4096,
   "embedding_dropout": 0.0,
   "embedding_size": 126464,
   "eos_token_id": 126081,
   "flash_attention": false,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
   "include_bias": false,
   "include_qkv_bias": false,
   "init_cutoff_factor": null,
   "init_device": "meta",
   "init_fn": "mitchell",
   "init_std": 0.02,
+  "initializer_range": 0.02,
   "input_emb_norm": false,
+  "intermediate_size": 11008,
   "layer_norm_type": "rms",
   "layer_norm_with_affine": true,
   "mask_token_id": 126336,
+  "max_position_embeddings": 2048,
   "max_sequence_length": 4096,
   "mlp_hidden_size": 12288,
   "mlp_ratio": 4,
@@ -39,16 +46,21 @@
   "n_heads": 32,
   "n_kv_heads": 32,
   "n_layers": 32,
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
   "pad_token_id": 126081,
   "precision": "amp_bf16",
+  "pretraining_tp": 1,
   "residual_dropout": 0.0,
   "rms_norm_eps": 1e-05,
   "rope": true,
   "rope_full_precision": true,
+  "rope_scaling": null,
   "rope_theta": 500000.0,
   "scale_logits": false,
+  "tie_word_embeddings": false,
   "torch_dtype": "float16",
-  "train_max_sequence_length": 1024,
   "transformers_version": "4.52.3",
   "use_cache": false,
   "vocab_size": 126464,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5587f3f4ddbbaae1869a6389609b91a93f206cebb44db643d6043469496f5d97
+oid sha256:b8939327b78d6c988f6265ad644f6ff3b5353dabf95d8b23342868e91c99d877
 size 9784