| { |
| "model_type": "deepseek_v3_moe", |
| "hf_model_type": "deepseek_v3", |
| "parameter_count": 2803272, |
| "tokenizer_size": 1003, |
| "final_eval_loss": 1.5254412926733494, |
| "final_eval_ppl": 4.59717181998998, |
| "final_train_lm_loss": 1.5200555527210236, |
| "router_usage": { |
| "model.layers.1.mlp.gate": [ |
| 0.1699, |
| 0.1683, |
| 0.1658, |
| 0.496 |
| ], |
| "model.layers.2.mlp.gate": [ |
| 0.2745, |
| 0.2254, |
| 0.2464, |
| 0.2538 |
| ], |
| "model.layers.3.mlp.gate": [ |
| 0.2655, |
| 0.2749, |
| 0.1966, |
| 0.263 |
| ], |
| "model.layers.4.mlp.gate": [ |
| 0.1976, |
| 0.14, |
| 0.3308, |
| 0.3316 |
| ] |
| }, |
| "args": { |
| "train_files": [ |
| "data/train-00000-of-00004-2d5a1467fff1081b.parquet", |
| "data/train-00001-of-00004-5852b56a2bd28fd9.parquet", |
| "data/train-00002-of-00004-a26307300439e943.parquet", |
| "data/train-00003-of-00004-d243063613e5a057.parquet" |
| ], |
| "output_dir": "runs/deepseek_v3_3m_fp32_100pct", |
| "max_rows": 0, |
| "data_fraction": 1.0, |
| "vocab_size": 1024, |
| "base_vocab_size": 1000, |
| "min_frequency": 2, |
| "tokenizer_from": "runs/deepseek_v3_3m_fp32_5pct/hf", |
| "legacy_tokenizer": true, |
| "legacy_special_token_ids": true, |
| "deepseek_arch": "v3", |
| "hidden_size": 216, |
| "intermediate_size": 432, |
| "num_hidden_layers": 5, |
| "num_attention_heads": 8, |
| "num_key_value_heads": 8, |
| "qk_nope_head_dim": 16, |
| "qk_rope_head_dim": 16, |
| "v_head_dim": 32, |
| "q_lora_rank": 0, |
| "kv_lora_rank": 64, |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "rms_norm_eps": 1e-06, |
| "initializer_range": 0.02, |
| "rope_theta": 10000, |
| "first_k_dense_replace": 1, |
| "n_routed_experts": 4, |
| "n_shared_experts": 1, |
| "num_experts_per_tok": 1, |
| "n_group": 0, |
| "topk_group": 0, |
| "topk_method": "greedy", |
| "norm_topk_prob": null, |
| "routed_scaling_factor": null, |
| "moe_intermediate_size": 128, |
| "router_aux_loss_alpha": null, |
| "v3_bias_update_speed": 0.01, |
| "official_router_init": true, |
| "max_position_embeddings": 2048, |
| "tie_word_embeddings": true, |
| "use_cache": false, |
| "mlp_bias": false, |
| "block_size": 256, |
| "batch_size": 16, |
| "max_steps": 152565, |
| "num_epochs": 1.0, |
| "learning_rate": 0.0002, |
| "weight_decay": 0.0, |
| "grad_clip": 1.0, |
| "warmup_steps": 2000, |
| "min_lr_ratio": 0.1, |
| "validation_fraction": 0.01, |
| "log_steps": 100, |
| "eval_steps": 5000, |
| "eval_batches": 32, |
| "save_steps": 10000, |
| "resume_from_checkpoint": "", |
| "seed": 1234, |
| "dtype": "float32", |
| "device": "cuda", |
| "num_workers": 0, |
| "generate_prompts": [ |
| "Once upon", |
| "There was a little", |
| "One day" |
| ], |
| "generate_new_tokens": 120, |
| "generation_temperature": 0.8, |
| "generation_top_p": 0.95, |
| "generation_top_k": 40, |
| "generation_repetition_penalty": 1.1, |
| "do_sample": true |
| }, |
| "config": { |
| "transformers_version": "5.14.1", |
| "architectures": [ |
| "DeepseekV3ForCausalLM" |
| ], |
| "output_hidden_states": false, |
| "return_dict": true, |
| "dtype": "float32", |
| "chunk_size_feed_forward": 0, |
| "is_encoder_decoder": false, |
| "id2label": { |
| "0": "LABEL_0", |
| "1": "LABEL_1" |
| }, |
| "label2id": { |
| "LABEL_0": 0, |
| "LABEL_1": 1 |
| }, |
| "problem_type": null, |
| "vocab_size": 1024, |
| "hidden_size": 216, |
| "intermediate_size": 432, |
| "moe_intermediate_size": 128, |
| "num_hidden_layers": 5, |
| "num_attention_heads": 8, |
| "num_key_value_heads": 8, |
| "n_shared_experts": 1, |
| "n_routed_experts": 4, |
| "routed_scaling_factor": 2.5, |
| "kv_lora_rank": 64, |
| "q_lora_rank": 64, |
| "qk_rope_head_dim": 16, |
| "v_head_dim": 32, |
| "qk_nope_head_dim": 16, |
| "n_group": 1, |
| "topk_group": 1, |
| "num_experts_per_tok": 1, |
| "first_k_dense_replace": 1, |
| "norm_topk_prob": true, |
| "hidden_act": "silu", |
| "max_position_embeddings": 2048, |
| "initializer_range": 0.02, |
| "rms_norm_eps": 1e-06, |
| "use_cache": false, |
| "pad_token_id": 1000, |
| "bos_token_id": 1000, |
| "eos_token_id": 1001, |
| "pretraining_tp": 1, |
| "tie_word_embeddings": true, |
| "rope_parameters": { |
| "rope_type": "default", |
| "rope_theta": 10000 |
| }, |
| "rope_interleave": true, |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "num_nextn_predict_layers": 0, |
| "qk_head_dim": 32, |
| "head_dim": 16, |
| "_name_or_path": "", |
| "aux_loss_alpha": 0.0001, |
| "seq_aux": true, |
| "model_type": "deepseek_v3", |
| "output_attentions": false |
| }, |
| "generations": { |
| "Once upon": "Once upon a time, there was a little girl named Lily. She had long black, white hair that she loved to play with every day. One day, Lily's mom asked her to clean up her bedroom. Lily didn't want to clean up, so she started to pick up her toys and put them away in the closet. \n\nWhile playing outside, Lily's mom came into the room and said, \"Lily, you need to clean your room before dinner.\" Lily smiled and said, \"Okay mommy, I will help you", |
| "There was a little": "There was a little girl who had an idea. She said, “Let’s go on an adventure!†\n\nSo, they all agreed to go. They went off on their journey, and soon they reached the big park. It was so exciting! There were lots of colorful trees, flowers and butterflies. Everywhere it went, people looked at the sky and smiled. They thought it was very interesting. \n\nWhen they got to the top of the", |
| "One day": "One day, he was walking in the park with his mom and he saw a small, wise old tree. He looked at the tree and asked why, \"What are you doing?\"\n\nThe tree replied, \"I'm looking for a place to study.\" His mom said, \"A place is very unique. I can help you find a place to stand and make a cozy place.\" \n\nSo, the little boy took out a big box of sparkly woods and put them" |
| } |
| } |
|
|
