artifact_metadata.json

{
  "model_type": "deepseek_v2_moe",
  "hf_model_type": "deepseek_v2",
  "parameter_count": 2928392,
  "tokenizer_size": 1003,
  "final_eval_loss": 1.4225675538182259,
  "final_eval_ppl": 4.1477563679496425,
  "final_train_lm_loss": 1.409641880750656,
  "router_usage": {
    "model.layers.1.mlp.gate": [
      0.2502,
      0.2487,
      0.2522,
      0.2488
    ],
    "model.layers.2.mlp.gate": [
      0.2538,
      0.2481,
      0.2487,
      0.2495
    ],
    "model.layers.3.mlp.gate": [
      0.2512,
      0.2454,
      0.2519,
      0.2514
    ],
    "model.layers.4.mlp.gate": [
      0.2465,
      0.2535,
      0.2542,
      0.2458
    ]
  },
  "args": {
    "train_files": [
      "data/train-00000-of-00004-2d5a1467fff1081b.parquet",
      "data/train-00001-of-00004-5852b56a2bd28fd9.parquet",
      "data/train-00002-of-00004-a26307300439e943.parquet",
      "data/train-00003-of-00004-d243063613e5a057.parquet"
    ],
    "output_dir": "runs/deepseek_v2_3m_fix2_100p_1epoch",
    "max_rows": 0,
    "data_fraction": 1.0,
    "vocab_size": 1024,
    "base_vocab_size": 1000,
    "min_frequency": 2,
    "legacy_tokenizer": true,
    "legacy_special_token_ids": true,
    "deepseek_arch": "v2",
    "hidden_size": 216,
    "intermediate_size": 432,
    "num_hidden_layers": 5,
    "num_attention_heads": 8,
    "num_key_value_heads": 8,
    "qk_nope_head_dim": 16,
    "qk_rope_head_dim": 16,
    "v_head_dim": 32,
    "q_lora_rank": -1,
    "kv_lora_rank": 64,
    "attention_bias": false,
    "attention_dropout": 0.0,
    "rms_norm_eps": 1e-06,
    "initializer_range": 0.02,
    "rope_theta": 10000,
    "first_k_dense_replace": 1,
    "n_routed_experts": 4,
    "n_shared_experts": 1,
    "num_experts_per_tok": 1,
    "n_group": 0,
    "topk_group": 0,
    "topk_method": "greedy",
    "norm_topk_prob": false,
    "routed_scaling_factor": 4.0,
    "moe_intermediate_size": 128,
    "router_aux_loss_alpha": 0.01,
    "official_router_init": true,
    "max_position_embeddings": 2048,
    "tie_word_embeddings": true,
    "use_cache": false,
    "mlp_bias": false,
    "block_size": 256,
    "batch_size": 16,
    "max_steps": 152568,
    "num_epochs": 1.0,
    "learning_rate": 0.0003,
    "weight_decay": 0.0,
    "grad_clip": 1.0,
    "warmup_steps": 2000,
    "min_lr_ratio": 0.1,
    "validation_fraction": 0.01,
    "log_steps": 500,
    "eval_steps": 10000,
    "eval_batches": 32,
    "save_steps": 10000,
    "resume_from_checkpoint": "",
    "seed": 1234,
    "dtype": "float32",
    "device": "cuda",
    "num_workers": 0,
    "generate_prompts": [
      "Once upon",
      "There was a little",
      "One day"
    ],
    "generate_new_tokens": 100,
    "generation_temperature": 0.8,
    "generation_top_p": 0.95,
    "generation_top_k": 40,
    "generation_repetition_penalty": 1.1,
    "do_sample": true
  },
  "config": {
    "transformers_version": "5.14.1",
    "architectures": [
      "DeepseekV2ForCausalLM"
    ],
    "output_hidden_states": false,
    "return_dict": true,
    "dtype": "float32",
    "chunk_size_feed_forward": 0,
    "is_encoder_decoder": false,
    "id2label": {
      "0": "LABEL_0",
      "1": "LABEL_1"
    },
    "label2id": {
      "LABEL_0": 0,
      "LABEL_1": 1
    },
    "problem_type": null,
    "vocab_size": 1024,
    "hidden_size": 216,
    "intermediate_size": 432,
    "num_hidden_layers": 5,
    "num_attention_heads": 8,
    "num_key_value_heads": 8,
    "hidden_act": "silu",
    "max_position_embeddings": 2048,
    "initializer_range": 0.02,
    "rms_norm_eps": 1e-06,
    "use_cache": false,
    "pad_token_id": 1000,
    "bos_token_id": 1000,
    "eos_token_id": 1001,
    "pretraining_tp": 1,
    "tie_word_embeddings": true,
    "rope_parameters": {
      "rope_type": "default",
      "rope_theta": 10000
    },
    "attention_bias": false,
    "attention_dropout": 0.0,
    "mlp_bias": false,
    "head_dim": 16,
    "first_k_dense_replace": 1,
    "kv_lora_rank": 64,
    "q_lora_rank": null,
    "n_group": null,
    "n_routed_experts": 4,
    "n_shared_experts": 1,
    "qk_nope_head_dim": 16,
    "qk_rope_head_dim": 16,
    "routed_scaling_factor": 4.0,
    "topk_group": null,
    "topk_method": "greedy",
    "norm_topk_prob": false,
    "v_head_dim": 32,
    "num_experts_per_tok": 1,
    "moe_intermediate_size": 128,
    "_name_or_path": "",
    "aux_loss_alpha": 0.01,
    "seq_aux": true,
    "model_type": "deepseek_v2",
    "output_attentions": false
  },
  "generations": {
    "Once upon": "Once upon a time, there was a little boy named Timmy. He loved to play in the sand and splash in the water. One day, he was playing with his toy car when it slipped on some rocks. His mom came and said, \"Oh no, I'm sorry! Let's bury something that is not ours so we can find out.\"\n\nTimmy didn't understand what \"oray\" meant, but",
    "There was a little": "There was a little girl named Lily who loved to explore. One day, she saw a big bush with lots of yummy peaches on it. She wanted to eat one, but her mom said no.\n\nLily didn't listen and kept trying to eat her peach. She tried and tried but the bush wouldn't eat. Finally, she ate all the peaches and started to feel sick. Her mom told her not to eat",
    "One day": "One day, the boy saw a palace in the park. He thought it was the most special palace he had ever seen! He wanted to play with it, so he called out to his friend, Alice.\n\nAlice asked the palace: \"Can I go in?\"\n\nHis friend replied: \"Yes, but you must be very kind and gentle. What do you think?\"\n\n"
  }
}