artifact_metadata.json

{
  "model_type": "gpt_oss_moe_3m",
  "hf_model_type": "gpt_oss",
  "parameter_count": 2895432,
  "tokenizer_size": 1003,
  "final_eval_loss": 1.4605633437633514,
  "final_eval_ppl": 4.308385947180532,
  "args": {
    "output_dir": "tinygptossmoe3m",
    "train_files": [
      "data/train-00000-of-00004-2d5a1467fff1081b.parquet",
      "data/train-00001-of-00004-5852b56a2bd28fd9.parquet",
      "data/train-00002-of-00004-a26307300439e943.parquet",
      "data/train-00003-of-00004-d243063613e5a057.parquet"
    ],
    "max_rows": 0,
    "vocab_size": 1024,
    "base_vocab_size": 1000,
    "min_frequency": 2,
    "legacy_tokenizer": true,
    "legacy_special_token_ids": true,
    "hidden_size": 128,
    "intermediate_size": 128,
    "num_hidden_layers": 6,
    "num_attention_heads": 4,
    "num_key_value_heads": 1,
    "head_dim": 32,
    "sliding_window": 128,
    "num_local_experts": 8,
    "num_experts_per_tok": 4,
    "router_aux_loss_coef": 0.0,
    "output_router_logits": false,
    "max_position_embeddings": 4096,
    "initial_context_length": 1024,
    "rope_theta": 150000.0,
    "rope_factor": 4.0,
    "layer_pattern": "sFsFsF",
    "swiglu_limit": 7.0,
    "attention_bias": true,
    "tie_word_embeddings": false,
    "rms_norm_eps": 1e-05,
    "block_size": 256,
    "batch_size": 32,
    "max_steps": 77053,
    "num_epochs": 1.0,
    "learning_rate": 0.0002,
    "weight_decay": 0.0,
    "grad_clip": 1.0,
    "error_on_nonfinite_gradients": true,
    "log_steps": 50,
    "eval_steps": 500,
    "eval_batches": 20,
    "seed": 1234,
    "device": "auto",
    "dtype": "auto",
    "num_workers": 0,
    "generate_prompts": [
      "Once upon",
      "There was a little",
      "One day"
    ],
    "generate_new_tokens": 100
  },
  "config": {
    "transformers_version": "5.9.0",
    "architectures": [
      "GptOssForCausalLM"
    ],
    "output_hidden_states": false,
    "return_dict": true,
    "dtype": "float32",
    "chunk_size_feed_forward": 0,
    "is_encoder_decoder": false,
    "id2label": {
      "0": "LABEL_0",
      "1": "LABEL_1"
    },
    "label2id": {
      "LABEL_0": 0,
      "LABEL_1": 1
    },
    "problem_type": null,
    "num_hidden_layers": 6,
    "num_local_experts": 8,
    "vocab_size": 1024,
    "hidden_size": 128,
    "intermediate_size": 128,
    "head_dim": 32,
    "num_attention_heads": 4,
    "num_key_value_heads": 1,
    "sliding_window": 128,
    "tie_word_embeddings": false,
    "hidden_act": "silu",
    "initializer_range": 0.02,
    "max_position_embeddings": 4096,
    "rms_norm_eps": 1e-05,
    "rope_parameters": {
      "rope_type": "yarn",
      "factor": 4.0,
      "original_max_position_embeddings": 1024,
      "beta_fast": 32.0,
      "beta_slow": 1.0,
      "truncate": false,
      "rope_theta": 150000.0
    },
    "attention_dropout": 0.0,
    "num_experts_per_tok": 4,
    "router_aux_loss_coef": 0.0,
    "output_router_logits": false,
    "use_cache": true,
    "layer_types": [
      "sliding_attention",
      "full_attention",
      "sliding_attention",
      "full_attention",
      "sliding_attention",
      "full_attention"
    ],
    "pad_token_id": 1000,
    "bos_token_id": 1000,
    "eos_token_id": 1001,
    "attention_bias": true,
    "_name_or_path": "",
    "initial_context_length": 1024,
    "experts_per_token": 4,
    "swiglu_limit": 7.0,
    "rope_theta": 150000.0,
    "model_type": "gpt_oss",
    "output_attentions": false
  },
  "generations": {
    "Once upon": "Once upon a time, there was a little girl named Lily. She loved to play outside in the sun. One day, she saw a big, scary dog. The dog was barking and running around. Lily wanted to pet the dog, but she didn't want to leave.\n\nSuddenly, a big dog came running towards her. Lily was scared and didn't know what to do. But then she remembered that she had a friend",
    "There was a little": "There was a little girl named Lily who was very curious. She wanted to know what was inside the box. She asked her mom, \"What is in the box?\" Her mom said, \"It's a box of candy. It's a special treat for you.\"\n\nLily was excited to open the box. She opened the box and found a big, shiny candy inside. She was so happy and said, \"Thank you, Mommy!\" Her mom smiled and",
    "One day": "One day, a little girl named Lily went to the park with her mom. She saw a big slide and wanted to try it. She asked her mom if she could go on the slide. Her mom said yes, but only if she promised to be careful.\n\nLily was so happy that she ran up the slide and started to slide down. She slid down the slide and laughed as she slid down. She felt so happy and free."
  }
}