{
  "llm_compressor_commit": "2d5242056a8a00028686a303771ffc8e2fe27d07",
  "compressed_tensors_commit": "e69c8dc58aa152e5f5e36e85800ec1d2e7de5271",
  "transformers": "5.17.0",
  "torch": "2.14.0+cu130",
  "vllm": "0.30.0",
  "base_architecture": "zai-org/GLM-4.5-Air",
  "base_revision": "a24ceef6ce4f3536971efe9b778bdaa1bab18daa",
  "fixture": "glm4-moe-native-depth",
  "variant": "source",
  "parameter_counts": {
    "architecture": "Glm4MoeForCausalLM",
    "backbone_parameters": 707149568,
    "mtp_parameters": 11513600,
    "total_parameters": 718663168,
    "native_depth": true,
    "model_patches": false,
    "changes": {
      "hidden_size": {
        "original": 4096,
        "tiny": 768
      },
      "intermediate_size": {
        "original": 10944,
        "tiny": 3072
      },
      "num_attention_heads": {
        "original": 96,
        "tiny": 8
      },
      "num_key_value_heads": {
        "original": 8,
        "tiny": 4
      },
      "moe_intermediate_size": {
        "original": 1408,
        "tiny": 384
      },
      "num_experts_per_tok": {
        "original": 8,
        "tiny": 4
      },
      "n_routed_experts": {
        "original": 128,
        "tiny": 8
      }
    }
  },
  "backbone_validation": {
    "name": "glm4-moe-native-depth",
    "passed": true,
    "reloaded_perplexity": 1.1919350007360576,
    "model_patches": false,
    "backbone_parameters": 707149568,
    "backbone_activated_parameters_estimate": 547897088,
    "generation": "The capital of France is Paris. Paris is the largest city in France and serves as the country's political"
  },
  "mtp_training": "synthetic initialized projections with trained final decoder block copied; not separately trained",
  "training": {
    "seed": 3225,
    "initialization": "random",
    "optimizer": "AdamW",
    "learning_rate": 0.0004,
    "weight_decay": 0.01,
    "batch_size": 2,
    "max_sequence_length": 160,
    "evaluation": "same toy corpus as training"
  },
  "quantization-results": {
    "name": "glm4-moe-native-depth",
    "world_size": 2,
    "mtp_fp8_layers": 31,
    "source": "fixtures/glm4-moe-native-depth/source",
    "quantized": "fixtures/glm4-moe-native-depth/fp8"
  },
  "vllm-source-results": {
    "name": "glm4-moe-native-depth",
    "mode": "source",
    "llm_args": {
      "model": "fixtures/glm4-moe-native-depth/source",
      "dtype": "bfloat16",
      "tensor_parallel_size": 1,
      "gpu_memory_utilization": 0.25,
      "max_model_len": 256,
      "max_num_seqs": 4,
      "enforce_eager": true,
      "enable_prefix_caching": false,
      "seed": 3225,
      "disable_log_stats": false
    },
    "generations": [
      {
        "prompt": "The capital of France is",
        "text": " Paris. Paris is the largest city in France and serves as the country's political",
        "token_ids": [
          12089,
          13,
          12089,
          374,
          279,
          7772,
          3283,
          304,
          9621,
          323,
          17026,
          438,
          279,
          3146,
          594,
          4948
        ],
        "cumulative_logprob": -0.30267810029909015,
        "toy_keyword_match": true
      },
      {
        "prompt": "The creator of the theory of relativity was Albert",
        "text": " Einstein. Einstein developed the special theory of relativity in 1905 and the",
        "token_ids": [
          53690,
          13,
          53690,
          7881,
          279,
          3281,
          10121,
          315,
          1351,
          43166,
          304,
          220,
          114146,
          20,
          323,
          279
        ],
        "cumulative_logprob": -0.2019934607669711,
        "toy_keyword_match": true
      },
      {
        "prompt": "The classic game is called rock, paper,",
        "text": " scissors. Rock beats scissors, scissors beats paper, and paper beats rock. rock",
        "token_ids": [
          71079,
          13,
          9124,
          33167,
          71079,
          11,
          71079,
          33167,
          5567,
          11,
          323,
          5567,
          33167,
          6946,
          13,
          6946
        ],
        "cumulative_logprob": -1.577314083231613,
        "toy_keyword_match": true
      },
      {
        "prompt": "Actions speak louder than",
        "text": " What you do matters more than. you say. you say. you do matters",
        "token_ids": [
          3555,
          498,
          653,
          12844,
          803,
          1091,
          13,
          498,
          1977,
          13,
          498,
          1977,
          13,
          498,
          653,
          12844
        ],
        "cumulative_logprob": -3.4432916017249227,
        "toy_keyword_match": false
      }
    ],
    "speculative_metrics": [],
    "passed": true
  },
  "vllm-fp8-results": {
    "name": "glm4-moe-native-depth",
    "mode": "fp8",
    "llm_args": {
      "model": "fixtures/glm4-moe-native-depth/fp8",
      "dtype": "bfloat16",
      "tensor_parallel_size": 1,
      "gpu_memory_utilization": 0.25,
      "max_model_len": 256,
      "max_num_seqs": 4,
      "enforce_eager": true,
      "enable_prefix_caching": false,
      "seed": 3225,
      "disable_log_stats": false
    },
    "generations": [
      {
        "prompt": "The capital of France is",
        "text": " Paris. Paris is the largest city in France and serves as the country's political",
        "token_ids": [
          12089,
          13,
          12089,
          374,
          279,
          7772,
          3283,
          304,
          9621,
          323,
          17026,
          438,
          279,
          3146,
          594,
          4948
        ],
        "cumulative_logprob": -0.30267687886953354,
        "toy_keyword_match": true
      },
      {
        "prompt": "The creator of the theory of relativity was Albert",
        "text": " Einstein. Einstein developed the special theory of relativity in 1905 and the",
        "token_ids": [
          53690,
          13,
          53690,
          7881,
          279,
          3281,
          10121,
          315,
          1351,
          43166,
          304,
          220,
          114146,
          20,
          323,
          279
        ],
        "cumulative_logprob": -0.20183802442625165,
        "toy_keyword_match": true
      },
      {
        "prompt": "The classic game is called rock, paper,",
        "text": " scissors. Rock beats scissors, scissors beats paper, and paper beats rock. rock",
        "token_ids": [
          71079,
          13,
          9124,
          33167,
          71079,
          11,
          71079,
          33167,
          5567,
          11,
          323,
          5567,
          33167,
          6946,
          13,
          6946
        ],
        "cumulative_logprob": -1.585465795127675,
        "toy_keyword_match": true
      },
      {
        "prompt": "Actions speak louder than",
        "text": " What you do matters more than. you say. you say. you do matters",
        "token_ids": [
          3555,
          498,
          653,
          12844,
          803,
          1091,
          13,
          498,
          1977,
          13,
          498,
          1977,
          13,
          498,
          653,
          12844
        ],
        "cumulative_logprob": -3.5015990268439054,
        "toy_keyword_match": false
      }
    ],
    "speculative_metrics": [],
    "passed": true
  },
  "vllm-mtp-results": {
    "name": "glm4-moe-native-depth",
    "mode": "mtp",
    "llm_args": {
      "model": "fixtures/glm4-moe-native-depth/fp8",
      "dtype": "bfloat16",
      "tensor_parallel_size": 1,
      "gpu_memory_utilization": 0.25,
      "max_model_len": 256,
      "max_num_seqs": 4,
      "enforce_eager": true,
      "enable_prefix_caching": false,
      "seed": 3225,
      "disable_log_stats": false,
      "speculative_config": {
        "method": "mtp",
        "num_speculative_tokens": 1
      }
    },
    "generations": [
      {
        "prompt": "The capital of France is",
        "text": " Paris. Paris is the largest city in France and serves as the country's political",
        "token_ids": [
          12089,
          13,
          12089,
          374,
          279,
          7772,
          3283,
          304,
          9621,
          323,
          17026,
          438,
          279,
          3146,
          594,
          4948
        ],
        "cumulative_logprob": -0.30267687886953354,
        "toy_keyword_match": true
      },
      {
        "prompt": "The creator of the theory of relativity was Albert",
        "text": " Einstein. Einstein developed the special theory of relativity in 1905 and the",
        "token_ids": [
          53690,
          13,
          53690,
          7881,
          279,
          3281,
          10121,
          315,
          1351,
          43166,
          304,
          220,
          114146,
          20,
          323,
          279
        ],
        "cumulative_logprob": -0.20183802442625165,
        "toy_keyword_match": true
      },
      {
        "prompt": "The classic game is called rock, paper,",
        "text": " scissors. Rock beats scissors, scissors beats paper, and paper beats rock. rock",
        "token_ids": [
          71079,
          13,
          9124,
          33167,
          71079,
          11,
          71079,
          33167,
          5567,
          11,
          323,
          5567,
          33167,
          6946,
          13,
          6946
        ],
        "cumulative_logprob": -1.585465795127675,
        "toy_keyword_match": true
      },
      {
        "prompt": "Actions speak louder than",
        "text": " What you do matters more than. you say. you say. you do matters",
        "token_ids": [
          3555,
          498,
          653,
          12844,
          803,
          1091,
          13,
          498,
          1977,
          13,
          498,
          1977,
          13,
          498,
          653,
          12844
        ],
        "cumulative_logprob": -3.5015990268439054,
        "toy_keyword_match": false
      }
    ],
    "speculative_metrics": [
      {
        "name": "vllm:spec_decode_num_drafts",
        "value": 60
      },
      {
        "name": "vllm:spec_decode_num_draft_tokens",
        "value": 60
      },
      {
        "name": "vllm:spec_decode_num_accepted_tokens",
        "value": 0
      },
      {
        "name": "vllm:spec_decode_num_accepted_tokens_per_pos",
        "value": null
      }
    ],
    "passed": true
  }
}