{
  "model_id": "MatterWaveAI/grt-30m-mor",
  "architecture": "GRTForCausalLM (Mixture-of-Rotations)",
  "base_model": "MatterWaveAI/grt-30m",
  "parameters": 31634086,
  "base_parameters": 29305498,
  "ffn_n_experts": 4,
  "mor_mode": "direct",
  "mor_router_mode": "rotational",
  "nn_linear_layers": 0,
  "before_upcycle_specialization": {
    "val_loss": 5.8467,
    "val_ppl": 346.102,
    "domain_val_loss": {
      "code_and_algorithms": 5.4092,
      "reasoning_and_language": 6.172,
      "rotational_geometry": 6.6421,
      "science_and_physics": 5.3337
    }
  },
  "after_upcycle_specialization": {
    "val_loss": 0.5115,
    "val_ppl": 1.6678,
    "domain_val_loss": {
      "code_and_algorithms": 0.4334,
      "reasoning_and_language": 0.6442,
      "rotational_geometry": 0.4991,
      "science_and_physics": 0.5293
    },
    "router_diagnostics": [
      {
        "layer": 0,
        "mean_expert_probs": [
          0.2494,
          0.2488,
          0.253,
          0.2487
        ],
        "marginal_entropy": 1.3863,
        "mean_top1_confidence": 0.2545,
        "expert_angle_l2_divergence": 2.5326
      },
      {
        "layer": 1,
        "mean_expert_probs": [
          0.2492,
          0.247,
          0.2574,
          0.2464
        ],
        "marginal_entropy": 1.3861,
        "mean_top1_confidence": 0.2599,
        "expert_angle_l2_divergence": 2.5353
      },
      {
        "layer": 2,
        "mean_expert_probs": [
          0.2526,
          0.2482,
          0.2528,
          0.2465
        ],
        "marginal_entropy": 1.3862,
        "mean_top1_confidence": 0.2541,
        "expert_angle_l2_divergence": 2.5524
      },
      {
        "layer": 3,
        "mean_expert_probs": [
          0.251,
          0.2486,
          0.2548,
          0.2457
        ],
        "marginal_entropy": 1.3862,
        "mean_top1_confidence": 0.2559,
        "expert_angle_l2_divergence": 2.5425
      },
      {
        "layer": 4,
        "mean_expert_probs": [
          0.2511,
          0.2486,
          0.2562,
          0.2441
        ],
        "marginal_entropy": 1.3861,
        "mean_top1_confidence": 0.2586,
        "expert_angle_l2_divergence": 2.5487
      },
      {
        "layer": 5,
        "mean_expert_probs": [
          0.2508,
          0.2485,
          0.2572,
          0.2436
        ],
        "marginal_entropy": 1.3861,
        "mean_top1_confidence": 0.26,
        "expert_angle_l2_divergence": 2.5342
      },
      {
        "layer": 6,
        "mean_expert_probs": [
          0.2504,
          0.2486,
          0.2571,
          0.2439
        ],
        "marginal_entropy": 1.3861,
        "mean_top1_confidence": 0.2602,
        "expert_angle_l2_divergence": 2.5501
      },
      {
        "layer": 7,
        "mean_expert_probs": [
          0.251,
          0.2488,
          0.2567,
          0.2435
        ],
        "marginal_entropy": 1.3861,
        "mean_top1_confidence": 0.2592,
        "expert_angle_l2_divergence": 2.5291
      },
      {
        "layer": 8,
        "mean_expert_probs": [
          0.2506,
          0.2461,
          0.2605,
          0.2427
        ],
        "marginal_entropy": 1.3859,
        "mean_top1_confidence": 0.2658,
        "expert_angle_l2_divergence": 2.5455
      },
      {
        "layer": 9,
        "mean_expert_probs": [
          0.2502,
          0.247,
          0.2582,
          0.2445
        ],
        "marginal_entropy": 1.3861,
        "mean_top1_confidence": 0.2647,
        "expert_angle_l2_divergence": 2.5364
      },
      {
        "layer": 10,
        "mean_expert_probs": [
          0.2501,
          0.2484,
          0.2563,
          0.2452
        ],
        "marginal_entropy": 1.3862,
        "mean_top1_confidence": 0.2636,
        "expert_angle_l2_divergence": 2.5368
      },
      {
        "layer": 11,
        "mean_expert_probs": [
          0.2509,
          0.2481,
          0.256,
          0.245
        ],
        "marginal_entropy": 1.3862,
        "mean_top1_confidence": 0.2588,
        "expert_angle_l2_divergence": 2.524
      }
    ]
  }
}