Release MatterWaveAI/grt-30m-mor (4-expert Mixture-of-Rotations, 0 nn.Linear layers)
0c8dbfe verified | { | |
| "model_id": "MatterWaveAI/grt-30m-mor", | |
| "architecture": "GRTForCausalLM (Mixture-of-Rotations)", | |
| "base_model": "MatterWaveAI/grt-30m", | |
| "parameters": 31634086, | |
| "base_parameters": 29305498, | |
| "ffn_n_experts": 4, | |
| "mor_mode": "direct", | |
| "mor_router_mode": "rotational", | |
| "nn_linear_layers": 0, | |
| "before_upcycle_specialization": { | |
| "val_loss": 5.8467, | |
| "val_ppl": 346.102, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 5.4092, | |
| "reasoning_and_language": 6.172, | |
| "rotational_geometry": 6.6421, | |
| "science_and_physics": 5.3337 | |
| } | |
| }, | |
| "after_upcycle_specialization": { | |
| "val_loss": 0.5115, | |
| "val_ppl": 1.6678, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 0.4334, | |
| "reasoning_and_language": 0.6442, | |
| "rotational_geometry": 0.4991, | |
| "science_and_physics": 0.5293 | |
| }, | |
| "router_diagnostics": [ | |
| { | |
| "layer": 0, | |
| "mean_expert_probs": [ | |
| 0.2494, | |
| 0.2488, | |
| 0.253, | |
| 0.2487 | |
| ], | |
| "marginal_entropy": 1.3863, | |
| "mean_top1_confidence": 0.2545, | |
| "expert_angle_l2_divergence": 2.5326 | |
| }, | |
| { | |
| "layer": 1, | |
| "mean_expert_probs": [ | |
| 0.2492, | |
| 0.247, | |
| 0.2574, | |
| 0.2464 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.2599, | |
| "expert_angle_l2_divergence": 2.5353 | |
| }, | |
| { | |
| "layer": 2, | |
| "mean_expert_probs": [ | |
| 0.2526, | |
| 0.2482, | |
| 0.2528, | |
| 0.2465 | |
| ], | |
| "marginal_entropy": 1.3862, | |
| "mean_top1_confidence": 0.2541, | |
| "expert_angle_l2_divergence": 2.5524 | |
| }, | |
| { | |
| "layer": 3, | |
| "mean_expert_probs": [ | |
| 0.251, | |
| 0.2486, | |
| 0.2548, | |
| 0.2457 | |
| ], | |
| "marginal_entropy": 1.3862, | |
| "mean_top1_confidence": 0.2559, | |
| "expert_angle_l2_divergence": 2.5425 | |
| }, | |
| { | |
| "layer": 4, | |
| "mean_expert_probs": [ | |
| 0.2511, | |
| 0.2486, | |
| 0.2562, | |
| 0.2441 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.2586, | |
| "expert_angle_l2_divergence": 2.5487 | |
| }, | |
| { | |
| "layer": 5, | |
| "mean_expert_probs": [ | |
| 0.2508, | |
| 0.2485, | |
| 0.2572, | |
| 0.2436 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.26, | |
| "expert_angle_l2_divergence": 2.5342 | |
| }, | |
| { | |
| "layer": 6, | |
| "mean_expert_probs": [ | |
| 0.2504, | |
| 0.2486, | |
| 0.2571, | |
| 0.2439 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.2602, | |
| "expert_angle_l2_divergence": 2.5501 | |
| }, | |
| { | |
| "layer": 7, | |
| "mean_expert_probs": [ | |
| 0.251, | |
| 0.2488, | |
| 0.2567, | |
| 0.2435 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.2592, | |
| "expert_angle_l2_divergence": 2.5291 | |
| }, | |
| { | |
| "layer": 8, | |
| "mean_expert_probs": [ | |
| 0.2506, | |
| 0.2461, | |
| 0.2605, | |
| 0.2427 | |
| ], | |
| "marginal_entropy": 1.3859, | |
| "mean_top1_confidence": 0.2658, | |
| "expert_angle_l2_divergence": 2.5455 | |
| }, | |
| { | |
| "layer": 9, | |
| "mean_expert_probs": [ | |
| 0.2502, | |
| 0.247, | |
| 0.2582, | |
| 0.2445 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.2647, | |
| "expert_angle_l2_divergence": 2.5364 | |
| }, | |
| { | |
| "layer": 10, | |
| "mean_expert_probs": [ | |
| 0.2501, | |
| 0.2484, | |
| 0.2563, | |
| 0.2452 | |
| ], | |
| "marginal_entropy": 1.3862, | |
| "mean_top1_confidence": 0.2636, | |
| "expert_angle_l2_divergence": 2.5368 | |
| }, | |
| { | |
| "layer": 11, | |
| "mean_expert_probs": [ | |
| 0.2509, | |
| 0.2481, | |
| 0.256, | |
| 0.245 | |
| ], | |
| "marginal_entropy": 1.3862, | |
| "mean_top1_confidence": 0.2588, | |
| "expert_angle_l2_divergence": 2.524 | |
| } | |
| ] | |
| } | |
| } | |