Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
| """GRM Evaluation Suite model scores. | |
| Scores use the 0-100 scale from ref/GRM_Score_Summary.csv. Missing or TBD | |
| values are stored as None and displayed as TBD in the UI. | |
| """ | |
| ScoreValue = float | None | |
| MODEL_METADATA: dict[str, dict[str, str | bool | float | None]] = { | |
| "Nano-4B": { | |
| "family": "Nemotron", | |
| "size": "4B", | |
| "parameter_b": 4.0, | |
| "open_weights": True, | |
| "precision": None, | |
| }, | |
| "Gemma-4-E4B": { | |
| "family": "Gemma", | |
| "size": "4B", | |
| "parameter_b": 4.0, | |
| "open_weights": True, | |
| "precision": None, | |
| }, | |
| "Qwen3.5-4B": { | |
| "family": "Qwen", | |
| "size": "4B", | |
| "parameter_b": 4.0, | |
| "open_weights": True, | |
| "precision": None, | |
| }, | |
| "Gemma-4-E2B": { | |
| "family": "Gemma", | |
| "size": "2B", | |
| "parameter_b": 2.0, | |
| "open_weights": True, | |
| "precision": None, | |
| }, | |
| "Qwen3.5-2B": { | |
| "family": "Qwen", | |
| "size": "2B", | |
| "parameter_b": 2.0, | |
| "open_weights": True, | |
| "precision": None, | |
| }, | |
| "Qwen3.5-9B": { | |
| "family": "Qwen", | |
| "size": "9B", | |
| "parameter_b": 9.0, | |
| "open_weights": True, | |
| "precision": None, | |
| }, | |
| "Nano-30B": { | |
| "family": "Nemotron", | |
| "size": "30B", | |
| "parameter_b": 30.0, | |
| "open_weights": True, | |
| "precision": "BF16", | |
| }, | |
| "Super-120B": { | |
| "family": "Nemotron", | |
| "size": "120B+", | |
| "parameter_b": 120.0, | |
| "open_weights": True, | |
| "precision": "BF16", | |
| }, | |
| } | |
| CSV_GRM_SCORES: dict[str, float] = { | |
| "Nano-4B": 50.2, | |
| "Gemma-4-E4B": 64.7, | |
| "Qwen3.5-4B": 61.5, | |
| "Gemma-4-E2B": 50.0, | |
| "Qwen3.5-2B": 49.9, | |
| "Qwen3.5-9B": 68.4, | |
| "Nano-30B": 55.8, | |
| "Super-120B": 61.4, | |
| } | |
| MODEL_SCORES: dict[str, dict[str, ScoreValue]] = { | |
| "Nano-4B": { | |
| "multichallenge": 35.3, | |
| "rolebench": 17.66, | |
| "rolemrc": 39.8, | |
| "eq_bench_v3": None, | |
| "grm_coherence": 27.6, | |
| "grm_response_diversity": 54.0, | |
| "bfcl_v3": 61.1, | |
| "when2call_mt": None, | |
| "toolsandbox": None, | |
| "tau2_bench": 29.2, | |
| "bfcl_v4": None, | |
| "grm_tool_recovery": 44.1, | |
| "iheval": 75.6, | |
| "ruler": 91.2, | |
| "longmemeval": 25.2, | |
| "agentif": 48.3, | |
| "grm_prompt_robustness": None, | |
| "grm_state_adaptation": None, | |
| "garage": None, | |
| "ragtruth": 71.4, | |
| "structeval_t": 63.5, | |
| "ifbench": 60.1, | |
| "grm_persona_actions": 78.0, | |
| "gsm8k": None, | |
| "humaneval": None, | |
| "mbpp": None, | |
| }, | |
| "Gemma-4-E4B": { | |
| "multichallenge": 36.2, | |
| "rolebench": 20.04, | |
| "rolemrc": 74.0, | |
| "eq_bench_v3": None, | |
| "grm_coherence": 89.7, | |
| "grm_response_diversity": 85.0, | |
| "bfcl_v3": 65.9, | |
| "when2call_mt": None, | |
| "toolsandbox": None, | |
| "tau2_bench": 42.2, | |
| "bfcl_v4": 42.4, | |
| "grm_tool_recovery": 64.7, | |
| "iheval": 68.6, | |
| "ruler": None, | |
| "longmemeval": 70.6, | |
| "agentif": 52.7, | |
| "grm_prompt_robustness": None, | |
| "grm_state_adaptation": None, | |
| "garage": None, | |
| "ragtruth": 77.7, | |
| "structeval_t": 78.6, | |
| "ifbench": 43.3, | |
| "grm_persona_actions": 92.0, | |
| "gsm8k": 85.9, | |
| "humaneval": 75.2, | |
| "mbpp": 77.4, | |
| }, | |
| "Qwen3.5-4B": { | |
| "multichallenge": 49.0, | |
| "rolebench": 16.21, | |
| "rolemrc": 56.5, | |
| "eq_bench_v3": 78.4, | |
| "grm_coherence": 41.4, | |
| "grm_response_diversity": 46.0, | |
| "bfcl_v3": 79.9, | |
| "when2call_mt": None, | |
| "toolsandbox": None, | |
| "tau2_bench": 79.9, | |
| "bfcl_v4": 50.3, | |
| "grm_tool_recovery": 41.2, | |
| "iheval": 84.6, | |
| "ruler": 95.0, | |
| "longmemeval": 72.4, | |
| "agentif": 51.7, | |
| "grm_prompt_robustness": None, | |
| "grm_state_adaptation": None, | |
| "garage": None, | |
| "ragtruth": 75.5, | |
| "structeval_t": 76.3, | |
| "ifbench": 59.2, | |
| "grm_persona_actions": 39.0, | |
| "gsm8k": 89.7, | |
| "humaneval": 75.9, | |
| "mbpp": 76.3, | |
| }, | |
| "Gemma-4-E2B": { | |
| "multichallenge": 33.0, | |
| "rolebench": 19.58, | |
| "rolemrc": 72.9, | |
| "eq_bench_v3": None, | |
| "grm_coherence": None, | |
| "grm_response_diversity": None, | |
| "bfcl_v3": 60.9, | |
| "when2call_mt": None, | |
| "toolsandbox": None, | |
| "tau2_bench": 24.5, | |
| "bfcl_v4": 40.2, | |
| "grm_tool_recovery": None, | |
| "iheval": 55.9, | |
| "ruler": None, | |
| "longmemeval": 57.2, | |
| "agentif": 52.5, | |
| "grm_prompt_robustness": None, | |
| "grm_state_adaptation": None, | |
| "garage": None, | |
| "ragtruth": 67.7, | |
| "structeval_t": 80.3, | |
| "ifbench": 37.8, | |
| "grm_persona_actions": None, | |
| "gsm8k": 83.0, | |
| "humaneval": 68.4, | |
| "mbpp": 68.1, | |
| }, | |
| "Qwen3.5-2B": { | |
| "multichallenge": 33.7, | |
| "rolebench": 13.11, | |
| "rolemrc": 43.7, | |
| "eq_bench_v3": 54.3, | |
| "grm_coherence": None, | |
| "grm_response_diversity": None, | |
| "bfcl_v3": 65.7, | |
| "when2call_mt": None, | |
| "toolsandbox": None, | |
| "tau2_bench": 48.8, | |
| "bfcl_v4": 43.6, | |
| "grm_tool_recovery": None, | |
| "iheval": 40.8, | |
| "ruler": 91.5, | |
| "longmemeval": 32.0, | |
| "agentif": 46.0, | |
| "grm_prompt_robustness": None, | |
| "grm_state_adaptation": None, | |
| "garage": None, | |
| "ragtruth": 58.4, | |
| "structeval_t": 54.4, | |
| "ifbench": 41.3, | |
| "grm_persona_actions": None, | |
| "gsm8k": 85.3, | |
| "humaneval": 70.2, | |
| "mbpp": 68.3, | |
| }, | |
| "Qwen3.5-9B": { | |
| "multichallenge": 54.5, | |
| "rolebench": 16.78, | |
| "rolemrc": 66.7, | |
| "eq_bench_v3": None, | |
| "grm_coherence": None, | |
| "grm_response_diversity": None, | |
| "bfcl_v3": 74.6, | |
| "when2call_mt": None, | |
| "toolsandbox": None, | |
| "tau2_bench": 79.1, | |
| "bfcl_v4": 66.1, | |
| "grm_tool_recovery": None, | |
| "iheval": 85.6, | |
| "ruler": 95.7, | |
| "longmemeval": 82.2, | |
| "agentif": 53.6, | |
| "grm_prompt_robustness": None, | |
| "grm_state_adaptation": None, | |
| "garage": None, | |
| "ragtruth": 72.2, | |
| "structeval_t": 68.3, | |
| "ifbench": 64.5, | |
| "grm_persona_actions": None, | |
| "gsm8k": 93.9, | |
| "humaneval": 84.8, | |
| "mbpp": 80.8, | |
| }, | |
| "Nano-30B": { | |
| "multichallenge": 43.2, | |
| "rolebench": 50.2, | |
| "rolemrc": 17.94, | |
| "eq_bench_v3": None, | |
| "grm_coherence": None, | |
| "grm_response_diversity": None, | |
| "bfcl_v3": 71.1, | |
| "when2call_mt": None, | |
| "toolsandbox": None, | |
| "tau2_bench": 49.0, | |
| "bfcl_v4": 53.8, | |
| "grm_tool_recovery": None, | |
| "iheval": 72.4, | |
| "ruler": 95.3, | |
| "longmemeval": 45.6, | |
| "agentif": 52.9, | |
| "grm_prompt_robustness": None, | |
| "grm_state_adaptation": None, | |
| "garage": None, | |
| "ragtruth": 73.0, | |
| "structeval_t": 76.9, | |
| "ifbench": 71.5, | |
| "grm_persona_actions": None, | |
| "gsm8k": 92.3, | |
| "humaneval": 78.1, | |
| "mbpp": 75.5, | |
| }, | |
| "Super-120B": { | |
| "multichallenge": 55.2, | |
| "rolebench": 57.0, | |
| "rolemrc": 19.48, | |
| "eq_bench_v3": None, | |
| "grm_coherence": None, | |
| "grm_response_diversity": None, | |
| "bfcl_v3": 74.4, | |
| "when2call_mt": None, | |
| "toolsandbox": None, | |
| "tau2_bench": 61.2, | |
| "bfcl_v4": 53.8, | |
| "grm_tool_recovery": None, | |
| "iheval": 76.3, | |
| "ruler": 96.8, | |
| "longmemeval": 75.6, | |
| "agentif": 52.7, | |
| "grm_prompt_robustness": None, | |
| "grm_state_adaptation": None, | |
| "garage": None, | |
| "ragtruth": 74.9, | |
| "structeval_t": 82.2, | |
| "ifbench": 72.6, | |
| "grm_persona_actions": None, | |
| "gsm8k": None, | |
| "humaneval": 79.4, | |
| "mbpp": 78.4, | |
| }, | |
| } | |