Spaces:
Running
Running
| { | |
| "updated": "2026-07-20", | |
| "latest_generation": "7-2026", | |
| "benchmarks": [ | |
| { | |
| "id": "bench-effortless-7-2026", | |
| "label": "Effortless", | |
| "tier": 1, | |
| "generation": "7-2026", | |
| "rows": 300, | |
| "metric": "exact_match", | |
| "dual_mode": true, | |
| "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-effortless-7-2026", | |
| "description": "Sanity-layer benchmark, v2 dual-mode schema: every item scored by generation AND log-likelihood. Balanced across all 17 categories." | |
| }, | |
| { | |
| "id": "bench-easy-7-2026", | |
| "label": "Easy", | |
| "tier": 2, | |
| "generation": "7-2026", | |
| "rows": 300, | |
| "metric": "hybrid_score", | |
| "dual_mode": true, | |
| "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-easy-7-2026", | |
| "description": "Easy-tier QA, v2 dual-mode schema with per-item scorer routing (strict / semantic / fuzzy). Balanced across all 17 categories." | |
| }, | |
| { | |
| "id": "bench-mid-7-2026", | |
| "label": "Mid", | |
| "tier": 3, | |
| "generation": "7-2026", | |
| "rows": 300, | |
| "metric": "soft_score_norm", | |
| "dual_mode": true, | |
| "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-mid-7-2026", | |
| "description": "Mid-tier QA, v2 dual-mode schema; headline is log-likelihood soft_score_norm, generation metrics also reported. Balanced across all 17 categories." | |
| }, | |
| { | |
| "id": "bench-effortless-6-2026", | |
| "label": "Effortless (6-2026)", | |
| "tier": 1, | |
| "generation": "6-2026", | |
| "rows": 240, | |
| "metric": "exact_match", | |
| "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-effortless-6-2026", | |
| "description": "Sanity-layer benchmark. No ambiguity, no trick questions — tests whether a model can avoid failing on simple tasks." | |
| }, | |
| { | |
| "id": "bench-easy-6-2026", | |
| "label": "Easy (6-2026)", | |
| "tier": 2, | |
| "generation": "6-2026", | |
| "rows": 238, | |
| "metric": "hybrid_score", | |
| "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-easy-6-2026", | |
| "description": "Effortless-to-Easy tier QA. Structured language understanding and controlled generalization." | |
| }, | |
| { | |
| "id": "bench-mid-6-2026", | |
| "label": "Mid (6-2026)", | |
| "tier": 3, | |
| "generation": "6-2026", | |
| "rows": 143, | |
| "metric": "soft_score_norm", | |
| "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-mid-6-2026", | |
| "description": "Easy-to-Mid tier multiple-choice QA scored via lm-eval loglikelihood, target_scores dict." | |
| }, | |
| { | |
| "id": "bench-AGI", | |
| "label": "AGI", | |
| "tier": 4, | |
| "generation": "6-2026", | |
| "rows": null, | |
| "metric": "rank_order", | |
| "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-AGI", | |
| "description": "Hard, open-ended questions with human-argued reasoning traces, graded blind by a multi-vendor panel. Rank-order evaluation, not absolute scoring.", | |
| "unavailable": true, | |
| "unavailable_reason": "Under maintenance. Scoring pipeline is being reworked — see the dataset README." | |
| } | |
| ], | |
| "categories": [ | |
| "Knowledge-basic", | |
| "Knowledge-definitions", | |
| "Math-arithmetic", | |
| "Math-pattern", | |
| "Math-reasoning", | |
| "Logic-deduction", | |
| "Logic-pattern", | |
| "Logic-consistency", | |
| "Language-comprehension", | |
| "Language-transformation", | |
| "Language-structure", | |
| "Commonsense-reasoning", | |
| "Commonsense-causality", | |
| "Commonsense-simulation", | |
| "Pattern-matching", | |
| "Pattern-recognition", | |
| "Pattern-generation" | |
| ], | |
| "models": [ | |
| { | |
| "id": "liquidai-lfm2.5-1.2b-base", | |
| "name": "LiquidAI/LFM2.5-1.2B-Base", | |
| "org": "LiquidAI", | |
| "params_b": null, | |
| "license": null, | |
| "architecture": null, | |
| "url": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Base", | |
| "model_revision": "f6a5d174bc3e52bd0df245d69133f9930b4828d8", | |
| "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", | |
| "runs": { | |
| "bench-effortless-6-2026": { | |
| "score": 0.0, | |
| "n": 240, | |
| "notes": "Exact-match, normalized.", | |
| "categories": { | |
| "Commonsense-reasoning": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Language-comprehension": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 40, | |
| "exact_match": 0.0 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": 0.3087, | |
| "n": 238, | |
| "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 10, | |
| "hybrid_score": 0.7338 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "hybrid_score": 0.7347 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "hybrid_score": 0.742 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 33, | |
| "hybrid_score": 0.0303 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 38, | |
| "hybrid_score": 0.8057 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "hybrid_score": 0.7448 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "hybrid_score": 0.2311 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "hybrid_score": 0.7685 | |
| }, | |
| "Logic-consistency": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 15, | |
| "hybrid_score": 0.0667 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-pattern": { | |
| "n": 14, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 15, | |
| "hybrid_score": 0.0868 | |
| }, | |
| "Pattern-matching": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": 0.621, | |
| "n": 143, | |
| "acc": 0.5385, | |
| "acc_norm": 0.6154, | |
| "soft_score": 0.5455, | |
| "soft_score_norm": 0.621, | |
| "stderr": 0.0404, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 5, | |
| "acc": 0.6, | |
| "acc_norm": 0.6 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.5 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.7 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 7, | |
| "acc": 0.5714, | |
| "acc_norm": 0.5714 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.9 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.7 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.4 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.7 | |
| }, | |
| "Logic-consistency": { | |
| "n": 5, | |
| "acc": 0.0, | |
| "acc_norm": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.3 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.6 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 8, | |
| "acc": 0.875, | |
| "acc_norm": 0.875 | |
| }, | |
| "Math-pattern": { | |
| "n": 7, | |
| "acc": 0.8571, | |
| "acc_norm": 0.8571 | |
| }, | |
| "Math-reasoning": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.3 | |
| }, | |
| "Pattern-generation": { | |
| "n": 4, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Pattern-matching": { | |
| "n": 8, | |
| "acc": 1.0, | |
| "acc_norm": 0.875 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 9, | |
| "acc": 0.4444, | |
| "acc_norm": 0.6667 | |
| } | |
| } | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| } | |
| } | |
| }, | |
| { | |
| "id": "liquidai-lfm2.5-1.2b-instruct", | |
| "name": "LiquidAI/LFM2.5-1.2B-Instruct", | |
| "org": "LiquidAI", | |
| "params_b": null, | |
| "license": null, | |
| "architecture": null, | |
| "url": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct", | |
| "model_revision": "868df74dd56ff8a0c2ac5dbf281690c2dbebe4c9", | |
| "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", | |
| "runs": { | |
| "bench-effortless-6-2026": { | |
| "score": 0.1208, | |
| "n": 240, | |
| "notes": "Exact-match, normalized.", | |
| "categories": { | |
| "Commonsense-reasoning": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 42, | |
| "exact_match": 0.119 | |
| }, | |
| "Language-comprehension": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 42, | |
| "exact_match": 0.0952 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 40, | |
| "exact_match": 0.475 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 37, | |
| "exact_match": 0.027 | |
| } | |
| } | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": 0.441, | |
| "n": 238, | |
| "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 10, | |
| "hybrid_score": 0.7508 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "hybrid_score": 0.7711 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "hybrid_score": 0.7765 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 33, | |
| "hybrid_score": 0.303 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 38, | |
| "hybrid_score": 0.8172 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "hybrid_score": 0.7459 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "hybrid_score": 0.4632 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "hybrid_score": 0.7849 | |
| }, | |
| "Logic-consistency": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 15, | |
| "hybrid_score": 0.0667 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "hybrid_score": 0.2 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 33, | |
| "hybrid_score": 0.2727 | |
| }, | |
| "Math-pattern": { | |
| "n": 14, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 15, | |
| "hybrid_score": 0.4654 | |
| }, | |
| "Pattern-matching": { | |
| "n": 10, | |
| "hybrid_score": 0.2 | |
| } | |
| } | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": 0.607, | |
| "n": 143, | |
| "acc": 0.5175, | |
| "acc_norm": 0.6014, | |
| "soft_score": 0.5231, | |
| "soft_score_norm": 0.607, | |
| "stderr": 0.0406, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 5, | |
| "acc": 0.8, | |
| "acc_norm": 0.8 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.5 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.6 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 7, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.9 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.6 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "acc": 0.2, | |
| "acc_norm": 0.3 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.6 | |
| }, | |
| "Logic-consistency": { | |
| "n": 5, | |
| "acc": 0.0, | |
| "acc_norm": 0.2 | |
| }, | |
| "Logic-deduction": { | |
| "n": 10, | |
| "acc": 0.0, | |
| "acc_norm": 0.3 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.3 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 8, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Math-pattern": { | |
| "n": 7, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.3 | |
| }, | |
| "Pattern-generation": { | |
| "n": 4, | |
| "acc": 0.75, | |
| "acc_norm": 0.75 | |
| }, | |
| "Pattern-matching": { | |
| "n": 8, | |
| "acc": 0.75, | |
| "acc_norm": 0.75 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 9, | |
| "acc": 0.5556, | |
| "acc_norm": 0.6667 | |
| } | |
| } | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| } | |
| } | |
| }, | |
| { | |
| "id": "liquidai-lfm2.5-230m-base", | |
| "name": "LiquidAI/LFM2.5-230M-Base", | |
| "org": "LiquidAI", | |
| "params_b": null, | |
| "license": null, | |
| "architecture": null, | |
| "url": "https://huggingface.co/LiquidAI/LFM2.5-230M-Base", | |
| "model_revision": "9d2be5519834990d30996f878b6771cccbd24f2c", | |
| "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", | |
| "runs": { | |
| "bench-effortless-6-2026": { | |
| "score": 0.0042, | |
| "n": 240, | |
| "notes": "Exact-match, normalized.", | |
| "categories": { | |
| "Commonsense-reasoning": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Language-comprehension": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 40, | |
| "exact_match": 0.0 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 37, | |
| "exact_match": 0.027 | |
| } | |
| } | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": 0.2876, | |
| "n": 238, | |
| "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 10, | |
| "hybrid_score": 0.7301 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "hybrid_score": 0.7137 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "hybrid_score": 0.7434 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 38, | |
| "hybrid_score": 0.7888 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "hybrid_score": 0.7182 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "hybrid_score": 0.2974 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "hybrid_score": 0.6079 | |
| }, | |
| "Logic-consistency": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 15, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-pattern": { | |
| "n": 14, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 15, | |
| "hybrid_score": 0.0247 | |
| }, | |
| "Pattern-matching": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": 0.5301, | |
| "n": 143, | |
| "acc": 0.4266, | |
| "acc_norm": 0.5245, | |
| "soft_score": 0.4357, | |
| "soft_score_norm": 0.5301, | |
| "stderr": 0.0415, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 5, | |
| "acc": 0.8, | |
| "acc_norm": 0.6 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.4 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.4 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 7, | |
| "acc": 0.5714, | |
| "acc_norm": 0.8571 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 10, | |
| "acc": 0.1, | |
| "acc_norm": 0.7 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.8 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "acc": 0.1, | |
| "acc_norm": 0.3 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "acc": 0.2, | |
| "acc_norm": 0.4 | |
| }, | |
| "Logic-consistency": { | |
| "n": 5, | |
| "acc": 0.0, | |
| "acc_norm": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.7 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.4 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 8, | |
| "acc": 0.875, | |
| "acc_norm": 0.875 | |
| }, | |
| "Math-pattern": { | |
| "n": 7, | |
| "acc": 0.8571, | |
| "acc_norm": 0.8571 | |
| }, | |
| "Math-reasoning": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.3 | |
| }, | |
| "Pattern-generation": { | |
| "n": 4, | |
| "acc": 0.75, | |
| "acc_norm": 0.5 | |
| }, | |
| "Pattern-matching": { | |
| "n": 8, | |
| "acc": 0.5, | |
| "acc_norm": 0.5 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 9, | |
| "acc": 0.3333, | |
| "acc_norm": 0.3333 | |
| } | |
| } | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| } | |
| } | |
| }, | |
| { | |
| "id": "liquidai-lfm2.5-350m", | |
| "name": "LiquidAI/LFM2.5-350M", | |
| "org": "LiquidAI", | |
| "params_b": null, | |
| "license": null, | |
| "architecture": null, | |
| "url": "https://huggingface.co/LiquidAI/LFM2.5-350M", | |
| "model_revision": "b9d6e4e2d75f440b12a2b4d731c808004ecbbd89", | |
| "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", | |
| "runs": { | |
| "bench-effortless-6-2026": { | |
| "score": 0.1375, | |
| "n": 240, | |
| "notes": "Exact-match, normalized.", | |
| "categories": { | |
| "Commonsense-reasoning": { | |
| "n": 37, | |
| "exact_match": 0.027 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 42, | |
| "exact_match": 0.0952 | |
| }, | |
| "Language-comprehension": { | |
| "n": 42, | |
| "exact_match": 0.2143 | |
| }, | |
| "Logic-deduction": { | |
| "n": 42, | |
| "exact_match": 0.4286 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 40, | |
| "exact_match": 0.0 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 37, | |
| "exact_match": 0.027 | |
| } | |
| } | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": 0.358, | |
| "n": 238, | |
| "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 10, | |
| "hybrid_score": 0.721 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "hybrid_score": 0.7313 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "hybrid_score": 0.75 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 33, | |
| "hybrid_score": 0.2424 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 38, | |
| "hybrid_score": 0.7996 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "hybrid_score": 0.741 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "hybrid_score": 0.3121 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "hybrid_score": 0.7689 | |
| }, | |
| "Logic-consistency": { | |
| "n": 10, | |
| "hybrid_score": 0.1 | |
| }, | |
| "Logic-deduction": { | |
| "n": 15, | |
| "hybrid_score": 0.2 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-pattern": { | |
| "n": 14, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 15, | |
| "hybrid_score": 0.1049 | |
| }, | |
| "Pattern-matching": { | |
| "n": 10, | |
| "hybrid_score": 0.1 | |
| } | |
| } | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": 0.5105, | |
| "n": 143, | |
| "acc": 0.4755, | |
| "acc_norm": 0.5035, | |
| "soft_score": 0.4825, | |
| "soft_score_norm": 0.5105, | |
| "stderr": 0.0417, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 5, | |
| "acc": 0.8, | |
| "acc_norm": 0.8 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.6 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.5 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 7, | |
| "acc": 0.8571, | |
| "acc_norm": 0.8571 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.8 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.8 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "acc": 0.2, | |
| "acc_norm": 0.1 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "acc": 0.2, | |
| "acc_norm": 0.4 | |
| }, | |
| "Logic-consistency": { | |
| "n": 5, | |
| "acc": 0.0, | |
| "acc_norm": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.4 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.3 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 8, | |
| "acc": 0.875, | |
| "acc_norm": 0.875 | |
| }, | |
| "Math-pattern": { | |
| "n": 7, | |
| "acc": 0.7143, | |
| "acc_norm": 0.7143 | |
| }, | |
| "Math-reasoning": { | |
| "n": 10, | |
| "acc": 0.2, | |
| "acc_norm": 0.1 | |
| }, | |
| "Pattern-generation": { | |
| "n": 4, | |
| "acc": 0.75, | |
| "acc_norm": 0.5 | |
| }, | |
| "Pattern-matching": { | |
| "n": 8, | |
| "acc": 0.75, | |
| "acc_norm": 0.375 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 9, | |
| "acc": 0.5556, | |
| "acc_norm": 0.5556 | |
| } | |
| } | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| } | |
| } | |
| }, | |
| { | |
| "id": "liquidai-lfm2.5-350m-base", | |
| "name": "LiquidAI/LFM2.5-350M-Base", | |
| "org": "LiquidAI", | |
| "params_b": null, | |
| "license": null, | |
| "architecture": null, | |
| "url": "https://huggingface.co/LiquidAI/LFM2.5-350M-Base", | |
| "model_revision": "9960764e30892e01f29a6dc23df2533fcd8bd5ae", | |
| "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", | |
| "runs": { | |
| "bench-effortless-6-2026": { | |
| "score": 0.0, | |
| "n": 240, | |
| "notes": "Exact-match, normalized.", | |
| "categories": { | |
| "Commonsense-reasoning": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Language-comprehension": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 40, | |
| "exact_match": 0.0 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": 0.2788, | |
| "n": 238, | |
| "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 10, | |
| "hybrid_score": 0.7048 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "hybrid_score": 0.6998 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "hybrid_score": 0.744 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 38, | |
| "hybrid_score": 0.7958 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "hybrid_score": 0.7201 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "hybrid_score": 0.146 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "hybrid_score": 0.5558 | |
| }, | |
| "Logic-consistency": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 15, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-pattern": { | |
| "n": 14, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 15, | |
| "hybrid_score": 0.0271 | |
| }, | |
| "Pattern-matching": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": 0.5105, | |
| "n": 143, | |
| "acc": 0.3986, | |
| "acc_norm": 0.5035, | |
| "soft_score": 0.4091, | |
| "soft_score_norm": 0.5105, | |
| "stderr": 0.0417, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 5, | |
| "acc": 0.8, | |
| "acc_norm": 0.6 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.5 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.3 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 7, | |
| "acc": 0.4286, | |
| "acc_norm": 0.5714 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 10, | |
| "acc": 0.1, | |
| "acc_norm": 0.7 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.8 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "acc": 0.0, | |
| "acc_norm": 0.2 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.7 | |
| }, | |
| "Logic-consistency": { | |
| "n": 5, | |
| "acc": 0.0, | |
| "acc_norm": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.6 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.5 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 8, | |
| "acc": 0.875, | |
| "acc_norm": 0.875 | |
| }, | |
| "Math-pattern": { | |
| "n": 7, | |
| "acc": 0.7143, | |
| "acc_norm": 0.7143 | |
| }, | |
| "Math-reasoning": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.3 | |
| }, | |
| "Pattern-generation": { | |
| "n": 4, | |
| "acc": 0.75, | |
| "acc_norm": 0.5 | |
| }, | |
| "Pattern-matching": { | |
| "n": 8, | |
| "acc": 0.625, | |
| "acc_norm": 0.375 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 9, | |
| "acc": 0.2222, | |
| "acc_norm": 0.2222 | |
| } | |
| } | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| } | |
| } | |
| }, | |
| { | |
| "id": "liquidai-lfm2.5-230m", | |
| "name": "LiquidAI/LFM2.5-230M", | |
| "org": "LiquidAI", | |
| "params_b": 0.23, | |
| "license": null, | |
| "architecture": "lfm2.5", | |
| "url": "https://huggingface.co/LiquidAI/LFM2.5-230M", | |
| "model_revision": "37b30cce3446f3f2e26a0d3f8c67c9167f5079d7", | |
| "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", | |
| "runs": { | |
| "bench-effortless-6-2026": { | |
| "score": 0.0, | |
| "n": 240, | |
| "notes": "Exact-match, normalized.", | |
| "categories": { | |
| "Commonsense-reasoning": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Language-comprehension": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 40, | |
| "exact_match": 0.0 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": 0.2953, | |
| "n": 238, | |
| "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 10, | |
| "hybrid_score": 0.7178 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "hybrid_score": 0.738 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "hybrid_score": 0.7269 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 38, | |
| "hybrid_score": 0.7906 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "hybrid_score": 0.7643 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "hybrid_score": 0.302 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "hybrid_score": 0.7019 | |
| }, | |
| "Logic-consistency": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 15, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-pattern": { | |
| "n": 14, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 15, | |
| "hybrid_score": 0.049 | |
| }, | |
| "Pattern-matching": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": 0.514, | |
| "n": 143, | |
| "acc": 0.4825, | |
| "acc_norm": 0.5105, | |
| "soft_score": 0.486, | |
| "soft_score_norm": 0.514, | |
| "stderr": 0.0417, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 5, | |
| "acc": 0.6, | |
| "acc_norm": 0.6 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.5 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.4 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 7, | |
| "acc": 0.8571, | |
| "acc_norm": 0.8571 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.7 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.7 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "acc": 0.2, | |
| "acc_norm": 0.1 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.4 | |
| }, | |
| "Logic-consistency": { | |
| "n": 5, | |
| "acc": 0.2, | |
| "acc_norm": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.6 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.4 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 8, | |
| "acc": 0.875, | |
| "acc_norm": 0.875 | |
| }, | |
| "Math-pattern": { | |
| "n": 7, | |
| "acc": 0.5714, | |
| "acc_norm": 0.5714 | |
| }, | |
| "Math-reasoning": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.3 | |
| }, | |
| "Pattern-generation": { | |
| "n": 4, | |
| "acc": 0.75, | |
| "acc_norm": 0.5 | |
| }, | |
| "Pattern-matching": { | |
| "n": 8, | |
| "acc": 0.625, | |
| "acc_norm": 0.75 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 9, | |
| "acc": 0.4444, | |
| "acc_norm": 0.4444 | |
| } | |
| } | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| } | |
| } | |
| }, | |
| { | |
| "id": "qwen-qwen2.5-1.5b", | |
| "name": "Qwen/Qwen2.5-1.5B", | |
| "org": "Qwen", | |
| "params_b": 1.5, | |
| "license": null, | |
| "architecture": null, | |
| "url": "https://huggingface.co/Qwen/Qwen2.5-1.5B", | |
| "model_revision": "8faed761d45a263340a0528343f099c05c9a4323", | |
| "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", | |
| "runs": { | |
| "bench-effortless-6-2026": { | |
| "score": 0.0, | |
| "n": 240, | |
| "notes": "Exact-match, normalized.", | |
| "categories": { | |
| "Commonsense-reasoning": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Language-comprehension": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 40, | |
| "exact_match": 0.0 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": 0.2563, | |
| "n": 238, | |
| "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 10, | |
| "hybrid_score": 0.5943 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "hybrid_score": 0.7341 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "hybrid_score": 0.7074 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 38, | |
| "hybrid_score": 0.676 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "hybrid_score": 0.6957 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "hybrid_score": 0.2408 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "hybrid_score": 0.3879 | |
| }, | |
| "Logic-consistency": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 15, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-pattern": { | |
| "n": 14, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 15, | |
| "hybrid_score": 0.1135 | |
| }, | |
| "Pattern-matching": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": 0.7517, | |
| "n": 143, | |
| "acc": 0.7483, | |
| "acc_norm": 0.7483, | |
| "soft_score": 0.7545, | |
| "soft_score_norm": 0.7517, | |
| "stderr": 0.036, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 5, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "acc": 1.0, | |
| "acc_norm": 0.8 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.8 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 7, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 10, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "acc": 0.7, | |
| "acc_norm": 0.9 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.6 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "acc": 0.9, | |
| "acc_norm": 0.9 | |
| }, | |
| "Logic-consistency": { | |
| "n": 5, | |
| "acc": 0.0, | |
| "acc_norm": 0.2 | |
| }, | |
| "Logic-deduction": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.4 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.3 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 8, | |
| "acc": 0.875, | |
| "acc_norm": 0.875 | |
| }, | |
| "Math-pattern": { | |
| "n": 7, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 10, | |
| "acc": 0.8, | |
| "acc_norm": 0.6 | |
| }, | |
| "Pattern-generation": { | |
| "n": 4, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Pattern-matching": { | |
| "n": 8, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 9, | |
| "acc": 0.5556, | |
| "acc_norm": 0.5556 | |
| } | |
| } | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| } | |
| } | |
| }, | |
| { | |
| "id": "qwen-qwen2.5-0.5b-instruct", | |
| "name": "Qwen/Qwen2.5-0.5B-Instruct", | |
| "org": "Qwen", | |
| "params_b": 0.5, | |
| "license": null, | |
| "architecture": null, | |
| "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct", | |
| "model_revision": "7ae557604adf67be50417f59c2c2f167def9a775", | |
| "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", | |
| "runs": { | |
| "bench-effortless-6-2026": { | |
| "score": 0.225, | |
| "n": 240, | |
| "notes": "Exact-match, normalized.", | |
| "categories": { | |
| "Commonsense-reasoning": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 42, | |
| "exact_match": 0.1905 | |
| }, | |
| "Language-comprehension": { | |
| "n": 42, | |
| "exact_match": 0.119 | |
| }, | |
| "Logic-deduction": { | |
| "n": 42, | |
| "exact_match": 0.0238 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 40, | |
| "exact_match": 0.825 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 37, | |
| "exact_match": 0.1892 | |
| } | |
| } | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": 0.4847, | |
| "n": 238, | |
| "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 10, | |
| "hybrid_score": 0.7134 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "hybrid_score": 0.7698 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "hybrid_score": 0.7428 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 33, | |
| "hybrid_score": 0.2424 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 38, | |
| "hybrid_score": 0.8114 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "hybrid_score": 0.7592 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "hybrid_score": 0.37 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "hybrid_score": 0.9131 | |
| }, | |
| "Logic-consistency": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 15, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "hybrid_score": 0.1 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 33, | |
| "hybrid_score": 0.8485 | |
| }, | |
| "Math-pattern": { | |
| "n": 14, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 15, | |
| "hybrid_score": 0.1892 | |
| }, | |
| "Pattern-matching": { | |
| "n": 10, | |
| "hybrid_score": 0.2 | |
| } | |
| } | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": 0.6636, | |
| "n": 143, | |
| "acc": 0.6154, | |
| "acc_norm": 0.6573, | |
| "soft_score": 0.6252, | |
| "soft_score_norm": 0.6636, | |
| "stderr": 0.0394, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 5, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "acc": 0.9, | |
| "acc_norm": 0.7 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.7 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 7, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 10, | |
| "acc": 0.7, | |
| "acc_norm": 0.9 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.8 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "acc": 0.2, | |
| "acc_norm": 0.3 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "acc": 0.6, | |
| "acc_norm": 0.8 | |
| }, | |
| "Logic-consistency": { | |
| "n": 5, | |
| "acc": 0.0, | |
| "acc_norm": 0.2 | |
| }, | |
| "Logic-deduction": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.4 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.4 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 8, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Math-pattern": { | |
| "n": 7, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.2 | |
| }, | |
| "Pattern-generation": { | |
| "n": 4, | |
| "acc": 0.75, | |
| "acc_norm": 0.75 | |
| }, | |
| "Pattern-matching": { | |
| "n": 8, | |
| "acc": 1.0, | |
| "acc_norm": 0.875 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 9, | |
| "acc": 0.4444, | |
| "acc_norm": 0.4444 | |
| } | |
| } | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| } | |
| } | |
| }, | |
| { | |
| "id": "qwen-qwen2.5-0.5b", | |
| "name": "Qwen/Qwen2.5-0.5B", | |
| "org": "Qwen", | |
| "params_b": 0.5, | |
| "license": null, | |
| "architecture": null, | |
| "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B", | |
| "model_revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", | |
| "script_sha256": "955411f9971c4c26a6eeb3aa43b1694fe66ac430a139011faa915543746d2921", | |
| "runs": { | |
| "bench-effortless-6-2026": { | |
| "score": 0.075, | |
| "n": 240, | |
| "notes": "Exact-match, normalized.", | |
| "categories": { | |
| "Commonsense-reasoning": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 42, | |
| "exact_match": 0.0238 | |
| }, | |
| "Language-comprehension": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 42, | |
| "exact_match": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 40, | |
| "exact_match": 0.425 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 37, | |
| "exact_match": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": 0.314, | |
| "n": 238, | |
| "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 10, | |
| "hybrid_score": 0.6992 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "hybrid_score": 0.6406 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "hybrid_score": 0.6482 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 33, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 38, | |
| "hybrid_score": 0.7609 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "hybrid_score": 0.6587 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "hybrid_score": 0.3333 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "hybrid_score": 0.4212 | |
| }, | |
| "Logic-consistency": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 15, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 33, | |
| "hybrid_score": 0.3333 | |
| }, | |
| "Math-pattern": { | |
| "n": 14, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 15, | |
| "hybrid_score": 0.0538 | |
| }, | |
| "Pattern-matching": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| } | |
| } | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": 0.5986, | |
| "n": 143, | |
| "acc": 0.5594, | |
| "acc_norm": 0.5874, | |
| "soft_score": 0.5657, | |
| "soft_score_norm": 0.5986, | |
| "stderr": 0.0406, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 5, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "acc": 0.9, | |
| "acc_norm": 0.6 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.7 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 7, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 10, | |
| "acc": 0.7, | |
| "acc_norm": 0.9 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "acc": 0.7, | |
| "acc_norm": 0.9 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "acc": 0.1, | |
| "acc_norm": 0.1 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.7 | |
| }, | |
| "Logic-consistency": { | |
| "n": 5, | |
| "acc": 0.0, | |
| "acc_norm": 0.2 | |
| }, | |
| "Logic-deduction": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.4 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "acc": 0.1, | |
| "acc_norm": 0.2 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 8, | |
| "acc": 0.875, | |
| "acc_norm": 0.875 | |
| }, | |
| "Math-pattern": { | |
| "n": 7, | |
| "acc": 0.7143, | |
| "acc_norm": 0.7143 | |
| }, | |
| "Math-reasoning": { | |
| "n": 10, | |
| "acc": 0.4, | |
| "acc_norm": 0.2 | |
| }, | |
| "Pattern-generation": { | |
| "n": 4, | |
| "acc": 0.75, | |
| "acc_norm": 0.75 | |
| }, | |
| "Pattern-matching": { | |
| "n": 8, | |
| "acc": 1.0, | |
| "acc_norm": 0.625 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 9, | |
| "acc": 0.4444, | |
| "acc_norm": 0.4444 | |
| } | |
| } | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| }, | |
| "bench-effortless-7-2026": { | |
| "score": 0.0033, | |
| "n": 300, | |
| "stderr": 0.0033, | |
| "metrics": { | |
| "generative": { | |
| "exact_match": 0.0033, | |
| "hybrid_score": 0.2351 | |
| }, | |
| "loglikelihood": { | |
| "acc": 0.7733, | |
| "acc_norm": 0.7433, | |
| "soft_score": 0.7733, | |
| "soft_score_norm": 0.7433 | |
| } | |
| }, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7778 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7778 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7222 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7778 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.8889 | |
| }, | |
| "Language-comprehension": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.8333 | |
| }, | |
| "Language-structure": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7222 | |
| }, | |
| "Language-transformation": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7778 | |
| }, | |
| "Logic-consistency": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.9444 | |
| }, | |
| "Logic-pattern": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7778 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.8824 | |
| }, | |
| "Math-pattern": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.9412 | |
| }, | |
| "Math-reasoning": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7059 | |
| }, | |
| "Pattern-generation": { | |
| "n": 17, | |
| "score": 0.0588, | |
| "exact_match": 0.0588, | |
| "acc_norm": 0.8235 | |
| }, | |
| "Pattern-matching": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7059 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5882 | |
| } | |
| } | |
| }, | |
| "bench-easy-7-2026": { | |
| "score": 0.266, | |
| "n": 300, | |
| "stderr": 0.02, | |
| "metrics": { | |
| "generative": { | |
| "exact_match": 0.02, | |
| "hybrid_score": 0.266 | |
| }, | |
| "loglikelihood": { | |
| "acc": 0.6767, | |
| "acc_norm": 0.64, | |
| "soft_score": 0.6767, | |
| "soft_score_norm": 0.64 | |
| } | |
| }, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 18, | |
| "score": 0.6793, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.8333 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 18, | |
| "score": 0.6656, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7778 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 18, | |
| "score": 0.6951, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5556 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7778 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 18, | |
| "score": 0.7592, | |
| "exact_match": 0.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Language-comprehension": { | |
| "n": 18, | |
| "score": 0.7092, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.8889 | |
| }, | |
| "Language-structure": { | |
| "n": 18, | |
| "score": 0.131, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5556 | |
| }, | |
| "Language-transformation": { | |
| "n": 18, | |
| "score": 0.5156, | |
| "exact_match": 0.0556, | |
| "acc_norm": 0.7778 | |
| }, | |
| "Logic-consistency": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 18, | |
| "score": 0.0556, | |
| "exact_match": 0.0556, | |
| "acc_norm": 0.7778 | |
| }, | |
| "Logic-pattern": { | |
| "n": 18, | |
| "score": 0.1111, | |
| "exact_match": 0.1111, | |
| "acc_norm": 0.6667 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1765 | |
| }, | |
| "Math-pattern": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7059 | |
| }, | |
| "Math-reasoning": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7647 | |
| }, | |
| "Pattern-generation": { | |
| "n": 17, | |
| "score": 0.1176, | |
| "exact_match": 0.1176, | |
| "acc_norm": 0.4706 | |
| }, | |
| "Pattern-matching": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5882 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5294 | |
| } | |
| } | |
| }, | |
| "bench-mid-7-2026": { | |
| "score": 0.5033, | |
| "n": 300, | |
| "stderr": 0.0289, | |
| "metrics": { | |
| "generative": { | |
| "exact_match": 0.0033, | |
| "hybrid_score": 0.2509 | |
| }, | |
| "loglikelihood": { | |
| "acc": 0.49, | |
| "acc_norm": 0.5033, | |
| "soft_score": 0.49, | |
| "soft_score_norm": 0.5033 | |
| } | |
| }, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 18, | |
| "score": 0.4444, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.4444 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 18, | |
| "score": 0.5556, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5556 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 18, | |
| "score": 0.5556, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5556 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 18, | |
| "score": 0.8333, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.8333 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 18, | |
| "score": 0.8333, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.8333 | |
| }, | |
| "Language-comprehension": { | |
| "n": 18, | |
| "score": 0.6667, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.6667 | |
| }, | |
| "Language-structure": { | |
| "n": 18, | |
| "score": 0.6667, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.6667 | |
| }, | |
| "Language-transformation": { | |
| "n": 18, | |
| "score": 0.7222, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.7222 | |
| }, | |
| "Logic-consistency": { | |
| "n": 18, | |
| "score": 0.1111, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1111 | |
| }, | |
| "Logic-deduction": { | |
| "n": 18, | |
| "score": 0.6111, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.6111 | |
| }, | |
| "Logic-pattern": { | |
| "n": 18, | |
| "score": 0.4444, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.4444 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 17, | |
| "score": 0.2353, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2353 | |
| }, | |
| "Math-pattern": { | |
| "n": 17, | |
| "score": 0.5882, | |
| "exact_match": 0.0588, | |
| "acc_norm": 0.5882 | |
| }, | |
| "Math-reasoning": { | |
| "n": 17, | |
| "score": 0.1176, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1176 | |
| }, | |
| "Pattern-generation": { | |
| "n": 17, | |
| "score": 0.4118, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.4118 | |
| }, | |
| "Pattern-matching": { | |
| "n": 17, | |
| "score": 0.4706, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.4706 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 17, | |
| "score": 0.2353, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2353 | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id": "qwen-qwen2.5-1.5b-instruct", | |
| "name": "Qwen/Qwen2.5-1.5B-Instruct", | |
| "org": "Qwen", | |
| "params_b": 1.5, | |
| "license": "Apache 2.0", | |
| "architecture": null, | |
| "url": "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct", | |
| "model_revision": "989aa7980e4cf806f80c7fef2b1adb7bc71aa306", | |
| "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", | |
| "runs": { | |
| "bench-effortless-6-2026": { | |
| "score": 0.7458, | |
| "n": 240, | |
| "notes": "Exact-match, normalized.", | |
| "categories": { | |
| "Commonsense-reasoning": { | |
| "n": 37, | |
| "exact_match": 0.5405 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 42, | |
| "exact_match": 0.5714 | |
| }, | |
| "Language-comprehension": { | |
| "n": 42, | |
| "exact_match": 0.6667 | |
| }, | |
| "Logic-deduction": { | |
| "n": 42, | |
| "exact_match": 0.881 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 40, | |
| "exact_match": 0.925 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 37, | |
| "exact_match": 0.8919 | |
| } | |
| } | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": 0.7463, | |
| "n": 238, | |
| "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 10, | |
| "hybrid_score": 0.8323 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "hybrid_score": 0.8156 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "hybrid_score": 0.7773 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 33, | |
| "hybrid_score": 0.7879 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 38, | |
| "hybrid_score": 0.8114 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "hybrid_score": 0.7737 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "hybrid_score": 0.7059 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "hybrid_score": 0.8567 | |
| }, | |
| "Logic-consistency": { | |
| "n": 10, | |
| "hybrid_score": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 15, | |
| "hybrid_score": 0.4667 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "hybrid_score": 0.5 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 33, | |
| "hybrid_score": 0.9697 | |
| }, | |
| "Math-pattern": { | |
| "n": 14, | |
| "hybrid_score": 0.8571 | |
| }, | |
| "Math-reasoning": { | |
| "n": 15, | |
| "hybrid_score": 0.8786 | |
| }, | |
| "Pattern-matching": { | |
| "n": 10, | |
| "hybrid_score": 0.4 | |
| } | |
| } | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": 0.7238, | |
| "n": 143, | |
| "acc": 0.6853, | |
| "acc_norm": 0.7203, | |
| "soft_score": 0.6951, | |
| "soft_score_norm": 0.7238, | |
| "stderr": 0.0373, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 5, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 10, | |
| "acc": 1.0, | |
| "acc_norm": 0.8 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 10, | |
| "acc": 0.7, | |
| "acc_norm": 0.9 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 7, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 10, | |
| "acc": 0.8, | |
| "acc_norm": 1.0 | |
| }, | |
| "Language-comprehension": { | |
| "n": 10, | |
| "acc": 0.7, | |
| "acc_norm": 0.7 | |
| }, | |
| "Language-structure": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.6 | |
| }, | |
| "Language-transformation": { | |
| "n": 10, | |
| "acc": 0.5, | |
| "acc_norm": 0.6 | |
| }, | |
| "Logic-consistency": { | |
| "n": 5, | |
| "acc": 0.0, | |
| "acc_norm": 0.2 | |
| }, | |
| "Logic-deduction": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.5 | |
| }, | |
| "Logic-pattern": { | |
| "n": 10, | |
| "acc": 0.3, | |
| "acc_norm": 0.2 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 8, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Math-pattern": { | |
| "n": 7, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Math-reasoning": { | |
| "n": 10, | |
| "acc": 0.7, | |
| "acc_norm": 0.6 | |
| }, | |
| "Pattern-generation": { | |
| "n": 4, | |
| "acc": 0.75, | |
| "acc_norm": 0.75 | |
| }, | |
| "Pattern-matching": { | |
| "n": 8, | |
| "acc": 1.0, | |
| "acc_norm": 1.0 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 9, | |
| "acc": 0.5556, | |
| "acc_norm": 0.5556 | |
| } | |
| } | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| } | |
| } | |
| }, | |
| { | |
| "id": "atomixlabs-photon-1.0-1m", | |
| "name": "AtomixLabs/Photon-1.0-1M", | |
| "org": "AtomixLabs", | |
| "params_b": 0.001, | |
| "license": "apache-2.0", | |
| "architecture": "LlamaForCausalLM", | |
| "url": "https://huggingface.co/AtomixLabs/Photon-1.0-1M", | |
| "model_revision": "a03eab42a7debdc27b797f8f5904a689720d0de4", | |
| "script_sha256": "955411f9971c4c26a6eeb3aa43b1694fe66ac430a139011faa915543746d2921", | |
| "runs": { | |
| "bench-effortless-7-2026": { | |
| "score": 0.0, | |
| "n": 300, | |
| "stderr": 0.0, | |
| "metrics": { | |
| "generative": { | |
| "exact_match": 0.0, | |
| "hybrid_score": 0.1942 | |
| }, | |
| "loglikelihood": { | |
| "acc": 0.33, | |
| "acc_norm": 0.2733, | |
| "soft_score": 0.33, | |
| "soft_score_norm": 0.2733 | |
| } | |
| }, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1111 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.4444 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2778 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1111 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.3889 | |
| }, | |
| "Language-comprehension": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5556 | |
| }, | |
| "Language-structure": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1667 | |
| }, | |
| "Language-transformation": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.3889 | |
| }, | |
| "Logic-consistency": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2778 | |
| }, | |
| "Logic-pattern": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.0556 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.3529 | |
| }, | |
| "Math-pattern": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1176 | |
| }, | |
| "Math-reasoning": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1765 | |
| }, | |
| "Pattern-generation": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.6471 | |
| }, | |
| "Pattern-matching": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2941 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2941 | |
| } | |
| } | |
| }, | |
| "bench-easy-7-2026": { | |
| "score": 0.203, | |
| "n": 300, | |
| "stderr": 0.0158, | |
| "metrics": { | |
| "generative": { | |
| "exact_match": 0.0, | |
| "hybrid_score": 0.203 | |
| }, | |
| "loglikelihood": { | |
| "acc": 0.2833, | |
| "acc_norm": 0.31, | |
| "soft_score": 0.2833, | |
| "soft_score_norm": 0.31 | |
| } | |
| }, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 18, | |
| "score": 0.5875, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2222 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 18, | |
| "score": 0.5981, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.6111 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 18, | |
| "score": 0.6044, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1667 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2222 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 18, | |
| "score": 0.6303, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.3889 | |
| }, | |
| "Language-comprehension": { | |
| "n": 18, | |
| "score": 0.5766, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5556 | |
| }, | |
| "Language-structure": { | |
| "n": 18, | |
| "score": 0.0829, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2778 | |
| }, | |
| "Language-transformation": { | |
| "n": 18, | |
| "score": 0.303, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5 | |
| }, | |
| "Logic-consistency": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.3333 | |
| }, | |
| "Logic-pattern": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2778 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.3529 | |
| }, | |
| "Math-pattern": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2353 | |
| }, | |
| "Math-reasoning": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1765 | |
| }, | |
| "Pattern-generation": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5294 | |
| }, | |
| "Pattern-matching": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1765 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 17, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2353 | |
| } | |
| } | |
| }, | |
| "bench-mid-7-2026": { | |
| "score": 0.2867, | |
| "n": 300, | |
| "stderr": 0.0262, | |
| "metrics": { | |
| "generative": { | |
| "exact_match": 0.0, | |
| "hybrid_score": 0.205 | |
| }, | |
| "loglikelihood": { | |
| "acc": 0.1933, | |
| "acc_norm": 0.2867, | |
| "soft_score": 0.1933, | |
| "soft_score_norm": 0.2867 | |
| } | |
| }, | |
| "categories": { | |
| "Commonsense-causality": { | |
| "n": 18, | |
| "score": 0.2778, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2778 | |
| }, | |
| "Commonsense-reasoning": { | |
| "n": 18, | |
| "score": 0.4444, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.4444 | |
| }, | |
| "Commonsense-simulation": { | |
| "n": 18, | |
| "score": 0.2778, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2778 | |
| }, | |
| "Knowledge-basic": { | |
| "n": 18, | |
| "score": 0.2778, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2778 | |
| }, | |
| "Knowledge-definitions": { | |
| "n": 18, | |
| "score": 0.4444, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.4444 | |
| }, | |
| "Language-comprehension": { | |
| "n": 18, | |
| "score": 0.3889, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.3889 | |
| }, | |
| "Language-structure": { | |
| "n": 18, | |
| "score": 0.2778, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2778 | |
| }, | |
| "Language-transformation": { | |
| "n": 18, | |
| "score": 0.1667, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1667 | |
| }, | |
| "Logic-consistency": { | |
| "n": 18, | |
| "score": 0.0, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.0 | |
| }, | |
| "Logic-deduction": { | |
| "n": 18, | |
| "score": 0.5, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5 | |
| }, | |
| "Logic-pattern": { | |
| "n": 18, | |
| "score": 0.2222, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.2222 | |
| }, | |
| "Math-arithmetic": { | |
| "n": 17, | |
| "score": 0.0588, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.0588 | |
| }, | |
| "Math-pattern": { | |
| "n": 17, | |
| "score": 0.1176, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.1176 | |
| }, | |
| "Math-reasoning": { | |
| "n": 17, | |
| "score": 0.0588, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.0588 | |
| }, | |
| "Pattern-generation": { | |
| "n": 17, | |
| "score": 0.5882, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.5882 | |
| }, | |
| "Pattern-matching": { | |
| "n": 17, | |
| "score": 0.4118, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.4118 | |
| }, | |
| "Pattern-recognition": { | |
| "n": 17, | |
| "score": 0.3529, | |
| "exact_match": 0.0, | |
| "acc_norm": 0.3529 | |
| } | |
| } | |
| }, | |
| "bench-effortless-6-2026": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| }, | |
| "bench-easy-6-2026": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| }, | |
| "bench-mid-6-2026": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| }, | |
| "bench-AGI": { | |
| "score": null, | |
| "n": null, | |
| "notes": "Not yet evaluated on this tier." | |
| } | |
| } | |
| } | |
| ], | |
| "t2i_benchmarks": [ | |
| { | |
| "id": "t2i-fid-coco", | |
| "label": "FID", | |
| "kind": "text-to-image", | |
| "metric": "fid", | |
| "lower_is_better": true, | |
| "description": "Frechet Inception Distance, self-reported per model card. Lower is better; measures distributional realism of generated images against real reference photos." | |
| }, | |
| { | |
| "id": "t2i-clip-coco", | |
| "label": "CLIP Score", | |
| "kind": "text-to-image", | |
| "metric": "clip_score", | |
| "lower_is_better": false, | |
| "description": "CLIP Score, self-reported per model card, 0-1 scale. Higher is better; measures text-image alignment." | |
| } | |
| ], | |
| "t2i_models": [ | |
| { | |
| "id": "bench-labs-pixelmodel-v4", | |
| "name": "PixelModel v4", | |
| "org": "Bench Labs", | |
| "params_b": 0.0401, | |
| "url": "https://huggingface.co/bench-labs/pixelmodel-v4", | |
| "resolution": "256x256", | |
| "release_date": "2026-07-25", | |
| "runs": { | |
| "t2i-fid-coco": { | |
| "score": 39.54, | |
| "n": null | |
| }, | |
| "t2i-clip-coco": { | |
| "score": 0.2804, | |
| "n": null | |
| } | |
| } | |
| }, | |
| { | |
| "id": "bench-labs-pixelmodel-v2", | |
| "name": "PixelModel v2", | |
| "org": "Bench Labs", | |
| "params_b": 0.0002, | |
| "url": "https://huggingface.co/bench-labs/pixelmodel-v2", | |
| "resolution": "64x64", | |
| "release_date": "2026-07-21", | |
| "runs": { | |
| "t2i-fid-coco": { | |
| "score": 390.68, | |
| "n": null | |
| }, | |
| "t2i-clip-coco": { | |
| "score": 0.2048, | |
| "n": null | |
| } | |
| } | |
| }, | |
| { | |
| "id": "bench-labs-pixelmodel-v3", | |
| "name": "PixelModel v3", | |
| "org": "Bench Labs", | |
| "params_b": 0.000919, | |
| "url": "https://huggingface.co/bench-labs/pixelmodel-v3", | |
| "resolution": "128x128", | |
| "release_date": "2026-07-24", | |
| "runs": { | |
| "t2i-fid-coco": { | |
| "score": 383.91, | |
| "n": null | |
| }, | |
| "t2i-clip-coco": { | |
| "score": 0.2073, | |
| "n": null | |
| } | |
| } | |
| }, | |
| { | |
| "id": "bench-labs-pixelmodel-v1", | |
| "name": "PixelModel v1", | |
| "org": "Bench Labs", | |
| "params_b": 2.37e-05, | |
| "url": "https://huggingface.co/bench-labs/pixelmodel-v1", | |
| "resolution": "64x64", | |
| "release_date": "2026-07-17", | |
| "runs": { | |
| "t2i-fid-coco": { | |
| "score": 420.75, | |
| "n": null | |
| }, | |
| "t2i-clip-coco": { | |
| "score": 0.201, | |
| "n": null | |
| } | |
| } | |
| }, | |
| { | |
| "id": "flamef0x-nushelld", | |
| "name": "nuShellD", | |
| "org": "FlameF0X", | |
| "params_b": 0.127, | |
| "url": "https://huggingface.co/FlameF0X/nuShellD", | |
| "resolution": "256x256", | |
| "release_date": "2026-07-16", | |
| "runs": { | |
| "t2i-fid-coco": { | |
| "score": 83.9199, | |
| "n": null | |
| }, | |
| "t2i-clip-coco": { | |
| "score": 0.22, | |
| "n": null | |
| } | |
| }, | |
| "space_url": "https://flamef0x-shelld-demo.hf.space/" | |
| }, | |
| { | |
| "id": "flamef0x-shelld-2026-07-16", | |
| "name": "ShellD (2026-07-16)", | |
| "org": "FlameF0X", | |
| "params_b": 0.0669, | |
| "url": "https://huggingface.co/FlameF0X/ShellD", | |
| "resolution": "256x256", | |
| "release_date": "2026-07-14", | |
| "runs": { | |
| "t2i-fid-coco": { | |
| "score": 4.8724, | |
| "n": null, | |
| "notes": "Label suggests a 07-16 release but releaseDate on this entry is 07-14, same as the other ShellD row below -- verify against the source before treating this FID as final." | |
| }, | |
| "t2i-clip-coco": { | |
| "score": 0.24, | |
| "n": null | |
| } | |
| } | |
| }, | |
| { | |
| "id": "flamef0x-shelld-2026-07-14", | |
| "name": "ShellD (2026-07-14)", | |
| "org": "FlameF0X", | |
| "params_b": 0.0669, | |
| "url": "https://huggingface.co/FlameF0X/ShellD", | |
| "resolution": "256x256", | |
| "release_date": "2026-07-14", | |
| "runs": { | |
| "t2i-fid-coco": { | |
| "score": 16.0159, | |
| "n": null, | |
| "notes": "Same name, org, params, and releaseDate as the other ShellD row above but a different FID -- likely two snapshots of the same repo; double check dates/commits before publishing both." | |
| }, | |
| "t2i-clip-coco": { | |
| "score": 0.21, | |
| "n": null | |
| } | |
| } | |
| }, | |
| { | |
| "id": "bench-labs-pixelmodel-v0", | |
| "name": "PixelModel v0", | |
| "org": "Bench Labs", | |
| "params_b": 0.000203, | |
| "url": "https://huggingface.co/bench-labs/pixelmodel", | |
| "resolution": "32x32", | |
| "release_date": "2026-07-14", | |
| "runs": { | |
| "t2i-fid-coco": { | |
| "score": 566.84, | |
| "n": null | |
| }, | |
| "t2i-clip-coco": { | |
| "score": 0.186, | |
| "n": null | |
| } | |
| } | |
| } | |
| ] | |
| } |