{ "updated": "2026-07-29", "latest_generation": "7-2026", "benchmarks": [ { "id": "bench-effortless-7-2026", "label": "Effortless", "tier": 1, "generation": "7-2026", "rows": 300, "metric": "exact_match", "dual_mode": true, "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-effortless-7-2026", "description": "Sanity-layer benchmark, v2 dual-mode schema: every item scored by generation AND log-likelihood. Balanced across all 17 categories." }, { "id": "bench-easy-7-2026", "label": "Easy", "tier": 2, "generation": "7-2026", "rows": 300, "metric": "hybrid_score", "dual_mode": true, "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-easy-7-2026", "description": "Easy-tier QA, v2 dual-mode schema with per-item scorer routing (strict / semantic / fuzzy). Balanced across all 17 categories." }, { "id": "bench-mid-7-2026", "label": "Mid", "tier": 3, "generation": "7-2026", "rows": 300, "metric": "soft_score_norm", "dual_mode": true, "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-mid-7-2026", "description": "Mid-tier QA, v2 dual-mode schema; headline is log-likelihood soft_score_norm, generation metrics also reported. Balanced across all 17 categories." }, { "id": "bench-effortless-6-2026", "label": "Effortless (6-2026)", "tier": 1, "generation": "6-2026", "rows": 240, "metric": "exact_match", "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-effortless-6-2026", "description": "Sanity-layer benchmark. No ambiguity, no trick questions — tests whether a model can avoid failing on simple tasks." }, { "id": "bench-easy-6-2026", "label": "Easy (6-2026)", "tier": 2, "generation": "6-2026", "rows": 238, "metric": "hybrid_score", "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-easy-6-2026", "description": "Effortless-to-Easy tier QA. Structured language understanding and controlled generalization." }, { "id": "bench-mid-6-2026", "label": "Mid (6-2026)", "tier": 3, "generation": "6-2026", "rows": 143, "metric": "soft_score_norm", "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-mid-6-2026", "description": "Easy-to-Mid tier multiple-choice QA scored via lm-eval loglikelihood, target_scores dict." }, { "id": "bench-AGI", "label": "AGI", "tier": 4, "generation": "6-2026", "rows": null, "metric": "rank_order", "dataset_url": "https://huggingface.co/datasets/bench-labs/bench-AGI", "description": "Hard, open-ended questions with human-argued reasoning traces, graded blind by a multi-vendor panel. Rank-order evaluation, not absolute scoring.", "unavailable": true, "unavailable_reason": "Under maintenance. Scoring pipeline is being reworked — see the dataset README." } ], "categories": [ "Knowledge-basic", "Knowledge-definitions", "Math-arithmetic", "Math-pattern", "Math-reasoning", "Logic-deduction", "Logic-pattern", "Logic-consistency", "Language-comprehension", "Language-transformation", "Language-structure", "Commonsense-reasoning", "Commonsense-causality", "Commonsense-simulation", "Pattern-matching", "Pattern-recognition", "Pattern-generation" ], "models": [ { "id": "liquidai-lfm2.5-1.2b-base", "name": "LiquidAI/LFM2.5-1.2B-Base", "org": "LiquidAI", "params_b": null, "license": null, "architecture": null, "url": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Base", "model_revision": "f6a5d174bc3e52bd0df245d69133f9930b4828d8", "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", "runs": { "bench-effortless-6-2026": { "score": 0.0, "n": 240, "notes": "Exact-match, normalized.", "categories": { "Commonsense-reasoning": { "n": 37, "exact_match": 0.0 }, "Knowledge-basic": { "n": 42, "exact_match": 0.0 }, "Language-comprehension": { "n": 42, "exact_match": 0.0 }, "Logic-deduction": { "n": 42, "exact_match": 0.0 }, "Math-arithmetic": { "n": 40, "exact_match": 0.0 }, "Pattern-recognition": { "n": 37, "exact_match": 0.0 } } }, "bench-easy-6-2026": { "score": 0.3087, "n": 238, "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", "categories": { "Commonsense-causality": { "n": 10, "hybrid_score": 0.7338 }, "Commonsense-reasoning": { "n": 10, "hybrid_score": 0.7347 }, "Commonsense-simulation": { "n": 10, "hybrid_score": 0.742 }, "Knowledge-basic": { "n": 33, "hybrid_score": 0.0303 }, "Knowledge-definitions": { "n": 38, "hybrid_score": 0.8057 }, "Language-comprehension": { "n": 10, "hybrid_score": 0.7448 }, "Language-structure": { "n": 10, "hybrid_score": 0.2311 }, "Language-transformation": { "n": 10, "hybrid_score": 0.7685 }, "Logic-consistency": { "n": 10, "hybrid_score": 0.0 }, "Logic-deduction": { "n": 15, "hybrid_score": 0.0667 }, "Logic-pattern": { "n": 10, "hybrid_score": 0.0 }, "Math-arithmetic": { "n": 33, "hybrid_score": 0.0 }, "Math-pattern": { "n": 14, "hybrid_score": 0.0 }, "Math-reasoning": { "n": 15, "hybrid_score": 0.0868 }, "Pattern-matching": { "n": 10, "hybrid_score": 0.0 } } }, "bench-mid-6-2026": { "score": 0.621, "n": 143, "acc": 0.5385, "acc_norm": 0.6154, "soft_score": 0.5455, "soft_score_norm": 0.621, "stderr": 0.0404, "categories": { "Commonsense-causality": { "n": 5, "acc": 0.6, "acc_norm": 0.6 }, "Commonsense-reasoning": { "n": 10, "acc": 0.6, "acc_norm": 0.5 }, "Commonsense-simulation": { "n": 10, "acc": 0.4, "acc_norm": 0.7 }, "Knowledge-basic": { "n": 7, "acc": 0.5714, "acc_norm": 0.5714 }, "Knowledge-definitions": { "n": 10, "acc": 0.3, "acc_norm": 0.9 }, "Language-comprehension": { "n": 10, "acc": 0.6, "acc_norm": 0.7 }, "Language-structure": { "n": 10, "acc": 0.5, "acc_norm": 0.4 }, "Language-transformation": { "n": 10, "acc": 0.5, "acc_norm": 0.7 }, "Logic-consistency": { "n": 5, "acc": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 10, "acc": 0.3, "acc_norm": 0.3 }, "Logic-pattern": { "n": 10, "acc": 0.5, "acc_norm": 0.6 }, "Math-arithmetic": { "n": 8, "acc": 0.875, "acc_norm": 0.875 }, "Math-pattern": { "n": 7, "acc": 0.8571, "acc_norm": 0.8571 }, "Math-reasoning": { "n": 10, "acc": 0.4, "acc_norm": 0.3 }, "Pattern-generation": { "n": 4, "acc": 1.0, "acc_norm": 1.0 }, "Pattern-matching": { "n": 8, "acc": 1.0, "acc_norm": 0.875 }, "Pattern-recognition": { "n": 9, "acc": 0.4444, "acc_norm": 0.6667 } } }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } }, { "id": "liquidai-lfm2.5-1.2b-instruct", "name": "LiquidAI/LFM2.5-1.2B-Instruct", "org": "LiquidAI", "params_b": null, "license": null, "architecture": null, "url": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct", "model_revision": "868df74dd56ff8a0c2ac5dbf281690c2dbebe4c9", "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", "runs": { "bench-effortless-6-2026": { "score": 0.1208, "n": 240, "notes": "Exact-match, normalized.", "categories": { "Commonsense-reasoning": { "n": 37, "exact_match": 0.0 }, "Knowledge-basic": { "n": 42, "exact_match": 0.119 }, "Language-comprehension": { "n": 42, "exact_match": 0.0 }, "Logic-deduction": { "n": 42, "exact_match": 0.0952 }, "Math-arithmetic": { "n": 40, "exact_match": 0.475 }, "Pattern-recognition": { "n": 37, "exact_match": 0.027 } } }, "bench-easy-6-2026": { "score": 0.441, "n": 238, "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", "categories": { "Commonsense-causality": { "n": 10, "hybrid_score": 0.7508 }, "Commonsense-reasoning": { "n": 10, "hybrid_score": 0.7711 }, "Commonsense-simulation": { "n": 10, "hybrid_score": 0.7765 }, "Knowledge-basic": { "n": 33, "hybrid_score": 0.303 }, "Knowledge-definitions": { "n": 38, "hybrid_score": 0.8172 }, "Language-comprehension": { "n": 10, "hybrid_score": 0.7459 }, "Language-structure": { "n": 10, "hybrid_score": 0.4632 }, "Language-transformation": { "n": 10, "hybrid_score": 0.7849 }, "Logic-consistency": { "n": 10, "hybrid_score": 0.0 }, "Logic-deduction": { "n": 15, "hybrid_score": 0.0667 }, "Logic-pattern": { "n": 10, "hybrid_score": 0.2 }, "Math-arithmetic": { "n": 33, "hybrid_score": 0.2727 }, "Math-pattern": { "n": 14, "hybrid_score": 0.0 }, "Math-reasoning": { "n": 15, "hybrid_score": 0.4654 }, "Pattern-matching": { "n": 10, "hybrid_score": 0.2 } } }, "bench-mid-6-2026": { "score": 0.607, "n": 143, "acc": 0.5175, "acc_norm": 0.6014, "soft_score": 0.5231, "soft_score_norm": 0.607, "stderr": 0.0406, "categories": { "Commonsense-causality": { "n": 5, "acc": 0.8, "acc_norm": 0.8 }, "Commonsense-reasoning": { "n": 10, "acc": 0.6, "acc_norm": 0.5 }, "Commonsense-simulation": { "n": 10, "acc": 0.3, "acc_norm": 0.6 }, "Knowledge-basic": { "n": 7, "acc": 1.0, "acc_norm": 1.0 }, "Knowledge-definitions": { "n": 10, "acc": 0.4, "acc_norm": 0.9 }, "Language-comprehension": { "n": 10, "acc": 0.6, "acc_norm": 0.6 }, "Language-structure": { "n": 10, "acc": 0.2, "acc_norm": 0.3 }, "Language-transformation": { "n": 10, "acc": 0.6, "acc_norm": 0.6 }, "Logic-consistency": { "n": 5, "acc": 0.0, "acc_norm": 0.2 }, "Logic-deduction": { "n": 10, "acc": 0.0, "acc_norm": 0.3 }, "Logic-pattern": { "n": 10, "acc": 0.4, "acc_norm": 0.3 }, "Math-arithmetic": { "n": 8, "acc": 1.0, "acc_norm": 1.0 }, "Math-pattern": { "n": 7, "acc": 1.0, "acc_norm": 1.0 }, "Math-reasoning": { "n": 10, "acc": 0.3, "acc_norm": 0.3 }, "Pattern-generation": { "n": 4, "acc": 0.75, "acc_norm": 0.75 }, "Pattern-matching": { "n": 8, "acc": 0.75, "acc_norm": 0.75 }, "Pattern-recognition": { "n": 9, "acc": 0.5556, "acc_norm": 0.6667 } } }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } }, { "id": "liquidai-lfm2.5-230m-base", "name": "LiquidAI/LFM2.5-230M-Base", "org": "LiquidAI", "params_b": null, "license": null, "architecture": null, "url": "https://huggingface.co/LiquidAI/LFM2.5-230M-Base", "model_revision": "9d2be5519834990d30996f878b6771cccbd24f2c", "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", "runs": { "bench-effortless-6-2026": { "score": 0.0042, "n": 240, "notes": "Exact-match, normalized.", "categories": { "Commonsense-reasoning": { "n": 37, "exact_match": 0.0 }, "Knowledge-basic": { "n": 42, "exact_match": 0.0 }, "Language-comprehension": { "n": 42, "exact_match": 0.0 }, "Logic-deduction": { "n": 42, "exact_match": 0.0 }, "Math-arithmetic": { "n": 40, "exact_match": 0.0 }, "Pattern-recognition": { "n": 37, "exact_match": 0.027 } } }, "bench-easy-6-2026": { "score": 0.2876, "n": 238, "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", "categories": { "Commonsense-causality": { "n": 10, "hybrid_score": 0.7301 }, "Commonsense-reasoning": { "n": 10, "hybrid_score": 0.7137 }, "Commonsense-simulation": { "n": 10, "hybrid_score": 0.7434 }, "Knowledge-basic": { "n": 33, "hybrid_score": 0.0 }, "Knowledge-definitions": { "n": 38, "hybrid_score": 0.7888 }, "Language-comprehension": { "n": 10, "hybrid_score": 0.7182 }, "Language-structure": { "n": 10, "hybrid_score": 0.2974 }, "Language-transformation": { "n": 10, "hybrid_score": 0.6079 }, "Logic-consistency": { "n": 10, "hybrid_score": 0.0 }, "Logic-deduction": { "n": 15, "hybrid_score": 0.0 }, "Logic-pattern": { "n": 10, "hybrid_score": 0.0 }, "Math-arithmetic": { "n": 33, "hybrid_score": 0.0 }, "Math-pattern": { "n": 14, "hybrid_score": 0.0 }, "Math-reasoning": { "n": 15, "hybrid_score": 0.0247 }, "Pattern-matching": { "n": 10, "hybrid_score": 0.0 } } }, "bench-mid-6-2026": { "score": 0.5301, "n": 143, "acc": 0.4266, "acc_norm": 0.5245, "soft_score": 0.4357, "soft_score_norm": 0.5301, "stderr": 0.0415, "categories": { "Commonsense-causality": { "n": 5, "acc": 0.8, "acc_norm": 0.6 }, "Commonsense-reasoning": { "n": 10, "acc": 0.5, "acc_norm": 0.4 }, "Commonsense-simulation": { "n": 10, "acc": 0.3, "acc_norm": 0.4 }, "Knowledge-basic": { "n": 7, "acc": 0.5714, "acc_norm": 0.8571 }, "Knowledge-definitions": { "n": 10, "acc": 0.1, "acc_norm": 0.7 }, "Language-comprehension": { "n": 10, "acc": 0.5, "acc_norm": 0.8 }, "Language-structure": { "n": 10, "acc": 0.1, "acc_norm": 0.3 }, "Language-transformation": { "n": 10, "acc": 0.2, "acc_norm": 0.4 }, "Logic-consistency": { "n": 5, "acc": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 10, "acc": 0.6, "acc_norm": 0.7 }, "Logic-pattern": { "n": 10, "acc": 0.3, "acc_norm": 0.4 }, "Math-arithmetic": { "n": 8, "acc": 0.875, "acc_norm": 0.875 }, "Math-pattern": { "n": 7, "acc": 0.8571, "acc_norm": 0.8571 }, "Math-reasoning": { "n": 10, "acc": 0.4, "acc_norm": 0.3 }, "Pattern-generation": { "n": 4, "acc": 0.75, "acc_norm": 0.5 }, "Pattern-matching": { "n": 8, "acc": 0.5, "acc_norm": 0.5 }, "Pattern-recognition": { "n": 9, "acc": 0.3333, "acc_norm": 0.3333 } } }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } }, { "id": "liquidai-lfm2.5-350m", "name": "LiquidAI/LFM2.5-350M", "org": "LiquidAI", "params_b": null, "license": null, "architecture": null, "url": "https://huggingface.co/LiquidAI/LFM2.5-350M", "model_revision": "b9d6e4e2d75f440b12a2b4d731c808004ecbbd89", "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", "runs": { "bench-effortless-6-2026": { "score": 0.1375, "n": 240, "notes": "Exact-match, normalized.", "categories": { "Commonsense-reasoning": { "n": 37, "exact_match": 0.027 }, "Knowledge-basic": { "n": 42, "exact_match": 0.0952 }, "Language-comprehension": { "n": 42, "exact_match": 0.2143 }, "Logic-deduction": { "n": 42, "exact_match": 0.4286 }, "Math-arithmetic": { "n": 40, "exact_match": 0.0 }, "Pattern-recognition": { "n": 37, "exact_match": 0.027 } } }, "bench-easy-6-2026": { "score": 0.358, "n": 238, "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", "categories": { "Commonsense-causality": { "n": 10, "hybrid_score": 0.721 }, "Commonsense-reasoning": { "n": 10, "hybrid_score": 0.7313 }, "Commonsense-simulation": { "n": 10, "hybrid_score": 0.75 }, "Knowledge-basic": { "n": 33, "hybrid_score": 0.2424 }, "Knowledge-definitions": { "n": 38, "hybrid_score": 0.7996 }, "Language-comprehension": { "n": 10, "hybrid_score": 0.741 }, "Language-structure": { "n": 10, "hybrid_score": 0.3121 }, "Language-transformation": { "n": 10, "hybrid_score": 0.7689 }, "Logic-consistency": { "n": 10, "hybrid_score": 0.1 }, "Logic-deduction": { "n": 15, "hybrid_score": 0.2 }, "Logic-pattern": { "n": 10, "hybrid_score": 0.0 }, "Math-arithmetic": { "n": 33, "hybrid_score": 0.0 }, "Math-pattern": { "n": 14, "hybrid_score": 0.0 }, "Math-reasoning": { "n": 15, "hybrid_score": 0.1049 }, "Pattern-matching": { "n": 10, "hybrid_score": 0.1 } } }, "bench-mid-6-2026": { "score": 0.5105, "n": 143, "acc": 0.4755, "acc_norm": 0.5035, "soft_score": 0.4825, "soft_score_norm": 0.5105, "stderr": 0.0417, "categories": { "Commonsense-causality": { "n": 5, "acc": 0.8, "acc_norm": 0.8 }, "Commonsense-reasoning": { "n": 10, "acc": 0.5, "acc_norm": 0.6 }, "Commonsense-simulation": { "n": 10, "acc": 0.4, "acc_norm": 0.5 }, "Knowledge-basic": { "n": 7, "acc": 0.8571, "acc_norm": 0.8571 }, "Knowledge-definitions": { "n": 10, "acc": 0.6, "acc_norm": 0.8 }, "Language-comprehension": { "n": 10, "acc": 0.4, "acc_norm": 0.8 }, "Language-structure": { "n": 10, "acc": 0.2, "acc_norm": 0.1 }, "Language-transformation": { "n": 10, "acc": 0.2, "acc_norm": 0.4 }, "Logic-consistency": { "n": 5, "acc": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 10, "acc": 0.3, "acc_norm": 0.4 }, "Logic-pattern": { "n": 10, "acc": 0.4, "acc_norm": 0.3 }, "Math-arithmetic": { "n": 8, "acc": 0.875, "acc_norm": 0.875 }, "Math-pattern": { "n": 7, "acc": 0.7143, "acc_norm": 0.7143 }, "Math-reasoning": { "n": 10, "acc": 0.2, "acc_norm": 0.1 }, "Pattern-generation": { "n": 4, "acc": 0.75, "acc_norm": 0.5 }, "Pattern-matching": { "n": 8, "acc": 0.75, "acc_norm": 0.375 }, "Pattern-recognition": { "n": 9, "acc": 0.5556, "acc_norm": 0.5556 } } }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } }, { "id": "liquidai-lfm2.5-350m-base", "name": "LiquidAI/LFM2.5-350M-Base", "org": "LiquidAI", "params_b": null, "license": null, "architecture": null, "url": "https://huggingface.co/LiquidAI/LFM2.5-350M-Base", "model_revision": "9960764e30892e01f29a6dc23df2533fcd8bd5ae", "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", "runs": { "bench-effortless-6-2026": { "score": 0.0, "n": 240, "notes": "Exact-match, normalized.", "categories": { "Commonsense-reasoning": { "n": 37, "exact_match": 0.0 }, "Knowledge-basic": { "n": 42, "exact_match": 0.0 }, "Language-comprehension": { "n": 42, "exact_match": 0.0 }, "Logic-deduction": { "n": 42, "exact_match": 0.0 }, "Math-arithmetic": { "n": 40, "exact_match": 0.0 }, "Pattern-recognition": { "n": 37, "exact_match": 0.0 } } }, "bench-easy-6-2026": { "score": 0.2788, "n": 238, "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", "categories": { "Commonsense-causality": { "n": 10, "hybrid_score": 0.7048 }, "Commonsense-reasoning": { "n": 10, "hybrid_score": 0.6998 }, "Commonsense-simulation": { "n": 10, "hybrid_score": 0.744 }, "Knowledge-basic": { "n": 33, "hybrid_score": 0.0 }, "Knowledge-definitions": { "n": 38, "hybrid_score": 0.7958 }, "Language-comprehension": { "n": 10, "hybrid_score": 0.7201 }, "Language-structure": { "n": 10, "hybrid_score": 0.146 }, "Language-transformation": { "n": 10, "hybrid_score": 0.5558 }, "Logic-consistency": { "n": 10, "hybrid_score": 0.0 }, "Logic-deduction": { "n": 15, "hybrid_score": 0.0 }, "Logic-pattern": { "n": 10, "hybrid_score": 0.0 }, "Math-arithmetic": { "n": 33, "hybrid_score": 0.0 }, "Math-pattern": { "n": 14, "hybrid_score": 0.0 }, "Math-reasoning": { "n": 15, "hybrid_score": 0.0271 }, "Pattern-matching": { "n": 10, "hybrid_score": 0.0 } } }, "bench-mid-6-2026": { "score": 0.5105, "n": 143, "acc": 0.3986, "acc_norm": 0.5035, "soft_score": 0.4091, "soft_score_norm": 0.5105, "stderr": 0.0417, "categories": { "Commonsense-causality": { "n": 5, "acc": 0.8, "acc_norm": 0.6 }, "Commonsense-reasoning": { "n": 10, "acc": 0.4, "acc_norm": 0.5 }, "Commonsense-simulation": { "n": 10, "acc": 0.3, "acc_norm": 0.3 }, "Knowledge-basic": { "n": 7, "acc": 0.4286, "acc_norm": 0.5714 }, "Knowledge-definitions": { "n": 10, "acc": 0.1, "acc_norm": 0.7 }, "Language-comprehension": { "n": 10, "acc": 0.3, "acc_norm": 0.8 }, "Language-structure": { "n": 10, "acc": 0.0, "acc_norm": 0.2 }, "Language-transformation": { "n": 10, "acc": 0.5, "acc_norm": 0.7 }, "Logic-consistency": { "n": 5, "acc": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 10, "acc": 0.4, "acc_norm": 0.6 }, "Logic-pattern": { "n": 10, "acc": 0.4, "acc_norm": 0.5 }, "Math-arithmetic": { "n": 8, "acc": 0.875, "acc_norm": 0.875 }, "Math-pattern": { "n": 7, "acc": 0.7143, "acc_norm": 0.7143 }, "Math-reasoning": { "n": 10, "acc": 0.4, "acc_norm": 0.3 }, "Pattern-generation": { "n": 4, "acc": 0.75, "acc_norm": 0.5 }, "Pattern-matching": { "n": 8, "acc": 0.625, "acc_norm": 0.375 }, "Pattern-recognition": { "n": 9, "acc": 0.2222, "acc_norm": 0.2222 } } }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } }, { "id": "liquidai-lfm2.5-230m", "name": "LiquidAI/LFM2.5-230M", "org": "LiquidAI", "params_b": 0.23, "license": null, "architecture": "lfm2.5", "url": "https://huggingface.co/LiquidAI/LFM2.5-230M", "model_revision": "37b30cce3446f3f2e26a0d3f8c67c9167f5079d7", "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", "runs": { "bench-effortless-6-2026": { "score": 0.0, "n": 240, "notes": "Exact-match, normalized.", "categories": { "Commonsense-reasoning": { "n": 37, "exact_match": 0.0 }, "Knowledge-basic": { "n": 42, "exact_match": 0.0 }, "Language-comprehension": { "n": 42, "exact_match": 0.0 }, "Logic-deduction": { "n": 42, "exact_match": 0.0 }, "Math-arithmetic": { "n": 40, "exact_match": 0.0 }, "Pattern-recognition": { "n": 37, "exact_match": 0.0 } } }, "bench-easy-6-2026": { "score": 0.2953, "n": 238, "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", "categories": { "Commonsense-causality": { "n": 10, "hybrid_score": 0.7178 }, "Commonsense-reasoning": { "n": 10, "hybrid_score": 0.738 }, "Commonsense-simulation": { "n": 10, "hybrid_score": 0.7269 }, "Knowledge-basic": { "n": 33, "hybrid_score": 0.0 }, "Knowledge-definitions": { "n": 38, "hybrid_score": 0.7906 }, "Language-comprehension": { "n": 10, "hybrid_score": 0.7643 }, "Language-structure": { "n": 10, "hybrid_score": 0.302 }, "Language-transformation": { "n": 10, "hybrid_score": 0.7019 }, "Logic-consistency": { "n": 10, "hybrid_score": 0.0 }, "Logic-deduction": { "n": 15, "hybrid_score": 0.0 }, "Logic-pattern": { "n": 10, "hybrid_score": 0.0 }, "Math-arithmetic": { "n": 33, "hybrid_score": 0.0 }, "Math-pattern": { "n": 14, "hybrid_score": 0.0 }, "Math-reasoning": { "n": 15, "hybrid_score": 0.049 }, "Pattern-matching": { "n": 10, "hybrid_score": 0.0 } } }, "bench-mid-6-2026": { "score": 0.514, "n": 143, "acc": 0.4825, "acc_norm": 0.5105, "soft_score": 0.486, "soft_score_norm": 0.514, "stderr": 0.0417, "categories": { "Commonsense-causality": { "n": 5, "acc": 0.6, "acc_norm": 0.6 }, "Commonsense-reasoning": { "n": 10, "acc": 0.5, "acc_norm": 0.5 }, "Commonsense-simulation": { "n": 10, "acc": 0.3, "acc_norm": 0.4 }, "Knowledge-basic": { "n": 7, "acc": 0.8571, "acc_norm": 0.8571 }, "Knowledge-definitions": { "n": 10, "acc": 0.3, "acc_norm": 0.7 }, "Language-comprehension": { "n": 10, "acc": 0.6, "acc_norm": 0.7 }, "Language-structure": { "n": 10, "acc": 0.2, "acc_norm": 0.1 }, "Language-transformation": { "n": 10, "acc": 0.4, "acc_norm": 0.4 }, "Logic-consistency": { "n": 5, "acc": 0.2, "acc_norm": 0.0 }, "Logic-deduction": { "n": 10, "acc": 0.6, "acc_norm": 0.6 }, "Logic-pattern": { "n": 10, "acc": 0.3, "acc_norm": 0.4 }, "Math-arithmetic": { "n": 8, "acc": 0.875, "acc_norm": 0.875 }, "Math-pattern": { "n": 7, "acc": 0.5714, "acc_norm": 0.5714 }, "Math-reasoning": { "n": 10, "acc": 0.4, "acc_norm": 0.3 }, "Pattern-generation": { "n": 4, "acc": 0.75, "acc_norm": 0.5 }, "Pattern-matching": { "n": 8, "acc": 0.625, "acc_norm": 0.75 }, "Pattern-recognition": { "n": 9, "acc": 0.4444, "acc_norm": 0.4444 } } }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } }, { "id": "qwen-qwen2.5-1.5b", "name": "Qwen/Qwen2.5-1.5B", "org": "Qwen", "params_b": 1.5, "license": null, "architecture": null, "url": "https://huggingface.co/Qwen/Qwen2.5-1.5B", "model_revision": "8faed761d45a263340a0528343f099c05c9a4323", "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", "runs": { "bench-effortless-6-2026": { "score": 0.0, "n": 240, "notes": "Exact-match, normalized.", "categories": { "Commonsense-reasoning": { "n": 37, "exact_match": 0.0 }, "Knowledge-basic": { "n": 42, "exact_match": 0.0 }, "Language-comprehension": { "n": 42, "exact_match": 0.0 }, "Logic-deduction": { "n": 42, "exact_match": 0.0 }, "Math-arithmetic": { "n": 40, "exact_match": 0.0 }, "Pattern-recognition": { "n": 37, "exact_match": 0.0 } } }, "bench-easy-6-2026": { "score": 0.2563, "n": 238, "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", "categories": { "Commonsense-causality": { "n": 10, "hybrid_score": 0.5943 }, "Commonsense-reasoning": { "n": 10, "hybrid_score": 0.7341 }, "Commonsense-simulation": { "n": 10, "hybrid_score": 0.7074 }, "Knowledge-basic": { "n": 33, "hybrid_score": 0.0 }, "Knowledge-definitions": { "n": 38, "hybrid_score": 0.676 }, "Language-comprehension": { "n": 10, "hybrid_score": 0.6957 }, "Language-structure": { "n": 10, "hybrid_score": 0.2408 }, "Language-transformation": { "n": 10, "hybrid_score": 0.3879 }, "Logic-consistency": { "n": 10, "hybrid_score": 0.0 }, "Logic-deduction": { "n": 15, "hybrid_score": 0.0 }, "Logic-pattern": { "n": 10, "hybrid_score": 0.0 }, "Math-arithmetic": { "n": 33, "hybrid_score": 0.0 }, "Math-pattern": { "n": 14, "hybrid_score": 0.0 }, "Math-reasoning": { "n": 15, "hybrid_score": 0.1135 }, "Pattern-matching": { "n": 10, "hybrid_score": 0.0 } } }, "bench-mid-6-2026": { "score": 0.7517, "n": 143, "acc": 0.7483, "acc_norm": 0.7483, "soft_score": 0.7545, "soft_score_norm": 0.7517, "stderr": 0.036, "categories": { "Commonsense-causality": { "n": 5, "acc": 1.0, "acc_norm": 1.0 }, "Commonsense-reasoning": { "n": 10, "acc": 1.0, "acc_norm": 0.8 }, "Commonsense-simulation": { "n": 10, "acc": 0.6, "acc_norm": 0.8 }, "Knowledge-basic": { "n": 7, "acc": 1.0, "acc_norm": 1.0 }, "Knowledge-definitions": { "n": 10, "acc": 1.0, "acc_norm": 1.0 }, "Language-comprehension": { "n": 10, "acc": 0.7, "acc_norm": 0.9 }, "Language-structure": { "n": 10, "acc": 0.5, "acc_norm": 0.6 }, "Language-transformation": { "n": 10, "acc": 0.9, "acc_norm": 0.9 }, "Logic-consistency": { "n": 5, "acc": 0.0, "acc_norm": 0.2 }, "Logic-deduction": { "n": 10, "acc": 0.5, "acc_norm": 0.4 }, "Logic-pattern": { "n": 10, "acc": 0.4, "acc_norm": 0.3 }, "Math-arithmetic": { "n": 8, "acc": 0.875, "acc_norm": 0.875 }, "Math-pattern": { "n": 7, "acc": 1.0, "acc_norm": 1.0 }, "Math-reasoning": { "n": 10, "acc": 0.8, "acc_norm": 0.6 }, "Pattern-generation": { "n": 4, "acc": 1.0, "acc_norm": 1.0 }, "Pattern-matching": { "n": 8, "acc": 1.0, "acc_norm": 1.0 }, "Pattern-recognition": { "n": 9, "acc": 0.5556, "acc_norm": 0.5556 } } }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } }, { "id": "qwen-qwen2.5-0.5b-instruct", "name": "Qwen/Qwen2.5-0.5B-Instruct", "org": "Qwen", "params_b": 0.5, "license": null, "architecture": null, "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct", "model_revision": "7ae557604adf67be50417f59c2c2f167def9a775", "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", "runs": { "bench-effortless-6-2026": { "score": 0.225, "n": 240, "notes": "Exact-match, normalized.", "categories": { "Commonsense-reasoning": { "n": 37, "exact_match": 0.0 }, "Knowledge-basic": { "n": 42, "exact_match": 0.1905 }, "Language-comprehension": { "n": 42, "exact_match": 0.119 }, "Logic-deduction": { "n": 42, "exact_match": 0.0238 }, "Math-arithmetic": { "n": 40, "exact_match": 0.825 }, "Pattern-recognition": { "n": 37, "exact_match": 0.1892 } } }, "bench-easy-6-2026": { "score": 0.4847, "n": 238, "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", "categories": { "Commonsense-causality": { "n": 10, "hybrid_score": 0.7134 }, "Commonsense-reasoning": { "n": 10, "hybrid_score": 0.7698 }, "Commonsense-simulation": { "n": 10, "hybrid_score": 0.7428 }, "Knowledge-basic": { "n": 33, "hybrid_score": 0.2424 }, "Knowledge-definitions": { "n": 38, "hybrid_score": 0.8114 }, "Language-comprehension": { "n": 10, "hybrid_score": 0.7592 }, "Language-structure": { "n": 10, "hybrid_score": 0.37 }, "Language-transformation": { "n": 10, "hybrid_score": 0.9131 }, "Logic-consistency": { "n": 10, "hybrid_score": 0.0 }, "Logic-deduction": { "n": 15, "hybrid_score": 0.0 }, "Logic-pattern": { "n": 10, "hybrid_score": 0.1 }, "Math-arithmetic": { "n": 33, "hybrid_score": 0.8485 }, "Math-pattern": { "n": 14, "hybrid_score": 0.0 }, "Math-reasoning": { "n": 15, "hybrid_score": 0.1892 }, "Pattern-matching": { "n": 10, "hybrid_score": 0.2 } } }, "bench-mid-6-2026": { "score": 0.6636, "n": 143, "acc": 0.6154, "acc_norm": 0.6573, "soft_score": 0.6252, "soft_score_norm": 0.6636, "stderr": 0.0394, "categories": { "Commonsense-causality": { "n": 5, "acc": 1.0, "acc_norm": 1.0 }, "Commonsense-reasoning": { "n": 10, "acc": 0.9, "acc_norm": 0.7 }, "Commonsense-simulation": { "n": 10, "acc": 0.5, "acc_norm": 0.7 }, "Knowledge-basic": { "n": 7, "acc": 1.0, "acc_norm": 1.0 }, "Knowledge-definitions": { "n": 10, "acc": 0.7, "acc_norm": 0.9 }, "Language-comprehension": { "n": 10, "acc": 0.6, "acc_norm": 0.8 }, "Language-structure": { "n": 10, "acc": 0.2, "acc_norm": 0.3 }, "Language-transformation": { "n": 10, "acc": 0.6, "acc_norm": 0.8 }, "Logic-consistency": { "n": 5, "acc": 0.0, "acc_norm": 0.2 }, "Logic-deduction": { "n": 10, "acc": 0.3, "acc_norm": 0.4 }, "Logic-pattern": { "n": 10, "acc": 0.5, "acc_norm": 0.4 }, "Math-arithmetic": { "n": 8, "acc": 1.0, "acc_norm": 1.0 }, "Math-pattern": { "n": 7, "acc": 1.0, "acc_norm": 1.0 }, "Math-reasoning": { "n": 10, "acc": 0.3, "acc_norm": 0.2 }, "Pattern-generation": { "n": 4, "acc": 0.75, "acc_norm": 0.75 }, "Pattern-matching": { "n": 8, "acc": 1.0, "acc_norm": 0.875 }, "Pattern-recognition": { "n": 9, "acc": 0.4444, "acc_norm": 0.4444 } } }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } }, { "id": "qwen-qwen2.5-0.5b", "name": "Qwen/Qwen2.5-0.5B", "org": "Qwen", "params_b": 0.5, "license": null, "architecture": null, "url": "https://huggingface.co/Qwen/Qwen2.5-0.5B", "model_revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "script_sha256": "955411f9971c4c26a6eeb3aa43b1694fe66ac430a139011faa915543746d2921", "runs": { "bench-effortless-6-2026": { "score": 0.075, "n": 240, "notes": "Exact-match, normalized.", "categories": { "Commonsense-reasoning": { "n": 37, "exact_match": 0.0 }, "Knowledge-basic": { "n": 42, "exact_match": 0.0238 }, "Language-comprehension": { "n": 42, "exact_match": 0.0 }, "Logic-deduction": { "n": 42, "exact_match": 0.0 }, "Math-arithmetic": { "n": 40, "exact_match": 0.425 }, "Pattern-recognition": { "n": 37, "exact_match": 0.0 } } }, "bench-easy-6-2026": { "score": 0.314, "n": 238, "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", "categories": { "Commonsense-causality": { "n": 10, "hybrid_score": 0.6992 }, "Commonsense-reasoning": { "n": 10, "hybrid_score": 0.6406 }, "Commonsense-simulation": { "n": 10, "hybrid_score": 0.6482 }, "Knowledge-basic": { "n": 33, "hybrid_score": 0.0 }, "Knowledge-definitions": { "n": 38, "hybrid_score": 0.7609 }, "Language-comprehension": { "n": 10, "hybrid_score": 0.6587 }, "Language-structure": { "n": 10, "hybrid_score": 0.3333 }, "Language-transformation": { "n": 10, "hybrid_score": 0.4212 }, "Logic-consistency": { "n": 10, "hybrid_score": 0.0 }, "Logic-deduction": { "n": 15, "hybrid_score": 0.0 }, "Logic-pattern": { "n": 10, "hybrid_score": 0.0 }, "Math-arithmetic": { "n": 33, "hybrid_score": 0.3333 }, "Math-pattern": { "n": 14, "hybrid_score": 0.0 }, "Math-reasoning": { "n": 15, "hybrid_score": 0.0538 }, "Pattern-matching": { "n": 10, "hybrid_score": 0.0 } } }, "bench-mid-6-2026": { "score": 0.5986, "n": 143, "acc": 0.5594, "acc_norm": 0.5874, "soft_score": 0.5657, "soft_score_norm": 0.5986, "stderr": 0.0406, "categories": { "Commonsense-causality": { "n": 5, "acc": 1.0, "acc_norm": 1.0 }, "Commonsense-reasoning": { "n": 10, "acc": 0.9, "acc_norm": 0.6 }, "Commonsense-simulation": { "n": 10, "acc": 0.3, "acc_norm": 0.7 }, "Knowledge-basic": { "n": 7, "acc": 1.0, "acc_norm": 1.0 }, "Knowledge-definitions": { "n": 10, "acc": 0.7, "acc_norm": 0.9 }, "Language-comprehension": { "n": 10, "acc": 0.7, "acc_norm": 0.9 }, "Language-structure": { "n": 10, "acc": 0.1, "acc_norm": 0.1 }, "Language-transformation": { "n": 10, "acc": 0.5, "acc_norm": 0.7 }, "Logic-consistency": { "n": 5, "acc": 0.0, "acc_norm": 0.2 }, "Logic-deduction": { "n": 10, "acc": 0.4, "acc_norm": 0.4 }, "Logic-pattern": { "n": 10, "acc": 0.1, "acc_norm": 0.2 }, "Math-arithmetic": { "n": 8, "acc": 0.875, "acc_norm": 0.875 }, "Math-pattern": { "n": 7, "acc": 0.7143, "acc_norm": 0.7143 }, "Math-reasoning": { "n": 10, "acc": 0.4, "acc_norm": 0.2 }, "Pattern-generation": { "n": 4, "acc": 0.75, "acc_norm": 0.75 }, "Pattern-matching": { "n": 8, "acc": 1.0, "acc_norm": 0.625 }, "Pattern-recognition": { "n": 9, "acc": 0.4444, "acc_norm": 0.4444 } } }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." }, "bench-effortless-7-2026": { "score": 0.0033, "n": 300, "stderr": 0.0033, "metrics": { "generative": { "exact_match": 0.0033, "hybrid_score": 0.2351 }, "loglikelihood": { "acc": 0.7733, "acc_norm": 0.7433, "soft_score": 0.7733, "soft_score_norm": 0.7433 } }, "categories": { "Commonsense-causality": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7778 }, "Commonsense-reasoning": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7778 }, "Commonsense-simulation": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7222 }, "Knowledge-basic": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7778 }, "Knowledge-definitions": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.8889 }, "Language-comprehension": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.8333 }, "Language-structure": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7222 }, "Language-transformation": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7778 }, "Logic-consistency": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.9444 }, "Logic-pattern": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7778 }, "Math-arithmetic": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.8824 }, "Math-pattern": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.9412 }, "Math-reasoning": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7059 }, "Pattern-generation": { "n": 17, "score": 0.0588, "exact_match": 0.0588, "acc_norm": 0.8235 }, "Pattern-matching": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7059 }, "Pattern-recognition": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.5882 } } }, "bench-easy-7-2026": { "score": 0.266, "n": 300, "stderr": 0.02, "metrics": { "generative": { "exact_match": 0.02, "hybrid_score": 0.266 }, "loglikelihood": { "acc": 0.6767, "acc_norm": 0.64, "soft_score": 0.6767, "soft_score_norm": 0.64 } }, "categories": { "Commonsense-causality": { "n": 18, "score": 0.6793, "exact_match": 0.0, "acc_norm": 0.8333 }, "Commonsense-reasoning": { "n": 18, "score": 0.6656, "exact_match": 0.0, "acc_norm": 0.7778 }, "Commonsense-simulation": { "n": 18, "score": 0.6951, "exact_match": 0.0, "acc_norm": 0.5556 }, "Knowledge-basic": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7778 }, "Knowledge-definitions": { "n": 18, "score": 0.7592, "exact_match": 0.0, "acc_norm": 1.0 }, "Language-comprehension": { "n": 18, "score": 0.7092, "exact_match": 0.0, "acc_norm": 0.8889 }, "Language-structure": { "n": 18, "score": 0.131, "exact_match": 0.0, "acc_norm": 0.5556 }, "Language-transformation": { "n": 18, "score": 0.5156, "exact_match": 0.0556, "acc_norm": 0.7778 }, "Logic-consistency": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 18, "score": 0.0556, "exact_match": 0.0556, "acc_norm": 0.7778 }, "Logic-pattern": { "n": 18, "score": 0.1111, "exact_match": 0.1111, "acc_norm": 0.6667 }, "Math-arithmetic": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.1765 }, "Math-pattern": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7059 }, "Math-reasoning": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7647 }, "Pattern-generation": { "n": 17, "score": 0.1176, "exact_match": 0.1176, "acc_norm": 0.4706 }, "Pattern-matching": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.5882 }, "Pattern-recognition": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.5294 } } }, "bench-mid-7-2026": { "score": 0.5033, "n": 300, "stderr": 0.0289, "metrics": { "generative": { "exact_match": 0.0033, "hybrid_score": 0.2509 }, "loglikelihood": { "acc": 0.49, "acc_norm": 0.5033, "soft_score": 0.49, "soft_score_norm": 0.5033 } }, "categories": { "Commonsense-causality": { "n": 18, "score": 0.4444, "exact_match": 0.0, "acc_norm": 0.4444 }, "Commonsense-reasoning": { "n": 18, "score": 0.5556, "exact_match": 0.0, "acc_norm": 0.5556 }, "Commonsense-simulation": { "n": 18, "score": 0.5556, "exact_match": 0.0, "acc_norm": 0.5556 }, "Knowledge-basic": { "n": 18, "score": 0.8333, "exact_match": 0.0, "acc_norm": 0.8333 }, "Knowledge-definitions": { "n": 18, "score": 0.8333, "exact_match": 0.0, "acc_norm": 0.8333 }, "Language-comprehension": { "n": 18, "score": 0.6667, "exact_match": 0.0, "acc_norm": 0.6667 }, "Language-structure": { "n": 18, "score": 0.6667, "exact_match": 0.0, "acc_norm": 0.6667 }, "Language-transformation": { "n": 18, "score": 0.7222, "exact_match": 0.0, "acc_norm": 0.7222 }, "Logic-consistency": { "n": 18, "score": 0.1111, "exact_match": 0.0, "acc_norm": 0.1111 }, "Logic-deduction": { "n": 18, "score": 0.6111, "exact_match": 0.0, "acc_norm": 0.6111 }, "Logic-pattern": { "n": 18, "score": 0.4444, "exact_match": 0.0, "acc_norm": 0.4444 }, "Math-arithmetic": { "n": 17, "score": 0.2353, "exact_match": 0.0, "acc_norm": 0.2353 }, "Math-pattern": { "n": 17, "score": 0.5882, "exact_match": 0.0588, "acc_norm": 0.5882 }, "Math-reasoning": { "n": 17, "score": 0.1176, "exact_match": 0.0, "acc_norm": 0.1176 }, "Pattern-generation": { "n": 17, "score": 0.4118, "exact_match": 0.0, "acc_norm": 0.4118 }, "Pattern-matching": { "n": 17, "score": 0.4706, "exact_match": 0.0, "acc_norm": 0.4706 }, "Pattern-recognition": { "n": 17, "score": 0.2353, "exact_match": 0.0, "acc_norm": 0.2353 } } } } }, { "id": "qwen-qwen2.5-1.5b-instruct", "name": "Qwen/Qwen2.5-1.5B-Instruct", "org": "Qwen", "params_b": 1.5, "license": "Apache 2.0", "architecture": null, "url": "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct", "model_revision": "989aa7980e4cf806f80c7fef2b1adb7bc71aa306", "script_sha256": "0c557dde5397ded8568280d621300759b4387a9f12227817c619a9325be8806a", "runs": { "bench-effortless-6-2026": { "score": 0.7458, "n": 240, "notes": "Exact-match, normalized.", "categories": { "Commonsense-reasoning": { "n": 37, "exact_match": 0.5405 }, "Knowledge-basic": { "n": 42, "exact_match": 0.5714 }, "Language-comprehension": { "n": 42, "exact_match": 0.6667 }, "Logic-deduction": { "n": 42, "exact_match": 0.881 }, "Math-arithmetic": { "n": 40, "exact_match": 0.925 }, "Pattern-recognition": { "n": 37, "exact_match": 0.8919 } } }, "bench-easy-6-2026": { "score": 0.7463, "n": 238, "notes": "Hybrid category-aware scoring (strict / flexible / semantic).", "categories": { "Commonsense-causality": { "n": 10, "hybrid_score": 0.8323 }, "Commonsense-reasoning": { "n": 10, "hybrid_score": 0.8156 }, "Commonsense-simulation": { "n": 10, "hybrid_score": 0.7773 }, "Knowledge-basic": { "n": 33, "hybrid_score": 0.7879 }, "Knowledge-definitions": { "n": 38, "hybrid_score": 0.8114 }, "Language-comprehension": { "n": 10, "hybrid_score": 0.7737 }, "Language-structure": { "n": 10, "hybrid_score": 0.7059 }, "Language-transformation": { "n": 10, "hybrid_score": 0.8567 }, "Logic-consistency": { "n": 10, "hybrid_score": 0.0 }, "Logic-deduction": { "n": 15, "hybrid_score": 0.4667 }, "Logic-pattern": { "n": 10, "hybrid_score": 0.5 }, "Math-arithmetic": { "n": 33, "hybrid_score": 0.9697 }, "Math-pattern": { "n": 14, "hybrid_score": 0.8571 }, "Math-reasoning": { "n": 15, "hybrid_score": 0.8786 }, "Pattern-matching": { "n": 10, "hybrid_score": 0.4 } } }, "bench-mid-6-2026": { "score": 0.7238, "n": 143, "acc": 0.6853, "acc_norm": 0.7203, "soft_score": 0.6951, "soft_score_norm": 0.7238, "stderr": 0.0373, "categories": { "Commonsense-causality": { "n": 5, "acc": 1.0, "acc_norm": 1.0 }, "Commonsense-reasoning": { "n": 10, "acc": 1.0, "acc_norm": 0.8 }, "Commonsense-simulation": { "n": 10, "acc": 0.7, "acc_norm": 0.9 }, "Knowledge-basic": { "n": 7, "acc": 1.0, "acc_norm": 1.0 }, "Knowledge-definitions": { "n": 10, "acc": 0.8, "acc_norm": 1.0 }, "Language-comprehension": { "n": 10, "acc": 0.7, "acc_norm": 0.7 }, "Language-structure": { "n": 10, "acc": 0.5, "acc_norm": 0.6 }, "Language-transformation": { "n": 10, "acc": 0.5, "acc_norm": 0.6 }, "Logic-consistency": { "n": 5, "acc": 0.0, "acc_norm": 0.2 }, "Logic-deduction": { "n": 10, "acc": 0.3, "acc_norm": 0.5 }, "Logic-pattern": { "n": 10, "acc": 0.3, "acc_norm": 0.2 }, "Math-arithmetic": { "n": 8, "acc": 1.0, "acc_norm": 1.0 }, "Math-pattern": { "n": 7, "acc": 1.0, "acc_norm": 1.0 }, "Math-reasoning": { "n": 10, "acc": 0.7, "acc_norm": 0.6 }, "Pattern-generation": { "n": 4, "acc": 0.75, "acc_norm": 0.75 }, "Pattern-matching": { "n": 8, "acc": 1.0, "acc_norm": 1.0 }, "Pattern-recognition": { "n": 9, "acc": 0.5556, "acc_norm": 0.5556 } } }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } }, { "id": "atomixlabs-photon-1.0-1m", "name": "AtomixLabs/Photon-1.0-1M", "org": "AtomixLabs", "params_b": 0.001, "license": "apache-2.0", "architecture": "LlamaForCausalLM", "url": "https://huggingface.co/AtomixLabs/Photon-1.0-1M", "model_revision": "a03eab42a7debdc27b797f8f5904a689720d0de4", "script_sha256": "955411f9971c4c26a6eeb3aa43b1694fe66ac430a139011faa915543746d2921", "runs": { "bench-effortless-7-2026": { "score": 0.0, "n": 300, "stderr": 0.0, "metrics": { "generative": { "exact_match": 0.0, "hybrid_score": 0.1942 }, "loglikelihood": { "acc": 0.33, "acc_norm": 0.2733, "soft_score": 0.33, "soft_score_norm": 0.2733 } }, "categories": { "Commonsense-causality": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.1111 }, "Commonsense-reasoning": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.4444 }, "Commonsense-simulation": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2778 }, "Knowledge-basic": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.1111 }, "Knowledge-definitions": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.3889 }, "Language-comprehension": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.5556 }, "Language-structure": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.1667 }, "Language-transformation": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.3889 }, "Logic-consistency": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2778 }, "Logic-pattern": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.0556 }, "Math-arithmetic": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.3529 }, "Math-pattern": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.1176 }, "Math-reasoning": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.1765 }, "Pattern-generation": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.6471 }, "Pattern-matching": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2941 }, "Pattern-recognition": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2941 } } }, "bench-easy-7-2026": { "score": 0.203, "n": 300, "stderr": 0.0158, "metrics": { "generative": { "exact_match": 0.0, "hybrid_score": 0.203 }, "loglikelihood": { "acc": 0.2833, "acc_norm": 0.31, "soft_score": 0.2833, "soft_score_norm": 0.31 } }, "categories": { "Commonsense-causality": { "n": 18, "score": 0.5875, "exact_match": 0.0, "acc_norm": 0.2222 }, "Commonsense-reasoning": { "n": 18, "score": 0.5981, "exact_match": 0.0, "acc_norm": 0.6111 }, "Commonsense-simulation": { "n": 18, "score": 0.6044, "exact_match": 0.0, "acc_norm": 0.1667 }, "Knowledge-basic": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2222 }, "Knowledge-definitions": { "n": 18, "score": 0.6303, "exact_match": 0.0, "acc_norm": 0.3889 }, "Language-comprehension": { "n": 18, "score": 0.5766, "exact_match": 0.0, "acc_norm": 0.5556 }, "Language-structure": { "n": 18, "score": 0.0829, "exact_match": 0.0, "acc_norm": 0.2778 }, "Language-transformation": { "n": 18, "score": 0.303, "exact_match": 0.0, "acc_norm": 0.5 }, "Logic-consistency": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.3333 }, "Logic-pattern": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2778 }, "Math-arithmetic": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.3529 }, "Math-pattern": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2353 }, "Math-reasoning": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.1765 }, "Pattern-generation": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.5294 }, "Pattern-matching": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.1765 }, "Pattern-recognition": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2353 } } }, "bench-mid-7-2026": { "score": 0.2867, "n": 300, "stderr": 0.0262, "metrics": { "generative": { "exact_match": 0.0, "hybrid_score": 0.205 }, "loglikelihood": { "acc": 0.1933, "acc_norm": 0.2867, "soft_score": 0.1933, "soft_score_norm": 0.2867 } }, "categories": { "Commonsense-causality": { "n": 18, "score": 0.2778, "exact_match": 0.0, "acc_norm": 0.2778 }, "Commonsense-reasoning": { "n": 18, "score": 0.4444, "exact_match": 0.0, "acc_norm": 0.4444 }, "Commonsense-simulation": { "n": 18, "score": 0.2778, "exact_match": 0.0, "acc_norm": 0.2778 }, "Knowledge-basic": { "n": 18, "score": 0.2778, "exact_match": 0.0, "acc_norm": 0.2778 }, "Knowledge-definitions": { "n": 18, "score": 0.4444, "exact_match": 0.0, "acc_norm": 0.4444 }, "Language-comprehension": { "n": 18, "score": 0.3889, "exact_match": 0.0, "acc_norm": 0.3889 }, "Language-structure": { "n": 18, "score": 0.2778, "exact_match": 0.0, "acc_norm": 0.2778 }, "Language-transformation": { "n": 18, "score": 0.1667, "exact_match": 0.0, "acc_norm": 0.1667 }, "Logic-consistency": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 18, "score": 0.5, "exact_match": 0.0, "acc_norm": 0.5 }, "Logic-pattern": { "n": 18, "score": 0.2222, "exact_match": 0.0, "acc_norm": 0.2222 }, "Math-arithmetic": { "n": 17, "score": 0.0588, "exact_match": 0.0, "acc_norm": 0.0588 }, "Math-pattern": { "n": 17, "score": 0.1176, "exact_match": 0.0, "acc_norm": 0.1176 }, "Math-reasoning": { "n": 17, "score": 0.0588, "exact_match": 0.0, "acc_norm": 0.0588 }, "Pattern-generation": { "n": 17, "score": 0.5882, "exact_match": 0.0, "acc_norm": 0.5882 }, "Pattern-matching": { "n": 17, "score": 0.4118, "exact_match": 0.0, "acc_norm": 0.4118 }, "Pattern-recognition": { "n": 17, "score": 0.3529, "exact_match": 0.0, "acc_norm": 0.3529 } } }, "bench-effortless-6-2026": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." }, "bench-easy-6-2026": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." }, "bench-mid-6-2026": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } }, { "id": "tobiaslogic-textmodel-v1", "name": "TobiasLogic/TextModel-v1", "org": "TobiasLogic", "params_b": 0.1, "license": "mit", "architecture": "Llama-style decoder-only transformer", "url": "https://huggingface.co/TobiasLogic/TextModel-v1", "model_revision": "95100784f78fc33c28246643a72b29930867c1d5", "script_sha256": "955411f9971c4c26a6eeb3aa43b1694fe66ac430a139011faa915543746d2921", "runs": { "bench-effortless-7-2026": { "score": 0.0033, "n": 300, "stderr": 0.0033, "metrics": { "generative": { "exact_match": 0.0033, "hybrid_score": 0.1331 }, "loglikelihood": { "acc": 0.43, "acc_norm": 0.41, "soft_score": 0.43, "soft_score_norm": 0.41 } }, "categories": { "Commonsense-causality": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.4444 }, "Commonsense-reasoning": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.5 }, "Commonsense-simulation": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.3333 }, "Knowledge-basic": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.3333 }, "Knowledge-definitions": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.6667 }, "Language-comprehension": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7222 }, "Language-structure": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.3333 }, "Language-transformation": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.5 }, "Logic-consistency": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7222 }, "Logic-pattern": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.8333 }, "Math-arithmetic": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.1765 }, "Math-pattern": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.1765 }, "Math-reasoning": { "n": 17, "score": 0.0588, "exact_match": 0.0588, "acc_norm": 0.2353 }, "Pattern-generation": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.4118 }, "Pattern-matching": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2941 }, "Pattern-recognition": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2353 } } }, "bench-easy-7-2026": { "score": 0.1538, "n": 300, "stderr": 0.0135, "metrics": { "generative": { "exact_match": 0.0, "hybrid_score": 0.1538 }, "loglikelihood": { "acc": 0.3733, "acc_norm": 0.4433, "soft_score": 0.3733, "soft_score_norm": 0.4433 } }, "categories": { "Commonsense-causality": { "n": 18, "score": 0.327, "exact_match": 0.0, "acc_norm": 0.6111 }, "Commonsense-reasoning": { "n": 18, "score": 0.2776, "exact_match": 0.0, "acc_norm": 0.6111 }, "Commonsense-simulation": { "n": 18, "score": 0.3022, "exact_match": 0.0, "acc_norm": 0.3333 }, "Knowledge-basic": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.3889 }, "Knowledge-definitions": { "n": 18, "score": 0.3525, "exact_match": 0.0, "acc_norm": 0.7222 }, "Language-comprehension": { "n": 18, "score": 0.2459, "exact_match": 0.0, "acc_norm": 0.8333 }, "Language-structure": { "n": 18, "score": 0.26, "exact_match": 0.0, "acc_norm": 0.2222 }, "Language-transformation": { "n": 18, "score": 0.7978, "exact_match": 0.0, "acc_norm": 0.5556 }, "Logic-consistency": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.0556 }, "Logic-deduction": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.6667 }, "Logic-pattern": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.7778 }, "Math-arithmetic": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.0588 }, "Math-pattern": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2941 }, "Math-reasoning": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.3529 }, "Pattern-generation": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2353 }, "Pattern-matching": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.5294 }, "Pattern-recognition": { "n": 17, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.2353 } } }, "bench-mid-7-2026": { "score": 0.3267, "n": 300, "stderr": 0.0271, "metrics": { "generative": { "exact_match": 0.0033, "hybrid_score": 0.1357 }, "loglikelihood": { "acc": 0.2667, "acc_norm": 0.3267, "soft_score": 0.2667, "soft_score_norm": 0.3267 } }, "categories": { "Commonsense-causality": { "n": 18, "score": 0.4444, "exact_match": 0.0, "acc_norm": 0.4444 }, "Commonsense-reasoning": { "n": 18, "score": 0.3333, "exact_match": 0.0, "acc_norm": 0.3333 }, "Commonsense-simulation": { "n": 18, "score": 0.3333, "exact_match": 0.0, "acc_norm": 0.3333 }, "Knowledge-basic": { "n": 18, "score": 0.5, "exact_match": 0.0, "acc_norm": 0.5 }, "Knowledge-definitions": { "n": 18, "score": 0.6667, "exact_match": 0.0, "acc_norm": 0.6667 }, "Language-comprehension": { "n": 18, "score": 0.3889, "exact_match": 0.0, "acc_norm": 0.3889 }, "Language-structure": { "n": 18, "score": 0.3889, "exact_match": 0.0556, "acc_norm": 0.3889 }, "Language-transformation": { "n": 18, "score": 0.5, "exact_match": 0.0, "acc_norm": 0.5 }, "Logic-consistency": { "n": 18, "score": 0.0, "exact_match": 0.0, "acc_norm": 0.0 }, "Logic-deduction": { "n": 18, "score": 0.5, "exact_match": 0.0, "acc_norm": 0.5 }, "Logic-pattern": { "n": 18, "score": 0.3333, "exact_match": 0.0, "acc_norm": 0.3333 }, "Math-arithmetic": { "n": 17, "score": 0.1765, "exact_match": 0.0, "acc_norm": 0.1765 }, "Math-pattern": { "n": 17, "score": 0.0588, "exact_match": 0.0, "acc_norm": 0.0588 }, "Math-reasoning": { "n": 17, "score": 0.1176, "exact_match": 0.0, "acc_norm": 0.1176 }, "Pattern-generation": { "n": 17, "score": 0.2941, "exact_match": 0.0, "acc_norm": 0.2941 }, "Pattern-matching": { "n": 17, "score": 0.3529, "exact_match": 0.0, "acc_norm": 0.3529 }, "Pattern-recognition": { "n": 17, "score": 0.1176, "exact_match": 0.0, "acc_norm": 0.1176 } } }, "bench-effortless-6-2026": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." }, "bench-easy-6-2026": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." }, "bench-mid-6-2026": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." }, "bench-AGI": { "score": null, "n": null, "notes": "Not yet evaluated on this tier." } } } ], "t2i_benchmarks": [ { "id": "t2i-fid-coco", "label": "FID", "kind": "text-to-image", "metric": "fid", "lower_is_better": true, "description": "Frechet Inception Distance, self-reported per model card. Lower is better; measures distributional realism of generated images against real reference photos." }, { "id": "t2i-clip-coco", "label": "CLIP Score", "kind": "text-to-image", "metric": "clip_score", "lower_is_better": false, "description": "CLIP Score, self-reported per model card, 0-1 scale. Higher is better; measures text-image alignment." } ], "t2i_models": [ { "id": "bench-labs-pixelmodel-v5", "name": "PixelModel v5", "org": "Bench Labs", "params_b": 0.0401, "url": "https://huggingface.co/bench-labs/pixelmodel-v5", "resolution": "256x256", "release_date": "2026-07-28", "runs": { "t2i-fid-coco": { "score": 34.06, "n": null }, "t2i-clip-coco": { "score": 0.2927, "n": null } } }, { "id": "bench-labs-pixelmodel-v4", "name": "PixelModel v4", "org": "Bench Labs", "params_b": 0.0401, "url": "https://huggingface.co/bench-labs/pixelmodel-v4", "resolution": "256x256", "release_date": "2026-07-25", "runs": { "t2i-fid-coco": { "score": 39.54, "n": null }, "t2i-clip-coco": { "score": 0.2804, "n": null } } }, { "id": "bench-labs-pixelmodel-v2", "name": "PixelModel v2", "org": "Bench Labs", "params_b": 0.0002, "url": "https://huggingface.co/bench-labs/pixelmodel-v2", "resolution": "64x64", "release_date": "2026-07-21", "runs": { "t2i-fid-coco": { "score": 390.68, "n": null }, "t2i-clip-coco": { "score": 0.2048, "n": null } } }, { "id": "bench-labs-pixelmodel-v3", "name": "PixelModel v3", "org": "Bench Labs", "params_b": 0.000919, "url": "https://huggingface.co/bench-labs/pixelmodel-v3", "resolution": "128x128", "release_date": "2026-07-24", "runs": { "t2i-fid-coco": { "score": 383.91, "n": null }, "t2i-clip-coco": { "score": 0.2073, "n": null } } }, { "id": "bench-labs-pixelmodel-v1", "name": "PixelModel v1", "org": "Bench Labs", "params_b": 2.37e-05, "url": "https://huggingface.co/bench-labs/pixelmodel-v1", "resolution": "64x64", "release_date": "2026-07-17", "runs": { "t2i-fid-coco": { "score": 420.75, "n": null }, "t2i-clip-coco": { "score": 0.201, "n": null } } }, { "id": "bench-labs-pixelmodel-v0", "name": "PixelModel v0", "org": "Bench Labs", "params_b": 0.000203, "url": "https://huggingface.co/bench-labs/pixelmodel", "resolution": "32x32", "release_date": "2026-07-14", "runs": { "t2i-fid-coco": { "score": 566.84, "n": null }, "t2i-clip-coco": { "score": 0.186, "n": null } } }, { "id": "flamef0x-shelld", "name": "ShellD", "org": "FlameF0X", "params_b": 0.0669, "url": "https://huggingface.co/FlameF0X/ShellD", "resolution": "256x256", "release_date": "2026-07-14", "runs": { "t2i-fid-coco": { "score": 348.92, "n": null }, "t2i-clip-coco": { "score": 0.1891, "n": null } }, "space_url": "https://flamef0x-shelld-demo.hf.space/" }, { "id": "fwkv-image", "name": "FWKV-Image", "org": "FWKV", "params_b": 0.192, "url": "https://huggingface.co/FWKV/FWKV-Image", "resolution": "256x256", "release_date": "2026-07-26", "runs": { "t2i-fid-coco": { "score": 173.84, "n": null }, "t2i-clip-coco": { "score": 0.2244, "n": null } } } ] }