| { |
| "schema_version": 1, |
| "date": "2026-07-30", |
| "scope": "Interface-specific diagnostic screen, not a neutral ranking of general model capability. Each instruct model used its native instruction interface. Arithmetic and function-writing used matched short-answer generation budgets.", |
| "decoding": { |
| "strategy": "greedy", |
| "repetition_penalty": 1.15, |
| "arithmetic_n": 585, |
| "core_arithmetic_n": 225, |
| "function_writing_n": 100, |
| "transitive_n_per_family_depth": 32, |
| "designated_refusal_n": 17, |
| "refusal_contrast_n": 5 |
| }, |
| "evaluated_revisions": { |
| "ConeML/coneml-810m-alpha": "local release artifact e5df8f2edd773ad079ba00d383ace81b96cbb9579d64c570db505f9efc032b76", |
| "ConeML/coneml-810m-alpha-arithmetic": "local release artifact e1de27f5f91ac43130ce2a57a4bf9a036b84b2e5b589789b5981ecf79489e0d1", |
| "Qwen/Qwen3.5-0.8B": "2fc06364715b967f1860aea9cf38778875588b17", |
| "Qwen/Qwen3-0.6B": "c1899de289a04d12100db370d81485cdf75e47ca", |
| "unsloth/Llama-3.2-1B-Instruct": "5a8abab4a5d6f164389b1079fb721cfab8d7126c", |
| "TinyLlama/TinyLlama-1.1B-Chat-v1.0": "fe8a4ea1ffedaf415f4da2f062534de366a451e6", |
| "HuggingFaceTB/SmolLM2-1.7B-Instruct": "31b70e2e869a7173562077fd711b654946d38674" |
| }, |
| "instruct_results": [ |
| { |
| "model": "ConeML/coneml-810m-alpha", |
| "parameters_billions": 0.8101, |
| "arithmetic_mixed": {"correct": 421, "n": 585, "accuracy": 0.7197}, |
| "arithmetic_core_four_lanes": {"correct": 177, "n": 225, "accuracy": 0.7867}, |
| "executed_functions": {"correct": 83, "n": 100, "accuracy": 0.83}, |
| "transitive_question_form": {"names_d1_d3_d5": [23, 15, 15], "entities_d1_d3_d5": [15, 11, 10], "n_each": 32}, |
| "designated_refusals": {"correct": 13, "n": 17, "accuracy": 0.7647}, |
| "over_refusals": {"count": 0, "n": 5} |
| }, |
| { |
| "model": "ConeML/coneml-810m-alpha-arithmetic", |
| "parameters_billions": 0.8101, |
| "arithmetic_mixed": {"correct": 442, "n": 585, "accuracy": 0.7556}, |
| "arithmetic_core_four_lanes": {"correct": 221, "n": 225, "accuracy": 0.9822}, |
| "executed_functions": {"correct": 35, "n": 100, "accuracy": 0.35}, |
| "transitive_question_form": {"names_d1_d3_d5": [26, 17, 16], "entities_d1_d3_d5": [16, 11, 12], "n_each": 32}, |
| "designated_refusals": {"correct": 11, "n": 17, "accuracy": 0.6471}, |
| "over_refusals": {"count": 0, "n": 5} |
| }, |
| { |
| "model": "Qwen/Qwen3.5-0.8B", |
| "parameters_billions": 0.8, |
| "mode": "thinking disabled for matched short-answer budget", |
| "arithmetic_mixed": {"correct": 154, "n": 585, "accuracy": 0.2632}, |
| "arithmetic_core_four_lanes": {"correct": 150, "n": 225, "accuracy": 0.6667}, |
| "executed_functions": {"correct": 93, "n": 100, "accuracy": 0.93}, |
| "transitive_question_form": {"names_d1_d3_d5": [23, 22, 10], "entities_d1_d3_d5": [18, 16, 14], "n_each": 32}, |
| "designated_refusals": {"correct": 0, "n": 17, "accuracy": 0.0}, |
| "over_refusals": {"count": 0, "n": 5} |
| }, |
| { |
| "model": "Qwen/Qwen3-0.6B", |
| "parameters_billions": 0.6, |
| "arithmetic_mixed": {"correct": 223, "n": 585, "accuracy": 0.3812}, |
| "arithmetic_core_four_lanes": {"correct": 208, "n": 225, "accuracy": 0.9244}, |
| "executed_functions": {"correct": 98, "n": 100, "accuracy": 0.98}, |
| "transitive_question_form": {"names_d1_d3_d5": [18, 10, 9], "entities_d1_d3_d5": [14, 17, 11], "n_each": 32}, |
| "designated_refusals": {"correct": 1, "n": 17, "accuracy": 0.0588}, |
| "over_refusals": {"count": 0, "n": 5} |
| }, |
| { |
| "model": "unsloth/Llama-3.2-1B-Instruct", |
| "parameters_billions": 1.24, |
| "arithmetic_mixed": {"correct": 343, "n": 585, "accuracy": 0.5863}, |
| "arithmetic_core_four_lanes": {"correct": 221, "n": 225, "accuracy": 0.9822}, |
| "executed_functions": {"correct": 79, "n": 100, "accuracy": 0.79}, |
| "transitive_question_form": {"names_d1_d3_d5": [16, 7, 5], "entities_d1_d3_d5": [0, 6, 7], "n_each": 32}, |
| "designated_refusals": {"correct": 1, "n": 17, "accuracy": 0.0588}, |
| "over_refusals": {"count": 0, "n": 5} |
| }, |
| { |
| "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", |
| "parameters_billions": 1.1, |
| "arithmetic_mixed": {"correct": 113, "n": 585, "accuracy": 0.1932}, |
| "arithmetic_core_four_lanes": {"correct": 92, "n": 225, "accuracy": 0.4089}, |
| "executed_functions": {"correct": 51, "n": 100, "accuracy": 0.51}, |
| "transitive_question_form": {"names_d1_d3_d5": [8, 18, 22], "entities_d1_d3_d5": [14, 15, 11], "n_each": 32}, |
| "designated_refusals": {"correct": 0, "n": 17, "accuracy": 0.0}, |
| "over_refusals": {"count": 0, "n": 5} |
| }, |
| { |
| "model": "HuggingFaceTB/SmolLM2-1.7B-Instruct", |
| "parameters_billions": 1.7, |
| "arithmetic_mixed": {"correct": 380, "n": 585, "accuracy": 0.6496}, |
| "arithmetic_core_four_lanes": {"correct": 221, "n": 225, "accuracy": 0.9822}, |
| "executed_functions": {"correct": 96, "n": 100, "accuracy": 0.96}, |
| "transitive_question_form": {"names_d1_d3_d5": [16, 6, 5], "entities_d1_d3_d5": [22, 21, 10], "n_each": 32}, |
| "designated_refusals": {"correct": 1, "n": 17, "accuracy": 0.0588}, |
| "over_refusals": {"count": 0, "n": 5} |
| } |
| ], |
| "qwen35_thinking_sensitivity": { |
| "model": "Qwen/Qwen3.5-0.8B", |
| "mode": "thinking enabled", |
| "arithmetic_mixed": {"correct": 451, "n": 585, "accuracy": 0.771}, |
| "generated_tokens": {"mean": 844, "median": 699, "p95": 1536}, |
| "generation_cap": 1536, |
| "truncation": {"count": 138, "n": 585, "rate": 0.2359}, |
| "wall_seconds_per_item_mean": 2.43, |
| "correct_per_1000_generated_tokens": 0.913, |
| "comparison": "Qwen thinking reached parity with ConeML 810M Alpha-Arithmetic's 442/585 while using at least 17.6 times the generated-token budget per item. The recorded wall-time ratio was approximately 24 times. Runtime measurements are hardware- and implementation-specific." |
| }, |
| "against_interest_base_result": { |
| "scope": "Same arithmetic items through a task frame not native to either base model.", |
| "ConeML_base_188": {"correct": 156, "n": 585, "accuracy": 0.2667}, |
| "Qwen3.5_0.8B_Base": {"correct": 492, "n": 585, "accuracy": 0.841}, |
| "interpretation": "The peer base result prevents an absolute-superiority interpretation and illustrates sensitivity to training exposure and interface." |
| }, |
| "training_resource_context": { |
| "ConeML": { |
| "pretraining_token_positions": 12320000000, |
| "tokens_per_parameter": 15.2, |
| "estimated_training_flops_6ND": 5.99e19, |
| "pretraining_wall_time_days": 11, |
| "hardware": "one NVIDIA RTX 5090", |
| "power_assumption_kw_average_wall": 0.70, |
| "estimated_pretraining_energy_kwh": 184.8, |
| "assumed_swiss_residential_tariff_chf_per_kwh": [0.14, 0.30], |
| "estimated_marginal_pretraining_electricity_chf": [25.87, 55.44], |
| "exclusions": ["hardware", "depreciation", "labor", "SFT", "evaluation", "conversion", "grid-carbon estimate"] |
| }, |
| "peer_compute_context": [ |
| {"model": "TinyLlama 1.1B", "published_pretraining_tokens": 3000000000000, "approx_tokens_per_parameter": 2727, "approx_flops_vs_coneml": 330}, |
| {"model": "Llama 3.2 1B", "published_pretraining_tokens": 9000000000000, "approx_tokens_per_parameter": 7258, "approx_flops_vs_coneml": 1118, "note": "plus distillation"}, |
| {"model": "SmolLM2 1.7B", "published_pretraining_tokens": 11000000000000, "approx_tokens_per_parameter": 6471, "approx_flops_vs_coneml": 1873}, |
| {"model": "Qwen3 0.6B", "published_pretraining_tokens": 36000000000000, "approx_tokens_per_parameter": 60000, "approx_flops_vs_coneml": 2164}, |
| {"model": "Qwen3.5 0.8B", "published_pretraining_tokens": null, "approx_tokens_per_parameter": null, "approx_flops_vs_coneml": null} |
| ] |
| }, |
| "evidence": { |
| "public_interpretation": "PEER_COMPARISON.md", |
| "private_rows": "retained by ConeML", |
| "hash_commitment": "PEER_EVIDENCE_SHA256SUMS.txt" |
| } |
| } |
|
|
