| { |
| "harness": "lm-evaluation-harness >=0.4.5, zero-shot, bootstrap off. Metrics: hellaswag/arc_challenge/piqa/openbookqa acc_norm; arc_easy/winogrande/sciq/lambada_openai/mmlu acc.", |
| "scorer_ours": "left-to-right chain rule over continuation tokens, later continuation tokens masked (dlm_lmeval.py)", |
| "models": { |
| "DiffuRefill-1B (ours)": {"params_B": 1.08, "tokens_T": 0.093, "hellaswag": 34.8, "arc_easy": 52.4, "arc_challenge": 25.0, "piqa": 62.7, "winogrande": 50.6, "openbookqa": 27.4, "sciq": 89.5, "lambada_openai": 34.0, "mmlu": 26.9}, |
| "EleutherAI/pythia-1b": {"params_B": 1.01, "tokens_T": 0.3, "hellaswag": 47.2, "arc_easy": 56.9, "arc_challenge": 27.0, "piqa": 69.4, "winogrande": 54.1, "openbookqa": 31.2, "sciq": 83.4, "lambada_openai": 55.8, "mmlu": 23.1}, |
| "HuggingFaceTB/SmolLM2-360M": {"params_B": 0.36, "tokens_T": 4, "hellaswag": 56.3, "arc_easy": 70.3, "arc_challenge": 38.1, "piqa": 71.5, "winogrande": 58.9, "openbookqa": 37.8, "sciq": 91.1, "lambada_openai": 53.4, "mmlu": 25.5}, |
| "Qwen/Qwen2.5-0.5B": {"params_B": 0.49, "tokens_T": 18, "hellaswag": 52.2, "arc_easy": 64.9, "arc_challenge": 31.8, "piqa": 69.9, "winogrande": 56.1, "openbookqa": 35.0, "sciq": 93.0, "lambada_openai": 52.0, "mmlu": 47.5}, |
| "HuggingFaceTB/SmolLM2-1.7B": {"params_B": 1.71, "tokens_T": 11, "hellaswag": 71.2, "arc_easy": 77.8, "arc_challenge": 47.1, "piqa": 77.6, "winogrande": 65.9, "openbookqa": 44.0, "sciq": 93.3, "lambada_openai": 67.5, "mmlu": 48.5}, |
| "Qwen/Qwen2.5-1.5B": {"params_B": 1.54, "tokens_T": 18, "hellaswag": 67.8, "arc_easy": 75.5, "arc_challenge": 45.0, "piqa": 75.9, "winogrande": 63.5, "openbookqa": 40.6, "sciq": 94.3, "lambada_openai": 62.2, "mmlu": 59.7} |
| }, |
| "excluded": "TinyLlama/TinyLlama_v1.1 and allenai/OLMo-1B-hf were run but load incorrectly under the current transformers (OLMo: LAMBADA 0%, SciQ 4%; TinyLlama: WinoGrande 49.7%), so they are not reported.", |
| "scorer_crosscheck": {"subset": "first 1000 docs", "hellaswag_acc_norm": {"chain": 42.5, "elbo_32": 40.8}, "winogrande_acc": {"chain": 50.3, "elbo_32": 51.3}}, |
| "speed_tok_per_s": { |
| "setup": "one RTX PRO 6000 Blackwell 96GB, bf16, 32-token prompt, 256 new tokens, greedy. AR models: HF generate() with KV cache. Ours: no KV cache, block 32, K tokens committed per pass.", |
| "DiffuRefill-1B": {"B1_K1": 193, "B1_K2": 384, "B1_K4": 771, "B1_K8": 1536, "B32_K1": 431, "B32_K2": 863, "B32_K4": 1725, "B32_K8": 3449}, |
| "SmolLM2-1.7B": {"B1": 101, "B32": 3119}, |
| "Qwen2.5-1.5B": {"B1": 82, "B32": 2612} |
| }, |
| "training_probes_final": { |
| "note": "fixed held-out masked CE (mask rates .3/.6/.9), wiki and code CE, PopQA fact probe (1050 facts)", |
| "global": {"heldout": 2.9503, "wiki": 3.5397, "code": 1.6668, "fact_mc": 0.558, "fact_nll": 5.899, "fact_em": 0.096}, |
| "ema_0.9": {"heldout": 2.9503, "wiki": 3.5397, "code": 1.6667, "fact_mc": 0.558, "fact_nll": 5.899, "fact_em": 0.096}, |
| "swa": {"heldout": 2.9501, "wiki": 3.5393, "code": 1.6639, "fact_mc": 0.561, "fact_nll": 5.894, "fact_em": 0.096}, |
| "avg_last3_snapshots": {"heldout": 2.9502, "wiki": 3.5398, "code": 1.6656, "fact_mc": 0.557, "fact_nll": 5.900, "fact_em": 0.097}, |
| "avg_last5_snapshots": {"heldout": 2.9503, "wiki": 3.5394, "code": 1.6641, "fact_mc": 0.559, "fact_nll": 5.895, "fact_em": 0.095} |
| } |
| } |
|
|
