outputs/table23_stats.json

{
  "rows": [
    {
      "task": "ImageNet ResNet-50",
      "metric": "Top-1 SGD",
      "ref": "77.01\u00b10.02",
      "flash": "77.16\u00b10.09",
      "delta": 0.15,
      "z_spread": 1.63,
      "z_sem": 2.82,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": true
    },
    {
      "task": "ImageNet ResNet-50",
      "metric": "Top-1 AdamW",
      "ref": "75.51\u00b10.09",
      "flash": "75.67\u00b10.04",
      "delta": 0.16,
      "z_spread": 1.62,
      "z_sem": 2.81,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": true
    },
    {
      "task": "Llama-3.1-8B OpenMathInstruct-2",
      "metric": "GSM8K AdamW",
      "ref": "75.09\u00b10.4",
      "flash": "74.98\u00b10.77",
      "delta": -0.11,
      "z_spread": 0.13,
      "z_sem": 0.22,
      "flash_worse": true,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "Val loss AdamW",
      "ref": "3.263\u00b10.001",
      "flash": "3.265\u00b10.001",
      "delta": 0.002,
      "z_spread": 1.41,
      "z_sem": 2.45,
      "flash_worse": true,
      "sig_degradation": true,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "HellaSwag AdamW",
      "ref": "31.9\u00b10.2",
      "flash": "31.9\u00b10.5",
      "delta": 0.0,
      "z_spread": 0.0,
      "z_sem": 0.0,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "ARC-E AdamW",
      "ref": "39.6\u00b10.7",
      "flash": "39.5\u00b10.9",
      "delta": -0.1,
      "z_spread": 0.09,
      "z_sem": 0.15,
      "flash_worse": true,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "CSQA AdamW",
      "ref": "25.9\u00b14.1",
      "flash": "30.8\u00b12.1",
      "delta": 4.9,
      "z_spread": 1.06,
      "z_sem": 1.84,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "PIQA AdamW",
      "ref": "64.3\u00b10.0",
      "flash": "64.5\u00b10.3",
      "delta": 0.2,
      "z_spread": 0.67,
      "z_sem": 1.15,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "LAMBADA AdamW",
      "ref": "31.0\u00b11.2",
      "flash": "31.9\u00b10.7",
      "delta": 0.9,
      "z_spread": 0.65,
      "z_sem": 1.12,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "Winograd AdamW",
      "ref": "57.3\u00b10.6",
      "flash": "59.1\u00b11.1",
      "delta": 1.8,
      "z_spread": 1.44,
      "z_sem": 2.49,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": true
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "BoolQ AdamW",
      "ref": "58.1\u00b13.6",
      "flash": "57.2\u00b14.8",
      "delta": -0.9,
      "z_spread": 0.15,
      "z_sem": 0.26,
      "flash_worse": true,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "Mean ICL AdamW",
      "ref": "44.0\u00b10.4",
      "flash": "45.0\u00b11.0",
      "delta": 1.0,
      "z_spread": 0.93,
      "z_sem": 1.61,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "Val loss Lion",
      "ref": "3.24\u00b10.002",
      "flash": "3.24\u00b10.001",
      "delta": 0.0,
      "z_spread": 0.0,
      "z_sem": 0.0,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "HellaSwag Lion",
      "ref": "32.3\u00b10.0",
      "flash": "32.4\u00b10.3",
      "delta": 0.1,
      "z_spread": 0.33,
      "z_sem": 0.58,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "ARC-E Lion",
      "ref": "40.0\u00b10.5",
      "flash": "40.8\u00b10.2",
      "delta": 0.8,
      "z_spread": 1.49,
      "z_sem": 2.57,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": true
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "CSQA Lion",
      "ref": "23.3\u00b11.8",
      "flash": "25.4\u00b12.9",
      "delta": 2.1,
      "z_spread": 0.62,
      "z_sem": 1.07,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "PIQA Lion",
      "ref": "63.8\u00b11.0",
      "flash": "64.2\u00b10.5",
      "delta": 0.4,
      "z_spread": 0.36,
      "z_sem": 0.62,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "LAMBADA Lion",
      "ref": "31.5\u00b10.2",
      "flash": "31.6\u00b10.5",
      "delta": 0.1,
      "z_spread": 0.19,
      "z_sem": 0.32,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "Winograd Lion",
      "ref": "58.9\u00b12.0",
      "flash": "59.1\u00b12.4",
      "delta": 0.2,
      "z_spread": 0.06,
      "z_sem": 0.11,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "BoolQ Lion",
      "ref": "58.1\u00b12.4",
      "flash": "59.1\u00b12.3",
      "delta": 1.0,
      "z_spread": 0.3,
      "z_sem": 0.52,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    },
    {
      "task": "GPT-2 124M pretrain",
      "metric": "Mean ICL Lion",
      "ref": "44.0\u00b10.5",
      "flash": "44.7\u00b10.5",
      "delta": 0.7,
      "z_spread": 0.99,
      "z_sem": 1.71,
      "flash_worse": false,
      "sig_degradation": false,
      "sig_improvement": false
    }
  ],
  "n_metrics": 21,
  "n_significant_degradations_z_sem_gt2": 1,
  "n_significant_improvements_z_sem_gt2": 4,
  "caveats": "n=3 seeds; z_sem assumes independent seeds and normality; PIQA/HellaSwag-Lion have std=0.0 in one arm (excluded from inf-z by pairing); this audits the paper's own numbers, it is NOT new measurement."
}