repro_bundle/outputs/summary.json

{
  "supernets": [
    {
      "dataset": "cifar10",
      "arch": "mobiletiny",
      "dfmpq_budgets": 5,
      "dfmpq_wins_vs_uniform_supernet": 2,
      "mean_acc_gain_vs_uniform_supernet_pp": -2.38,
      "max_acc": 0.5772,
      "min_acc": 0.1319,
      "best_dfmpq_size_bits": 455008,
      "ref_acc_at_rep_budget": 0.6767,
      "ref_size_bits": 910016
    },
    {
      "dataset": "cifar10",
      "arch": "resnet8",
      "dfmpq_budgets": 5,
      "dfmpq_wins_vs_uniform_supernet": 2,
      "mean_acc_gain_vs_uniform_supernet_pp": -6.06,
      "max_acc": 0.524,
      "min_acc": 0.1438,
      "best_dfmpq_size_bits": 389216,
      "ref_acc_at_rep_budget": 0.6915,
      "ref_size_bits": 778432
    },
    {
      "dataset": "cifar100",
      "arch": "mobiletiny",
      "dfmpq_budgets": 5,
      "dfmpq_wins_vs_uniform_supernet": 2,
      "mean_acc_gain_vs_uniform_supernet_pp": -1.23,
      "max_acc": 0.1346,
      "min_acc": 0.0174,
      "best_dfmpq_size_bits": 478048,
      "ref_acc_at_rep_budget": 0.2288,
      "ref_size_bits": 956096
    },
    {
      "dataset": "cifar100",
      "arch": "resnet8",
      "dfmpq_budgets": 5,
      "dfmpq_wins_vs_uniform_supernet": 2,
      "mean_acc_gain_vs_uniform_supernet_pp": -0.61,
      "max_acc": 0.0792,
      "min_acc": 0.0344,
      "best_dfmpq_size_bits": 400736,
      "ref_acc_at_rep_budget": 0.2302,
      "ref_size_bits": 801472
    }
  ],
  "headline": [
    "cifar10/mobiletiny: DFMPQ beats same-size uniform supernet in 2/5 budgets, ++2.38 pp (mean), max acc=57.72%, min=13.19%.",
    "cifar10/resnet8: DFMPQ beats same-size uniform supernet in 2/5 budgets, ++6.06 pp (mean), max acc=52.40%, min=14.38%.",
    "cifar100/mobiletiny: DFMPQ beats same-size uniform supernet in 2/5 budgets, ++1.23 pp (mean), max acc=13.46%, min=1.74%.",
    "cifar100/resnet8: DFMPQ beats same-size uniform supernet in 2/5 budgets, ++0.61 pp (mean), max acc=7.92%, min=3.44%."
  ]
}