analysis/aggregate_summary.json

{
  "final_accuracy": {
    "bp_checkpoint": {
      "mean": 0.7099609375,
      "std": 0.023700509959983587
    },
    "fmad": {
      "mean": 0.2841796875,
      "std": 0.014778072217208551
    },
    "zo": {
      "mean": 0.3076171875,
      "std": 0.02046125667945016
    }
  },
  "accuracy_gap_percentage_points": {
    "bp_minus_fmad": {
      "mean": 42.578125,
      "std": 3.7327177448469517
    },
    "bp_minus_zo": {
      "mean": 40.234375,
      "std": 2.3089043784251735
    }
  },
  "peak_memory_mb": {
    "bp_checkpoint": {
      "mean": 58.61767578125,
      "std": 0.0
    },
    "fmad": {
      "mean": 57.091796875,
      "std": 0.0
    },
    "zo": {
      "mean": 57.150390625,
      "std": 0.0
    }
  },
  "bp_threshold": {
    "accuracy": 0.45,
    "steps": {
      "mean": 266.6666666666667,
      "std": 115.47005383792515
    },
    "train_seconds": {
      "mean": 2.721017178332962,
      "std": 1.0941186816836288
    },
    "flops": {
      "mean": 1409030553600.0,
      "std": 610128127063.0255
    }
  },
  "alternatives_threshold": "0/3 seeds reached 45% by 4,000 steps",
  "censored_lower_bounds": {
    "fmad_compute_ratio_over_bp": {
      "mean": 5.298812563662495,
      "std": 1.8355625160095475
    },
    "zo_compute_ratio_over_bp": {
      "mean": 30.80047689700668,
      "std": 10.669598176593395
    },
    "bp_faster_than_fmad_percent": {
      "mean": 94.21806957039674,
      "std": 2.3902114480860948
    },
    "bp_faster_than_zo_percent": {
      "mean": 89.02650377320361,
      "std": 4.519499232535944
    }
  },
  "same_batch_seed0": {
    "scope": "scaled AG News proxy; not an exact full-paper replication",
    "dataset": "https://huggingface.co/datasets/fancyzhx/ag_news",
    "official_code": "https://github.com/Astuary/Gradient_Estimation_Methods/tree/bc5798add2339d1317664462648167fc6b987671",
    "hardware": {
      "device": "cuda",
      "gpu": "NVIDIA A10G",
      "torch": "2.7.1+cu128",
      "cuda": "12.8",
      "job_id": null
    },
    "configuration": {
      "output_dir": "/work/seed0",
      "train_size": 8192,
      "eval_size": 1024,
      "vocab_size": 12000,
      "seq_len": 64,
      "width": 192,
      "depth": 8,
      "rank": 1,
      "batch_size": 128,
      "steps": 400,
      "eval_every": 25,
      "threshold": 0.55,
      "perturbations": 1,
      "zo_eps": 0.001,
      "grad_clip": 5.0,
      "lr_bp": 0.003,
      "lr_fmad": 0.001,
      "lr_zo": 0.001,
      "seed": 0,
      "synthetic": false,
      "device": "cuda"
    },
    "methods": [
      {
        "method": "bp_checkpoint",
        "initial_accuracy": 0.25,
        "final_accuracy": 0.541015625,
        "best_accuracy": 0.541015625,
        "total_train_s": 4.09987770199848,
        "peak_memory_mb": 105.111328125,
        "step_flops": 14090305536,
        "threshold": 0.55,
        "threshold_step": null,
        "threshold_train_s": null,
        "threshold_flops": null,
        "threshold_censored": true
      },
      {
        "method": "fmad",
        "initial_accuracy": 0.25,
        "final_accuracy": 0.2509765625,
        "best_accuracy": 0.2548828125,
        "total_train_s": 4.869865859001152,
        "peak_memory_mb": 267.61181640625,
        "step_flops": 13439139840,
        "threshold": 0.55,
        "threshold_step": null,
        "threshold_train_s": null,
        "threshold_flops": null,
        "threshold_censored": true
      },
      {
        "method": "zo",
        "initial_accuracy": 0.25,
        "final_accuracy": 0.2509765625,
        "best_accuracy": 0.2548828125,
        "total_train_s": 2.5066474040000912,
        "peak_memory_mb": 57.150390625,
        "step_flops": 9764732928,
        "threshold": 0.55,
        "threshold_step": null,
        "threshold_train_s": null,
        "threshold_flops": null,
        "threshold_censored": true
      }
    ],
    "comparisons": {
      "bp_vs_fmad": {
        "final_accuracy_gap_percentage_points": 29.00390625,
        "final_accuracy_relative_gain_percent": 115.56420233463034,
        "peak_memory_ratio_bp_over_other": 0.392775362226289,
        "total_wall_time_ratio_other_over_bp": 1.1878075915843398
      },
      "bp_vs_zo": {
        "final_accuracy_gap_percentage_points": 29.00390625,
        "final_accuracy_relative_gain_percent": 115.56420233463034,
        "peak_memory_ratio_bp_over_other": 1.8392057687707186,
        "total_wall_time_ratio_other_over_bp": 0.6113956527967235
      }
    }
  }
}