analysis/same_batch_seed0_summary.json

{
  "scope": "scaled AG News proxy; not an exact full-paper replication",
  "dataset": "https://huggingface.co/datasets/fancyzhx/ag_news",
  "official_code": "https://github.com/Astuary/Gradient_Estimation_Methods/tree/bc5798add2339d1317664462648167fc6b987671",
  "hardware": {
    "device": "cuda",
    "gpu": "NVIDIA A10G",
    "torch": "2.7.1+cu128",
    "cuda": "12.8",
    "job_id": null
  },
  "configuration": {
    "output_dir": "/work/seed0",
    "train_size": 8192,
    "eval_size": 1024,
    "vocab_size": 12000,
    "seq_len": 64,
    "width": 192,
    "depth": 8,
    "rank": 1,
    "batch_size": 128,
    "steps": 400,
    "eval_every": 25,
    "threshold": 0.55,
    "perturbations": 1,
    "zo_eps": 0.001,
    "grad_clip": 5.0,
    "lr_bp": 0.003,
    "lr_fmad": 0.001,
    "lr_zo": 0.001,
    "seed": 0,
    "synthetic": false,
    "device": "cuda"
  },
  "methods": [
    {
      "method": "bp_checkpoint",
      "initial_accuracy": 0.25,
      "final_accuracy": 0.541015625,
      "best_accuracy": 0.541015625,
      "total_train_s": 4.09987770199848,
      "peak_memory_mb": 105.111328125,
      "step_flops": 14090305536,
      "threshold": 0.55,
      "threshold_step": null,
      "threshold_train_s": null,
      "threshold_flops": null,
      "threshold_censored": true
    },
    {
      "method": "fmad",
      "initial_accuracy": 0.25,
      "final_accuracy": 0.2509765625,
      "best_accuracy": 0.2548828125,
      "total_train_s": 4.869865859001152,
      "peak_memory_mb": 267.61181640625,
      "step_flops": 13439139840,
      "threshold": 0.55,
      "threshold_step": null,
      "threshold_train_s": null,
      "threshold_flops": null,
      "threshold_censored": true
    },
    {
      "method": "zo",
      "initial_accuracy": 0.25,
      "final_accuracy": 0.2509765625,
      "best_accuracy": 0.2548828125,
      "total_train_s": 2.5066474040000912,
      "peak_memory_mb": 57.150390625,
      "step_flops": 9764732928,
      "threshold": 0.55,
      "threshold_step": null,
      "threshold_train_s": null,
      "threshold_flops": null,
      "threshold_censored": true
    }
  ],
  "comparisons": {
    "bp_vs_fmad": {
      "final_accuracy_gap_percentage_points": 29.00390625,
      "final_accuracy_relative_gain_percent": 115.56420233463034,
      "peak_memory_ratio_bp_over_other": 0.392775362226289,
      "total_wall_time_ratio_other_over_bp": 1.1878075915843398
    },
    "bp_vs_zo": {
      "final_accuracy_gap_percentage_points": 29.00390625,
      "final_accuracy_relative_gain_percent": 115.56420233463034,
      "peak_memory_ratio_bp_over_other": 1.8392057687707186,
      "total_wall_time_ratio_other_over_bp": 0.6113956527967235
    }
  }
}