analysis/memory_matched_seed2_summary.json

{
  "scope": "scaled AG News proxy; not an exact full-paper replication",
  "dataset": "https://huggingface.co/datasets/fancyzhx/ag_news",
  "official_code": "https://github.com/Astuary/Gradient_Estimation_Methods/tree/bc5798add2339d1317664462648167fc6b987671",
  "hardware": {
    "device": "cuda",
    "gpu": "NVIDIA A10G",
    "torch": "2.7.1+cu128",
    "cuda": "12.8",
    "job_id": null
  },
  "configuration": {
    "output_dir": "/work/memory_matched_seed2",
    "train_size": 8192,
    "eval_size": 1024,
    "vocab_size": 12000,
    "seq_len": 64,
    "width": 192,
    "depth": 8,
    "rank": 1,
    "batch_size": 128,
    "batch_size_bp": 48,
    "batch_size_fmad": 16,
    "batch_size_zo": 128,
    "steps": 4000,
    "eval_every": 200,
    "threshold": 0.45,
    "perturbations": 1,
    "zo_eps": 0.001,
    "grad_clip": 5.0,
    "lr_bp": 0.003,
    "lr_fmad": 0.001,
    "lr_zo": 0.001,
    "seed": 2,
    "synthetic": false,
    "device": "cuda"
  },
  "methods": [
    {
      "method": "bp_checkpoint",
      "batch_size": 48,
      "initial_accuracy": 0.267578125,
      "final_accuracy": 0.6904296875,
      "best_accuracy": 0.7119140625,
      "total_train_s": 39.12202145099832,
      "peak_memory_mb": 58.61767578125,
      "step_flops": 5283864576,
      "threshold": 0.45,
      "threshold_step": 200,
      "threshold_train_s": 2.042041762999361,
      "threshold_flops": 1056772915200,
      "threshold_censored": false
    },
    {
      "method": "fmad",
      "batch_size": 16,
      "initial_accuracy": 0.267578125,
      "final_accuracy": 0.2890625,
      "best_accuracy": 0.2939453125,
      "total_train_s": 46.945645891000254,
      "peak_memory_mb": 57.091796875,
      "step_flops": 1679892480,
      "threshold": 0.45,
      "threshold_step": null,
      "threshold_train_s": null,
      "threshold_flops": null,
      "threshold_censored": true
    },
    {
      "method": "zo",
      "batch_size": 128,
      "initial_accuracy": 0.267578125,
      "final_accuracy": 0.28515625,
      "best_accuracy": 0.322265625,
      "total_train_s": 24.68838089699625,
      "peak_memory_mb": 57.150390625,
      "step_flops": 9764732928,
      "threshold": 0.45,
      "threshold_step": null,
      "threshold_train_s": null,
      "threshold_flops": null,
      "threshold_censored": true
    }
  ],
  "comparisons": {
    "bp_vs_fmad": {
      "final_accuracy_gap_percentage_points": 40.13671875,
      "final_accuracy_relative_gain_percent": 138.85135135135135,
      "peak_memory_ratio_bp_over_other": 1.0267267626834524,
      "total_wall_time_ratio_other_over_bp": 1.1999800662090352
    },
    "bp_vs_zo": {
      "final_accuracy_gap_percentage_points": 40.52734375,
      "final_accuracy_relative_gain_percent": 142.12328767123287,
      "peak_memory_ratio_bp_over_other": 1.0256741054646117,
      "total_wall_time_ratio_other_over_bp": 0.6310609723456979
    }
  }
}