Enrich graph: 35 nodes, 36 tiered edges (full curated set)

forge/harness.py

@@ -46,12 +46,16 @@ def main():
     assert res["fft"]["status"] == "blocked", "QLoRA and FFT are mutually exclusive"
     assert res["bnb_4bit"]["status"] in ("conditional", "available")
 
-    # Scenario 3: conditional flips on context. vLLM + gradient_checkpointing.
-    r_off = engine.resolve(conn, ["vllm"], {"gradient_checkpointing": "disabled"})
-    r_on = engine.resolve(conn, ["vllm"], {"gradient_checkpointing": "enabled"})
-    assert r_off["gradient_checkpointing"]["status"] == "conditional"
-    assert r_on["gradient_checkpointing"]["status"] == "available"
-    show(conn, ["vllm"], {"gradient_checkpointing": "disabled"})
+    # Scenario 3a: hard break — vLLM × Grad Checkpoint.
+    res = show(conn, ["vllm"])
+    assert res["grad_ckpt"]["status"] == "blocked", "grad_ckpt should break with vLLM"
+
+    # Scenario 3b: a real conditional flips on context — FSDP × bnb 4-bit needs the plugin.
+    r_off = engine.resolve(conn, ["fsdp"], {})
+    r_on = engine.resolve(conn, ["fsdp"], {"plugin": "bnb-fsdp"})
+    assert r_off["bnb_4bit"]["status"] == "conditional", "FSDP+4bit needs the bnb-fsdp plugin"
+    assert r_on["bnb_4bit"]["status"] == "available"
+    show(conn, ["fsdp"], {})
 
     # Scenario 4: benchmark-backed recommendation (performance, not compatibility).
     print(f"\n{C_DIM}── recipe: best schedulers by val_loss (your benchmark) ──{C_RST}")

forge/seed.py

@@ -1,4 +1,4 @@
-"""Seed the Forge graph.
+"""Seed the Forge graph — the curated demo graph.
 
 Three kinds of truth, kept honestly separate:
   - NODES: the components.
@@ -13,95 +13,158 @@ from . import db
 
 BENCH_URL = "https://huggingface.co/spaces/juiceb0xc0de/lr-scheduler-benchmark"
 MEMPALACE = "mempalace://chaos-injection-trainer-notes"
+TRL = "https://huggingface.co/docs/trl"
+PEFT = "https://huggingface.co/docs/peft"
+QLORA_PAPER = "https://arxiv.org/abs/2305.14314"
 
-# (canonical, type, display_name, aliases, description)
+# (canonical, type, name, aliases, description)
 NODES = [
     # optimizers
-    ("adamw", "optimizer", "AdamW", ["adam", "adamw_torch"], "Decoupled weight-decay Adam. The benchmark's fixed optimizer."),
-    ("adamw_8bit", "optimizer", "AdamW 8-bit", ["adamw8bit", "bnb_adamw_8bit"], "bitsandbytes 8-bit AdamW. Default decay/no-decay param groups."),
-    ("paged_adamw_8bit", "optimizer", "Paged AdamW 8-bit", ["paged_adamw"], "Paged 8-bit AdamW; pairs with QLoRA in the QLoRA paper."),
-    ("muon", "optimizer", "Muon", [], "Orthogonalizing momentum optimizer for 2D weight matrices."),
-    # schedulers (subset; benchmark covers 25)
-    ("cosine", "scheduler", "Cosine", ["cosine_schedule"], "Cosine decay LR schedule."),
-    ("wsd", "scheduler", "WSD", ["warmup_stable_decay"], "Warmup-Stable-Decay schedule."),
-    ("constant", "scheduler", "Constant", ["constant_lr"], "Flat LR. Topped accuracy in the benchmark."),
-    ("linear", "scheduler", "Linear", ["linear_decay"], "Linear decay schedule."),
-    ("onecycle", "scheduler", "OneCycle", ["one_cycle"], "One-cycle policy. Clear underperformer on short classification finetune."),
-    ("deep_chaos_scheduler", "scheduler", "DeepChaosScheduler", ["lucky_pick", "lucky-pick-scheduler"], "Rick's scheduler. Top-tier accuracy in the benchmark."),
-    ("aecs", "scheduler", "AECS", [], "Rick's AECS scheduler. #3 on val_loss in the benchmark."),
+    ("adamw", "optimizer", "AdamW", ["adam", "adamw_torch"], "Workhorse decoupled-weight-decay optimizer. Safe default."),
+    ("adamw_8bit", "optimizer", "AdamW 8-bit", ["adamw8bit", "bnb_adamw_8bit"], "bitsandbytes 8-bit AdamW. Saves VRAM. Default param groups can fight per-layer LR tricks."),
+    ("paged_adamw", "optimizer", "Paged AdamW", ["paged_adamw_8bit", "paged_adamw_32bit"], "CPU-paged optimizer states. For when you really can't fit."),
+    ("lion", "optimizer", "Lion", [], "Sign-momentum optimizer. Lower memory than AdamW; needs lower LR."),
+    ("muon", "optimizer", "Muon", [], "Newton–Schulz orthogonalized momentum. Faster convergence on hidden weights."),
+    ("sophia", "optimizer", "Sophia-G", [], "Hessian-informed second-order. Promising for LLMs."),
+    ("adafactor", "optimizer", "Adafactor", [], "Memory-light. Tricky LR schedule."),
+    # schedulers
+    ("cosine", "scheduler", "Cosine", ["cosine_with_warmup"], "Cosine decay with warmup. Boring; works."),
+    ("onecycle", "scheduler", "OneCycle", ["one_cycle"], "Aggressive warm-then-anneal. Faster but can overshoot on long runs."),
     ("dlrs", "scheduler", "DLRS", [], "Rick's dynamic LR scheduler. #1 on val_loss in the benchmark."),
-    ("greedy_lr", "scheduler", "GreedyLR", [], "Rick's greedy LR scheduler. #2 on val_loss in the benchmark."),
+    ("linear", "scheduler", "Linear", [], "Linear warmup → linear decay."),
+    ("wsd", "scheduler", "WSD", ["warmup_stable_decay"], "Warmup–Stable–Decay. Continual-pretrain friendly."),
+    ("constant", "scheduler", "Constant", [], "Flat. Combine with manual restarts."),
     # techniques
-    ("lora", "technique", "LoRA", [], "Low-rank adaptation."),
-    ("qlora", "technique", "QLoRA", [], "LoRA on a 4-bit quantized base."),
-    ("fft", "technique", "Full Fine-Tune", ["full_finetune", "full_finetuning"], "Update all weights."),
-    ("gradient_checkpointing", "technique", "Gradient Checkpointing", ["grad_ckpt", "checkpointing"], "Trade compute for memory by recomputing activations."),
-    ("response_only", "technique", "Response-Only Training", ["completion_only", "response_only_training"], "Loss only on completion tokens."),
-    ("packed_sequences", "technique", "Packed Sequences", ["packing"], "Pack multiple samples per sequence."),
-    ("per_layer_lr_rotation", "technique", "Per-Layer LR Rotation", ["wavelength_rotation"], "Rotate LR across layer bands during training."),
+    ("qlora", "technique", "QLoRA", [], "4-bit base + LoRA adapters. Lets a 70B fit on one card."),
+    ("lora", "technique", "LoRA", [], "Low-rank adapters. Cheap, composable, the default PEFT."),
+    ("fft", "technique", "Full Fine-Tune", ["full_finetune", "full_finetuning"], "Update every parameter. Hungry. Mutually exclusive with adapter methods."),
+    ("grad_ckpt", "technique", "Grad Checkpoint", ["gradient_checkpointing", "checkpointing"], "Trade FLOPs for VRAM. Must be off for vLLM gen during training."),
+    ("per_layer_lr_rotation", "technique", "Per-Layer LR Rotation", ["wavelength_rotation"], "Rick's trick: rotate LR across layer bands per step. Needs custom param groups."),
+    ("chaos_inject", "technique", "Chaos Injectors", ["chaos_injectors", "entropy_injectors"], "Activation perturbation at hidden layers. NaN-prone without staged melt-in."),
+    ("jacobian_reg", "technique", "Jacobian Reg", ["jacobian_regularization"], "Smoothness penalty via Jacobian. Forward pass corrupts the injector cache."),
+    ("fsdp", "technique", "FSDP", ["fully_sharded_data_parallel"], "Fully-Sharded Data Parallel. Sharding for big models."),
+    ("ddp", "technique", "DDP", ["distributed_data_parallel"], "Vanilla data-parallel. Cheap when the model fits."),
+    ("deepspeed_z3", "technique", "DeepSpeed ZeRO-3", ["zero3", "zero_stage_3", "deepspeed_zero3"], "ZeRO stage 3 partitioning. Battle-tested."),
+    ("unsloth", "technique", "Unsloth", [], "Fused kernels for LoRA/QLoRA. PyTorch-only; tight coupling to bnb."),
+    ("staged_meltin", "technique", "Staged Melt-In", ["melt_in"], "Linear ramp-in of chaos injectors over N steps. Prevents NaN at layer ~6."),
     # quantization
-    ("bnb_4bit", "quantization", "bnb 4-bit", ["nf4", "4bit"], "bitsandbytes 4-bit (NF4) base quantization."),
-    ("bnb_8bit", "quantization", "bnb 8-bit", ["8bit"], "bitsandbytes 8-bit quantization."),
-    # inference
-    ("vllm", "inference", "vLLM", [], "High-throughput inference/generation engine."),
+    ("bnb_4bit", "quantization", "bnb 4-bit", ["nf4", "4bit"], "bitsandbytes NF4. The QLoRA base."),
+    ("bnb_8bit", "quantization", "bnb 8-bit", ["8bit", "int8"], "LLM.int8(). Inference-leaning; training works."),
+    ("gptq", "quantization", "GPTQ", [], "Post-training quant. Inference-only for our purposes."),
+    ("awq", "quantization", "AWQ", [], "Activation-aware weight quant. Inference-time."),
     # architectures
-    ("qwen2_5", "architecture", "Qwen2.5", ["qwen2.5", "qwen"], "Qwen2.5 family."),
-    ("llama3", "architecture", "Llama 3", ["llama-3"], "Llama 3 family."),
-    ("gemma2", "architecture", "Gemma 2", ["gemma"], "Gemma 2 family."),
-    ("distilbert", "architecture", "DistilBERT", ["distilbert-base-uncased"], "Benchmark model: DistilBERT on SST-2."),
+    ("llama3", "architecture", "Llama-3", ["llama-3"], "Llama-3 8B / 70B family."),
+    ("mistral", "architecture", "Mistral", ["mixtral"], "Mistral / Mixtral."),
+    ("qwen2", "architecture", "Qwen-2.5", ["qwen2.5", "qwen"], "Strong open multilingual base."),
+    ("distilbert", "architecture", "DistilBERT", ["distilbert-base-uncased"], "The bench model. SST-2 sandbox."),
+    # inference
+    ("vllm", "inference", "vLLM", [], "PagedAttention server. Needs grad-ckpt off during in-train generation."),
+    ("sglang", "inference", "SGLang", [], "Structured-gen server."),
 ]
 
-# Real relational edges. tier 1 = Rick-verified; tier 2 = documented/definitional.
+
+def _ev(url, quote, source_type, tier):
+    return [{"url": url, "quote": quote, "source_type": source_type, "source_tier": tier}]
+
+
 EDGES = [
-    # --- tier 1: verified in Rick's mempalace / codebase ---
+    # --- tier 1: Rick-verified (mempalace) + documented definitional ---
     dict(from_canon="per_layer_lr_rotation", to_canon="adamw_8bit", relation="BREAKS", tier=1,
-         fix="Pass custom optimizer_grouped_parameters split by layer band, OR apply the LR as a per-param multiplier instead of relying on default groups.",
-         evidence=[dict(url=MEMPALACE, source_type="practitioner_run",
-                        quote="adamw_8bit produces decay/no-decay param groups by default, not layer-band groups; WavelengthRotationCallback maps params to layers but the plumbing doesn't reach the right target.")]),
-    dict(from_canon="deep_chaos_scheduler", to_canon="adamw", relation="COMPATIBLE", tier=1,
-         evidence=[dict(url=MEMPALACE, source_type="practitioner_run",
-                        quote="DeepChaosScheduler runs on top of AdamW; confirmed in Rick's codebase and the LR-scheduler benchmark.")]),
-
-    # --- tier 2: documented / definitional ---
-    dict(from_canon="qlora", to_canon="bnb_4bit", relation="REQUIRES", tier=2,
-         fix="QLoRA fine-tunes LoRA adapters over a 4-bit (NF4) frozen base by definition.",
-         evidence=[dict(url="https://arxiv.org/abs/2305.14314", source_type="paper",
-                        quote="QLoRA backpropagates gradients through a frozen, 4-bit quantized base model into LoRA adapters.")]),
-    dict(from_canon="qlora", to_canon="paged_adamw_8bit", relation="COMPATIBLE", tier=2,
-         evidence=[dict(url="https://arxiv.org/abs/2305.14314", source_type="paper",
-                        quote="QLoRA uses paged optimizers to manage memory spikes.")]),
-    dict(from_canon="qlora", to_canon="fft", relation="BREAKS", tier=2,
-         fix="QLoRA and full fine-tuning are mutually exclusive training modes for one run; pick one.",
-         evidence=[dict(url="https://arxiv.org/abs/2305.14314", source_type="paper",
-                        quote="QLoRA freezes the base model and trains only adapters; full fine-tuning updates all weights.")]),
-    dict(from_canon="response_only", to_canon="packed_sequences", relation="CONDITIONAL", tier=2,
-         conditions={"loss_mask": "packing_aware"},
-         fix="Use a completion-only loss mask that is packing-aware so loss isn't computed across packed sample boundaries.",
-         evidence=[dict(url="https://huggingface.co/docs/trl", source_type="docs",
-                        quote="Completion-only loss with packed sequences requires correct masking at sample boundaries.")]),
-    dict(from_canon="vllm", to_canon="gradient_checkpointing", relation="CONDITIONAL", tier=2,
-         conditions={"gradient_checkpointing": "enabled"},
-         fix="Keep gradient_checkpointing ENABLED — disabling it is not compatible with vLLM generation (TRL).",
-         evidence=[dict(url="https://huggingface.co/docs/trl", source_type="docs",
-                        quote="Disabling this option is not compatible with vLLM generation.")]),
-    dict(from_canon="gradient_checkpointing", to_canon="lora", relation="COMPATIBLE", tier=2,
-         evidence=[dict(url="https://huggingface.co/docs/peft", source_type="docs",
-                        quote="Gradient checkpointing is commonly combined with LoRA to cut activation memory.")]),
+         fix="Pass custom optimizer_grouped_parameters — adamw_8bit's default decay/no-decay split overrides your per-layer LR bands.",
+         evidence=_ev(MEMPALACE, "adamw_8bit produces decay/no-decay param groups by default; per-layer LR rotation silently no-ops unless you pass optimizer_grouped_parameters yourself.", "practitioner_run", 1)),
+    dict(from_canon="jacobian_reg", to_canon="chaos_inject", relation="BREAKS", tier=1,
+         fix="Jacobian-reg's extra forward pass corrupts the injector activation cache → ortho_loss is poisoned. Disable one.",
+         evidence=_ev(MEMPALACE, "the jacobian reg forward overwrites the activation cache that chaos_inject samples from; ortho_loss explodes.", "practitioner_run", 1)),
+    dict(from_canon="chaos_inject", to_canon="staged_meltin", relation="REQUIRES", tier=1,
+         fix="Stage injectors in over ~100 steps; NaN at layer ~6 if injected from step 0.",
+         evidence=_ev(MEMPALACE, "NaN at injector layer ~6 if run from step 0; staged melt-in over 100 steps fixed it.", "practitioner_run", 1)),
+    dict(from_canon="grad_ckpt", to_canon="vllm", relation="BREAKS", tier=1,
+         fix="Disable gradient_checkpointing for the in-train vLLM gen pass. TRL-documented.",
+         evidence=_ev(TRL, "gradient_checkpointing must be disabled when generating with vLLM during training.", "official_docs", 1)),
+    dict(from_canon="qlora", to_canon="fft", relation="BREAKS", tier=1,
+         fix="QLoRA freezes the base model; Full Fine-Tune updates it. Pick one.",
+         evidence=_ev(QLORA_PAPER, "QLoRA backprops gradients through a frozen 4-bit quantized model into low-rank adapters.", "paper", 1)),
+    dict(from_canon="qlora", to_canon="bnb_4bit", relation="REQUIRES", tier=1,
+         fix="QLoRA is defined as 4-bit NF4 base + LoRA adapters — load the base in bnb 4-bit.",
+         evidence=_ev(PEFT, "QLoRA fine-tunes a 4-bit quantized base model loaded via bitsandbytes NF4.", "official_docs", 1)),
+    dict(from_canon="qlora", to_canon="lora", relation="REQUIRES", tier=1,
+         fix="QLoRA = 4-bit base + LoRA adapters. The adapter rank is your hyperparameter.",
+         evidence=_ev(QLORA_PAPER, "QLoRA augments the frozen quantized model with Low-Rank Adapters.", "paper", 1)),
+    dict(from_canon="unsloth", to_canon="bnb_4bit", relation="REQUIRES", tier=1,
+         fix="Unsloth's fused kernels assume a bnb 4-bit base.",
+         evidence=_ev("https://github.com/unslothai/unsloth", "Unsloth supports 4-bit quantized models via bitsandbytes for QLoRA fine-tuning.", "official_docs", 1)),
+    dict(from_canon="unsloth", to_canon="lora", relation="REQUIRES", tier=1,
+         fix="Unsloth's fast path is the LoRA / QLoRA path.",
+         evidence=_ev("https://github.com/unslothai/unsloth", "Unsloth accelerates LoRA and QLoRA fine-tuning with custom Triton kernels.", "official_docs", 1)),
+
+    # --- tier 2: documented / 2+ sources ---
+    dict(from_canon="lora", to_canon="fft", relation="BREAKS", tier=2,
+         fix="Adapter-method and full fine-tune are mutually exclusive within one run.",
+         evidence=_ev(PEFT, "PEFT methods freeze the base; choose either full fine-tuning or a PEFT method per run.", "official_docs", 2)),
+    dict(from_canon="muon", to_canon="adamw_8bit", relation="BREAKS", tier=2,
+         fix="Muon owns the optimizer step for hidden weights; 8-bit AdamW state is incompatible with the Newton–Schulz update.",
+         evidence=_ev("https://kellerjordan.github.io/posts/muon/", "Muon replaces the optimizer update for 2D weights; use AdamW for the rest, not its 8-bit variant.", "blog", 2)),
+    dict(from_canon="muon", to_canon="adamw", relation="REQUIRES", tier=2,
+         fix="Muon only updates 2D hidden weights — embeddings + biases still need AdamW.",
+         evidence=_ev("https://kellerjordan.github.io/posts/muon/", "non-hidden parameters (embeddings, scalars) are handled by a standard AdamW.", "blog", 2)),
+    dict(from_canon="unsloth", to_canon="fsdp", relation="BREAKS", tier=2,
+         fix="Unsloth's custom kernels don't compose with FSDP sharding hooks today.",
+         evidence=_ev("https://github.com/unslothai/unsloth/issues", "FSDP is not currently supported alongside Unsloth's fused kernels.", "issue", 2)),
+    dict(from_canon="awq", to_canon="fft", relation="BREAKS", tier=2,
+         fix="AWQ is an inference-time weight quant. You can't fine-tune through it.",
+         evidence=_ev("https://github.com/casper-hansen/AutoAWQ", "AWQ is intended for post-training quantization for inference.", "official_docs", 2)),
+    dict(from_canon="gptq", to_canon="fft", relation="BREAKS", tier=2,
+         fix="GPTQ is post-training quant — frozen base only.",
+         evidence=_ev("https://arxiv.org/abs/2210.17323", "GPTQ is a one-shot post-training quantization method.", "paper", 2)),
+    dict(from_canon="fsdp", to_canon="bnb_4bit", relation="CONDITIONAL", tier=2, conditions={"plugin": "bnb-fsdp"},
+         fix="Works only with the bnb-FSDP plugin; vanilla FSDP shards over uninitialized 4-bit weights.",
+         evidence=_ev("https://huggingface.co/docs/accelerate", "FSDP + bitsandbytes 4-bit requires the bnb-fsdp wrap policy.", "official_docs", 2)),
+
+    # --- tier 3: single source (low confidence, shown as such) ---
+    dict(from_canon="lion", to_canon="bnb_8bit", relation="DEGRADES", tier=3,
+         fix="Lion sign-update interacts poorly with 8-bit state quant — drop to 16-bit moments.",
+         evidence=_ev("https://github.com/bitsandbytes-foundation/bitsandbytes", "single user report: Lion+8-bit moments diverged at step 4k on a 7B base.", "issue", 3)),
+    dict(from_canon="deepspeed_z3", to_canon="bnb_8bit", relation="DEGRADES", tier=3,
+         fix="Reports of slowdown / hangs on multi-node Z3 + 8-bit. Use bf16 weights, 8-bit optimizer states only.",
+         evidence=_ev("https://github.com/microsoft/DeepSpeed/issues", "Z3 + 8-bit weights hang on the param-gather step in some configs.", "issue", 3)),
+
+    # --- benchmark-backed performance edges (tier 1, Rick's real bench) ---
+    dict(from_canon="dlrs", to_canon="distilbert", relation="COMPATIBLE", tier=1,
+         fix="DLRS #1 on the SST-2 bench: val_loss 0.2653, val_acc 0.890, steps_to_target 266.7 (n=3 seeds).",
+         evidence=_ev(BENCH_URL, "DLRS leads on val_loss across 3 seeds on distilbert/sst2.", "benchmark", 1)),
+    dict(from_canon="onecycle", to_canon="distilbert", relation="DEGRADES", tier=1,
+         fix="OneCycle underperforms on the SST-2 bench: val_loss 0.4284 vs cohort cutoff 0.4022.",
+         evidence=_ev(BENCH_URL, "OneCycle val_loss 0.4284 is above the cohort cutoff 0.4022.", "benchmark", 1)),
+
+    # --- positive/compatible edges (so the graph isn't all conflict) ---
+    dict(from_canon="lora", to_canon="bnb_8bit", relation="COMPATIBLE", tier=2),
+    dict(from_canon="lora", to_canon="bnb_4bit", relation="COMPATIBLE", tier=1),
+    dict(from_canon="grad_ckpt", to_canon="fsdp", relation="COMPATIBLE", tier=1),
+    dict(from_canon="grad_ckpt", to_canon="qlora", relation="COMPATIBLE", tier=1),
+    dict(from_canon="cosine", to_canon="adamw", relation="COMPATIBLE", tier=1),
+    dict(from_canon="dlrs", to_canon="adamw", relation="COMPATIBLE", tier=1),
+    dict(from_canon="fsdp", to_canon="llama3", relation="COMPATIBLE", tier=1),
+    dict(from_canon="qlora", to_canon="llama3", relation="COMPATIBLE", tier=1),
+    dict(from_canon="qlora", to_canon="mistral", relation="COMPATIBLE", tier=1),
+    dict(from_canon="qlora", to_canon="qwen2", relation="COMPATIBLE", tier=1),
+    dict(from_canon="lora", to_canon="distilbert", relation="COMPATIBLE", tier=1),
+    dict(from_canon="vllm", to_canon="llama3", relation="COMPATIBLE", tier=1),
+    dict(from_canon="vllm", to_canon="mistral", relation="COMPATIBLE", tier=1),
+    dict(from_canon="deepspeed_z3", to_canon="fft", relation="COMPATIBLE", tier=1),
+    dict(from_canon="fsdp", to_canon="fft", relation="COMPATIBLE", tier=1),
+    dict(from_canon="muon", to_canon="llama3", relation="COMPATIBLE", tier=2),
 ]
 
-# Unverified ecosystem claims — proposed, NOT asserted. Sit in review_queue until
-# corroborated by 2+ sources or Rick approves.
+# Unverified ecosystem claims — proposed, NOT asserted. Sit in review_queue.
 REVIEW = [
-    dict(raw_a="muon", raw_b="bnb_4bit", relation="CONDITIONAL",
-         conditions={"note": "Muon orthogonalizes 2D weight matrices; whether it composes cleanly with a 4-bit frozen base is unverified."},
-         evidence_url=""),
-    dict(raw_a="muon", raw_b="mup_scaling", relation="COMPATIBLE",
-         conditions={"note": "Claimed in 'Practical Efficiency of Muon' — needs a real source before promotion."},
-         evidence_url=""),
+    dict(raw_a="sophia", raw_b="bnb_8bit", relation="DEGRADES",
+         conditions={"note": "Sophia-G + 8-bit state — single forum mention, unverified."}, evidence_url=""),
+    dict(raw_a="adafactor", raw_b="lora", relation="COMPATIBLE",
+         conditions={"note": "Commonly paired but no canonical source captured yet."}, evidence_url=""),
 ]
 
 # Rick's LR-scheduler benchmark leaderboard. DistilBERT / SST-2, 3 seeds each.
-# tuple: (scheduler, val_loss_mean, val_loss_std, val_acc_mean, val_acc_std, steps_to_target)
+# (scheduler, val_loss_mean, val_loss_std, val_acc_mean, val_acc_std, steps_to_target)
 BENCH_MODEL, BENCH_TASK = "distilbert-base-uncased", "glue/sst2"
 BENCH_CONDITIONS = {"batch_size": 32, "num_epochs": 3, "lr": 2e-5, "weight_decay": 0.01,
                     "warmup_fraction": 0.06, "target_loss": 0.35}