ablation-lab / scripts /generate_configs.py
philippyt's picture
setting up training kind A and kind B sets, improvements to frontend
e7b5745
Raw History Blame Contribute Delete
3.48 kB
from __future__ import annotations
import itertools
import json
import os
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
OUT_DIR = ROOT / "training" / "configs"
BASE_ARCH = {
"d_model": 128,
"n_layers": 4,
"n_heads": 8,
"head_dim": 16,
"ctx": 256,
"ffn_mult": 4,
"tie_embeddings": True,
}
BASE_TRAIN = {
"seed": 1337,
"lr": 3e-4,
"batch_size": 64,
"steps": 3000,
"warmup_steps": 100,
"eval_interval": 250,
"eval_iters": 40,
"weight_decay": 0.01,
"grad_clip": 1.0,
"dataset_path": "data/tinystories.txt",
"device": "auto",
}
NORMS = ["layernorm", "rmsnorm"]
ATTENTIONS = [
("mha", {"n_kv_heads": 8, "attention_type": "standard"}),
("gqa", {"n_kv_heads": 2, "attention_type": "standard"}),
("mqa", {"n_kv_heads": 1, "attention_type": "standard"}),
("mla", {"n_kv_heads": 1, "attention_type": "mla"}),
]
FFNS = [
("dense", {"ffn": "dense"}),
("moe", {"ffn": "moe", "n_experts": 8, "top_k": 2}),
]
ACTIVATIONS = ["silu", "gelu", "relu2"]
QK_NORMS = [False, True]
POS_EMBEDS = ["rope", "absolute"]
def short_norm(n: str) -> str:
return "ln" if n == "layernorm" else "rms"
def short_act(a: str) -> str:
return {"silu": "silu", "gelu": "gelu", "relu2": "rl2"}[a]
def short_pos(p: str) -> str:
return "rope" if p == "rope" else "abs"
def main() -> None:
OUT_DIR.mkdir(parents=True, exist_ok=True)
existing = {p.name for p in OUT_DIR.glob("*.json")}
written = 0
pruned = 0
for norm, (attn_name, attn_patch), (ffn_name, ffn_patch), act, qkn, pos in itertools.product(
NORMS, ATTENTIONS, FFNS, ACTIVATIONS, QK_NORMS, POS_EMBEDS
):
# PRUNE: MLA × absolute pos is architecturally incoherent
if attn_name == "mla" and pos == "absolute":
pruned += 1
continue
arch = dict(BASE_ARCH)
arch.update(attn_patch)
arch.update(ffn_patch)
arch["norm"] = norm
arch["activation"] = act
arch["qk_norm"] = qkn
arch["pos_embed"] = pos
cfg_id = f"{short_norm(norm)}_{attn_name}_{ffn_name}_{short_act(act)}_qkn{'1' if qkn else '0'}_{short_pos(pos)}"
name = f"{norm}+{attn_name}+{ffn_name}+{act}+qkn={'on' if qkn else 'off'}+{pos}"
note = "auto-generated Kind-A config"
cfg = {"name": name, "_note": note, "arch": arch, "train": dict(BASE_TRAIN)}
out_path = OUT_DIR / f"{cfg_id}.json"
with out_path.open("w") as f:
json.dump(cfg, f, indent=2, sort_keys=True)
written += 1
# bonus: small MoE (4 experts, top-1) on rms+gqa+silu+qkn=off+rope baseline
arch = dict(BASE_ARCH)
arch.update({"n_kv_heads": 2, "attention_type": "standard"})
arch.update({"ffn": "moe", "n_experts": 4, "top_k": 1})
arch["norm"] = "rmsnorm"
arch["activation"] = "silu"
arch["qk_norm"] = False
arch["pos_embed"] = "rope"
cfg = {
"name": "rmsnorm+gqa+moe_small(4,1)+silu+qkn=off+rope",
"_note": "bonus: small-MoE (4 experts, top-1) to demonstrate total/active gap shrinking toward dense",
"arch": arch,
"train": dict(BASE_TRAIN),
}
out_path = OUT_DIR / "rms_gqa_moe-small_silu_qkn0_rope.json"
with out_path.open("w") as f:
json.dump(cfg, f, indent=2, sort_keys=True)
written += 1
print(f"wrote {written} configs to {OUT_DIR} (pruned {pruned} MLA×absolute combos)")
if __name__ == "__main__":
main()