ablation-lab / frontend /src /content /rules.json
philippyt's picture
setting up training kind A and kind B sets, improvements to frontend
e7b5745
Raw History Blame Contribute Delete
6.3 kB
[
{
"id": "pos_embed_kindb_swap",
"trigger": {
"fields_changed": ["pos_embed"],
"new_value_in": { "pos_embed": ["alibi", "sinusoidal", "nope"] },
"old_value_in": { "pos_embed": ["rope", "alibi", "sinusoidal", "nope"] }
},
"template": "pos_embed swapped from {old.pos_embed} to {new.pos_embed}. The trained checkpoint expects {old.pos_embed} positional info baked into Q and K. At inference the model is now seeing {new.pos_embed} instead, with the same trained weights underneath. This is a Kind-B intervention applied live (no retraining), so the output will degrade. The degradation is the architectural lesson, not a crash. Compare against the rope baseline at the same shape to see what those positional bits were actually doing for quality."
},
{
"id": "pos_embed_kindb_nope_abs",
"trigger": {
"fields_changed": ["pos_embed"],
"new_value_in": { "pos_embed": ["nope"] },
"old_value_in": { "pos_embed": ["absolute"] }
},
"template": "pos_embed swapped from absolute to nope. The trained absolute position table is zeroed at inference. The model loses every positional cue it learned. Output degrades. The degradation is the lesson."
},
{
"id": "gqa_kv_reduction",
"trigger": {
"fields_changed": ["n_kv_heads"],
"metric": "kv_cache_full_ctx",
"direction": "decrease",
"min_relative_change": 0.1
},
"template": "n_kv_heads went from {old.n_kv_heads} to {new.n_kv_heads}. KV cache at full context fell {pct} ({bytes.old} to {bytes.new}). Standard GQA shrinkage; each attention layer now stores {ratio}x less K and V."
},
{
"id": "mha_kv_growth",
"trigger": {
"fields_changed": ["n_kv_heads"],
"metric": "kv_cache_full_ctx",
"direction": "increase",
"min_relative_change": 0.1
},
"template": "n_kv_heads went from {old.n_kv_heads} to {new.n_kv_heads}. KV cache at full context grew {pct} ({bytes.old} to {bytes.new}). More heads cache independent K and V."
},
{
"id": "ffn_dense_to_moe",
"trigger": {
"fields_changed": ["ffn"],
"metric": "total_params",
"direction": "increase"
},
"template": "FFN switched to MoE with {new.n_experts} experts, top-{new.top_k}. Total params increased {pct} ({params.old} to {params.new}); active params per token stayed near one expert's worth."
},
{
"id": "ffn_moe_to_dense",
"trigger": {
"fields_changed": ["ffn"],
"metric": "total_params",
"direction": "decrease"
},
"template": "FFN switched back to dense. Total and active params collapse onto a single FFN stack ({params.old} to {params.new})."
},
{
"id": "norm_kind",
"trigger": {
"fields_changed": ["norm"]
},
"template": "norm changed from {old.norm} to {new.norm}. Per-norm parameter count changes accordingly (LayerNorm has weight+bias, RMSNorm has weight only); the difference is {params.diff} across the full stack."
},
{
"id": "pos_to_absolute",
"trigger": {
"fields_changed": ["pos_embed"],
"metric": "total_params",
"direction": "increase"
},
"template": "pos_embed went to absolute. A learned position table of {pos_size} parameters was added to the embedding bank. Other position schemes (RoPE, ALiBi, NoPE) cost zero parameters."
},
{
"id": "pos_from_absolute",
"trigger": {
"fields_changed": ["pos_embed"],
"metric": "total_params",
"direction": "decrease"
},
"template": "pos_embed left absolute. The {pos_size}-param position table is gone; new scheme stores no learned positional weights."
},
{
"id": "ctx_change",
"trigger": {
"fields_changed": ["ctx"],
"metric": "kv_cache_full_ctx",
"min_relative_change": 0.05
},
"template": "ctx changed from {old.ctx} to {new.ctx}. KV cache at full context scales linearly: {bytes.old} to {bytes.new}."
},
{
"id": "tie_to_untie",
"trigger": {
"fields_changed": ["tie_embeddings"],
"metric": "total_params"
},
"template": "tie_embeddings flipped from {old.tie_embeddings} to {new.tie_embeddings}. LM head and token embedding now {tied_state}; total params changed by {params.diff} (vocab_size·d_model = {head_size})."
},
{
"id": "qk_norm_on",
"trigger": {
"fields_changed": ["qk_norm"]
},
"template": "qk_norm went from {old.qk_norm} to {new.qk_norm}. Q/K normalization adds {qkn_size} parameters per attention block; stabilizes attention logits at large widths."
},
{
"id": "mla_switch",
"trigger": {
"fields_changed": ["attention_type"]
},
"template": "attention_type changed from {old.attention_type} to {new.attention_type}. KV per token is now {bytes.kv.new} (was {bytes.kv.old}). MLA caches a low-rank latent instead of full K and V."
},
{
"id": "layers_change",
"trigger": {
"fields_changed": ["n_layers"],
"metric": "total_params",
"min_relative_change": 0.05
},
"template": "n_layers changed from {old.n_layers} to {new.n_layers}. Block params scale linearly: {params.old} to {params.new}. KV cache scales linearly too."
},
{
"id": "d_model_change",
"trigger": {
"fields_changed": ["d_model"],
"metric": "total_params"
},
"template": "d_model changed from {old.d_model} to {new.d_model}. Attention QKVO scales quadratically with d_model, FFN hidden scales linearly. Total params {params.old} to {params.new}."
},
{
"id": "ablation_attn_off",
"trigger": {
"fields_changed": ["enable_attention"]
},
"template": "enable_attention toggled to {new.enable_attention}. Attention sublayer params and KV cache zero out when disabled; the block becomes a pure FFN residual stack."
},
{
"id": "ablation_ffn_off",
"trigger": {
"fields_changed": ["enable_ffn"]
},
"template": "enable_ffn toggled to {new.enable_ffn}. FFN sublayer params zero out when disabled; the block becomes attention-only."
},
{
"id": "ablation_norms_off",
"trigger": {
"fields_changed": ["enable_norms"]
},
"template": "enable_norms toggled to {new.enable_norms}. All normalization layers zero out when disabled; activations are unscaled across the residual stream."
}
]