Spaces:
Running
Running
Download content/consequences/rules.json from philippyt/ablation-lab: direct link, hf CLI and curl.
- Browser
- Download file 6.3 kB
-
https://huggingface.co/spaces/philippyt/ablation-lab/resolve/main/content/consequences/rules.json
- Command line
-
hf download hf://spaces/philippyt/ablation-lab/content/consequences/rules.json
-
curl -L -o rules.json https://huggingface.co/spaces/philippyt/ablation-lab/resolve/main/content/consequences/rules.json
6.3 kB
| [ | |
| { | |
| "id": "pos_embed_kindb_swap", | |
| "trigger": { | |
| "fields_changed": ["pos_embed"], | |
| "new_value_in": { "pos_embed": ["alibi", "sinusoidal", "nope"] }, | |
| "old_value_in": { "pos_embed": ["rope", "alibi", "sinusoidal", "nope"] } | |
| }, | |
| "template": "pos_embed swapped from {old.pos_embed} to {new.pos_embed}. The trained checkpoint expects {old.pos_embed} positional info baked into Q and K. At inference the model is now seeing {new.pos_embed} instead, with the same trained weights underneath. This is a Kind-B intervention applied live (no retraining), so the output will degrade. The degradation is the architectural lesson, not a crash. Compare against the rope baseline at the same shape to see what those positional bits were actually doing for quality." | |
| }, | |
| { | |
| "id": "pos_embed_kindb_nope_abs", | |
| "trigger": { | |
| "fields_changed": ["pos_embed"], | |
| "new_value_in": { "pos_embed": ["nope"] }, | |
| "old_value_in": { "pos_embed": ["absolute"] } | |
| }, | |
| "template": "pos_embed swapped from absolute to nope. The trained absolute position table is zeroed at inference. The model loses every positional cue it learned. Output degrades. The degradation is the lesson." | |
| }, | |
| { | |
| "id": "gqa_kv_reduction", | |
| "trigger": { | |
| "fields_changed": ["n_kv_heads"], | |
| "metric": "kv_cache_full_ctx", | |
| "direction": "decrease", | |
| "min_relative_change": 0.1 | |
| }, | |
| "template": "n_kv_heads went from {old.n_kv_heads} to {new.n_kv_heads}. KV cache at full context fell {pct} ({bytes.old} to {bytes.new}). Standard GQA shrinkage; each attention layer now stores {ratio}x less K and V." | |
| }, | |
| { | |
| "id": "mha_kv_growth", | |
| "trigger": { | |
| "fields_changed": ["n_kv_heads"], | |
| "metric": "kv_cache_full_ctx", | |
| "direction": "increase", | |
| "min_relative_change": 0.1 | |
| }, | |
| "template": "n_kv_heads went from {old.n_kv_heads} to {new.n_kv_heads}. KV cache at full context grew {pct} ({bytes.old} to {bytes.new}). More heads cache independent K and V." | |
| }, | |
| { | |
| "id": "ffn_dense_to_moe", | |
| "trigger": { | |
| "fields_changed": ["ffn"], | |
| "metric": "total_params", | |
| "direction": "increase" | |
| }, | |
| "template": "FFN switched to MoE with {new.n_experts} experts, top-{new.top_k}. Total params increased {pct} ({params.old} to {params.new}); active params per token stayed near one expert's worth." | |
| }, | |
| { | |
| "id": "ffn_moe_to_dense", | |
| "trigger": { | |
| "fields_changed": ["ffn"], | |
| "metric": "total_params", | |
| "direction": "decrease" | |
| }, | |
| "template": "FFN switched back to dense. Total and active params collapse onto a single FFN stack ({params.old} to {params.new})." | |
| }, | |
| { | |
| "id": "norm_kind", | |
| "trigger": { | |
| "fields_changed": ["norm"] | |
| }, | |
| "template": "norm changed from {old.norm} to {new.norm}. Per-norm parameter count changes accordingly (LayerNorm has weight+bias, RMSNorm has weight only); the difference is {params.diff} across the full stack." | |
| }, | |
| { | |
| "id": "pos_to_absolute", | |
| "trigger": { | |
| "fields_changed": ["pos_embed"], | |
| "metric": "total_params", | |
| "direction": "increase" | |
| }, | |
| "template": "pos_embed went to absolute. A learned position table of {pos_size} parameters was added to the embedding bank. Other position schemes (RoPE, ALiBi, NoPE) cost zero parameters." | |
| }, | |
| { | |
| "id": "pos_from_absolute", | |
| "trigger": { | |
| "fields_changed": ["pos_embed"], | |
| "metric": "total_params", | |
| "direction": "decrease" | |
| }, | |
| "template": "pos_embed left absolute. The {pos_size}-param position table is gone; new scheme stores no learned positional weights." | |
| }, | |
| { | |
| "id": "ctx_change", | |
| "trigger": { | |
| "fields_changed": ["ctx"], | |
| "metric": "kv_cache_full_ctx", | |
| "min_relative_change": 0.05 | |
| }, | |
| "template": "ctx changed from {old.ctx} to {new.ctx}. KV cache at full context scales linearly: {bytes.old} to {bytes.new}." | |
| }, | |
| { | |
| "id": "tie_to_untie", | |
| "trigger": { | |
| "fields_changed": ["tie_embeddings"], | |
| "metric": "total_params" | |
| }, | |
| "template": "tie_embeddings flipped from {old.tie_embeddings} to {new.tie_embeddings}. LM head and token embedding now {tied_state}; total params changed by {params.diff} (vocab_size·d_model = {head_size})." | |
| }, | |
| { | |
| "id": "qk_norm_on", | |
| "trigger": { | |
| "fields_changed": ["qk_norm"] | |
| }, | |
| "template": "qk_norm went from {old.qk_norm} to {new.qk_norm}. Q/K normalization adds {qkn_size} parameters per attention block; stabilizes attention logits at large widths." | |
| }, | |
| { | |
| "id": "mla_switch", | |
| "trigger": { | |
| "fields_changed": ["attention_type"] | |
| }, | |
| "template": "attention_type changed from {old.attention_type} to {new.attention_type}. KV per token is now {bytes.kv.new} (was {bytes.kv.old}). MLA caches a low-rank latent instead of full K and V." | |
| }, | |
| { | |
| "id": "layers_change", | |
| "trigger": { | |
| "fields_changed": ["n_layers"], | |
| "metric": "total_params", | |
| "min_relative_change": 0.05 | |
| }, | |
| "template": "n_layers changed from {old.n_layers} to {new.n_layers}. Block params scale linearly: {params.old} to {params.new}. KV cache scales linearly too." | |
| }, | |
| { | |
| "id": "d_model_change", | |
| "trigger": { | |
| "fields_changed": ["d_model"], | |
| "metric": "total_params" | |
| }, | |
| "template": "d_model changed from {old.d_model} to {new.d_model}. Attention QKVO scales quadratically with d_model, FFN hidden scales linearly. Total params {params.old} to {params.new}." | |
| }, | |
| { | |
| "id": "ablation_attn_off", | |
| "trigger": { | |
| "fields_changed": ["enable_attention"] | |
| }, | |
| "template": "enable_attention toggled to {new.enable_attention}. Attention sublayer params and KV cache zero out when disabled; the block becomes a pure FFN residual stack." | |
| }, | |
| { | |
| "id": "ablation_ffn_off", | |
| "trigger": { | |
| "fields_changed": ["enable_ffn"] | |
| }, | |
| "template": "enable_ffn toggled to {new.enable_ffn}. FFN sublayer params zero out when disabled; the block becomes attention-only." | |
| }, | |
| { | |
| "id": "ablation_norms_off", | |
| "trigger": { | |
| "fields_changed": ["enable_norms"] | |
| }, | |
| "template": "enable_norms toggled to {new.enable_norms}. All normalization layers zero out when disabled; activations are unscaled across the residual stream." | |
| } | |
| ] | |