epic_quant/layers.py

"""
Layer-aware parameter key layout for Gemma 4 E4B.

Verified directly against the safetensors header (2130 tensors).
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import List, Tuple


@dataclass
class LayerDims:
    """Dimensions for a single layer, variant-aware."""
    layer_idx: int
    is_global: bool            # full_attention vs sliding_attention
    head_dim: int              # 256 sliding, 512 global
    q_out: int                 # 2048 sliding, 4096 global
    kv_out: int                # 512 sliding, 1024 global
    hidden: int                # 2560 always


def layer_param_keys(layer_idx: int) -> List[str]:
    """The 17 BF16 parameter keys that make up one language-model decoder block.

    Plus the shared PLE contribution that this layer reads from the *single*
    2D embed_tokens_per_layer table.
    """
    base = f"model.language_model.layers.{layer_idx}"
    return [
        f"{base}.input_layernorm.weight",
        f"{base}.post_attention_layernorm.weight",
        f"{base}.pre_feedforward_layernorm.weight",
        f"{base}.post_feedforward_layernorm.weight",
        f"{base}.post_per_layer_input_norm.weight",
        f"{base}.self_attn.q_proj.weight",
        f"{base}.self_attn.k_proj.weight",
        f"{base}.self_attn.v_proj.weight",
        f"{base}.self_attn.o_proj.weight",
        f"{base}.self_attn.q_norm.weight",
        f"{base}.self_attn.k_norm.weight",
        f"{base}.mlp.gate_proj.weight",
        f"{base}.mlp.up_proj.weight",
        f"{base}.mlp.down_proj.weight",
        f"{base}.per_layer_input_gate.weight",
        f"{base}.per_layer_projection.weight",
        f"{base}.layer_scalar",
    ]


def ple_columns_for_layer(layer_idx: int, num_layers: int = 42,
                          per_layer_dim: int = 256) -> Tuple[int, int]:
    """PLE is a single 2D matrix [vocab, num_layers * per_layer_dim].

    Each layer's slice is columns [layer_idx * per_layer_dim,
    (layer_idx + 1) * per_layer_dim)."""
    start = layer_idx * per_layer_dim
    end = start + per_layer_dim
    return start, end


def get_layer_dims(layer_idx: int, layer_types: List[str]) -> LayerDims:
    is_global = layer_types[layer_idx] == "full_attention"
    if is_global:
        return LayerDims(layer_idx=layer_idx, is_global=True,
                         head_dim=512, q_out=4096, kv_out=1024, hidden=2560)
    return LayerDims(layer_idx=layer_idx, is_global=False,
                     head_dim=256, q_out=2048, kv_out=512, hidden=2560)