reorg: remove superseded path svd-llm-v2/

svd-llm-v2/README.md

@@ -1,43 +0,0 @@
----
-license: llama2
-base_model: NousResearch/Llama-2-7b-chat-hf
-tags:
-  - svd
-  - low-rank
-  - model-compression
-  - llama2
----
-
-# SVD-LLM V2 (NAACL 2025): LLaMA-2-7B-chat @ 4.02B, KD-healed
-
-SVD-LLM V2: heterogeneous compression-ratio allocation from theoretical truncation loss (Algorithm 1) + loss-optimized weight truncation (Algorithm 2). Official code was never released; this is our faithful implementation of the published algorithms (per-layer ranks in ranks.json show the allocation).
-
-- **4.023B params** (down from 6.74B, ~40% reduction)
-- Healed with pure forward-KL distillation from the dense 7B teacher: **60M tokens**,
-  held-out KL **2.5806 -> 0.3365**, probe MMLU (5-shot, limit 40/subtask) **25.70% -> 40.83%**
-- Formal full-benchmark evaluation under the locked protocol (lm-eval 0.4.12, fp16, seed
-  1234, 5/10/0/5/5-shot MMLU/HS/BQ/TQA/GSM): **[being finalized; see
-  the TR-3 paper's Table 2]**
-
-| Reference | MMLU | HellaSwag | BoolQ | TriviaQA | GSM8K |
-|---|---:|---:|---:|---:|---:|
-| unhealed 4.02B (same structure) | 26.85 | 34.58 | 37.83 | 1.81 | 0.00 |
-| TR-3 (same budget + same healing) | 43.21 | 70.75 | 77.09 | 36.93 | 9.40 |
-| FP16 dense 6.74B | 47.22 | 78.59 | 80.58 | 57.17 | 22.67 |
-
-## How to load
-
-```python
-import json
-from lowrank_linear import build, materialize_all
-
-model = build("NousResearch/Llama-2-7b-chat-hf",
-              json.load(open("ranks.json")), "state_dict_fp16_healed.pt",
-              device="cuda")
-# for generation/eval: fuse the factors first (inference-only)
-materialize_all(model)
-```
-
-`ranks.json` maps each compressed projection path to its integer rank; untouched weights are
-stock LLaMA-2-7B-chat tensors. The full healing-provenance record (data mixture & revisions,
-decontamination, SHA256 of inputs/checkpoint, 5M milestone trajectory) is in `provenance.json`.

svd-llm-v2/config.json

@@ -1,28 +0,0 @@
-{
-  "_name_or_path": "NousResearch/Llama-2-7b-chat-hf",
-  "architectures": [
-    "LlamaForCausalLM"
-  ],
-  "attention_bias": false,
-  "bos_token_id": 1,
-  "eos_token_id": 2,
-  "hidden_act": "silu",
-  "hidden_size": 4096,
-  "initializer_range": 0.02,
-  "intermediate_size": 11008,
-  "max_position_embeddings": 4096,
-  "model_type": "llama",
-  "num_attention_heads": 32,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "pad_token_id": 0,
-  "pretraining_tp": 1,
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
-  "rope_theta": 10000.0,
-  "tie_word_embeddings": false,
-  "torch_dtype": "float16",
-  "transformers_version": "4.35.2",
-  "use_cache": true,
-  "vocab_size": 32000
-}

svd-llm-v2/lowrank_linear.py

@@ -1,83 +0,0 @@
-"""Minimal standalone loader for the SVD-baseline compressed LLaMA-2-7B checkpoints.
-
-No dependency beyond torch + transformers. The state_dict stores every compressed
-projection as two factors of a LowRankLinear (A: out x k, B: k x in); ranks.json
-holds the per-layer integer rank of each projection. Untouched parameters
-(embeddings, norms) keep their stock transformers names.
-
-Usage:
-    import json, torch
-    from lowrank_linear import build
-    model = build(
-        base_id="NousResearch/Llama-2-7b-chat-hf",
-        ranks=json.load(open("ranks.json")),
-        state_dict_path="state_dict_fp16.pt",
-        device="cuda", dtype=torch.float16,
-    )
-    tokenizer = AutoTokenizer.from_pretrained("NousResearch/Llama-2-7b-chat-hf")
-    out = model.generate(tokenizer("2+2=", return_tensors="pt").to("cuda").input_ids,
-                         max_new_tokens=16, do_sample=False)
-    print(tokenizer.decode(out[0]))
-"""
-
-from __future__ import annotations
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-class LowRankLinear(nn.Module):
-    """y = A (B x).  A: out x r, B: r x in.  (LLaMA projections carry no bias.)"""
-
-    def __init__(self, A: torch.Tensor, B: torch.Tensor):
-        super().__init__()
-        self.A = nn.Parameter(A)
-        self.B = nn.Parameter(B)
-
-    def forward(self, x):
-        return F.linear(F.linear(x, self.B), self.A)
-
-    @torch.no_grad()
-    def materialize_(self) -> nn.Linear:
-        """Replace this module with the dense W = A @ B (inference only; never call
-        while training -- it cuts the gradient path)."""
-        W = (self.A @ self.B).to(self.A.dtype)
-        dense = nn.Linear(self.B.shape[1], self.A.shape[0], bias=False,
-                          device=W.device, dtype=W.dtype)
-        dense.weight.data = W
-        return dense
-
-
-def build(base_id: str, ranks: dict, state_dict_path: str,
-          device: str = "cuda", dtype=torch.float16):
-    from transformers import AutoModelForCausalLM
-
-    model = AutoModelForCausalLM.from_pretrained(base_id, torch_dtype=dtype)
-    search = {}
-    for name, mod in model.named_modules():
-        if isinstance(mod, nn.Linear):
-            search[name] = mod
-    for path, k in ranks.items():
-        out_f, in_f = search[path].weight.shape
-        model_mod = model
-        for p in path.split(".")[:-1]:
-            model_mod = getattr(model_mod, p)
-        setattr(model_mod, path.split(".")[-1],
-                LowRankLinear(torch.zeros(out_f, int(k), dtype=dtype),
-                              torch.zeros(int(k), in_f, dtype=dtype)))
-    sd = torch.load(state_dict_path, map_location="cpu", weights_only=True)
-    model.load_state_dict(sd, strict=True)
-    del sd
-    return model.to(device)
-
-
-def materialize_all(model):
-    """Fuse every LowRankLinear back into a dense nn.Linear in place."""
-    for name, mod in list(model.named_modules()):
-        if isinstance(mod, LowRankLinear):
-            parent = model
-            for p in name.split(".")[:-1]:
-                parent = getattr(parent, p)
-            setattr(parent, name.split(".")[-1], mod.materialize_())
-    return model

svd-llm-v2/provenance.json

@@ -1,196 +0,0 @@
-{
-  "method": "KD healing: alpha*T^2*KL(teacher||student) + (1-alpha)*CE",
-  "student_init": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/svdllm2_r04193_tr3fmt/state_dict_fp16.pt",
-  "teacher": "NousResearch/Llama-2-7b-chat-hf",
-  "alpha": 1.0,
-  "temperature": 1.0,
-  "lr": 2e-05,
-  "tokens_seen": 59998208,
-  "epochs": 0.228,
-  "global_batch_tokens": 32768,
-  "dataset_mixture_pre_exclusion": {
-    "alpaca": 50000,
-    "openhermes": 200000,
-    "ultrachat": 100000
-  },
-  "dataset_revisions": {
-    "tatsu-lab/alpaca": "dce01c9b08f87459cf36a430d809084718273017",
-    "teknium/OpenHermes-2.5": "b82037821055c377bed0d495e72e46de3bc72e84",
-    "HuggingFaceH4/ultrachat_200k": "8049631c405ae6576f93f445c6b8166f76f5505a"
-  },
-  "canonical_pool_excluded_texts": 10000,
-  "train_texts_after_exclusion": 356000,
-  "chat_texts": 340000,
-  "mmlu_aux_texts_kept": 16000,
-  "mmlu_aux_split": "cais/mmlu auxiliary_train (official TRAIN split, disjoint from the evaluated test split; audit verified 0/99842 exact overlap)",
-  "mmlu_aux_stats": {
-    "requested_groups": 16000,
-    "rendered_groups": 16000,
-    "shots": 5,
-    "rows_total": 99842,
-    "rows_dropped_by_decontam": 71,
-    "rows_clean": 99771,
-    "revision": "c30699e8356da336a370243923dbaf21066bb9fe",
-    "format": "MMLU-shaped (NOT byte-identical to lm-eval; auxiliary_train has no subject)"
-  },
-  "gold_label_supervision": false,
-  "attribution_note": "alpha=1.0 => pure KD, no gold-label CE, so any MMLU change is attributable to distillation on in-domain text. alpha<1 adds CE on text containing answers = labeled MCQ training, which is a DIFFERENT and weaker claim.",
-  "train_tokens_total": 263112704,
-  "decontamination": {
-    "removed": {
-      "alpaca": 11,
-      "openhermes": 120,
-      "ultrachat": 33
-    }
-  },
-  "start_kl": 2.5805691182613373,
-  "final_kl": 0.33650884963572025,
-  "start_mmlu": 0.25701754385964914,
-  "final_mmlu": 0.4083333333333333,
-  "history": [
-    {
-      "tokens_m": 5.01,
-      "step": 153,
-      "held_out_kl": 0.647051740437746,
-      "mmlu": 0.2894736842105263
-    },
-    {
-      "tokens_m": 10.03,
-      "step": 306,
-      "held_out_kl": 0.594494417309761,
-      "mmlu": 0.368859649122807
-    },
-    {
-      "tokens_m": 15.01,
-      "step": 458,
-      "held_out_kl": 0.5670272633433342,
-      "mmlu": 0.3592105263157895
-    },
-    {
-      "tokens_m": 20.02,
-      "step": 611,
-      "held_out_kl": 0.5243954174220562,
-      "mmlu": 0.36622807017543857
-    },
-    {
-      "tokens_m": 25.0,
-      "step": 763,
-      "held_out_kl": 0.5039057768881321,
-      "mmlu": 0.38859649122807016
-    },
-    {
-      "tokens_m": 30.02,
-      "step": 916,
-      "held_out_kl": 0.44497134163975716,
-      "mmlu": 0.37543859649122807
-    },
-    {
-      "tokens_m": 35.03,
-      "step": 1069,
-      "held_out_kl": 0.402653768658638,
-      "mmlu": 0.39342105263157895
-    },
-    {
-      "tokens_m": 40.01,
-      "step": 1221,
-      "held_out_kl": 0.380719942972064,
-      "mmlu": 0.393859649122807
-    },
-    {
-      "tokens_m": 45.02,
-      "step": 1374,
-      "held_out_kl": 0.3613550402224064,
-      "mmlu": 0.4105263157894737
-    },
-    {
-      "tokens_m": 50.0,
-      "step": 1526,
-      "held_out_kl": 0.3475096616894007,
-      "mmlu": 0.39780701754385966
-    },
-    {
-      "tokens_m": 55.02,
-      "step": 1679,
-      "held_out_kl": 0.34086795896291733,
-      "mmlu": 0.40482456140350875
-    },
-    {
-      "tokens_m": 60.0,
-      "step": 1831,
-      "held_out_kl": 0.33650884963572025,
-      "mmlu": 0.4083333333333333,
-      "final": true
-    }
-  ],
-  "teacher_mmlu_same_limit": 0.47543859649122805,
-  "checkpoint": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm2r04193_h1/state_dict_fp16.pt",
-  "sha256": "2c48578d7ddbe5f937c2436f35ad8ffbda01d3be45b4c4eab62e1e1c6104501c",
-  "seed": 1234,
-  "mmlu_limit_per_subtask": 40,
-  "teacher_revision": "351844e75ed0bcbbe3f10671b3c808d2b83894ee",
-  "student_init_sha256": "33e0dba497c0bfe3b4532b4c6c0e07274a2ca85219b475fb2c26978d5ae6dfbd",
-  "ranks_file": "/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/svdllm2_r04193_tr3fmt/ranks.json",
-  "ranks_sha256": "90ba50885421afe8d0d19024aaf52bfbbee39433f99b1e934c4bf15a20f556a3",
-  "input_identity_gate": {
-    "student_ckpt": {
-      "key": "svdllm2_r04193_tr3fmt",
-      "sha256": "33e0dba497c0bfe3b4532b4c6c0e07274a2ca85219b475fb2c26978d5ae6dfbd",
-      "status": "unregistered_allowed"
-    },
-    "ranks": {
-      "key": "svdllm2_r04193_tr3fmt",
-      "sha256": "90ba50885421afe8d0d19024aaf52bfbbee39433f99b1e934c4bf15a20f556a3",
-      "status": "unregistered_allowed"
-    }
-  },
-  "train_seed": 1234,
-  "mmlu_seed": 1234,
-  "dynamic_batch": false,
-  "dbl": null,
-  "snapshot_milestones": true,
-  "snapshots": [
-    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm2r04193_h1_snap50m'), 50.003968)",
-    "(PosixPath('/data1/rkp00022/zixuan/projects/TR3_main/checkpoints/kdheal_svdllm2r04193_h1_snap55m'), 55.017472)"
-  ],
-  "stream_skip_tokens_m": 0.0,
-  "stream_blocks_consumed": [
-    0,
-    58592
-  ],
-  "stream_note": "the block order is a seeded permutation consumed from index 0; stages sharing a --seed with stream_skip=0 re-train on the SAME blocks. [start, end] block indices are recorded so any continuation can be checked for overlap with what earlier stages consumed.",
-  "student_master_dtype": "bf16",
-  "fake_quant": null,
-  "rebalance_report": null,
-  "student_param_count": 4023119360,
-  "micro": 16,
-  "grad_accum": 2,
-  "seq_len": 1024,
-  "held_out_source": "canonical: mixture 2000/4000/4000, seed 1234, seq 1024, blocks 256:264 (identical to the compression gates)",
-  "decontam_stats": {
-    "mmlu": {
-      "items": 14042,
-      "new_ngrams": 764261,
-      "revision": "c30699e8356da336a370243923dbaf21066bb9fe"
-    },
-    "hellaswag": {
-      "items": 10042,
-      "new_ngrams": 1334302,
-      "revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76"
-    },
-    "boolq": {
-      "items": 3270,
-      "new_ngrams": 276806,
-      "revision": "3de24cf8022e94f4ee4b9d55a6f539891524d646"
-    },
-    "triviaqa": {
-      "items": 17944,
-      "new_ngrams": 32608,
-      "revision": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f"
-    },
-    "gsm8k": {
-      "items": 1319,
-      "new_ngrams": 46281,
-      "revision": "740312add88f781978c0658806c59bc2815b9866"
-    }
-  }
-}

svd-llm-v2/ranks.json

@@ -1,226 +0,0 @@
-{
- "model.layers.0.self_attn.q_proj": 2048,
- "model.layers.0.self_attn.k_proj": 2048,
- "model.layers.0.self_attn.v_proj": 2048,
- "model.layers.0.self_attn.o_proj": 2048,
- "model.layers.0.mlp.gate_proj": 2341,
- "model.layers.0.mlp.up_proj": 2332,
- "model.layers.0.mlp.down_proj": 2887,
- "model.layers.1.self_attn.q_proj": 1911,
- "model.layers.1.self_attn.k_proj": 1960,
- "model.layers.1.self_attn.v_proj": 1826,
- "model.layers.1.self_attn.o_proj": 2048,
- "model.layers.1.mlp.gate_proj": 2115,
- "model.layers.1.mlp.up_proj": 2094,
- "model.layers.1.mlp.down_proj": 2280,
- "model.layers.2.self_attn.q_proj": 1404,
- "model.layers.2.self_attn.k_proj": 1405,
- "model.layers.2.self_attn.v_proj": 1400,
- "model.layers.2.self_attn.o_proj": 1662,
- "model.layers.2.mlp.gate_proj": 1931,
- "model.layers.2.mlp.up_proj": 1928,
- "model.layers.2.mlp.down_proj": 2094,
- "model.layers.3.self_attn.q_proj": 1252,
- "model.layers.3.self_attn.k_proj": 1247,
- "model.layers.3.self_attn.v_proj": 1285,
- "model.layers.3.self_attn.o_proj": 1598,
- "model.layers.3.mlp.gate_proj": 1821,
- "model.layers.3.mlp.up_proj": 1826,
- "model.layers.3.mlp.down_proj": 2002,
- "model.layers.4.self_attn.q_proj": 1255,
- "model.layers.4.self_attn.k_proj": 1252,
- "model.layers.4.self_attn.v_proj": 1282,
- "model.layers.4.self_attn.o_proj": 1490,
- "model.layers.4.mlp.gate_proj": 1780,
- "model.layers.4.mlp.up_proj": 1789,
- "model.layers.4.mlp.down_proj": 1911,
- "model.layers.5.self_attn.q_proj": 1239,
- "model.layers.5.self_attn.k_proj": 1236,
- "model.layers.5.self_attn.v_proj": 1266,
- "model.layers.5.self_attn.o_proj": 1456,
- "model.layers.5.mlp.gate_proj": 1757,
- "model.layers.5.mlp.up_proj": 1765,
- "model.layers.5.mlp.down_proj": 1855,
- "model.layers.6.self_attn.q_proj": 1218,
- "model.layers.6.self_attn.k_proj": 1214,
- "model.layers.6.self_attn.v_proj": 1244,
- "model.layers.6.self_attn.o_proj": 1393,
- "model.layers.6.mlp.gate_proj": 1733,
- "model.layers.6.mlp.up_proj": 1743,
- "model.layers.6.mlp.down_proj": 1807,
- "model.layers.7.self_attn.q_proj": 1209,
- "model.layers.7.self_attn.k_proj": 1208,
- "model.layers.7.self_attn.v_proj": 1237,
- "model.layers.7.self_attn.o_proj": 1356,
- "model.layers.7.mlp.gate_proj": 1718,
- "model.layers.7.mlp.up_proj": 1726,
- "model.layers.7.mlp.down_proj": 1770,
- "model.layers.8.self_attn.q_proj": 1205,
- "model.layers.8.self_attn.k_proj": 1201,
- "model.layers.8.self_attn.v_proj": 1238,
- "model.layers.8.self_attn.o_proj": 1314,
- "model.layers.8.mlp.gate_proj": 1715,
- "model.layers.8.mlp.up_proj": 1718,
- "model.layers.8.mlp.down_proj": 1747,
- "model.layers.9.self_attn.q_proj": 1193,
- "model.layers.9.self_attn.k_proj": 1190,
- "model.layers.9.self_attn.v_proj": 1229,
- "model.layers.9.self_attn.o_proj": 1276,
- "model.layers.9.mlp.gate_proj": 1711,
- "model.layers.9.mlp.up_proj": 1711,
- "model.layers.9.mlp.down_proj": 1725,
- "model.layers.10.self_attn.q_proj": 1201,
- "model.layers.10.self_attn.k_proj": 1194,
- "model.layers.10.self_attn.v_proj": 1230,
- "model.layers.10.self_attn.o_proj": 1252,
- "model.layers.10.mlp.gate_proj": 1708,
- "model.layers.10.mlp.up_proj": 1706,
- "model.layers.10.mlp.down_proj": 1713,
- "model.layers.11.self_attn.q_proj": 1199,
- "model.layers.11.self_attn.k_proj": 1196,
- "model.layers.11.self_attn.v_proj": 1213,
- "model.layers.11.self_attn.o_proj": 1258,
- "model.layers.11.mlp.gate_proj": 1699,
- "model.layers.11.mlp.up_proj": 1694,
- "model.layers.11.mlp.down_proj": 1699,
- "model.layers.12.self_attn.q_proj": 1180,
- "model.layers.12.self_attn.k_proj": 1179,
- "model.layers.12.self_attn.v_proj": 1206,
- "model.layers.12.self_attn.o_proj": 1234,
- "model.layers.12.mlp.gate_proj": 1692,
- "model.layers.12.mlp.up_proj": 1686,
- "model.layers.12.mlp.down_proj": 1683,
- "model.layers.13.self_attn.q_proj": 1176,
- "model.layers.13.self_attn.k_proj": 1170,
- "model.layers.13.self_attn.v_proj": 1190,
- "model.layers.13.self_attn.o_proj": 1231,
- "model.layers.13.mlp.gate_proj": 1688,
- "model.layers.13.mlp.up_proj": 1678,
- "model.layers.13.mlp.down_proj": 1661,
- "model.layers.14.self_attn.q_proj": 1181,
- "model.layers.14.self_attn.k_proj": 1179,
- "model.layers.14.self_attn.v_proj": 1192,
- "model.layers.14.self_attn.o_proj": 1213,
- "model.layers.14.mlp.gate_proj": 1679,
- "model.layers.14.mlp.up_proj": 1669,
- "model.layers.14.mlp.down_proj": 1644,
- "model.layers.15.self_attn.q_proj": 1177,
- "model.layers.15.self_attn.k_proj": 1174,
- "model.layers.15.self_attn.v_proj": 1187,
- "model.layers.15.self_attn.o_proj": 1198,
- "model.layers.15.mlp.gate_proj": 1670,
- "model.layers.15.mlp.up_proj": 1661,
- "model.layers.15.mlp.down_proj": 1621,
- "model.layers.16.self_attn.q_proj": 1176,
- "model.layers.16.self_attn.k_proj": 1174,
- "model.layers.16.self_attn.v_proj": 1173,
- "model.layers.16.self_attn.o_proj": 1175,
- "model.layers.16.mlp.gate_proj": 1662,
- "model.layers.16.mlp.up_proj": 1655,
- "model.layers.16.mlp.down_proj": 1595,
- "model.layers.17.self_attn.q_proj": 1163,
- "model.layers.17.self_attn.k_proj": 1163,
- "model.layers.17.self_attn.v_proj": 1167,
- "model.layers.17.self_attn.o_proj": 1193,
- "model.layers.17.mlp.gate_proj": 1647,
- "model.layers.17.mlp.up_proj": 1643,
- "model.layers.17.mlp.down_proj": 1584,
- "model.layers.18.self_attn.q_proj": 1150,
- "model.layers.18.self_attn.k_proj": 1149,
- "model.layers.18.self_attn.v_proj": 1151,
- "model.layers.18.self_attn.o_proj": 1190,
- "model.layers.18.mlp.gate_proj": 1632,
- "model.layers.18.mlp.up_proj": 1631,
- "model.layers.18.mlp.down_proj": 1568,
- "model.layers.19.self_attn.q_proj": 1156,
- "model.layers.19.self_attn.k_proj": 1157,
- "model.layers.19.self_attn.v_proj": 1152,
- "model.layers.19.self_attn.o_proj": 1184,
- "model.layers.19.mlp.gate_proj": 1623,
- "model.layers.19.mlp.up_proj": 1623,
- "model.layers.19.mlp.down_proj": 1557,
- "model.layers.20.self_attn.q_proj": 1155,
- "model.layers.20.self_attn.k_proj": 1155,
- "model.layers.20.self_attn.v_proj": 1146,
- "model.layers.20.self_attn.o_proj": 1172,
- "model.layers.20.mlp.gate_proj": 1617,
- "model.layers.20.mlp.up_proj": 1619,
- "model.layers.20.mlp.down_proj": 1540,
- "model.layers.21.self_attn.q_proj": 1147,
- "model.layers.21.self_attn.k_proj": 1147,
- "model.layers.21.self_attn.v_proj": 1136,
- "model.layers.21.self_attn.o_proj": 1174,
- "model.layers.21.mlp.gate_proj": 1607,
- "model.layers.21.mlp.up_proj": 1611,
- "model.layers.21.mlp.down_proj": 1531,
- "model.layers.22.self_attn.q_proj": 1138,
- "model.layers.22.self_attn.k_proj": 1138,
- "model.layers.22.self_attn.v_proj": 1129,
- "model.layers.22.self_attn.o_proj": 1149,
- "model.layers.22.mlp.gate_proj": 1599,
- "model.layers.22.mlp.up_proj": 1604,
- "model.layers.22.mlp.down_proj": 1517,
- "model.layers.23.self_attn.q_proj": 1122,
- "model.layers.23.self_attn.k_proj": 1123,
- "model.layers.23.self_attn.v_proj": 1115,
- "model.layers.23.self_attn.o_proj": 1139,
- "model.layers.23.mlp.gate_proj": 1589,
- "model.layers.23.mlp.up_proj": 1593,
- "model.layers.23.mlp.down_proj": 1508,
- "model.layers.24.self_attn.q_proj": 1138,
- "model.layers.24.self_attn.k_proj": 1139,
- "model.layers.24.self_attn.v_proj": 1118,
- "model.layers.24.self_attn.o_proj": 1135,
- "model.layers.24.mlp.gate_proj": 1580,
- "model.layers.24.mlp.up_proj": 1585,
- "model.layers.24.mlp.down_proj": 1501,
- "model.layers.25.self_attn.q_proj": 1116,
- "model.layers.25.self_attn.k_proj": 1116,
- "model.layers.25.self_attn.v_proj": 1100,
- "model.layers.25.self_attn.o_proj": 1141,
- "model.layers.25.mlp.gate_proj": 1573,
- "model.layers.25.mlp.up_proj": 1577,
- "model.layers.25.mlp.down_proj": 1496,
- "model.layers.26.self_attn.q_proj": 1127,
- "model.layers.26.self_attn.k_proj": 1126,
- "model.layers.26.self_attn.v_proj": 1103,
- "model.layers.26.self_attn.o_proj": 1121,
- "model.layers.26.mlp.gate_proj": 1565,
- "model.layers.26.mlp.up_proj": 1569,
- "model.layers.26.mlp.down_proj": 1487,
- "model.layers.27.self_attn.q_proj": 1108,
- "model.layers.27.self_attn.k_proj": 1107,
- "model.layers.27.self_attn.v_proj": 1095,
- "model.layers.27.self_attn.o_proj": 1109,
- "model.layers.27.mlp.gate_proj": 1557,
- "model.layers.27.mlp.up_proj": 1561,
- "model.layers.27.mlp.down_proj": 1475,
- "model.layers.28.self_attn.q_proj": 1113,
- "model.layers.28.self_attn.k_proj": 1111,
- "model.layers.28.self_attn.v_proj": 1090,
- "model.layers.28.self_attn.o_proj": 1105,
- "model.layers.28.mlp.gate_proj": 1555,
- "model.layers.28.mlp.up_proj": 1556,
- "model.layers.28.mlp.down_proj": 1462,
- "model.layers.29.self_attn.q_proj": 1128,
- "model.layers.29.self_attn.k_proj": 1127,
- "model.layers.29.self_attn.v_proj": 1094,
- "model.layers.29.self_attn.o_proj": 1094,
- "model.layers.29.mlp.gate_proj": 1552,
- "model.layers.29.mlp.up_proj": 1553,
- "model.layers.29.mlp.down_proj": 1450,
- "model.layers.30.self_attn.q_proj": 1116,
- "model.layers.30.self_attn.k_proj": 1115,
- "model.layers.30.self_attn.v_proj": 1087,
- "model.layers.30.self_attn.o_proj": 1093,
- "model.layers.30.mlp.gate_proj": 1552,
- "model.layers.30.mlp.up_proj": 1556,
- "model.layers.30.mlp.down_proj": 1439,
- "model.layers.31.self_attn.q_proj": 1156,
- "model.layers.31.self_attn.k_proj": 1152,
- "model.layers.31.self_attn.v_proj": 1126,
- "model.layers.31.self_attn.o_proj": 1102,
- "model.layers.31.mlp.gate_proj": 1570,
- "model.layers.31.mlp.up_proj": 1576,
- "model.layers.31.mlp.down_proj": 1429
-}

svd-llm-v2/state_dict_fp16_healed.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2c48578d7ddbe5f937c2436f35ad8ffbda01d3be45b4c4eab62e1e1c6104501c
-size 8046409836