diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_01_merge.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_01_merge.json new file mode 100644 index 0000000000000000000000000000000000000000..628314c914a5a6f9f064e0ef7be99bf3d14f7dad --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_01_merge.json @@ -0,0 +1,110 @@ +{ + "cycle": 1, + "anchor": 12, + "removed_original_layer": 13, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "ea43767088fa2f8cc0b7ba197024635f7b8a7e19fdd34170b499f470c2461672", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.12798181447783583, + "calibration_cosine": 0.9338195359584089, + "calibration_norm_ratio": 0.9329591782339236, + "transport_relative_identity_distance": 0.4961368441581726, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "67540e2474547f1d622b738128696574720591fddabb8157d2f6812ecc3af308", + "self_attn.k_proj.weight": "efbe8b71b99ef90277849155ebc45e9aad5ffe746f2958c077c7c7130d783c7b", + "self_attn.v_proj.weight": "c53545e1ac9a6447efa83242434a0b28bcda5fe12ef3866e173dd13c3da45e38", + "self_attn.o_proj.weight": "8812afef76626e47fb5dffe3a77a0cb07a07e8a7f5cf3a97453368a1284b0ffa", + "self_attn.q_norm.weight": "79bbb659cb1c2af322ca85abf7d26b088c837bcd284e70efa95a6449f69b5673", + "self_attn.k_norm.weight": "e50a57833625792c16439de078eeb810df393e1a49ef7bbe43f5af6a93923adc", + "mlp.gate_proj.weight": "6bc0a4412e6639d40e3a7d5c9099a5a05f2573bd12744019550fc11da2c52c55", + "mlp.up_proj.weight": "8715dd4fa81b5d58a577a1b2fdfb24592b23775921014b36f832fe8dbf2bbf51", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b", + "input_layernorm.weight": "c419d161cc4932ecddf0ec98cbe5737b92688f0d5aa71a7e95d2006ca824ffca", + "post_attention_layernorm.weight": "ee6a9a96c2b4f35eb0807060229f10bc28080072ed9ecfa0a17e1aaa3cb4df40" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "67540e2474547f1d622b738128696574720591fddabb8157d2f6812ecc3af308", + "self_attn.k_proj.weight": "efbe8b71b99ef90277849155ebc45e9aad5ffe746f2958c077c7c7130d783c7b", + "self_attn.v_proj.weight": "c53545e1ac9a6447efa83242434a0b28bcda5fe12ef3866e173dd13c3da45e38", + "self_attn.o_proj.weight": "8812afef76626e47fb5dffe3a77a0cb07a07e8a7f5cf3a97453368a1284b0ffa", + "self_attn.q_norm.weight": "79bbb659cb1c2af322ca85abf7d26b088c837bcd284e70efa95a6449f69b5673", + "self_attn.k_norm.weight": "e50a57833625792c16439de078eeb810df393e1a49ef7bbe43f5af6a93923adc", + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b", + "input_layernorm.weight": "c419d161cc4932ecddf0ec98cbe5737b92688f0d5aa71a7e95d2006ca824ffca", + "post_attention_layernorm.weight": "ee6a9a96c2b4f35eb0807060229f10bc28080072ed9ecfa0a17e1aaa3cb4df40" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "3e138d6d890dfaeeeb88ae9b8a48c084aea69752230749e87f1586b975880ddd", + "mlp.up_proj.weight": "1f3dacc4c4a3af2a55d6273cfc2374077a2eda18cec7cf8237e05c54cc1b8ea5", + "mlp.down_proj.weight": "7a34a8a9bcc1a25c928397e3be5c409081b9de0ec283d13425482c0020e6db43" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "67540e2474547f1d622b738128696574720591fddabb8157d2f6812ecc3af308", + "self_attn.k_proj.weight": "efbe8b71b99ef90277849155ebc45e9aad5ffe746f2958c077c7c7130d783c7b", + "self_attn.v_proj.weight": "c53545e1ac9a6447efa83242434a0b28bcda5fe12ef3866e173dd13c3da45e38", + "self_attn.o_proj.weight": "8812afef76626e47fb5dffe3a77a0cb07a07e8a7f5cf3a97453368a1284b0ffa", + "self_attn.q_norm.weight": "79bbb659cb1c2af322ca85abf7d26b088c837bcd284e70efa95a6449f69b5673", + "self_attn.k_norm.weight": "e50a57833625792c16439de078eeb810df393e1a49ef7bbe43f5af6a93923adc", + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b", + "input_layernorm.weight": "c419d161cc4932ecddf0ec98cbe5737b92688f0d5aa71a7e95d2006ca824ffca", + "post_attention_layernorm.weight": "ee6a9a96c2b4f35eb0807060229f10bc28080072ed9ecfa0a17e1aaa3cb4df40" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_01_transport.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_01_transport.pt new file mode 100644 index 0000000000000000000000000000000000000000..e58dab4f7fe06e20db5edaf72b51682dd62858cb --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_01_transport.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce29905e20c096a8d0a555134ba437d93e2e81d3e91cbfdff0b3e62513583bbb +size 67110518 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_02_merge.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_02_merge.json new file mode 100644 index 0000000000000000000000000000000000000000..fb5d0948bc5b1627d6702f884ee7211386b7525c --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_02_merge.json @@ -0,0 +1,110 @@ +{ + "cycle": 2, + "anchor": 12, + "removed_original_layer": 14, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "2f2164ca8bffcf85ff9cfb9506308fa1340559aba7e44bda37d1e050e6e82d36", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.10099672573787757, + "calibration_cosine": 0.9481582440752598, + "calibration_norm_ratio": 0.9472741933087646, + "transport_relative_identity_distance": 0.45524656772613525, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "f57ad388d57bb515af2ac9324d86dc9e19c4cc44bc73879a7b409f9c07597c15", + "self_attn.k_proj.weight": "09cba09d72eea6b683087478aeaaf6d321423335a9eb57d47006adbc1f336c63", + "self_attn.v_proj.weight": "af4adee7b325a938beb530808104ab27a70afc0d494134b3de1f5ea8ef060773", + "self_attn.o_proj.weight": "6888fff18d668ea322341951eb6ab19a3fbbc4cf94b758ed0f8dba411a262d1f", + "self_attn.q_norm.weight": "e3e9ec56f12aa5ec410018bbac99e6953ad52f3b8b796d73c5bd0cc6545fb93b", + "self_attn.k_norm.weight": "a5bf1576b21d93da019094f1a61912669136a5d9e89e8d2301c88f6bf2def290", + "mlp.gate_proj.weight": "1d90e8cf937190e4027f0c7bc34bbabd5bb43c5f3d081537e0d866da8a3efe3f", + "mlp.up_proj.weight": "dc867f2d0718d761bb166f69bd21878837006daf9ae392c955a2e9578a15ad97", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d", + "input_layernorm.weight": "c21d58d80fa7aca24ff0dac67092f8204b3876926fa095bb7e6ceb3a16060706", + "post_attention_layernorm.weight": "1e599b9bd0ed7fdbcef34cff94651bf467bf923745a76a96d33a74ae551ad5d5" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "f57ad388d57bb515af2ac9324d86dc9e19c4cc44bc73879a7b409f9c07597c15", + "self_attn.k_proj.weight": "09cba09d72eea6b683087478aeaaf6d321423335a9eb57d47006adbc1f336c63", + "self_attn.v_proj.weight": "af4adee7b325a938beb530808104ab27a70afc0d494134b3de1f5ea8ef060773", + "self_attn.o_proj.weight": "6888fff18d668ea322341951eb6ab19a3fbbc4cf94b758ed0f8dba411a262d1f", + "self_attn.q_norm.weight": "e3e9ec56f12aa5ec410018bbac99e6953ad52f3b8b796d73c5bd0cc6545fb93b", + "self_attn.k_norm.weight": "a5bf1576b21d93da019094f1a61912669136a5d9e89e8d2301c88f6bf2def290", + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d", + "input_layernorm.weight": "c21d58d80fa7aca24ff0dac67092f8204b3876926fa095bb7e6ceb3a16060706", + "post_attention_layernorm.weight": "1e599b9bd0ed7fdbcef34cff94651bf467bf923745a76a96d33a74ae551ad5d5" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "f57ad388d57bb515af2ac9324d86dc9e19c4cc44bc73879a7b409f9c07597c15", + "self_attn.k_proj.weight": "09cba09d72eea6b683087478aeaaf6d321423335a9eb57d47006adbc1f336c63", + "self_attn.v_proj.weight": "af4adee7b325a938beb530808104ab27a70afc0d494134b3de1f5ea8ef060773", + "self_attn.o_proj.weight": "6888fff18d668ea322341951eb6ab19a3fbbc4cf94b758ed0f8dba411a262d1f", + "self_attn.q_norm.weight": "e3e9ec56f12aa5ec410018bbac99e6953ad52f3b8b796d73c5bd0cc6545fb93b", + "self_attn.k_norm.weight": "a5bf1576b21d93da019094f1a61912669136a5d9e89e8d2301c88f6bf2def290", + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d", + "input_layernorm.weight": "c21d58d80fa7aca24ff0dac67092f8204b3876926fa095bb7e6ceb3a16060706", + "post_attention_layernorm.weight": "1e599b9bd0ed7fdbcef34cff94651bf467bf923745a76a96d33a74ae551ad5d5" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_02_transport.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_02_transport.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dd3e83bbd6869321e62b3849fb31fec096dfec3 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_02_transport.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:771567c95fea3376299e7efde066b56970e55b94ec143fe51c9236dce5599fc0 +size 67110518 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_03_merge.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_03_merge.json new file mode 100644 index 0000000000000000000000000000000000000000..1f834ed1b6a08d276d733eaed1b11df75077ba61 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_03_merge.json @@ -0,0 +1,110 @@ +{ + "cycle": 3, + "anchor": 12, + "removed_original_layer": 15, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "c74bae7cb32fe1396e861248ba6bf41c2fd708fc354660e485db8f232fea727e", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.11038890826345682, + "calibration_cosine": 0.9431923979113933, + "calibration_norm_ratio": 0.9422936536202322, + "transport_relative_identity_distance": 0.480193167924881, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "6005f741b04aefb4b83452f774f7d013b010b6218020e382a98c92a45d8878ac", + "self_attn.k_proj.weight": "e4d7d1464c351cfcb69799eb3e3bf63cc597383cbfb0d07263236ba2d8eb3d5b", + "self_attn.v_proj.weight": "25063dfdce05fdcb1a48c5d71c3e0107d7a894b7fb6c0f7defa2e768c68ba3d7", + "self_attn.o_proj.weight": "e96bf888e3b0629075b2ee05288e147fd875119467d978c2269ecfeb3b0db4eb", + "self_attn.q_norm.weight": "d77168c19fe27a0d64a2611360be089b4435adc62444d0e82a986419be1d9b44", + "self_attn.k_norm.weight": "2cbb9d53874b3fe1d350a9c9031d64cd138a1dbac549529d07da2927106a10ff", + "mlp.gate_proj.weight": "3acdf3c7c3b8dd224f2837ef92c5e72e8bc7963baa021e9ba2a032da6084c86a", + "mlp.up_proj.weight": "6a0bdd43c6eab5e133991697799089710e9077591b56bcbde02440054dbfebbc", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea", + "input_layernorm.weight": "3f2e5d516a4c80f8d93f0221e5ecbc2e7e692f111118c7c629f210ee9d03419d", + "post_attention_layernorm.weight": "3a237234fa5182946c09967a08faec9d9060f4bd780bdd6627f6159edebe2f98" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "6005f741b04aefb4b83452f774f7d013b010b6218020e382a98c92a45d8878ac", + "self_attn.k_proj.weight": "e4d7d1464c351cfcb69799eb3e3bf63cc597383cbfb0d07263236ba2d8eb3d5b", + "self_attn.v_proj.weight": "25063dfdce05fdcb1a48c5d71c3e0107d7a894b7fb6c0f7defa2e768c68ba3d7", + "self_attn.o_proj.weight": "e96bf888e3b0629075b2ee05288e147fd875119467d978c2269ecfeb3b0db4eb", + "self_attn.q_norm.weight": "d77168c19fe27a0d64a2611360be089b4435adc62444d0e82a986419be1d9b44", + "self_attn.k_norm.weight": "2cbb9d53874b3fe1d350a9c9031d64cd138a1dbac549529d07da2927106a10ff", + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea", + "input_layernorm.weight": "3f2e5d516a4c80f8d93f0221e5ecbc2e7e692f111118c7c629f210ee9d03419d", + "post_attention_layernorm.weight": "3a237234fa5182946c09967a08faec9d9060f4bd780bdd6627f6159edebe2f98" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "6005f741b04aefb4b83452f774f7d013b010b6218020e382a98c92a45d8878ac", + "self_attn.k_proj.weight": "e4d7d1464c351cfcb69799eb3e3bf63cc597383cbfb0d07263236ba2d8eb3d5b", + "self_attn.v_proj.weight": "25063dfdce05fdcb1a48c5d71c3e0107d7a894b7fb6c0f7defa2e768c68ba3d7", + "self_attn.o_proj.weight": "e96bf888e3b0629075b2ee05288e147fd875119467d978c2269ecfeb3b0db4eb", + "self_attn.q_norm.weight": "d77168c19fe27a0d64a2611360be089b4435adc62444d0e82a986419be1d9b44", + "self_attn.k_norm.weight": "2cbb9d53874b3fe1d350a9c9031d64cd138a1dbac549529d07da2927106a10ff", + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea", + "input_layernorm.weight": "3f2e5d516a4c80f8d93f0221e5ecbc2e7e692f111118c7c629f210ee9d03419d", + "post_attention_layernorm.weight": "3a237234fa5182946c09967a08faec9d9060f4bd780bdd6627f6159edebe2f98" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_03_transport.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_03_transport.pt new file mode 100644 index 0000000000000000000000000000000000000000..53aff087999c989476fadc6a936c9d5624950286 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_03_transport.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:092a56f0df3e4eb2717935143213ae33bb0fc618abe15caefafdc39ff542cc90 +size 67110518 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_04_merge.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_04_merge.json new file mode 100644 index 0000000000000000000000000000000000000000..93e8f3f584743d06c301b3a9a42c3004b3a57ff8 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_04_merge.json @@ -0,0 +1,110 @@ +{ + "cycle": 4, + "anchor": 12, + "removed_original_layer": 16, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "49bbea5f7162f9479174a850b673f69481d7c2e8e01b01c5a110ec706d662fce", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09866496995725485, + "calibration_cosine": 0.9493870783680597, + "calibration_norm_ratio": 0.9484957145281611, + "transport_relative_identity_distance": 0.46458062529563904, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "a9874967bd53acfd480ed2bd5552fab4e1a06f610af242d411e176c279e0191a", + "self_attn.k_proj.weight": "0a285bc594b54a921f698937c566bce629295ff4befcebb203038dd0d97eb3f0", + "self_attn.v_proj.weight": "dd8847c5b10e200df29bf79b30321ec7f1d90427ab9b3d273e18a4cbd83e597d", + "self_attn.o_proj.weight": "e6afd2a9ba7c9e6f1911d773062b9d37442a01ead339c9aa9255ff20f146b196", + "self_attn.q_norm.weight": "48de60cd86dec981224231f275146bc8d17990394835783504bbf5f0e9c5bb1a", + "self_attn.k_norm.weight": "d6ef88be0fc007a2bc5e9f1a0a70b7eac493495b5b54b4f72582378340b9bb4d", + "mlp.gate_proj.weight": "171beb5701a83f117d0a59a14a78be2936a062c7aa5700ec39efd9c816af96db", + "mlp.up_proj.weight": "e04d543903d934aae9062d2034f71ac4d3fb1ac69bd5f31d365f404c764e535e", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1", + "input_layernorm.weight": "45d58615dedc0906944d998c52fed1d0d139338362abd51fc1b98ae12edec92a", + "post_attention_layernorm.weight": "af332049d823017194704880c92765ab0cf4a519c8819753d0539106ea951cef" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "a9874967bd53acfd480ed2bd5552fab4e1a06f610af242d411e176c279e0191a", + "self_attn.k_proj.weight": "0a285bc594b54a921f698937c566bce629295ff4befcebb203038dd0d97eb3f0", + "self_attn.v_proj.weight": "dd8847c5b10e200df29bf79b30321ec7f1d90427ab9b3d273e18a4cbd83e597d", + "self_attn.o_proj.weight": "e6afd2a9ba7c9e6f1911d773062b9d37442a01ead339c9aa9255ff20f146b196", + "self_attn.q_norm.weight": "48de60cd86dec981224231f275146bc8d17990394835783504bbf5f0e9c5bb1a", + "self_attn.k_norm.weight": "d6ef88be0fc007a2bc5e9f1a0a70b7eac493495b5b54b4f72582378340b9bb4d", + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1", + "input_layernorm.weight": "45d58615dedc0906944d998c52fed1d0d139338362abd51fc1b98ae12edec92a", + "post_attention_layernorm.weight": "af332049d823017194704880c92765ab0cf4a519c8819753d0539106ea951cef" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "a9874967bd53acfd480ed2bd5552fab4e1a06f610af242d411e176c279e0191a", + "self_attn.k_proj.weight": "0a285bc594b54a921f698937c566bce629295ff4befcebb203038dd0d97eb3f0", + "self_attn.v_proj.weight": "dd8847c5b10e200df29bf79b30321ec7f1d90427ab9b3d273e18a4cbd83e597d", + "self_attn.o_proj.weight": "e6afd2a9ba7c9e6f1911d773062b9d37442a01ead339c9aa9255ff20f146b196", + "self_attn.q_norm.weight": "48de60cd86dec981224231f275146bc8d17990394835783504bbf5f0e9c5bb1a", + "self_attn.k_norm.weight": "d6ef88be0fc007a2bc5e9f1a0a70b7eac493495b5b54b4f72582378340b9bb4d", + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1", + "input_layernorm.weight": "45d58615dedc0906944d998c52fed1d0d139338362abd51fc1b98ae12edec92a", + "post_attention_layernorm.weight": "af332049d823017194704880c92765ab0cf4a519c8819753d0539106ea951cef" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_04_transport.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_04_transport.pt new file mode 100644 index 0000000000000000000000000000000000000000..54aa437e5aebb34d266ff11d808354ea71e487e5 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_04_transport.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:030b90624e604f764c93dc04a03b3b58095bbc3d2b1a21e22226d92b950f95d9 +size 67110518 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_05_merge.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_05_merge.json new file mode 100644 index 0000000000000000000000000000000000000000..1083bf9c61c277056f566d0dc8bcf86324fd07ad --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_05_merge.json @@ -0,0 +1,110 @@ +{ + "cycle": 5, + "anchor": 12, + "removed_original_layer": 17, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "b968cc0681a32eab25ab339122b95bf8f1b90e3ced55b7638652d052fbcdbcc3", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09917833468174767, + "calibration_cosine": 0.9491166565617855, + "calibration_norm_ratio": 0.9482434757273703, + "transport_relative_identity_distance": 0.4882379472255707, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "93af1649dbcf27dcc906cff079a1448af3e0b78dc0ae7fcd0038e95d05f79fdc", + "self_attn.k_proj.weight": "919f1a2454b2957fa1f611232b1c0aaccdab718aed512b435923cf972cd52015", + "self_attn.v_proj.weight": "2a660e851663561147921fae640f303f3f599da61afc8d68545fed320cc9b63d", + "self_attn.o_proj.weight": "bfb11d528abdacfc054070ed7a55f90ddde74fb17704bc742ceffa065adcc717", + "self_attn.q_norm.weight": "a4538fa22891825dbfbac330127ff29569ac04b5ab0ee89e60db7e7bc64f26a3", + "self_attn.k_norm.weight": "f26654e4f51003513e7ff0cd2f23e99387481dc5c639b01444903ec9632ae685", + "mlp.gate_proj.weight": "1ec134d886ea6246b31de9cb6cbb4c451fddfb333fca1e90375f8659eb27d81c", + "mlp.up_proj.weight": "b7549bcdd3b5089f09aebb1847587e2049463294f52f4335b88b7e64888edd45", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f", + "input_layernorm.weight": "70025e492885453112e0108ebf4311dfd994db35383e90da14292277834fb949", + "post_attention_layernorm.weight": "41357e377104509608d219bc3f4d8a51dde14749a9a6e6b592f6d4f4594faa85" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "93af1649dbcf27dcc906cff079a1448af3e0b78dc0ae7fcd0038e95d05f79fdc", + "self_attn.k_proj.weight": "919f1a2454b2957fa1f611232b1c0aaccdab718aed512b435923cf972cd52015", + "self_attn.v_proj.weight": "2a660e851663561147921fae640f303f3f599da61afc8d68545fed320cc9b63d", + "self_attn.o_proj.weight": "bfb11d528abdacfc054070ed7a55f90ddde74fb17704bc742ceffa065adcc717", + "self_attn.q_norm.weight": "a4538fa22891825dbfbac330127ff29569ac04b5ab0ee89e60db7e7bc64f26a3", + "self_attn.k_norm.weight": "f26654e4f51003513e7ff0cd2f23e99387481dc5c639b01444903ec9632ae685", + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f", + "input_layernorm.weight": "70025e492885453112e0108ebf4311dfd994db35383e90da14292277834fb949", + "post_attention_layernorm.weight": "41357e377104509608d219bc3f4d8a51dde14749a9a6e6b592f6d4f4594faa85" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "93af1649dbcf27dcc906cff079a1448af3e0b78dc0ae7fcd0038e95d05f79fdc", + "self_attn.k_proj.weight": "919f1a2454b2957fa1f611232b1c0aaccdab718aed512b435923cf972cd52015", + "self_attn.v_proj.weight": "2a660e851663561147921fae640f303f3f599da61afc8d68545fed320cc9b63d", + "self_attn.o_proj.weight": "bfb11d528abdacfc054070ed7a55f90ddde74fb17704bc742ceffa065adcc717", + "self_attn.q_norm.weight": "a4538fa22891825dbfbac330127ff29569ac04b5ab0ee89e60db7e7bc64f26a3", + "self_attn.k_norm.weight": "f26654e4f51003513e7ff0cd2f23e99387481dc5c639b01444903ec9632ae685", + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f", + "input_layernorm.weight": "70025e492885453112e0108ebf4311dfd994db35383e90da14292277834fb949", + "post_attention_layernorm.weight": "41357e377104509608d219bc3f4d8a51dde14749a9a6e6b592f6d4f4594faa85" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_05_transport.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_05_transport.pt new file mode 100644 index 0000000000000000000000000000000000000000..72af5abb8fedcdf529b307ce191b1ef7d03c36e7 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_05_transport.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6889d5ef721bde145df816af6b64c40e1f943e4ddd06411a06a3196a78d20c2 +size 67110518 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_06_merge.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_06_merge.json new file mode 100644 index 0000000000000000000000000000000000000000..2977f0178d7ae65fc09257eb048794bf399eb0e9 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_06_merge.json @@ -0,0 +1,110 @@ +{ + "cycle": 6, + "anchor": 12, + "removed_original_layer": 18, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "b1c313ea0a3aa1df29a0618ae2ef9e5bc49705d8fb26ca876cad9c127db74e9e", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09820423130187533, + "calibration_cosine": 0.9496297005441647, + "calibration_norm_ratio": 0.9487355766850942, + "transport_relative_identity_distance": 0.48003679513931274, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "3d50a07d21693613e0d831209868828215164c78606ad76bb5339d0093e2b167", + "self_attn.k_proj.weight": "055922b3d6d8ae628f8a4761be8273b2b35bd979a899e599e03a4aebc9b31b15", + "self_attn.v_proj.weight": "35d50fc9ef07106a763c826a6bed55f5f8de393edeac002703ab161116d8dead", + "self_attn.o_proj.weight": "a211c679b699b735e0587b7e0bab5610d182a4c54868438be766544a8f36b4eb", + "self_attn.q_norm.weight": "41ed057917753085abc49146d705c39743047800a2ac65dba3e227fb675ad494", + "self_attn.k_norm.weight": "f01c939b42b6e7b3a03519eadb0a08a563897458717fc05d98b7c7d158621f02", + "mlp.gate_proj.weight": "a60ac64753890f45bc06d840148fad55807f022c4f14d6437d926bfe08d6e401", + "mlp.up_proj.weight": "f614342e9f77c5b73e0b93c34e9a6cbfb1692decfd132853765e9da75c5a2edc", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7", + "input_layernorm.weight": "2c7f7059994a701ae1bfd696bd109340fa7a22df9deee3bfd6c4afb890155523", + "post_attention_layernorm.weight": "d85cdb5cf77b8d6d42c74fbd6aff643661b72fcfbe4f343311c4d31db4afc67f" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "3d50a07d21693613e0d831209868828215164c78606ad76bb5339d0093e2b167", + "self_attn.k_proj.weight": "055922b3d6d8ae628f8a4761be8273b2b35bd979a899e599e03a4aebc9b31b15", + "self_attn.v_proj.weight": "35d50fc9ef07106a763c826a6bed55f5f8de393edeac002703ab161116d8dead", + "self_attn.o_proj.weight": "a211c679b699b735e0587b7e0bab5610d182a4c54868438be766544a8f36b4eb", + "self_attn.q_norm.weight": "41ed057917753085abc49146d705c39743047800a2ac65dba3e227fb675ad494", + "self_attn.k_norm.weight": "f01c939b42b6e7b3a03519eadb0a08a563897458717fc05d98b7c7d158621f02", + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7", + "input_layernorm.weight": "2c7f7059994a701ae1bfd696bd109340fa7a22df9deee3bfd6c4afb890155523", + "post_attention_layernorm.weight": "d85cdb5cf77b8d6d42c74fbd6aff643661b72fcfbe4f343311c4d31db4afc67f" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "3d50a07d21693613e0d831209868828215164c78606ad76bb5339d0093e2b167", + "self_attn.k_proj.weight": "055922b3d6d8ae628f8a4761be8273b2b35bd979a899e599e03a4aebc9b31b15", + "self_attn.v_proj.weight": "35d50fc9ef07106a763c826a6bed55f5f8de393edeac002703ab161116d8dead", + "self_attn.o_proj.weight": "a211c679b699b735e0587b7e0bab5610d182a4c54868438be766544a8f36b4eb", + "self_attn.q_norm.weight": "41ed057917753085abc49146d705c39743047800a2ac65dba3e227fb675ad494", + "self_attn.k_norm.weight": "f01c939b42b6e7b3a03519eadb0a08a563897458717fc05d98b7c7d158621f02", + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7", + "input_layernorm.weight": "2c7f7059994a701ae1bfd696bd109340fa7a22df9deee3bfd6c4afb890155523", + "post_attention_layernorm.weight": "d85cdb5cf77b8d6d42c74fbd6aff643661b72fcfbe4f343311c4d31db4afc67f" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_06_transport.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_06_transport.pt new file mode 100644 index 0000000000000000000000000000000000000000..365b6f7df89a3ac961c9059b8ca1fec2945d0db4 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_06_transport.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1400002d389e07d4a0bb71829c43f6aa137b91935e5ef07a925fea5f5040cee +size 67110518 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_07_merge.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_07_merge.json new file mode 100644 index 0000000000000000000000000000000000000000..01bdaed372bdaa6dd166ca63ac05d917bc5fffa5 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_07_merge.json @@ -0,0 +1,110 @@ +{ + "cycle": 7, + "anchor": 12, + "removed_original_layer": 19, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "4b807a71a3f5c920f7bbe9842c8bc0c89de7ebf5b02893847374cb46eeebbaae", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.103606409701293, + "calibration_cosine": 0.9467810247760247, + "calibration_norm_ratio": 0.9459333354270779, + "transport_relative_identity_distance": 0.5275545120239258, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "d14bc25171528405c3db791fd017b4e0fe37d06a9b07a9878cb81c1f0eb98ae6", + "self_attn.k_proj.weight": "8911d7363eb8d7e35ff407ac925183a06d54d624a8d2bd0ff41db97423515a54", + "self_attn.v_proj.weight": "4781046a7d6224a77c3e34dda92a2ef5748eb777d9d554acf5621c4e5742be2a", + "self_attn.o_proj.weight": "90f47fc1bc507ddfad16299dfcecb78a6b4efac01e21d3a6402c6ef3eae56348", + "self_attn.q_norm.weight": "6915d87fcc2402518b382dee5f5929f5774006c5d82904db327d2f118d6a5c99", + "self_attn.k_norm.weight": "cb20faa93398541fe6b2fbda2a527ea3243337db753acebdfd045bf309cbc4ae", + "mlp.gate_proj.weight": "aaf6792caf14dc5a810bfa937c966c7989238e23b21080752c784428eedbf79c", + "mlp.up_proj.weight": "5a9cf857484fc35820ab3032e13383687631e54fc1a7c7ff0443ec36e962cf17", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6", + "input_layernorm.weight": "316472831e20dec19263b56944811a059efb241cc842d8ace4fce706c98a8b11", + "post_attention_layernorm.weight": "7a462c4c15d73900fe1275d32448251905112e9530f615fe6776622c416f0f52" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "d14bc25171528405c3db791fd017b4e0fe37d06a9b07a9878cb81c1f0eb98ae6", + "self_attn.k_proj.weight": "8911d7363eb8d7e35ff407ac925183a06d54d624a8d2bd0ff41db97423515a54", + "self_attn.v_proj.weight": "4781046a7d6224a77c3e34dda92a2ef5748eb777d9d554acf5621c4e5742be2a", + "self_attn.o_proj.weight": "90f47fc1bc507ddfad16299dfcecb78a6b4efac01e21d3a6402c6ef3eae56348", + "self_attn.q_norm.weight": "6915d87fcc2402518b382dee5f5929f5774006c5d82904db327d2f118d6a5c99", + "self_attn.k_norm.weight": "cb20faa93398541fe6b2fbda2a527ea3243337db753acebdfd045bf309cbc4ae", + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6", + "input_layernorm.weight": "316472831e20dec19263b56944811a059efb241cc842d8ace4fce706c98a8b11", + "post_attention_layernorm.weight": "7a462c4c15d73900fe1275d32448251905112e9530f615fe6776622c416f0f52" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "d14bc25171528405c3db791fd017b4e0fe37d06a9b07a9878cb81c1f0eb98ae6", + "self_attn.k_proj.weight": "8911d7363eb8d7e35ff407ac925183a06d54d624a8d2bd0ff41db97423515a54", + "self_attn.v_proj.weight": "4781046a7d6224a77c3e34dda92a2ef5748eb777d9d554acf5621c4e5742be2a", + "self_attn.o_proj.weight": "90f47fc1bc507ddfad16299dfcecb78a6b4efac01e21d3a6402c6ef3eae56348", + "self_attn.q_norm.weight": "6915d87fcc2402518b382dee5f5929f5774006c5d82904db327d2f118d6a5c99", + "self_attn.k_norm.weight": "cb20faa93398541fe6b2fbda2a527ea3243337db753acebdfd045bf309cbc4ae", + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6", + "input_layernorm.weight": "316472831e20dec19263b56944811a059efb241cc842d8ace4fce706c98a8b11", + "post_attention_layernorm.weight": "7a462c4c15d73900fe1275d32448251905112e9530f615fe6776622c416f0f52" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_07_transport.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_07_transport.pt new file mode 100644 index 0000000000000000000000000000000000000000..96e6e4327002e3a498ef4831f277fa4e16bcbb57 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_07_transport.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d501ae2b2629df66facd8e4c516ff89d4312221d3383c886067b857ce6f86764 +size 67110518 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_08_merge.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_08_merge.json new file mode 100644 index 0000000000000000000000000000000000000000..99978c923e28c206d3c4d4c9b2fa6edcb262eee8 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_08_merge.json @@ -0,0 +1,110 @@ +{ + "cycle": 8, + "anchor": 12, + "removed_original_layer": 20, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "2c7b2775e51300a42d0d75c89a6c22bee3e7c935b89f0db3cf2b3fd9863b8fd6", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.10606533290714001, + "calibration_cosine": 0.9454815765850116, + "calibration_norm_ratio": 0.9446186925723196, + "transport_relative_identity_distance": 0.5891098976135254, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "6bdb1d9174c7bf6d06b12d5d55b3a7649f4f880714e3e2d5a6f90c1449ad1a50", + "self_attn.k_proj.weight": "e3a98bfec07bba4f0fc6aa01b510f13fce379d8a5fdce7a70ce0870da65a2e43", + "self_attn.v_proj.weight": "f250ec94ab67ea3d397297fa0c70e7e24c7f77ca609a3a83cc8f7e792e4bc608", + "self_attn.o_proj.weight": "18f0f0ec64b28cbc88f0937245cb010fd86dcf094de2d96d1bd2162b76020229", + "self_attn.q_norm.weight": "2172344fe2c1fda6f261ece920fbd07a4b03d2385d2c0848c497aae4f07e69b5", + "self_attn.k_norm.weight": "a2c8f7815e194c45bad87d1e291c9ff2ad3a36929b1201d024354ec501563a4e", + "mlp.gate_proj.weight": "8334c424ddf9edb282ccb34d6f90321e9530eff5e81b7881ba9b3d74417a9672", + "mlp.up_proj.weight": "db91586fd2907e8ecccb075f036e5c180641fa35f7cd899a3f81411caa9d10df", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd", + "input_layernorm.weight": "96fb39db2f64c716ae6891acb4e4e7f2a1276921a97d0d9a50d746727e4bbcd0", + "post_attention_layernorm.weight": "cd331d158252933122eae174cda361e7f3eb260b867b079b375650b368a4ae06" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "6bdb1d9174c7bf6d06b12d5d55b3a7649f4f880714e3e2d5a6f90c1449ad1a50", + "self_attn.k_proj.weight": "e3a98bfec07bba4f0fc6aa01b510f13fce379d8a5fdce7a70ce0870da65a2e43", + "self_attn.v_proj.weight": "f250ec94ab67ea3d397297fa0c70e7e24c7f77ca609a3a83cc8f7e792e4bc608", + "self_attn.o_proj.weight": "18f0f0ec64b28cbc88f0937245cb010fd86dcf094de2d96d1bd2162b76020229", + "self_attn.q_norm.weight": "2172344fe2c1fda6f261ece920fbd07a4b03d2385d2c0848c497aae4f07e69b5", + "self_attn.k_norm.weight": "a2c8f7815e194c45bad87d1e291c9ff2ad3a36929b1201d024354ec501563a4e", + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd", + "input_layernorm.weight": "96fb39db2f64c716ae6891acb4e4e7f2a1276921a97d0d9a50d746727e4bbcd0", + "post_attention_layernorm.weight": "cd331d158252933122eae174cda361e7f3eb260b867b079b375650b368a4ae06" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "6bdb1d9174c7bf6d06b12d5d55b3a7649f4f880714e3e2d5a6f90c1449ad1a50", + "self_attn.k_proj.weight": "e3a98bfec07bba4f0fc6aa01b510f13fce379d8a5fdce7a70ce0870da65a2e43", + "self_attn.v_proj.weight": "f250ec94ab67ea3d397297fa0c70e7e24c7f77ca609a3a83cc8f7e792e4bc608", + "self_attn.o_proj.weight": "18f0f0ec64b28cbc88f0937245cb010fd86dcf094de2d96d1bd2162b76020229", + "self_attn.q_norm.weight": "2172344fe2c1fda6f261ece920fbd07a4b03d2385d2c0848c497aae4f07e69b5", + "self_attn.k_norm.weight": "a2c8f7815e194c45bad87d1e291c9ff2ad3a36929b1201d024354ec501563a4e", + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd", + "input_layernorm.weight": "96fb39db2f64c716ae6891acb4e4e7f2a1276921a97d0d9a50d746727e4bbcd0", + "post_attention_layernorm.weight": "cd331d158252933122eae174cda361e7f3eb260b867b079b375650b368a4ae06" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_08_transport.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_08_transport.pt new file mode 100644 index 0000000000000000000000000000000000000000..ebd23a7089eac6f40ed56ebb5a2fc869862e05a4 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_08_transport.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43deee7cc5037a42f1952463af7065212e8cfabff7be44e11268becab95c3bb0 +size 67110518 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_09_merge.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_09_merge.json new file mode 100644 index 0000000000000000000000000000000000000000..8b17ceef677ecb206cf3ddf8293ea31ca638d830 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_09_merge.json @@ -0,0 +1,110 @@ +{ + "cycle": 9, + "anchor": 12, + "removed_original_layer": 21, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "7ad34658a18ce5ffbab2f8af974892f191e7db80d7b5da8d19e347938544b107", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.13979247738453124, + "calibration_cosine": 0.927474114888108, + "calibration_norm_ratio": 0.9266308047370905, + "transport_relative_identity_distance": 0.6954833269119263, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "1839858bb3153a7d2907bbc4837cab44494ea4f8f73db2dac5a973f1500537c0", + "self_attn.k_proj.weight": "34776759a30508b301e6eeb4c03e74afc6e315a0e4005b788772f03267fa6bad", + "self_attn.v_proj.weight": "25ed7a7bfff05b2a63a9489ed945f49ed624baa5ffa7f971c263e4d3aade1d04", + "self_attn.o_proj.weight": "a8c4e7b3c03654158c39ea33456a291809aee2893e9dde1fa2d530da216b313c", + "self_attn.q_norm.weight": "21aa357e3be95df8e0b87310db2cf759d951d8c9a706c9e1bee207de27cbb87f", + "self_attn.k_norm.weight": "0dfad6f8b52d3a2c224e7cd2b5c1fe87db29b927a7d7fb4161f231df6df2ac02", + "mlp.gate_proj.weight": "4d745750b386ff30e96c0fe5ac21229e0672de87a3c4d30b7a877c333f250622", + "mlp.up_proj.weight": "b0b002f9bd65d4a6b4b37b6cf06139deb86a51ebc96fc27b4ebcff016f3e450b", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a", + "input_layernorm.weight": "2974f6d5be715ca00edf542f4e51e5eb5ced2a292277c1974a92f5f6fec44871", + "post_attention_layernorm.weight": "a00da323dd47889f5a6238369523374a092cab49267ad69f4716d7b1c3778d5f" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "1839858bb3153a7d2907bbc4837cab44494ea4f8f73db2dac5a973f1500537c0", + "self_attn.k_proj.weight": "34776759a30508b301e6eeb4c03e74afc6e315a0e4005b788772f03267fa6bad", + "self_attn.v_proj.weight": "25ed7a7bfff05b2a63a9489ed945f49ed624baa5ffa7f971c263e4d3aade1d04", + "self_attn.o_proj.weight": "a8c4e7b3c03654158c39ea33456a291809aee2893e9dde1fa2d530da216b313c", + "self_attn.q_norm.weight": "21aa357e3be95df8e0b87310db2cf759d951d8c9a706c9e1bee207de27cbb87f", + "self_attn.k_norm.weight": "0dfad6f8b52d3a2c224e7cd2b5c1fe87db29b927a7d7fb4161f231df6df2ac02", + "mlp.gate_proj.weight": "6b132cf55ede8fb805c51a5b57c84cddd3ac2e84d4d94d13538a0f3eae146ba2", + "mlp.up_proj.weight": "d8ae064ab88056091a57a96a007959b9d01bb6c1f037cdab35d76d5f5694b3ae", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a", + "input_layernorm.weight": "2974f6d5be715ca00edf542f4e51e5eb5ced2a292277c1974a92f5f6fec44871", + "post_attention_layernorm.weight": "a00da323dd47889f5a6238369523374a092cab49267ad69f4716d7b1c3778d5f" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "1839858bb3153a7d2907bbc4837cab44494ea4f8f73db2dac5a973f1500537c0", + "self_attn.k_proj.weight": "34776759a30508b301e6eeb4c03e74afc6e315a0e4005b788772f03267fa6bad", + "self_attn.v_proj.weight": "25ed7a7bfff05b2a63a9489ed945f49ed624baa5ffa7f971c263e4d3aade1d04", + "self_attn.o_proj.weight": "a8c4e7b3c03654158c39ea33456a291809aee2893e9dde1fa2d530da216b313c", + "self_attn.q_norm.weight": "21aa357e3be95df8e0b87310db2cf759d951d8c9a706c9e1bee207de27cbb87f", + "self_attn.k_norm.weight": "0dfad6f8b52d3a2c224e7cd2b5c1fe87db29b927a7d7fb4161f231df6df2ac02", + "mlp.gate_proj.weight": "6b132cf55ede8fb805c51a5b57c84cddd3ac2e84d4d94d13538a0f3eae146ba2", + "mlp.up_proj.weight": "d8ae064ab88056091a57a96a007959b9d01bb6c1f037cdab35d76d5f5694b3ae", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a", + "input_layernorm.weight": "2974f6d5be715ca00edf542f4e51e5eb5ced2a292277c1974a92f5f6fec44871", + "post_attention_layernorm.weight": "a00da323dd47889f5a6238369523374a092cab49267ad69f4716d7b1c3778d5f" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "6b132cf55ede8fb805c51a5b57c84cddd3ac2e84d4d94d13538a0f3eae146ba2", + "mlp.up_proj.weight": "d8ae064ab88056091a57a96a007959b9d01bb6c1f037cdab35d76d5f5694b3ae", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_09_transport.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_09_transport.pt new file mode 100644 index 0000000000000000000000000000000000000000..8217bbc99f999a36c0fcd83c0fd60e1564d79df2 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/cycle_09_transport.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f785d329bfa8289c73722d71a8d82628c2c22f8a25f05a7201cc3f792ec3481 +size 67110518 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/data_manifest.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/data_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..460dfdbcb9b802235868b5d0ac3803cf9a63ffc1 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/data_manifest.json @@ -0,0 +1,15 @@ +{ + "model": "Qwen/Qwen3-8B-Base", + "model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "dataset": "DKYoon/SlimPajama-6B", + "revision": "b5f90f419b7489cdba26fdbc8c022fcb5562f968", + "split": "train", + "packing": "historical nonstreaming dataset.shuffle(seed42); concatenate encoded text without separators or BOS", + "samples": 64, + "seq_len": 1024, + "batch_size": 2, + "seed": 42, + "valid_tokens": 65536, + "input_ids_sha256": "85a84b9a68cf0856fd363186d3a273860886b1c17f876cae145fd092454aff55", + "attention_mask_sha256": "b1476de1907209012e63fa502715fd97db26e595352faa808d8c10433a1472a4" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/selection.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/selection.json new file mode 100644 index 0000000000000000000000000000000000000000..e04c8a6a1a6a0b35697cac96520e4702415cd7a9 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/audit/selection.json @@ -0,0 +1,992 @@ +[ + { + "cycle": 1, + "anchor": 12, + "removed_original_layer": 13, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "ea43767088fa2f8cc0b7ba197024635f7b8a7e19fdd34170b499f470c2461672", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.12798181447783583, + "calibration_cosine": 0.9338195359584089, + "calibration_norm_ratio": 0.9329591782339236, + "transport_relative_identity_distance": 0.4961368441581726, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "67540e2474547f1d622b738128696574720591fddabb8157d2f6812ecc3af308", + "self_attn.k_proj.weight": "efbe8b71b99ef90277849155ebc45e9aad5ffe746f2958c077c7c7130d783c7b", + "self_attn.v_proj.weight": "c53545e1ac9a6447efa83242434a0b28bcda5fe12ef3866e173dd13c3da45e38", + "self_attn.o_proj.weight": "8812afef76626e47fb5dffe3a77a0cb07a07e8a7f5cf3a97453368a1284b0ffa", + "self_attn.q_norm.weight": "79bbb659cb1c2af322ca85abf7d26b088c837bcd284e70efa95a6449f69b5673", + "self_attn.k_norm.weight": "e50a57833625792c16439de078eeb810df393e1a49ef7bbe43f5af6a93923adc", + "mlp.gate_proj.weight": "6bc0a4412e6639d40e3a7d5c9099a5a05f2573bd12744019550fc11da2c52c55", + "mlp.up_proj.weight": "8715dd4fa81b5d58a577a1b2fdfb24592b23775921014b36f832fe8dbf2bbf51", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b", + "input_layernorm.weight": "c419d161cc4932ecddf0ec98cbe5737b92688f0d5aa71a7e95d2006ca824ffca", + "post_attention_layernorm.weight": "ee6a9a96c2b4f35eb0807060229f10bc28080072ed9ecfa0a17e1aaa3cb4df40" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "67540e2474547f1d622b738128696574720591fddabb8157d2f6812ecc3af308", + "self_attn.k_proj.weight": "efbe8b71b99ef90277849155ebc45e9aad5ffe746f2958c077c7c7130d783c7b", + "self_attn.v_proj.weight": "c53545e1ac9a6447efa83242434a0b28bcda5fe12ef3866e173dd13c3da45e38", + "self_attn.o_proj.weight": "8812afef76626e47fb5dffe3a77a0cb07a07e8a7f5cf3a97453368a1284b0ffa", + "self_attn.q_norm.weight": "79bbb659cb1c2af322ca85abf7d26b088c837bcd284e70efa95a6449f69b5673", + "self_attn.k_norm.weight": "e50a57833625792c16439de078eeb810df393e1a49ef7bbe43f5af6a93923adc", + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b", + "input_layernorm.weight": "c419d161cc4932ecddf0ec98cbe5737b92688f0d5aa71a7e95d2006ca824ffca", + "post_attention_layernorm.weight": "ee6a9a96c2b4f35eb0807060229f10bc28080072ed9ecfa0a17e1aaa3cb4df40" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "3e138d6d890dfaeeeb88ae9b8a48c084aea69752230749e87f1586b975880ddd", + "mlp.up_proj.weight": "1f3dacc4c4a3af2a55d6273cfc2374077a2eda18cec7cf8237e05c54cc1b8ea5", + "mlp.down_proj.weight": "7a34a8a9bcc1a25c928397e3be5c409081b9de0ec283d13425482c0020e6db43" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "67540e2474547f1d622b738128696574720591fddabb8157d2f6812ecc3af308", + "self_attn.k_proj.weight": "efbe8b71b99ef90277849155ebc45e9aad5ffe746f2958c077c7c7130d783c7b", + "self_attn.v_proj.weight": "c53545e1ac9a6447efa83242434a0b28bcda5fe12ef3866e173dd13c3da45e38", + "self_attn.o_proj.weight": "8812afef76626e47fb5dffe3a77a0cb07a07e8a7f5cf3a97453368a1284b0ffa", + "self_attn.q_norm.weight": "79bbb659cb1c2af322ca85abf7d26b088c837bcd284e70efa95a6449f69b5673", + "self_attn.k_norm.weight": "e50a57833625792c16439de078eeb810df393e1a49ef7bbe43f5af6a93923adc", + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b", + "input_layernorm.weight": "c419d161cc4932ecddf0ec98cbe5737b92688f0d5aa71a7e95d2006ca824ffca", + "post_attention_layernorm.weight": "ee6a9a96c2b4f35eb0807060229f10bc28080072ed9ecfa0a17e1aaa3cb4df40" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + }, + { + "cycle": 2, + "anchor": 12, + "removed_original_layer": 14, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "2f2164ca8bffcf85ff9cfb9506308fa1340559aba7e44bda37d1e050e6e82d36", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.10099672573787757, + "calibration_cosine": 0.9481582440752598, + "calibration_norm_ratio": 0.9472741933087646, + "transport_relative_identity_distance": 0.45524656772613525, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "f57ad388d57bb515af2ac9324d86dc9e19c4cc44bc73879a7b409f9c07597c15", + "self_attn.k_proj.weight": "09cba09d72eea6b683087478aeaaf6d321423335a9eb57d47006adbc1f336c63", + "self_attn.v_proj.weight": "af4adee7b325a938beb530808104ab27a70afc0d494134b3de1f5ea8ef060773", + "self_attn.o_proj.weight": "6888fff18d668ea322341951eb6ab19a3fbbc4cf94b758ed0f8dba411a262d1f", + "self_attn.q_norm.weight": "e3e9ec56f12aa5ec410018bbac99e6953ad52f3b8b796d73c5bd0cc6545fb93b", + "self_attn.k_norm.weight": "a5bf1576b21d93da019094f1a61912669136a5d9e89e8d2301c88f6bf2def290", + "mlp.gate_proj.weight": "1d90e8cf937190e4027f0c7bc34bbabd5bb43c5f3d081537e0d866da8a3efe3f", + "mlp.up_proj.weight": "dc867f2d0718d761bb166f69bd21878837006daf9ae392c955a2e9578a15ad97", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d", + "input_layernorm.weight": "c21d58d80fa7aca24ff0dac67092f8204b3876926fa095bb7e6ceb3a16060706", + "post_attention_layernorm.weight": "1e599b9bd0ed7fdbcef34cff94651bf467bf923745a76a96d33a74ae551ad5d5" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "f57ad388d57bb515af2ac9324d86dc9e19c4cc44bc73879a7b409f9c07597c15", + "self_attn.k_proj.weight": "09cba09d72eea6b683087478aeaaf6d321423335a9eb57d47006adbc1f336c63", + "self_attn.v_proj.weight": "af4adee7b325a938beb530808104ab27a70afc0d494134b3de1f5ea8ef060773", + "self_attn.o_proj.weight": "6888fff18d668ea322341951eb6ab19a3fbbc4cf94b758ed0f8dba411a262d1f", + "self_attn.q_norm.weight": "e3e9ec56f12aa5ec410018bbac99e6953ad52f3b8b796d73c5bd0cc6545fb93b", + "self_attn.k_norm.weight": "a5bf1576b21d93da019094f1a61912669136a5d9e89e8d2301c88f6bf2def290", + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d", + "input_layernorm.weight": "c21d58d80fa7aca24ff0dac67092f8204b3876926fa095bb7e6ceb3a16060706", + "post_attention_layernorm.weight": "1e599b9bd0ed7fdbcef34cff94651bf467bf923745a76a96d33a74ae551ad5d5" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "f57ad388d57bb515af2ac9324d86dc9e19c4cc44bc73879a7b409f9c07597c15", + "self_attn.k_proj.weight": "09cba09d72eea6b683087478aeaaf6d321423335a9eb57d47006adbc1f336c63", + "self_attn.v_proj.weight": "af4adee7b325a938beb530808104ab27a70afc0d494134b3de1f5ea8ef060773", + "self_attn.o_proj.weight": "6888fff18d668ea322341951eb6ab19a3fbbc4cf94b758ed0f8dba411a262d1f", + "self_attn.q_norm.weight": "e3e9ec56f12aa5ec410018bbac99e6953ad52f3b8b796d73c5bd0cc6545fb93b", + "self_attn.k_norm.weight": "a5bf1576b21d93da019094f1a61912669136a5d9e89e8d2301c88f6bf2def290", + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d", + "input_layernorm.weight": "c21d58d80fa7aca24ff0dac67092f8204b3876926fa095bb7e6ceb3a16060706", + "post_attention_layernorm.weight": "1e599b9bd0ed7fdbcef34cff94651bf467bf923745a76a96d33a74ae551ad5d5" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + }, + { + "cycle": 3, + "anchor": 12, + "removed_original_layer": 15, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "c74bae7cb32fe1396e861248ba6bf41c2fd708fc354660e485db8f232fea727e", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.11038890826345682, + "calibration_cosine": 0.9431923979113933, + "calibration_norm_ratio": 0.9422936536202322, + "transport_relative_identity_distance": 0.480193167924881, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "6005f741b04aefb4b83452f774f7d013b010b6218020e382a98c92a45d8878ac", + "self_attn.k_proj.weight": "e4d7d1464c351cfcb69799eb3e3bf63cc597383cbfb0d07263236ba2d8eb3d5b", + "self_attn.v_proj.weight": "25063dfdce05fdcb1a48c5d71c3e0107d7a894b7fb6c0f7defa2e768c68ba3d7", + "self_attn.o_proj.weight": "e96bf888e3b0629075b2ee05288e147fd875119467d978c2269ecfeb3b0db4eb", + "self_attn.q_norm.weight": "d77168c19fe27a0d64a2611360be089b4435adc62444d0e82a986419be1d9b44", + "self_attn.k_norm.weight": "2cbb9d53874b3fe1d350a9c9031d64cd138a1dbac549529d07da2927106a10ff", + "mlp.gate_proj.weight": "3acdf3c7c3b8dd224f2837ef92c5e72e8bc7963baa021e9ba2a032da6084c86a", + "mlp.up_proj.weight": "6a0bdd43c6eab5e133991697799089710e9077591b56bcbde02440054dbfebbc", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea", + "input_layernorm.weight": "3f2e5d516a4c80f8d93f0221e5ecbc2e7e692f111118c7c629f210ee9d03419d", + "post_attention_layernorm.weight": "3a237234fa5182946c09967a08faec9d9060f4bd780bdd6627f6159edebe2f98" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "6005f741b04aefb4b83452f774f7d013b010b6218020e382a98c92a45d8878ac", + "self_attn.k_proj.weight": "e4d7d1464c351cfcb69799eb3e3bf63cc597383cbfb0d07263236ba2d8eb3d5b", + "self_attn.v_proj.weight": "25063dfdce05fdcb1a48c5d71c3e0107d7a894b7fb6c0f7defa2e768c68ba3d7", + "self_attn.o_proj.weight": "e96bf888e3b0629075b2ee05288e147fd875119467d978c2269ecfeb3b0db4eb", + "self_attn.q_norm.weight": "d77168c19fe27a0d64a2611360be089b4435adc62444d0e82a986419be1d9b44", + "self_attn.k_norm.weight": "2cbb9d53874b3fe1d350a9c9031d64cd138a1dbac549529d07da2927106a10ff", + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea", + "input_layernorm.weight": "3f2e5d516a4c80f8d93f0221e5ecbc2e7e692f111118c7c629f210ee9d03419d", + "post_attention_layernorm.weight": "3a237234fa5182946c09967a08faec9d9060f4bd780bdd6627f6159edebe2f98" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "6005f741b04aefb4b83452f774f7d013b010b6218020e382a98c92a45d8878ac", + "self_attn.k_proj.weight": "e4d7d1464c351cfcb69799eb3e3bf63cc597383cbfb0d07263236ba2d8eb3d5b", + "self_attn.v_proj.weight": "25063dfdce05fdcb1a48c5d71c3e0107d7a894b7fb6c0f7defa2e768c68ba3d7", + "self_attn.o_proj.weight": "e96bf888e3b0629075b2ee05288e147fd875119467d978c2269ecfeb3b0db4eb", + "self_attn.q_norm.weight": "d77168c19fe27a0d64a2611360be089b4435adc62444d0e82a986419be1d9b44", + "self_attn.k_norm.weight": "2cbb9d53874b3fe1d350a9c9031d64cd138a1dbac549529d07da2927106a10ff", + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea", + "input_layernorm.weight": "3f2e5d516a4c80f8d93f0221e5ecbc2e7e692f111118c7c629f210ee9d03419d", + "post_attention_layernorm.weight": "3a237234fa5182946c09967a08faec9d9060f4bd780bdd6627f6159edebe2f98" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + }, + { + "cycle": 4, + "anchor": 12, + "removed_original_layer": 16, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "49bbea5f7162f9479174a850b673f69481d7c2e8e01b01c5a110ec706d662fce", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09866496995725485, + "calibration_cosine": 0.9493870783680597, + "calibration_norm_ratio": 0.9484957145281611, + "transport_relative_identity_distance": 0.46458062529563904, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "a9874967bd53acfd480ed2bd5552fab4e1a06f610af242d411e176c279e0191a", + "self_attn.k_proj.weight": "0a285bc594b54a921f698937c566bce629295ff4befcebb203038dd0d97eb3f0", + "self_attn.v_proj.weight": "dd8847c5b10e200df29bf79b30321ec7f1d90427ab9b3d273e18a4cbd83e597d", + "self_attn.o_proj.weight": "e6afd2a9ba7c9e6f1911d773062b9d37442a01ead339c9aa9255ff20f146b196", + "self_attn.q_norm.weight": "48de60cd86dec981224231f275146bc8d17990394835783504bbf5f0e9c5bb1a", + "self_attn.k_norm.weight": "d6ef88be0fc007a2bc5e9f1a0a70b7eac493495b5b54b4f72582378340b9bb4d", + "mlp.gate_proj.weight": "171beb5701a83f117d0a59a14a78be2936a062c7aa5700ec39efd9c816af96db", + "mlp.up_proj.weight": "e04d543903d934aae9062d2034f71ac4d3fb1ac69bd5f31d365f404c764e535e", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1", + "input_layernorm.weight": "45d58615dedc0906944d998c52fed1d0d139338362abd51fc1b98ae12edec92a", + "post_attention_layernorm.weight": "af332049d823017194704880c92765ab0cf4a519c8819753d0539106ea951cef" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "a9874967bd53acfd480ed2bd5552fab4e1a06f610af242d411e176c279e0191a", + "self_attn.k_proj.weight": "0a285bc594b54a921f698937c566bce629295ff4befcebb203038dd0d97eb3f0", + "self_attn.v_proj.weight": "dd8847c5b10e200df29bf79b30321ec7f1d90427ab9b3d273e18a4cbd83e597d", + "self_attn.o_proj.weight": "e6afd2a9ba7c9e6f1911d773062b9d37442a01ead339c9aa9255ff20f146b196", + "self_attn.q_norm.weight": "48de60cd86dec981224231f275146bc8d17990394835783504bbf5f0e9c5bb1a", + "self_attn.k_norm.weight": "d6ef88be0fc007a2bc5e9f1a0a70b7eac493495b5b54b4f72582378340b9bb4d", + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1", + "input_layernorm.weight": "45d58615dedc0906944d998c52fed1d0d139338362abd51fc1b98ae12edec92a", + "post_attention_layernorm.weight": "af332049d823017194704880c92765ab0cf4a519c8819753d0539106ea951cef" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "a9874967bd53acfd480ed2bd5552fab4e1a06f610af242d411e176c279e0191a", + "self_attn.k_proj.weight": "0a285bc594b54a921f698937c566bce629295ff4befcebb203038dd0d97eb3f0", + "self_attn.v_proj.weight": "dd8847c5b10e200df29bf79b30321ec7f1d90427ab9b3d273e18a4cbd83e597d", + "self_attn.o_proj.weight": "e6afd2a9ba7c9e6f1911d773062b9d37442a01ead339c9aa9255ff20f146b196", + "self_attn.q_norm.weight": "48de60cd86dec981224231f275146bc8d17990394835783504bbf5f0e9c5bb1a", + "self_attn.k_norm.weight": "d6ef88be0fc007a2bc5e9f1a0a70b7eac493495b5b54b4f72582378340b9bb4d", + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1", + "input_layernorm.weight": "45d58615dedc0906944d998c52fed1d0d139338362abd51fc1b98ae12edec92a", + "post_attention_layernorm.weight": "af332049d823017194704880c92765ab0cf4a519c8819753d0539106ea951cef" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + }, + { + "cycle": 5, + "anchor": 12, + "removed_original_layer": 17, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "b968cc0681a32eab25ab339122b95bf8f1b90e3ced55b7638652d052fbcdbcc3", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09917833468174767, + "calibration_cosine": 0.9491166565617855, + "calibration_norm_ratio": 0.9482434757273703, + "transport_relative_identity_distance": 0.4882379472255707, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "93af1649dbcf27dcc906cff079a1448af3e0b78dc0ae7fcd0038e95d05f79fdc", + "self_attn.k_proj.weight": "919f1a2454b2957fa1f611232b1c0aaccdab718aed512b435923cf972cd52015", + "self_attn.v_proj.weight": "2a660e851663561147921fae640f303f3f599da61afc8d68545fed320cc9b63d", + "self_attn.o_proj.weight": "bfb11d528abdacfc054070ed7a55f90ddde74fb17704bc742ceffa065adcc717", + "self_attn.q_norm.weight": "a4538fa22891825dbfbac330127ff29569ac04b5ab0ee89e60db7e7bc64f26a3", + "self_attn.k_norm.weight": "f26654e4f51003513e7ff0cd2f23e99387481dc5c639b01444903ec9632ae685", + "mlp.gate_proj.weight": "1ec134d886ea6246b31de9cb6cbb4c451fddfb333fca1e90375f8659eb27d81c", + "mlp.up_proj.weight": "b7549bcdd3b5089f09aebb1847587e2049463294f52f4335b88b7e64888edd45", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f", + "input_layernorm.weight": "70025e492885453112e0108ebf4311dfd994db35383e90da14292277834fb949", + "post_attention_layernorm.weight": "41357e377104509608d219bc3f4d8a51dde14749a9a6e6b592f6d4f4594faa85" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "93af1649dbcf27dcc906cff079a1448af3e0b78dc0ae7fcd0038e95d05f79fdc", + "self_attn.k_proj.weight": "919f1a2454b2957fa1f611232b1c0aaccdab718aed512b435923cf972cd52015", + "self_attn.v_proj.weight": "2a660e851663561147921fae640f303f3f599da61afc8d68545fed320cc9b63d", + "self_attn.o_proj.weight": "bfb11d528abdacfc054070ed7a55f90ddde74fb17704bc742ceffa065adcc717", + "self_attn.q_norm.weight": "a4538fa22891825dbfbac330127ff29569ac04b5ab0ee89e60db7e7bc64f26a3", + "self_attn.k_norm.weight": "f26654e4f51003513e7ff0cd2f23e99387481dc5c639b01444903ec9632ae685", + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f", + "input_layernorm.weight": "70025e492885453112e0108ebf4311dfd994db35383e90da14292277834fb949", + "post_attention_layernorm.weight": "41357e377104509608d219bc3f4d8a51dde14749a9a6e6b592f6d4f4594faa85" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "93af1649dbcf27dcc906cff079a1448af3e0b78dc0ae7fcd0038e95d05f79fdc", + "self_attn.k_proj.weight": "919f1a2454b2957fa1f611232b1c0aaccdab718aed512b435923cf972cd52015", + "self_attn.v_proj.weight": "2a660e851663561147921fae640f303f3f599da61afc8d68545fed320cc9b63d", + "self_attn.o_proj.weight": "bfb11d528abdacfc054070ed7a55f90ddde74fb17704bc742ceffa065adcc717", + "self_attn.q_norm.weight": "a4538fa22891825dbfbac330127ff29569ac04b5ab0ee89e60db7e7bc64f26a3", + "self_attn.k_norm.weight": "f26654e4f51003513e7ff0cd2f23e99387481dc5c639b01444903ec9632ae685", + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f", + "input_layernorm.weight": "70025e492885453112e0108ebf4311dfd994db35383e90da14292277834fb949", + "post_attention_layernorm.weight": "41357e377104509608d219bc3f4d8a51dde14749a9a6e6b592f6d4f4594faa85" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + }, + { + "cycle": 6, + "anchor": 12, + "removed_original_layer": 18, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "b1c313ea0a3aa1df29a0618ae2ef9e5bc49705d8fb26ca876cad9c127db74e9e", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09820423130187533, + "calibration_cosine": 0.9496297005441647, + "calibration_norm_ratio": 0.9487355766850942, + "transport_relative_identity_distance": 0.48003679513931274, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "3d50a07d21693613e0d831209868828215164c78606ad76bb5339d0093e2b167", + "self_attn.k_proj.weight": "055922b3d6d8ae628f8a4761be8273b2b35bd979a899e599e03a4aebc9b31b15", + "self_attn.v_proj.weight": "35d50fc9ef07106a763c826a6bed55f5f8de393edeac002703ab161116d8dead", + "self_attn.o_proj.weight": "a211c679b699b735e0587b7e0bab5610d182a4c54868438be766544a8f36b4eb", + "self_attn.q_norm.weight": "41ed057917753085abc49146d705c39743047800a2ac65dba3e227fb675ad494", + "self_attn.k_norm.weight": "f01c939b42b6e7b3a03519eadb0a08a563897458717fc05d98b7c7d158621f02", + "mlp.gate_proj.weight": "a60ac64753890f45bc06d840148fad55807f022c4f14d6437d926bfe08d6e401", + "mlp.up_proj.weight": "f614342e9f77c5b73e0b93c34e9a6cbfb1692decfd132853765e9da75c5a2edc", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7", + "input_layernorm.weight": "2c7f7059994a701ae1bfd696bd109340fa7a22df9deee3bfd6c4afb890155523", + "post_attention_layernorm.weight": "d85cdb5cf77b8d6d42c74fbd6aff643661b72fcfbe4f343311c4d31db4afc67f" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "3d50a07d21693613e0d831209868828215164c78606ad76bb5339d0093e2b167", + "self_attn.k_proj.weight": "055922b3d6d8ae628f8a4761be8273b2b35bd979a899e599e03a4aebc9b31b15", + "self_attn.v_proj.weight": "35d50fc9ef07106a763c826a6bed55f5f8de393edeac002703ab161116d8dead", + "self_attn.o_proj.weight": "a211c679b699b735e0587b7e0bab5610d182a4c54868438be766544a8f36b4eb", + "self_attn.q_norm.weight": "41ed057917753085abc49146d705c39743047800a2ac65dba3e227fb675ad494", + "self_attn.k_norm.weight": "f01c939b42b6e7b3a03519eadb0a08a563897458717fc05d98b7c7d158621f02", + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7", + "input_layernorm.weight": "2c7f7059994a701ae1bfd696bd109340fa7a22df9deee3bfd6c4afb890155523", + "post_attention_layernorm.weight": "d85cdb5cf77b8d6d42c74fbd6aff643661b72fcfbe4f343311c4d31db4afc67f" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "3d50a07d21693613e0d831209868828215164c78606ad76bb5339d0093e2b167", + "self_attn.k_proj.weight": "055922b3d6d8ae628f8a4761be8273b2b35bd979a899e599e03a4aebc9b31b15", + "self_attn.v_proj.weight": "35d50fc9ef07106a763c826a6bed55f5f8de393edeac002703ab161116d8dead", + "self_attn.o_proj.weight": "a211c679b699b735e0587b7e0bab5610d182a4c54868438be766544a8f36b4eb", + "self_attn.q_norm.weight": "41ed057917753085abc49146d705c39743047800a2ac65dba3e227fb675ad494", + "self_attn.k_norm.weight": "f01c939b42b6e7b3a03519eadb0a08a563897458717fc05d98b7c7d158621f02", + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7", + "input_layernorm.weight": "2c7f7059994a701ae1bfd696bd109340fa7a22df9deee3bfd6c4afb890155523", + "post_attention_layernorm.weight": "d85cdb5cf77b8d6d42c74fbd6aff643661b72fcfbe4f343311c4d31db4afc67f" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + }, + { + "cycle": 7, + "anchor": 12, + "removed_original_layer": 19, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "4b807a71a3f5c920f7bbe9842c8bc0c89de7ebf5b02893847374cb46eeebbaae", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.103606409701293, + "calibration_cosine": 0.9467810247760247, + "calibration_norm_ratio": 0.9459333354270779, + "transport_relative_identity_distance": 0.5275545120239258, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "d14bc25171528405c3db791fd017b4e0fe37d06a9b07a9878cb81c1f0eb98ae6", + "self_attn.k_proj.weight": "8911d7363eb8d7e35ff407ac925183a06d54d624a8d2bd0ff41db97423515a54", + "self_attn.v_proj.weight": "4781046a7d6224a77c3e34dda92a2ef5748eb777d9d554acf5621c4e5742be2a", + "self_attn.o_proj.weight": "90f47fc1bc507ddfad16299dfcecb78a6b4efac01e21d3a6402c6ef3eae56348", + "self_attn.q_norm.weight": "6915d87fcc2402518b382dee5f5929f5774006c5d82904db327d2f118d6a5c99", + "self_attn.k_norm.weight": "cb20faa93398541fe6b2fbda2a527ea3243337db753acebdfd045bf309cbc4ae", + "mlp.gate_proj.weight": "aaf6792caf14dc5a810bfa937c966c7989238e23b21080752c784428eedbf79c", + "mlp.up_proj.weight": "5a9cf857484fc35820ab3032e13383687631e54fc1a7c7ff0443ec36e962cf17", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6", + "input_layernorm.weight": "316472831e20dec19263b56944811a059efb241cc842d8ace4fce706c98a8b11", + "post_attention_layernorm.weight": "7a462c4c15d73900fe1275d32448251905112e9530f615fe6776622c416f0f52" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "d14bc25171528405c3db791fd017b4e0fe37d06a9b07a9878cb81c1f0eb98ae6", + "self_attn.k_proj.weight": "8911d7363eb8d7e35ff407ac925183a06d54d624a8d2bd0ff41db97423515a54", + "self_attn.v_proj.weight": "4781046a7d6224a77c3e34dda92a2ef5748eb777d9d554acf5621c4e5742be2a", + "self_attn.o_proj.weight": "90f47fc1bc507ddfad16299dfcecb78a6b4efac01e21d3a6402c6ef3eae56348", + "self_attn.q_norm.weight": "6915d87fcc2402518b382dee5f5929f5774006c5d82904db327d2f118d6a5c99", + "self_attn.k_norm.weight": "cb20faa93398541fe6b2fbda2a527ea3243337db753acebdfd045bf309cbc4ae", + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6", + "input_layernorm.weight": "316472831e20dec19263b56944811a059efb241cc842d8ace4fce706c98a8b11", + "post_attention_layernorm.weight": "7a462c4c15d73900fe1275d32448251905112e9530f615fe6776622c416f0f52" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "d14bc25171528405c3db791fd017b4e0fe37d06a9b07a9878cb81c1f0eb98ae6", + "self_attn.k_proj.weight": "8911d7363eb8d7e35ff407ac925183a06d54d624a8d2bd0ff41db97423515a54", + "self_attn.v_proj.weight": "4781046a7d6224a77c3e34dda92a2ef5748eb777d9d554acf5621c4e5742be2a", + "self_attn.o_proj.weight": "90f47fc1bc507ddfad16299dfcecb78a6b4efac01e21d3a6402c6ef3eae56348", + "self_attn.q_norm.weight": "6915d87fcc2402518b382dee5f5929f5774006c5d82904db327d2f118d6a5c99", + "self_attn.k_norm.weight": "cb20faa93398541fe6b2fbda2a527ea3243337db753acebdfd045bf309cbc4ae", + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6", + "input_layernorm.weight": "316472831e20dec19263b56944811a059efb241cc842d8ace4fce706c98a8b11", + "post_attention_layernorm.weight": "7a462c4c15d73900fe1275d32448251905112e9530f615fe6776622c416f0f52" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + }, + { + "cycle": 8, + "anchor": 12, + "removed_original_layer": 20, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "2c7b2775e51300a42d0d75c89a6c22bee3e7c935b89f0db3cf2b3fd9863b8fd6", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.10606533290714001, + "calibration_cosine": 0.9454815765850116, + "calibration_norm_ratio": 0.9446186925723196, + "transport_relative_identity_distance": 0.5891098976135254, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "6bdb1d9174c7bf6d06b12d5d55b3a7649f4f880714e3e2d5a6f90c1449ad1a50", + "self_attn.k_proj.weight": "e3a98bfec07bba4f0fc6aa01b510f13fce379d8a5fdce7a70ce0870da65a2e43", + "self_attn.v_proj.weight": "f250ec94ab67ea3d397297fa0c70e7e24c7f77ca609a3a83cc8f7e792e4bc608", + "self_attn.o_proj.weight": "18f0f0ec64b28cbc88f0937245cb010fd86dcf094de2d96d1bd2162b76020229", + "self_attn.q_norm.weight": "2172344fe2c1fda6f261ece920fbd07a4b03d2385d2c0848c497aae4f07e69b5", + "self_attn.k_norm.weight": "a2c8f7815e194c45bad87d1e291c9ff2ad3a36929b1201d024354ec501563a4e", + "mlp.gate_proj.weight": "8334c424ddf9edb282ccb34d6f90321e9530eff5e81b7881ba9b3d74417a9672", + "mlp.up_proj.weight": "db91586fd2907e8ecccb075f036e5c180641fa35f7cd899a3f81411caa9d10df", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd", + "input_layernorm.weight": "96fb39db2f64c716ae6891acb4e4e7f2a1276921a97d0d9a50d746727e4bbcd0", + "post_attention_layernorm.weight": "cd331d158252933122eae174cda361e7f3eb260b867b079b375650b368a4ae06" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "6bdb1d9174c7bf6d06b12d5d55b3a7649f4f880714e3e2d5a6f90c1449ad1a50", + "self_attn.k_proj.weight": "e3a98bfec07bba4f0fc6aa01b510f13fce379d8a5fdce7a70ce0870da65a2e43", + "self_attn.v_proj.weight": "f250ec94ab67ea3d397297fa0c70e7e24c7f77ca609a3a83cc8f7e792e4bc608", + "self_attn.o_proj.weight": "18f0f0ec64b28cbc88f0937245cb010fd86dcf094de2d96d1bd2162b76020229", + "self_attn.q_norm.weight": "2172344fe2c1fda6f261ece920fbd07a4b03d2385d2c0848c497aae4f07e69b5", + "self_attn.k_norm.weight": "a2c8f7815e194c45bad87d1e291c9ff2ad3a36929b1201d024354ec501563a4e", + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd", + "input_layernorm.weight": "96fb39db2f64c716ae6891acb4e4e7f2a1276921a97d0d9a50d746727e4bbcd0", + "post_attention_layernorm.weight": "cd331d158252933122eae174cda361e7f3eb260b867b079b375650b368a4ae06" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "6bdb1d9174c7bf6d06b12d5d55b3a7649f4f880714e3e2d5a6f90c1449ad1a50", + "self_attn.k_proj.weight": "e3a98bfec07bba4f0fc6aa01b510f13fce379d8a5fdce7a70ce0870da65a2e43", + "self_attn.v_proj.weight": "f250ec94ab67ea3d397297fa0c70e7e24c7f77ca609a3a83cc8f7e792e4bc608", + "self_attn.o_proj.weight": "18f0f0ec64b28cbc88f0937245cb010fd86dcf094de2d96d1bd2162b76020229", + "self_attn.q_norm.weight": "2172344fe2c1fda6f261ece920fbd07a4b03d2385d2c0848c497aae4f07e69b5", + "self_attn.k_norm.weight": "a2c8f7815e194c45bad87d1e291c9ff2ad3a36929b1201d024354ec501563a4e", + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd", + "input_layernorm.weight": "96fb39db2f64c716ae6891acb4e4e7f2a1276921a97d0d9a50d746727e4bbcd0", + "post_attention_layernorm.weight": "cd331d158252933122eae174cda361e7f3eb260b867b079b375650b368a4ae06" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + }, + { + "cycle": 9, + "anchor": 12, + "removed_original_layer": 21, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "7ad34658a18ce5ffbab2f8af974892f191e7db80d7b5da8d19e347938544b107", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.13979247738453124, + "calibration_cosine": 0.927474114888108, + "calibration_norm_ratio": 0.9266308047370905, + "transport_relative_identity_distance": 0.6954833269119263, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "1839858bb3153a7d2907bbc4837cab44494ea4f8f73db2dac5a973f1500537c0", + "self_attn.k_proj.weight": "34776759a30508b301e6eeb4c03e74afc6e315a0e4005b788772f03267fa6bad", + "self_attn.v_proj.weight": "25ed7a7bfff05b2a63a9489ed945f49ed624baa5ffa7f971c263e4d3aade1d04", + "self_attn.o_proj.weight": "a8c4e7b3c03654158c39ea33456a291809aee2893e9dde1fa2d530da216b313c", + "self_attn.q_norm.weight": "21aa357e3be95df8e0b87310db2cf759d951d8c9a706c9e1bee207de27cbb87f", + "self_attn.k_norm.weight": "0dfad6f8b52d3a2c224e7cd2b5c1fe87db29b927a7d7fb4161f231df6df2ac02", + "mlp.gate_proj.weight": "4d745750b386ff30e96c0fe5ac21229e0672de87a3c4d30b7a877c333f250622", + "mlp.up_proj.weight": "b0b002f9bd65d4a6b4b37b6cf06139deb86a51ebc96fc27b4ebcff016f3e450b", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a", + "input_layernorm.weight": "2974f6d5be715ca00edf542f4e51e5eb5ced2a292277c1974a92f5f6fec44871", + "post_attention_layernorm.weight": "a00da323dd47889f5a6238369523374a092cab49267ad69f4716d7b1c3778d5f" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "1839858bb3153a7d2907bbc4837cab44494ea4f8f73db2dac5a973f1500537c0", + "self_attn.k_proj.weight": "34776759a30508b301e6eeb4c03e74afc6e315a0e4005b788772f03267fa6bad", + "self_attn.v_proj.weight": "25ed7a7bfff05b2a63a9489ed945f49ed624baa5ffa7f971c263e4d3aade1d04", + "self_attn.o_proj.weight": "a8c4e7b3c03654158c39ea33456a291809aee2893e9dde1fa2d530da216b313c", + "self_attn.q_norm.weight": "21aa357e3be95df8e0b87310db2cf759d951d8c9a706c9e1bee207de27cbb87f", + "self_attn.k_norm.weight": "0dfad6f8b52d3a2c224e7cd2b5c1fe87db29b927a7d7fb4161f231df6df2ac02", + "mlp.gate_proj.weight": "6b132cf55ede8fb805c51a5b57c84cddd3ac2e84d4d94d13538a0f3eae146ba2", + "mlp.up_proj.weight": "d8ae064ab88056091a57a96a007959b9d01bb6c1f037cdab35d76d5f5694b3ae", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a", + "input_layernorm.weight": "2974f6d5be715ca00edf542f4e51e5eb5ced2a292277c1974a92f5f6fec44871", + "post_attention_layernorm.weight": "a00da323dd47889f5a6238369523374a092cab49267ad69f4716d7b1c3778d5f" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "1839858bb3153a7d2907bbc4837cab44494ea4f8f73db2dac5a973f1500537c0", + "self_attn.k_proj.weight": "34776759a30508b301e6eeb4c03e74afc6e315a0e4005b788772f03267fa6bad", + "self_attn.v_proj.weight": "25ed7a7bfff05b2a63a9489ed945f49ed624baa5ffa7f971c263e4d3aade1d04", + "self_attn.o_proj.weight": "a8c4e7b3c03654158c39ea33456a291809aee2893e9dde1fa2d530da216b313c", + "self_attn.q_norm.weight": "21aa357e3be95df8e0b87310db2cf759d951d8c9a706c9e1bee207de27cbb87f", + "self_attn.k_norm.weight": "0dfad6f8b52d3a2c224e7cd2b5c1fe87db29b927a7d7fb4161f231df6df2ac02", + "mlp.gate_proj.weight": "6b132cf55ede8fb805c51a5b57c84cddd3ac2e84d4d94d13538a0f3eae146ba2", + "mlp.up_proj.weight": "d8ae064ab88056091a57a96a007959b9d01bb6c1f037cdab35d76d5f5694b3ae", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a", + "input_layernorm.weight": "2974f6d5be715ca00edf542f4e51e5eb5ced2a292277c1974a92f5f6fec44871", + "post_attention_layernorm.weight": "a00da323dd47889f5a6238369523374a092cab49267ad69f4716d7b1c3778d5f" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "6b132cf55ede8fb805c51a5b57c84cddd3ac2e84d4d94d13538a0f3eae146ba2", + "mlp.up_proj.weight": "d8ae064ab88056091a57a96a007959b9d01bb6c1f037cdab35d76d5f5694b3ae", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + } +] diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/2026-10-11_qwen3-base-nofit-merge4.md b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/2026-10-11_qwen3-base-nofit-merge4.md new file mode 100644 index 0000000000000000000000000000000000000000..f42c0e0e01ed0a9303f1839fe9016b4ecd843a4f --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/2026-10-11_qwen3-base-nofit-merge4.md @@ -0,0 +1,52 @@ +# Qwen3 Base no FiT merge controls + +Compare Wa, transported Wb, equal average, and corrected MKA initialization without FiT on our fixed nine-block Qwen3 Base span. Reuse the completed Fisher plus Transport no-FiT baseline. Report common-dense-teacher Local MSE, WikiText2 PPL, and Bench14 macro. MKA merges attention and MLP; the other four methods merge MLP only, so the MKA comparison also changes merge scope. + +## Resources and provenance + +- Session alias: `vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011`. +- Branch: `codex/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011`; immutable execution commit recorded at launch and in each backup. +- Instance 55359231, offer 51341854, machine 117301, Mississippi US. Four A100 SXM4 40 GB, reported allocated RAM 515807 MiB, disk 120 GB. Total listed cost USD 2.7444444444444445 per hour including disk, traffic extra. Offer reliability 0.9847917, download 1767 Mbps, upload 1891.5 Mbps. +- Earlier instance 55357778, machine 149846, stalled in Vast container initialization with no SSH port. It never received code or started an experiment. Destroyed October 11 on user instruction to replace it; no experiment data lost. +- Image `pytorch/pytorch:2.9.1-cuda12.8-cudnn9-devel`; transformers 5.15.1, datasets 3.6.0, huggingface_hub 1.33.0, lm_eval 0.4.12, scipy, scikit-learn. BF16 model/evaluation; source transport solves in FP32. +- External engine: vendor base bundle 39e59afe, immutable three-file source overlay from Hub revision `61734e7b1a77fd279fe1e2584f7a0fdc26c175c2`, progressive-loader helpers and instruction-mask patch. + +## Inputs and merge rules + +Model `Qwen/Qwen3-8B-Base`, revision `49e3418fbbbca6ecbdf9608b4d22e5a407081db4`. Keep original anchor 12 and progressively remove original layers 13 through 21, span `[13,22)`, depth 36 to 27. Never switch to native MKA pair selection. + +Calibration is the historical non-streaming `DKYoon/SlimPajama-6B` shuffle, revision `b5f90f419b7489cdba26fdbc8c022fcb5562f968`, seed 42, 64 packed sequences of 1024 tokens, batch 2. Token SHA256 must equal `85a84b9a68cf0856fd363186d3a273860886b1c17f876cae145fd092454aff55`. This is not the concurrent mixed SlimPajama and MCQA-like experiment. Prepare and save exact token IDs once, share with all arms. + +Existing source feature Transport remains ON, ridge relative 0.001. It rewrites source MLP gate/up input projections only; down and attention projections are not transported. Save each transport matrix and pre/post operand hashes. In every cycle initialize from the actual post-transport source, not a stale pre-transport copy. + +Wa, Wb, Average merge the three MLP projection weights with respective left coefficients 1, 0, and 0.5. Attention, residual norms, and Q/K norms remain Wa. MKA merges all seven attention and MLP projection weights with the corrected local upstream coefficient, leaving all norms Wa. MKA geometry uses these same 64 calibration sequences, sequence mean pooling, sigma 8, alpha 0.5, two diffusion dimensions, five MI neighbors, 50 entropy neighbors, beta 1, MI seed 0, and is recomputed each stage before transport. This corrected local implementation is not an exact paper reproduction. + +No FiT loader, optimizer, additive U, backward pass, Fisher calculation, head permutation, LoRA, or residual-scale fitting is permitted in the four new arms. The transport ridge solve is the retained coordinate-alignment step, not MSE parameter matching. + +## Evaluation and retention + +Local MSE uses the same original dense teacher for every arm, including restored existing Fisher. Inputs are the first 64 packed 1024-token WikiText2 test windows under the HCHC full-stream packing. Capture dense layer12 input and layer21 output before final RMSNorm. Compare student anchor12 output with the dense output, verifying identical anchor inputs. Evaluate only; no optimization. Report squared difference sum divided by token count and hidden width, plus auxiliary energy-normalized MSE. BF16 eager activations, FP32 differences, FP64 aggregate. Exact evaluation input IDs and teacher-output hashes are retained. + +WikiText PPL preserves existing HCHC packing, sequence length 2048, BF16 eager, expected 145 windows and 296815 predicted tokens; batch 1 to fit 40 GB. Bench14 uses the established OpenCompass-five and lm-eval-nine zero-shot protocols with strict local caches. AVG means unweighted 14-task macro; separately report ZS12, MMLU, CMMLU. Preserve existing RACE protocols and earlier results. + +Existing Fisher scores are reused from `results/qwen3_8b_base_nofit_bench14_20261009/summary.json`, Transport ON row: PPL 15.585997450543665, ZS12 0.5125956256004165, MMLU 0.25630252100840334, CMMLU 0.2646347781039544, Bench14 0.4765774861655254. Restore its final anchor from immutable Hub revision `b2a437466b134a6311d3ab625b0b37f9aa1a96d6` and measure only new Local MSE. Do not confuse this no-FiT baseline with historical Fisher plus FiT. + +Keep all nine compact anchor stages, original model pin, stage topology, calibration and evaluation token IDs, every transport matrix, MKA coefficients/geometry, metrics and code. This permits future evaluation against each previous-stage teacher without retraining. Upload each arm under `LJYAI/paper_revision/runs/SESSION/20261011-qwen3-base-nofit-merge4/ARM`, verify every payload at its immutable revision, commit results on the remote session branch and recover/push locally. Do not return the replacement server without further instruction. + +## Commands and status + +Bootstrap: `bash scripts/bootstrap_qwen3_nofit_merge4.sh PROJECT EXTERNAL ROOT MODEL`. + +Controller: `bash scripts/launch_qwen3_nofit_merge4.sh PROJECT EXTERNAL ROOT MODEL SESSION`. + +GPU assignment: Wa 0, Wb 1, Average 2, MKA 3. Each must pass a two-stage smoke with all 64 calibration sequences before any production arm proceeds to nine stages, Local MSE, PPL, Bench14 and verified all-stage backup. Controller commands and PID provenance are recorded in `ROOT/execution.json`. + +Exact merge tests passed: four cases covering Wa, Wb, Average and MKA, BF16 deployed values, unchanged source operands and correct frozen parameters. Local syntax compilation passed; local Python lacks torch, so these CPU-only tests ran in an isolated temporary directory using the existing server's environment without changing its project or GPU workload. Replacement instance is preparing; no new pruning has started at this checkpoint. + +At 16:59 KST the replacement was running and SSH setup completed at `23.127.144.217:26556`; all four devices reported A100 SXM4 40 GB. Code commit `c5846595387bf856a3feb780d8c21d86feca194e` was sent by bundle to the isolated session branch. Bootstrap PID 576 and controller PID 577 launched at 07:59:48 UTC. The pinned evaluation archives were transferred and the controller is waiting on exact input preparation. At 17:00 KST dependencies were ready and model/corpus snapshots were downloading without failed markers. + +Local monitor PID 56032 checks every 300 seconds, writing `C:/Users/CELL/orca/artifacts/paper_revision/qwen3-base-nofit-merge4-55359231-20261011-monitor/poll.log`; on complete it recovers the result commit and pushes from this computer. It is a status/recovery process, not an autonomous failure-repair agent. All-stage compact packaging and backup use hard links within the same experiment filesystem to retain every stage without storing three physical copies. Hub uploads still contain complete standalone stage files. + +By 17:07 KST the historical calibration SHA256 matched exactly and all 131 local evaluation configurations validated without Hub requests. Common dense teacher input/output collection completed, and the restored Fisher anchor passed exact common-prefix-input equality; new evaluation-only Local MSE is 30.159363496315123, normalized MSE 0.5497266315473979. Teacher input IDs SHA256 is `3f1977a32c7a533356dbc14d142c64b470e67dc9942344ff2942e3b1f8b9e2a1`, layer12 input SHA256 `57378a13b898fd63fc40d2c0ccb209ec5d5b636026accc93521b928669200250`, layer21 output SHA256 `8c246a6cca66306de3cd208e8ec8ef5df8eb9d3e3976f16ad34329cb1216ab40`. + +By 17:08 KST all four two-stage smokes passed scope, exact data, zero-FiT and topology verification. All four nine-stage production processes started. Wa/Wb/Average cycles took about 8.1 seconds; MKA about 19 seconds including per-stage geometry. Inspected GPU memory remained below 40 GB. Remote test total is eight passing tests, including the explicit hard-link retry/storage check and existing compact regressions. Source provenance includes subsequent note/test-only commits `e6faf799` and `5546ad91`; merge implementation remains commit `c5846595`. Final four-arm metrics and Hub receipts are pending; do not report the existing Fisher Local MSE as a newly completed four-arm comparison. diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/abprune_compact.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/abprune_compact.py new file mode 100644 index 0000000000000000000000000000000000000000..ffabddabd8ced8b87666f63fb9f99fae8b36cb38 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/abprune_compact.py @@ -0,0 +1,284 @@ +"""Package and reconstruct compact ABPrune stage checkpoints. + +ABPrune already writes ``fused_layer.pt`` for every cycle. This utility makes +those files portable by pairing them with an explicit, immutable stage record; +the dense base model plus that record is sufficient to reconstruct a student. +""" +from __future__ import annotations + +import argparse, hashlib, json, shutil +from pathlib import Path +from typing import Any + +import torch + + +def copy_stage_state(src: Path, dst: Path, *, hard_link: bool = False) -> None: + """Optionally share immutable stage bytes within one experiment filesystem.""" + if not hard_link: + shutil.copy2(src, dst) + elif dst.exists(): + if not src.samefile(dst): + raise FileExistsError(f"Refusing to replace existing non-linked stage: {dst}") + else: + # Fail instead of silently consuming space if cross-device linking is + # unavailable. The caller explicitly selected the storage strategy. + dst.hardlink_to(src) + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(8 * 1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def post_merge_scale(source_cycle: Path) -> dict | None: + """Return the deployable post-merge scale stored for a cycle, if present.""" + metadata_path = source_cycle / "cycle_metadata.json" + if not metadata_path.is_file(): + return None + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + distill = metadata.get("distill", {}) + scale = distill.get("post_merge_nll_scale") or {} + source = "post_merge_nll" + if not scale.get("enabled"): + scale = (distill.get("reparam_stats") or {}).get("residual_scale") or {} + source = "scale_aware_fit" + if not scale.get("enabled"): + return None + gamma = scale["gamma"] + cumulative_gamma = scale["cumulative_gamma"] + if not isinstance(gamma, list): + gamma = float(gamma) + else: + gamma = [float(value) for value in gamma] + if not isinstance(cumulative_gamma, list): + cumulative_gamma = float(cumulative_gamma) + else: + cumulative_gamma = [float(value) for value in cumulative_gamma] + result = { + "enabled": True, + "mode": scale.get("mode", "residual"), + "scale_parameterization": scale.get("scale_parameterization", "scalar"), + "gamma": gamma, + "cumulative_gamma": cumulative_gamma, + "source": source, + } + for field in ( + "optimization_mode", + "gamma_gradient_sources", + "theta_gradient_sources", + "nll_used", + ): + if field in scale: + result[field] = scale[field] + if "cumulative_gamma_statistics" in scale: + result["cumulative_gamma_statistics"] = scale["cumulative_gamma_statistics"] + if "raw" in scale and "ppl" in scale["raw"]: + result["calibration_raw_ppl"] = float(scale["raw"]["ppl"]) + if "fitted" in scale and "ppl" in scale["fitted"]: + result["calibration_fitted_ppl"] = float(scale["fitted"]["ppl"]) + return result + + +def read_compact_metadata(compact_stage: Path) -> dict[str, Any]: + return json.loads((compact_stage / "metadata.json").read_text(encoding="utf-8")) + + +def compact_scale_targets(metadata: dict[str, Any]) -> list[dict[str, Any]]: + """Return residual scales and current student indices for a compact stage.""" + if metadata.get("format") == "abprune_compact_stage_v2": + active = [int(value) for value in metadata["active_original_layers"]] + current_index = {original: index for index, original in enumerate(active)} + targets = [] + for state in metadata.get("retained_layer_states", []): + scale = state.get("post_merge_scale") or {} + if scale.get("enabled") and scale.get("mode") == "residual": + original = int(state["original_layer"]) + targets.append( + { + "original_layer": original, + "student_index": current_index[original], + "scale": scale, + } + ) + return targets + + scale = metadata.get("post_merge_scale") or {} + if scale.get("enabled") and scale.get("mode") == "residual": + return [ + { + "original_layer": int(metadata["anchor_index"]), + "student_index": int(metadata["anchor_index"]), + "scale": scale, + } + ] + return [] + + +def _package_topology_aware(args, source: Path, out: Path, order: list[int]) -> None: + """Package cycles whose recipients may switch between pair boundaries.""" + if int(getattr(args, "start_cycle", 1)) != 1: + raise ValueError("topology-aware compact packaging must start at cycle 1") + + removed: set[int] = set() + latest_states: dict[int, dict[str, Any]] = {} + stages: list[dict[str, Any]] = [] + original_layer_count: int | None = None + + for cycle in range(1, args.cycles + 1): + source_cycle = source / f"cycle_{cycle}" + src = source_cycle / "fused_layer.pt" + if not src.is_file(): + raise FileNotFoundError(src) + cycle_metadata_path = source_cycle / "cycle_metadata.json" + if not cycle_metadata_path.is_file(): + raise FileNotFoundError(cycle_metadata_path) + cycle_metadata = json.loads(cycle_metadata_path.read_text(encoding="utf-8")) + recipient = int(cycle_metadata["recipient_original_layer"]) + removed_original = int(cycle_metadata["removed_original_layer"]) + recipient_side = str(cycle_metadata["recipient_side"]) + if recipient_side not in {"left", "right"}: + raise ValueError(f"invalid recipient_side at cycle {cycle}: {recipient_side}") + if original_layer_count is None: + original_layer_count = int(cycle_metadata["num_layers_before"]) + if removed_original in removed: + raise ValueError(f"original layer {removed_original} removed more than once") + if recipient in removed: + raise ValueError(f"cycle {cycle} selects removed recipient {recipient}") + + removed.add(removed_original) + latest_states.pop(removed_original, None) + latest_states[recipient] = { + "original_layer": recipient, + "source_cycle": cycle, + "recipient_side": recipient_side, + "source_state": src, + "post_merge_scale": post_merge_scale(source_cycle), + } + + stage_dir = out / f"stage_{cycle:02d}" + stage_dir.mkdir(exist_ok=True) + retained_states = [] + for original, record in sorted(latest_states.items()): + state_name = f"layer_original_{original:04d}.pt" + state_path = stage_dir / state_name + copy_stage_state(record["source_state"], state_path, hard_link=bool(getattr(args, "hard_link", False))) + packaged = { + "original_layer": original, + "state_file": state_name, + "sha256": sha256(state_path), + "source_cycle": int(record["source_cycle"]), + "recipient_side": record["recipient_side"], + } + if record["post_merge_scale"] is not None: + packaged["post_merge_scale"] = record["post_merge_scale"] + retained_states.append(packaged) + + assert original_layer_count is not None + active = [ + original + for original in range(original_layer_count) + if original not in removed + ] + meta = { + "format": "abprune_compact_stage_v2", + "model": args.model, + "base_model_revision": args.base_revision, + "run_id": args.run_id, + "target_prune_count": args.cycles, + "stage": cycle, + "removed_count": len(removed), + "original_layer_count": original_layer_count, + "layer_order": order, + "latest_pair_index": order[cycle - 1], + "latest_recipient_side": recipient_side, + "latest_recipient_original_layer": recipient, + "latest_removed_original_layer": removed_original, + "removed_original_layers": sorted(removed), + "active_original_layers": active, + "retained_layer_states": retained_states, + "source_cycle": f"cycle_{cycle}", + } + (stage_dir / "metadata.json").write_text( + json.dumps(meta, indent=2) + "\n", encoding="utf-8" + ) + stages.append(meta) + + manifest = { + "format": "abprune_compact_manifest_v2", + "model": args.model, + "base_model_revision": args.base_revision, + "run_id": args.run_id, + "target_prune_count": args.cycles, + "original_layer_count": original_layer_count, + "layer_order": order, + "stages": stages, + } + (out / "manifest.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" + ) + + +def package(args): + source = Path(args.run_root); out = Path(args.output_root); out.mkdir(parents=True, exist_ok=True) + order = [int(x) for x in args.layer_order.replace(',', ' ').split()] + start_cycle = int(getattr(args, "start_cycle", 1)) + expected_order = args.cycles - start_cycle + 1 + if not 1 <= start_cycle <= args.cycles: + raise ValueError("start_cycle must be between 1 and cycles") + if len(order) != expected_order: + raise ValueError("layer_order length must equal cycles - start_cycle + 1") + first_metadata_path = source / f"cycle_{start_cycle}" / "cycle_metadata.json" + if first_metadata_path.is_file(): + first_metadata = json.loads(first_metadata_path.read_text(encoding="utf-8")) + topology_fields = { + "recipient_side", + "recipient_original_layer", + "removed_original_layer", + } + if topology_fields.issubset(first_metadata): + _package_topology_aware(args, source, out, order) + return + stages = [] + for cycle in range(start_cycle, args.cycles + 1): + order_index = cycle - start_cycle + source_cycle = source / f"cycle_{cycle}" + src = source_cycle / "fused_layer.pt" + if not src.exists(): raise FileNotFoundError(src) + dst_dir = out / f"stage_{cycle:02d}"; dst_dir.mkdir(exist_ok=True) + dst = dst_dir / "anchor_layer.pt"; copy_stage_state(src, dst, hard_link=bool(getattr(args, "hard_link", False))) + meta = { + "format": "abprune_compact_stage_v1", "model": args.model, + "base_model_revision": args.base_revision, "run_id": args.run_id, + "target_prune_count": args.cycles, "stage": cycle, + "removed_count": cycle, "anchor_index": order[order_index], + "layer_order": order, + "packaged_stage_range": [start_cycle, args.cycles], + "removed_original_layers": list(range(order[order_index] + 1, order[order_index] + cycle + 1)), + "source_cycle": f"cycle_{cycle}", "anchor_state_file": "anchor_layer.pt", + "sha256": sha256(dst), + } + scale = post_merge_scale(source_cycle) + if scale is not None: + meta["post_merge_scale"] = scale + (dst_dir / "metadata.json").write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8") + stages.append(meta) + manifest = {"format": "abprune_compact_manifest_v1", "model": args.model, + "base_model_revision": args.base_revision, "run_id": args.run_id, + "target_prune_count": args.cycles, "layer_order": order, "stages": stages} + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + +def main(): + p = argparse.ArgumentParser(); p.add_argument("--run-root", required=True); p.add_argument("--output-root", required=True) + p.add_argument("--model", required=True); p.add_argument("--base-revision", required=True); p.add_argument("--run-id", required=True) + p.add_argument("--cycles", type=int, required=True); p.add_argument("--layer-order", required=True) + p.add_argument("--start-cycle", type=int, default=1) + p.add_argument("--hard-link", action="store_true", help="Hard-link immutable stage weights on the same filesystem") + package(p.parse_args()) + + +if __name__ == "__main__": main() diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/abprune_stagewise_geometry.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/abprune_stagewise_geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..8ee3d79254a828fc6f93cde8887159cead689abc --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/abprune_stagewise_geometry.py @@ -0,0 +1,227 @@ +"""Stage-wise activation geometry diagnostics for progressive ABPrune runs. + +The ABPrune checkpoints are treated as black-box students. The dense model is +the fixed teacher and boundaries are aligned by the original depth represented +by the fused anchor (rather than by student layer index). +""" +from __future__ import annotations + +import argparse, gc, json, math +from pathlib import Path +from typing import Dict, Iterable + +import torch + +from .abprune_compact import read_compact_metadata +from .abprune_recoverability import capture_module_output, load_model +from .span_diagnostics import build_token_chunks, find_layers, get_dtype + + +def _layer_container(model): + for path in ("model.layers", "model.decoder.layers", "transformer.h", "transformer.blocks", "gpt_neox.layers"): + parent = model + parts = path.split('.') + for part in parts[:-1]: + if not hasattr(parent, part): break + parent = getattr(parent, part) + else: + if hasattr(parent, parts[-1]): + container = getattr(parent, parts[-1]) + if isinstance(container, torch.nn.ModuleList): return container + raise ValueError("Could not locate a mutable decoder layer ModuleList") + + +def _reindex_layer_modules(layers) -> None: + """Align cache-facing layer indices with the compact decoder positions.""" + for index, layer in enumerate(layers): + for module in layer.modules(): + if hasattr(module, "layer_idx"): + module.layer_idx = index + + +def load_compact_model( + base_ref: str, + compact_stage: Path, + *, + cycle: int | None, + anchor: int | None, + dtype: torch.dtype, + device: torch.device, + attn: str, + revision: str | None = None, +): + """Reconstruct legacy single-anchor or topology-aware compact stages.""" + model = load_model(base_ref, dtype=dtype, device=device, attn=attn, revision=revision) + container = _layer_container(model) + layers = list(container) + metadata = read_compact_metadata(compact_stage) + if metadata.get("format") == "abprune_compact_stage_v2": + if cycle is not None and int(metadata["stage"]) != int(cycle): + raise ValueError( + f"compact stage mismatch: metadata={metadata['stage']} requested={cycle}" + ) + removed = {int(value) for value in metadata["removed_original_layers"]} + expected_active = [int(value) for value in metadata["active_original_layers"]] + active = [index for index in range(len(layers)) if index not in removed] + if active != expected_active: + raise ValueError( + f"compact topology mismatch: computed={active} metadata={expected_active}" + ) + for entry in metadata["retained_layer_states"]: + original = int(entry["original_layer"]) + state_path = compact_stage / entry["state_file"] + if not state_path.is_file(): + raise FileNotFoundError(state_path) + state = torch.load(state_path, map_location="cpu", weights_only=True) + layers[original].load_state_dict(state, strict=True) + layers = [layers[index] for index in active] + else: + if cycle is None or anchor is None: + raise ValueError("legacy compact stages require cycle and anchor") + active = [ + index + for index in range(len(layers)) + if index not in range(anchor + 1, anchor + 1 + cycle) + ] + state = torch.load( + compact_stage / "anchor_layer.pt", map_location="cpu", weights_only=True + ) + layers[anchor].load_state_dict(state, strict=True) + del layers[anchor + 1: anchor + 1 + cycle] + for idx in range(len(container) - 1, -1, -1): + del container[idx] + container.extend(layers) + _reindex_layer_modules(container) + if hasattr(model.config, "num_hidden_layers"): + model.config.num_hidden_layers = len(layers) + if getattr(model.config, "layer_types", None) is not None: + model.config.layer_types = [ + model.config.layer_types[index] for index in active + ] + return model + + +def _metric(student: torch.Tensor, teacher: torch.Tensor, mask: torch.Tensor) -> Dict[str, float]: + s = student.float().squeeze(0)[mask] + t = teacher.float().squeeze(0)[mask] + denom = t.square().sum().item() + sden = s.square().sum().item() + dot = (s * t).sum().item() + sn = s.norm(dim=-1).clamp_min(1e-12) + tn = t.norm(dim=-1).clamp_min(1e-12) + alpha = dot / max(sden, 1e-30) + return { + "nmse": float((s - t).square().sum().item() / max(denom, 1e-30)), + "scaled_nmse": float((alpha * s - t).square().sum().item() / max(denom, 1e-30)), + "cosine": float(((s * t).sum(dim=-1) / (sn * tn)).mean().item()), + "norm_ratio": float((sn / tn).mean().item()), + "optimal_alpha": float(alpha), + "tokens": int(s.shape[0]), + } + + +def _teacher_cache(model, chunks, boundaries: Iterable[int], device): + wanted = sorted(set(boundaries)) + cache = {b: [] for b in wanted} + layers = find_layers(model) + with torch.inference_mode(): + for i, chunk in enumerate(chunks): + ids = chunk[None].to(device) + out = model(input_ids=ids, use_cache=False, output_hidden_states=True) + for b in wanted: + if b < 0 or b >= len(layers): + raise ValueError(f"invalid teacher boundary {b}") + cache[b].append(out.hidden_states[b + 1].detach().cpu().half()) + final_hidden = getattr(out, "last_hidden_state", None) + if final_hidden is None: + final_hidden = out.hidden_states[-1] + cache.setdefault("final", []).append(final_hidden.detach().cpu().half()) + if (i + 1) % 4 == 0 or i + 1 == len(chunks): + print(f"[stagewise] teacher {i+1}/{len(chunks)}", flush=True) + return cache + + +def _student_stage(model, chunks, teacher, *, anchor, original_end, device): + layers = find_layers(model) + if anchor >= len(layers): + raise ValueError(f"anchor {anchor} not present in {len(layers)} student layers") + local, final = [], [] + target_local = teacher[original_end] + with torch.inference_mode(): + for i, chunk in enumerate(chunks): + ids = chunk[None].to(device) + with capture_module_output(layers[anchor]) as hook: + out = model(input_ids=ids, use_cache=False, output_hidden_states=True) + local.append(hook["output"].cpu().half()) + final_hidden = getattr(out, "last_hidden_state", None) + if final_hidden is None: + final_hidden = out.hidden_states[-1] + final.append(final_hidden.detach().cpu().half()) + return local, final + + +def run(args): + device = torch.device(args.device) + dtype = get_dtype(args.dtype) + from transformers import AutoTokenizer + tok = AutoTokenizer.from_pretrained(args.model) + chunks = build_token_chunks(tok, dataset_name="Salesforce/wikitext", dataset_config="wikitext-2-raw-v1", + split="test", text_field="text", seq_len=args.seq_len, num_sequences=args.num_sequences, seed=args.seed) + if len(chunks) != args.num_sequences: + raise RuntimeError(f"only built {len(chunks)} held-out chunks") + teacher_model = load_model(args.model, dtype=dtype, device=device, attn=args.attn) + depth = len(find_layers(teacher_model)) + stages = list(args.stages) if args.stages else list(range(1, args.cycles + 1)) + if not stages or any(stage < 1 or stage > args.cycles for stage in stages): + raise ValueError("--stages must contain values in [1, --cycles]") + boundaries = [args.anchor + stage for stage in stages] + teacher = _teacher_cache(teacher_model, chunks, boundaries, device) + # ``final`` is cached alongside the requested original block boundaries. + del teacher_model; gc.collect(); torch.cuda.empty_cache() + rows = [] + for cycle in stages: + if args.compact_root: + ref = Path(args.compact_root) / f"stage_{cycle:02d}" + if not (ref / "metadata.json").exists(): raise FileNotFoundError(ref) + student = load_compact_model(args.model, ref, cycle=cycle, anchor=args.anchor, + dtype=dtype, device=device, attn=args.attn) + else: + ref = Path(args.student_root) / f"cycle_{cycle}" / "full_model" + if not (ref / "config.json").exists(): raise FileNotFoundError(ref) + student = load_model(str(ref), dtype=dtype, device=device, attn=args.attn) + local, final = _student_stage(student, chunks, teacher, anchor=args.anchor, + original_end=args.anchor + cycle, device=device) + for pos0_excluded in (True, False): + local_acc = {k: [] for k in ("nmse", "scaled_nmse", "cosine", "norm_ratio", "optimal_alpha")} + final_acc = {k: [] for k in local_acc} + for i in range(len(chunks)): + mask = torch.ones((args.seq_len,), dtype=torch.bool) + if pos0_excluded: mask[0] = False + lm = _metric(local[i], teacher[args.anchor + cycle][i], mask) + fm = _metric(final[i], teacher["final"][i], mask) + for k in local_acc: local_acc[k].append(lm[k]); final_acc[k].append(fm[k]) + rows.append({"model": args.model, "target_prune_count": args.cycles, "stage": cycle, + "removed_count": cycle, "anchor": args.anchor, "merged_original_layers": list(range(args.anchor, args.anchor + cycle + 1)), + "teacher_boundary_index": args.anchor + cycle, "student_layer_index": args.anchor, + "position0_excluded": pos0_excluded, + "local": {k: float(sum(v)/len(v)) for k,v in local_acc.items()}, + "final": {k: float(sum(v)/len(v)) for k,v in final_acc.items()}, + "num_sequences": len(chunks), "seq_len": args.seq_len, "seed": args.seed}) + print(f"[stagewise] completed cycle {cycle}/{args.cycles}", flush=True) + del student; gc.collect(); torch.cuda.empty_cache() + out = Path(args.output); out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps({"model": args.model, "anchor": args.anchor, "cycles": args.cycles, "rows": rows}, indent=2) + "\n") + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--model", required=True); p.add_argument("--student-root", required=True); p.add_argument("--anchor", type=int, required=True) + p.add_argument("--cycles", type=int, required=True); p.add_argument("--output", required=True) + p.add_argument("--stages", type=int, nargs="*", default=None, + help="Optional subset of compact stages to analyze (default: every stage)") + p.add_argument("--compact-root", default=None, help="stage_XX/anchor_layer.pt package; overrides --student-root") + p.add_argument("--device", default="cuda"); p.add_argument("--dtype", default="bf16"); p.add_argument("--attn", default="eager") + p.add_argument("--num-sequences", type=int, default=16); p.add_argument("--seq-len", type=int, default=512); p.add_argument("--seed", type=int, default=12345) + args = p.parse_args(); run(args) + +if __name__ == "__main__": main() diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/aggregate_transport_rho_fit_14bench.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/aggregate_transport_rho_fit_14bench.py new file mode 100644 index 0000000000000000000000000000000000000000..b6dfcea6eb20c3fc5653b5e2ec8e1cf3ff6553c3 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/aggregate_transport_rho_fit_14bench.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Combine the OpenCompass-five and lm-eval-nine evaluation reports.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +OPENCOMPASS_KEYS = { + "race_high": "race-high", + "race_middle": "race-middle", + "c3": "C3", + "cmnli": "cmnli", + "chid": "chid-dev", +} +LM_EVAL_KEYS = { + "piqa", + "winogrande", + "hellaswag", + "arc_challenge", + "arc_easy", + "openbookqa", + "sciq", + "mmlu", + "cmmlu", +} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("run_root", type=Path) + args = parser.parse_args() + + eval_dir = args.run_root / "eval" + opencompass = json.loads( + (eval_dir / "final_opencompass5.json").read_text(encoding="utf-8") + ) + lm_eval = json.loads( + (eval_dir / "final_lmeval9.json").read_text(encoding="utf-8") + ) + if set(lm_eval["task_scores"]) != LM_EVAL_KEYS: + raise RuntimeError( + f"unexpected lm-eval tasks: {sorted(lm_eval['task_scores'])}" + ) + + scores = { + name: opencompass["results"][source]["accuracy"] + for name, source in OPENCOMPASS_KEYS.items() + } + scores.update(lm_eval["task_scores"]) + if len(scores) != 14 or any(not 0.0 <= float(value) <= 1.0 for value in scores.values()): + raise RuntimeError(f"invalid 14-task score set: {scores}") + + report = { + "task_scores": scores, + "macro_mean_14": sum(scores.values()) / len(scores), + "tasks": list(scores), + "opencompass_protocol": opencompass["protocol"], + "lm_eval_num_fewshot": lm_eval["num_fewshot"], + } + output = eval_dir / "final_14bench.json" + output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + (args.run_root / "BENCH14_COMPLETE").touch() + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/bootstrap_qwen3_nofit_bench8.sh b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/bootstrap_qwen3_nofit_bench8.sh new file mode 100644 index 0000000000000000000000000000000000000000..3d8cae12debb749d6eb992c32ef4c2d20ef2600b --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/bootstrap_qwen3_nofit_bench8.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail +project="$1"; root="$2"; model="$3" +export HF_HOME=/workspace/.hf_home HF_TOKEN_PATH=/root/.cache/huggingface/token +export HF_DATASETS_CACHE=/workspace/.hf_datasets36 PYTHONUNBUFFERED=1 OMP_NUM_THREADS=4 +mkdir -p "$root" +trap 'rc=$?; if ((rc)); then touch "$root/BOOTSTRAP_FAILED"; fi' EXIT +python -m pip install 'transformers==5.15.1' 'datasets==3.6.0' 'huggingface_hub==1.33.0' \ + 'lm_eval==0.4.12' accelerate scipy sentencepiece protobuf +python - "$root" "$model" <<'PY' +from concurrent.futures import ThreadPoolExecutor +from huggingface_hub import snapshot_download,hf_hub_download +from pathlib import Path +import hashlib,json,shutil,sys +r=Path(sys.argv[1]) +prefix='runs/vast-paper_revision-qwen3-base-nofit-transport-zs6-54974094-20261009/20261009-qwen3-base-nofit-transport-zs6' +pins={'off':'71252a138f0f97349f29c743206160b77e45899e','on':'b2a437466b134a6311d3ab625b0b37f9aa1a96d6'} +def base(): + return snapshot_download('Qwen/Qwen3-8B-Base',revision='49e3418fbbbca6ecbdf9608b4d22e5a407081db4', + local_dir=sys.argv[2],allow_patterns=['*.json','*.safetensors','tokenizer*','*.model','*.jinja'],max_workers=8) +def restore(arm): + rev=pins[arm]; path=f'{prefix}/{arm}' + manifest=json.loads(Path(hf_hub_download('LJYAI/paper_revision',f'{path}/manifest.json',revision=rev)).read_text()) + out=r/'full'/arm; restored=[] + for item in manifest['files']: + rel=item['path'] + if not (rel.startswith('compact/stage_09/') or rel=='eval/zs6.json'): continue + src=Path(hf_hub_download('LJYAI/paper_revision',f'{path}/{rel}',revision=rev)) + assert src.stat().st_size==item['size'] + assert hashlib.sha256(src.read_bytes()).hexdigest()==item['sha256'] + dst=out/rel;dst.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(src,dst);restored.append(item) + meta=json.loads((out/'compact/stage_09/metadata.json').read_text()) + assert meta['anchor_index']==12 and meta['removed_original_layers']==list(range(13,22)) + assert meta['stage']==9 and meta['format']=='abprune_compact_stage_v1' + assert (out/'eval/zs6.json').is_file() and len(list((out/'compact/stage_09').glob('*.pt')))==1 + receipt={'arm':arm,'repository':'LJYAI/paper_revision','path':path,'revision':rev, + 'restored_files':restored,'all_restored_hashes_match':True} + (out/'restore_receipt.json').write_text(json.dumps(receipt,indent=2)+'\n') + return receipt +with ThreadPoolExecutor(max_workers=3) as p: + futures=[p.submit(base),p.submit(restore,'off'),p.submit(restore,'on')] + for f in futures: print('PINNED_INPUT_READY',f.result(),flush=True) +(r/'BASE_READY').touch() +PY +while [[ ! -f "$root/EVAL_ASSETS_READY" ]]; do sleep 3; done +python - "$root" <<'PY' +from pathlib import Path +import hashlib,sys +r=Path(sys.argv[1]) +for name,expected in [('opencompass5-data.tar.gz','b49605cc272cee94ddc177d2350e06d70f0f610926a2abf20c8ccd35ad8cd353'), + ('lmeval-cache-131.tar.gz','e5a319ed00b52181df3d80e47e4939e78fddd393803c29a9bcb63c2dd8a6d468')]: + assert hashlib.sha256((r/name).read_bytes()).hexdigest()==expected +print('EVALUATION_ARCHIVE_HASHES_VERIFIED',flush=True) +PY +mkdir -p "$root/eval_data" +tar -xzf "$root/opencompass5-data.tar.gz" -C "$root/eval_data" +tar -xzf "$root/lmeval-cache-131.tar.gz" -C "$root" +HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 python "$project/scripts/prepare_lmeval9_local_cache.py" "$root/lmeval_cache" --validate-local +touch "$root/EVAL_READY" "$root/BOOTSTRAP_COMPLETE" diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/bootstrap_qwen3_nofit_merge4.sh b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/bootstrap_qwen3_nofit_merge4.sh new file mode 100644 index 0000000000000000000000000000000000000000..e7b466054d96a1a9fa0acbe2932c901554c823ca --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/bootstrap_qwen3_nofit_merge4.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail +project="$1"; external="$2"; root="$3"; model="$4" +export HF_HOME=/workspace/.hf_home HF_TOKEN_PATH=/root/.cache/huggingface/token +export HF_DATASETS_CACHE=/workspace/.hf_datasets36 PYTHONUNBUFFERED=1 +export PYTHONPATH="$external/src:$project/src:$project" TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=4 +mkdir -p "$root" +trap 'rc=$?; if ((rc)); then touch "$root/BOOTSTRAP_FAILED"; fi' EXIT +# Pinned engine + dense model + the original non-streaming SlimPajama cache. +# One download/cache preparation is shared by all four GPU workers. +bash "$project/scripts/bootstrap_qwen3_8b_mergeinit_fit2.sh" "$project" "$external" "$root" "$model" +python -m pytest -q "$project/tests/test_nofit_merge_control.py" > "$root/unit_tests.log" 2>&1 +python "$project/scripts/prepare_qwen3_nofit_merge_inputs.py" --external-root "$external" \ + --model "$model" --output "$root/inputs" > "$root/data_prepare.log" 2>&1 +touch "$root/DATA_READY" +python - "$root" <<'PY' +from pathlib import Path +from huggingface_hub import hf_hub_download +from datasets import load_dataset +import hashlib,json,shutil,sys +r=Path(sys.argv[1]); prefix='runs/vast-paper_revision-qwen3-base-nofit-transport-zs6-54974094-20261009/20261009-qwen3-base-nofit-transport-zs6/on' +revision='b2a437466b134a6311d3ab625b0b37f9aa1a96d6' +manifest=json.loads(Path(hf_hub_download('LJYAI/paper_revision',prefix+'/manifest.json',revision=revision)).read_text()) +restored=[] +for item in manifest['files']: + if not item['path'].startswith('compact/stage_09/'): continue + src=Path(hf_hub_download('LJYAI/paper_revision',prefix+'/'+item['path'],revision=revision)) + assert src.stat().st_size==item['size'] and hashlib.sha256(src.read_bytes()).hexdigest()==item['sha256'] + dst=r/'fisher_reference'/item['path'];dst.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(src,dst);restored.append(item) +assert len(restored)==2 +(r/'fisher_reference/restore_receipt.json').write_text(json.dumps(dict(repo='LJYAI/paper_revision',path=prefix,revision=revision,files=restored,verified=True),indent=2)+'\n') +load_dataset('Salesforce/wikitext','wikitext-2-raw-v1') +PY +python "$project/scripts/prepare_offline_ppl_cache.py" "$root" +while [[ ! -f "$root/EVAL_ASSETS_READY" ]]; do sleep 3; done +python - "$root" <<'PY' +import hashlib,sys +from pathlib import Path +r=Path(sys.argv[1]) +for name,h in [('opencompass5-data.tar.gz','b49605cc272cee94ddc177d2350e06d70f0f610926a2abf20c8ccd35ad8cd353'),('lmeval-cache-131.tar.gz','e5a319ed00b52181df3d80e47e4939e78fddd393803c29a9bcb63c2dd8a6d468')]: + assert hashlib.sha256((r/name).read_bytes()).hexdigest()==h +PY +mkdir -p "$root/eval_data" +tar -xzf "$root/opencompass5-data.tar.gz" -C "$root/eval_data" +tar -xzf "$root/lmeval-cache-131.tar.gz" -C "$root" +HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 python "$project/scripts/prepare_lmeval9_local_cache.py" "$root/lmeval_cache" --validate-local +touch "$root/EVAL_READY" +CUDA_VISIBLE_DEVICES=0 HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ + python "$project/scripts/qwen3_nofit_local_mse.py" --model "$model" --cache "$root/local_mse_reference" \ + --wikitext-cache "$root/wikitext2_cache" --prepare > "$root/local_mse_prepare.log" 2>&1 +touch "$root/COMMON_TEACHER_READY" +CUDA_VISIBLE_DEVICES=0 HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ + python "$project/scripts/qwen3_nofit_local_mse.py" --model "$model" --cache "$root/local_mse_reference" \ + --compact-stage "$root/fisher_reference/compact/stage_09" --output "$root/fisher_reference/local_mse.json" > "$root/fisher_mse.log" 2>&1 +touch "$root/INPUTS_READY" diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/common_lm_data.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/common_lm_data.py new file mode 100644 index 0000000000000000000000000000000000000000..bfa1733a8a6f38fe03dc714353fadc373029ee9c --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/common_lm_data.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 +"""Shared LM dataset helpers for fair cross-method comparisons.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, Iterable, Iterator, List, Optional, Tuple + +import torch + +from dm500_rows import configured_indices, dataset_revision_kwargs, select_exact_rows + +try: + from datasets import load_dataset + from datasets import Dataset as HFDataset +except Exception: # pragma: no cover - optional dependency + load_dataset = None + HFDataset = None + + +def _normalize_config(config: Optional[str]) -> Optional[str]: + if config is None: + return None + if config.strip().lower() in {"none", "null", "-"}: + return None + return config + + +def guess_text_field(dataset) -> str: + if hasattr(dataset, "column_names") and dataset.column_names: + if "text" in dataset.column_names: + return "text" + return dataset.column_names[0] + if hasattr(dataset, "features"): + names = list(dataset.features.keys()) + if "text" in names: + return "text" + if names: + return names[0] + return "text" + + +def normalize_dataset_name(name: str) -> str: + normalized = name.strip().lower() + aliases = { + "bookcorpus": "bookcorpus", + "boockcorpus": "bookcorpus", + "slimpajama": "slimpajama", + "dkyoon/slimpajama-6b": "slimpajama", + "slimorca": "slimorca", + "orca": "slimorca", + "open-orca/slimorca": "slimorca", + "fineweb": "fineweb", + "huggingfacefw/fineweb": "fineweb", + } + if normalized not in aliases: + raise ValueError(f"Unsupported dataset: {name}") + return aliases[normalized] + + +def resolve_dataset_spec( + name: str, + config: Optional[str] = None, + split: str = "train", +) -> Tuple[str, Optional[str], str]: + normalized = normalize_dataset_name(name) + if normalized == "bookcorpus": + return "bookcorpus", _normalize_config(config), split + if normalized == "slimpajama": + return "DKYoon/SlimPajama-6B", _normalize_config(config), split + if normalized == "slimorca": + return "Open-Orca/SlimOrca", _normalize_config(config), split + if normalized == "fineweb": + return "HuggingFaceFW/fineweb", _normalize_config(config), split + raise ValueError(f"Unsupported dataset: {name}") + + +def is_instruction_dataset(name: str) -> bool: + return normalize_dataset_name(name) == "slimorca" + + +def should_stream_dataset(name: str) -> bool: + return normalize_dataset_name(name) == "fineweb" + + +def load_resolved_dataset( + dataset_name: str, + hf_name: str, + hf_config: Optional[str], + hf_split: str, +): + kwargs = { + "split": hf_split, + **dataset_revision_kwargs(), + } + if should_stream_dataset(dataset_name): + kwargs["streaming"] = True + return load_dataset(hf_name, hf_config, **kwargs) + + +def _build_messages_text(messages: List[Dict[str, str]]) -> str: + chunks: List[str] = [] + for message in messages: + role = str(message.get("role", "")).strip().title() or "User" + content = str(message.get("content", "")).strip() + if content: + chunks.append(f"{role}: {content}") + return "\n\n".join(chunks) + + +def _build_slimorca_text(row: Dict[str, object]) -> Optional[str]: + conversations = row.get("conversations") + if not isinstance(conversations, list) or len(conversations) < 2: + return None + + start_idx = 0 + if str(conversations[0].get("from", "")).strip().lower() != "human": + start_idx = 1 + if start_idx + 1 >= len(conversations): + return None + + user_turn = conversations[start_idx] + assistant_turn = conversations[start_idx + 1] + if ( + str(user_turn.get("from", "")).strip().lower() != "human" + or str(assistant_turn.get("from", "")).strip().lower() != "gpt" + ): + return None + + user_content = str(user_turn.get("value", "")).strip() + assistant_content = str(assistant_turn.get("value", "")).strip() + if not user_content or not assistant_content: + return None + + messages = [ + {"role": "user", "content": user_content}, + {"role": "assistant", "content": assistant_content}, + ] + return _build_messages_text(messages) + + +def _build_slimorca_record(row: Dict[str, object]) -> Optional[Dict[str, object]]: + conversations = row.get("conversations") + if not isinstance(conversations, list) or len(conversations) < 2: + return None + + start_idx = 0 + if str(conversations[0].get("from", "")).strip().lower() != "human": + start_idx = 1 + if start_idx + 1 >= len(conversations): + return None + + user_turn = conversations[start_idx] + assistant_turn = conversations[start_idx + 1] + if ( + str(user_turn.get("from", "")).strip().lower() != "human" + or str(assistant_turn.get("from", "")).strip().lower() != "gpt" + ): + return None + + user_content = str(user_turn.get("value", "")).strip() + assistant_content = str(assistant_turn.get("value", "")).strip() + if not user_content or not assistant_content: + return None + + messages = [ + {"role": "user", "content": user_content}, + {"role": "assistant", "content": assistant_content}, + ] + return {"messages": messages, "text": _build_messages_text(messages)} + + +def _tokenize_instruction_record(record: Dict[str, object], tokenizer) -> List[int]: + chat_template = getattr(tokenizer, "chat_template", None) + if ( + "messages" in record + and hasattr(tokenizer, "apply_chat_template") + and chat_template + ): + ids = tokenizer.apply_chat_template( + record["messages"], + tokenize=True, + add_generation_prompt=False, + ) + else: + ids = tokenizer.encode(str(record.get("text", "")), add_special_tokens=False) + + if hasattr(ids, "input_ids"): + ids = ids.input_ids + if isinstance(ids, torch.Tensor): + return ids.tolist() + if isinstance(ids, list): + return ids + return list(ids) + + +def _round_up_to_multiple(value: int, multiple: int) -> int: + if multiple <= 1: + return value + return ((value + multiple - 1) // multiple) * multiple + + +def _sample_dataset_rows(dataset, target: int, seed: int) -> List[Dict[str, object]]: + if target <= 0: + return [] + exact = select_exact_rows(dataset, target) + if exact is not None: + return [row for row in exact] + try: + dataset = dataset.shuffle(seed=seed) + except Exception: + pass + + if hasattr(dataset, "__len__"): + limit = min(target, len(dataset)) + dataset = dataset.select(range(limit)) + return [row for row in dataset] + + rows = [] + for row in dataset: + rows.append(row) + if len(rows) >= target: + break + return rows + + +def _iter_dataset_rows(dataset, seed: int) -> Iterator[Dict[str, object]]: + exact = select_exact_rows(dataset) + if exact is not None: + for row in exact: + yield row + return + try: + dataset = dataset.shuffle(seed=seed) + except Exception: + pass + for row in dataset: + yield row + + +def load_named_texts( + dataset_name: str, + *, + config: Optional[str] = None, + split: str = "train", + text_field: Optional[str] = None, + num_samples: int = 0, + seed: int = 0, +) -> List[str]: + if load_dataset is None: + raise SystemExit("datasets is required for shared LM dataloaders") + + hf_name, hf_config, hf_split = resolve_dataset_spec(dataset_name, config, split) + dataset = load_resolved_dataset(dataset_name, hf_name, hf_config, hf_split) + rows = dataset if num_samples <= 0 else _sample_dataset_rows(dataset, num_samples, seed) + field = text_field or guess_text_field(dataset) + + texts: List[str] = [] + for row in rows: + if not isinstance(row, dict): + continue + if is_instruction_dataset(dataset_name): + value = _build_slimorca_text(row) + else: + value = row.get(field, None) + if isinstance(value, str) and value.strip(): + texts.append(value) + return texts + + +def build_token_chunks_from_rows( + rows: Iterable[Dict[str, object]], + *, + dataset_name: Optional[str] = None, + text_field: str, + tokenizer, + seq_len: int, + num_sequences: int = 0, + add_bos: bool = False, + max_rows: int = 0, +) -> List[torch.Tensor]: + chunks: List[torch.Tensor] = [] + buffer: List[int] = [] + limit = None if num_sequences <= 0 else num_sequences + rows_seen = 0 + + for row in rows: + if max_rows > 0 and rows_seen >= max_rows: + break + rows_seen += 1 + + if not isinstance(row, dict): + continue + if dataset_name is not None and is_instruction_dataset(dataset_name): + value = _build_slimorca_text(row) + else: + value = row.get(text_field, None) + if not isinstance(value, str) or not value.strip(): + continue + + ids = tokenizer.encode(value, add_special_tokens=False) + if add_bos and tokenizer.bos_token_id is not None: + ids = [tokenizer.bos_token_id] + ids + if not ids: + continue + + buffer.extend(ids) + while len(buffer) >= seq_len and (limit is None or len(chunks) < limit): + chunk = buffer[:seq_len] + buffer = buffer[seq_len:] + chunks.append(torch.tensor(chunk, dtype=torch.long)) + if limit is not None and len(chunks) >= limit: + break + + return chunks + + +def collect_texts_from_rows( + rows: Iterable[Dict[str, object]], + *, + dataset_name: Optional[str] = None, + text_field: str, + tokenizer, + target_tokens: int = 0, + add_bos: bool = False, + max_rows: int = 0, +) -> List[str]: + texts: List[str] = [] + token_count = 0 + rows_seen = 0 + + for row in rows: + if max_rows > 0 and rows_seen >= max_rows: + break + rows_seen += 1 + + if not isinstance(row, dict): + continue + if dataset_name is not None and is_instruction_dataset(dataset_name): + value = _build_slimorca_text(row) + else: + value = row.get(text_field, None) + if not isinstance(value, str) or not value.strip(): + continue + + texts.append(value) + if target_tokens > 0: + ids = tokenizer.encode(value, add_special_tokens=False) + if add_bos and tokenizer.bos_token_id is not None: + ids = [tokenizer.bos_token_id] + ids + token_count += len(ids) + if token_count >= target_tokens: + break + + return texts + + +def build_token_chunks( + texts: Iterable[str], + tokenizer, + seq_len: int, + num_sequences: int = 0, + add_bos: bool = False, +) -> List[torch.Tensor]: + chunks: List[torch.Tensor] = [] + buffer: List[int] = [] + limit = None if num_sequences <= 0 else num_sequences + + for text in texts: + ids = tokenizer.encode(text, add_special_tokens=False) + if add_bos and tokenizer.bos_token_id is not None: + ids = [tokenizer.bos_token_id] + ids + if not ids: + continue + + buffer.extend(ids) + while len(buffer) >= seq_len and (limit is None or len(chunks) < limit): + chunk = buffer[:seq_len] + buffer = buffer[seq_len:] + chunks.append(torch.tensor(chunk, dtype=torch.long)) + if limit is not None and len(chunks) >= limit: + break + + return chunks + + +class TokenChunkDataset(torch.utils.data.Dataset): + def __init__(self, chunks: List[torch.Tensor]) -> None: + self.chunks = chunks + + def __len__(self) -> int: + return len(self.chunks) + + def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: + input_ids = self.chunks[idx] + attention_mask = torch.ones_like(input_ids) + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "labels": input_ids.clone(), + } + + +class TokenOnlyDataset(torch.utils.data.Dataset): + def __init__(self, chunks: List[torch.Tensor]) -> None: + self.chunks = chunks + + def __len__(self) -> int: + return len(self.chunks) + + def __getitem__(self, idx: int) -> torch.Tensor: + return self.chunks[idx] + + +class TokenInputMaskDataset(torch.utils.data.Dataset): + def __init__(self, chunks: List[torch.Tensor]) -> None: + self.chunks = chunks + + def __len__(self) -> int: + return len(self.chunks) + + def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: + input_ids = self.chunks[idx] + return { + "input_ids": input_ids, + "attention_mask": torch.ones_like(input_ids), + } + + +@dataclass +class SharedLMDataSpec: + dataset: str + config: Optional[str] = None + split: str = "train" + text_field: Optional[str] = None + num_samples: int = 0 + seq_len: int = 2048 + num_sequences: int = 0 + target_tokens: int = 0 + batch_size: int = 1 + shuffle: bool = False + num_workers: int = 0 + seed: int = 0 + add_bos: bool = False + + +def build_chunks(spec: SharedLMDataSpec, tokenizer) -> List[torch.Tensor]: + if load_dataset is None: + raise SystemExit("datasets is required for shared LM dataloaders") + + hf_name, hf_config, hf_split = resolve_dataset_spec(spec.dataset, spec.config, spec.split) + dataset = load_resolved_dataset(spec.dataset, hf_name, hf_config, hf_split) + + target_sequences = spec.num_sequences + if spec.target_tokens > 0: + token_sequences = (spec.target_tokens + spec.seq_len - 1) // spec.seq_len + target_sequences = max(target_sequences, token_sequences) + row_limit = spec.num_samples if target_sequences <= 0 else 0 + + rows = _iter_dataset_rows(dataset, spec.seed) + text_field = spec.text_field or guess_text_field(dataset) + chunks = build_token_chunks_from_rows( + rows, + dataset_name=spec.dataset, + text_field=text_field, + tokenizer=tokenizer, + seq_len=spec.seq_len, + num_sequences=target_sequences, + add_bos=spec.add_bos, + max_rows=row_limit, + ) + return chunks + + +def build_dataloader(spec: SharedLMDataSpec, tokenizer) -> torch.utils.data.DataLoader: + chunks = build_chunks(spec, tokenizer) + dataset = TokenChunkDataset(chunks) + return torch.utils.data.DataLoader( + dataset, + batch_size=spec.batch_size, + shuffle=spec.shuffle, + num_workers=spec.num_workers, + ) + + +def build_text_dataloader(spec: SharedLMDataSpec, tokenizer) -> torch.utils.data.DataLoader: + if load_dataset is None: + raise SystemExit("datasets is required for shared LM dataloaders") + + hf_name, hf_config, hf_split = resolve_dataset_spec(spec.dataset, spec.config, spec.split) + dataset = load_resolved_dataset(spec.dataset, hf_name, hf_config, hf_split) + rows = _iter_dataset_rows(dataset, spec.seed) + text_field = spec.text_field or guess_text_field(dataset) + row_limit = spec.num_samples + texts = collect_texts_from_rows( + rows, + dataset_name=spec.dataset, + text_field=text_field, + tokenizer=tokenizer, + target_tokens=spec.target_tokens, + add_bos=spec.add_bos, + max_rows=row_limit, + ) + return torch.utils.data.DataLoader( + texts, + batch_size=spec.batch_size, + shuffle=spec.shuffle, + num_workers=spec.num_workers, + drop_last=True, + ) + + +def build_uidl_post_train_dataloader( + spec: SharedLMDataSpec, + tokenizer, +) -> torch.utils.data.DataLoader: + dataset = TokenChunkDataset(build_chunks(spec, tokenizer)) + return torch.utils.data.DataLoader( + dataset, + batch_size=spec.batch_size, + shuffle=spec.shuffle, + num_workers=spec.num_workers, + ) + + +def build_uidl_similarity_dataloader( + spec: SharedLMDataSpec, + tokenizer, +) -> torch.utils.data.DataLoader: + dataset = TokenInputMaskDataset(build_chunks(spec, tokenizer)) + return torch.utils.data.DataLoader( + dataset, + batch_size=spec.batch_size, + shuffle=spec.shuffle, + num_workers=spec.num_workers, + ) + + +def build_shortened_llm_dataloader( + spec: SharedLMDataSpec, + tokenizer, +) -> torch.utils.data.DataLoader: + dataset = TokenOnlyDataset(build_chunks(spec, tokenizer)) + return torch.utils.data.DataLoader( + dataset, + batch_size=spec.batch_size, + shuffle=spec.shuffle, + num_workers=spec.num_workers, + ) + + +def build_shortened_llm_examples(spec: SharedLMDataSpec, tokenizer) -> torch.Tensor: + chunks = build_chunks(spec, tokenizer) + if not chunks: + return torch.empty((0, spec.seq_len), dtype=torch.long) + return torch.stack(chunks, dim=0) + + +def build_llmpruner_examples(spec: SharedLMDataSpec, tokenizer) -> torch.Tensor: + chunks = build_chunks(spec, tokenizer) + if not chunks: + return torch.empty((0, spec.seq_len), dtype=torch.long) + return torch.stack(chunks, dim=0) + + +def build_replaceme_dataloader( + spec: SharedLMDataSpec, + tokenizer, +) -> torch.utils.data.DataLoader: + return build_text_dataloader(spec, tokenizer) + + +def build_hf_causal_dataset(spec: SharedLMDataSpec, tokenizer): + if HFDataset is None: + raise SystemExit("datasets is required for shared LM dataloaders") + + if is_instruction_dataset(spec.dataset): + if load_dataset is None: + raise SystemExit("datasets is required for shared LM dataloaders") + + hf_name, hf_config, hf_split = resolve_dataset_spec(spec.dataset, spec.config, spec.split) + dataset = load_resolved_dataset(spec.dataset, hf_name, hf_config, hf_split) + rows = dataset if spec.num_samples <= 0 else _sample_dataset_rows(dataset, spec.num_samples, spec.seed) + + records: List[Dict[str, object]] = [] + tokenized_records: List[List[int]] = [] + max_observed_len = 0 + for row in rows: + if not isinstance(row, dict): + continue + record = _build_slimorca_record(row) + if record is None: + continue + ids = _tokenize_instruction_record(record, tokenizer) + if not ids: + continue + ids = ids[: spec.seq_len] + if not ids: + continue + records.append(record) + tokenized_records.append(ids) + max_observed_len = max(max_observed_len, len(ids)) + + if not tokenized_records: + raise SystemExit( + f"No instruction samples built for dataset={spec.dataset} split={spec.split}." + ) + + if configured_indices() is not None: + effective_max_len = spec.seq_len + else: + effective_max_len = min(spec.seq_len, max_observed_len) + effective_max_len = min(spec.seq_len, _round_up_to_multiple(effective_max_len, 8)) + print( + "[common_lm_data] " + f"dataset={spec.dataset} split={spec.split} samples={len(tokenized_records)} " + f"max_observed_len={max_observed_len} effective_max_len={effective_max_len}" + ) + pad_id = tokenizer.pad_token_id + if pad_id is None: + pad_id = tokenizer.eos_token_id or 0 + + input_ids_payload: List[List[int]] = [] + attention_masks_payload: List[List[int]] = [] + labels_payload: List[List[int]] = [] + for ids in tokenized_records: + ids = ids[:effective_max_len] + attention_mask = [1] * len(ids) + if len(ids) < effective_max_len: + pad_len = effective_max_len - len(ids) + ids = ids + [pad_id] * pad_len + attention_mask = attention_mask + [0] * pad_len + input_ids_payload.append(ids) + attention_masks_payload.append(attention_mask) + labels_payload.append(list(ids)) + + return HFDataset.from_dict( + { + "input_ids": input_ids_payload, + "attention_mask": attention_masks_payload, + "labels": labels_payload, + } + ) + + chunks = build_chunks(spec, tokenizer) + payload = { + "input_ids": [chunk.tolist() for chunk in chunks], + "attention_mask": [torch.ones_like(chunk).tolist() for chunk in chunks], + "labels": [chunk.tolist() for chunk in chunks], + } + return HFDataset.from_dict(payload) diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/dm500_rows.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/dm500_rows.py new file mode 100644 index 0000000000000000000000000000000000000000..97705f7f88004eec5fa66a40c7dfd8863c2abdf4 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/dm500_rows.py @@ -0,0 +1,130 @@ +"""Shared exact-row selection for data-matched experiments.""" + +from __future__ import annotations + +import json +import os +import hashlib +from pathlib import Path +from typing import Any, Optional + + +def _manifest_canonical_row(value: Any) -> Any: + """Canonicalize rows for manifest hash validation. + + Hugging Face datasets may materialize nullable struct fields that were absent + when a manifest was generated. SlimOrca's `conversations[*].weight: null` is + such a schema-only field and is not consumed by the ABPrune/ReplaceMe text + builders. Dropping only null `weight` fields preserves text identity while + keeping manifest validation strict for actual content changes. + """ + + if isinstance(value, dict): + return { + key: _manifest_canonical_row(child) + for key, child in value.items() + if not (key == "weight" and child is None) + } + if isinstance(value, list): + return [_manifest_canonical_row(child) for child in value] + return value + + +def _json_hash(row: Any) -> str: + payload = json.dumps( + row, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _canonical_hash(row: Any) -> str: + return _json_hash(_manifest_canonical_row(row)) + + +def _load_manifest(path: str) -> tuple[list[int], dict[int, str]]: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + values = payload.get("indices") if isinstance(payload, dict) else payload + if not isinstance(values, list) or not values: + raise ValueError(f"indices manifest is empty or invalid: {path}") + indices = [int(value) for value in values] + if len(indices) != len(set(indices)) or min(indices) < 0: + raise ValueError(f"indices manifest has duplicates or negative values: {path}") + expected_hashes: dict[int, str] = {} + if isinstance(payload, dict): + records = payload.get("records") + if isinstance(records, list): + for record in records: + if not isinstance(record, dict): + continue + raw_index = record.get("raw_index") + content_sha256 = record.get("content_sha256") + if raw_index is None or not content_sha256: + continue + expected_hashes[int(raw_index)] = str(content_sha256) + return indices, expected_hashes + + +def _configured_path(manifest_env_var: str) -> str: + path = os.getenv(manifest_env_var, "").strip() + if not path and manifest_env_var != "ABPRUNE_DATA_INDICES_FILE": + path = os.getenv("ABPRUNE_DATA_INDICES_FILE", "").strip() + return path + + +def configured_indices( + manifest_env_var: str = "ABPRUNE_DATA_INDICES_FILE", +) -> Optional[list[int]]: + path = _configured_path(manifest_env_var) + if not path: + return None + return _load_manifest(path)[0] + + +def select_rows_from_manifest(dataset, manifest_path: str, target: int = 0): + indices, expected_hashes = _load_manifest(manifest_path) + if target > 0: + if target > len(indices): + raise ValueError( + f"requested {target} rows but manifest contains {len(indices)}" + ) + indices = indices[:target] + if hasattr(dataset, "__len__") and indices and max(indices) >= len(dataset): + raise ValueError( + f"manifest index {max(indices)} exceeds dataset length {len(dataset)}" + ) + if not hasattr(dataset, "select"): + raise TypeError("exact DM500 row selection requires a dataset.select implementation") + selected = dataset.select(indices) + if expected_hashes: + for raw_index, row in zip(indices, selected): + expected = expected_hashes.get(raw_index) + if expected is None: + continue + raw_actual = _json_hash(row) + canonical_actual = _canonical_hash(row) + if expected not in {raw_actual, canonical_actual}: + raise ValueError( + "manifest content hash mismatch for raw_index " + f"{raw_index}: expected {expected}, got raw={raw_actual}, " + f"canonical={canonical_actual}" + ) + return selected + + +def select_exact_rows( + dataset, + target: int = 0, + manifest_env_var: str = "ABPRUNE_DATA_INDICES_FILE", +): + path = _configured_path(manifest_env_var) + if not path: + return None + return select_rows_from_manifest(dataset, path, target=target) + + +def dataset_revision_kwargs() -> dict[str, str]: + revision = os.getenv("ABPRUNE_DATASET_REVISION", "").strip() + return {"revision": revision} if revision else {} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/eval_llama31_8b_12bench.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/eval_llama31_8b_12bench.py new file mode 100644 index 0000000000000000000000000000000000000000..8bbc8713a4d0f04209bec7ee454a24744f9d3bb8 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/eval_llama31_8b_12bench.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Evaluate the lm-eval portion of the requested Llama benchmark suite. + +RACE-H/M, C3, CMNLI, and CHID are evaluated with their canonical OpenCompass +PPL task definitions by the companion launcher. This script handles the +remaining eight tasks plus full zero-shot MMLU and CMMLU. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + + +TASKS = ( + "piqa", + "winogrande", + "hellaswag", + "arc_challenge", + "arc_easy", + "openbookqa", + "sciq", + "mmlu", + "cmmlu", +) + +# Use normalized accuracy where the standard harness exposes it. The +# extraction code falls back to plain accuracy for tasks/versions without it. +PREFERRED_METRICS = { + "piqa": "acc_norm,none", + "hellaswag": "acc_norm,none", + "arc_challenge": "acc_norm,none", + "arc_easy": "acc_norm,none", + "openbookqa": "acc_norm,none", + "sciq": "acc_norm,none", +} + + +def _score(task: str, result: dict[str, Any]) -> tuple[str, float]: + preferred = PREFERRED_METRICS.get(task, "acc,none") + candidates = (preferred, "acc_norm,none", "acc,none") + for metric in candidates: + value = result.get(metric) + if value is not None: + return metric, float(value) + available = sorted(key for key in result if "," in key and not key.endswith("_stderr,none")) + raise KeyError(f"No supported accuracy metric for {task}; available={available}") + + +def _load_model(args: argparse.Namespace): + if args.compact_stage is None: + model = AutoModelForCausalLM.from_pretrained( + args.model, + revision=args.revision, + torch_dtype=torch.bfloat16, + attn_implementation="eager", + local_files_only=args.local_files_only, + ).cuda().eval() + tokenizer = AutoTokenizer.from_pretrained( + args.model, + revision=args.revision, + local_files_only=args.local_files_only, + ) + return model, tokenizer + + if args.cycle is None or args.anchor is None: + raise ValueError("--compact-stage requires --cycle and --anchor") + from fusegpt.abprune_stagewise_geometry import load_compact_model + + model = load_compact_model( + args.model, + args.compact_stage, + cycle=args.cycle, + anchor=args.anchor, + dtype=torch.bfloat16, + device=torch.device("cuda"), + attn="eager", + revision=args.revision, + ) + tokenizer = AutoTokenizer.from_pretrained( + args.model, revision=args.revision, local_files_only=args.local_files_only) + return model, tokenizer + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", default=None) + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--compact-stage", type=Path, default=None) + parser.add_argument("--cycle", type=int, default=None) + parser.add_argument("--anchor", type=int, default=None) + parser.add_argument("--batch-size", default="auto:4") + parser.add_argument("--limit", type=float, default=None) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--dataset-cache", type=Path, default=None) + parser.add_argument("--tasks", nargs="+", choices=TASKS, default=list(TASKS), + help="Evaluate a subset without repeating already scored tasks") + args = parser.parse_args() + if len(set(args.tasks)) != len(args.tasks): + parser.error("--tasks must not contain duplicates") + + # CMMLU is still distributed through a Hugging Face dataset loading + # script. Opt in explicitly so non-interactive experiment jobs do not + # stop at the datasets trust prompt. + os.environ.setdefault("HF_DATASETS_TRUST_REMOTE_CODE", "1") + + from lm_eval import evaluator + from lm_eval.models.huggingface import HFLM + + if args.dataset_cache is not None: + from prepare_lmeval9_local_cache import install_local_loader + install_local_loader(args.dataset_cache) + + model, tokenizer = _load_model(args) + model.config.use_cache = False + if getattr(model, "generation_config", None) is not None: + model.generation_config.use_cache = False + for layer_idx, layer in enumerate(model.model.layers): + if hasattr(layer, "self_attn") and hasattr(layer.self_attn, "layer_idx"): + layer.self_attn.layer_idx = layer_idx + + lm = HFLM( + pretrained=model, + tokenizer=tokenizer, + backend="causal", + batch_size=args.batch_size, + ) + raw = evaluator.simple_evaluate( + model=lm, + tasks=args.tasks, + num_fewshot=0, + batch_size=args.batch_size, + random_seed=0, + numpy_random_seed=1234, + torch_random_seed=1234, + fewshot_random_seed=1234, + bootstrap_iters=0, + limit=args.limit, + ) + + scores: dict[str, float] = {} + metrics: dict[str, str] = {} + for task in args.tasks: + source = raw.get("groups", {}) if task in {"mmlu", "cmmlu"} else raw["results"] + if task not in source: + raise KeyError( + f"Requested task {task!r} missing from lm-eval results: " + f"results={sorted(raw['results'])}, groups={sorted(raw.get('groups', {}))}" + ) + metric, value = _score(task, source[task]) + metrics[task] = metric + scores[task] = value + + report = { + "model": args.model, + "revision": args.revision, + "compact_stage": str(args.compact_stage) if args.compact_stage else None, + "cycle": args.cycle, + "anchor": args.anchor, + "tasks": args.tasks, + "num_fewshot": 0, + "batch_size": args.batch_size, + "limit": args.limit, + "dataset_cache": str(args.dataset_cache) if args.dataset_cache else None, + "preferred_metrics": metrics, + "task_scores": scores, + "macro_mean_all_rows": sum(scores.values()) / len(scores), + "lm_eval_results": raw["results"], + "lm_eval_groups": raw.get("groups"), + "lm_eval_versions": raw.get("versions"), + "lm_eval_config": raw.get("config"), + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(report, indent=2, ensure_ascii=False, default=str) + "\n", + encoding="utf-8", + ) + print( + json.dumps( + {"task_scores": scores, "macro_mean_all_rows": report["macro_mean_all_rows"]}, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/eval_llama31_fit_ablation_one.sh b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/eval_llama31_fit_ablation_one.sh new file mode 100644 index 0000000000000000000000000000000000000000..b64dcaa488d68367fedff793ba3a68caa6919491 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/eval_llama31_fit_ablation_one.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail +project="$1"; root="$2"; model="$3"; data="$4"; python="$5" +anchor="$6"; cycles="$7"; gpu="$8" +stage="$root/compact/stage_$(printf '%02d' "$cycles")" +export HF_HOME=/workspace/.hf_home HF_TOKEN_PATH=/root/.cache/huggingface/token +export HF_DATASETS_CACHE=/workspace/.hf_datasets36 HF_DATASETS_TRUST_REMOTE_CODE=1 +export TOKENIZERS_PARALLELISM=false PYTHONUNBUFFERED=1 PYTHONPATH="$project/src" +revision="${MODEL_REVISION:-49e3418fbbbca6ecbdf9608b4d22e5a407081db4}" +if [[ -n "${LMEVAL_LOCAL_CACHE:-}" ]]; then + export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 + [[ -d "$model" ]] || { echo 'Offline bench requires a local model directory'; exit 1; } + "$python" "$project/scripts/prepare_lmeval9_local_cache.py" "$LMEVAL_LOCAL_CACHE" --validate-local + for file in race/test/high.jsonl race/test/middle.jsonl CLUE/C3/dev_0.json CLUE/cmnli/cmnli_public/dev.json FewCLUE/chid/dev_few_all.json; do + [[ -s "$data/$file" ]] || { echo "Offline OpenCompass input missing: $file"; exit 1; } + done + echo 'STRICT_OFFLINE_BENCH_PREFLIGHT_OK' +fi +opencompass_gpu="$gpu"; lmeval_gpu="${9:-$gpu}" +if [[ -f "$root/eval/gpu_assignment.json" ]]; then + read -r opencompass_gpu lmeval_gpu < <("$python" - "$root/eval/gpu_assignment.json" <<'PY' +import json, sys +assignment = json.load(open(sys.argv[1])) +values = [assignment[key] for key in ("opencompass_gpu", "lmeval_gpu")] +assert all(type(value) is int and value >= 0 for value in values) +print(*values) +PY + ) +fi +run_opencompass() { + [[ -f "$root/eval/final_opencompass5.json" ]] && return 0 + CUDA_VISIBLE_DEVICES="$opencompass_gpu" "$python" "$project/scripts/eval_opencompass_ppl_fast.py" \ + --model "$model" --revision "$revision" --tokenizer "$model" \ + --compact-stage "$stage" --cycle "$cycles" --anchor "$anchor" \ + --data-root "$data" --max-batch 32 --token-budget 8192 \ + --output "$root/eval/final_opencompass5.json" +} +run_lmeval() { + [[ -f "$root/eval/final_lmeval9.json" ]] && return 0 + local cache_args=() + if [[ -n "${LMEVAL_LOCAL_CACHE:-}" ]]; then + [[ -f "$LMEVAL_LOCAL_CACHE/manifest.json" ]] || return 1 + cache_args=(--dataset-cache "$LMEVAL_LOCAL_CACHE" --local-files-only) + export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 + fi + CUDA_VISIBLE_DEVICES="$lmeval_gpu" "$python" "$project/scripts/eval_llama31_8b_12bench.py" \ + --model "$model" --revision "$revision" --compact-stage "$stage" \ + --cycle "$cycles" --anchor "$anchor" --batch-size auto:4 \ + --output "$root/eval/final_lmeval9.json" "${cache_args[@]}" +} +if [[ "$opencompass_gpu" != "$lmeval_gpu" ]]; then + echo "Parallel evaluation: OpenCompass GPU=$opencompass_gpu; lm-eval GPU=$lmeval_gpu" + run_opencompass >> "$root/logs/eval_opencompass.log" 2>&1 & oc_pid=$! + run_lmeval >> "$root/logs/eval_lmeval.log" 2>&1 & lm_pid=$! + status=0 + wait "$oc_pid" || status=1 + wait "$lm_pid" || status=1 + if ((status)); then exit 1; fi +else + run_opencompass >> "$root/logs/eval14.log" 2>&1 + run_lmeval >> "$root/logs/eval14.log" 2>&1 +fi +"$python" "$project/scripts/aggregate_transport_rho_fit_14bench.py" "$root" diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/eval_opencompass_ppl_fast.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/eval_opencompass_ppl_fast.py new file mode 100644 index 0000000000000000000000000000000000000000..e32bb5fdc36375a58debd2042f8a60c706ab3ef6 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/eval_opencompass_ppl_fast.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Fast, equivalent evaluator for selected OpenCompass zero-shot PPL tasks. + +The prompts, labels, split choices, full-prompt mean-CE scoring, and lowest-PPL +decision rule mirror OpenCompass 0.5.4's canonical configs for RACE, C3, +CMNLI, and CHID. Unlike the stock runner, this evaluator disables KV cache +and length-batches all choices, which matters substantially on 40 GB GPUs. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Iterable + +import torch +import torch.nn.functional as F +from transformers import AutoModelForCausalLM, AutoTokenizer + + +def read_jsonl(path: Path) -> Iterable[dict]: + with path.open("r", encoding="utf-8") as stream: + for line in stream: + yield json.loads(line) + + +def race(path: Path) -> tuple[list[list[str]], list[int]]: + prompts, golds = [], [] + for row in read_jsonl(path): + human = ( + "Read the article, and answer the question by replying A, B, C or D.\n\n" + f"Article:\n{row['article']}\n\nQ: {row['question']}\n\n" + f"A. {row['options'][0]}\nB. {row['options'][1]}\n" + f"C. {row['options'][2]}\nD. {row['options'][3]}" + ) + prompts.append([human + "\n" + f"A: {answer}" for answer in "ABCD"]) + golds.append("ABCD".index(row["answer"])) + return prompts, golds + + +def c3(path: Path) -> tuple[list[list[str]], list[int]]: + prompts, golds = [], [] + rows = json.loads(path.read_text(encoding="utf-8")) + for row in rows: + content = " ".join("".join(paragraph) for paragraph in row[0]) + for question in row[1]: + choices = list(question["choice"]) + golds.append(choices.index(question["answer"])) + while len(choices) < 4: + choices.append(choices[0]) + human = f"文章：{content}\n问题：{question['question']}" + prompts.append([human + "\n" + f"答案：{choice}" for choice in choices[:4]]) + return prompts, golds + + +def cmnli(path: Path) -> tuple[list[list[str]], list[int]]: + labels = ("contradiction", "entailment", "neutral") + verbalizers = ("矛盾", "蕴含", "无关") + prompts, golds = [], [] + for row in read_jsonl(path): + if row["label"] == "-": + continue + human = ( + f"语句一：“{row['sentence1']}”\n语句二：“{row['sentence2']}”\n" + "请问这两句话是什么关系？" + ) + prompts.append([human + "\n" + word for word in verbalizers]) + golds.append(labels.index(row["label"])) + return prompts, golds + + +def chid(path: Path) -> tuple[list[list[str]], list[int]]: + prompts, golds = [], [] + for row in read_jsonl(path): + candidates = [row["content"].replace("#idiom#", candidate) for candidate in row["candidates"]] + prompts.append([ + f"以下句子是否通顺？\n{content}\n这个句子是通顺的。" for content in candidates + ]) + golds.append(int(row["answer"])) + return prompts, golds + + +def score_task(model, tokenizer, choices: list[list[str]], golds: list[int], max_batch: int, + token_budget: int) -> dict: + flat = [(sample_idx, choice_idx, prompt) + for sample_idx, row in enumerate(choices) + for choice_idx, prompt in enumerate(row)] + encoded = tokenizer( + [item[2] for item in flat], + add_special_tokens=True, + padding=False, + truncation=True, + max_length=model.config.max_position_embeddings, + )["input_ids"] + order = sorted(range(len(flat)), key=lambda idx: len(encoded[idx]), reverse=True) + losses = [0.0] * len(flat) + cursor = 0 + while cursor < len(order): + longest = len(encoded[order[cursor]]) + batch_size = min(max_batch, max(1, token_budget // longest), len(order) - cursor) + while True: + batch_indices = order[cursor:cursor + batch_size] + max_len = max(len(encoded[idx]) for idx in batch_indices) + input_ids = torch.full( + (batch_size, max_len), tokenizer.pad_token_id, dtype=torch.long, device="cuda" + ) + attention = torch.zeros((batch_size, max_len), dtype=torch.long, device="cuda") + for row_idx, flat_idx in enumerate(batch_indices): + values = torch.tensor(encoded[flat_idx], dtype=torch.long, device="cuda") + input_ids[row_idx, : len(values)] = values + attention[row_idx, : len(values)] = 1 + try: + with torch.inference_mode(): + logits = model(input_ids=input_ids, attention_mask=attention, use_cache=False).logits + token_loss = F.cross_entropy( + logits[:, :-1].float().reshape(-1, logits.shape[-1]), + input_ids[:, 1:].reshape(-1), + reduction="none", + ).view(batch_size, -1) + # OpenCompass divides by the number of non-padding input + # tokens (including BOS), while scoring shifted labels. + sequence_loss = ( + token_loss * attention[:, 1:] + ).sum(dim=1) / attention.sum(dim=1) + break + except torch.OutOfMemoryError: + del input_ids, attention + torch.cuda.empty_cache() + if batch_size == 1: + raise + batch_size = max(1, batch_size // 2) + for flat_idx, value in zip(batch_indices, sequence_loss.tolist()): + losses[flat_idx] = value + cursor += batch_size + if cursor % 1000 < batch_size or cursor == len(order): + print(f"choices={cursor}/{len(order)}", flush=True) + + score_rows = [[0.0] * len(row) for row in choices] + for flat_idx, (sample_idx, choice_idx, _) in enumerate(flat): + score_rows[sample_idx][choice_idx] = losses[flat_idx] + predictions = [min(range(len(row)), key=row.__getitem__) for row in score_rows] + correct = sum(prediction == gold for prediction, gold in zip(predictions, golds)) + return { + "accuracy": correct / len(golds), + "correct": correct, + "samples": len(golds), + "choices": len(flat), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", default=None) + parser.add_argument("--tokenizer", required=True) + parser.add_argument("--compact-stage", type=Path, default=None) + parser.add_argument("--cycle", type=int, default=None) + parser.add_argument("--anchor", type=int, default=None) + parser.add_argument("--data-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--max-batch", type=int, default=32) + parser.add_argument("--token-budget", type=int, default=8192) + parser.add_argument("--limit", type=int, default=None) + args = parser.parse_args() + + tokenizer = AutoTokenizer.from_pretrained(args.tokenizer, local_files_only=True) + if tokenizer.pad_token_id is None: + tokenizer.pad_token_id = tokenizer.eos_token_id + tokenizer.padding_side = "right" + if args.compact_stage is None: + model = AutoModelForCausalLM.from_pretrained( + args.model, + revision=args.revision, + torch_dtype=torch.bfloat16, + attn_implementation="sdpa", + local_files_only=True, + ).cuda().eval() + else: + if args.cycle is None or args.anchor is None: + raise ValueError("--compact-stage requires --cycle and --anchor") + from fusegpt.abprune_stagewise_geometry import load_compact_model + + model = load_compact_model( + args.model, + args.compact_stage, + cycle=args.cycle, + anchor=args.anchor, + dtype=torch.bfloat16, + device=torch.device("cuda"), + attn="sdpa", + revision=args.revision, + ) + model.config.use_cache = False + + datasets = { + "race-middle": race(args.data_root / "race/test/middle.jsonl"), + "race-high": race(args.data_root / "race/test/high.jsonl"), + "C3": c3(args.data_root / "CLUE/C3/dev_0.json"), + "cmnli": cmnli(args.data_root / "CLUE/cmnli/cmnli_public/dev.json"), + "chid-dev": chid(args.data_root / "FewCLUE/chid/dev_few_all.json"), + } + results = {} + for name, (choices, golds) in datasets.items(): + if args.limit is not None: + choices, golds = choices[: args.limit], golds[: args.limit] + print(f"task={name} samples={len(golds)}", flush=True) + results[name] = score_task( + model, tokenizer, choices, golds, args.max_batch, args.token_budget + ) + print(json.dumps({name: results[name]}, ensure_ascii=False), flush=True) + + report = { + "model": args.model, + "revision": args.revision, + "compact_stage": str(args.compact_stage) if args.compact_stage else None, + "cycle": args.cycle, + "anchor": args.anchor, + "tokenizer": args.tokenizer, + "protocol": "OpenCompass 0.5.4 canonical PPL prompts; full-prompt mean CE; lowest PPL", + "max_batch": args.max_batch, + "token_budget": args.token_budget, + "limit": args.limit, + "results": results, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/evaluate_abprune_compact_ppl.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/evaluate_abprune_compact_ppl.py new file mode 100644 index 0000000000000000000000000000000000000000..40ba37eaa42227fb5065410015f6407e126237e1 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/evaluate_abprune_compact_ppl.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Evaluate an unscaled or residual-scaled compact ABPrune checkpoint.""" + +from __future__ import annotations + +import argparse +import json +from contextlib import ExitStack +from pathlib import Path +import sys + +from offline_eval_policy import apply_run_offline_policy + +OFFLINE_POLICY = apply_run_offline_policy(sys.argv) + +import torch + +from evaluate_hchc_wikitext2_ppl import evaluate_perplexity, packed_wikitext_tokens +from fusegpt.abprune_residual_scale import residual_scale_hook +from fusegpt.abprune_compact import compact_scale_targets, read_compact_metadata +from fusegpt.abprune_stagewise_geometry import load_compact_model +from fusegpt.hadamard_channel_fusion import linear_patch_hook + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--compact-stage", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--anchor", type=int, default=None) + parser.add_argument("--cycles", type=int, required=True) + parser.add_argument("--original-layers", type=int, required=True) + parser.add_argument("--seq-len", type=int, default=2048) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--device", default="cuda") + parser.add_argument("--apply-post-merge-scale", action="store_true") + parser.add_argument("--linear-patch", type=Path, default=None) + args = parser.parse_args() + + # Long training supervisors may be asked to stop after model construction. + # A run-local marker makes that request durable without killing training. + run_root = args.output.parent.parent + if (run_root / "DEFER_EVAL").exists(): + print(f"[eval] deferred by {run_root / 'DEFER_EVAL'}", flush=True) + raise SystemExit(75) + + device = torch.device(args.device) + model = load_compact_model( + args.model, + args.compact_stage, + cycle=args.cycles, + anchor=args.anchor, + dtype=torch.bfloat16, + device=device, + attn="eager", + revision=args.revision, + ) + model.eval() + expected_depth = args.original_layers - args.cycles + if len(model.model.layers) != expected_depth: + raise RuntimeError( + f"Unexpected decoder depth: {len(model.model.layers)} != {expected_depth}" + ) + + from transformers import AutoTokenizer + + if OFFLINE_POLICY and not Path(args.model).is_dir(): + raise ValueError('Strict offline PPL requires a local model directory') + tokenizer = AutoTokenizer.from_pretrained( + args.model, revision=args.revision, local_files_only=bool(OFFLINE_POLICY)) + applied_scales = [] + scale_stack = ExitStack() + linear_patch = None + if args.apply_post_merge_scale: + metadata = read_compact_metadata(args.compact_stage) + targets = compact_scale_targets(metadata) + if not targets: + raise RuntimeError( + "--apply-post-merge-scale requires enabled residual scale metadata" + ) + for target in targets: + stored_scale = target["scale"]["cumulative_gamma"] + applied = ( + torch.tensor(stored_scale, device=device, dtype=torch.float32) + if isinstance(stored_scale, list) + else float(stored_scale) + ) + scale_stack.enter_context( + residual_scale_hook(model, int(target["student_index"]), applied) + ) + applied_scales.append( + { + "original_layer": int(target["original_layer"]), + "student_index": int(target["student_index"]), + "value": applied, + } + ) + if args.linear_patch is not None: + linear_patch = json.loads(args.linear_patch.read_text(encoding="utf-8")) + patch_scale = torch.tensor( + linear_patch["scale"], device=device, dtype=torch.float32 + ) + patch_anchor = int(linear_patch["anchor_student_index"]) + scale_stack.enter_context( + linear_patch_hook( + model.model.layers[patch_anchor], + patch_scale, + hadamard=linear_patch["mode"] == "hadamard", + ) + ) + + with scale_stack: + tokens = packed_wikitext_tokens( + tokenizer, "test", dataset_cache=( + Path(OFFLINE_POLICY['wikitext_cache']) if OFFLINE_POLICY else None)) + metrics = evaluate_perplexity( + model=model, + tokens=tokens, + sequence_length=args.seq_len, + batch_size=args.batch_size, + ) + + report = { + "model": args.model, + "revision": args.revision, + "compact_stage": str(args.compact_stage), + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "offline_policy": OFFLINE_POLICY, + "protocol": ( + "HCHC experiment.py: filter empty rows, join with double newline, " + "then tokenize the complete stream once" + ), + "sequence_length": args.seq_len, + "batch_size": args.batch_size, + "dtype": "bfloat16", + "metrics": metrics, + "anchor": args.anchor, + "cycles": args.cycles, + "original_layers": args.original_layers, + "decoder_layers": len(model.model.layers), + "post_merge_residual_scale": ( + applied_scales[0]["value"].tolist() + if len(applied_scales) == 1 + and torch.is_tensor(applied_scales[0]["value"]) + else ( + applied_scales[0]["value"] + if len(applied_scales) == 1 + else None + ) + ), + "post_merge_residual_scales": [ + { + "original_layer": item["original_layer"], + "student_index": item["student_index"], + "parameterization": ( + "vector" if torch.is_tensor(item["value"]) else "scalar" + ), + "value": ( + item["value"].tolist() + if torch.is_tensor(item["value"]) + else item["value"] + ), + } + for item in applied_scales + ], + "parameterization": ( + "X_interface <- X_interface H D H.T" + if linear_patch and linear_patch["mode"] == "hadamard" + else ( + "X_interface <- X_interface D" + if linear_patch + else ( + "h_out = h_in + gamma * (block(h_in) - h_in)" + if applied_scales else "unscaled compact ABPrune" + ) + ) + ), + "linear_patch": linear_patch, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/evaluate_hchc_wikitext2_ppl.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/evaluate_hchc_wikitext2_ppl.py new file mode 100644 index 0000000000000000000000000000000000000000..28ea302a1e40246a770b97e6ee6fd9198cf57c28 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/evaluate_hchc_wikitext2_ppl.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Evaluate a saved causal LM with the HCHC WikiText-2 PPL protocol.""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + +import torch +from datasets import load_dataset, load_from_disk + + +def primary(output: Any) -> torch.Tensor: + if torch.is_tensor(output): + return output + if isinstance(output, (tuple, list)) and output and torch.is_tensor(output[0]): + return output[0] + raise TypeError(f"Unsupported decoder-layer output type: {type(output)!r}") + + +def replace_primary(output: Any, hidden: torch.Tensor) -> Any: + if torch.is_tensor(output): + return hidden + if isinstance(output, tuple): + return (hidden, *output[1:]) + if isinstance(output, list): + return [hidden, *output[1:]] + raise TypeError(f"Unsupported decoder-layer output type: {type(output)!r}") + + +def register_residual_scale_hook(layer, gamma: float): + """Install h_out = h_in + gamma * (block(h_in) - h_in).""" + scale = torch.tensor(gamma, device=next(layer.parameters()).device) + + def hook(_module, inputs, output): + if not inputs or not torch.is_tensor(inputs[0]): + raise RuntimeError("Could not capture decoder-layer residual input") + layer_input = inputs[0] + corrected = layer_input + scale * (primary(output) - layer_input) + return replace_primary(output, corrected) + + return layer.register_forward_hook(hook) + + +def packed_wikitext_tokens(tokenizer, split: str, *, dataset_cache: Path | None = None) -> torch.Tensor: + """Reproduce the canonical HCHC WikiText-2 packing protocol.""" + if dataset_cache is not None: + dataset = load_from_disk(str(dataset_cache))[split] + else: + dataset = load_dataset( + "Salesforce/wikitext", "wikitext-2-raw-v1", split=split + ) + text = "\n\n".join(row for row in dataset["text"] if row.strip()) + return tokenizer( + text, add_special_tokens=False, return_tensors="pt" + ).input_ids[0] + + +@torch.no_grad() +def evaluate_perplexity( + model, + tokens: torch.Tensor, + sequence_length: int, + batch_size: int, +) -> dict[str, float | int]: + num_sequences = tokens.numel() // sequence_length + ids = tokens[: num_sequences * sequence_length].view(num_sequences, sequence_length) + total_nll = 0.0 + total_predicted_tokens = 0 + model.eval() + for start in range(0, num_sequences, batch_size): + batch = ids[start : start + batch_size].to("cuda") + output = model(input_ids=batch, labels=batch, use_cache=False) + predicted = batch.shape[0] * (batch.shape[1] - 1) + total_nll += output.loss.float().item() * predicted + total_predicted_tokens += predicted + mean_nll = total_nll / total_predicted_tokens + return { + "nll": mean_nll, + "perplexity": math.exp(mean_nll), + "tokens": total_predicted_tokens, + "sequences": num_sequences, + "source_tokens": int(tokens.numel()), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--sequence-length", type=int, default=2048) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--residual-anchor", type=int) + parser.add_argument("--residual-gamma", type=float) + args = parser.parse_args() + + if (args.residual_anchor is None) != (args.residual_gamma is None): + parser.error("--residual-anchor and --residual-gamma must be supplied together") + + from transformers import AutoModelForCausalLM, AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True) + model = AutoModelForCausalLM.from_pretrained( + args.model, + torch_dtype=torch.bfloat16, + local_files_only=True, + low_cpu_mem_usage=True, + attn_implementation="sdpa", + ).to("cuda") + model.config.use_cache = False + + handle = None + if args.residual_gamma is not None: + if args.residual_anchor >= len(model.model.layers): + raise ValueError( + f"Residual anchor {args.residual_anchor} is outside decoder depth " + f"{len(model.model.layers)}" + ) + handle = register_residual_scale_hook( + model.model.layers[args.residual_anchor], args.residual_gamma + ) + + tokens = packed_wikitext_tokens(tokenizer, "test") + try: + metrics = evaluate_perplexity( + model, tokens, args.sequence_length, args.batch_size + ) + finally: + if handle is not None: + handle.remove() + + report = { + "model": str(args.model), + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: non-empty rows joined with double newline before one-shot tokenization", + "sequence_length": args.sequence_length, + "batch_size": args.batch_size, + "dtype": "bfloat16", + "residual_anchor": args.residual_anchor, + "residual_gamma": args.residual_gamma, + "metrics": metrics, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fit_ablation_mka.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fit_ablation_mka.py new file mode 100644 index 0000000000000000000000000000000000000000..5d0ff3b1ab1c31a493f7d54d607aef04cafd8046 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fit_ablation_mka.py @@ -0,0 +1,482 @@ +"""Corrected MKA variant: adjacent NPIB selection, upstream lambda, real fusion. + +Not an exact reproduction of the paper: use sequence-level pooled activations, +recompute geometry after every merge, and resolve NPIB ties back-to-front. +The seven projection weights are fused; residual and Q/K norms stay at anchor. +""" + +from __future__ import annotations + +import hashlib +import math +import random +from collections.abc import Callable, Iterable, Sequence +from dataclasses import asdict, dataclass + +import numpy as np +import torch +from scipy.linalg import eigh +from scipy.spatial.distance import pdist, squareform +from sklearn.feature_selection import mutual_info_regression +from sklearn.neighbors import NearestNeighbors +from torch import Tensor, nn + +PROJECTION_WEIGHTS = ( + "mlp.down_proj.weight", + "mlp.up_proj.weight", + "mlp.gate_proj.weight", + "self_attn.k_proj.weight", + "self_attn.o_proj.weight", + "self_attn.q_proj.weight", + "self_attn.v_proj.weight", +) +UPSTREAM = "https://github.com/SempraETY/Pruning-via-Merging" +UPSTREAM_COMMIT = "3b60b002a0963266e2f4ca78f133b4e091bc607a" + + +@dataclass(frozen=True) +class GeometryConfig: + sigma: float = 8.0 + alpha: float = 0.5 + dimensions: int = 2 + mi_neighbors: int = 5 + entropy_neighbors: int = 50 + beta: float = 1.0 + seed: int = 0 + pooling: str = "mean" + + def validate(self) -> None: + if not math.isfinite(self.sigma) or self.sigma <= 0: + raise ValueError("sigma must be positive and finite") + if not math.isfinite(self.alpha) or not 0 <= self.alpha <= 1: + raise ValueError("alpha must be finite and in [0, 1]") + if self.dimensions < 1 or min(self.mi_neighbors, self.entropy_neighbors) < 1: + raise ValueError("dimensions and neighbor counts must be positive") + if not math.isfinite(self.beta) or self.beta < 0: + raise ValueError("beta must be non-negative and finite") + if self.pooling not in {"mean", "last"}: + raise ValueError("pooling must be 'mean' or 'last'") + + +def sample_slimpajama( + tokenizer, + *, + samples: int = 100, + sequence_length: int = 512, + seed: int = 0, + revision: str = "b5f90f419b7489cdba26fdbc8c022fcb5562f968", + shuffle_buffer: int = 10000, + max_documents: int = 100000, + documents: Iterable[dict] | None = None, +) -> tuple[list[dict[str, Tensor]], dict]: + """Exactly one random fixed-length, no-BOS window from each distinct document. + + Unlike CoMe's row-packing loader, ``samples`` is the number of sequences. + Persist returned IDs and provenance; do not sample again between stages. + """ + if min(samples, sequence_length, shuffle_buffer, max_documents) < 1: + raise ValueError("sampling counts, length and buffer must be positive") + if documents is None: + from datasets import load_dataset + + documents = load_dataset( + "DKYoon/SlimPajama-6B", revision=revision, split="train", streaming=True + ).shuffle(seed=seed, buffer_size=shuffle_buffer) + rng = random.Random(seed) + batches, records, seen = [], [], set() + iterator = iter(documents) + try: + for row_index, row in enumerate(iterator): + if row_index >= max_documents: + break + text = row.get("text") + if not isinstance(text, str) or not text.strip(): + continue + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + if digest in seen: + continue + seen.add(digest) + ids = tokenizer.encode(text, add_special_tokens=False) + if len(ids) < sequence_length: + continue + start = rng.randrange(len(ids) - sequence_length + 1) + tensor = torch.tensor( + ids[start : start + sequence_length], dtype=torch.long + )[None, :] + batches.append( + {"input_ids": tensor, "attention_mask": torch.ones_like(tensor)} + ) + records.append( + { + "stream_row": row_index, + "document_sha256": digest, + "document_tokens": len(ids), + "window_start": start, + } + ) + if len(batches) == samples: + break + finally: + close = getattr(iterator, "close", None) + if close is not None: + close() + if len(batches) != samples: + raise RuntimeError( + f"Only found {len(batches)}/{samples} distinct sufficiently long documents" + ) + checksum = hashlib.sha256() + for batch in batches: + checksum.update(batch["input_ids"].numpy().astype(" list[np.ndarray]: + """One backbone forward per batch; block hooks avoid hidden-state off-by-one. + + Collect outputs *before* the final backbone norm, including the final block. + Pool only valid tokens; activations are CPU float32 after pooling. + """ + if pooling not in {"mean", "last"} or not batches: + raise ValueError("valid pooling and nonempty calibration batches are required") + layers = model.model.layers + features: list[list[Tensor]] = [[] for _ in layers] + current_mask: Tensor | None = None + + def hook(index): + def capture(module, inputs, output): + hidden = output[0] if isinstance(output, (tuple, list)) else output + mask = current_mask.to(hidden.device, dtype=torch.bool) + if hidden.shape[:2] != mask.shape or (mask.sum(-1) == 0).any(): + raise ValueError("block output and valid-token mask do not match") + if pooling == "mean": + value = (hidden.float() * mask[:, :, None]).sum(1) / mask.sum(1)[ + :, None + ] + else: + positions = torch.arange(mask.shape[1], device=mask.device)[None, :] + last = positions.expand_as(mask).masked_fill(~mask, -1).max(1).values + value = hidden[ + torch.arange(hidden.shape[0], device=hidden.device), last + ].float() + if not torch.isfinite(value).all(): + raise FloatingPointError(f"nonfinite activation in block {index}") + features[index].append(value.cpu()) + + return capture + + handles = [layer.register_forward_hook(hook(i)) for i, layer in enumerate(layers)] + was_training = model.training + model.eval() + try: + device = model.get_input_embeddings().weight.device + for batch in batches: + prepared = {key: value.to(device) for key, value in batch.items()} + current_mask = prepared.get( + "attention_mask", torch.ones_like(prepared["input_ids"]) + ) + model.model(**prepared, use_cache=False, return_dict=True) + finally: + for handle in handles: + handle.remove() + model.train(was_training) + expected = sum(batch["input_ids"].shape[0] for batch in batches) + arrays = [torch.cat(parts, dim=0).numpy() for parts in features] + if any(array.shape[0] != expected for array in arrays): + raise RuntimeError("not all calibration examples reached each decoder block") + return arrays + + +def diffusion_embedding( + features: np.ndarray, config: GeometryConfig +) -> tuple[np.ndarray, dict]: + """Correctly shaped sample-by-sample diffusion kernel, following upstream SVD geometry.""" + config.validate() + x = np.asarray(features, dtype=np.float64) + if x.ndim != 2 or not np.isfinite(x).all() or len(x) <= config.dimensions + 1: + raise ValueError( + "features must be finite [samples, features], samples > dimensions + 1" + ) + distance = squareform(pdist(x, metric="euclidean")) + if distance.max() <= np.finfo(float).eps: + raise ValueError( + "constant calibration features: cannot estimate manifold geometry" + ) + kernel = np.exp(-np.sqrt(distance / config.sigma)) + p = kernel.sum(0) + normalized = kernel / np.outer(p, p) ** config.alpha + v = np.sqrt(normalized.sum(0)) + operator = normalized / np.outer(v, v) + condition = float(np.linalg.cond(operator)) + regularization = 1e-6 if not math.isfinite(condition) or condition > 1e12 else 0.0 + if regularization: + operator += regularization * np.eye(len(x)) + values, vectors = eigh( + operator, subset_by_index=(len(x) - config.dimensions - 1, len(x) - 1) + ) + values, vectors = values[::-1], vectors[:, ::-1] + # Fix arbitrary eigenvector signs; the leading vector of this positive kernel is positive. + for dim in range(vectors.shape[1]): + if vectors[np.argmax(np.abs(vectors[:, dim])), dim] < 0: + vectors[:, dim] *= -1 + if np.any(np.abs(vectors[:, 0]) < 1e-12): + raise FloatingPointError( + "diffusion normalization has a near-zero leading eigenvector" + ) + embedding = vectors[:, 1:] / vectors[:, :1] + if not np.isfinite(embedding).all(): + raise FloatingPointError("nonfinite diffusion embedding") + # Upstream uses per-axis rank normalization before its KNN estimators. + embedding = np.argsort( + np.argsort(embedding, axis=0, kind="stable"), axis=0, kind="stable" + ) / len(x) + return embedding, { + "samples": len(x), + "condition_number": condition if math.isfinite(condition) else None, + "regularization": regularization, + "eigenvalues": values.tolist(), + } + + +def entropy_knn(x: np.ndarray, neighbors: int) -> float: + """Upstream differential-entropy estimator, evaluated in log space.""" + n, dimensions = x.shape + k = min(neighbors, n - 1) + distances = NearestNeighbors(n_neighbors=k + 1).fit(x).kneighbors(x)[0][:, -1] + if np.any(distances <= 0) or not np.isfinite(distances).all(): + raise FloatingPointError( + "duplicate/degenerate manifold points in entropy estimator" + ) + return float(math.log(n / k) + dimensions * np.log(distances).mean()) + + +def npib_matrix( + embeddings: Sequence[np.ndarray], config: GeometryConfig +) -> tuple[np.ndarray, list[float]]: + """Match upstream axis-wise MI average and symmetric sqrt(H_i*H_j) normalization. + + Differential entropies can be negative. Preserve upstream's formula when + their product is positive; fail clearly for zero/negative products or NaNs. + This estimator is a heuristic, not a guaranteed [0, 1] similarity. + """ + config.validate() + if not embeddings or len({x.shape for x in embeddings}) != 1: + raise ValueError("embeddings must have identical sample/coordinate shapes") + n = len(embeddings[0]) + if n < 3 or any(not np.isfinite(x).all() for x in embeddings): + raise ValueError("need at least three finite manifold points") + entropies = [entropy_knn(x, config.entropy_neighbors) for x in embeddings] + scores = np.empty((len(embeddings), len(embeddings)), dtype=np.float64) + for i, left in enumerate(embeddings): + for j in range(i, len(embeddings)): + right = embeddings[j] + product = entropies[i] * entropies[j] + if not math.isfinite(product) or product <= 1e-12: + raise FloatingPointError( + f"invalid entropy product at ({i}, {j}): {entropies[i]}, {entropies[j]}" + ) + mi = np.mean( + [ + np.mean( + mutual_info_regression( + left, + right[:, dim], + n_neighbors=min(config.mi_neighbors, n - 1), + random_state=config.seed, + ) + ) + for dim in range(right.shape[1]) + ] + ) + scores[i, j] = scores[j, i] = mi / math.sqrt(product) + if not np.isfinite(scores).all(): + raise FloatingPointError("NPIB contains nonfinite scores") + return scores, entropies + + +def upstream_lambda( + scores: np.ndarray, left: int, right: int, beta: float = 1.0 +) -> float: + """Exactly upstream's sigmoid(beta * row_mean_i / (row_mean_i + row_mean_j)).""" + means = np.mean(scores[[left, right]], axis=1) + total = float(means.sum()) + if not np.isfinite(means).all() or (means < 0).any() or total <= 1e-12: + raise FloatingPointError( + "cannot compute upstream lambda from nonpositive/nonfinite row means" + ) + if not math.isfinite(beta) or beta < 0: + raise ValueError("beta must be non-negative and finite") + value = beta * float(means[0]) / total + return 1.0 / (1.0 + math.exp(-value)) + + +def select_pair( + scores: np.ndarray, + *, + selection: str = "npib", + protected_first: int = 0, + protected_last: int = 0, +) -> int: + """Adjacent pairs only; NPIB descending, and deeper pair wins exact ties.""" + n = len(scores) + if scores.shape != (n, n) or not np.isfinite(scores).all(): + raise ValueError("scores must be a finite square matrix") + if min(protected_first, protected_last) < 0: + raise ValueError("protected layer counts must be non-negative") + candidates = range(protected_first, n - protected_last - 1) + if not candidates: + raise ValueError("no eligible adjacent pair remains") + if selection == "back-to-front": + return candidates[-1] + if selection != "npib": + raise ValueError("selection must be 'npib' or 'back-to-front'") + return max(candidates, key=lambda i: (float(scores[i, i + 1]), i)) + + +@torch.inference_mode() +def merge_pair(model: nn.Module, left: int, ratio: float) -> int: + """FP32 weighted sum copied into real parameters, then delete the right block.""" + layers = model.model.layers + if ( + not 0 <= left < len(layers) - 1 + or not math.isfinite(ratio) + or not 0 <= ratio <= 1 + ): + raise ValueError("invalid adjacent pair or merge ratio") + destination, source = layers[left], layers[left + 1] + pairs = [] + for name in PROJECTION_WEIGHTS: + pairs.append((destination.get_parameter(name), source.get_parameter(name))) + # Qwen2.5 has Q/K/V biases. Fuse them with the same coefficient; + # Llama and Qwen3 (bias-free projections) retain their existing behavior. + module_name = name.rsplit('.', 1)[0] + a_bias = getattr(destination.get_submodule(module_name), 'bias', None) + b_bias = getattr(source.get_submodule(module_name), 'bias', None) + if (a_bias is None) != (b_bias is None): + raise ValueError('projection bias presence differs between merge layers') + if a_bias is not None: + pairs.append((a_bias, b_bias)) + # Validate all shapes/values/config before any model mutation. + if any(a.shape != b.shape for a, b in pairs): + raise ValueError("projection shapes differ between merge layers") + if any(not torch.isfinite(p).all() for pair in pairs for p in pair): + raise FloatingPointError("nonfinite source weight") + configs = [model.config, model.model.config] + for config in configs: + types = getattr(config, "layer_types", None) + if types is not None and len(types) != len(layers): + raise ValueError("config.layer_types length does not match decoder depth") + changed = 0 + for target, other in pairs: + merged = torch.lerp( + other.detach().to(target.device, dtype=torch.float32), target.float(), ratio + ) + merged = merged.to(target.dtype) + if not torch.isfinite(merged).all(): + raise FloatingPointError("merge overflow after conversion to model dtype") + changed += int(not torch.equal(target, merged)) + target.copy_(merged) + del layers[left + 1] + handled = set() + for config in configs: + if id(config) in handled: + continue + handled.add(id(config)) + config.num_hidden_layers = len(layers) + if getattr(config, "layer_types", None) is not None: + config.layer_types = list(config.layer_types[: left + 1]) + list( + config.layer_types[left + 2 :] + ) + for index, layer in enumerate(layers): + if hasattr(layer.self_attn, "layer_idx"): + layer.self_attn.layer_idx = index + return changed + + +def progressive_prune( + model: nn.Module, + batches: Sequence[dict[str, Tensor]], + *, + target_layers: int, + geometry: GeometryConfig | None = None, + selection: str = "npib", + protected_first: int = 0, + protected_last: int = 0, + on_stage: Callable[[nn.Module, dict], None] | None = None, +) -> list[dict]: + """Recompute activations/manifolds/NPIB on fixed data after every accepted merge. + + No threshold or forced fallbacks; stop at the requested exact depth. Repeated + merging of a previously merged block is allowed and recorded in its lineage. + """ + geometry = geometry or GeometryConfig() + geometry.validate() + depth = len(model.model.layers) + if min(protected_first, protected_last) < 0 or not 1 <= target_layers <= depth: + raise ValueError("invalid target depth or protected counts") + if target_layers < protected_first + protected_last + 1: + raise ValueError("target depth would remove protected layers") + if selection not in {"npib", "back-to-front"}: + raise ValueError("invalid selection mode") + lineage = [[i] for i in range(depth)] + records = [] + model.eval() + while len(model.model.layers) > target_layers: + step = len(records) + 1 + print( + f"MKA stage {step}: collecting features for {len(lineage)} layers", + flush=True, + ) + features = collect_features(model, batches, pooling=geometry.pooling) + results = [diffusion_embedding(x, geometry) for x in features] + del features + scores, entropies = npib_matrix([x[0] for x in results], geometry) + left = select_pair( + scores, + selection=selection, + protected_first=protected_first, + protected_last=protected_last, + ) + ratio = upstream_lambda(scores, left, left + 1, geometry.beta) + record = { + "step": step, + "selection": selection, + "left_index": left, + "source_layers": [lineage[left][:], lineage[left + 1][:]], + "pair_npib": float(scores[left, left + 1]), + "lambda_left": ratio, + "lambda_right": 1 - ratio, + "row_means": scores.mean(1).tolist(), + "npib_matrix": scores.tolist(), + "entropies": entropies, + "geometry": asdict(geometry), + "manifolds": [x[1] for x in results], + } + record["changed_parameter_tensors"] = merge_pair(model, left, ratio) + lineage[left] += lineage.pop(left + 1) + record["lineage_after"] = [x[:] for x in lineage] + record["remaining_layers"] = len(lineage) + records.append(record) + print( + f"MKA stage {step}: merge {left}/{left + 1}, NPIB={record['pair_npib']:.6g}, " + f"lambda={ratio:.6g}, remaining={len(lineage)}", + flush=True, + ) + if on_stage is not None: + on_stage(model, record) + return records diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fixed_span_merge_init.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fixed_span_merge_init.py new file mode 100644 index 0000000000000000000000000000000000000000..9ed2738d3265bc7c960633e71e12ecdf9415fec4 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fixed_span_merge_init.py @@ -0,0 +1,59 @@ +"""MLP-only merge rules; fixed topology and feature transport belong to caller. + +CoMe importance/neuron scores and MKA manifold scores are measured on the +previous-cycle teacher, on the same calibration data. This helper consumes the +actual post-transport operands. No selector, attention merge, norm merge, or +teacher mutation is allowed here. These are hybrid controls, not native methods. +""" +from __future__ import annotations + +import copy +import hashlib +from types import SimpleNamespace + +import torch + +MLP_NAMES = ("mlp.gate_proj.weight", "mlp.up_proj.weight", "mlp.down_proj.weight") + + +def tensor_hash(value): + raw = value.detach().cpu().contiguous().view(torch.uint8).numpy().tobytes() + return hashlib.sha256(raw).hexdigest() + + +def frozen_hashes(layer): + return {n: tensor_hash(p) for n, p in layer.named_parameters() if n not in MLP_NAMES} + + +def build_initialization(a, transported_b, method, payload): + before_a, before_b = frozen_hashes(a), frozen_hashes(transported_b) + a_mlp = {n: tensor_hash(a.get_parameter(n)) for n in MLP_NAMES} + b_mlp = {n: tensor_hash(transported_b.get_parameter(n)) for n in MLP_NAMES} + if method == "come": + from fusegpt.come import merge_mlp + # Copy only the two MLPs, not the entire blocks/models. Preserve native + # channel concatenation order and BF16 rounding of the CoMe helper. + pair = [SimpleNamespace(mlp=copy.deepcopy(layer.mlp).cpu()) for layer in (a, transported_b)] + counts = merge_mlp(pair, payload["ratios"], payload["neuron_scores"]) + result = {"mlp." + n: p.detach().float().clone() for n, p in pair[0].mlp.named_parameters()} + detail = {"retention_ratios": payload["ratios"], "retained_mlp_neurons": counts} + del pair + elif method == "mka": + ratio = float(payload["lambda_left"]) + if not 0 <= ratio <= 1: + raise ValueError("Invalid MKA merge coefficient") + result = {n: torch.lerp(transported_b.get_parameter(n).detach().float(), + a.get_parameter(n).detach().float(), ratio) for n in MLP_NAMES} + detail = {"lambda_left": ratio, "lambda_right": 1 - ratio} + else: + raise ValueError(method) + assert set(result) == set(MLP_NAMES) + assert frozen_hashes(a) == before_a and frozen_hashes(transported_b) == before_b + assert {n: tensor_hash(a.get_parameter(n)) for n in MLP_NAMES} == a_mlp + assert {n: tensor_hash(transported_b.get_parameter(n)) for n in MLP_NAMES} == b_mlp + assert all(torch.isfinite(v).all() for v in result.values()) + return result, {**detail, "source_weights": "post_feature_transport", + "scoring_context": "previous_cycle_teacher_before_transport", + "anchor_mlp_sha256": a_mlp, "transported_source_mlp_sha256": b_mlp, + "initial_mlp_sha256": {n: tensor_hash(v) for n, v in result.items()}, + "frozen_anchor_sha256": before_a, "operands_unchanged_by_initializer": True} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fuse_layers.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fuse_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..5cca981f0f2f951ce21d88119fd12d26c6d8a089 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fuse_layers.py @@ -0,0 +1,3600 @@ +#!/usr/bin/env python3 +"""Fuse adjacent layers via attention head alignment + Fisher-barycentric merge.""" + +import argparse +import copy +import gc +import json +import os +import random +import time +from dataclasses import dataclass +from typing import Dict, List, Optional, Set, Tuple + +import torch + +try: + import numpy as np +except Exception: # pragma: no cover - optional dependency + np = None + +try: + from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig +except Exception as exc: # pragma: no cover - fail early with clear error + raise SystemExit("transformers is required: pip install transformers") from exc + +try: + import ppl_eval +except Exception as exc: # pragma: no cover - optional dependency + raise SystemExit("ppl_eval.py is required (missing or invalid)") from exc + +try: + import eval_ppl as legacy_eval_ppl +except Exception: # pragma: no cover - optional dependency + legacy_eval_ppl = None + +from fuse_layers_data import ( + FixedSeqDataset, + build_token_chunks, + expand_dataset_configs, + load_instruction_records, + load_texts, + load_texts_from_datasets, +) +from common_lm_data import SharedLMDataSpec, build_chunks, build_dataloader +from fuse_layers_distill import ( + commutator_precondition, + compute_fisher_gate_priors, + distill_reparam_merge, + lora_ce_finetune, + summarize_fisher_gate_prior_inputs, +) +from fuse_layers_model import ( + apply_norm_policy, + build_head_permutation, + clone_state_dict, + compute_fisher, + compute_head_means, + decrement_config, + drop_layer, + find_attention_module, + find_colon_modules, + find_layer_container, + get_dtype, + get_norm_pair, + merge_layers, + permute_attention_heads, +) +from fuse_layers_select import select_best_region_by_cosine, select_layer_auto +from progressive_loader import ( + load_causal_lm, + load_progressive_model, + model_is_sharded, + model_parameter_devices, + model_primary_device, + resolve_model_device_map_spec, +) +from source_feature_transport import ( + compute_same_anchor_contextual_fisher, + fit_and_apply_source_feature_transport, +) +try: + from post_merge_nll_scale import ( + build_scale_chunks, + fit_and_fold_post_merge_scales, + ) +except ImportError: + # The zero-safe / sequential-rho experiments do not enable the optional + # post-merge NLL-scaling ablation. Keep the core ABPrune runner usable + # when that standalone ablation module is not present, but fail clearly if + # a caller explicitly requests it. + def _missing_post_merge_nll_scale(*_args, **_kwargs): + raise RuntimeError( + "post_merge_nll_scale.py is required when " + "--post_merge_nll_scale is enabled" + ) + + build_scale_chunks = _missing_post_merge_nll_scale + fit_and_fold_post_merge_scales = _missing_post_merge_nll_scale + + +def _sync_for_timing() -> None: + if torch.cuda.is_available(): + torch.cuda.synchronize() + + +def _timing_start() -> float: + _sync_for_timing() + return time.perf_counter() + + +def _timing_elapsed(start: float) -> float: + _sync_for_timing() + return time.perf_counter() - start + + +def _timing_summary(seconds: Dict[str, float]) -> Dict[str, object]: + cycle_total = float(seconds.get("cycle_total", 0.0)) + component_seconds = { + key: float(value) + for key, value in seconds.items() + if key != "cycle_total" and isinstance(value, (int, float)) + } + tracked_total = float(sum(component_seconds.values())) + if cycle_total <= 0.0: + cycle_total = tracked_total + return { + "seconds": {key: float(value) for key, value in seconds.items()}, + "tracked_component_total_seconds": tracked_total, + "cycle_total_seconds": cycle_total, + "ratios_of_tracked_components": { + key: (value / tracked_total if tracked_total > 0.0 else None) + for key, value in component_seconds.items() + }, + "ratios_of_cycle_total": { + key: (value / cycle_total if cycle_total > 0.0 else None) + for key, value in component_seconds.items() + }, + } + + +def _print_cycle_timing(cycle_idx: int, timing: Dict[str, object]) -> None: + seconds = timing.get("seconds", {}) + ratios = timing.get("ratios_of_cycle_total", {}) + if not isinstance(seconds, dict) or not isinstance(ratios, dict): + return + print(f"[timing] Cycle {cycle_idx} component wall-clock:") + preferred = [ + "selection", + "rebase", + "head_permutation", + "fisher", + "fit", + "materialize_save_layer", + "metadata_write", + "teacher_snapshot", + "cycle_cleanup", + ] + for name in preferred: + if name not in seconds: + continue + ratio = ratios.get(name) + ratio_text = "n/a" if ratio is None else f"{100.0 * float(ratio):.2f}%" + print(f"[timing] {name}: {float(seconds[name]):.3f}s ({ratio_text} of cycle)") + cycle_total = timing.get("cycle_total_seconds") + tracked_total = timing.get("tracked_component_total_seconds") + if isinstance(cycle_total, (int, float)): + print(f"[timing] cycle_total: {float(cycle_total):.3f}s") + if isinstance(tracked_total, (int, float)): + print(f"[timing] tracked_components_total: {float(tracked_total):.3f}s") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Fuse layer i and i+1 using head alignment + Fisher barycenter." + ) + parser.add_argument("--model", required=True, help="HF model id or local path") + parser.add_argument( + "--model_cache_dir", + default=None, + help="Optional cache dir for model/tokenizer downloads", + ) + parser.add_argument( + "--layer", + type=str, + default="auto", + help="Layer index i (int) or 'auto' to select via auto metric", + ) + parser.add_argument( + "--selection_method", + choices=["dwce", "sequential"], + default="dwce", + help=( + "Pair selection policy for progressive pruning. " + "'dwce' uses downstream-weighted composition error; " + "'sequential' always takes the next available pair." + ), + ) + parser.add_argument( + "--exclude_pairs", + "--exclude_layers", + nargs="*", + default=None, + dest="exclude_pairs", + help=( + "Exclude pair indices from consideration for any fusion. Indices refer to " + "pair start positions in [0..N-2]. Negative indices count from the end " + "(-1 = last pair, -2 = second last). Accepts space- or comma-separated ints. " + "Alias: --exclude_layers (deprecated)." + ), + ) + parser.add_argument( + "--output_dir", required=True, help="Directory to write fused model" + ) + parser.add_argument( + "--dataset", + action="append", + default=[], + help=( + "HF dataset name (repeatable). Optional if using --text or --text_file." + ), + ) + parser.add_argument( + "--dataset_config", + action="append", + default=[], + help="Optional dataset config (repeatable or single shared config).", + ) + parser.add_argument( + "--dataset_split", + default="train", + help="Dataset split to use (default: train)", + ) + parser.add_argument( + "--dataset_text_field", + default=None, + help="Text field in dataset (default: auto-detect, applies to all datasets)", + ) + parser.add_argument( + "--text", + action="append", + default=[], + help="Inline text samples (can pass multiple)", + ) + parser.add_argument( + "--text_file", + default=None, + help="Path to a text file for calibration data", + ) + parser.add_argument( + "--num_samples", + type=int, + default=128, + help="Number of token sequences to use", + ) + parser.add_argument( + "--target_tokens", + type=int, + default=0, + help="Target token budget for common_lm_data-backed calibration/distillation (0 = disabled)", + ) + parser.add_argument("--seq_len", type=int, default=256, help="Sequence length") + parser.add_argument("--batch_size", type=int, default=2, help="Batch size") + parser.add_argument( + "--device", + default="cuda" if torch.cuda.is_available() else "cpu", + help="Device for model + compute", + ) + parser.add_argument( + "--dtype", + default="auto", + choices=["auto", "float32", "float16", "bfloat16"], + help="Model dtype", + ) + parser.add_argument( + "--model_device_map", + default=None, + help=( + "Optional transformers device_map for the student model. " + "Use auto/balanced/balanced_low_0/sequential, split_half, " + "split_half:cuda:0,cuda:1, or split_layers:cuda:0,cuda:1,..." + ), + ) + parser.add_argument( + "--layer_path", + default=None, + help="Override layer attribute path (e.g., model.layers)", + ) + parser.add_argument( + "--fisher_mode", + default="tensor", + choices=["tensor", "param"], + help="Fisher approximation granularity", + ) + parser.add_argument( + "--no_head_permute", + action="store_true", + help=( + "Deprecated alias for --no_head_permute_merge. " + "Disables merge-stage head permutation only." + ), + ) + parser.add_argument( + "--no_head_permute_merge", + action="store_true", + help="Disable attention head permutation alignment before merge", + ) + parser.add_argument( + "--no_head_permute_select", + action="store_true", + help="Disable attention head permutation alignment during auto selection", + ) + parser.add_argument("--eps", type=float, default=1e-8, help="Stability epsilon") + parser.add_argument("--seed", type=int, default=0, help="Random seed") + parser.add_argument( + "--trust_remote_code", + action="store_true", + help="Allow custom model code from hub", + ) + parser.add_argument( + "--save_metadata", + action="store_true", + help="Backward-compatible no-op; metadata is always written.", + ) + parser.add_argument( + "--skip_eval", + action="store_true", + help="Skip pre/post perplexity evaluation", + ) + parser.add_argument( + "--eval_dataset", + action="append", + default=[], + help="Evaluation dataset name (repeatable). Defaults to wikitext.", + ) + parser.add_argument( + "--eval_dataset_config", + action="append", + default=[], + help="Evaluation dataset config (repeatable or single shared config).", + ) + parser.add_argument( + "--eval_split", + default="test", + help="Evaluation dataset split (default: test)", + ) + parser.add_argument( + "--eval_text_field", + default=None, + help="Evaluation text field override (default: auto-detect)", + ) + parser.add_argument( + "--eval_model_family", + type=str, + choices=["auto", "llama", "qwen"], + default="auto", + help="Model family for BOS handling during eval", + ) + parser.add_argument( + "--eval_add_bos", + type=str, + choices=["single", "auto", "always", "never"], + default="single", + help="Whether to prepend BOS to each eval sample", + ) + parser.add_argument( + "--eval_num_samples", + type=int, + default=0, + help="Number of token sequences per eval dataset (0 = all)", + ) + parser.add_argument( + "--eval_seq_len", + type=int, + default=1024, + help="Sequence length for eval", + ) + parser.add_argument( + "--eval_batch_size", + type=int, + default=4, + help="Batch size for eval (default: 4 to match run_eval_ppl.sh)", + ) + parser.add_argument( + "--eval_dtype", + default="float16", + choices=["auto", "float32", "float16", "bfloat16"], + help="Model dtype for eval (default: float16 to match run_eval_ppl.sh)", + ) + parser.add_argument( + "--eval_max_batches", + type=int, + default=None, + help="Optional max number of eval batches per dataset", + ) + parser.add_argument( + "--eval_cache_dir", + default=None, + help="Optional datasets cache dir for eval", + ) + parser.add_argument( + "--eval_num_workers", + type=int, + default=0, + help="Eval DataLoader workers", + ) + parser.add_argument( + "--eval_device", + default=None, + help="Device for eval (defaults to --device)", + ) + parser.add_argument( + "--eval_merge_only_ppl", + action="store_true", + help=( + "Evaluate PPL after a temporary Fisher merge and before reparam/distill." + ), + ) + parser.add_argument( + "--skip_distill", + action="store_true", + help="Skip reparameterized distillation after head alignment/Fisher setup", + ) + parser.add_argument( + "--distill_calib_samples", + type=int, + default=256, + help="Number of distillation sequences from calibration datasets", + ) + parser.add_argument( + "--distill_inst_samples", + type=int, + default=0, + help="Number of distillation sequences from instruction dataset (0 = all)", + ) + parser.add_argument( + "--distill_seq_len", + type=int, + default=512, + help="Sequence length for distillation", + ) + parser.add_argument( + "--distill_batch_size", + type=int, + default=2, + help="Batch size for distillation", + ) + parser.add_argument( + "--distill_epochs", + type=float, + default=1.0, + help="Number of distillation epochs (float allowed, e.g. 0.5)", + ) + parser.add_argument( + "--distill_lr", + type=float, + default=1e-4, + help="Learning rate for distillation", + ) + parser.add_argument( + "--distill_method", + choices=["reparam"], + default="reparam", + help="Distillation strategy (reparam only).", + ) + parser.add_argument( + "--distill_kl_weight", + type=float, + default=1e-2, + help="Weight for KL loss on logits", + ) + parser.add_argument( + "--distill_kl_temp", + type=float, + default=4.0, + help="Temperature for KL distillation on logits", + ) + parser.add_argument( + "--kl_chunk_size", + type=int, + default=16384, + help=( + "Vocabulary chunk size for streaming KL calculations. The default " + "preserves the previous implementation; lower values reduce peak memory." + ), + ) + parser.add_argument( + "--distill_hidden_mse_weight", + type=float, + default=1.0, + help="Weight for hidden-state MSE in reparam distillation (0 disables it)", + ) + parser.add_argument( + "--distill_attn_mse_weight", + type=float, + default=0.0, + help="Weight for auxiliary attention-output MSE in reparam distillation", + ) + parser.add_argument( + "--distill_mlp_mse_weight", + type=float, + default=0.0, + help="Weight for auxiliary MLP-output MSE in reparam distillation", + ) + parser.add_argument( + "--reparam_eta", + type=float, + default=1e-2, + help="Eta: ||lambda - lambda_gate||^2 regularizer weight for --distill_method reparam", + ) + parser.add_argument( + "--reparam_gamma", + type=float, + default=1e-4, + help="Gamma: ||U - U0||^2 regularizer weight for --distill_method reparam", + ) + parser.add_argument( + "--reparam_attn_reg_scale", + type=float, + default=1.0, + help="Relative scale applied to attention-parameter reparam regularizers", + ) + parser.add_argument( + "--reparam_mlp_reg_scale", + type=float, + default=1.0, + help="Relative scale applied to MLP-parameter reparam regularizers", + ) + parser.add_argument( + "--reparam_param_subset", + type=str, + choices=["all", "mlp", "attn", "down_proj"], + default="all", + help=( + "Restrict reparam merge/recovery capacity to only this parameter family " + "or module subset" + ), + ) + parser.add_argument( + "--reparam_fisher_init_subset", + type=str, + choices=["same", "all", "mlp", "attn", "down_proj", "mlp_o", "mlp_v", "mlp_vo"], + default="same", + help=( + "Restrict Fisher initialization to this subset while still materializing " + "and fitting --reparam_param_subset. Parameters outside this subset " + "start exactly at the anchor value." + ), + ) + parser.add_argument( + "--reparam_train_subset", + type=str, + choices=["same", "all", "mlp", "attn", "down_proj"], + default="same", + help=( + "Restrict which materialized reparam parameters are optimized. " + "Default 'same' preserves --reparam_param_subset behavior." + ), + ) + parser.add_argument( + "--reparam_skip_train", + action="store_true", + help=( + "Materialize the Fisher-prior reparam merge without optimizing U/gates. " + "Useful for separating Fisher initialization from reparam training." + ), + ) + parser.add_argument( + "--reparam_rho_objective", + choices=["none", "full", "hidden_kl", "kl_only"], + default="none", + help=( + "Replace trainable element-wise Fisher gates with gate/up/down rho " + "coefficients. U still uses the full FiT loss; rho receives only the " + "selected objective." + ), + ) + parser.add_argument( + "--reparam_rho_lr", + type=float, + default=2e-3, + help="Learning rate for projection-level Fisher-undo rho coefficients", + ) + parser.add_argument( + "--reparam_rho_schedule", + choices=["joint", "sequential"], + default="joint", + help=( + "joint optimizes rho and U together; sequential first optimizes rho " + "on the fixed Fisher-to-anchor axis, then freezes rho and trains an " + "additive U correction with the full FiT objective." + ), + ) + parser.add_argument( + "--reparam_additive_u", + action="store_true", + help=( + "Freeze the Fisher interpolation point and train a zero-initialized " + "additive U correction: W = W_F + U_add. This removes the " + "coordinatewise (2*lambda_F-1) gradient mask used by the standard " + "W0/U parameterization." + ), + ) + parser.add_argument( + "--reparam_rho_epochs", + type=float, + default=1.0, + help="Rho-only epochs before U training in sequential mode", + ) + parser.add_argument("--reparam_rho_min", type=float, default=-1.0) + parser.add_argument("--reparam_rho_max", type=float, default=2.0) + parser.add_argument( + "--source_feature_transport", + action="store_true", + help=( + "Before each progressive merge, fit X_anchor @ T ~= X_source and " + "rewrite the source MLP gate/up input projections as W_source @ T.T. " + "Then measure anchor and transported-source Fisher in the same " + "post-pruning anchor slot before merging." + ), + ) + parser.add_argument( + "--source_feature_transport_ridge_relative", + type=float, + default=1e-3, + help="Relative ridge used by the stage-local feature transport OLS solve", + ) + parser.add_argument( + "--source_feature_transport_max_batches", + type=int, + default=0, + help="Maximum calibration batches per transport solve (0 uses all)", + ) + parser.add_argument( + "--post_merge_nll_scale", + choices=["none", "mlp", "joint"], + default="none", + help="After every progressive merge, fit MLP-only or joint attention/MLP output scales with next-token NLL and fold them into output projections.", + ) + parser.add_argument("--post_merge_nll_scale_dataset", default="DKYoon/SlimPajama-6B") + parser.add_argument("--post_merge_nll_scale_config", default="none") + parser.add_argument("--post_merge_nll_scale_split", default="train") + parser.add_argument("--post_merge_nll_scale_text_field", default="text") + parser.add_argument("--post_merge_nll_scale_sequences", type=int, default=16) + parser.add_argument("--post_merge_nll_scale_seq_len", type=int, default=512) + parser.add_argument("--post_merge_nll_scale_seed", type=int, default=42) + parser.add_argument("--post_merge_nll_scale_steps", type=int, default=128) + parser.add_argument("--post_merge_nll_scale_lr", type=float, default=0.02) + parser.add_argument("--post_merge_nll_scale_regularization", type=float, default=1e-4) + parser.add_argument("--post_merge_nll_scale_min", type=float, default=0.25) + parser.add_argument("--post_merge_nll_scale_max", type=float, default=4.0) + parser.add_argument( + "--norm_policy", + type=str, + choices=["hybrid", "merge_all", "copy_n1", "copy_n1_n2"], + default="hybrid", + help="Norm merge policy (default: hybrid)", + ) + parser.add_argument( + "--distill_weight_decay", + type=float, + default=0.0, + help="Weight decay for distillation", + ) + parser.add_argument( + "--distill_max_grad_norm", + type=float, + default=1.0, + help="Max grad norm for distillation", + ) + parser.add_argument( + "--distill_grad_accum_steps", + type=int, + default=1, + help="Gradient accumulation steps for distillation", + ) + parser.add_argument( + "--distill_log_steps", + type=int, + default=100, + help="Log distillation loss every N steps", + ) + parser.add_argument( + "--distill_eval_every", + type=int, + default=0, + help="Evaluate PPL every N distill steps (0 = disable)", + ) + parser.add_argument( + "--distill_eval_max_batches", + type=int, + default=None, + help="Max eval batches per dataset during distill (default: all)", + ) + parser.add_argument( + "--distill_teacher_device", + default=None, + help="Device for teacher model during distillation (defaults to --device)", + ) + parser.add_argument( + "--distill_teacher_device_map", + default=None, + help=( + "Optional transformers device_map for the teacher model. " + "Use auto/balanced/balanced_low_0/sequential, split_half, " + "split_half:cuda:2,cuda:3, or split_layers:cuda:0,cuda:1,..." + ), + ) + parser.add_argument( + "--disable_previous_cycle_teacher_cache", + action="store_true", + help=( + "Disable the in-memory previous-cycle teacher snapshot and always reload " + "teachers from disk/base checkpoints. Useful when the student model is sharded " + "or when the teacher runs on CPU." + ), + ) + parser.add_argument( + "--distill_teacher_load_in_4bit", + action="store_true", + help=( + "Prototype-only rescue path: load the distillation/previous-cycle teacher in " + "4-bit NF4 on a CUDA device to reduce memory pressure." + ), + ) + parser.add_argument( + "--comm_enabled", + action="store_true", + help=( + "Enable commutator-style preconditioning before each progressive " + "cycle's fusion." + ), + ) + parser.add_argument( + "--comm_include_cycle1", + action="store_true", + help="Run commutator preconditioning for cycle 1 as well (default: skip cycle 1).", + ) + parser.add_argument( + "--comm_data_source", + choices=["auto", "calib", "distill"], + default="auto", + help=( + "Data source for commutator preconditioning. 'auto' preserves the " + "existing behavior: use distillation data when available, otherwise " + "fall back to calibration data." + ), + ) + parser.add_argument( + "--comm_topk", + type=int, + default=1, + help="Top-K lowest-score pairs used as the commutator candidate set", + ) + parser.add_argument( + "--comm_sample_eta", + type=float, + default=0.5, + help="Mixture weight between uniform and score-biased candidate sampling", + ) + parser.add_argument( + "--comm_sample_dwce_scale", + type=float, + default=1.0, + help="Scale c in softmax(-c * score(i)) for commutator pair sampling", + ) + parser.add_argument( + "--comm_temp", + type=float, + default=2.0, + help="Temperature for teacher-anchor KL in commutator preconditioning", + ) + parser.add_argument( + "--comm_steps_ratio", + type=float, + default=0.1, + help="Run this fraction of distillation optimizer steps for commutator phase", + ) + parser.add_argument( + "--comm_lr_scale", + type=float, + default=0.1, + help="Commutator LR = --distill_lr * this scale", + ) + parser.add_argument( + "--comm_train_mode", + choices=["lora", "full"], + default="lora", + help=( + "Commutator trainable parameter mode: " + "'lora' updates LoRA adapters on sampled receiver layers; " + "'full' updates full receiver-layer weights." + ), + ) + parser.add_argument( + "--comm_interaction_mode", + choices=["mse", "relative"], + default="relative", + help="Interaction loss form: plain MSE or relative MSE", + ) + parser.add_argument( + "--comm_interaction_eps", + type=float, + default=1e-8, + help="Epsilon for relative commutator interaction normalization", + ) + parser.add_argument( + "--comm_mu", + type=float, + default=None, + help=( + "Weight for interaction loss. Defaults to 0.1 for --comm_interaction_mode=mse " + "and 0.5 for --comm_interaction_mode=relative." + ), + ) + parser.add_argument( + "--comm_mu_auto", + action="store_true", + help="Enable automatic mu scaling via gradient-norm balancing", + ) + parser.add_argument( + "--comm_mu_auto_rho", + type=float, + default=0.1, + help="Target anchor-to-interaction gradient ratio constant for auto-mu", + ) + parser.add_argument( + "--comm_mu_auto_eps", + type=float, + default=1e-8, + help="Numerical epsilon in auto-mu denominator", + ) + parser.add_argument( + "--comm_log_steps", + type=int, + default=50, + help="Log commutator preconditioning loss every N optimizer steps", + ) + parser.add_argument( + "--comm_skip_post_reselect", + action="store_true", + help=( + "Keep the pre-comm selected fusion pair and skip recomputing " + "selection after commutator preconditioning." + ), + ) + parser.add_argument( + "--comm_measure_gap", + action="store_true", + help=( + "Measure commutator residual gap before and after preconditioning. " + "Uses prepared eval data when available, otherwise the commutator dataloader." + ), + ) + parser.add_argument( + "--comm_gap_max_batches", + type=int, + default=8, + help="Max batches for --comm_measure_gap (0 = all available batches).", + ) + parser.add_argument( + "--redistrib_teacher_source", + type=str, + choices=["base_model", "previous_cycle"], + default="base_model", + help=( + "Teacher source for commutator preconditioning teacher loading. " + "'base_model' uses --model for all cycles; " + "'previous_cycle' uses cycle-1 checkpoint (cycle 1 falls back to base_model)." + ), + ) + parser.add_argument( + "--lora_epochs", + type=float, + default=1.0, + help="LoRA CE finetuning epochs after distill (0 = disable)", + ) + parser.add_argument( + "--lora_rank", + type=int, + default=8, + help="LoRA rank (r)", + ) + parser.add_argument( + "--lora_alpha", + type=float, + default=16.0, + help="LoRA alpha", + ) + parser.add_argument( + "--lora_dropout", + type=float, + default=0.0, + help="LoRA dropout", + ) + parser.add_argument( + "--lora_kl_enabled", + action="store_true", + help="Add KL regularization between pre/post LoRA logits", + ) + parser.add_argument( + "--lora_kl_weight", + type=float, + default=1e-1, + help="KL weight for LoRA regularization", + ) + parser.add_argument( + "--lora_kl_temp", + type=float, + default=4.0, + help="Temperature for LoRA KL regularization", + ) + parser.add_argument( + "--lora_target_modules", + nargs="*", + default=[ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj", + ], + help="Module name suffixes to LoRA-wrap", + ) + parser.add_argument( + "--lora_respect_exclude_pairs", + action="store_true", + help=( + "When attaching LoRA adapters, skip linear modules under layers touched by " + "--exclude_pairs (i and i+1 for each excluded pair)." + ), + ) + parser.add_argument( + "--lora_lr", + type=float, + default=1e-4, + help="Learning rate for LoRA finetuning", + ) + parser.add_argument( + "--lora_weight_decay", + type=float, + default=0.0, + help="Weight decay for LoRA finetuning", + ) + parser.add_argument( + "--lora_max_grad_norm", + type=float, + default=1.0, + help="Max grad norm for LoRA finetuning", + ) + parser.add_argument( + "--lora_grad_accum_steps", + type=int, + default=1, + help="Gradient accumulation steps for LoRA finetuning", + ) + parser.add_argument( + "--lora_log_steps", + type=int, + default=100, + help="Log LoRA loss every N steps", + ) + parser.add_argument( + "--lora_eval_every", + type=int, + default=0, + help="Evaluate PPL every N LoRA steps (0 = disable)", + ) + parser.add_argument( + "--lora_eval_max_batches", + type=int, + default=None, + help="Max eval batches per dataset during LoRA (default: all)", + ) + parser.add_argument( + "--instruction_dataset", + default=None, + help="HF dataset name for alpaca-style instruction data", + ) + parser.add_argument( + "--instruction_config", + default=None, + help="Optional instruction dataset config", + ) + parser.add_argument( + "--instruction_split", + default="train", + help="Instruction dataset split", + ) + parser.add_argument( + "--instruction_field_instruction", + default="instruction", + help="Instruction field name", + ) + parser.add_argument( + "--instruction_field_input", + default="input", + help="Optional input field name", + ) + parser.add_argument( + "--instruction_field_output", + default="output", + help="Response/output field name", + ) + parser.add_argument( + "--auto_max_batches", + type=int, + default=0, + help="Max calibration batches for auto selection scoring (0 = all)", + ) + parser.add_argument( + "--auto_metric", + type=str, + choices=[ + "dwce", + "final_hidden_relative_error", + "cosine", + "region_cosine_dwce", + "hybrid", + "hybrid_cosine", + "hybrid_global_rel", + ], + default="dwce", + help=( + "Auto pair scoring metric. 'dwce' uses downstream-weighted composition error; " + "'final_hidden_relative_error' measures surrogate-fusion error at the final hidden state; " + "'cosine' uses average token-level cosine distance between adjacent layer outputs; " + "'region_cosine_dwce' first finds the most cosine-redundant contiguous region " + "and then applies DWCE only within that region; " + "'hybrid'/'hybrid_cosine' use DWCE to shortlist then adjacent cosine for final scoring; " + "'hybrid_global_rel' uses DWCE to shortlist then reranks by the change in " + "pair-to-final-layer cosine relation after surrogate fusion." + ), + ) + parser.add_argument( + "--auto_cosine_topk", + type=int, + default=3, + help="Top-K DWCE candidates to rescore with cosine in --auto_metric=hybrid", + ) + parser.add_argument( + "--auto_region_width", + type=int, + default=4, + help=( + "Contiguous region width in adjacent pairs for " + "--auto_metric=region_cosine_dwce" + ), + ) + parser.add_argument( + "--fixed_region_pairs", + type=str, + default=None, + help=( + "Optional fixed original pair-start indices to restrict auto selection to. " + "Accepts comma-separated integers or ranges like 22-27. When set, " + "cosine region search is skipped and only these original pair starts are " + "considered by DWCE." + ), + ) + parser.add_argument( + "--auto_norm", + type=str, + choices=["relative", "none"], + default="relative", + help="Normalization mode for DWCE scoring (ignored for cosine)", + ) + parser.add_argument( + "--auto_dwce_mode", + type=str, + choices=["separate", "shared"], + default="separate", + help=( + "DWCE implementation for auto scoring. " + "'separate' runs distinct Fisher and DWCE backward passes; " + "'shared' reuses one backward pass and replays DWCE with cached gradients." + ), + ) + parser.add_argument( + "--auto_dwce_grad_power", + type=int, + choices=[1, 2], + default=2, + help="Exponent for DWCE gradient weighting. 2 matches the original DWCE; 1 uses |grad|.", + ) + parser.add_argument( + "--num_progressive", + type=int, + default=0, + help="Number of progressive fusions (>0 required)", + ) + parser.add_argument( + "--resume_from_cycle", + type=int, + default=0, + help=( + "Resume from this completed cycle index. When > 0, --model should point " + "to the saved full model directory for that cycle." + ), + ) + parser.add_argument( + "--save_full_model_cycles", + nargs="*", + default=[], + help=( + "Cycle indices whose full models should be saved. Requesting cycle c " + "also saves cycle c-1 automatically (c=1 saves only cycle 1)." + ), + ) + parser.add_argument( + "--profile_no_save", + action="store_true", + help=( + "Profiling-only mode: write metadata/timing logs but skip all weight " + "artifacts, including per-cycle fused layers and final save_pretrained outputs." + ), + ) + parser.add_argument( + "--skip_final_model_save", + action="store_true", + help=( + "Compact-checkpoint mode: keep each cycle's fused_layer.pt and metadata " + "but skip the duplicate final pre-LoRA and final model save_pretrained outputs." + ), + ) + return parser.parse_args() + + +def parse_exclude_pairs(exclude_raw: Optional[List[str]], num_pairs: int) -> List[int]: + """Parse --exclude_pairs into normalized pair indices for the current model. + + Indices refer to the start of an adjacent pair (i, i+1) and must be in [0..N-2]. + Negative indices count from the end (-1 = last pair). + """ + if not exclude_raw: + return [] + exclude: List[int] = [] + for item in exclude_raw: + if item is None: + continue + for part in str(item).split(","): + part = part.strip() + if not part: + continue + try: + idx = int(part) + except ValueError as exc: + raise SystemExit("--exclude_pairs must contain integers.") from exc + if idx < 0: + idx = num_pairs + idx + if 0 <= idx < num_pairs: + exclude.append(idx) + return sorted(set(exclude)) + + +def parse_cycle_list(raw_values: Optional[List[str]]) -> List[int]: + if not raw_values: + return [] + cycles: List[int] = [] + for item in raw_values: + if item is None: + continue + for part in str(item).split(","): + part = part.strip() + if not part: + continue + try: + cycles.append(int(part)) + except ValueError as exc: + raise SystemExit( + "--save_full_model_cycles must contain integers." + ) from exc + return cycles + + +def resolve_full_model_save_cycles( + requested_cycles: List[int], num_progressive: int +) -> Set[int]: + resolved: Set[int] = set() + for cycle in requested_cycles: + if cycle <= 0 or cycle > num_progressive: + raise SystemExit( + "--save_full_model_cycles entries must be within [1, --num_progressive]." + ) + resolved.add(cycle) + if cycle > 1: + resolved.add(cycle - 1) + return resolved + + +def load_resume_metadata(model_path: str) -> Optional[Dict[str, object]]: + resume_meta_path = os.path.join(model_path, "resume_info.json") + if not os.path.exists(resume_meta_path): + return None + with open(resume_meta_path, "r", encoding="utf-8") as handle: + loaded = json.load(handle) + return loaded if isinstance(loaded, dict) else None + + +def build_generator(seed: int) -> torch.Generator: + generator = torch.Generator(device="cpu") + generator.manual_seed(int(seed)) + return generator + + +def capture_rng_state() -> Dict[str, object]: + state: Dict[str, object] = { + "python_random_state": random.getstate(), + "torch_cpu_rng_state": torch.get_rng_state(), + } + if np is not None: + state["numpy_random_state"] = np.random.get_state() + if torch.cuda.is_available(): + state["torch_cuda_rng_state_all"] = torch.cuda.get_rng_state_all() + return state + + +def restore_rng_state(state: Dict[str, object]) -> None: + python_state = state.get("python_random_state") + if python_state is not None: + random.setstate(python_state) + + numpy_state = state.get("numpy_random_state") + if numpy_state is not None and np is not None: + np.random.set_state(numpy_state) + + torch_cpu_state = state.get("torch_cpu_rng_state") + if torch_cpu_state is not None: + torch.set_rng_state(torch_cpu_state) + + torch_cuda_state = state.get("torch_cuda_rng_state_all") + if torch_cuda_state is not None and torch.cuda.is_available(): + torch.cuda.set_rng_state_all(torch_cuda_state) + + +def save_rng_state(path: str) -> None: + torch.save(capture_rng_state(), path) + + +def load_rng_state(path: str) -> Optional[Dict[str, object]]: + if not os.path.exists(path): + return None + loaded = torch.load(path, map_location="cpu", weights_only=False) + return loaded if isinstance(loaded, dict) else None + + +def configure_reproducibility(seed: int) -> None: + random.seed(seed) + if np is not None: + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + if hasattr(torch.backends, "cudnn"): + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + if hasattr(torch, "use_deterministic_algorithms"): + torch.use_deterministic_algorithms(True, warn_only=True) + + +def save_loader_generator_state( + base_dir: str, + *, + distill_generator: Optional[torch.Generator] = None, + lora_generator: Optional[torch.Generator] = None, +) -> None: + state: Dict[str, object] = {} + if distill_generator is not None: + state["distill_generator_state"] = distill_generator.get_state() + if lora_generator is not None: + state["lora_generator_state"] = lora_generator.get_state() + if state: + torch.save(state, os.path.join(base_dir, "loader_generators.pt")) + + +def load_loader_generator_state(base_dir: str) -> Optional[Dict[str, object]]: + path = os.path.join(base_dir, "loader_generators.pt") + if not os.path.exists(path): + return None + loaded = torch.load(path, map_location="cpu") + return loaded if isinstance(loaded, dict) else None + + +def parse_fixed_region_pairs(spec: Optional[str]) -> Optional[List[int]]: + if spec is None: + return None + text = str(spec).strip() + if not text: + return None + values: List[int] = [] + for part in text.split(","): + token = part.strip() + if not token: + continue + if "-" in token: + pieces = token.split("-") + if len(pieces) != 2: + raise SystemExit( + "--fixed_region_pairs accepts comma-separated integers or simple ranges like 22-27" + ) + start = int(pieces[0].strip()) + end = int(pieces[1].strip()) + if end < start: + start, end = end, start + values.extend(range(start, end + 1)) + else: + values.append(int(token)) + if not values: + return None + return sorted(set(values)) + + +def resolve_layer_idx( + args: argparse.Namespace, + model, + layers: List[torch.nn.Module], + dataloader, + cosine_dataloader, + previous_scores, + start_index: int, + exclude_pairs: Set[int], +): + layer_arg = str(getattr(args, "layer", "auto")).strip().lower() + selection_method = str(getattr(args, "selection_method", "dwce")).strip().lower() + + if layer_arg != "auto": + try: + layer_idx = int(layer_arg) + except ValueError as exc: + raise SystemExit("--layer must be 'auto' or an integer index") from exc + num_pairs = max(len(layers) - 1, 0) + if layer_idx < 0: + layer_idx += num_pairs + if layer_idx in exclude_pairs: + raise SystemExit(f"--layer resolved to excluded pair index {layer_idx}") + return layer_idx, previous_scores, {"method": "manual", "exclude_pairs": sorted(exclude_pairs)} + + if selection_method == "sequential": + num_pairs = len(layers) - 1 + for layer_idx in range(max(start_index, 0), num_pairs): + if layer_idx not in exclude_pairs: + return layer_idx, previous_scores, { + "method": "sequential", + "start_index": max(start_index, 0), + "exclude_pairs": sorted(exclude_pairs), + } + raise SystemExit("No eligible layer pairs remain after exclusions") + + layer_idx, dwce_scores, dwce_meta = select_layer_auto( + model, + layers, + dataloader, + cosine_dataloader, + args, + previous_scores=previous_scores, + start_index=start_index, + exclude_pairs=exclude_pairs, + ) + return layer_idx, dwce_scores, dwce_meta + + +@dataclass +class PreparedData: + calib_loader: torch.utils.data.DataLoader + calib_num_sequences: int + distill_loader: Optional[torch.utils.data.DataLoader] + distill_generator: Optional[torch.Generator] + distill_meta: Dict[str, object] + lora_loader: Optional[torch.utils.data.DataLoader] + lora_generator: Optional[torch.Generator] + lora_meta: Dict[str, object] + eval_datasets: List[str] + eval_configs: List[Optional[str]] + eval_dataloaders: Optional[Dict[str, torch.utils.data.DataLoader]] + + +def resolve_eval_datasets(args: argparse.Namespace) -> Tuple[List[str], List[Optional[str]]]: + eval_datasets = args.eval_dataset or ["wikitext2"] + eval_configs = args.eval_dataset_config or [] + eval_configs = ppl_eval._expand_dataset_configs(eval_datasets, eval_configs) + return eval_datasets, eval_configs + + +def _legacy_eval_compatible( + args: argparse.Namespace, + eval_datasets: List[str], + eval_configs: List[Optional[str]], +) -> bool: + # The legacy eval_ppl parser hardcodes the benchmark text field and split, + # so only use it for the default fuse_layers evaluation settings. + if legacy_eval_ppl is None: + return False + if args.eval_text_field is not None: + return False + if str(args.eval_split).strip().lower() != "test": + return False + return all( + legacy_eval_ppl.is_supported_dataset(dataset_name, config) + for dataset_name, config in zip(eval_datasets, eval_configs) + ) + + +def _legacy_eval_bos_mode(args: argparse.Namespace) -> str: + if args.eval_add_bos == "single": + return "single" + if args.eval_add_bos == "always": + return "every" + if args.eval_add_bos == "never": + return "none" + # Match script/run_eval_ppl.sh default behavior: one BOS at corpus start. + return "single" + + +def _ppl_eval_add_bos_mode(args: argparse.Namespace) -> str: + if args.eval_add_bos == "single": + # ppl_eval has no exact "single BOS at corpus start" mode. + # Fall back to its closest default behavior when legacy eval is unavailable. + return "auto" + return args.eval_add_bos + + +def _legacy_eval_labels( + eval_datasets: List[str], + eval_configs: List[Optional[str]], +) -> List[str]: + return [ + dataset_name if config is None else f"{dataset_name}:{config}" + for dataset_name, config in zip(eval_datasets, eval_configs) + ] + + +def _build_legacy_eval_dataloaders( + tokenizer, + eval_datasets: List[str], + eval_configs: List[Optional[str]], + args: argparse.Namespace, +) -> Dict[str, torch.utils.data.DataLoader]: + return legacy_eval_ppl.build_ppl_dataloaders( + eval_datasets, + tokenizer, + seq_len=args.eval_seq_len, + batch_size=args.eval_batch_size or args.batch_size, + configs=eval_configs, + labels=_legacy_eval_labels(eval_datasets, eval_configs), + bos_mode=_legacy_eval_bos_mode(args), + num_samples=args.eval_num_samples, + ) + + +def _resolve_eval_runtime_device( + model: torch.nn.Module, + requested_device: str, + *, + context: str, +) -> str: + if not model_is_sharded(model): + return requested_device + + primary_device = str(model_primary_device(model)) + devices = ", ".join(model_parameter_devices(model)) + if primary_device != requested_device: + print( + f"[device] {context}: keeping sharded model on {devices}; " + f"using primary device {primary_device} instead of requested {requested_device}." + ) + else: + print(f"[device] {context}: using existing sharded model placement {devices}.") + return primary_device + + +def _teacher_compute_dtype(args: argparse.Namespace) -> torch.dtype: + dtype = get_dtype(args.dtype) + if dtype is not None: + return dtype + return torch.bfloat16 + + +def _resolve_teacher_device_map( + args: argparse.Namespace, + model_id_or_path: str, +): + spec_raw = getattr(args, "distill_teacher_device_map", None) + if spec_raw is None: + return None + spec = str(spec_raw).strip() + if not spec or spec.lower() in {"none", "null"}: + return None + if bool(getattr(args, "distill_teacher_load_in_4bit", False)): + raise SystemExit( + "--distill_teacher_device_map cannot be combined with " + "--distill_teacher_load_in_4bit; use bf16 sharding or 4-bit single-device " + "teacher loading, not both." + ) + return resolve_model_device_map_spec( + spec, + model_path_or_id=model_id_or_path, + trust_remote_code=args.trust_remote_code, + cache_dir=args.model_cache_dir, + ) + + +def _teacher_load_kwargs( + args: argparse.Namespace, + teacher_device: str, + model_id_or_path: str, +) -> Tuple[Dict[str, object], bool]: + kwargs: Dict[str, object] = { + "cache_dir": args.model_cache_dir, + } + needs_manual_to = True + teacher_device_map = _resolve_teacher_device_map(args, model_id_or_path) + if teacher_device_map is not None: + kwargs["device_map"] = teacher_device_map + needs_manual_to = False + return kwargs, needs_manual_to + if bool(getattr(args, "distill_teacher_load_in_4bit", False)): + normalized = str(teacher_device).strip().lower() + if not normalized.startswith("cuda"): + raise SystemExit( + "--distill_teacher_load_in_4bit requires --distill_teacher_device to be a CUDA device." + ) + kwargs["quantization_config"] = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=_teacher_compute_dtype(args), + bnb_4bit_quant_type="nf4", + bnb_4bit_use_double_quant=True, + ) + target_device = teacher_device if normalized != "cuda" else "cuda:0" + kwargs["device_map"] = {"": target_device} + needs_manual_to = False + return kwargs, needs_manual_to + + +def _load_teacher_causal_lm( + model_id_or_path: str, + args: argparse.Namespace, + teacher_device: str, +) -> torch.nn.Module: + teacher_kwargs, needs_manual_to = _teacher_load_kwargs( + args, + teacher_device, + model_id_or_path, + ) + model = load_causal_lm( + model_id_or_path, + torch_dtype=get_dtype(args.dtype), + trust_remote_code=args.trust_remote_code, + **teacher_kwargs, + ) + if needs_manual_to: + model.to(teacher_device) + model.eval() + return model + + +def run_ppl_eval( + model_id_or_path: str, + eval_datasets: List[str], + eval_configs: List[Optional[str]], + args: argparse.Namespace, + prepared_eval_dataloaders: Optional[Dict[str, torch.utils.data.DataLoader]] = None, +) -> Dict[str, float]: + eval_device = args.eval_device or args.device + dtype = get_dtype(args.eval_dtype) + resolved_device_map = resolve_model_device_map_spec( + getattr(args, "model_device_map", None), + model_path_or_id=model_id_or_path, + trust_remote_code=args.trust_remote_code, + ) + + eval_model = load_causal_lm( + model_id_or_path, + torch_dtype=dtype, + trust_remote_code=args.trust_remote_code, + device_map=resolved_device_map, + ) + + if model_is_sharded(eval_model): + eval_device = _resolve_eval_runtime_device( + eval_model, + eval_device, + context="run_ppl_eval", + ) + else: + eval_model.to(eval_device) + if prepared_eval_dataloaders is not None: + results = ppl_eval.evaluate_ppl_dataloaders( + eval_model, + prepared_eval_dataloaders, + eval_device, + max_batches=args.eval_max_batches, + ) + elif _legacy_eval_compatible(args, eval_datasets, eval_configs): + eval_tokenizer = AutoTokenizer.from_pretrained( + model_id_or_path, trust_remote_code=args.trust_remote_code + ) + if eval_tokenizer.pad_token is None and eval_tokenizer.eos_token is not None: + eval_tokenizer.pad_token = eval_tokenizer.eos_token + results = ppl_eval.evaluate_ppl_dataloaders( + eval_model, + _build_legacy_eval_dataloaders( + eval_tokenizer, + eval_datasets, + eval_configs, + args, + ), + eval_device, + max_batches=args.eval_max_batches, + ) + else: + eval_batch_size = args.eval_batch_size or args.batch_size + eval_tokenizer = AutoTokenizer.from_pretrained( + model_id_or_path, trust_remote_code=args.trust_remote_code + ) + if eval_tokenizer.pad_token is None and eval_tokenizer.eos_token is not None: + eval_tokenizer.pad_token = eval_tokenizer.eos_token + + results = ppl_eval.evaluate_ppl_datasets( + eval_model, + eval_tokenizer, + datasets=eval_datasets, + configs=eval_configs, + split=args.eval_split, + text_field=args.eval_text_field, + num_samples=args.eval_num_samples, + seq_len=args.eval_seq_len, + batch_size=eval_batch_size, + device=eval_device, + seed=args.seed, + shuffle=False, + model_family=args.eval_model_family, + add_bos=_ppl_eval_add_bos_mode(args), + max_batches=args.eval_max_batches, + cache_dir=args.eval_cache_dir, + num_workers=args.eval_num_workers, + ) + + del eval_model + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + return results + + +def build_calibration_dataloader( + args: argparse.Namespace, tokenizer +) -> Tuple[List[str], List[torch.Tensor], torch.utils.data.DataLoader]: + if args.dataset: + datasets = list(args.dataset) + configs = expand_dataset_configs(datasets, list(args.dataset_config)) + chunks: List[torch.Tensor] = [] + for idx, (dataset_name, config) in enumerate(zip(datasets, configs)): + spec = SharedLMDataSpec( + dataset=dataset_name, + config=config, + split=args.dataset_split, + text_field=args.dataset_text_field, + seq_len=args.seq_len, + num_sequences=args.num_samples, + seed=args.seed + idx, + ) + chunks.extend(build_chunks(spec, tokenizer)) + if not chunks: + raise SystemExit("Not enough text to build token sequences.") + input_ids = torch.stack(chunks) + attention_mask = torch.ones_like(input_ids) + dataset = torch.utils.data.TensorDataset(input_ids, attention_mask) + dataloader = torch.utils.data.DataLoader( + dataset, batch_size=args.batch_size, shuffle=False + ) + return [], chunks, dataloader + + texts = load_texts(args) + if not texts: + raise SystemExit( + "No calibration text found. Provide --dataset, --text, or --text_file." + ) + + chunks = build_token_chunks(texts, tokenizer, args.seq_len, args.num_samples) + if not chunks: + raise SystemExit("Not enough text to build token sequences.") + + input_ids = torch.stack(chunks) + attention_mask = torch.ones_like(input_ids) + dataset = torch.utils.data.TensorDataset(input_ids, attention_mask) + dataloader = torch.utils.data.DataLoader( + dataset, batch_size=args.batch_size, shuffle=False + ) + return texts, chunks, dataloader + + +def prepare_distillation_data( + args: argparse.Namespace, tokenizer, include_instruction: bool = True +) -> Tuple[Optional[torch.utils.data.DataLoader], Optional[torch.Generator], Dict[str, object]]: + if ( + include_instruction + and args.distill_inst_samples != 0 + and not args.instruction_dataset + ): + print( + "Warning: --distill_inst_samples > 0 but no --instruction_dataset " + "provided; instruction distillation will be skipped." + ) + + calib_texts: List[str] = [] + calib_dataset = None + if args.target_tokens > 0 and args.dataset: + datasets = list(args.dataset) + configs = expand_dataset_configs(datasets, list(args.dataset_config)) + per_dataset = args.target_tokens // len(datasets) + remainder = args.target_tokens % len(datasets) + calib_chunks: List[torch.Tensor] = [] + for idx, (dataset_name, config) in enumerate(zip(datasets, configs)): + dataset_tokens = per_dataset + (remainder if idx == 0 else 0) + spec = SharedLMDataSpec( + dataset=dataset_name, + config=config, + split=args.dataset_split, + text_field=args.dataset_text_field, + seq_len=args.distill_seq_len, + target_tokens=dataset_tokens, + seed=args.seed + 17 + idx, + ) + calib_chunks.extend(build_chunks(spec, tokenizer)) + if calib_chunks: + input_ids = torch.stack(calib_chunks) + attention_mask = torch.ones_like(input_ids) + calib_dataset = torch.utils.data.TensorDataset(input_ids, attention_mask) + else: + calib_texts = load_texts_from_datasets( + datasets=list(args.dataset), + configs=expand_dataset_configs(list(args.dataset), list(args.dataset_config)), + split=args.dataset_split, + text_field=args.dataset_text_field, + num_samples=args.distill_calib_samples, + seed=args.seed + 17, + manifest_env_var="ABPRUNE_DISTILL_DATA_INDICES_FILE", + ) + inst_records = [] + if include_instruction: + inst_records = load_instruction_records( + args, + args.distill_inst_samples, + "ABPRUNE_DISTILL_DATA_INDICES_FILE", + ) + + distill_datasets = [] + if calib_dataset is not None: + distill_datasets.append(calib_dataset) + elif calib_texts: + calib_records = [{"text": text} for text in calib_texts] + distill_datasets.append( + FixedSeqDataset(calib_records, tokenizer, args.distill_seq_len) + ) + if inst_records: + distill_datasets.append( + FixedSeqDataset(inst_records, tokenizer, args.distill_seq_len) + ) + + distill_meta: Dict[str, object] = { + "calib_texts": len(calib_texts), + "calib_sequences": len(calib_dataset) if calib_dataset is not None else len(calib_texts), + "inst_sequences": len(inst_records), + "total_sequences": 0, + } + + if not distill_datasets: + return None, None, distill_meta + if len(distill_datasets) == 1: + distill_dataset = distill_datasets[0] + else: + distill_dataset = torch.utils.data.ConcatDataset(distill_datasets) + + distill_meta["total_sequences"] = len(distill_dataset) + distill_generator = build_generator( + args.seed + 1000 + (1000000 if include_instruction else 0) + ) + distill_loader = torch.utils.data.DataLoader( + distill_dataset, + batch_size=args.distill_batch_size, + shuffle=True, + generator=distill_generator, + ) + return distill_loader, distill_generator, distill_meta + + +def prepare_eval_dataloaders( + args: argparse.Namespace, + tokenizer, + model: torch.nn.Module, + eval_datasets: List[str], + eval_configs: List[Optional[str]], +) -> Optional[Dict[str, torch.utils.data.DataLoader]]: + if args.skip_eval: + return None + + if _legacy_eval_compatible(args, eval_datasets, eval_configs): + return _build_legacy_eval_dataloaders( + tokenizer, + eval_datasets, + eval_configs, + args, + ) + + eval_batch_size = args.eval_batch_size or args.batch_size + resolved_family = args.eval_model_family + if resolved_family == "auto": + resolved_family = ppl_eval._infer_model_family(model) + + return ppl_eval.prepare_ppl_dataloaders( + tokenizer=tokenizer, + datasets=eval_datasets, + configs=eval_configs, + split=args.eval_split, + text_field=args.eval_text_field, + num_samples=args.eval_num_samples, + seq_len=args.eval_seq_len, + batch_size=eval_batch_size, + seed=args.seed, + shuffle=False, + model_family=resolved_family, + add_bos=_ppl_eval_add_bos_mode(args), + cache_dir=args.eval_cache_dir, + num_workers=args.eval_num_workers, + model=model, + ) + + +def prepare_all_data( + args: argparse.Namespace, + tokenizer, + model: torch.nn.Module, + eval_datasets: List[str], + eval_configs: List[Optional[str]], + loader_generator_state: Optional[Dict[str, object]] = None, +) -> PreparedData: + texts, chunks, calib_loader = build_calibration_dataloader(args, tokenizer) + calib_num_sequences = len(chunks) + del texts + del chunks + + distill_loader = None + distill_generator = None + distill_meta: Dict[str, object] = { + "calib_texts": 0, + "calib_sequences": 0, + "inst_sequences": 0, + "total_sequences": 0, + } + lora_loader = None + lora_generator = None + lora_meta: Dict[str, object] = { + "calib_texts": 0, + "calib_sequences": 0, + "inst_sequences": 0, + "total_sequences": 0, + } + if (not args.skip_distill) or bool(getattr(args, "comm_enabled", False)): + distill_loader, distill_generator, distill_meta = prepare_distillation_data( + args, tokenizer, include_instruction=False + ) + if ( + distill_generator is not None + and loader_generator_state is not None + and loader_generator_state.get("distill_generator_state") is not None + ): + distill_generator.set_state(loader_generator_state["distill_generator_state"]) + if args.lora_epochs > 0: + lora_loader, lora_generator, lora_meta = prepare_distillation_data( + args, tokenizer, include_instruction=True + ) + if ( + lora_generator is not None + and loader_generator_state is not None + and loader_generator_state.get("lora_generator_state") is not None + ): + lora_generator.set_state(loader_generator_state["lora_generator_state"]) + + eval_dataloaders = prepare_eval_dataloaders( + args, tokenizer, model, eval_datasets, eval_configs + ) + + return PreparedData( + calib_loader=calib_loader, + calib_num_sequences=calib_num_sequences, + distill_loader=distill_loader, + distill_generator=distill_generator, + distill_meta=distill_meta, + lora_loader=lora_loader, + lora_generator=lora_generator, + lora_meta=lora_meta, + eval_datasets=eval_datasets, + eval_configs=eval_configs, + eval_dataloaders=eval_dataloaders, + ) + + +def evaluate_ppl_model( + model: torch.nn.Module, + tokenizer, + eval_datasets: List[str], + eval_configs: List[Optional[str]], + args: argparse.Namespace, + max_batches: Optional[int] = None, + prepared_eval_dataloaders: Optional[Dict[str, torch.utils.data.DataLoader]] = None, +) -> Dict[str, float]: + eval_device = args.eval_device or args.device + prev_mode = model.training + eval_dtype = get_dtype(args.eval_dtype) + sharded = model_is_sharded(model) + try: + first_param = next(model.parameters()) + prev_device = first_param.device + prev_dtype = first_param.dtype + except StopIteration: + prev_device = torch.device(eval_device) + prev_dtype = None + + model.eval() + if sharded: + eval_device = _resolve_eval_runtime_device( + model, + eval_device, + context="evaluate_ppl_model", + ) + if eval_dtype is not None and prev_dtype is not None and prev_dtype != eval_dtype: + print( + f"[eval] Sharded model kept at dtype={prev_dtype}; " + f"ignoring requested eval dtype {eval_dtype}." + ) + elif str(prev_device) != eval_device or ( + eval_dtype is not None and prev_dtype is not None and prev_dtype != eval_dtype + ): + move_kwargs = {"device": eval_device} + if eval_dtype is not None: + move_kwargs["dtype"] = eval_dtype + model.to(**move_kwargs) + + if prepared_eval_dataloaders is not None: + results = ppl_eval.evaluate_ppl_dataloaders( + model, + prepared_eval_dataloaders, + eval_device, + max_batches=max_batches if max_batches is not None else args.eval_max_batches, + ) + elif _legacy_eval_compatible(args, eval_datasets, eval_configs): + results = ppl_eval.evaluate_ppl_dataloaders( + model, + _build_legacy_eval_dataloaders( + tokenizer, + eval_datasets, + eval_configs, + args, + ), + eval_device, + max_batches=max_batches if max_batches is not None else args.eval_max_batches, + ) + else: + eval_batch_size = args.eval_batch_size or args.batch_size + results = ppl_eval.evaluate_ppl_datasets( + model, + tokenizer, + datasets=eval_datasets, + configs=eval_configs, + split=args.eval_split, + text_field=args.eval_text_field, + num_samples=args.eval_num_samples, + seq_len=args.eval_seq_len, + batch_size=eval_batch_size, + device=eval_device, + seed=args.seed, + shuffle=False, + model_family=args.eval_model_family, + add_bos=_ppl_eval_add_bos_mode(args), + max_batches=max_batches if max_batches is not None else args.eval_max_batches, + cache_dir=args.eval_cache_dir, + num_workers=args.eval_num_workers, + ) + + if prev_mode: + model.train() + if (not sharded) and ( + str(prev_device) != eval_device or ( + eval_dtype is not None and prev_dtype is not None and prev_dtype != eval_dtype + ) + ): + restore_kwargs = {"device": prev_device} + if prev_dtype is not None: + restore_kwargs["dtype"] = prev_dtype + model.to(**restore_kwargs) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + return results + + +def evaluate_temporary_fisher_merge_ppl( + model: torch.nn.Module, + tokenizer, + parent: object, + layer_attr: str, + container: object, + layer_idx: int, + layer_a: torch.nn.Module, + layer_b: torch.nn.Module, + fisher_sums, + num_batches: int, + param_numels, + norm_policy: str, + norm1_state: Optional[Dict[str, torch.Tensor]], + norm2_state: Optional[Dict[str, torch.Tensor]], + norm_names: Tuple[Optional[str], Optional[str]], + eval_datasets: List[str], + eval_configs: List[Optional[str]], + args: argparse.Namespace, + prepared_eval_dataloaders: Optional[Dict[str, torch.utils.data.DataLoader]] = None, +) -> Dict[str, float]: + layer_a_state = clone_state_dict(layer_a) + original_container = container + original_training = model.training + try: + with torch.no_grad(): + merge_layers( + layer_a, + layer_b, + fisher_sums[0], + fisher_sums[1], + num_batches, + param_numels[0], + param_numels[1], + fisher_mode=args.fisher_mode, + eps=args.eps, + ) + apply_norm_policy( + layer_a, + norm_policy, + norm1_state, + norm2_state, + norm_names, + ) + merged_container = drop_layer(original_container, layer_idx + 1) + setattr(parent, layer_attr, merged_container) + + return evaluate_ppl_model( + model, + tokenizer, + eval_datasets, + eval_configs, + args, + prepared_eval_dataloaders=prepared_eval_dataloaders, + ) + finally: + setattr(parent, layer_attr, original_container) + layer_a.load_state_dict(layer_a_state) + if original_training: + model.train() + else: + model.eval() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + +def has_post_fusion_data( + distill_loader: Optional[torch.utils.data.DataLoader], + distill_meta: Optional[Dict[str, object]], +) -> bool: + if distill_loader is None or distill_meta is None: + return False + return distill_meta.get("total_sequences", 0) > 0 + + +def summarize_gate_lambdas(gates: Dict[str, torch.Tensor]) -> Dict[str, object]: + if not gates: + return {"num_tensors": 0, "num_elements": 0} + + total_sum = 0.0 + total_elems = 0 + per_tensor_mean: Dict[str, Optional[float]] = {} + for name, gate in gates.items(): + g = gate.detach().float() + if g.numel() == 0: + per_tensor_mean[name] = None + continue + per_tensor_mean[name] = float(g.mean().item()) + total_sum += float(g.sum().item()) + total_elems += int(g.numel()) + + global_mean = None if total_elems == 0 else total_sum / float(total_elems) + return { + "num_tensors": len(gates), + "num_elements": total_elems, + "global_mean": global_mean, + "per_tensor_mean": per_tensor_mean, + } + + +def compute_path_bytes(path: str) -> int: + if os.path.isfile(path): + return os.path.getsize(path) + + total = 0 + for root, _, files in os.walk(path): + for name in files: + file_path = os.path.join(root, name) + if os.path.islink(file_path): + continue + try: + total += os.path.getsize(file_path) + except OSError: + continue + return total + + +def save_stage_checkpoint( + model: torch.nn.Module, + tokenizer, + stage_dir: str, + stage_name: str, + ppl_results: Optional[Dict[str, float]], +) -> Dict[str, object]: + os.makedirs(stage_dir, exist_ok=True) + colon_modules = find_colon_modules(model) + if colon_modules: + raise RuntimeError( + "Unexpected module names with ':' detected before save: " + f"{', '.join(colon_modules)}." + ) + + model.save_pretrained(stage_dir) + tokenizer.save_pretrained(stage_dir) + + stage_meta = { + "stage": stage_name, + "path": stage_dir, + "weight_bytes": compute_path_bytes(stage_dir), + "post_ppl": ppl_results, + } + with open( + os.path.join(stage_dir, "stage_metrics.json"), + "w", + encoding="utf-8", + ) as handle: + json.dump(stage_meta, handle, indent=2) + return stage_meta + + +def save_cycle_full_model( + model: torch.nn.Module, + tokenizer, + cycle_dir: str, + cycle_idx: int, + args: argparse.Namespace, + ppl_results: Optional[Dict[str, float]], +) -> Dict[str, object]: + full_model_dir = os.path.join(cycle_dir, "full_model") + stage_meta = save_stage_checkpoint( + model=model, + tokenizer=tokenizer, + stage_dir=full_model_dir, + stage_name=f"cycle_{cycle_idx}_full_model", + ppl_results=ppl_results, + ) + resume_meta = { + "base_model": getattr(args, "base_model_id", args.model), + "cycle": cycle_idx, + "output_dir": args.output_dir, + "layer_path": args.layer_path, + "rng_state": "rng_state.pt", + "loader_generators": "loader_generators.pt", + } + with open( + os.path.join(full_model_dir, "resume_info.json"), + "w", + encoding="utf-8", + ) as handle: + json.dump(resume_meta, handle, indent=2) + stage_meta["resume_info"] = "resume_info.json" + return stage_meta + + +def run_lora_phase( + model: torch.nn.Module, + tokenizer, + eval_datasets: List[str], + eval_configs: List[Optional[str]], + args: argparse.Namespace, + lora_loader: Optional[torch.utils.data.DataLoader] = None, + lora_meta: Optional[Dict[str, object]] = None, + eval_dataloaders: Optional[Dict[str, torch.utils.data.DataLoader]] = None, + cycle_idx: Optional[int] = None, + num_cycles: Optional[int] = None, +) -> List[Dict[str, object]]: + lora_eval_history: List[Dict[str, object]] = [] + if args.lora_epochs <= 0: + return lora_eval_history + if not has_post_fusion_data(lora_loader, lora_meta): + print("No post-fusion sequences built; skipping LoRA finetuning.") + return lora_eval_history + + lora_ce_finetune( + model=model, + dataloader=lora_loader, + eval_tokenizer=tokenizer, + eval_datasets=eval_datasets, + eval_configs=eval_configs, + eval_history=lora_eval_history, + args=args, + eval_dataloaders=eval_dataloaders, + progressive_cycle=cycle_idx, + progressive_total=num_cycles, + ) + return lora_eval_history + + +def run_progressive( + args: argparse.Namespace, + model: torch.nn.Module, + tokenizer, + prepared: PreparedData, +) -> None: + eval_datasets = prepared.eval_datasets + eval_configs = prepared.eval_configs + student_sharded = model_is_sharded(model) + if student_sharded: + print( + "[device] student model loaded across " + f"{', '.join(model_parameter_devices(model))}." + ) + + dataloader = prepared.calib_loader + num_sequences = prepared.calib_num_sequences + selection_args = args + fixed_region_start: Optional[int] = None + fixed_region_pairs: Optional[List[int]] = None + fixed_region_nodes: Optional[Set[int]] = None + fixed_region_score: Optional[float] = None + fixed_region_meta: Optional[Dict[str, object]] = None + active_layer_orig_ids: List[int] = list(range(len(list(find_layer_container(model, args.layer_path)[2])))) + manual_fixed_region_pairs = parse_fixed_region_pairs(getattr(args, "fixed_region_pairs", None)) + + if manual_fixed_region_pairs is not None: + fixed_region_pairs = list(manual_fixed_region_pairs) + fixed_region_nodes = set(fixed_region_pairs) + if fixed_region_pairs: + fixed_region_nodes.add(fixed_region_pairs[-1] + 1) + fixed_region_start = fixed_region_pairs[0] + fixed_region_score = None + fixed_region_meta = { + "source": "manual", + "region_pairs": list(fixed_region_pairs), + } + print( + "[auto] using manual fixed region pairs: " + f"{','.join(str(x) for x in fixed_region_pairs)}" + ) + selection_args = copy.copy(args) + selection_args.auto_metric = "dwce" + elif str(getattr(args, "auto_metric", "dwce")).strip().lower() == "region_cosine_dwce": + cosine_loader = ( + prepared.distill_loader + if prepared.distill_loader is not None + else prepared.calib_loader + ) + if cosine_loader is None: + raise SystemExit( + "--auto_metric=region_cosine_dwce requires calibration or distillation data." + ) + if not student_sharded: + model.to(args.device) + initial_exclude_pairs = set( + parse_exclude_pairs(args.exclude_pairs, max(len(active_layer_orig_ids) - 1, 0)) + ) + ( + fixed_region_start, + fixed_region_pairs, + fixed_region_score, + fixed_region_meta, + ) = select_best_region_by_cosine( + model, + list(find_layer_container(model, args.layer_path)[2]), + cosine_loader, + args, + start_index=0, + exclude_pairs=initial_exclude_pairs, + ) + print( + "[auto] fixed cosine region " + f"{fixed_region_start}-{fixed_region_pairs[-1] + 1} " + f"(pair-width={len(fixed_region_pairs)}, mean_cosine_distance={fixed_region_score:.6e})" + ) + dwce_candidate_pairs = fixed_region_meta.get("dwce_candidate_pairs") + if isinstance(dwce_candidate_pairs, list) and dwce_candidate_pairs: + fixed_region_pairs = [int(x) for x in dwce_candidate_pairs] + fixed_region_nodes = set(fixed_region_pairs) + fixed_region_nodes.add(fixed_region_pairs[-1] + 1) + print( + "[auto] remapped DWCE candidate pairs " + f"{fixed_region_pairs[0]}-{fixed_region_pairs[-1]} " + "(left-anchor interpretation)" + ) + selection_args = copy.copy(args) + selection_args.auto_metric = "dwce" + if not student_sharded: + model.cpu() + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + if not student_sharded: + model.to(args.device) + + os.makedirs(args.output_dir, exist_ok=True) + progressive_meta_path = os.path.join(args.output_dir, "progressive_metadata.json") + existing_meta: Dict[str, object] = {} + if args.resume_from_cycle > 0 and os.path.exists(progressive_meta_path): + with open(progressive_meta_path, "r", encoding="utf-8") as handle: + loaded_meta = json.load(handle) + if isinstance(loaded_meta, dict): + existing_meta = loaded_meta + + bootstrap_meta = { + "base_model": getattr(args, "base_model_id", args.model), + "num_progressive": args.num_progressive, + "layer_path": args.layer_path, + "resume_from_cycle": args.resume_from_cycle, + "save_full_model_cycles": sorted(args.full_model_save_cycles), + "student_device": args.device, + "student_model_device_map": getattr(args, "model_device_map", None), + "distill_teacher_device": args.distill_teacher_device or args.device, + "distill_teacher_device_map": getattr(args, "distill_teacher_device_map", None), + "distill_teacher_load_in_4bit": bool( + getattr(args, "distill_teacher_load_in_4bit", False) + ), + "kl_chunk_size": args.kl_chunk_size, + "cycles": ( + existing_meta.get("cycles", []) + if isinstance(existing_meta.get("cycles"), list) + else [] + ), + } + with open( + progressive_meta_path, + "w", + encoding="utf-8", + ) as handle: + json.dump(bootstrap_meta, handle, indent=2) + + pre_eval = None + if not args.skip_eval: + pre_eval = evaluate_ppl_model( + model, + tokenizer, + eval_datasets, + eval_configs, + args, + prepared_eval_dataloaders=prepared.eval_dataloaders, + ) + print("Pre-pruning perplexity:") + for dataset_name, ppl in pre_eval.items(): + print(f"{dataset_name}: {ppl:.4f}") + + post_merge_scale_chunks = None + if str(getattr(args, "post_merge_nll_scale", "none")) != "none": + post_merge_scale_chunks = build_scale_chunks(args, tokenizer) + print( + "[post-merge-nll-scale] prepared " + f"{len(post_merge_scale_chunks)} deterministic calibration chunks", + flush=True, + ) + + parent, name, container = find_layer_container(model, args.layer_path) + layers = list(container) + if args.num_progressive > (len(layers) - 1 + args.resume_from_cycle): + raise SystemExit( + f"--num_progressive ({args.num_progressive}) exceeds available pairs " + f"after resume offset ({len(layers) - 1 + args.resume_from_cycle})" + ) + + dwce_scores = None + dwce_meta = None + last_fused_idx = 0 + cycle_summaries: List[Dict[str, object]] = [] + existing_cycles = existing_meta.get("cycles", []) + if isinstance(existing_cycles, list): + for entry in existing_cycles: + if not isinstance(entry, dict): + continue + cycle_value = entry.get("cycle") + if isinstance(cycle_value, int) and cycle_value <= args.resume_from_cycle: + cycle_summaries.append(entry) + + comm_enabled = bool(getattr(args, "comm_enabled", False)) + comm_teacher_model = None + comm_teacher_cycle: Optional[int] = None + teacher_device = args.distill_teacher_device or args.device + teacher_cache_disabled_reason = None + if bool(getattr(args, "disable_previous_cycle_teacher_cache", False)): + teacher_cache_disabled_reason = "explicit flag" + elif student_sharded: + teacher_cache_disabled_reason = "sharded student model" + elif getattr(args, "distill_teacher_device_map", None): + teacher_cache_disabled_reason = "teacher device_map" + elif str(teacher_device).strip().lower() == "cpu": + teacher_cache_disabled_reason = "teacher_device=cpu" + if teacher_cache_disabled_reason is not None: + print( + "[teacher] previous-cycle in-memory cache disabled " + f"({teacher_cache_disabled_reason}); reloading from disk/base as needed." + ) + previous_cycle_teacher_model = None + previous_cycle_teacher_cycle: Optional[int] = None + + def _release_comm_teacher() -> None: + nonlocal comm_teacher_model, comm_teacher_cycle + if comm_teacher_model is not None: + del comm_teacher_model + comm_teacher_model = None + comm_teacher_cycle = None + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + def _release_previous_cycle_teacher() -> None: + nonlocal previous_cycle_teacher_model, previous_cycle_teacher_cycle + if previous_cycle_teacher_model is not None: + del previous_cycle_teacher_model + previous_cycle_teacher_model = None + previous_cycle_teacher_cycle = None + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + def _snapshot_previous_cycle_teacher(cycle_idx: int) -> None: + nonlocal previous_cycle_teacher_model, previous_cycle_teacher_cycle + if teacher_cache_disabled_reason is not None: + return + _release_previous_cycle_teacher() + previous_cycle_teacher_model = copy.deepcopy(model) + previous_cycle_teacher_model.to(teacher_device) + previous_cycle_teacher_model.eval() + previous_cycle_teacher_cycle = cycle_idx + + def _get_previous_cycle_teacher( + cycle_idx: int, + ) -> Tuple[Optional[torch.nn.Module], str, Optional[int]]: + prev_cycle = cycle_idx - 1 + if prev_cycle <= 0: + return None, "base_model", 0 + if ( + previous_cycle_teacher_model is not None + and previous_cycle_teacher_cycle == prev_cycle + ): + return previous_cycle_teacher_model, "previous_cycle_memory", prev_cycle + teacher_model_path_for_map = os.path.join( + args.output_dir, + f"cycle_{prev_cycle}", + "full_model", + ) + if not os.path.isdir(teacher_model_path_for_map): + teacher_model_path_for_map = getattr(args, "base_model_id", args.model) + teacher_kwargs, teacher_needs_manual_to = _teacher_load_kwargs( + args, + teacher_device, + teacher_model_path_for_map, + ) + teacher_model = load_progressive_model( + getattr(args, "base_model_id", args.model), + args.output_dir, + cycle=prev_cycle, + device=teacher_device if teacher_needs_manual_to else None, + dtype=args.dtype, + trust_remote_code=args.trust_remote_code, + layer_path=args.layer_path, + **teacher_kwargs, + ) + teacher_model.eval() + return teacher_model, "previous_cycle_disk", prev_cycle + + def _get_comm_teacher(cycle_idx: int) -> Tuple[Optional[torch.nn.Module], str, Optional[int]]: + nonlocal comm_teacher_model, comm_teacher_cycle + if not comm_enabled: + return None, "disabled", None + + source = str(getattr(args, "redistrib_teacher_source", "base_model")) + if source == "base_model": + if comm_teacher_model is None: + print( + "[comm] Loading fixed base teacher for anchor loss " + f"(device={teacher_device})." + ) + comm_teacher_model = _load_teacher_causal_lm( + getattr(args, "base_model_id", args.model), + args, + teacher_device, + ) + comm_teacher_cycle = 0 + return comm_teacher_model, "base_model", 0 + + prev_cycle = cycle_idx - 1 + if prev_cycle <= 0: + if comm_teacher_model is None or comm_teacher_cycle != 0: + _release_comm_teacher() + print( + "[comm] --redistrib_teacher_source=previous_cycle but cycle 1 " + "has no prior checkpoint; using base teacher." + ) + comm_teacher_model = _load_teacher_causal_lm( + getattr(args, "base_model_id", args.model), + args, + teacher_device, + ) + comm_teacher_cycle = 0 + return comm_teacher_model, "base_model", 0 + + if ( + previous_cycle_teacher_model is not None + and previous_cycle_teacher_cycle == prev_cycle + ): + if comm_teacher_model is not previous_cycle_teacher_model: + _release_comm_teacher() + comm_teacher_model = previous_cycle_teacher_model + comm_teacher_cycle = prev_cycle + return comm_teacher_model, "previous_cycle_memory", prev_cycle + + if comm_teacher_model is None or comm_teacher_cycle != prev_cycle: + _release_comm_teacher() + print( + "[comm] Loading teacher from previous cycle " + f"{prev_cycle} (device={teacher_device})." + ) + teacher_model_path_for_map = os.path.join( + args.output_dir, + f"cycle_{prev_cycle}", + "full_model", + ) + if not os.path.isdir(teacher_model_path_for_map): + teacher_model_path_for_map = getattr(args, "base_model_id", args.model) + teacher_kwargs, teacher_needs_manual_to = _teacher_load_kwargs( + args, + teacher_device, + teacher_model_path_for_map, + ) + comm_teacher_model = load_progressive_model( + getattr(args, "base_model_id", args.model), + args.output_dir, + cycle=prev_cycle, + device=teacher_device if teacher_needs_manual_to else None, + dtype=args.dtype, + trust_remote_code=args.trust_remote_code, + layer_path=args.layer_path, + **teacher_kwargs, + ) + comm_teacher_model.eval() + comm_teacher_cycle = prev_cycle + return comm_teacher_model, "previous_cycle_disk", prev_cycle + + def _comm_gap_dataloader(): + if prepared.eval_dataloaders: + first_key = next(iter(prepared.eval_dataloaders)) + return prepared.eval_dataloaders[first_key] + return prepared.calib_loader + + def _comm_train_dataloader(): + source = str(getattr(args, "comm_data_source", "auto")).strip().lower() + if source == "calib": + return prepared.calib_loader, "calib" + if source == "distill": + return prepared.distill_loader, "distill" + if prepared.distill_loader is not None: + return prepared.distill_loader, "distill" + return prepared.calib_loader, "calib" + + if args.resume_from_cycle > 0: + _snapshot_previous_cycle_teacher(args.resume_from_cycle) + + start_cycle = args.resume_from_cycle + 1 + for cycle_idx in range(start_cycle, args.num_progressive + 1): + cycle_timing_start = _timing_start() + component_seconds: Dict[str, float] = {} + + def _record_component(name: str, start: float) -> float: + elapsed = _timing_elapsed(start) + component_seconds[name] = component_seconds.get(name, 0.0) + elapsed + return elapsed + + print(f"[progressive] Cycle {cycle_idx}/{args.num_progressive}") + run_comm = comm_enabled and ( + cycle_idx > 1 or bool(getattr(args, "comm_include_cycle1", False)) + ) + comm_stats: Dict[str, object] = {"enabled": False} + comm_post_eval = None + comm_teacher_model_loaded = None + if run_comm: + # Preconditioning updates model weights, so DWCE reuse is unreliable. + start_index = 0 + reuse_scores = None + else: + start_index = last_fused_idx if cycle_idx > 1 else 0 + reuse_scores = dwce_scores + exclude_pairs = set(parse_exclude_pairs(args.exclude_pairs, max(len(layers) - 1, 0))) + if fixed_region_nodes is not None: + for idx in range(max(len(layers) - 1, 0)): + left_orig = active_layer_orig_ids[idx] + right_orig = active_layer_orig_ids[idx + 1] + if left_orig not in fixed_region_nodes or right_orig not in fixed_region_nodes: + exclude_pairs.add(idx) + selection_timing_start = _timing_start() + layer_idx, dwce_scores, dwce_meta = resolve_layer_idx( + selection_args, + model, + layers, + dataloader, + prepared.distill_loader, + reuse_scores, + start_index, + exclude_pairs, + ) + _record_component("selection", selection_timing_start) + if fixed_region_meta is not None: + dwce_meta = dict(dwce_meta or {}) + dwce_meta["fixed_cosine_region"] = { + "region_start": fixed_region_start, + "region_pairs": list( + fixed_region_meta.get("cosine_region_pairs", fixed_region_pairs or []) + ), + "dwce_candidate_pairs": list(fixed_region_pairs or []), + "region_score": fixed_region_score, + "region_meta": fixed_region_meta, + } + + rebase_timing_start = _timing_start() if run_comm else None + component_seconds["rebase"] = 0.0 + if run_comm: + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + dwce_scores_pre_comm = dwce_scores + comm_dataloader, comm_data_source = _comm_train_dataloader() + if comm_dataloader is None: + requested_source = str( + getattr(args, "comm_data_source", "auto") + ).strip().lower() + if requested_source in {"calib", "distill"}: + raise RuntimeError( + "comm_enabled but the requested " + f"--comm_data_source={requested_source} loader was not built." + ) + print( + "[comm] Enabled but no distillation/calibration sequences were built; skipping." + ) + else: + ( + comm_teacher_model_loaded, + comm_teacher_source, + comm_teacher_cycle_idx, + ) = _get_comm_teacher(cycle_idx) + if comm_teacher_model_loaded is None: + raise RuntimeError("comm_enabled but teacher model was not loaded.") + comm_stats = commutator_precondition( + student_model=model, + student_layers=layers, + teacher_model=comm_teacher_model_loaded, + dataloader=comm_dataloader, + dwce_scores=dwce_scores_pre_comm, + exclude_pairs=exclude_pairs, + args=args, + progressive_cycle=cycle_idx, + progressive_total=args.num_progressive, + measure_dataloader=( + _comm_gap_dataloader() + if bool(getattr(args, "comm_measure_gap", False)) + else None + ), + ) + if comm_stats.get("enabled"): + comm_stats["teacher_source"] = comm_teacher_source + comm_stats["teacher_cycle"] = comm_teacher_cycle_idx + comm_stats["data_source"] = comm_data_source + comm_stats["dwce_scores_pre"] = dwce_scores_pre_comm + print( + "[comm] Done:" + f" opt_steps={comm_stats.get('opt_steps')}" + f" lr={comm_stats.get('lr')}" + ) + if not args.skip_eval: + comm_post_eval = evaluate_ppl_model( + model, + tokenizer, + eval_datasets, + eval_configs, + args, + prepared_eval_dataloaders=prepared.eval_dataloaders, + ) + comm_stats["post_ppl"] = comm_post_eval + print(f"[progressive] Cycle {cycle_idx} post-comm perplexity:") + for dataset_name, ppl in comm_post_eval.items(): + print(f"{dataset_name}: {ppl:.4f}") + + if bool(getattr(args, "comm_skip_post_reselect", False)): + comm_stats["post_selection_recomputed"] = False + comm_stats["selected_layer_post"] = int(layer_idx) + print( + "[comm] Keeping pre-comm DWCE pair selection for fusion." + ) + else: + print( + "[comm] Recomputing DWCE after preconditioning for fusion selection." + ) + layer_idx, dwce_scores, dwce_meta = resolve_layer_idx( + selection_args, + model, + layers, + dataloader, + prepared.distill_loader, + None, + 0, + exclude_pairs, + ) + comm_stats["post_selection_recomputed"] = True + comm_stats["selected_layer_post"] = int(layer_idx) + if ( + run_comm + and comm_teacher_model_loaded is not None + and teacher_cache_disabled_reason is not None + ): + reparam_teacher_cycle_for_release = cycle_idx - 1 + if reparam_teacher_cycle_for_release <= 0: + comm_teacher_matches_reparam = ( + comm_teacher_source == "base_model" + and comm_teacher_cycle_idx == 0 + ) + else: + comm_teacher_matches_reparam = ( + comm_teacher_source in {"previous_cycle_memory", "previous_cycle_disk"} + and comm_teacher_cycle_idx == reparam_teacher_cycle_for_release + ) + keep_comm_teacher_for_reparam = ( + (not args.skip_distill) + and has_post_fusion_data(prepared.distill_loader, prepared.distill_meta) + and (not bool(getattr(args, "reparam_skip_train", False))) + and float(args.distill_epochs) > 0.0 + and comm_teacher_matches_reparam + ) + if keep_comm_teacher_for_reparam: + print( + "[comm] Keeping comm teacher for FiT/reparam phase " + f"(source={comm_teacher_source}, cycle={comm_teacher_cycle_idx})." + ) + else: + print( + "[comm] Releasing comm teacher after comm phase " + f"({teacher_cache_disabled_reason})." + ) + comm_teacher_model_loaded = None + _release_comm_teacher() + if rebase_timing_start is not None: + _record_component("rebase", rebase_timing_start) + + if layer_idx < 0 or layer_idx >= len(layers) - 1: + raise SystemExit("--layer must be in [0, num_layers-2]") + + num_layers_before = len(layers) + layer_a = layers[layer_idx] + layer_b = layers[layer_idx + 1] + + norm1_state = None + norm2_state = None + norm1, norm2, norm_names = get_norm_pair(layer_a) + if norm1 is not None: + norm1_state = clone_state_dict(norm1) + if norm2 is not None: + norm2_state = clone_state_dict(norm2) + + attn_a = find_attention_module(layer_a) + attn_b = find_attention_module(layer_b) + hidden_size = getattr(model.config, "hidden_size", None) + if hidden_size is None: + hidden_size = getattr(model.config, "n_embd", None) + if hidden_size is None: + raise SystemExit("Model config missing hidden_size/n_embd") + + no_head_permute_merge = bool( + getattr(args, "no_head_permute_merge", False) + or getattr(args, "no_head_permute", False) + ) + if no_head_permute_merge: + print("[fuse] Head permutation disabled; merging with original head order.") + else: + head_timing_start = _timing_start() + mean_a, mean_b, num_heads, num_kv_heads, head_dim = compute_head_means( + model, + attn_a, + attn_b, + dataloader, + args.device, + hidden_size, + ) + + perm = build_head_permutation( + mean_a, + mean_b, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + eps=args.eps, + ) + permute_attention_heads( + attn_b, perm, num_heads, num_kv_heads, head_dim=head_dim + ) + _record_component("head_permutation", head_timing_start) + + source_feature_transport_stats = None + contextual_fisher_stats = None + component_seconds["source_feature_transport"] = 0.0 + if bool(getattr(args, "source_feature_transport", False)): + transport_timing_start = _timing_start() + source_feature_transport_stats = fit_and_apply_source_feature_transport( + model=model, + layer_a=layer_a, + layer_b=layer_b, + dataloader=dataloader, + device=args.device, + ridge_relative=float(args.source_feature_transport_ridge_relative), + max_batches=int(args.source_feature_transport_max_batches), + ) + _record_component("source_feature_transport", transport_timing_start) + print( + "[source-feature-transport] " + f"cycle={cycle_idx} batches={source_feature_transport_stats['batches']} " + f"tokens={source_feature_transport_stats['tokens']} " + f"nmse={source_feature_transport_stats['calibration_nmse']:.6f} " + f"cosine={source_feature_transport_stats['calibration_cosine']:.6f} " + f"norm_ratio={source_feature_transport_stats['calibration_norm_ratio']:.6f}", + flush=True, + ) + + fisher_timing_start = _timing_start() + if bool(getattr(args, "source_feature_transport", False)): + ( + fisher_sums, + num_batches, + param_numels, + contextual_fisher_stats, + ) = compute_same_anchor_contextual_fisher( + model=model, + parent=parent, + layer_attr=name, + container=container, + layer_idx=layer_idx, + layer_a=layer_a, + layer_b=layer_b, + dataloader=dataloader, + device=args.device, + fisher_mode=args.fisher_mode, + ) + source_feature_transport_stats["fisher_order"] = ( + "source transport first; same-anchor post-pruning contextual Fisher second" + ) + print( + "[source-feature-transport] contextual-fisher " + f"cycle={cycle_idx} " + f"anchor_nll={contextual_fisher_stats['anchor_candidate_mean_nll']:.6f} " + f"source_nll={contextual_fisher_stats['transported_source_candidate_mean_nll']:.6f}", + flush=True, + ) + else: + fisher_sums, num_batches, param_numels = compute_fisher( + model, + layer_a, + layer_b, + dataloader, + fisher_mode=args.fisher_mode, + device=args.device, + ) + _record_component("fisher", fisher_timing_start) + + fit_timing_start = _timing_start() + distill_ready = has_post_fusion_data( + prepared.distill_loader, prepared.distill_meta + ) + teacher_cycle = cycle_idx - 1 + teacher_source = "previous_cycle" if teacher_cycle > 0 else "base_model" + + merge_method = "fisher" + distill_method = str(getattr(args, "distill_method", "reparam")) + reparam_stats: Optional[Dict[str, object]] = None + reparam_gate_summary: Optional[Dict[str, object]] = None + merge_only_eval = None + if bool(getattr(args, "eval_merge_only_ppl", False)) and not args.skip_eval: + print( + f"[progressive] Cycle {cycle_idx} temporary merge-only perplexity " + f"for pair {layer_idx}-{layer_idx + 1}:" + ) + merge_only_eval = evaluate_temporary_fisher_merge_ppl( + model=model, + tokenizer=tokenizer, + parent=parent, + layer_attr=name, + container=container, + layer_idx=layer_idx, + layer_a=layer_a, + layer_b=layer_b, + fisher_sums=fisher_sums, + num_batches=num_batches, + param_numels=param_numels, + norm_policy=args.norm_policy, + norm1_state=norm1_state, + norm2_state=norm2_state, + norm_names=norm_names, + eval_datasets=eval_datasets, + eval_configs=eval_configs, + args=args, + prepared_eval_dataloaders=prepared.eval_dataloaders, + ) + for dataset_name, ppl in merge_only_eval.items(): + print(f"{dataset_name}: {ppl:.4f}") + + reparam_teacher_cycle = cycle_idx - 1 + reparam_teacher_source = ( + "previous_cycle" if reparam_teacher_cycle > 0 else "base_model" + ) + needs_teacher_for_reparam = ( + (not args.skip_distill) + and distill_ready + and (not bool(getattr(args, "reparam_skip_train", False))) + and float(args.distill_epochs) > 0.0 + ) + def _comm_teacher_matches_reparam_teacher() -> bool: + if comm_teacher_model_loaded is None: + return False + if reparam_teacher_cycle <= 0: + return comm_teacher_source == "base_model" and comm_teacher_cycle_idx == 0 + return ( + comm_teacher_source in {"previous_cycle_memory", "previous_cycle_disk"} + and comm_teacher_cycle_idx == reparam_teacher_cycle + ) + + can_reuse_comm_teacher_for_reparam = ( + needs_teacher_for_reparam and _comm_teacher_matches_reparam_teacher() + ) + teacher_model = None + teacher_parent = None + teacher_layer_attr = None + teacher_layers: Optional[List[torch.nn.Module]] = None + teacher_from_cache = False + reused_comm_teacher_for_reparam = False + if needs_teacher_for_reparam: + if can_reuse_comm_teacher_for_reparam: + teacher_model = comm_teacher_model_loaded + teacher_source = comm_teacher_source + teacher_cycle = comm_teacher_cycle_idx + teacher_from_cache = True + reused_comm_teacher_for_reparam = True + print( + "[reparam] Reusing comm teacher for FiT/reparam phase " + f"(source={teacher_source}, cycle={teacher_cycle})." + ) + else: + teacher_model, teacher_source, teacher_cycle = _get_previous_cycle_teacher( + cycle_idx + ) + teacher_from_cache = ( + teacher_source == "previous_cycle_memory" + and teacher_model is previous_cycle_teacher_model + ) + if teacher_model is None: + teacher_model = _load_teacher_causal_lm( + getattr(args, "base_model_id", args.model), + args, + teacher_device, + ) + teacher_source = "base_model" + teacher_cycle = 0 + teacher_parent, teacher_layer_attr, teacher_container = find_layer_container( + teacher_model, args.layer_path + ) + teacher_layers = list(teacher_container) + + do_reparam = ( + (not args.skip_distill) + and distill_ready + and prepared.distill_loader is not None + ) + if (not args.skip_distill) and not do_reparam: + print("[reparam] No distillation sequences built; skipping reparam distill.") + distill_post_eval = None + if do_reparam: + lambda_source = "fisher_prior" + fisher_prior_input_summary = summarize_fisher_gate_prior_inputs( + layer_a=layer_a, + layer_b=layer_b, + fisher_a=fisher_sums[0], + fisher_b=fisher_sums[1], + num_batches=num_batches, + numels_a=param_numels[0], + numels_b=param_numels[1], + fisher_mode=args.fisher_mode, + eps=float(args.eps), + param_subset=args.reparam_param_subset, + ) + print( + "[reparam] fisher_prior_inputs " + f"fa_mean={fisher_prior_input_summary.get('fa_mean')} " + f"fb_mean={fisher_prior_input_summary.get('fb_mean')} " + f"active_ratio={fisher_prior_input_summary.get('active_ratio')} " + f"rel_contrast={fisher_prior_input_summary.get('mean_relative_contrast')} " + f"prior_mean={fisher_prior_input_summary.get('prior_mean')} " + f"prior_active_mean={fisher_prior_input_summary.get('prior_active_mean')}" + ) + reparam_gate_targets: Dict[str, object] = compute_fisher_gate_priors( + layer_a=layer_a, + layer_b=layer_b, + fisher_a=fisher_sums[0], + fisher_b=fisher_sums[1], + num_batches=num_batches, + numels_a=param_numels[0], + numels_b=param_numels[1], + fisher_mode=args.fisher_mode, + eps=float(args.eps), + ) + if not reparam_gate_targets: + raise SystemExit("[reparam] No mergeable parameters found; cannot continue.") + reparam_train_enabled = ( + (not bool(getattr(args, "reparam_skip_train", False))) + and float(args.distill_epochs) > 0.0 + ) + if reparam_train_enabled and (teacher_model is None or teacher_layers is None): + raise SystemExit("--distill_method reparam requires a teacher model.") + reparam_action = ( + "training U + gates" + if reparam_train_enabled + else "materializing Fisher-prior U + gates without training" + ) + print( + f"[reparam] Cycle {cycle_idx}: {reparam_action} for pair " + f"{layer_idx}-{layer_idx + 1} (epochs={args.distill_epochs}, " + f"skip_train={bool(getattr(args, 'reparam_skip_train', False))}, " + f"hidden_mse_w={args.distill_hidden_mse_weight}, " + f"attn_mse_w={args.distill_attn_mse_weight}, " + f"mlp_mse_w={args.distill_mlp_mse_weight}, " + f"eta={args.reparam_eta}, gamma={args.reparam_gamma}, " + f"attn_reg_scale={args.reparam_attn_reg_scale}, " + f"mlp_reg_scale={args.reparam_mlp_reg_scale}, " + f"param_subset={args.reparam_param_subset}, " + f"fisher_init_subset={args.reparam_fisher_init_subset}, " + f"train_subset={args.reparam_train_subset}, " + f"lambda_init={lambda_source})." + ) + merged, final_gates, reparam_stats = distill_reparam_merge( + student_model=model, + student_parent=parent, + student_layer_attr=name, + student_layers=layers, + teacher_model=teacher_model, + teacher_parent=teacher_parent, + teacher_layer_attr=teacher_layer_attr, + teacher_layers=teacher_layers, + layer_idx=layer_idx, + gate_lambdas=reparam_gate_targets, + dataloader=prepared.distill_loader, + args=args, + progressive_cycle=cycle_idx, + progressive_total=args.num_progressive, + ) + reparam_gate_summary = summarize_gate_lambdas(final_gates) + merge_method = "reparam" + if reparam_stats is not None: + reparam_stats["lambda_init"] = lambda_source + reparam_stats["fisher_prior_input_summary"] = fisher_prior_input_summary + else: + merged = merge_layers( + layer_a, + layer_b, + fisher_sums[0], + fisher_sums[1], + num_batches, + param_numels[0], + param_numels[1], + fisher_mode=args.fisher_mode, + eps=args.eps, + ) + + apply_norm_policy( + layer_a, + args.norm_policy, + norm1_state, + norm2_state, + norm_names, + ) + if teacher_model is not None and not teacher_from_cache: + del teacher_model + teacher_model = None + teacher_parent = None + teacher_layer_attr = None + teacher_layers = None + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + if reused_comm_teacher_for_reparam: + teacher_model = None + teacher_parent = None + teacher_layer_attr = None + teacher_layers = None + comm_teacher_model_loaded = None + _release_comm_teacher() + _record_component("fit", fit_timing_start) + + materialize_timing_start = _timing_start() + new_container = drop_layer(container, layer_idx + 1) + setattr(parent, name, new_container) + decrement_config(model.config) + layers = list(new_container) + del active_layer_orig_ids[layer_idx + 1] + + post_merge_nll_scale_stats = None + if post_merge_scale_chunks is not None: + post_merge_nll_scale_stats = fit_and_fold_post_merge_scales( + model, + layers[layer_idx], + post_merge_scale_chunks, + args, + cycle=cycle_idx, + ) + + lora_post_eval = None + if (not args.skip_eval) and (not args.skip_distill) and do_reparam: + distill_post_eval = evaluate_ppl_model( + model, + tokenizer, + eval_datasets, + eval_configs, + args, + prepared_eval_dataloaders=prepared.eval_dataloaders, + ) + print(f"[progressive] Cycle {cycle_idx} post-distill perplexity:") + for dataset_name, ppl in distill_post_eval.items(): + print(f"{dataset_name}: {ppl:.4f}") + + post_eval = None + if not args.skip_eval: + if distill_post_eval is not None: + post_eval = distill_post_eval + else: + post_eval = evaluate_ppl_model( + model, + tokenizer, + eval_datasets, + eval_configs, + args, + prepared_eval_dataloaders=prepared.eval_dataloaders, + ) + print(f"[progressive] Cycle {cycle_idx} perplexity:") + for dataset_name, ppl in post_eval.items(): + print(f"{dataset_name}: {ppl:.4f}") + + cycle_dir = os.path.join(args.output_dir, f"cycle_{cycle_idx}") + os.makedirs(cycle_dir, exist_ok=True) + fused_layer_file = None + if args.profile_no_save: + print( + f"[profile] Skipping fused layer weight save for cycle {cycle_idx}." + ) + else: + fused_layer_file = "fused_layer.pt" + fused_layer_path = os.path.join(cycle_dir, fused_layer_file) + torch.save(layers[layer_idx].state_dict(), fused_layer_path) + _record_component("materialize_save_layer", materialize_timing_start) + + cycle_meta: Dict[str, object] = { + "cycle": cycle_idx, + "layer_merged": layer_idx, + "num_layers_before": num_layers_before, + "num_layers_after": num_layers_before - 1, + "fused_layer_state": fused_layer_file, + "profile_no_save": bool(args.profile_no_save), + "dwce_score": dwce_scores[layer_idx] if dwce_scores else None, + "dwce_scores": dwce_scores, + "dwce_meta": dwce_meta, + "fisher_num_batches": num_batches, + "merge_method": merge_method, + "merged_params": merged, + "num_sequences": num_sequences, + "teacher_source": teacher_source, + "teacher_cycle": teacher_cycle, + "eval": { + "datasets": eval_datasets, + "configs": eval_configs, + "split": args.eval_split, + "num_samples": args.eval_num_samples, + "seq_len": args.eval_seq_len, + "post_ppl": post_eval, + "merge_only_ppl": merge_only_eval, + }, + "comm": comm_stats, + "distill": { + "enabled": not args.skip_distill, + "method": distill_method, + "calib_samples": args.distill_calib_samples, + "inst_samples": args.distill_inst_samples, + "seq_len": args.distill_seq_len, + "batch_size": args.distill_batch_size, + "epochs": args.distill_epochs, + "lr": args.distill_lr, + "kl_weight": args.distill_kl_weight, + "kl_temp": args.distill_kl_temp, + "kl_chunk_size": args.kl_chunk_size, + "hidden_mse_weight": args.distill_hidden_mse_weight, + "attn_mse_weight": args.distill_attn_mse_weight, + "mlp_mse_weight": args.distill_mlp_mse_weight, + "reparam_eta": args.reparam_eta, + "reparam_gamma": args.reparam_gamma, + "reparam_attn_reg_scale": args.reparam_attn_reg_scale, + "reparam_mlp_reg_scale": args.reparam_mlp_reg_scale, + "reparam_param_subset": args.reparam_param_subset, + "reparam_train_subset": args.reparam_train_subset, + "reparam_skip_train": bool(getattr(args, "reparam_skip_train", False)), + "reparam_stats": reparam_stats, + "reparam_gate_summary": reparam_gate_summary, + "post_merge_nll_scale": post_merge_nll_scale_stats, + "merge_only_ppl": merge_only_eval, + "post_ppl": distill_post_eval, + "weight_decay": args.distill_weight_decay, + "max_grad_norm": args.distill_max_grad_norm, + "grad_accum_steps": args.distill_grad_accum_steps, + "instruction_dataset": args.instruction_dataset, + "instruction_config": args.instruction_config, + "instruction_split": args.instruction_split, + }, + "lora": { + "enabled": args.lora_epochs > 0, + "seq_len": args.distill_seq_len, + "batch_size": args.distill_batch_size, + "epochs": args.lora_epochs, + "rank": args.lora_rank, + "alpha": args.lora_alpha, + "dropout": args.lora_dropout, + "target_modules": args.lora_target_modules, + "respect_exclude_pairs": args.lora_respect_exclude_pairs, + "kl_enabled": args.lora_kl_enabled, + "kl_weight": args.lora_kl_weight, + "kl_temp": args.lora_kl_temp, + "post_ppl": lora_post_eval, + "lr": args.lora_lr, + "weight_decay": args.lora_weight_decay, + "max_grad_norm": args.lora_max_grad_norm, + "grad_accum_steps": args.lora_grad_accum_steps, + "log_steps": args.lora_log_steps, + "eval_every": args.lora_eval_every, + "eval_max_batches": args.lora_eval_max_batches, + }, + "norm_policy": args.norm_policy, + "source_feature_transport": source_feature_transport_stats, + "same_anchor_contextual_fisher": contextual_fisher_stats, + } + + saved_full_model_dir = None + if cycle_idx in args.full_model_save_cycles: + cycle_meta["full_model_saved"] = True + cycle_meta["full_model"] = save_cycle_full_model( + model=model, + tokenizer=tokenizer, + cycle_dir=cycle_dir, + cycle_idx=cycle_idx, + args=args, + ppl_results=post_eval, + ) + saved_full_model_dir = os.path.join(cycle_dir, "full_model") + else: + cycle_meta["full_model_saved"] = False + + metadata_timing_start = _timing_start() + cycle_meta_path = os.path.join(cycle_dir, "cycle_metadata.json") + with open( + cycle_meta_path, + "w", + encoding="utf-8", + ) as handle: + json.dump(cycle_meta, handle, indent=2) + _record_component("metadata_write", metadata_timing_start) + + cycle_summary: Dict[str, object] = { + "cycle": cycle_idx, + "layer_merged": layer_idx, + "dwce_score": dwce_scores[layer_idx] if dwce_scores else None, + "comm_post_ppl": comm_post_eval, + "merge_only_ppl": merge_only_eval, + "distill_post_ppl": distill_post_eval, + "lora_post_ppl": lora_post_eval, + "post_ppl": post_eval, + "cycle_dir": f"cycle_{cycle_idx}", + } + cycle_summaries.append(cycle_summary) + + last_fused_idx = layer_idx + snapshot_timing_start = _timing_start() + _snapshot_previous_cycle_teacher(cycle_idx) + _record_component("teacher_snapshot", snapshot_timing_start) + + cleanup_timing_start = _timing_start() + parent, name, container = find_layer_container(model, args.layer_path) + layers = list(container) + if dwce_scores: + dwce_scores = dwce_scores[: max(len(layers) - 1, 0)] + + # Encourage allocator to release cached blocks between cycles. + if torch.cuda.is_available(): + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + gc.collect() + _record_component("cycle_cleanup", cleanup_timing_start) + + if saved_full_model_dir is not None: + save_rng_state(os.path.join(saved_full_model_dir, "rng_state.pt")) + save_loader_generator_state( + saved_full_model_dir, + distill_generator=prepared.distill_generator, + lora_generator=prepared.lora_generator, + ) + + component_seconds["cycle_total"] = _timing_elapsed(cycle_timing_start) + timing_meta = _timing_summary(component_seconds) + cycle_meta["timing"] = timing_meta + cycle_summary["timing"] = timing_meta + with open(cycle_meta_path, "w", encoding="utf-8") as handle: + json.dump(cycle_meta, handle, indent=2) + _print_cycle_timing(cycle_idx, timing_meta) + + _release_comm_teacher() + _release_previous_cycle_teacher() + + final_pre_lora_eval = cycle_summaries[-1]["post_ppl"] if cycle_summaries else None + final_pre_lora_dir = f"{os.path.abspath(args.output_dir.rstrip(os.sep))}_final_pre_lora_hf" + if args.profile_no_save or args.skip_final_model_save: + skip_reason = "profile_no_save" if args.profile_no_save else "compact_only" + final_pre_lora_meta = { + "stage": "final_pre_lora", + "path": final_pre_lora_dir, + "skipped": True, + "reason": skip_reason, + "post_ppl": final_pre_lora_eval, + } + print(f"[save] Skipping final pre-LoRA weight save ({skip_reason}).") + else: + final_pre_lora_meta = save_stage_checkpoint( + model=model, + tokenizer=tokenizer, + stage_dir=final_pre_lora_dir, + stage_name="final_pre_lora", + ppl_results=final_pre_lora_eval, + ) + + # Optional final LoRA finetune after all pruning cycles. + lora_eval_history: List[Dict[str, object]] = [] + lora_post_eval = None + lora_ready = has_post_fusion_data(prepared.lora_loader, prepared.lora_meta) + if args.lora_epochs > 0: + if not lora_ready: + print("No post-fusion sequences built; skipping LoRA finetuning.") + else: + print( + f"[progressive] Running final LoRA finetuning (epochs={args.lora_epochs})." + ) + lora_eval_history = run_lora_phase( + model=model, + tokenizer=tokenizer, + eval_datasets=eval_datasets, + eval_configs=eval_configs, + args=args, + lora_loader=prepared.lora_loader, + lora_meta=prepared.lora_meta, + eval_dataloaders=prepared.eval_dataloaders, + cycle_idx=args.num_progressive, + num_cycles=args.num_progressive, + ) + + if not args.skip_eval: + lora_post_eval = evaluate_ppl_model( + model, + tokenizer, + eval_datasets, + eval_configs, + args, + prepared_eval_dataloaders=prepared.eval_dataloaders, + ) + print("[progressive] Post-LoRA perplexity:") + for dataset_name, ppl in lora_post_eval.items(): + print(f"{dataset_name}: {ppl:.4f}") + + # Update final cycle metadata and summary with the post-LoRA PPL. + if cycle_summaries: + cycle_summaries[-1]["lora_post_ppl"] = lora_post_eval + if lora_post_eval is not None: + cycle_summaries[-1]["post_ppl"] = lora_post_eval + + final_cycle_dir = os.path.join( + args.output_dir, f"cycle_{args.num_progressive}" + ) + final_cycle_meta_path = os.path.join(final_cycle_dir, "cycle_metadata.json") + if os.path.exists(final_cycle_meta_path): + with open(final_cycle_meta_path, "r", encoding="utf-8") as handle: + final_cycle_meta = json.load(handle) + + lora_meta_entry = final_cycle_meta.get("lora") + if not isinstance(lora_meta_entry, dict): + lora_meta_entry = {} + final_cycle_meta["lora"] = lora_meta_entry + lora_meta_entry["ran"] = True + lora_meta_entry["post_ppl"] = lora_post_eval + if lora_post_eval is not None and isinstance( + final_cycle_meta.get("eval"), dict + ): + final_cycle_meta["eval"]["post_ppl"] = lora_post_eval + + if lora_eval_history: + lora_path = os.path.join(final_cycle_dir, "ppl_over_lora.json") + with open(lora_path, "w", encoding="utf-8") as handle: + json.dump(lora_eval_history, handle, indent=2) + lora_meta_entry["ppl_over_lora"] = "ppl_over_lora.json" + + with open(final_cycle_meta_path, "w", encoding="utf-8") as handle: + json.dump(final_cycle_meta, handle, indent=2) + + os.makedirs(args.output_dir, exist_ok=True) + if args.profile_no_save or args.skip_final_model_save: + skip_reason = "profile_no_save" if args.profile_no_save else "compact_only" + final_post_lora_meta = { + "stage": "final_post_lora" if lora_post_eval is not None else "final_model", + "path": args.output_dir, + "skipped": True, + "reason": skip_reason, + "post_ppl": lora_post_eval, + } + print(f"[save] Skipping final model weight save ({skip_reason}).") + else: + final_post_lora_meta = save_stage_checkpoint( + model=model, + tokenizer=tokenizer, + stage_dir=args.output_dir, + stage_name="final_post_lora" if lora_post_eval is not None else "final_model", + ppl_results=lora_post_eval, + ) + + progressive_meta = { + "base_model": getattr(args, "base_model_id", args.model), + "num_progressive": args.num_progressive, + "layer_path": args.layer_path, + "resume_from_cycle": args.resume_from_cycle, + "profile_no_save": bool(args.profile_no_save), + "skip_final_model_save": bool(args.skip_final_model_save), + "save_full_model_cycles": sorted(args.full_model_save_cycles), + "num_sequences": num_sequences, + "seq_len": args.seq_len, + "lora": { + "enabled": args.lora_epochs > 0, + "ran": args.lora_epochs > 0 and lora_ready, + "seq_len": args.distill_seq_len, + "batch_size": args.distill_batch_size, + "epochs": args.lora_epochs, + "rank": args.lora_rank, + "alpha": args.lora_alpha, + "dropout": args.lora_dropout, + "target_modules": args.lora_target_modules, + "respect_exclude_pairs": args.lora_respect_exclude_pairs, + "kl_enabled": args.lora_kl_enabled, + "kl_weight": args.lora_kl_weight, + "kl_temp": args.lora_kl_temp, + "post_ppl": lora_post_eval, + "ppl_over_lora": ( + f"cycle_{args.num_progressive}/ppl_over_lora.json" + if lora_eval_history + else None + ), + "lr": args.lora_lr, + "weight_decay": args.lora_weight_decay, + "max_grad_norm": args.lora_max_grad_norm, + "grad_accum_steps": args.lora_grad_accum_steps, + "log_steps": args.lora_log_steps, + "eval_every": args.lora_eval_every, + "eval_max_batches": args.lora_eval_max_batches, + }, + "artifacts": { + "final_pre_lora": final_pre_lora_meta, + "final_post_lora": final_post_lora_meta, + }, + "eval": { + "datasets": eval_datasets, + "configs": eval_configs, + "split": args.eval_split, + "num_samples": args.eval_num_samples, + "seq_len": args.eval_seq_len, + "pre_ppl": pre_eval, + "post_ppl": cycle_summaries[-1]["post_ppl"] if cycle_summaries else None, + }, + "cycles": cycle_summaries, + "final_num_layers": len(layers), + } + + with open( + os.path.join(args.output_dir, "progressive_metadata.json"), + "w", + encoding="utf-8", + ) as handle: + json.dump(progressive_meta, handle, indent=2) + + print( + f"[progressive] Completed {args.num_progressive} cycles. " + + ( + f"Profiling metadata written to {args.output_dir}; final model save skipped." + if args.profile_no_save + else ( + f"Compact weights written to {args.output_dir}; final dense model save skipped." + if args.skip_final_model_save else f"Final model saved to {args.output_dir}." + ) + ) + ) + + +def main() -> None: + args = parse_args() + if args.num_progressive <= 0: + raise SystemExit( + "Single-cycle mode has been removed. Pass --num_progressive > 0." + ) + if args.resume_from_cycle < 0: + raise SystemExit("--resume_from_cycle must be >= 0.") + if args.resume_from_cycle >= args.num_progressive: + raise SystemExit("--resume_from_cycle must be smaller than --num_progressive.") + args.full_model_save_cycles = resolve_full_model_save_cycles( + parse_cycle_list(args.save_full_model_cycles), + args.num_progressive, + ) + if args.profile_no_save: + args.full_model_save_cycles = set() + args.base_model_id = args.model + if args.resume_from_cycle > 0: + resume_meta = load_resume_metadata(args.model) + if resume_meta is None: + raise SystemExit( + "--resume_from_cycle requires --model to point to a saved cycle full model " + "directory containing resume_info.json." + ) + resume_cycle = resume_meta.get("cycle") + if resume_cycle is not None and int(resume_cycle) != args.resume_from_cycle: + raise SystemExit( + "resume_info.json cycle does not match --resume_from_cycle." + ) + base_model = resume_meta.get("base_model") + if isinstance(base_model, str) and base_model: + args.base_model_id = base_model + configure_reproducibility(args.seed) + + eval_datasets, eval_configs = resolve_eval_datasets(args) + dtype = get_dtype(args.dtype) + student_device_map = resolve_model_device_map_spec( + args.model_device_map, + model_path_or_id=args.model, + trust_remote_code=args.trust_remote_code, + cache_dir=args.model_cache_dir, + ) + if student_device_map is not None: + print(f"[device] student model_device_map={args.model_device_map}") + model = load_causal_lm( + args.model, + torch_dtype=dtype, + trust_remote_code=args.trust_remote_code, + cache_dir=args.model_cache_dir, + device_map=student_device_map, + ) + loader_generator_state = None + if args.resume_from_cycle > 0: + rng_state_path = os.path.join(args.model, "rng_state.pt") + rng_state = load_rng_state(rng_state_path) + if rng_state is not None: + restore_rng_state(rng_state) + loader_generator_state = load_loader_generator_state(args.model) + tokenizer = AutoTokenizer.from_pretrained( + args.model, + trust_remote_code=args.trust_remote_code, + cache_dir=args.model_cache_dir, + ) + print(model) + if tokenizer.pad_token is None and tokenizer.eos_token is not None: + tokenizer.pad_token = tokenizer.eos_token + + # for llama? + model.config.use_cache = False + + prepared = prepare_all_data( + args, + tokenizer, + model, + eval_datasets, + eval_configs, + loader_generator_state=loader_generator_state, + ) + run_progressive(args, model, tokenizer, prepared) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fuse_layers_distill.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fuse_layers_distill.py new file mode 100644 index 0000000000000000000000000000000000000000..3112fcf513cbf4963f90b2b9707c1972dba458e4 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/fuse_layers_distill.py @@ -0,0 +1,3824 @@ +#!/usr/bin/env python3 +"""Distillation helpers for fuse_layers.""" + +import argparse +import gc +import itertools +import math +import os +from contextlib import contextmanager, nullcontext +from typing import Dict, List, Optional, Set, Tuple + +import torch +import torch.nn.functional as F + +try: + import ppl_eval +except Exception as exc: # pragma: no cover - optional dependency + raise SystemExit("ppl_eval.py is required (missing or invalid)") from exc +try: + from tqdm import tqdm +except Exception: # pragma: no cover - optional dependency + tqdm = None + +try: + from torch.func import functional_call as _functional_call +except Exception: # pragma: no cover - depends on torch version + try: + from torch.nn.utils.stateless import functional_call as _functional_call + except Exception: # pragma: no cover - depends on torch version + _functional_call = None + +from fuse_layers_model import find_attention_module, find_mlp_module + + +def _tqdm_enabled() -> bool: + value = os.environ.get("DISABLE_TQDM", os.environ.get("TQDM_DISABLE", "0")) + return value.strip().lower() not in {"1", "true", "yes", "on"} + + +@contextmanager +def temporary_layers(parent: object, name: str, new_layers: torch.nn.Module): + original = getattr(parent, name) + setattr(parent, name, new_layers) + try: + yield + finally: + setattr(parent, name, original) + + +@contextmanager +def temporary_norm(parent: object): + if hasattr(parent, "norm"): + original = getattr(parent, "norm") + setattr(parent, "norm", torch.nn.Identity()) + try: + yield + finally: + setattr(parent, "norm", original) + else: + yield + + +def forward_truncated( + parent: torch.nn.Module, + layer_attr: str, + layers: List[torch.nn.Module], + upto: int, + input_ids: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, +) -> torch.Tensor: + truncated = torch.nn.ModuleList(layers[:upto]) + with temporary_layers(parent, layer_attr, truncated), temporary_norm(parent): + outputs = parent( + input_ids=input_ids, + attention_mask=attention_mask, + use_cache=False, + ) + if hasattr(outputs, "last_hidden_state"): + return outputs.last_hidden_state + return outputs[0] + + +def _masked_hidden_mse(diff: torch.Tensor, attention_mask: torch.Tensor) -> Optional[torch.Tensor]: + diff_f = diff.float() + mask = attention_mask.to(device=diff.device, dtype=torch.float32) + denom = mask.sum() * diff_f.size(-1) + if denom.item() == 0: + return None + return (diff_f.pow(2) * mask.unsqueeze(-1)).sum() / denom + + +def _extract_hidden_like(output) -> Optional[torch.Tensor]: + if torch.is_tensor(output): + return output + if isinstance(output, (tuple, list)) and output: + first = output[0] + if torch.is_tensor(first): + return first + if hasattr(output, "last_hidden_state"): + hidden = getattr(output, "last_hidden_state") + if torch.is_tensor(hidden): + return hidden + return None + + +def _add_weighted_loss( + total_loss: Optional[torch.Tensor], + term: Optional[torch.Tensor], + weight: float, +) -> Optional[torch.Tensor]: + if term is None: + return total_loss + weighted = float(weight) * term + if total_loss is None: + return weighted + if weighted.device != total_loss.device: + weighted = weighted.to(total_loss.device) + return total_loss + weighted + + +def _move_tensor_tree_to_device(value: object, device: torch.device) -> object: + if isinstance(value, torch.Tensor): + return value.to(device=device) + if isinstance(value, tuple): + return tuple(_move_tensor_tree_to_device(item, device) for item in value) + if isinstance(value, list): + return [_move_tensor_tree_to_device(item, device) for item in value] + if isinstance(value, dict): + return { + key: _move_tensor_tree_to_device(item, device) + for key, item in value.items() + } + return value + + +def _sum_grad_to_shape(grad: torch.Tensor, shape: torch.Size) -> torch.Tensor: + while grad.dim() > len(shape): + grad = grad.sum(dim=0) + for dim, size in enumerate(shape): + if size == 1 and grad.shape[dim] != 1: + grad = grad.sum(dim=dim, keepdim=True) + return grad + + +def _to_reparam_work_tensor(tensor: torch.Tensor, device: torch.device) -> torch.Tensor: + return tensor.detach().to(device=device, dtype=torch.float32) + + +def _initial_reparam_u( + param_a: torch.Tensor, + param_b: torch.Tensor, + device: torch.device, +) -> torch.Tensor: + return 0.5 * ( + _to_reparam_work_tensor(param_a, device) + - _to_reparam_work_tensor(param_b, device) + ) + + +class _ReparamMergedParamFn(torch.autograd.Function): + @staticmethod + def forward( + ctx, + param_a: torch.Tensor, + param_b: torch.Tensor, + gate: torch.Tensor, + u: torch.Tensor, + out_dtype: torch.dtype, + ) -> torch.Tensor: + ctx.save_for_backward(gate, u) + ctx.gate_shape = gate.shape + + work = torch.empty_like(u, dtype=torch.float32) + work.copy_(param_a.detach()) + work.add_(param_b.detach().to(device=work.device, dtype=work.dtype)).mul_(0.5) + scale = torch.sigmoid(gate.to(dtype=torch.float32)).mul_(2.0).sub_(1.0) + work.addcmul_(scale, u.to(dtype=torch.float32)) + return work.to(dtype=out_dtype) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + gate, u = ctx.saved_tensors + grad_f = grad_output.to(device=u.device, dtype=torch.float32) + scale = torch.sigmoid(gate.to(dtype=torch.float32)).mul_(2.0).sub_(1.0) + + grad_gate = grad_f * u.to(dtype=torch.float32) + grad_u = grad_f.mul_(scale) + scale.square_().neg_().add_(1.0).mul_(0.5) + grad_gate.mul_(scale) + grad_gate = _sum_grad_to_shape(grad_gate, ctx.gate_shape) + return None, None, grad_gate, grad_u, None + + +def _merge_reparam_param( + param_a: torch.Tensor, + param_b: torch.Tensor, + gate: torch.Tensor, + u: torch.Tensor, +) -> torch.Tensor: + return _ReparamMergedParamFn.apply(param_a, param_b, gate, u, param_a.dtype) + + +def _merge_reparam_param_no_grad( + param_a: torch.Tensor, + param_b: torch.Tensor, + gate: torch.Tensor, + u: torch.Tensor, +) -> torch.Tensor: + work = torch.empty_like(u, dtype=torch.float32) + work.copy_(param_a.detach()) + work.add_(param_b.detach().to(device=work.device, dtype=work.dtype)).mul_(0.5) + scale = torch.sigmoid(gate.to(dtype=torch.float32)).mul_(2.0).sub_(1.0) + work.addcmul_(scale, u.to(dtype=torch.float32)) + return work.to(dtype=param_a.dtype) + + +class _AdditiveUMergedParamFn(torch.autograd.Function): + """Frozen Fisher initialization plus a free additive FiT correction.""" + + @staticmethod + def forward( + ctx, + param_a: torch.Tensor, + param_b: torch.Tensor, + fisher_gate: torch.Tensor, + u_add: torch.Tensor, + out_dtype: torch.dtype, + ) -> torch.Tensor: + fisher_lambda = torch.sigmoid(fisher_gate.detach().float()) + a = param_a.detach().to(device=u_add.device, dtype=torch.float32) + b = param_b.detach().to(device=u_add.device, dtype=torch.float32) + fisher = b + fisher_lambda * (a - b) + ctx.u_device = u_add.device + return (fisher + u_add.float()).to(dtype=out_dtype) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + grad_u = grad_output.to(device=ctx.u_device, dtype=torch.float32) + return None, None, None, grad_u, None + + +def _merge_reparam_param_additive_u( + param_a: torch.Tensor, + param_b: torch.Tensor, + fisher_gate: torch.Tensor, + u_add: torch.Tensor, +) -> torch.Tensor: + return _AdditiveUMergedParamFn.apply( + param_a, param_b, fisher_gate, u_add, param_a.dtype + ) + + +def _merge_reparam_param_additive_u_no_grad( + param_a: torch.Tensor, + param_b: torch.Tensor, + fisher_gate: torch.Tensor, + u_add: torch.Tensor, +) -> torch.Tensor: + fisher_lambda = torch.sigmoid(fisher_gate.detach().float()) + a = param_a.detach().to(device=u_add.device, dtype=torch.float32) + b = param_b.detach().to(device=u_add.device, dtype=torch.float32) + fisher = b + fisher_lambda * (a - b) + return (fisher + u_add.detach().float()).to(dtype=param_a.dtype) + + +def _rho_projection(name: str) -> Optional[str]: + for projection in ("gate_proj", "up_proj", "down_proj"): + if f".{projection}." in f".{name}.": + return projection + return None + + +class _RhoMergedParamFn(torch.autograd.Function): + """Reparam merge with a frozen Fisher gate and one projection-level rho.""" + + @staticmethod + def forward( + ctx, + param_a: torch.Tensor, + param_b: torch.Tensor, + fisher_gate: torch.Tensor, + u: torch.Tensor, + rho: torch.Tensor, + out_dtype: torch.dtype, + ) -> torch.Tensor: + fisher_lambda = torch.sigmoid(fisher_gate.detach().float()) + effective_lambda = fisher_lambda + rho.float() * (1.0 - fisher_lambda) + scale = effective_lambda.mul(2.0).sub(1.0) + ctx.save_for_backward(fisher_lambda, scale, u) + work = torch.empty_like(u, dtype=torch.float32) + work.copy_(param_a.detach()) + work.add_(param_b.detach().to(device=work.device, dtype=work.dtype)).mul_(0.5) + work.addcmul_(scale, u.float()) + return work.to(dtype=out_dtype) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + fisher_lambda, scale, u = ctx.saved_tensors + grad_f = grad_output.to(device=u.device, dtype=torch.float32) + grad_u = grad_f * scale + grad_rho = (grad_f * u.float() * (2.0 * (1.0 - fisher_lambda))).sum() + return None, None, None, grad_u, grad_rho, None + + +def _merge_reparam_param_rho( + param_a: torch.Tensor, + param_b: torch.Tensor, + fisher_gate: torch.Tensor, + u: torch.Tensor, + rho: torch.Tensor, +) -> torch.Tensor: + return _RhoMergedParamFn.apply( + param_a, param_b, fisher_gate, u, rho, param_a.dtype + ) + + +def _merge_reparam_param_rho_no_grad( + param_a: torch.Tensor, + param_b: torch.Tensor, + fisher_gate: torch.Tensor, + u: torch.Tensor, + rho: torch.Tensor, +) -> torch.Tensor: + fisher_lambda = torch.sigmoid(fisher_gate.detach().float()) + effective_lambda = fisher_lambda + rho.detach().float() * (1.0 - fisher_lambda) + work = torch.empty_like(u, dtype=torch.float32) + work.copy_(param_a.detach()) + work.add_(param_b.detach().to(device=work.device, dtype=work.dtype)).mul_(0.5) + work.addcmul_(effective_lambda.mul(2.0).sub(1.0), u.float()) + return work.to(dtype=param_a.dtype) + + +class _SequentialRhoMergedParamFn(torch.autograd.Function): + """Fixed Fisher-to-anchor rho direction plus an additive FiT correction.""" + + @staticmethod + def forward( + ctx, + param_a: torch.Tensor, + param_b: torch.Tensor, + fisher_gate: torch.Tensor, + u_add: torch.Tensor, + rho: torch.Tensor, + out_dtype: torch.dtype, + ) -> torch.Tensor: + fisher_lambda = torch.sigmoid(fisher_gate.detach().float()) + a = param_a.detach().to(device=u_add.device, dtype=torch.float32) + b = param_b.detach().to(device=u_add.device, dtype=torch.float32) + fisher = b + fisher_lambda * (a - b) + anchor_delta = a - fisher + ctx.save_for_backward(anchor_delta) + work = fisher + rho.float() * anchor_delta + u_add.float() + return work.to(dtype=out_dtype) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + (anchor_delta,) = ctx.saved_tensors + grad_f = grad_output.to(device=anchor_delta.device, dtype=torch.float32) + grad_u = grad_f + grad_rho = (grad_f * anchor_delta).sum() + return None, None, None, grad_u, grad_rho, None + + +def _merge_reparam_param_sequential_rho( + param_a: torch.Tensor, + param_b: torch.Tensor, + fisher_gate: torch.Tensor, + u_add: torch.Tensor, + rho: torch.Tensor, +) -> torch.Tensor: + return _SequentialRhoMergedParamFn.apply( + param_a, param_b, fisher_gate, u_add, rho, param_a.dtype + ) + + +def _merge_reparam_param_sequential_rho_no_grad( + param_a: torch.Tensor, + param_b: torch.Tensor, + fisher_gate: torch.Tensor, + u_add: torch.Tensor, + rho: torch.Tensor, +) -> torch.Tensor: + fisher_lambda = torch.sigmoid(fisher_gate.detach().float()) + a = param_a.detach().to(device=u_add.device, dtype=torch.float32) + b = param_b.detach().to(device=u_add.device, dtype=torch.float32) + fisher = b + fisher_lambda * (a - b) + return ( + fisher + rho.detach().float() * (a - fisher) + u_add.detach().float() + ).to(dtype=param_a.dtype) + + +@contextmanager +def capture_module_output(module: torch.nn.Module): + cache: Dict[str, Optional[torch.Tensor]] = {"output": None} + + def hook(_module, _inputs, output): + cache["output"] = _extract_hidden_like(output) + + handle = module.register_forward_hook(hook) + try: + yield cache + finally: + handle.remove() + + +_ATTN_NAME_FRAGMENTS = ( + "self_attn.", + "attn.", + "attention.", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "q_norm", + "k_norm", +) +_MLP_NAME_FRAGMENTS = ( + "mlp.", + "ffn.", + "feed_forward", + "feedforward", + "gate_proj", + "up_proj", + "down_proj", + "fc1", + "fc2", + "dense_h_to_4h", + "dense_4h_to_h", + "w1", + "w2", + "w3", +) + + +def _classify_param_family(name: str) -> str: + lowered = name.lower() + if any(fragment in lowered for fragment in _MLP_NAME_FRAGMENTS): + return "mlp" + if any(fragment in lowered for fragment in _ATTN_NAME_FRAGMENTS): + return "attn" + return "other" + + +def _family_reg_scale(family: str, attn_scale: float, mlp_scale: float) -> float: + if family == "attn": + return attn_scale + if family == "mlp": + return mlp_scale + return 1.0 + + +def _subset_allows_param(name: str, subset: str) -> bool: + if subset == "all": + return True + if subset == "down_proj": + parts = name.lower().split(".") + return len(parts) >= 2 and parts[-2] == "down_proj" + return _classify_param_family(name) == subset + + +def _resolve_reparam_train_subset(param_subset: str, train_subset: object) -> str: + value = str(train_subset).strip().lower() + if value in {"", "same", "param_subset"}: + return param_subset + if value not in {"all", "mlp", "attn", "down_proj"}: + raise SystemExit( + "--reparam_train_subset must be one of: same, all, mlp, attn, down_proj" + ) + return value + + +def _resolve_reparam_fisher_init_subset(param_subset: str, init_subset: object) -> str: + value = str(init_subset).strip().lower() + if value in {"", "same", "param_subset"}: + return param_subset + valid = {"all", "mlp", "attn", "down_proj", "mlp_o", "mlp_v", "mlp_vo"} + if value not in valid: + raise SystemExit( + "--reparam_fisher_init_subset must be one of: same, all, mlp, attn, " + "down_proj, mlp_o, mlp_v, mlp_vo" + ) + return value + + +def _gate_logit_from_prior(prior: torch.Tensor) -> torch.Tensor: + # Stable logit: log(p) - log(1 - p). + return torch.log(prior) - torch.log1p(-prior) + + +def _tensor_debug_summary(tensors: Dict[str, torch.Tensor]) -> Dict[str, object]: + total_sum = 0.0 + total_sq_sum = 0.0 + total_abs_from_half = 0.0 + total_elems = 0 + global_min: Optional[float] = None + global_max: Optional[float] = None + per_tensor: Dict[str, Dict[str, float]] = {} + + with torch.no_grad(): + for name, tensor in tensors.items(): + t = tensor.detach().float() + if t.numel() == 0: + continue + numel = int(t.numel()) + t_sum = float(t.sum().item()) + t_sq_sum = float(t.pow(2).sum().item()) + t_min = float(t.min().item()) + t_max = float(t.max().item()) + t_abs_half = float((t - 0.5).abs().sum().item()) + per_tensor[name] = { + "mean": t_sum / float(numel), + "std": math.sqrt(max(t_sq_sum / float(numel) - (t_sum / float(numel)) ** 2, 0.0)), + "min": t_min, + "max": t_max, + "mean_abs_from_0_5": t_abs_half / float(numel), + } + total_sum += t_sum + total_sq_sum += t_sq_sum + total_abs_from_half += t_abs_half + total_elems += numel + global_min = t_min if global_min is None else min(global_min, t_min) + global_max = t_max if global_max is None else max(global_max, t_max) + + if total_elems == 0: + return {"num_tensors": len(tensors), "num_elements": 0} + mean = total_sum / float(total_elems) + return { + "num_tensors": len(tensors), + "num_elements": total_elems, + "mean": mean, + "std": math.sqrt(max(total_sq_sum / float(total_elems) - mean ** 2, 0.0)), + "min": global_min, + "max": global_max, + "mean_abs_from_0_5": total_abs_from_half / float(total_elems), + "per_tensor": per_tensor, + } + + +def _lambda_delta_summary( + final_lambdas: Dict[str, torch.Tensor], + init_lambdas: Dict[str, object], +) -> Dict[str, object]: + total_abs = 0.0 + total_sq = 0.0 + total_elems = 0 + max_abs = 0.0 + per_tensor: Dict[str, Dict[str, float]] = {} + + with torch.no_grad(): + for name, final in final_lambdas.items(): + init = init_lambdas.get(name) + if init is None: + init_t = torch.full_like(final.detach().float(), 0.5) + elif isinstance(init, torch.Tensor): + init_t = init.detach().to(device=final.device, dtype=torch.float32) + else: + init_t = torch.full_like(final.detach().float(), float(init)) + diff = final.detach().float() - init_t + if diff.numel() == 0: + continue + abs_diff = diff.abs() + numel = int(diff.numel()) + abs_sum = float(abs_diff.sum().item()) + sq_sum = float(diff.pow(2).sum().item()) + tensor_max_abs = float(abs_diff.max().item()) + per_tensor[name] = { + "mean_abs": abs_sum / float(numel), + "rms": math.sqrt(sq_sum / float(numel)), + "max_abs": tensor_max_abs, + } + total_abs += abs_sum + total_sq += sq_sum + total_elems += numel + max_abs = max(max_abs, tensor_max_abs) + + if total_elems == 0: + return {"num_tensors": len(final_lambdas), "num_elements": 0} + return { + "num_tensors": len(final_lambdas), + "num_elements": total_elems, + "mean_abs": total_abs / float(total_elems), + "rms": math.sqrt(total_sq / float(total_elems)), + "max_abs": max_abs, + "per_tensor": per_tensor, + } + + +def _lambda_summary_from_gates( + name_map: Dict[str, str], + gates: torch.nn.ParameterDict, +) -> Dict[str, object]: + total_sum = 0.0 + total_sq_sum = 0.0 + total_abs_from_half = 0.0 + total_elems = 0 + global_min: Optional[float] = None + global_max: Optional[float] = None + per_tensor: Dict[str, Dict[str, float]] = {} + + with torch.no_grad(): + for orig, safe in name_map.items(): + t = torch.sigmoid(gates[safe]).detach().float() + if t.numel() == 0: + continue + numel = int(t.numel()) + t_sum = float(t.sum().item()) + t_sq_sum = float(t.pow(2).sum().item()) + t_min = float(t.min().item()) + t_max = float(t.max().item()) + t_abs_half = float((t - 0.5).abs().sum().item()) + per_tensor[orig] = { + "mean": t_sum / float(numel), + "std": math.sqrt(max(t_sq_sum / float(numel) - (t_sum / float(numel)) ** 2, 0.0)), + "min": t_min, + "max": t_max, + "mean_abs_from_0_5": t_abs_half / float(numel), + } + total_sum += t_sum + total_sq_sum += t_sq_sum + total_abs_from_half += t_abs_half + total_elems += numel + global_min = t_min if global_min is None else min(global_min, t_min) + global_max = t_max if global_max is None else max(global_max, t_max) + + if total_elems == 0: + return {"num_tensors": len(name_map), "num_elements": 0} + mean = total_sum / float(total_elems) + return { + "num_tensors": len(name_map), + "num_elements": total_elems, + "mean": mean, + "std": math.sqrt(max(total_sq_sum / float(total_elems) - mean ** 2, 0.0)), + "min": global_min, + "max": global_max, + "mean_abs_from_0_5": total_abs_from_half / float(total_elems), + "per_tensor": per_tensor, + } + + +def _grad_debug_summary(params: List[torch.nn.Parameter]) -> Dict[str, object]: + total_sq = 0.0 + total_abs = 0.0 + total_elems = 0 + max_abs = 0.0 + none_count = 0 + with torch.no_grad(): + for param in params: + grad = param.grad + if grad is None: + none_count += 1 + continue + g = grad.detach().float() + if g.numel() == 0: + continue + abs_g = g.abs() + total_sq += float(g.pow(2).sum().item()) + total_abs += float(abs_g.sum().item()) + total_elems += int(g.numel()) + max_abs = max(max_abs, float(abs_g.max().item())) + return { + "num_tensors": len(params), + "num_tensors_without_grad": none_count, + "num_elements": total_elems, + "l2": math.sqrt(total_sq), + "mean_abs": None if total_elems == 0 else total_abs / float(total_elems), + "max_abs": max_abs, + } + + +def _u_delta_debug_summary( + name_map: Dict[str, str], + u_params: torch.nn.ParameterDict, + params_a: Dict[str, torch.nn.Parameter], + params_b: Dict[str, torch.nn.Parameter], + initial_scale: float = 1.0, +) -> Dict[str, object]: + total_sum = 0.0 + total_sq_sum = 0.0 + total_abs = 0.0 + total_elems = 0 + global_min: Optional[float] = None + global_max: Optional[float] = None + per_tensor: Dict[str, Dict[str, float]] = {} + + with torch.no_grad(): + for orig, safe in name_map.items(): + param_a = params_a.get(orig) + param_b = params_b.get(orig) + if param_a is None or param_b is None or param_b.shape != param_a.shape: + continue + u0 = float(initial_scale) * _initial_reparam_u( + param_a, param_b, u_params[safe].device + ) + delta = u_params[safe].detach().float() - u0 + if delta.numel() == 0: + continue + numel = int(delta.numel()) + delta_sum = float(delta.sum().item()) + delta_sq_sum = float(delta.pow(2).sum().item()) + delta_abs_sum = float(delta.abs().sum().item()) + delta_min = float(delta.min().item()) + delta_max = float(delta.max().item()) + per_tensor[orig] = { + "mean": delta_sum / float(numel), + "std": math.sqrt(max(delta_sq_sum / float(numel) - (delta_sum / float(numel)) ** 2, 0.0)), + "rms": math.sqrt(delta_sq_sum / float(numel)), + "mean_abs": delta_abs_sum / float(numel), + "min": delta_min, + "max": delta_max, + } + total_sum += delta_sum + total_sq_sum += delta_sq_sum + total_abs += delta_abs_sum + total_elems += numel + global_min = delta_min if global_min is None else min(global_min, delta_min) + global_max = delta_max if global_max is None else max(global_max, delta_max) + + if total_elems == 0: + return {"num_tensors": len(name_map), "num_elements": 0} + mean = total_sum / float(total_elems) + return { + "num_tensors": len(name_map), + "num_elements": total_elems, + "mean": mean, + "std": math.sqrt(max(total_sq_sum / float(total_elems) - mean ** 2, 0.0)), + "rms": math.sqrt(total_sq_sum / float(total_elems)), + "mean_abs": total_abs / float(total_elems), + "min": global_min, + "max": global_max, + "per_tensor": per_tensor, + } + + +def _build_gate_priors( + layer_a: torch.nn.Module, + layer_b: torch.nn.Module, + fisher_a: Dict[str, object], + fisher_b: Dict[str, object], + num_batches: int, + numels_a: Dict[str, int], + numels_b: Dict[str, int], + fisher_mode: str, + eps: float, + clamp_eps: float, +) -> Dict[str, torch.Tensor]: + """Return lambda priors for parameters that can be merged.""" + priors: Dict[str, torch.Tensor] = {} + params_b = {name: param for name, param in layer_b.named_parameters()} + for name, param_a in layer_a.named_parameters(): + param_b = params_b.get(name) + if param_b is None or param_b.shape != param_a.shape: + continue + if fisher_mode == "param": + fa = fisher_a[name] / max(num_batches, 1) + fb = fisher_b[name] / max(num_batches, 1) + denom = fa + fb + if not isinstance(denom, torch.Tensor): + denom = torch.tensor(float(denom)) + # If Fisher is uninformative, default to symmetric init. + prior = torch.where( + denom > eps, + fa / (denom + eps), + torch.full_like(denom, 0.5), + ) + prior = prior.clamp(clamp_eps, 1.0 - clamp_eps) + priors[name] = prior + else: + fa = fisher_a[name] / (max(num_batches, 1) * numels_a[name]) + fb = fisher_b[name] / (max(num_batches, 1) * numels_b[name]) + denom = fa + fb + if denom <= eps: + prior_val = 0.5 + else: + prior_val = float(fa / (denom + eps)) + prior_val = min(max(prior_val, clamp_eps), 1.0 - clamp_eps) + priors[name] = torch.tensor(prior_val, dtype=torch.float32) + return priors + + +def compute_fisher_gate_priors( + layer_a: torch.nn.Module, + layer_b: torch.nn.Module, + fisher_a: Dict[str, object], + fisher_b: Dict[str, object], + num_batches: int, + numels_a: Dict[str, int], + numels_b: Dict[str, int], + fisher_mode: str, + eps: float, + clamp_eps: float = 1e-4, +) -> Dict[str, torch.Tensor]: + """Compute Fisher prior gate lambdas (lambda_prior) for mergeable parameters.""" + return _build_gate_priors( + layer_a=layer_a, + layer_b=layer_b, + fisher_a=fisher_a, + fisher_b=fisher_b, + num_batches=num_batches, + numels_a=numels_a, + numels_b=numels_b, + fisher_mode=fisher_mode, + eps=eps, + clamp_eps=clamp_eps, + ) + + +_FISHER_DISTRIBUTION_QUANTILES = ( + ("p0", 0.0), + ("p1", 0.01), + ("p10", 0.10), + ("p25", 0.25), + ("p50", 0.50), + ("p75", 0.75), + ("p90", 0.90), + ("p95", 0.95), + ("p99", 0.99), + ("p99_9", 0.999), + ("p100", 1.0), +) + + +def _deterministic_fisher_sample( + values: torch.Tensor, + max_samples: int = 131_072, +) -> torch.Tensor: + """Return an evenly strided, deterministic CPU sample for quantiles.""" + flat = values.detach().float().reshape(-1) + if flat.numel() <= max_samples: + return flat + stride = max(1, math.ceil(flat.numel() / max_samples)) + return flat[::stride][:max_samples] + + +def _sampled_fisher_quantiles(sample: torch.Tensor) -> Dict[str, float]: + if sample.numel() == 0: + return {} + probabilities = torch.tensor( + [probability for _, probability in _FISHER_DISTRIBUTION_QUANTILES], + dtype=torch.float32, + device=sample.device, + ) + values = torch.quantile(sample.float(), probabilities) + return { + name: float(value.item()) + for (name, _), value in zip(_FISHER_DISTRIBUTION_QUANTILES, values) + } + + +def _summarize_fisher_distribution( + values: torch.Tensor, + eps: float, +) -> Dict[str, object]: + """Summarize one non-negative Fisher tensor without retaining it.""" + flat = values.detach().float().reshape(-1) + numel = int(flat.numel()) + if numel == 0: + return {"num_elements": 0} + finite = torch.isfinite(flat) + finite_count = int(finite.sum().item()) + if finite_count != numel: + finite_values = flat[finite] + else: + finite_values = flat + if finite_values.numel() == 0: + return { + "num_elements": numel, + "finite_count": 0, + "finite_ratio": 0.0, + } + + finite_numel = int(finite_values.numel()) + value_sum = float(finite_values.sum().item()) + value_sq_sum = float(finite_values.square().sum().item()) + mean = value_sum / float(finite_numel) + variance = max(value_sq_sum / float(finite_numel) - mean * mean, 0.0) + zero_count = int((finite_values == 0).sum().item()) + le_eps_count = int((finite_values <= eps).sum().item()) + sample = _deterministic_fisher_sample(finite_values) + return { + "num_elements": numel, + "finite_count": finite_count, + "finite_ratio": finite_count / float(numel), + "zero_count": zero_count, + "zero_ratio": zero_count / float(finite_numel), + "le_eps_count": le_eps_count, + "le_eps_ratio": le_eps_count / float(finite_numel), + "mean": mean, + "std": math.sqrt(variance), + "min": float(finite_values.min().item()), + "max": float(finite_values.max().item()), + "quantile_method": "deterministic_even_stride_sample", + "quantile_sample_count": int(sample.numel()), + "quantiles": _sampled_fisher_quantiles(sample), + } + + +def summarize_fisher_gate_prior_inputs( + layer_a: torch.nn.Module, + layer_b: torch.nn.Module, + fisher_a: Dict[str, object], + fisher_b: Dict[str, object], + num_batches: int, + numels_a: Dict[str, int], + numels_b: Dict[str, int], + fisher_mode: str, + eps: float, + param_subset: str = "all", +) -> Dict[str, object]: + """Summarize Fisher inputs that determine lambda priors without storing tensors.""" + params_b = {name: param for name, param in layer_b.named_parameters()} + per_tensor: Dict[str, Dict[str, object]] = {} + global_samples: Dict[str, List[torch.Tensor]] = { + "fa": [], + "fb": [], + "denom": [], + } + total_elems = 0 + total_fa = 0.0 + total_fb = 0.0 + total_denom = 0.0 + total_abs_diff = 0.0 + total_contrast = 0.0 + total_active = 0.0 + total_prior = 0.0 + total_prior_active = 0.0 + active_elems = 0 + + with torch.no_grad(): + for name, param_a in layer_a.named_parameters(): + param_b = params_b.get(name) + if param_b is None or param_b.shape != param_a.shape: + continue + if not _subset_allows_param(name, param_subset): + continue + + if fisher_mode == "param": + fa = (fisher_a[name] / max(num_batches, 1)).detach().float() + fb = (fisher_b[name] / max(num_batches, 1)).detach().float() + else: + fa_val = float( + fisher_a[name] + / (max(num_batches, 1) * max(numels_a.get(name, 1), 1)) + ) + fb_val = float( + fisher_b[name] + / (max(num_batches, 1) * max(numels_b.get(name, 1), 1)) + ) + fa = torch.tensor(fa_val, dtype=torch.float32) + fb = torch.tensor(fb_val, dtype=torch.float32) + + denom = fa + fb + active = denom > eps + prior = torch.where( + active, + fa / (denom + eps), + torch.full_like(denom, 0.5), + ) + contrast = (fa - fb).abs() / (denom + eps) + numel = int(denom.numel()) + active_count = int(active.sum().item()) + fa_sum = float(fa.sum().item()) + fb_sum = float(fb.sum().item()) + denom_sum = float(denom.sum().item()) + abs_diff_sum = float((fa - fb).abs().sum().item()) + contrast_sum = float(contrast.sum().item()) + prior_sum = float(prior.sum().item()) + if active_count > 0: + prior_active_mean = float(prior[active].mean().item()) + else: + prior_active_mean = 0.5 + + fa_distribution = _summarize_fisher_distribution(fa, eps) + fb_distribution = _summarize_fisher_distribution(fb, eps) + denom_distribution = _summarize_fisher_distribution(denom, eps) + global_samples["fa"].append(_deterministic_fisher_sample(fa)) + global_samples["fb"].append(_deterministic_fisher_sample(fb)) + global_samples["denom"].append(_deterministic_fisher_sample(denom)) + + per_tensor[name] = { + "num_elements": numel, + "fa_mean": fa_sum / float(numel), + "fb_mean": fb_sum / float(numel), + "denom_mean": denom_sum / float(numel), + "active_ratio": active_count / float(numel), + "mean_abs_fa_minus_fb": abs_diff_sum / float(numel), + "mean_relative_contrast": contrast_sum / float(numel), + "prior_mean": prior_sum / float(numel), + "prior_active_mean": prior_active_mean, + "fa_distribution": fa_distribution, + "fb_distribution": fb_distribution, + "denom_distribution": denom_distribution, + } + + total_elems += numel + total_fa += fa_sum + total_fb += fb_sum + total_denom += denom_sum + total_abs_diff += abs_diff_sum + total_contrast += contrast_sum + total_active += float(active_count) + total_prior += prior_sum + total_prior_active += prior_active_mean * float(active_count) + active_elems += active_count + + if total_elems == 0: + return {"num_tensors": 0, "num_elements": 0, "per_tensor": per_tensor} + + distributions = {} + for distribution_name, samples in global_samples.items(): + sample = torch.cat(samples) if samples else torch.empty(0, dtype=torch.float32) + distributions[distribution_name] = { + "quantile_method": "concatenated_per_tensor_even_stride_samples", + "quantile_sample_count": int(sample.numel()), + "quantiles": _sampled_fisher_quantiles(sample), + } + + return { + "num_tensors": len(per_tensor), + "num_elements": total_elems, + "fisher_mode": fisher_mode, + "eps": float(eps), + "param_subset": param_subset, + "fa_mean": total_fa / float(total_elems), + "fb_mean": total_fb / float(total_elems), + "denom_mean": total_denom / float(total_elems), + "active_ratio": total_active / float(total_elems), + "mean_abs_fa_minus_fb": total_abs_diff / float(total_elems), + "mean_relative_contrast": total_contrast / float(total_elems), + "prior_mean": total_prior / float(total_elems), + "prior_active_mean": ( + 0.5 if active_elems == 0 else total_prior_active / float(active_elems) + ), + "distributions": distributions, + "per_tensor": per_tensor, + } + + +class ReparamMergedLayer(torch.nn.Module): + """Virtual layer that merges parameters via W0/U reparameterization. + + Parameters of layer_a/layer_b are treated as frozen (detached). We train: + - gate logits s (lambda = sigmoid(s)) + - U (initialized as U0 = (W_a - W_b) / 2) + + Forward uses: + W_merge = W0 + (2 * lambda - 1) * U + where W0 = (W_a + W_b) / 2 + """ + + def __init__( + self, + layer_a: torch.nn.Module, + layer_b: torch.nn.Module, + gate_targets: Dict[str, object], + param_subset: str = "all", + clamp_eps: float = 1e-4, + tensor_rho: bool = False, + sequential_rho: bool = False, + additive_u: bool = False, + ) -> None: + super().__init__() + if sequential_rho and additive_u: + raise ValueError("sequential_rho and additive_u are mutually exclusive") + self.layer_a = layer_a + self.layer_b = layer_b + self.param_subset = param_subset + self.sequential_rho = bool(sequential_rho) + self.additive_u = bool(additive_u) + self._name_map: Dict[str, str] = {} + + self.gates = torch.nn.ParameterDict() + self.u = torch.nn.ParameterDict() + self.rho = torch.nn.ParameterDict() + + params_b = {name: param for name, param in layer_b.named_parameters()} + try: + device = next(layer_a.parameters()).device + except StopIteration: + device = torch.device("cpu") + + for name, param_a in layer_a.named_parameters(): + param_b = params_b.get(name) + if param_b is None or param_b.shape != param_a.shape: + continue + if not _subset_allows_param(name, self.param_subset): + continue + + target = gate_targets.get(name) + if target is None: + target_t = torch.tensor(0.5, device=device, dtype=torch.float32) + elif isinstance(target, torch.Tensor): + target_t = target.detach().to(device=device, dtype=torch.float32) + else: + target_t = torch.tensor(float(target), device=device, dtype=torch.float32) + + target_t = target_t.clamp(clamp_eps, 1.0 - clamp_eps) + s0 = _gate_logit_from_prior(target_t) + u0 = _initial_reparam_u(param_a, param_b, device) + if self.sequential_rho or self.additive_u: + # Both modes use an explicit additive FiT correction. Additive-U + # fixes rho at zero: W = W_F + U_add. + u0 = torch.zeros_like(u0) + + safe = name.replace(".", "__") + if safe in self.gates: + safe = f"{safe}_{len(self.gates)}" + self._name_map[name] = safe + self.gates[safe] = torch.nn.Parameter(s0) + self.u[safe] = torch.nn.Parameter(u0) + projection = _rho_projection(name) + if tensor_rho and projection is not None and projection not in self.rho: + self.rho[projection] = torch.nn.Parameter( + torch.zeros((), device=device, dtype=torch.float32) + ) + + def __getattr__(self, name: str): + # Delegate model-specific attributes (e.g. Qwen's `attention_type`) to + # the underlying layer so the parent forward doesn't break. + try: + return super().__getattr__(name) + except AttributeError as exc: + try: + layer_a = super().__getattr__("layer_a") + if hasattr(layer_a, name): + return getattr(layer_a, name) + except AttributeError: + pass + try: + layer_b = super().__getattr__("layer_b") + if hasattr(layer_b, name): + return getattr(layer_b, name) + except AttributeError: + pass + raise exc + + def _safe_for(self, orig: str) -> Optional[str]: + return self._name_map.get(orig) + + def gate_lambdas(self) -> Dict[str, torch.Tensor]: + out: Dict[str, torch.Tensor] = {} + for orig, safe in self._name_map.items(): + fisher_lambda = torch.sigmoid(self.gates[safe]).detach() + projection = _rho_projection(orig) + if projection is not None and projection in self.rho: + fisher_lambda = fisher_lambda + self.rho[projection].detach() * ( + 1.0 - fisher_lambda + ) + out[orig] = fisher_lambda + return out + + def rho_values(self) -> Dict[str, float]: + return { + name: float(value.detach().float().cpu()) + for name, value in self.rho.items() + } + + def _merged_params(self) -> Dict[str, torch.Tensor]: + params_a = {name: p for name, p in self.layer_a.named_parameters()} + params_b = {name: p for name, p in self.layer_b.named_parameters()} + merged_params: Dict[str, torch.Tensor] = {} + + for name, param_a in params_a.items(): + param_b = params_b.get(name) + safe = self._safe_for(name) + if safe is None or param_b is None or param_b.shape != param_a.shape: + merged_params[name] = param_a.detach() + continue + + projection = _rho_projection(name) + if self.additive_u: + merged_params[name] = _merge_reparam_param_additive_u( + param_a.detach(), + param_b.detach(), + self.gates[safe], + self.u[safe], + ) + elif projection is not None and projection in self.rho: + if self.sequential_rho: + merged_params[name] = _merge_reparam_param_sequential_rho( + param_a.detach(), + param_b.detach(), + self.gates[safe], + self.u[safe], + self.rho[projection], + ) + else: + merged_params[name] = _merge_reparam_param_rho( + param_a.detach(), + param_b.detach(), + self.gates[safe], + self.u[safe], + self.rho[projection], + ) + else: + merged_params[name] = _merge_reparam_param( + param_a.detach(), + param_b.detach(), + self.gates[safe], + self.u[safe], + ) + return merged_params + + def forward(self, *args, **kwargs): + if _functional_call is None: + raise RuntimeError( + "Reparam distillation requires torch.func.functional_call" + ) + + merged_params = self._merged_params() + return _functional_call(self.layer_a, merged_params, args, kwargs) + + def materialize_into_layer_a(self) -> int: + merged = 0 + params_a = {name: p for name, p in self.layer_a.named_parameters()} + params_b = {name: p for name, p in self.layer_b.named_parameters()} + with torch.no_grad(): + for orig, safe in self._name_map.items(): + param_a = params_a.get(orig) + param_b = params_b.get(orig) + if param_a is None or param_b is None or param_b.shape != param_a.shape: + continue + projection = _rho_projection(orig) + if self.additive_u: + merged_param = _merge_reparam_param_additive_u_no_grad( + param_a.detach(), + param_b.detach(), + self.gates[safe], + self.u[safe], + ) + elif projection is not None and projection in self.rho: + if self.sequential_rho: + merged_param = _merge_reparam_param_sequential_rho_no_grad( + param_a.detach(), + param_b.detach(), + self.gates[safe], + self.u[safe], + self.rho[projection], + ) + else: + merged_param = _merge_reparam_param_rho_no_grad( + param_a.detach(), + param_b.detach(), + self.gates[safe], + self.u[safe], + self.rho[projection], + ) + else: + merged_param = _merge_reparam_param_no_grad( + param_a.detach(), + param_b.detach(), + self.gates[safe], + self.u[safe], + ) + param_a.copy_(merged_param.to(dtype=param_a.dtype)) + merged += 1 + return merged + + +def _sequential_u_fisher_orthogonality( + reparam_layer: ReparamMergedLayer, +) -> Dict[str, object]: + """Summarize additive FiT U versus the original Fisher displacement. + + The fixed reference direction is Delta_F = W_F - W_anchor, before rho or + U training. Statistics are accumulated without concatenating the large + projection tensors. + """ + if not (reparam_layer.sequential_rho or reparam_layer.additive_u): + return {} + params_a = dict(reparam_layer.layer_a.named_parameters()) + params_b = dict(reparam_layer.layer_b.named_parameters()) + accum: Dict[str, Dict[str, float]] = {} + + def add(group: str, u: torch.Tensor, delta_f: torch.Tensor) -> None: + bucket = accum.setdefault( + group, + {"dot": 0.0, "u_sq": 0.0, "fisher_sq": 0.0, "num_elements": 0.0}, + ) + bucket["dot"] += float((u * delta_f).sum().item()) + bucket["u_sq"] += float(u.square().sum().item()) + bucket["fisher_sq"] += float(delta_f.square().sum().item()) + bucket["num_elements"] += float(u.numel()) + + with torch.no_grad(): + for orig, safe in reparam_layer._name_map.items(): + projection = _rho_projection(orig) + if projection is None: + continue + if reparam_layer.sequential_rho and projection not in reparam_layer.rho: + continue + param_a = params_a.get(orig) + param_b = params_b.get(orig) + if param_a is None or param_b is None or param_a.shape != param_b.shape: + continue + lam = torch.sigmoid(reparam_layer.gates[safe].detach().float()) + a = param_a.detach().to(device=lam.device, dtype=torch.float32) + b = param_b.detach().to(device=lam.device, dtype=torch.float32) + delta_f = (b + lam * (a - b)) - a + u = reparam_layer.u[safe].detach().float() + add(projection, u, delta_f) + add("global", u, delta_f) + + result: Dict[str, object] = {} + for group, values in accum.items(): + u_norm = math.sqrt(max(values["u_sq"], 0.0)) + fisher_norm = math.sqrt(max(values["fisher_sq"], 0.0)) + denom = u_norm * fisher_norm + cosine = None if denom == 0.0 else values["dot"] / denom + if cosine is not None: + cosine = max(-1.0, min(1.0, cosine)) + result[group] = { + "num_elements": int(values["num_elements"]), + "cosine_u_vs_fisher_delta": cosine, + "orthogonal_fraction_of_u": ( + None if cosine is None else math.sqrt(max(0.0, 1.0 - cosine * cosine)) + ), + "u_norm": u_norm, + "fisher_delta_norm": fisher_norm, + "u_over_fisher_delta_norm": ( + None if fisher_norm == 0.0 else u_norm / fisher_norm + ), + "projection_coefficient_on_fisher": ( + None + if values["fisher_sq"] == 0.0 + else values["dot"] / values["fisher_sq"] + ), + } + return result + + +def distill_reparam_merge( + student_model: torch.nn.Module, + student_parent: object, + student_layer_attr: str, + student_layers: List[torch.nn.Module], + teacher_model: torch.nn.Module, + teacher_parent: object, + teacher_layer_attr: str, + teacher_layers: List[torch.nn.Module], + layer_idx: int, + gate_lambdas: Dict[str, object], + dataloader, + args: argparse.Namespace, + progressive_cycle: Optional[int] = None, + progressive_total: Optional[int] = None, +) -> Tuple[int, Dict[str, torch.Tensor], Dict[str, object]]: + """Reparameterized distillation that materializes a fused layer into layer_a. + + Trains U and gate logits s (lambda = sigmoid(s)) using: + - composition MSE + distill-KL + - eta * ||lambda - lambda_gate||^2 + gamma * ||U - U0||^2 + """ + requested_epochs = float(args.distill_epochs) + skip_train = bool(getattr(args, "reparam_skip_train", False)) + total_epochs = 0.0 if skip_train else requested_epochs + + hidden_mse_weight = float(getattr(args, "distill_hidden_mse_weight", 1.0)) + if hidden_mse_weight < 0.0: + raise SystemExit("--distill_hidden_mse_weight must be >= 0") + attn_mse_weight = float(getattr(args, "distill_attn_mse_weight", 0.0)) + if attn_mse_weight < 0.0: + raise SystemExit("--distill_attn_mse_weight must be >= 0") + mlp_mse_weight = float(getattr(args, "distill_mlp_mse_weight", 0.0)) + if mlp_mse_weight < 0.0: + raise SystemExit("--distill_mlp_mse_weight must be >= 0") + param_subset = str(getattr(args, "reparam_param_subset", "all")) + if param_subset not in {"all", "mlp", "attn", "down_proj"}: + raise SystemExit( + "--reparam_param_subset must be one of: all, mlp, attn, down_proj" + ) + train_subset = _resolve_reparam_train_subset( + param_subset, + getattr(args, "reparam_train_subset", "same"), + ) + fisher_init_subset = _resolve_reparam_fisher_init_subset( + param_subset, + getattr(args, "reparam_fisher_init_subset", "same"), + ) + + kl_weight = float(args.distill_kl_weight) + kl_temp = float(args.distill_kl_temp) + if kl_weight < 0.0: + raise SystemExit("--distill_kl_weight must be >= 0") + if kl_temp <= 0.0: + raise SystemExit("--distill_kl_temp must be > 0") + + rho_objective = str(getattr(args, "reparam_rho_objective", "none")) + valid_rho_objectives = {"none", "full", "hidden_kl", "kl_only"} + if rho_objective not in valid_rho_objectives: + raise SystemExit( + "--reparam_rho_objective must be one of: " + + ", ".join(sorted(valid_rho_objectives)) + ) + rho_enabled = rho_objective != "none" + rho_schedule = str(getattr(args, "reparam_rho_schedule", "joint")) + if rho_schedule not in {"joint", "sequential"}: + raise SystemExit("--reparam_rho_schedule must be joint or sequential") + if not rho_enabled and rho_schedule == "sequential": + raise SystemExit("sequential rho schedule requires an enabled rho objective") + rho_epochs = float(getattr(args, "reparam_rho_epochs", 1.0)) + if rho_epochs <= 0.0: + raise SystemExit("--reparam_rho_epochs must be > 0") + sequential_rho = rho_enabled and rho_schedule == "sequential" + if sequential_rho and abs(rho_epochs - round(rho_epochs)) > 1e-8: + raise SystemExit("sequential rho currently requires integer --reparam_rho_epochs") + if sequential_rho and not skip_train: + total_epochs = rho_epochs + requested_epochs + additive_u = bool(getattr(args, "reparam_additive_u", False)) + if additive_u and rho_enabled: + raise SystemExit("--reparam_additive_u cannot be combined with rho") + rho_lr = float(getattr(args, "reparam_rho_lr", 2e-3)) + rho_min = float(getattr(args, "reparam_rho_min", -1.0)) + rho_max = float(getattr(args, "reparam_rho_max", 2.0)) + if rho_lr <= 0.0: + raise SystemExit("--reparam_rho_lr must be > 0") + if rho_min >= rho_max: + raise SystemExit("--reparam_rho_min must be below --reparam_rho_max") + if rho_enabled and kl_weight <= 0.0 and rho_objective in {"hidden_kl", "kl_only"}: + raise SystemExit("rho objective requires nonzero distillation KL weight") + + eta = float(getattr(args, "reparam_eta", 0.0)) + gamma = float(getattr(args, "reparam_gamma", 0.0)) + if eta < 0.0: + raise SystemExit("--reparam_eta must be >= 0") + if gamma < 0.0: + raise SystemExit("--reparam_gamma must be >= 0") + attn_reg_scale = float(getattr(args, "reparam_attn_reg_scale", 1.0)) + mlp_reg_scale = float(getattr(args, "reparam_mlp_reg_scale", 1.0)) + if attn_reg_scale < 0.0: + raise SystemExit("--reparam_attn_reg_scale must be >= 0") + if mlp_reg_scale < 0.0: + raise SystemExit("--reparam_mlp_reg_scale must be >= 0") + if ( + total_epochs > 0.0 + and hidden_mse_weight == 0.0 + and attn_mse_weight == 0.0 + and mlp_mse_weight == 0.0 + and kl_weight == 0.0 + and eta == 0.0 + and gamma == 0.0 + ): + raise SystemExit( + "Reparam distillation has no active loss terms. " + "Enable hidden/attention/MLP MSE, KL, or at least one reparam regularizer." + ) + + if not gate_lambdas: + raise SystemExit("Reparam distillation requires non-empty gate lambdas.") + + layer_a = student_layers[layer_idx] + layer_b = student_layers[layer_idx + 1] + + reparam_layer = ReparamMergedLayer( + layer_a, + layer_b, + gate_lambdas, + param_subset=param_subset, + clamp_eps=1e-4, + tensor_rho=rho_enabled, + sequential_rho=sequential_rho, + additive_u=additive_u, + ) + if not reparam_layer._name_map: + raise RuntimeError( + "No mergeable parameters found for reparam distillation under " + f"--reparam_param_subset={param_subset!r}." + ) + + teacher_attn = None + student_attn = None + if total_epochs > 0.0 and attn_mse_weight > 0.0: + try: + teacher_attn = find_attention_module(teacher_layers[layer_idx + 1]) + student_attn = find_attention_module(reparam_layer.layer_a) + except ValueError as exc: + raise SystemExit( + "Attention-output preservation was requested but an attention module " + f"could not be resolved: {exc}" + ) from exc + + teacher_mlp = None + student_mlp = None + if total_epochs > 0.0 and mlp_mse_weight > 0.0: + try: + teacher_mlp = find_mlp_module(teacher_layers[layer_idx + 1]) + student_mlp = find_mlp_module(reparam_layer.layer_a) + except ValueError as exc: + raise SystemExit( + "MLP-output preservation was requested but an MLP module could not be " + f"resolved: {exc}" + ) from exc + + # Virtual layer list: replace layer_a with reparam layer and remove layer_b. + virtual_layers = list(student_layers) + virtual_layers[layer_idx] = reparam_layer + del virtual_layers[layer_idx + 1] + + # Only selected reparam (U, s) tensors are trainable. Parameters outside + # train_subset are still materialized from their Fisher-prior initialization. + for param in student_model.parameters(): + param.requires_grad_(False) + trainable_safe_names = set() + train_family_counts: Dict[str, int] = {"attn": 0, "mlp": 0, "other": 0} + for orig, safe in reparam_layer._name_map.items(): + is_trainable = _subset_allows_param(orig, train_subset) + reparam_layer.gates[safe].requires_grad_( + is_trainable and not rho_enabled and not additive_u + ) + reparam_layer.u[safe].requires_grad_(is_trainable) + if is_trainable: + trainable_safe_names.add(safe) + train_family_counts[_classify_param_family(orig)] += 1 + for rho in reparam_layer.rho.values(): + rho.requires_grad_(rho_enabled) + trainable_base_params = [ + param + for param in itertools.chain( + reparam_layer.gates.parameters(), + reparam_layer.u.parameters(), + ) + if param.requires_grad + ] + trainable_rho_params = [ + param for param in reparam_layer.rho.parameters() if param.requires_grad + ] + trainable_reparam_params = [*trainable_base_params, *trainable_rho_params] + if total_epochs > 0.0 and not trainable_reparam_params: + raise SystemExit( + "Reparam training requested but --reparam_train_subset selects no " + "materialized parameters." + ) + + do_train = total_epochs > 0.0 + if do_train: + teacher_model.eval() + student_model.train() + elif skip_train: + print("[reparam] skip_train enabled; using Fisher-prior initialization only.") + + # Rough memory heads-up (esp. when --fisher_mode param makes per-element gates). + total_gate_elems = sum(int(p.numel()) for p in reparam_layer.gates.parameters()) + total_u_elems = sum(int(p.numel()) for p in reparam_layer.u.parameters()) + gate_mib = total_gate_elems * 4.0 / (1024.0 * 1024.0) + u_mib = total_u_elems * 4.0 / (1024.0 * 1024.0) + family_counts: Dict[str, int] = {"attn": 0, "mlp": 0, "other": 0} + for orig in reparam_layer._name_map: + family_counts[_classify_param_family(orig)] += 1 + lambda_init_summary = _lambda_summary_from_gates( + reparam_layer._name_map, + reparam_layer.gates, + ) + print( + f"[reparam] subset={param_subset} gates={len(reparam_layer.gates)} " + f"(attn={family_counts['attn']}, mlp={family_counts['mlp']}, other={family_counts['other']}) " + f"train_subset={train_subset} " + f"(attn={train_family_counts['attn']}, mlp={train_family_counts['mlp']}, " + f"other={train_family_counts['other']}) " + f"elems={total_gate_elems} (~{gate_mib:.1f} MiB), " + f"U_elems={total_u_elems} (~{u_mib:.1f} MiB; +optimizer state)" + ) + print( + "[reparam] lambda_init " + f"mean={lambda_init_summary.get('mean')} " + f"std={lambda_init_summary.get('std')} " + f"min={lambda_init_summary.get('min')} " + f"max={lambda_init_summary.get('max')} " + f"mean_abs_from_0.5={lambda_init_summary.get('mean_abs_from_0_5')}" + ) + if rho_enabled: + print( + f"[reparam-rho] objective={rho_objective} schedule={rho_schedule} " + f"rho_epochs={rho_epochs if sequential_rho else 0.0} lr={rho_lr} " + f"bounds=[{rho_min}, {rho_max}] projections={sorted(reparam_layer.rho)}" + ) + if additive_u: + print("[reparam] additive_u enabled: W = W_F + U_add, U_add initialized at zero") + + optimizer = None + if do_train: + groups = [] + if trainable_base_params: + groups.append( + { + "params": trainable_base_params, + "lr": float(args.distill_lr), + "weight_decay": float(args.distill_weight_decay), + } + ) + if trainable_rho_params: + groups.append( + { + "params": trainable_rho_params, + "lr": rho_lr, + "weight_decay": 0.0, + } + ) + optimizer = torch.optim.AdamW(groups) + + device_type = torch.device(args.device).type + amp_dtype = None + if args.dtype == "float16": + amp_dtype = torch.float16 + elif args.dtype == "bfloat16": + amp_dtype = torch.bfloat16 + use_amp = do_train and amp_dtype is not None and device_type == "cuda" + use_scaler = use_amp and amp_dtype == torch.float16 + if rho_enabled and use_scaler: + raise SystemExit("tensor-rho gradient routing requires bf16 or fp32 training") + scaler = torch.cuda.amp.GradScaler() if use_scaler else None + teacher_runtime_device = torch.device(args.distill_teacher_device or args.device) + if teacher_model is not None: + try: + teacher_runtime_device = next(teacher_model.parameters()).device + except StopIteration: + pass + + full_epochs = int(total_epochs) if do_train else 0 + fractional = (total_epochs - full_epochs) if do_train else 0.0 + if fractional < 1e-8: + fractional = 0.0 + + epoch_plan = [(epoch_idx, None) for epoch_idx in range(full_epochs)] + if fractional > 0: + try: + batches_per_epoch = len(dataloader) + except TypeError as exc: + raise SystemExit( + "Fractional distill epochs require a dataloader with finite length." + ) from exc + if batches_per_epoch > 0: + frac_batches = int(round(fractional * batches_per_epoch)) + if frac_batches <= 0: + frac_batches = 1 + epoch_plan.append((full_epochs, frac_batches)) + + grad_accum = int(getattr(args, "distill_grad_accum_steps", 1)) + if grad_accum <= 0: + raise SystemExit("--distill_grad_accum_steps must be >= 1") + if rho_enabled and grad_accum != 1: + raise SystemExit("tensor-rho gradient routing currently requires grad accumulation 1") + + log_steps = int(getattr(args, "distill_log_steps", 100)) + max_grad_norm = getattr(args, "distill_max_grad_norm", 1.0) + + params_a = {name: p for name, p in layer_a.named_parameters()} + params_b = {name: p for name, p in layer_b.named_parameters()} + + first_grad_summary: Dict[str, object] = {} + rho_after_phase: Optional[Dict[str, float]] = None + step = 0 + for epoch_idx, max_batches in epoch_plan: + if sequential_rho: + phase = "rho" if epoch_idx < int(round(rho_epochs)) else "u" + for param in trainable_base_params: + param.requires_grad_(phase == "u") + for param in trainable_rho_params: + param.requires_grad_(phase == "rho") + if phase == "u" and rho_after_phase is None: + rho_after_phase = reparam_layer.rho_values() + print(f"[reparam-rho] rho phase complete values={rho_after_phase}") + else: + phase = "joint" + if max_batches is None: + epoch_iter = dataloader + else: + epoch_iter = itertools.islice(dataloader, max_batches) + iterator = epoch_iter + if tqdm is not None and _tqdm_enabled(): + if progressive_cycle is not None: + if progressive_total is not None: + desc = ( + f"Reparam (cycle {progressive_cycle}/{progressive_total}, " + f"epoch {epoch_idx+1})" + ) + else: + desc = f"Reparam (cycle {progressive_cycle}, epoch {epoch_idx+1})" + else: + desc = f"Reparam (epoch {epoch_idx+1})" + iterator = tqdm(epoch_iter, desc=desc, unit="batch", total=max_batches) + + for batch in iterator: + input_ids = batch[0].to(args.device) + attention_mask = batch[1].to(args.device) + teacher_ids = input_ids.to(teacher_runtime_device) + teacher_mask = attention_mask.to(teacher_runtime_device) + + teacher_depth = layer_idx + 2 + student_depth = layer_idx + 1 + + autocast_ctx = ( + torch.autocast(device_type=device_type, dtype=amp_dtype) + if use_amp + else nullcontext() + ) + with autocast_ctx: + teacher_attn_ctx = ( + capture_module_output(teacher_attn) + if teacher_attn is not None + else nullcontext({"output": None}) + ) + teacher_mlp_ctx = ( + capture_module_output(teacher_mlp) + if teacher_mlp is not None + else nullcontext({"output": None}) + ) + with torch.no_grad(): + with teacher_attn_ctx as teacher_attn_cache, teacher_mlp_ctx as teacher_mlp_cache: + teacher_hidden = forward_truncated( + teacher_parent, + teacher_layer_attr, + teacher_layers, + teacher_depth, + teacher_ids, + attention_mask=teacher_mask, + ) + + student_attn_ctx = ( + capture_module_output(student_attn) + if student_attn is not None + else nullcontext({"output": None}) + ) + student_mlp_ctx = ( + capture_module_output(student_mlp) + if student_mlp is not None + else nullcontext({"output": None}) + ) + with student_attn_ctx as student_attn_cache, student_mlp_ctx as student_mlp_cache: + student_hidden = forward_truncated( + student_parent, + student_layer_attr, + virtual_layers, + student_depth, + input_ids, + attention_mask=attention_mask, + ) + + if teacher_hidden.device != student_hidden.device: + teacher_hidden = teacher_hidden.to(student_hidden.device) + + mse_loss = None + if hidden_mse_weight > 0.0: + diff = student_hidden - teacher_hidden + mse_loss = _masked_hidden_mse(diff, attention_mask) + if mse_loss is None: + continue + + attn_aux_loss = None + if attn_mse_weight > 0.0: + teacher_attn_hidden = teacher_attn_cache.get("output") + student_attn_hidden = student_attn_cache.get("output") + if teacher_attn_hidden is None or student_attn_hidden is None: + raise RuntimeError( + "Attention-output preservation is enabled, but the forward " + "hook did not capture attention outputs." + ) + if teacher_attn_hidden.device != student_attn_hidden.device: + teacher_attn_hidden = teacher_attn_hidden.to(student_attn_hidden.device) + attn_aux_loss = _masked_hidden_mse( + student_attn_hidden - teacher_attn_hidden, + attention_mask, + ) + if attn_aux_loss is None: + continue + + mlp_aux_loss = None + if mlp_mse_weight > 0.0: + teacher_mlp_hidden = teacher_mlp_cache.get("output") + student_mlp_hidden = student_mlp_cache.get("output") + if teacher_mlp_hidden is None or student_mlp_hidden is None: + raise RuntimeError( + "MLP-output preservation is enabled, but the forward hook " + "did not capture MLP outputs." + ) + if teacher_mlp_hidden.device != student_mlp_hidden.device: + teacher_mlp_hidden = teacher_mlp_hidden.to(student_mlp_hidden.device) + mlp_aux_loss = _masked_hidden_mse( + student_mlp_hidden - teacher_mlp_hidden, + attention_mask, + ) + if mlp_aux_loss is None: + continue + + kl_loss = None + if kl_weight > 0.0: + with torch.no_grad(): + teacher_outputs = teacher_model( + input_ids=teacher_ids, + attention_mask=teacher_mask, + use_cache=False, + ) + teacher_logits = teacher_outputs.logits + + virtual_container = torch.nn.ModuleList(virtual_layers) + with temporary_layers( + student_parent, student_layer_attr, virtual_container + ): + student_outputs = student_model( + input_ids=input_ids, + attention_mask=attention_mask, + use_cache=False, + ) + student_logits = student_outputs.logits + kl_loss = _masked_kl( + teacher_logits, + student_logits, + attention_mask, + temp=kl_temp, + detach_p=True, + chunk_size=int(getattr(args, "kl_chunk_size", 16384)), + ) + if kl_loss is None: + continue + + lambda_reg = None + if eta > 0.0: + reg_sum: Optional[torch.Tensor] = None + reg_elems = 0 + for orig, safe in reparam_layer._name_map.items(): + if safe not in trainable_safe_names: + continue + lam = torch.sigmoid(reparam_layer.gates[safe]).float() + target = gate_lambdas.get(orig) + if target is None: + target_t = 0.5 + elif isinstance(target, torch.Tensor): + target_t = target.to(device=lam.device, dtype=lam.dtype) + else: + target_t = float(target) + diff_lam = lam - target_t + family = _classify_param_family(orig) + scale = _family_reg_scale( + family, + attn_scale=attn_reg_scale, + mlp_scale=mlp_reg_scale, + ) + if scale <= 0.0: + continue + part = diff_lam.pow(2).sum() * scale + reg_sum = part if reg_sum is None else reg_sum + part + reg_elems += int(float(diff_lam.numel()) * scale) + if reg_elems > 0 and reg_sum is not None: + lambda_reg = reg_sum / float(reg_elems) + + u_reg = None + if gamma > 0.0: + reg_sum: Optional[torch.Tensor] = None + reg_elems = 0 + for orig, safe in reparam_layer._name_map.items(): + if safe not in trainable_safe_names: + continue + u = reparam_layer.u[safe].float() + param_a = params_a.get(orig) + param_b = params_b.get(orig) + if param_a is None or param_b is None or param_b.shape != param_a.shape: + continue + u0 = ( + torch.zeros_like(u) + if sequential_rho or additive_u + else _initial_reparam_u(param_a, param_b, u.device) + ) + diff_u = u - u0 + family = _classify_param_family(orig) + scale = _family_reg_scale( + family, + attn_scale=attn_reg_scale, + mlp_scale=mlp_reg_scale, + ) + if scale <= 0.0: + continue + part = diff_u.pow(2).sum() * scale + reg_sum = part if reg_sum is None else reg_sum + part + reg_elems += int(float(diff_u.numel()) * scale) + if reg_elems > 0 and reg_sum is not None: + u_reg = reg_sum / float(reg_elems) + + total_loss = None + total_loss = _add_weighted_loss(total_loss, mse_loss, hidden_mse_weight) + total_loss = _add_weighted_loss(total_loss, attn_aux_loss, attn_mse_weight) + total_loss = _add_weighted_loss(total_loss, mlp_aux_loss, mlp_mse_weight) + total_loss = _add_weighted_loss(total_loss, kl_loss, kl_weight * (kl_temp ** 2)) + total_loss = _add_weighted_loss(total_loss, lambda_reg, eta) + total_loss = _add_weighted_loss(total_loss, u_reg, gamma) + if total_loss is None: + continue + + rho_loss = None + rho_grads = None + if rho_enabled and (not sequential_rho or phase == "rho"): + if rho_objective in {"full", "hidden_kl"}: + rho_loss = _add_weighted_loss( + rho_loss, mse_loss, hidden_mse_weight + ) + if rho_objective == "full": + rho_loss = _add_weighted_loss( + rho_loss, mlp_aux_loss, mlp_mse_weight + ) + rho_loss = _add_weighted_loss( + rho_loss, kl_loss, kl_weight * (kl_temp ** 2) + ) + if rho_loss is None: + raise RuntimeError( + f"rho objective {rho_objective!r} has no active loss term" + ) + if not sequential_rho: + rho_grads = torch.autograd.grad( + rho_loss, + trainable_rho_params, + retain_graph=True, + allow_unused=False, + ) + + backward_loss = rho_loss if sequential_rho and phase == "rho" else total_loss + if backward_loss is None: + raise RuntimeError(f"no active backward loss for phase {phase!r}") + if grad_accum > 1: + backward_loss = backward_loss / grad_accum + if use_scaler: + scaler.scale(backward_loss).backward() + else: + backward_loss.backward() + + if rho_grads is not None: + for rho_param, rho_grad in zip(trainable_rho_params, rho_grads): + rho_param.grad = rho_grad.detach() + + if (step + 1) % grad_accum == 0: + if use_scaler: + scaler.unscale_(optimizer) + if phase not in first_grad_summary: + trainable_gates = [ + param + for safe, param in reparam_layer.gates.items() + if safe in trainable_safe_names + ] + trainable_u = [ + param + for safe, param in reparam_layer.u.items() + if safe in trainable_safe_names + ] + trainable_rho = list(reparam_layer.rho.parameters()) + phase_grad_summary = { + "step": step + 1, + "phase": phase, + "gate": _grad_debug_summary(trainable_gates), + "u": _grad_debug_summary(trainable_u), + "rho": _grad_debug_summary(trainable_rho), + } + first_grad_summary[phase] = phase_grad_summary + print( + "[reparam] first_grad " + f"phase={phase} step={phase_grad_summary['step']} " + f"gate_l2={phase_grad_summary['gate']['l2']:.6e} " + f"gate_mean_abs={phase_grad_summary['gate']['mean_abs']} " + f"gate_max_abs={phase_grad_summary['gate']['max_abs']:.6e} " + f"u_l2={phase_grad_summary['u']['l2']:.6e} " + f"u_mean_abs={phase_grad_summary['u']['mean_abs']} " + f"u_max_abs={phase_grad_summary['u']['max_abs']:.6e} " + f"rho_l2={phase_grad_summary['rho']['l2']:.6e}" + ) + if max_grad_norm is not None: + if trainable_base_params and (not sequential_rho or phase == "u"): + torch.nn.utils.clip_grad_norm_( + trainable_base_params, + float(max_grad_norm), + ) + if trainable_rho_params and (not sequential_rho or phase == "rho"): + torch.nn.utils.clip_grad_norm_( + trainable_rho_params, + float(max_grad_norm), + ) + if use_scaler: + scaler.step(optimizer) + scaler.update() + else: + optimizer.step() + if rho_enabled and (not sequential_rho or phase == "rho"): + with torch.no_grad(): + for rho in reparam_layer.rho.values(): + rho.clamp_(rho_min, rho_max) + optimizer.zero_grad(set_to_none=True) + + if log_steps and (step == 0 or (step + 1) % log_steps == 0): + log_parts = [f"phase={phase}", f"loss={total_loss.item():.6e}"] + if rho_loss is not None: + log_parts.append(f"rho_loss={rho_loss.item():.6e}") + if mse_loss is not None: + log_parts.append(f"mse={mse_loss.item():.6e}") + else: + log_parts.append("mse=disabled") + if attn_aux_loss is not None: + log_parts.append(f"attn_mse={attn_aux_loss.item():.6e}") + elif attn_mse_weight > 0.0: + log_parts.append("attn_mse=nan") + if mlp_aux_loss is not None: + log_parts.append(f"mlp_mse={mlp_aux_loss.item():.6e}") + elif mlp_mse_weight > 0.0: + log_parts.append("mlp_mse=nan") + if kl_loss is not None: + log_parts.append(f"kl={kl_loss.item():.6e}") + if lambda_reg is not None: + log_parts.append(f"lam_reg={lambda_reg.item():.6e}") + if u_reg is not None: + log_parts.append(f"u_reg={u_reg.item():.6e}") + lambda_now = _lambda_summary_from_gates( + reparam_layer._name_map, + reparam_layer.gates, + ) + log_parts.append(f"lambda_mean={lambda_now.get('mean')}") + log_parts.append(f"lambda_std={lambda_now.get('std')}") + log_parts.append( + f"lambda_abs0.5={lambda_now.get('mean_abs_from_0_5')}" + ) + if rho_enabled: + log_parts.append(f"rho={reparam_layer.rho_values()}") + print( + f"[reparam] epoch={epoch_idx+1} step={step+1} " + " ".join(log_parts) + ) + # Release graph-heavy batch tensors before the next forward. On 70B + # reparam runs, AdamW state leaves little headroom, so carrying the + # previous logits/outputs into the next batch can trip OOM. + teacher_outputs = None + teacher_logits = None + student_outputs = None + student_logits = None + teacher_hidden = None + student_hidden = None + teacher_attn_hidden = None + student_attn_hidden = None + teacher_mlp_hidden = None + student_mlp_hidden = None + teacher_attn_cache = None + teacher_mlp_cache = None + student_attn_cache = None + student_mlp_cache = None + diff = None + mse_loss = None + attn_aux_loss = None + mlp_aux_loss = None + kl_loss = None + lambda_reg = None + u_reg = None + rho_loss = None + rho_grads = None + total_loss = None + if device_type == "cuda" and torch.cuda.is_available(): + gc.collect() + torch.cuda.empty_cache() + + step += 1 + + final_lambdas = reparam_layer.gate_lambdas() + lambda_final_summary = _tensor_debug_summary(final_lambdas) + lambda_delta_from_init = _lambda_delta_summary(final_lambdas, gate_lambdas) + u_delta_summary = _u_delta_debug_summary( + reparam_layer._name_map, + reparam_layer.u, + params_a, + params_b, + initial_scale=0.0 if sequential_rho or additive_u else 1.0, + ) + u_fisher_orthogonality = _sequential_u_fisher_orthogonality(reparam_layer) + merged = reparam_layer.materialize_into_layer_a() + stats: Dict[str, object] = { + "enabled": True, + "epochs": total_epochs, + "requested_epochs": requested_epochs, + "skip_train": skip_train, + "trained": bool(do_train), + "lr": float(args.distill_lr), + "hidden_mse_weight": hidden_mse_weight, + "attn_mse_weight": attn_mse_weight, + "mlp_mse_weight": mlp_mse_weight, + "eta": eta, + "gamma": gamma, + "attn_reg_scale": attn_reg_scale, + "mlp_reg_scale": mlp_reg_scale, + "param_subset": param_subset, + "train_subset": train_subset, + "num_gates": len(final_lambdas), + "num_attn_gates": family_counts["attn"], + "num_mlp_gates": family_counts["mlp"], + "num_other_gates": family_counts["other"], + "num_trainable_attn_gates": train_family_counts["attn"], + "num_trainable_mlp_gates": train_family_counts["mlp"], + "num_trainable_other_gates": train_family_counts["other"], + "lambda_init_summary": lambda_init_summary, + "lambda_final_summary": lambda_final_summary, + "lambda_delta_from_init": lambda_delta_from_init, + "first_grad_summary": first_grad_summary, + "u_delta_summary": u_delta_summary, + "rho_objective": rho_objective, + "rho_schedule": rho_schedule, + "additive_u": additive_u, + "rho_epochs": rho_epochs if sequential_rho else 0.0, + "rho_lr": rho_lr if rho_enabled else None, + "rho_bounds": [rho_min, rho_max] if rho_enabled else None, + "rho_after_phase": rho_after_phase, + "rho_final": reparam_layer.rho_values(), + "u_fisher_orthogonality": u_fisher_orthogonality, + } + return merged, final_lambdas, stats + + +class LoRALinear(torch.nn.Module): + def __init__( + self, + base: torch.nn.Linear, + rank: int, + alpha: float, + dropout: float, + ) -> None: + super().__init__() + if rank <= 0: + raise ValueError("LoRA rank must be positive") + self.base = base + self.rank = int(rank) + self.alpha = float(alpha) + self.scaling = self.alpha / float(self.rank) + self.enabled = True + if dropout > 0: + self.dropout = torch.nn.Dropout(dropout) + else: + self.dropout = torch.nn.Identity() + + self.lora_A = torch.nn.Linear(base.in_features, self.rank, bias=False) + self.lora_B = torch.nn.Linear(self.rank, base.out_features, bias=False) + torch.nn.init.kaiming_uniform_(self.lora_A.weight, a=math.sqrt(5)) + torch.nn.init.zeros_(self.lora_B.weight) + + self.lora_A.to(device=base.weight.device, dtype=base.weight.dtype) + self.lora_B.to(device=base.weight.device, dtype=base.weight.dtype) + self.merged = False + + def lora_parameters(self) -> List[torch.nn.Parameter]: + return [*self.lora_A.parameters(), *self.lora_B.parameters()] + + def forward(self, x: torch.Tensor) -> torch.Tensor: + result = self.base(x) + if self.merged or not self.enabled: + return result + lora_out = self.lora_B(self.lora_A(self.dropout(x))) + return result + lora_out * self.scaling + + def merge(self) -> None: + if self.merged: + return + delta = torch.matmul(self.lora_B.weight, self.lora_A.weight) + delta = delta.to(dtype=self.base.weight.dtype) * self.scaling + self.base.weight.data.add_(delta) + self.merged = True + + +def _get_child_module(parent: torch.nn.Module, part: str) -> torch.nn.Module: + if isinstance(parent, (torch.nn.ModuleList, torch.nn.Sequential)) and part.isdigit(): + return parent[int(part)] + if isinstance(parent, torch.nn.ModuleDict): + return parent[part] + return getattr(parent, part) + + +def _set_child_module(parent: torch.nn.Module, part: str, module: torch.nn.Module) -> None: + if isinstance(parent, (torch.nn.ModuleList, torch.nn.Sequential)) and part.isdigit(): + parent[int(part)] = module + return + if isinstance(parent, torch.nn.ModuleDict): + parent[part] = module + return + setattr(parent, part, module) + + +def _resolve_parent_module( + root: torch.nn.Module, module_name: str +) -> Optional[tuple]: + if not module_name: + return None + parts = module_name.split(".") + parent = root + for part in parts[:-1]: + parent = _get_child_module(parent, part) + return parent, parts[-1] + + +def _resolve_module_by_path(root: torch.nn.Module, module_path: str) -> Optional[torch.nn.Module]: + if not module_path: + return None + parts = [part for part in module_path.split(".") if part] + node = root + for part in parts: + try: + node = _get_child_module(node, part) + except Exception: + return None + return node + + +def _resolve_layer_container_for_lora( + model: torch.nn.Module, layer_path: Optional[str] +) -> Tuple[Optional[str], Optional[object]]: + """Resolve transformer layer container with optional auto-detection. + + Mirrors the candidate path strategy used elsewhere, so LoRA filtering can work + even when --layer_path is not provided. + """ + if isinstance(layer_path, str) and layer_path and layer_path.lower() != "none": + container = _resolve_module_by_path(model, layer_path) + if container is not None: + try: + list(container) + return layer_path, container + except TypeError: + pass + + candidate_paths = [ + "model.layers", # LLaMA, Mistral, Qwen2, Gemma + "model.decoder.layers", # OPT + "transformer.h", # GPT-2, GPT-J, Bloom, Falcon + "transformer.blocks", # MPT + "gpt_neox.layers", # GPT-NeoX + "layers", # fallback + ] + for path in candidate_paths: + container = _resolve_module_by_path(model, path) + if container is None: + continue + try: + list(container) + except TypeError: + continue + return path, container + + return None, None + + +def _parse_exclude_pairs_local(raw_values, num_pairs: int) -> Set[int]: + if not raw_values or num_pairs <= 0: + return set() + exclude: Set[int] = set() + for item in raw_values: + if item is None: + continue + for part in str(item).split(","): + part = part.strip() + if not part: + continue + try: + idx = int(part) + except ValueError as exc: + raise SystemExit("--exclude_pairs must contain integers.") from exc + if idx < 0: + idx = num_pairs + idx + if 0 <= idx < num_pairs: + exclude.add(idx) + return exclude + + +def _extract_layer_index_from_module_name( + module_name: str, layer_path: str +) -> Optional[int]: + if not layer_path: + return None + prefix = f"{layer_path}." + if not module_name.startswith(prefix): + return None + rest = module_name[len(prefix) :] + if not rest: + return None + idx_text = rest.split(".", 1)[0] + if not idx_text.isdigit(): + return None + return int(idx_text) + + +def _select_linear_modules_for_lora_targets( + model: torch.nn.Module, + args: argparse.Namespace, + *, + log_tag: str, +) -> Tuple[List[Tuple[str, torch.nn.Linear]], Optional[Set[str]], Set[int], Optional[str]]: + raw_targets = getattr(args, "lora_target_modules", None) + target_modules: Optional[Set[str]] = None + if raw_targets: + target_modules = {str(item) for item in raw_targets if str(item)} + + exclude_layer_indices: Set[int] = set() + resolved_layer_path: Optional[str] = None + if bool(getattr(args, "lora_respect_exclude_pairs", False)): + requested_layer_path = getattr(args, "layer_path", None) + resolved_layer_path, layer_container = _resolve_layer_container_for_lora( + model, requested_layer_path + ) + if isinstance(layer_container, (torch.nn.ModuleList, list, tuple)): + num_pairs = max(len(layer_container) - 1, 0) + exclude_pairs = _parse_exclude_pairs_local( + getattr(args, "exclude_pairs", None), num_pairs + ) + for pair_idx in exclude_pairs: + exclude_layer_indices.add(pair_idx) + exclude_layer_indices.add(pair_idx + 1) + else: + print( + f"[{log_tag}] Warning: --lora_respect_exclude_pairs enabled, but " + f"could not resolve layer path '{requested_layer_path}'." + ) + + linear_modules = [ + (name, module) + for name, module in model.named_modules() + if isinstance(module, torch.nn.Linear) + and (target_modules is None or name.split(".")[-1] in target_modules) + and ( + not exclude_layer_indices + or _extract_layer_index_from_module_name(name, resolved_layer_path or "") + not in exclude_layer_indices + ) + ] + return linear_modules, target_modules, exclude_layer_indices, resolved_layer_path + + +def apply_lora_adapters( + model: torch.nn.Module, args: argparse.Namespace +) -> List[LoRALinear]: + if args.lora_rank <= 0: + raise SystemExit("--lora_rank must be > 0 when --lora_epochs > 0") + linear_modules, target_modules, exclude_layer_indices, _ = ( + _select_linear_modules_for_lora_targets(model, args, log_tag="lora") + ) + if not linear_modules: + raise SystemExit( + "No Linear modules found for LoRA adapters " + "(check --lora_target_modules / --exclude_pairs / --lora_respect_exclude_pairs)." + ) + + lora_modules: List[LoRALinear] = [] + for name, module in linear_modules: + resolved = _resolve_parent_module(model, name) + if resolved is None: + continue + parent, attr = resolved + wrapped = LoRALinear( + base=module, + rank=args.lora_rank, + alpha=args.lora_alpha, + dropout=args.lora_dropout, + ) + _set_child_module(parent, attr, wrapped) + lora_modules.append(wrapped) + + for param in model.parameters(): + param.requires_grad_(False) + for lora_module in lora_modules: + for param in lora_module.lora_parameters(): + param.requires_grad_(True) + + total_params = sum(p.numel() for p in model.parameters()) + trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + percent = 100.0 * trainable_params / max(total_params, 1) + target_note = "" + if target_modules is not None: + target_note = f" target={sorted(target_modules)}" + exclude_note = "" + if exclude_layer_indices: + exclude_note = f" excluded_layers={sorted(exclude_layer_indices)}" + print( + "[lora] Applied adapters to " + f"{len(lora_modules)} linear modules " + f"({trainable_params}/{total_params} trainable, {percent:.4f}%)." + f"{target_note}{exclude_note}" + ) + return lora_modules + + +def merge_lora_adapters(model: torch.nn.Module) -> None: + lora_entries = [ + (name, module) + for name, module in model.named_modules() + if isinstance(module, LoRALinear) + ] + for name, module in lora_entries: + module.merge() + resolved = _resolve_parent_module(model, name) + if resolved is None: + continue + parent, attr = resolved + _set_child_module(parent, attr, module.base) + + +def set_lora_enabled(lora_modules: List[LoRALinear], enabled: bool) -> None: + for module in lora_modules: + module.enabled = enabled + + +def lora_ce_finetune( + model: torch.nn.Module, + dataloader, + eval_tokenizer, + eval_datasets: List[str], + eval_configs: List[Optional[str]], + eval_history: List[Dict[str, object]], + args: argparse.Namespace, + eval_dataloaders: Optional[Dict[str, object]] = None, + progressive_cycle: Optional[int] = None, + progressive_total: Optional[int] = None, +) -> None: + total_epochs = float(args.lora_epochs) + if total_epochs <= 0: + return + + use_kl = bool(getattr(args, "lora_kl_enabled", False)) + kl_weight = float(getattr(args, "lora_kl_weight", 0.0)) + kl_temp = float(getattr(args, "lora_kl_temp", 1.0)) + if use_kl: + if kl_weight < 0.0: + raise SystemExit("--lora_kl_weight must be >= 0") + if kl_temp <= 0.0: + raise SystemExit("--lora_kl_temp must be > 0") + if kl_weight == 0.0: + use_kl = False + + lora_modules = apply_lora_adapters(model, args) + if not lora_modules: + return + + model.train() + + lora_params = [] + for module in lora_modules: + lora_params.extend(module.lora_parameters()) + + optimizer = torch.optim.AdamW( + lora_params, + lr=args.lora_lr, + weight_decay=args.lora_weight_decay, + ) + + device_type = torch.device(args.device).type + amp_dtype = None + if args.dtype == "float16": + amp_dtype = torch.float16 + elif args.dtype == "bfloat16": + amp_dtype = torch.bfloat16 + use_amp = amp_dtype is not None and device_type == "cuda" + use_scaler = use_amp and amp_dtype == torch.float16 + scaler = torch.cuda.amp.GradScaler() if use_scaler else None + + full_epochs = int(total_epochs) + fractional = total_epochs - full_epochs + if fractional < 1e-8: + fractional = 0.0 + + epoch_plan = [(epoch_idx, None) for epoch_idx in range(full_epochs)] + if fractional > 0: + try: + batches_per_epoch = len(dataloader) + except TypeError as exc: + raise SystemExit( + "Fractional lora epochs require a dataloader with finite length." + ) from exc + if batches_per_epoch > 0: + frac_batches = int(round(fractional * batches_per_epoch)) + if frac_batches <= 0: + frac_batches = 1 + epoch_plan.append((full_epochs, frac_batches)) + + step = 0 + for epoch_idx, max_batches in epoch_plan: + if max_batches is None: + epoch_iter = dataloader + else: + epoch_iter = itertools.islice(dataloader, max_batches) + iterator = epoch_iter + if tqdm is not None and _tqdm_enabled(): + if progressive_cycle is not None: + if progressive_total is not None: + desc = ( + f"LoRA (cycle {progressive_cycle}/{progressive_total}, " + f"epoch {epoch_idx+1})" + ) + else: + desc = f"LoRA (cycle {progressive_cycle}, epoch {epoch_idx+1})" + else: + desc = f"LoRA (epoch {epoch_idx+1})" + iterator = tqdm( + epoch_iter, + desc=desc, + unit="batch", + total=max_batches, + ) + for batch in iterator: + input_ids = batch[0].to(args.device) + attention_mask = batch[1].to(args.device) + autocast_ctx = ( + torch.autocast(device_type=device_type, dtype=amp_dtype) + if use_amp + else nullcontext() + ) + with autocast_ctx: + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + use_cache=False, + ) + logits = outputs.logits + shift_logits = logits[:, :-1, :].contiguous() + shift_labels = input_ids[:, 1:].contiguous() + shift_mask = attention_mask[:, 1:].contiguous() + ce_flat = F.cross_entropy( + shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1), + reduction="none", + ) + ce_denom = shift_mask.sum() + if ce_denom.item() == 0: + continue + ce_loss = ( + ce_flat * shift_mask.view(-1).to(ce_flat.dtype) + ).sum() / ce_denom + kl_loss = None + if use_kl: + set_lora_enabled(lora_modules, False) + with torch.no_grad(): + base_outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + use_cache=False, + ) + base_logits = base_outputs.logits + set_lora_enabled(lora_modules, True) + if base_logits.device != shift_logits.device: + base_logits = base_logits.to(shift_logits.device) + shift_base_logits = base_logits[:, :-1, :].contiguous() + log_p_pre = F.log_softmax(shift_base_logits / kl_temp, dim=-1) + log_p_post = F.log_softmax(shift_logits / kl_temp, dim=-1) + p_pre = log_p_pre.exp() + kl_flat = (p_pre * (log_p_pre - log_p_post)).sum(dim=-1) + kl_loss = ( + kl_flat * shift_mask.to(kl_flat.dtype) + ).sum() / ce_denom + + total_loss = ce_loss + if kl_loss is not None: + total_loss = total_loss + (kl_weight * (kl_temp ** 2) * kl_loss) + + if args.lora_grad_accum_steps > 1: + total_loss = total_loss / args.lora_grad_accum_steps + if use_scaler: + scaler.scale(total_loss).backward() + else: + total_loss.backward() + + if (step + 1) % args.lora_grad_accum_steps == 0: + if args.lora_max_grad_norm is not None: + if use_scaler: + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_( + lora_params, + args.lora_max_grad_norm, + ) + if use_scaler: + scaler.step(optimizer) + scaler.update() + else: + optimizer.step() + optimizer.zero_grad(set_to_none=True) + + if args.lora_eval_every and (step + 1) % args.lora_eval_every == 0: + prev_mode = model.training + model.eval() + eval_device = args.eval_device or args.device + if eval_dataloaders is not None: + results = ppl_eval.evaluate_ppl_dataloaders( + model, + eval_dataloaders, + eval_device, + max_batches=args.lora_eval_max_batches, + ) + else: + results = ppl_eval.evaluate_ppl_datasets( + model, + eval_tokenizer, + datasets=eval_datasets, + configs=eval_configs, + split=args.eval_split, + text_field=args.eval_text_field, + num_samples=args.eval_num_samples, + seq_len=args.eval_seq_len, + batch_size=args.eval_batch_size or args.batch_size, + device=eval_device, + seed=args.seed, + shuffle=False, + model_family=args.eval_model_family, + add_bos=args.eval_add_bos, + max_batches=args.lora_eval_max_batches, + cache_dir=args.eval_cache_dir, + num_workers=args.eval_num_workers, + ) + eval_history.append({"step": step + 1, "ppl": results}) + print(f"[lora] eval step={step+1}: {results}") + if prev_mode: + model.train() + + if args.lora_log_steps and ( + step == 0 or (step + 1) % args.lora_log_steps == 0 + ): + log_parts = [f"loss={total_loss.item():.6f}"] + if kl_loss is not None: + log_parts.append(f"kl={kl_loss.item():.6f}") + print( + f"[lora] epoch={epoch_idx+1} step={step+1} " + + " ".join(log_parts) + ) + step += 1 + + merge_lora_adapters(model) + + +def _masked_kl( + logits_p: torch.Tensor, + logits_q: torch.Tensor, + attention_mask: torch.Tensor, + temp: float, + detach_p: bool = True, + chunk_size: int = 16384, +) -> Optional[torch.Tensor]: + if chunk_size <= 0: + raise ValueError("chunk_size must be positive") + shift_mask = attention_mask[:, 1:].contiguous() + denom = shift_mask.sum() + if denom.item() == 0: + return None + + # Avoid materializing a full shifted logits copy on GPU. We stream over the + # vocab dimension below, so the sliced view is sufficient and lowers peak + # memory for sharded comm preconditioning. + p = logits_p[:, :-1, :] + q = logits_q[:, :-1, :] + q_device = q.device + q_dtype = q.dtype + + if (not detach_p) and p.device != q_device: + p = p.to(q_device) + + def _streaming_logsumexp( + logits: torch.Tensor, + target_device: torch.device, + target_dtype: torch.dtype, + ) -> torch.Tensor: + max_logits = logits.amax(dim=-1, keepdim=True) + max_scaled = max_logits / temp + sum_exp = None + vocab = logits.size(-1) + for start in range(0, vocab, chunk_size): + end = min(start + chunk_size, vocab) + chunk = logits[..., start:end] + if chunk.device != target_device or chunk.dtype != target_dtype: + chunk = chunk.to(device=target_device, dtype=target_dtype) + else: + chunk = chunk.to(dtype=target_dtype) + chunk_scaled = chunk / temp + partial = (chunk_scaled - max_scaled.to(device=chunk_scaled.device, dtype=chunk_scaled.dtype)).exp().sum( + dim=-1, + keepdim=True, + ) + sum_exp = partial if sum_exp is None else sum_exp + partial + return max_scaled.to(device=target_device, dtype=target_dtype) + sum_exp.log() + + logsumexp_q = _streaming_logsumexp(q, q_device, q_dtype) + logsumexp_p = _streaming_logsumexp( + p, + q_device if detach_p else p.device, + q_dtype if detach_p else p.dtype, + ) + if detach_p: + logsumexp_p = logsumexp_p.detach() + + kl_flat = None + vocab = q.size(-1) + for start in range(0, vocab, chunk_size): + end = min(start + chunk_size, vocab) + q_chunk = q[..., start:end].to(dtype=q_dtype) + log_q_chunk = (q_chunk / temp) - logsumexp_q + + p_chunk = p[..., start:end] + if p_chunk.device != q_device or p_chunk.dtype != q_dtype: + p_chunk = p_chunk.to(device=q_device, dtype=q_dtype) + else: + p_chunk = p_chunk.to(dtype=q_dtype) + log_p_chunk = (p_chunk / temp) - logsumexp_p.to(device=q_device, dtype=q_dtype) + if detach_p: + log_p_chunk = log_p_chunk.detach() + p_probs_chunk = log_p_chunk.exp() + chunk_kl = (p_probs_chunk * (log_p_chunk - log_q_chunk)).sum(dim=-1) + kl_flat = chunk_kl if kl_flat is None else kl_flat + chunk_kl + + mask = shift_mask.to(device=kl_flat.device, dtype=kl_flat.dtype) + denom = denom.to(device=kl_flat.device, dtype=kl_flat.dtype) + return (kl_flat * mask).sum() / denom + + +def _extract_hidden_tensor(output: object) -> Optional[torch.Tensor]: + if isinstance(output, torch.Tensor): + return output + if isinstance(output, (tuple, list)) and output: + first = output[0] + if isinstance(first, torch.Tensor): + return first + return None + + +def _grad_l2_norm(grads: List[Optional[torch.Tensor]]) -> float: + total = 0.0 + for grad in grads: + if grad is None: + continue + total += float(grad.detach().float().pow(2).sum().item()) + if total <= 0.0: + return 0.0 + return float(math.sqrt(total)) + + +def _register_forward_pre_hook_with_optional_kwargs(layer, hook): + try: + handle = layer.register_forward_pre_hook(hook, with_kwargs=True) + return handle + except TypeError: + def wrapper(module, inputs): + return hook(module, inputs, None) + + return layer.register_forward_pre_hook(wrapper) + + +def _batch_input_ids_attention(batch) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: + if isinstance(batch, torch.Tensor): + return batch, None + if isinstance(batch, dict): + input_ids = batch.get("input_ids") + attention_mask = batch.get("attention_mask") + return input_ids, attention_mask + if ( + isinstance(batch, (tuple, list)) + and len(batch) == 1 + and isinstance(batch[0], torch.Tensor) + ): + return batch[0], None + if isinstance(batch, (tuple, list)) and len(batch) >= 2: + return batch[0], batch[1] + return None, None + + +def _new_gap_accumulator() -> Dict[str, float]: + return { + "mse_sum": 0.0, + "relative_sum": 0.0, + "cosine_sum": 0.0, + "r1_norm_sum": 0.0, + "r0_norm_sum": 0.0, + "token_count": 0.0, + "elem_count": 0.0, + "batches": 0.0, + } + + +def _accumulate_gap_component( + acc: Dict[str, float], + r1_like: torch.Tensor, + r0_like: torch.Tensor, + attention_mask: torch.Tensor, + interaction_eps: float, +) -> bool: + r1 = r1_like.float() + r0 = r0_like.float() + if r0.device != r1.device: + r0 = r0.to(r1.device) + if r0.shape != r1.shape or r1.dim() < 3: + return False + + mask = attention_mask.to(device=r1.device, dtype=torch.float32) + mask_sum = float(mask.sum().item()) + if mask_sum <= 0.0: + return False + + hidden_dim = int(r1.size(-1)) + mask_3d = mask.unsqueeze(-1) + gap = r1 - r0 + gap_sq = gap.pow(2).sum(dim=-1) + r1_sq = r1.pow(2).sum(dim=-1) + r0_sq = r0.pow(2).sum(dim=-1) + relative = gap_sq / (r1_sq + float(interaction_eps)) + cosine = F.cosine_similarity(r1, r0, dim=-1, eps=float(interaction_eps)) + + acc["mse_sum"] += float((gap.pow(2) * mask_3d).sum().item()) + acc["relative_sum"] += float((relative * mask).sum().item()) + acc["cosine_sum"] += float((cosine * mask).sum().item()) + acc["r1_norm_sum"] += float((r1_sq * mask).sum().item()) + acc["r0_norm_sum"] += float((r0_sq * mask).sum().item()) + acc["token_count"] += mask_sum + acc["elem_count"] += mask_sum * hidden_dim + acc["batches"] += 1.0 + return True + + +def _finalize_gap_component(acc: Dict[str, float]) -> Optional[Dict[str, object]]: + token_count = float(acc.get("token_count", 0.0)) + elem_count = float(acc.get("elem_count", 0.0)) + if token_count <= 0.0 or elem_count <= 0.0: + return None + mean_cosine = float(acc["cosine_sum"] / token_count) + return { + "batches": int(acc.get("batches", 0.0)), + "tokens": int(token_count), + "mse": float(acc["mse_sum"] / elem_count), + "relative": float(acc["relative_sum"] / token_count), + "cosine": mean_cosine, + "cosine_distance": float(1.0 - mean_cosine), + "r1_norm_mean": float(acc["r1_norm_sum"] / token_count), + "r0_norm_mean": float(acc["r0_norm_sum"] / token_count), + } + + +def _format_comm_gap_stats(stats: Dict[str, object]) -> str: + parts = [ + f"relative={stats.get('relative'):.6e}", + f"mse={stats.get('mse'):.6e}", + f"cos_dist={stats.get('cosine_distance'):.6e}", + ] + components = stats.get("components") + if isinstance(components, dict): + for name in ("attn", "mlp", "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj"): + value = components.get(name) + if isinstance(value, dict) and "relative" in value: + parts.append(f"{name}_relative={value.get('relative'):.6e}") + if "cosine_distance" in value: + parts.append(f"{name}_cos_dist={value.get('cosine_distance'):.6e}") + return " ".join(parts) + + +def _safe_ratio(numer: object, denom: object) -> Optional[float]: + try: + n = float(numer) + d = float(denom) + except (TypeError, ValueError): + return None + if not math.isfinite(n) or not math.isfinite(d) or d == 0.0: + return None + return n / d + + +def _comm_gap_ratio(pre_stats: Dict[str, object], post_stats: Dict[str, object]) -> Dict[str, object]: + ratio: Dict[str, object] = {} + for key in ("mse", "relative", "cosine_distance"): + if key not in pre_stats or key not in post_stats: + continue + value = _safe_ratio(post_stats.get(key), pre_stats.get(key)) + if value is not None: + ratio[key] = value + mse_ratio = ratio.get("mse") + if isinstance(mse_ratio, float) and mse_ratio >= 0.0: + ratio["l2_norm_from_mse"] = math.sqrt(mse_ratio) + + pre_components = pre_stats.get("components") + post_components = post_stats.get("components") + if isinstance(pre_components, dict) and isinstance(post_components, dict): + component_ratios = {} + for name, pre_value in pre_components.items(): + post_value = post_components.get(name) + if not isinstance(pre_value, dict) or not isinstance(post_value, dict): + continue + one_component: Dict[str, float] = {} + for key in ("mse", "relative", "cosine_distance"): + if key not in pre_value or key not in post_value: + continue + value = _safe_ratio(post_value.get(key), pre_value.get(key)) + if value is not None: + one_component[key] = value + mse_ratio = one_component.get("mse") + if isinstance(mse_ratio, float) and mse_ratio >= 0.0: + one_component["l2_norm_from_mse"] = math.sqrt(mse_ratio) + if one_component: + component_ratios[name] = one_component + if component_ratios: + ratio["components"] = component_ratios + return ratio + + +def _measure_commutator_gap( + student_model: torch.nn.Module, + student_layers: List[torch.nn.Module], + dataloader, + pair_idx: int, + args: argparse.Namespace, + max_batches: int, + interaction_eps: float, +) -> Dict[str, object]: + """Measure r1-r0 on a fixed pair without updating model weights.""" + if dataloader is None: + return {"enabled": False, "reason": "no_dataloader"} + if pair_idx < 0 or pair_idx >= len(student_layers) - 1: + return {"enabled": False, "reason": "invalid_pair"} + + was_training = student_model.training + student_model.eval() + + device_type = torch.device(args.device).type + amp_dtype = None + if args.dtype == "float16": + amp_dtype = torch.float16 + elif args.dtype == "bfloat16": + amp_dtype = torch.bfloat16 + use_amp = amp_dtype is not None and device_type == "cuda" + autocast_ctx = ( + torch.autocast(device_type=device_type, dtype=amp_dtype) + if use_amp + else nullcontext() + ) + + block_acc = _new_gap_accumulator() + component_accs: Dict[str, Dict[str, float]] = {} + batches = 0 + skipped = 0 + receiver_attn = None + receiver_mlp = None + try: + receiver_attn = find_attention_module(student_layers[pair_idx + 1]) + except Exception: + receiver_attn = None + try: + receiver_mlp = find_mlp_module(student_layers[pair_idx + 1]) + except Exception: + receiver_mlp = None + if receiver_attn is not None: + component_accs["attn"] = _new_gap_accumulator() + mlp_submodules: Dict[str, torch.nn.Module] = {} + if receiver_mlp is not None: + component_accs["mlp"] = _new_gap_accumulator() + for sub_name in ("gate_proj", "up_proj", "down_proj"): + submodule = getattr(receiver_mlp, sub_name, None) + if isinstance(submodule, torch.nn.Module): + component_name = f"mlp.{sub_name}" + mlp_submodules[component_name] = submodule + component_accs[component_name] = _new_gap_accumulator() + + limit = int(max_batches) + try: + iterator = iter(dataloader) + with torch.no_grad(): + for batch in iterator: + if limit > 0 and batches >= limit: + break + input_ids, attention_mask = _batch_input_ids_attention(batch) + if input_ids is None: + skipped += 1 + continue + input_ids = input_ids.to(args.device) + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + attention_mask = attention_mask.to(args.device) + + capture: Dict[str, object] = { + "h_l": None, + "h_lp1": None, + "y1": None, + "attn1": None, + "mlp1": None, + "recv_args": None, + "recv_kwargs": None, + } + + def _hook_l(_module, inputs, _output): + if inputs and isinstance(inputs[0], torch.Tensor): + capture["h_l"] = inputs[0] + + def _hook_recv_pre(_module, inputs, kwargs): + capture["recv_args"] = inputs + capture["recv_kwargs"] = kwargs + + def _hook_recv(_module, inputs, output): + if inputs and isinstance(inputs[0], torch.Tensor): + capture["h_lp1"] = inputs[0] + capture["y1"] = _extract_hidden_tensor(output) + + def _hook_attn1(_module, _inputs, output): + capture["attn1"] = _extract_hidden_tensor(output) + + def _hook_mlp1(_module, _inputs, output): + capture["mlp1"] = _extract_hidden_tensor(output) + + def _make_component_hook(storage: Dict[str, object], key: str): + def _hook(_module, _inputs, output): + storage[key] = _extract_hidden_tensor(output) + + return _hook + + handles: List[object] = [ + student_layers[pair_idx].register_forward_hook(_hook_l), + _register_forward_pre_hook_with_optional_kwargs( + student_layers[pair_idx + 1], _hook_recv_pre + ), + student_layers[pair_idx + 1].register_forward_hook(_hook_recv), + ] + if receiver_attn is not None: + handles.append(receiver_attn.register_forward_hook(_hook_attn1)) + if receiver_mlp is not None: + handles.append(receiver_mlp.register_forward_hook(_hook_mlp1)) + for name, module in mlp_submodules.items(): + handles.append( + module.register_forward_hook( + _make_component_hook(capture, f"{name}1") + ) + ) + try: + with autocast_ctx: + student_model( + input_ids=input_ids, + attention_mask=attention_mask, + use_cache=False, + ) + finally: + for handle in handles: + try: + handle.remove() + except Exception: + pass + + h_l = capture.get("h_l") + h_lp1 = capture.get("h_lp1") + y1 = capture.get("y1") + recv_args = capture.get("recv_args") + recv_kwargs = capture.get("recv_kwargs") + if not ( + isinstance(h_l, torch.Tensor) + and isinstance(h_lp1, torch.Tensor) + and isinstance(y1, torch.Tensor) + and isinstance(recv_args, tuple) + and len(recv_args) > 0 + and isinstance(recv_args[0], torch.Tensor) + ): + skipped += 1 + continue + + first_hidden = recv_args[0] + receiver_device = first_hidden.device + call_args = list(_move_tensor_tree_to_device(recv_args, receiver_device)) + call_args[0] = h_l.detach().to( + device=receiver_device, + dtype=first_hidden.dtype, + ) + call_kwargs = ( + _move_tensor_tree_to_device(dict(recv_kwargs), receiver_device) + if isinstance(recv_kwargs, dict) + else {} + ) + y0_capture: Dict[str, object] = {"attn0": None, "mlp0": None} + + def _hook_attn0(_module, _inputs, output): + y0_capture["attn0"] = _extract_hidden_tensor(output) + + def _hook_mlp0(_module, _inputs, output): + y0_capture["mlp0"] = _extract_hidden_tensor(output) + + y0_handles: List[object] = [] + if receiver_attn is not None: + y0_handles.append(receiver_attn.register_forward_hook(_hook_attn0)) + if receiver_mlp is not None: + y0_handles.append(receiver_mlp.register_forward_hook(_hook_mlp0)) + for name, module in mlp_submodules.items(): + y0_handles.append( + module.register_forward_hook( + _make_component_hook(y0_capture, f"{name}0") + ) + ) + with autocast_ctx: + try: + y0_raw = student_layers[pair_idx + 1](*tuple(call_args), **call_kwargs) + finally: + for handle in y0_handles: + try: + handle.remove() + except Exception: + pass + y0 = _extract_hidden_tensor(y0_raw) + if not isinstance(y0, torch.Tensor): + skipped += 1 + continue + + if y0.device != y1.device: + y0 = y0.to(y1.device) + h_lp1_detached = h_lp1.detach().to(device=y1.device, dtype=y1.dtype) + h_l_for_res = h_l.detach().to(device=y0.device, dtype=y0.dtype) + r1 = (y1 - h_lp1_detached).float() + r0 = (y0 - h_l_for_res).float() + if not _accumulate_gap_component( + block_acc, + r1, + r0, + attention_mask, + interaction_eps, + ): + skipped += 1 + continue + + attn1 = capture.get("attn1") + attn0 = y0_capture.get("attn0") + if isinstance(attn1, torch.Tensor) and isinstance(attn0, torch.Tensor): + _accumulate_gap_component( + component_accs["attn"], + attn1, + attn0, + attention_mask, + interaction_eps, + ) + mlp1 = capture.get("mlp1") + mlp0 = y0_capture.get("mlp0") + if isinstance(mlp1, torch.Tensor) and isinstance(mlp0, torch.Tensor): + _accumulate_gap_component( + component_accs["mlp"], + mlp1, + mlp0, + attention_mask, + interaction_eps, + ) + for name in mlp_submodules: + sub1 = capture.get(f"{name}1") + sub0 = y0_capture.get(f"{name}0") + if isinstance(sub1, torch.Tensor) and isinstance(sub0, torch.Tensor): + _accumulate_gap_component( + component_accs[name], + sub1, + sub0, + attention_mask, + interaction_eps, + ) + batches += 1 + finally: + if was_training: + student_model.train() + + block_stats = _finalize_gap_component(block_acc) + if block_stats is None: + return { + "enabled": False, + "reason": "no_valid_batches", + "batches": int(batches), + "skipped_batches": int(skipped), + } + + components = {} + for name, acc in component_accs.items(): + component_stats = _finalize_gap_component(acc) + if component_stats is not None: + components[name] = component_stats + + return { + "enabled": True, + "pair": int(pair_idx), + "batches": int(batches), + "skipped_batches": int(skipped), + "tokens": block_stats["tokens"], + "mse": block_stats["mse"], + "relative": block_stats["relative"], + "cosine": block_stats["cosine"], + "cosine_distance": block_stats["cosine_distance"], + "r1_norm_mean": block_stats["r1_norm_mean"], + "r0_norm_mean": block_stats["r0_norm_mean"], + "components": components, + } + + +def commutator_precondition( + student_model: torch.nn.Module, + student_layers: List[torch.nn.Module], + teacher_model: torch.nn.Module, + dataloader, + dwce_scores: Optional[List[float]], + args: argparse.Namespace, + exclude_pairs: Optional[Set[int]] = None, + progressive_cycle: Optional[int] = None, + progressive_total: Optional[int] = None, + measure_dataloader=None, +) -> Dict[str, object]: + """Run commutator-style preconditioning before pair fusion. + + Objective on each sampled pair i: + L = T^2 * KL(p_teacher || p_student) + mu * L_interaction(i) + + Interaction loss is computed locally on block (i+1): + r1 = B_{i+1}(h_{i+1}) - h_{i+1} + r0 = B_{i+1}(h_i) - h_i + L_interaction = ||r1-r0||^2 (or relative form). + """ + if not bool(getattr(args, "comm_enabled", False)): + return {"enabled": False} + if not student_layers or len(student_layers) < 2: + return {"enabled": False, "reason": "need_at_least_2_layers"} + + temp = float(getattr(args, "comm_temp", 2.0)) + steps_ratio = float(getattr(args, "comm_steps_ratio", 0.1)) + lr_scale = float(getattr(args, "comm_lr_scale", 0.1)) + sample_eta = float(getattr(args, "comm_sample_eta", 0.5)) + sample_dwce_scale = float(getattr(args, "comm_sample_dwce_scale", 1.0)) + top_k = int(getattr(args, "comm_topk", 1)) + interaction_mode = str(getattr(args, "comm_interaction_mode", "relative")).strip().lower() + interaction_eps = float(getattr(args, "comm_interaction_eps", 1e-8)) + mu_cfg = getattr(args, "comm_mu", None) + mu_auto = bool(getattr(args, "comm_mu_auto", False)) + mu_auto_rho = float(getattr(args, "comm_mu_auto_rho", 0.1)) + mu_auto_eps = float(getattr(args, "comm_mu_auto_eps", 1e-8)) + comm_train_mode = str(getattr(args, "comm_train_mode", "lora")).strip().lower() + log_steps = int(getattr(args, "comm_log_steps", 50)) + + if temp <= 0.0: + raise SystemExit("--comm_temp must be > 0") + if steps_ratio < 0.0: + raise SystemExit("--comm_steps_ratio must be >= 0") + if lr_scale <= 0.0: + raise SystemExit("--comm_lr_scale must be > 0") + if not (0.0 <= sample_eta <= 1.0): + raise SystemExit("--comm_sample_eta must be in [0, 1]") + if top_k <= 0: + raise SystemExit("--comm_topk must be >= 1") + if interaction_mode not in {"mse", "relative"}: + raise SystemExit("--comm_interaction_mode must be one of: mse, relative") + if comm_train_mode not in {"lora", "full"}: + raise SystemExit("--comm_train_mode must be one of: lora, full") + if interaction_eps <= 0.0: + raise SystemExit("--comm_interaction_eps must be > 0") + if mu_auto_rho < 0.0: + raise SystemExit("--comm_mu_auto_rho must be >= 0") + if mu_auto_eps <= 0.0: + raise SystemExit("--comm_mu_auto_eps must be > 0") + + if mu_cfg is None: + base_mu = 0.5 if interaction_mode == "relative" else 0.1 + else: + base_mu = float(mu_cfg) + if base_mu < 0.0: + raise SystemExit("--comm_mu must be >= 0") + + distill_epochs = float(getattr(args, "distill_epochs", 1.0)) + if distill_epochs <= 0.0: + distill_epochs = 1.0 + grad_accum = int(getattr(args, "distill_grad_accum_steps", 1)) + if grad_accum <= 0: + grad_accum = 1 + + try: + batches_per_epoch = len(dataloader) + except TypeError as exc: + raise SystemExit( + "Commutator preconditioning requires a finite-length distillation dataloader." + ) from exc + if batches_per_epoch <= 0: + return {"enabled": False, "reason": "empty_dataloader"} + + full_epochs = int(distill_epochs) + fractional = distill_epochs - full_epochs + if fractional < 1e-8: + fractional = 0.0 + total_batches = full_epochs * batches_per_epoch + if fractional > 0.0: + frac_batches = int(round(fractional * batches_per_epoch)) + if frac_batches <= 0: + frac_batches = 1 + total_batches += frac_batches + + distill_opt_steps = int(math.ceil(total_batches / float(grad_accum))) + target_opt_steps = int(round(steps_ratio * distill_opt_steps)) + if target_opt_steps <= 0: + target_opt_steps = 1 + + num_pairs = max(len(student_layers) - 1, 0) + exclude_set = { + int(idx) + for idx in (exclude_pairs or set()) + if isinstance(idx, int) and 0 <= int(idx) < num_pairs + } + allowed_pairs = [i for i in range(num_pairs) if i not in exclude_set] + if not allowed_pairs: + return {"enabled": False, "reason": "all_pairs_excluded"} + + ranked_pairs = list(allowed_pairs) + if dwce_scores is not None and len(dwce_scores) >= num_pairs: + finite_pairs = [] + for idx in allowed_pairs: + value = float(dwce_scores[idx]) + if math.isfinite(value): + finite_pairs.append(idx) + if finite_pairs: + ranked_pairs = sorted(finite_pairs, key=lambda i: float(dwce_scores[i])) + else: + ranked_pairs = list(allowed_pairs) + candidate_pairs = ranked_pairs[: min(top_k, len(ranked_pairs))] + if not candidate_pairs: + return {"enabled": False, "reason": "no_candidate_pairs"} + + layer_trainable_params: List[List[torch.nn.Parameter]] = [] + trainable_params: List[torch.nn.Parameter] = [] + if comm_train_mode == "lora": + # LoRA comm preconditioning: update LoRA adapters on receiver layer (i+1). + lora_modules = apply_lora_adapters(student_model, args) + if not lora_modules: + return {"enabled": False, "reason": "no_lora_modules"} + + trainable_seen: Set[int] = set() + for module in lora_modules: + for param in module.lora_parameters(): + pid = id(param) + if pid in trainable_seen: + continue + trainable_seen.add(pid) + trainable_params.append(param) + + for layer in student_layers: + seen: Set[int] = set() + params: List[torch.nn.Parameter] = [] + for module in layer.modules(): + if not isinstance(module, LoRALinear): + continue + for param in module.lora_parameters(): + pid = id(param) + if pid in seen: + continue + seen.add(pid) + params.append(param) + layer_trainable_params.append(params) + else: + # Full-weight comm preconditioning: update full receiver-layer weights. + for layer in student_layers: + seen: Set[int] = set() + params: List[torch.nn.Parameter] = [] + for param in layer.parameters(): + if not isinstance(param, torch.nn.Parameter): + continue + pid = id(param) + if pid in seen: + continue + seen.add(pid) + params.append(param) + layer_trainable_params.append(params) + + candidate_pairs = [ + i + for i in candidate_pairs + if (i + 1) < len(layer_trainable_params) and layer_trainable_params[i + 1] + ] + if not candidate_pairs: + if comm_train_mode == "lora": + merge_lora_adapters(student_model) + return {"enabled": False, "reason": "no_trainable_receiver_layers"} + + if comm_train_mode == "full": + trainable_seen: Set[int] = set() + for pair_idx in candidate_pairs: + for param in layer_trainable_params[pair_idx + 1]: + pid = id(param) + if pid in trainable_seen: + continue + trainable_seen.add(pid) + trainable_params.append(param) + if not trainable_params: + return {"enabled": False, "reason": "no_trainable_receiver_layers"} + + # Freeze non-comm params to reduce grad memory. + for param in student_model.parameters(): + param.requires_grad_(False) + for param in trainable_params: + param.requires_grad_(True) + + if not trainable_params: + if comm_train_mode == "lora": + merge_lora_adapters(student_model) + return {"enabled": False, "reason": "no_trainable_params"} + + candidate_probs = torch.full( + (len(candidate_pairs),), + 1.0 / float(len(candidate_pairs)), + dtype=torch.float32, + ) + if dwce_scores is not None and len(dwce_scores) >= num_pairs and sample_eta > 0.0: + score_vec = torch.tensor( + [float(dwce_scores[i]) for i in candidate_pairs], dtype=torch.float32 + ) + score_vec = torch.nan_to_num(score_vec, nan=1e9, posinf=1e9, neginf=-1e9) + biased = torch.softmax(-float(sample_dwce_scale) * score_vec, dim=0) + candidate_probs = (1.0 - sample_eta) * candidate_probs + sample_eta * biased + candidate_probs = candidate_probs / candidate_probs.sum() + + probs_by_pair = [0.0 for _ in range(num_pairs)] + for pos, pair_idx in enumerate(candidate_pairs): + probs_by_pair[pair_idx] = float(candidate_probs[pos].item()) + + measured_pair = int(candidate_pairs[0]) + gap_pre = None + if bool(getattr(args, "comm_measure_gap", False)): + max_gap_batches = int(getattr(args, "comm_gap_max_batches", 8)) + gap_pre = _measure_commutator_gap( + student_model=student_model, + student_layers=student_layers, + dataloader=measure_dataloader if measure_dataloader is not None else dataloader, + pair_idx=measured_pair, + args=args, + max_batches=max_gap_batches, + interaction_eps=interaction_eps, + ) + if gap_pre.get("enabled"): + print( + "[comm-gap] pre " + f"pair={measured_pair}-{measured_pair + 1} " + f"{_format_comm_gap_stats(gap_pre)} " + f"batches={gap_pre.get('batches')}" + ) + else: + print(f"[comm-gap] pre measurement skipped: {gap_pre.get('reason')}") + + lr = float(getattr(args, "distill_lr", 1e-4)) * lr_scale + optimizer = torch.optim.AdamW( + trainable_params, + lr=lr, + weight_decay=float(getattr(args, "distill_weight_decay", 0.0)), + ) + + device_type = torch.device(args.device).type + amp_dtype = None + if args.dtype == "float16": + amp_dtype = torch.float16 + elif args.dtype == "bfloat16": + amp_dtype = torch.bfloat16 + use_amp = amp_dtype is not None and device_type == "cuda" + use_scaler = use_amp and amp_dtype == torch.float16 + scaler = torch.cuda.amp.GradScaler() if use_scaler else None + + teacher_device = next(teacher_model.parameters()).device + teacher_model.eval() + student_model.train() + + gen = torch.Generator(device="cpu") + seed = int(getattr(args, "seed", 0)) + if progressive_cycle is not None: + seed += int(progressive_cycle) * 100003 + gen.manual_seed(seed) + + opt_step = 0 + total_loss_sum = 0.0 + anchor_sum = 0.0 + interaction_sum = 0.0 + mu_sum = 0.0 + counted = 0 + pair_counts = [0 for _ in range(num_pairs)] + + desc = "Comm" + if progressive_cycle is not None: + if progressive_total is not None: + desc = f"Comm (cycle {progressive_cycle}/{progressive_total})" + else: + desc = f"Comm (cycle {progressive_cycle})" + iterator = range(target_opt_steps) + if tqdm is not None and _tqdm_enabled(): + iterator = tqdm(iterator, desc=desc, unit="step") + + data_iter = iter(dataloader) + autocast_ctx = ( + torch.autocast(device_type=device_type, dtype=amp_dtype) + if use_amp + else nullcontext() + ) + + for _ in iterator: + optimizer.zero_grad(set_to_none=True) + accum_done = 0 + while accum_done < grad_accum: + try: + batch = next(data_iter) + except StopIteration: + data_iter = iter(dataloader) + batch = next(data_iter) + + input_ids = batch[0].to(args.device) + attention_mask = batch[1].to(args.device) + sampled_pos = int(torch.multinomial(candidate_probs, 1, generator=gen).item()) + pair_idx = int(candidate_pairs[sampled_pos]) + pair_counts[pair_idx] += 1 + + receiver_params = layer_trainable_params[pair_idx + 1] + receiver_param_ids = {id(param) for param in receiver_params} + + teacher_ids = input_ids.to(teacher_device) + teacher_mask = attention_mask.to(teacher_device) + with torch.no_grad(), autocast_ctx: + teacher_outputs = teacher_model( + input_ids=teacher_ids, + attention_mask=teacher_mask, + use_cache=False, + ) + teacher_logits = teacher_outputs.logits + + capture: Dict[str, object] = { + "h_l": None, + "h_lp1": None, + "y1": None, + "recv_args": None, + "recv_kwargs": None, + } + + def _hook_l(_module, inputs, _output): + if inputs and isinstance(inputs[0], torch.Tensor): + capture["h_l"] = inputs[0] + + def _hook_recv_pre(_module, inputs, kwargs): + capture["recv_args"] = inputs + capture["recv_kwargs"] = kwargs + + def _hook_recv(_module, inputs, output): + if inputs and isinstance(inputs[0], torch.Tensor): + capture["h_lp1"] = inputs[0] + capture["y1"] = _extract_hidden_tensor(output) + + handles: List[object] = [ + student_layers[pair_idx].register_forward_hook(_hook_l), + _register_forward_pre_hook_with_optional_kwargs( + student_layers[pair_idx + 1], _hook_recv_pre + ), + student_layers[pair_idx + 1].register_forward_hook(_hook_recv), + ] + try: + with autocast_ctx: + student_outputs = student_model( + input_ids=input_ids, + attention_mask=attention_mask, + use_cache=False, + ) + student_logits = student_outputs.logits + finally: + for handle in handles: + try: + handle.remove() + except Exception: + pass + + with autocast_ctx: + anchor_kl = _masked_kl( + teacher_logits, + student_logits, + attention_mask, + temp=temp, + detach_p=True, + chunk_size=int(getattr(args, "kl_chunk_size", 16384)), + ) + if anchor_kl is None: + continue + anchor_loss = (temp ** 2) * anchor_kl + + interaction_loss = None + h_l = capture.get("h_l") + h_lp1 = capture.get("h_lp1") + y1 = capture.get("y1") + recv_args = capture.get("recv_args") + recv_kwargs = capture.get("recv_kwargs") + if ( + isinstance(h_l, torch.Tensor) + and isinstance(h_lp1, torch.Tensor) + and isinstance(y1, torch.Tensor) + and isinstance(recv_args, tuple) + and len(recv_args) > 0 + and isinstance(recv_args[0], torch.Tensor) + ): + first_hidden = recv_args[0] + receiver_device = first_hidden.device + call_args = list( + _move_tensor_tree_to_device(recv_args, receiver_device) + ) + h_l_detached = h_l.detach().to( + device=receiver_device, + dtype=first_hidden.dtype, + ) + call_args[0] = h_l_detached + call_kwargs = ( + _move_tensor_tree_to_device(dict(recv_kwargs), receiver_device) + if isinstance(recv_kwargs, dict) + else {} + ) + + y0_raw = student_layers[pair_idx + 1](*tuple(call_args), **call_kwargs) + y0 = _extract_hidden_tensor(y0_raw) + if isinstance(y0, torch.Tensor): + if y0.device != y1.device: + y0 = y0.to(y1.device) + h_lp1_detached = h_lp1.detach().to(device=y1.device, dtype=y1.dtype) + h_l_for_res = h_l.detach().to(device=y0.device, dtype=y0.dtype) + r1 = y1 - h_lp1_detached + r0 = y0 - h_l_for_res + mask = attention_mask.to(device=r1.device, dtype=r1.dtype) + mask_sum = mask.sum() + if mask_sum.item() > 0: + if interaction_mode == "relative": + num = (r1 - r0).float().pow(2).sum(dim=-1) + den = r1.float().pow(2).sum(dim=-1) + float(interaction_eps) + ratio = (num / den) * mask.to(num.dtype) + interaction_loss = ratio.sum() / (mask_sum + 1e-8) + else: + denom = mask_sum * r1.size(-1) + if denom.item() > 0: + interaction_loss = ( + (r1 - r0).pow(2) * mask.unsqueeze(-1) + ).sum() / denom + + mu_effective = float(base_mu) + if ( + mu_auto + and interaction_loss is not None + and receiver_params + and mu_auto_rho > 0.0 + ): + anchor_grads = torch.autograd.grad( + anchor_loss, + receiver_params, + retain_graph=True, + allow_unused=True, + ) + interaction_grads = torch.autograd.grad( + interaction_loss, + receiver_params, + retain_graph=True, + allow_unused=True, + ) + anchor_norm = _grad_l2_norm(list(anchor_grads)) + interaction_norm = _grad_l2_norm(list(interaction_grads)) + if interaction_norm > 0.0: + mu_effective = float( + mu_auto_rho + * (anchor_norm / (interaction_norm + float(mu_auto_eps))) + ) + else: + mu_effective = float(base_mu) + if not math.isfinite(mu_effective): + mu_effective = float(base_mu) + + total_loss = anchor_loss + if interaction_loss is not None: + if interaction_loss.device != total_loss.device: + interaction_loss = interaction_loss.to(total_loss.device) + total_loss = total_loss + (float(mu_effective) * interaction_loss) + + if grad_accum > 1: + total_loss = total_loss / float(grad_accum) + + if use_scaler: + scaler.scale(total_loss).backward() + else: + total_loss.backward() + + # Only the sampled receiver layer updates on this micro-batch. + for param in trainable_params: + if id(param) in receiver_param_ids: + continue + if param.grad is not None: + if comm_train_mode == "lora": + param.grad.zero_() + else: + param.grad = None + + total_loss_sum += float(total_loss.detach().float().item()) + anchor_sum += float(anchor_loss.detach().float().item()) + if interaction_loss is not None: + interaction_sum += float(interaction_loss.detach().float().item()) + mu_sum += float(mu_effective) + counted += 1 + accum_done += 1 + + if args.distill_max_grad_norm is not None: + if use_scaler: + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_( + trainable_params, + float(args.distill_max_grad_norm), + ) + + if use_scaler: + scaler.step(optimizer) + scaler.update() + else: + optimizer.step() + + opt_step += 1 + if log_steps and (opt_step == 1 or opt_step % log_steps == 0): + denom = max(counted, 1) + print( + f"[comm] step={opt_step}/{target_opt_steps} " + f"loss={total_loss_sum/denom:.6f} " + f"anchor={anchor_sum/denom:.6f} " + f"int={interaction_sum/denom:.6f} " + f"mu={mu_sum/denom:.6f}" + ) + + if comm_train_mode == "lora": + merge_lora_adapters(student_model) + + gap_post = None + if bool(getattr(args, "comm_measure_gap", False)): + max_gap_batches = int(getattr(args, "comm_gap_max_batches", 8)) + gap_post = _measure_commutator_gap( + student_model=student_model, + student_layers=student_layers, + dataloader=measure_dataloader if measure_dataloader is not None else dataloader, + pair_idx=measured_pair, + args=args, + max_batches=max_gap_batches, + interaction_eps=interaction_eps, + ) + if gap_post.get("enabled"): + print( + "[comm-gap] post " + f"pair={measured_pair}-{measured_pair + 1} " + f"{_format_comm_gap_stats(gap_post)} " + f"batches={gap_post.get('batches')}" + ) + else: + print(f"[comm-gap] post measurement skipped: {gap_post.get('reason')}") + + stats: Dict[str, object] = { + "enabled": True, + "train_mode": comm_train_mode, + "opt_steps": int(target_opt_steps), + "grad_accum_steps": int(grad_accum), + "lr": float(lr), + "temp": float(temp), + "steps_ratio": float(steps_ratio), + "lr_scale": float(lr_scale), + "interaction_mode": interaction_mode, + "interaction_eps": float(interaction_eps), + "mu": float(base_mu), + "mu_auto": bool(mu_auto), + "mu_auto_rho": float(mu_auto_rho), + "mu_auto_eps": float(mu_auto_eps), + "sample_eta": float(sample_eta), + "sample_dwce_scale": float(sample_dwce_scale), + "topk": int(top_k), + "candidate_pairs": [int(i) for i in candidate_pairs], + "trainable_params": int(sum(int(param.numel()) for param in trainable_params)), + } + if gap_pre is not None or gap_post is not None: + stats["gap_pair"] = int(measured_pair) + stats["gap_pre"] = gap_pre + stats["gap_post"] = gap_post + if ( + isinstance(gap_pre, dict) + and isinstance(gap_post, dict) + and gap_pre.get("enabled") + and gap_post.get("enabled") + ): + stats["gap_delta"] = { + key: float(gap_post[key] - gap_pre[key]) + for key in ("mse", "relative", "cosine_distance") + if key in gap_pre and key in gap_post + } + pre_components = gap_pre.get("components") + post_components = gap_post.get("components") + if isinstance(pre_components, dict) and isinstance(post_components, dict): + component_delta = {} + for name, pre_value in pre_components.items(): + post_value = post_components.get(name) + if not isinstance(pre_value, dict) or not isinstance(post_value, dict): + continue + component_delta[name] = { + key: float(post_value[key] - pre_value[key]) + for key in ("mse", "relative", "cosine_distance") + if key in pre_value and key in post_value + } + if component_delta: + stats["gap_delta"]["components"] = component_delta + stats["gap_ratio_post_over_pre"] = _comm_gap_ratio(gap_pre, gap_post) + total_samples = int(sum(pair_counts)) + probs_list = [float(x) for x in probs_by_pair] + freqs = ( + [float(c) / float(total_samples) for c in pair_counts] + if total_samples > 0 + else [0.0 for _ in pair_counts] + ) + top_show = min(10, num_pairs) + top_indices = sorted(range(num_pairs), key=lambda i: pair_counts[i], reverse=True)[:top_show] + top_pairs = [ + { + "pair": int(i), + "count": int(pair_counts[i]), + "freq": float(freqs[i]), + "prob": float(probs_list[i]) if i < len(probs_list) else None, + } + for i in top_indices + if pair_counts[i] > 0 + ] + stats["pair_selection"] = { + "num_pairs": int(num_pairs), + "excluded_pairs": sorted(exclude_set), + "candidate_pairs": [int(i) for i in candidate_pairs], + "total_samples": total_samples, + "unique_pairs": int(sum(1 for c in pair_counts if c > 0)), + "counts": [int(c) for c in pair_counts], + "freqs": freqs, + "probs": probs_list, + "top_pairs": top_pairs, + } + + if total_samples > 0 and top_pairs: + top_str = ", ".join( + f"{entry['pair']}-{entry['pair'] + 1}: {entry['count']} " + f"(obs={entry['freq']:.3f}, exp={entry['prob']:.3f})" + for entry in top_pairs + if entry.get("prob") is not None + ) + if not top_str: + top_str = ", ".join( + f"{entry['pair']}-{entry['pair'] + 1}: {entry['count']} " + f"(obs={entry['freq']:.3f})" + for entry in top_pairs + ) + print( + f"[comm] Pair sampling stats: total={total_samples} " + f"unique={stats['pair_selection']['unique_pairs']}/{num_pairs} " + f"top={top_str}" + ) + + if counted > 0: + stats["avg_loss"] = float(total_loss_sum / float(counted)) + stats["avg_anchor"] = float(anchor_sum / float(counted)) + stats["avg_interaction"] = float(interaction_sum / float(counted)) + stats["avg_mu"] = float(mu_sum / float(counted)) + return stats diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/launch_qwen3_nofit_bench8.sh b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/launch_qwen3_nofit_bench8.sh new file mode 100644 index 0000000000000000000000000000000000000000..7a57605cf03cad75ed33d54e8c11e6c3d7fca131 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/launch_qwen3_nofit_bench8.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail +project="$1"; root="$2"; model="$3"; alias="$4" +export HF_HOME=/workspace/.hf_home HF_TOKEN_PATH=/root/.cache/huggingface/token +export HF_DATASETS_CACHE=/workspace/.hf_datasets36 HF_DATASETS_TRUST_REMOTE_CODE=1 +export PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=4 PYTHONPATH="$project/src" +mkdir -p "$root" +trap 'rc=$?; if ((rc)); then touch "$root/FAILED"; fi' EXIT +while [[ ! -f "$root/EVAL_READY" ]]; do [[ ! -f "$root/BOOTSTRAP_FAILED" ]] || exit 1; sleep 3; done +export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 +run_arm() ( + arm="$1"; export CUDA_VISIBLE_DEVICES="$2"; out="$root/full/$arm"; stage="$out/compact/stage_09" + trap 'rc=$?; if ((rc)); then touch "$root/${arm}_FAILED"; fi' EXIT + mkdir -p "$out/logs" + if [[ ! -f "$out/eval/final_opencompass5.json" ]]; then + python "$project/scripts/eval_opencompass_ppl_fast.py" --model "$model" --tokenizer "$model" \ + --revision 49e3418fbbbca6ecbdf9608b4d22e5a407081db4 --compact-stage "$stage" --cycle 9 --anchor 12 \ + --data-root "$root/eval_data/data" --max-batch 32 --token-budget 8192 \ + --output "$out/eval/final_opencompass5.json" > "$out/logs/opencompass.log" 2>&1 + fi + if [[ ! -f "$out/eval/new_lmeval3.json" ]]; then + python "$project/scripts/eval_llama31_8b_12bench.py" --model "$model" \ + --revision 49e3418fbbbca6ecbdf9608b4d22e5a407081db4 --compact-stage "$stage" --cycle 9 --anchor 12 \ + --batch-size auto:4 --tasks sciq mmlu cmmlu --dataset-cache "$root/lmeval_cache" --local-files-only \ + --output "$out/eval/new_lmeval3.json" > "$out/logs/lmeval3.log" 2>&1 + fi + python "$project/scripts/merge_nofit_existing_zs6.py" "$out/eval/zs6.json" "$out/eval/new_lmeval3.json" "$out/eval/final_lmeval9.json" + python "$project/scripts/aggregate_transport_rho_fit_14bench.py" "$out" > "$out/logs/aggregate.log" + touch "$out/EVAL_COMPLETE" +) +run_arm off 0 > "$root/off_pipeline.log" 2>&1 & off_pid=$! +run_arm on 1 > "$root/on_pipeline.log" 2>&1 & on_pid=$! +failed=0;for pid in "$off_pid" "$on_pid";do wait "$pid" || failed=1;done +((failed==0)) || exit 1 +python - "$root" "$project" <<'PY' +import json,shutil,subprocess,sys +from pathlib import Path +r,project=map(Path,sys.argv[1:]); backup=r/'backup';backup.mkdir(exist_ok=True) +old=json.loads((project/'results/qwen3_8b_base_nofit_transport_zs6_20261009/summary_with_ppl.json').read_text()) +rows=[] +for row in old['rows']: + arm=row['transport'];out=r/'full'/arm + report=json.loads((out/'eval/final_14bench.json').read_text());scores=report['task_scores'] + assert len(scores)==14 + assert all(scores[k]==v for k,v in row['task_scores'].items()) + row.update(task_scores=scores,zs12_avg=sum(v for k,v in scores.items() if k not in {'mmlu','cmmlu'})/12, + mmlu=scores['mmlu'],cmmlu=scores['cmmlu'],bench14_avg=report['macro_mean_14'], + zs6_preserved=True,new_task_count=8,weight_restore=json.loads((out/'restore_receipt.json').read_text())) + rows.append(row) + target=backup/arm;target.mkdir(exist_ok=True) + shutil.copytree(out/'eval',target/'eval',dirs_exist_ok=True) + shutil.copytree(out/'logs',target/'logs',dirs_exist_ok=True) + shutil.copy2(out/'restore_receipt.json',target/'restore_receipt.json') +summary={'rows':rows,'source_commit':subprocess.check_output(['git','-C',str(project),'rev-parse','HEAD'],text=True).strip(), + 'protocol':'Unchanged legacy OpenCompass5 + lm-eval9, BF16 eager, zero-shot; immutable existing ZS6 retained; only missing eight newly evaluated'} +(r/'summary.json').write_text(json.dumps(summary,indent=2)+'\n');shutil.copy2(r/'summary.json',backup/'summary.json') +code=backup/'code';code.mkdir(exist_ok=True) +for name in ['bootstrap_qwen3_nofit_bench8.sh','launch_qwen3_nofit_bench8.sh','merge_nofit_existing_zs6.py', + 'eval_llama31_8b_12bench.py','eval_opencompass_ppl_fast.py','aggregate_transport_rho_fit_14bench.py']: + shutil.copy2(project/'scripts'/name,code/name) +shutil.copy2(project/'exp_md/2026-10-09_qwen3-base-nofit-bench14-supplement.md',backup/'protocol.md') +PY +python "$project/scripts/write_tree_manifest.py" "$root/backup" +HF_HUB_OFFLINE=0 HF_DATASETS_OFFLINE=0 TRANSFORMERS_OFFLINE=0 \ + python "$project/scripts/upload_verified_hf_tree.py" "$root/backup" --repo-id LJYAI/paper_revision \ + --path-in-repo "runs/$alias/20261009-qwen3-base-nofit-bench14-supplement" --verify-all-files \ + --commit-message 'Qwen3 Base Fisher Transport OFF ON no-FiT remaining eight tasks, existing ZS6 preserved' > "$root/backup.json" +touch "$root/COMPLETE" +echo 'COMPLETE: results backed up; instance retained by explicit user request.' diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/launch_qwen3_nofit_merge4.sh b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/launch_qwen3_nofit_merge4.sh new file mode 100644 index 0000000000000000000000000000000000000000..ccfb64d06f0c6762b1719a4fda69641c6de3d1a1 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/launch_qwen3_nofit_merge4.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail +project="$1"; external="$2"; root="$3"; model="$4"; alias="$5" +export HF_HOME=/workspace/.hf_home HF_TOKEN_PATH=/root/.cache/huggingface/token +export HF_DATASETS_CACHE=/workspace/.hf_datasets36 PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=4 +export PYTHONPATH="$external/src:$project/src:$project" +export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 +revision=49e3418fbbbca6ecbdf9608b4d22e5a407081db4 +trap 'rc=$?; if ((rc)); then touch "$root/FAILED"; fi' EXIT +while [[ ! -f "$root/INPUTS_READY" ]]; do [[ ! -f "$root/BOOTSTRAP_FAILED" ]] || exit 1; sleep 3; done +run_prune() { + local arm="$1" gpu="$2" out="$3" cycles="$4" + mkdir -p "$out/logs" "$out/audit" "$out/eval" + CUDA_VISIBLE_DEVICES="$gpu" python "$project/scripts/run_qwen3_nofit_merge_control.py" \ + --external-root "$external" --inputs "$root/inputs" --audit-root "$out/audit" --initializer "$arm" \ + --model "$model" --output_dir "$out/run" --layer 12 --num_progressive "$cycles" \ + --dataset slimpajama --dataset_config none --num_samples 64 --seed 42 --seq_len 1024 --batch_size 2 \ + --skip_distill --norm_policy copy_n1_n2 --fisher_mode param --eps 0 \ + --no_head_permute_merge --no_head_permute_select --source_feature_transport \ + --source_feature_transport_ridge_relative 1e-3 --dtype bfloat16 --eval_dtype bfloat16 \ + --lora_epochs 0 --skip_eval --skip_final_model_save > "$out/logs/prune.log" 2>&1 + python -m fusegpt.abprune_compact --run-root "$out/run" --output-root "$out/compact" \ + --model Qwen/Qwen3-8B-Base --base-revision "$revision" --run-id "qwen3_base_nofit_merge_${arm}_20261011" \ + --cycles "$cycles" --hard-link --layer-order "$(python -c "print(' '.join(['12']*$cycles))")" + python "$project/scripts/verify_qwen3_nofit_merge.py" "$out" "$arm" "$cycles" + touch "$out/PRUNE_COMPLETE" +} +job() ( + arm="$1"; gpu="$2"; out="$root/full/$arm"; smoke="$root/smoke/$arm" + trap 'rc=$?; if ((rc)); then touch "$root/${arm}_FAILED"; fi' EXIT + [[ -f "$smoke/PRUNE_COMPLETE" ]] || run_prune "$arm" "$gpu" "$smoke" 2 + for other in wa wb average mka; do + while [[ ! -f "$root/smoke/$other/PRUNE_COMPLETE" ]]; do + [[ ! -f "$root/${other}_FAILED" ]] || exit 1; sleep 3 + done + done + [[ -f "$out/PRUNE_COMPLETE" ]] || run_prune "$arm" "$gpu" "$out" 9 + cp "$root/full/base/offline_eval.json" "$out/offline_eval.json" + [[ -f "$out/eval/local_mse.json" ]] || CUDA_VISIBLE_DEVICES="$gpu" python "$project/scripts/qwen3_nofit_local_mse.py" \ + --model "$model" --cache "$root/local_mse_reference" --compact-stage "$out/compact/stage_09" \ + --output "$out/eval/local_mse.json" > "$out/logs/local_mse.log" 2>&1 + [[ -f "$out/eval/final_ppl.json" ]] || CUDA_VISIBLE_DEVICES="$gpu" python "$project/scripts/evaluate_abprune_compact_ppl.py" \ + --model "$model" --revision "$revision" --compact-stage "$out/compact/stage_09" --anchor 12 --cycles 9 \ + --original-layers 36 --seq-len 2048 --batch-size 1 --output "$out/eval/final_ppl.json" > "$out/logs/ppl.log" 2>&1 + MODEL_REVISION="$revision" LMEVAL_LOCAL_CACHE="$root/lmeval_cache" \ + bash "$project/scripts/eval_llama31_fit_ablation_one.sh" "$project" "$out" "$model" "$root/eval_data/data" "$(command -v python)" 12 9 "$gpu" + touch "$out/EVAL_COMPLETE" + backup="$root/backup/$arm" + python "$project/scripts/package_qwen3_nofit_merge_backup.py" "$out" "$backup" "$project" "$external" "$root" + python "$project/scripts/write_tree_manifest.py" "$backup" + HF_HUB_OFFLINE=0 HF_DATASETS_OFFLINE=0 TRANSFORMERS_OFFLINE=0 python "$project/scripts/upload_verified_hf_tree.py" \ + "$backup" --repo-id LJYAI/paper_revision --path-in-repo "runs/$alias/20261011-qwen3-base-nofit-merge4/$arm" \ + --verify-all-files --commit-message "Qwen3 base no-FiT merge control $arm; all9 stages" > "$root/${arm}_backup.json" + touch "$out/BACKUP_COMPLETE" +) +pids=(); gpu=0 +for arm in wa wb average mka; do job "$arm" "$gpu" > "$root/${arm}_pipeline.log" 2>&1 & pids+=("$!"); gpu=$((gpu+1)); done +failed=0; for pid in "${pids[@]}"; do wait "$pid" || failed=1; done +((failed==0)) || exit 1 +python "$project/scripts/summarize_qwen3_nofit_merge.py" "$root" "$project" +git -C "$project" add results/qwen3_base_nofit_merge4_20261011 +git -C "$project" commit -m 'Record four Qwen3 no-FiT transported merge controls and verified all-stage backups' +touch "$root/COMPLETE" "$root/RECORDS_COMMITTED" +echo 'Complete; retain instance until explicitly authorized to return.' diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/launch_qwen3_nofit_transport_zs6.sh b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/launch_qwen3_nofit_transport_zs6.sh new file mode 100644 index 0000000000000000000000000000000000000000..7830d1ac70d353aee0382e962145e3a3beb1c4e0 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/launch_qwen3_nofit_transport_zs6.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail +project="$1"; external="$2"; root="$3"; model="$4"; alias="$5" +source "$project/scripts/qwen3_nofit_transport_config.sh" +export HF_HOME=/workspace/.hf_home HF_TOKEN_PATH=/root/.cache/huggingface/token +export HF_DATASETS_CACHE=/workspace/.hf_datasets36 PYTHONUNBUFFERED=1 +export PYTHONPATH="$project/src:$external/src:$project" +trap 'rc=$?; if ((rc)); then touch "$root/FAILED"; fi' EXIT +prepare_eval() ( + trap 'rc=$?; if ((rc)); then touch "$root/EVAL_FAILED"; fi' EXIT + while [[ ! -f "$root/DEPS_READY" || ! -f "$root/EVAL_ASSETS_READY" ]]; do [[ ! -f "$root/BOOTSTRAP_FAILED" ]] || exit 1; sleep 3; done + tar -xzf "$root/lmeval-cache-131.tar.gz" -C "$root" + HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 python "$project/scripts/prepare_lmeval9_local_cache.py" "$root/lmeval_cache" --validate-local + touch "$root/EVAL_READY" +) +prepare_eval > "$root/eval_prepare.log" 2>&1 & eval_pid=$! +verify() { + python - "$1" "$2" "$3" <<'PY' +import json,sys,os +from pathlib import Path +r=Path(sys.argv[1]); arm=sys.argv[2]; cycles=int(sys.argv[3]) +anchor=int(os.environ['NOFIT_ANCHOR']) +for c in range(1,cycles+1): + m=json.loads((r/f'run/cycle_{c}/cycle_metadata.json').read_text()) + f=json.loads((r/f'audit/cycle_{c:02d}_fit.json').read_text()) + assert m['layer_merged']==anchor and m['num_layers_after']==36-c + assert m['fisher_num_batches']==32 + assert m['distill']['reparam_skip_train'] and not m['distill']['reparam_stats']['trained'] + assert f['attention_norm_unchanged'] and f['stats']['epochs']==0 + assert bool(m['source_feature_transport'])==(arm=='on') + assert not m['lora']['enabled'] and not (m['comm'] or {}).get('enabled',False) +meta=json.loads((r/f'compact/stage_{cycles:02d}/metadata.json').read_text()) +assert meta['removed_original_layers']==list(range(anchor+1,anchor+1+cycles)) +assert meta['model']==os.environ['NOFIT_MODEL_ID'] and meta['base_model_revision']==os.environ['NOFIT_MODEL_REVISION'] +if meta['format']=='abprune_compact_stage_v2': + assert meta['active_original_layers']==list(range(anchor+1))+list(range(anchor+1+cycles,36)) +else: + assert meta['anchor_index']==anchor and meta['removed_count']==cycles +print('MATCHED_FISHER_NO_FIT_VERIFIED',arm,cycles,flush=True) +PY +} +job() ( + arm="$1"; gpu="$2" + trap 'rc=$?; if ((rc)); then touch "$root/${arm}_FAILED"; fi' EXIT + while [[ ! -f "$root/BASE_READY" ]]; do [[ ! -f "$root/BOOTSTRAP_FAILED" ]] || exit 1; sleep 3; done + smoke="$root/smoke/$arm"; out="$root/full/$arm" + if [[ ! -f "$smoke/VERIFIED" ]]; then + if [[ ! -f "$smoke/PRUNE_COMPLETE" ]]; then + bash "$project/scripts/run_qwen3_nofit_transport_arm.sh" "$project" "$external" "$smoke" "$model" "$arm" "$gpu" 2 + fi + verify "$smoke" "$arm" 2; touch "$smoke/VERIFIED" + fi + while [[ ! -f "$root/smoke/off/VERIFIED" || ! -f "$root/smoke/on/VERIFIED" ]]; do + [[ ! -f "$root/off_FAILED" && ! -f "$root/on_FAILED" ]] || exit 1; sleep 2 + done + python - "$root" <<'PY' +import json,sys +from pathlib import Path +r=Path(sys.argv[1]) +assert json.loads((r/'smoke/off/audit/data_fingerprints.json').read_text())==json.loads((r/'smoke/on/audit/data_fingerprints.json').read_text()) +print('PAIRED_SMOKE_DATA_IDENTICAL',flush=True) +PY + if [[ ! -f "$out/PRUNE_COMPLETE" ]]; then + bash "$project/scripts/run_qwen3_nofit_transport_arm.sh" "$project" "$external" "$out" "$model" "$arm" "$gpu" 9 + fi + verify "$out" "$arm" 9 + while [[ ! -f "$root/EVAL_READY" ]]; do [[ ! -f "$root/EVAL_FAILED" ]] || exit 1; sleep 3; done + if [[ ! -f "$out/eval/zs6.json" ]]; then + CUDA_VISIBLE_DEVICES="$gpu" HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ + python "$project/scripts/eval_abprune_compact_zeroshot.py" --model "$model" \ + --revision "$NOFIT_MODEL_REVISION" --compact-stage "$out/compact/stage_09" \ + --cycle 9 --anchor "$NOFIT_ANCHOR" --batch-size 8 --dataset-cache "$root/lmeval_cache" --output "$out/eval/zs6.json" > "$out/logs/zs6.log" 2>&1 + fi + if [[ "$NOFIT_MEASURE_PPL" == 1 && ! -f "$out/eval/final_ppl.json" ]]; then + CUDA_VISIBLE_DEVICES="$gpu" HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ + python "$project/scripts/evaluate_abprune_compact_ppl.py" --model "$model" --revision "$NOFIT_MODEL_REVISION" \ + --compact-stage "$out/compact/stage_09" --anchor "$NOFIT_ANCHOR" --cycles 9 --original-layers 36 \ + --seq-len 2048 --batch-size 4 --output "$out/eval/final_ppl.json" > "$out/logs/ppl.log" 2>&1 + fi + touch "$out/EVAL_COMPLETE" + backup="$root/backup/$arm" + if [[ ! -d "$backup" ]]; then + python "$project/scripts/package_final_fit_backup.py" "$out" "$backup" "$project" "$external" \ + --protocol "$NOFIT_PROTOCOL" --cycles 9 --original-layers 36 + cp "$project/scripts/run_qwen3_nofit_transport_control.py" "$project/scripts/run_qwen3_nofit_transport_arm.sh" \ + "$project/scripts/launch_qwen3_nofit_transport_zs6.sh" "$project/scripts/qwen3_nofit_transport_config.sh" \ + "$project/scripts/eval_abprune_compact_zeroshot.py" "$backup/code/" + git -C "$project" rev-parse HEAD > "$backup/code/project_commit.txt" + python "$project/scripts/write_tree_manifest.py" "$backup" + fi + HF_HUB_OFFLINE=0 HF_DATASETS_OFFLINE=0 TRANSFORMERS_OFFLINE=0 \ + python "$project/scripts/upload_verified_hf_tree.py" "$backup" --repo-id LJYAI/paper_revision \ + --path-in-repo "runs/$alias/$NOFIT_RUN_SLUG/$arm" --verify-all-files \ + --commit-message "Qwen3 ${QWEN_NOFIT_VARIANT:-base} matched Fisher transport $arm no FiT" > "$root/${arm}_backup.json" + touch "$out/BACKUP_COMPLETE" +) +job off 0 > "$root/off_pipeline.log" 2>&1 & off_pid=$! +job on 1 > "$root/on_pipeline.log" 2>&1 & on_pid=$! +failed=0; for pid in "$off_pid" "$on_pid" "$eval_pid"; do wait "$pid" || failed=1; done +((failed==0)) || exit 1 +python - "$root" <<'PY' +import json,sys,os +from pathlib import Path +r=Path(sys.argv[1]); rows=[]; fingerprints=[] +anchor=int(os.environ['NOFIT_ANCHOR']) +for arm in ['off','on']: + out=r/'full'/arm; result=json.loads((out/'eval/zs6.json').read_text()) + fp=json.loads((out/'audit/data_fingerprints.json').read_text()); fingerprints.append(fp) + rows.append(dict(model=os.environ['NOFIT_MODEL_ID'],model_revision=os.environ['NOFIT_MODEL_REVISION'],removed_blocks=9,anchor=anchor,removed_original_layers=list(range(anchor+1,anchor+10)), + transport=arm,fit=False,fisher='same_anchor_contextual',task_scores=result['task_scores'], + zs6=result['six_task_mean'],backup=json.loads((r/f'{arm}_backup.json').read_text()))) + if os.environ['NOFIT_MEASURE_PPL']=='1': + ppl=json.loads((out/'eval/final_ppl.json').read_text()) + assert ppl['metrics']['sequences']==145 and ppl['metrics']['tokens']==296815 + rows[-1]['ppl']=ppl['metrics']['perplexity'] +assert fingerprints[0]==fingerprints[1] +(r/'summary.json').write_text(json.dumps(rows,indent=2)+'\n') +PY +touch "$root/COMPLETE" diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/monitor_qwen3_nofit_bench8.ps1 b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/monitor_qwen3_nofit_bench8.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..7d7a4f5ec3a4e3553ed33b3e197e4ce31f41ba78 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/monitor_qwen3_nofit_bench8.ps1 @@ -0,0 +1,58 @@ +param( + [string]$HostAlias='vast-paper_revision-qwen3-base-nofit-bench8-54984557-20261009', + [string]$LogDirectory='C:\Users\CELL\orca\artifacts\paper_revision\qwen3-base-nofit-bench8-20261009-monitor', + [ValidatePattern('^/workspace/artifacts/[A-Za-z0-9._-]+$')] + [string]$RemoteRoot='/workspace/artifacts/qwen3-base-nofit-bench8-20261009', + [ValidateRange(30,3600)][int]$IntervalSeconds=180 +) +$ErrorActionPreference='Stop' +New-Item -ItemType Directory -Force -Path $LogDirectory | Out-Null +$poll=@' +from pathlib import Path +import json,re,time,subprocess +r=Path('__ROOT__') +def tail(p): + return [re.sub(r'https?://\S+','[URL omitted]',s)[-500:] for s in p.read_text(errors='replace').splitlines()[-2:]] if p.exists() else [] +arms={} +for arm in ['off','on']: + d=r/'full'/arm + arms[arm]={'markers':[m for m in ['EVAL_COMPLETE','BENCH14_COMPLETE'] if (d/m).exists()], + 'logs':{n:tail(d/'logs'/n) for n in ['opencompass.log','lmeval3.log']}, + 'results':[p.name for p in (d/'eval').glob('*.json')]} +print('SNAPSHOT_JSON='+json.dumps({'utc':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()), + 'failed':any((r/m).exists() for m in ['FAILED','BOOTSTRAP_FAILED','off_FAILED','on_FAILED']), + 'complete':(r/'COMPLETE').exists(),'markers':[m for m in ['BASE_READY','EVAL_ASSETS_READY','EVAL_READY','BOOTSTRAP_COMPLETE'] if (r/m).exists()], + 'arms':arms,'logs':{n:tail(r/n) for n in ['bootstrap.log','controller.log','off_pipeline.log','on_pipeline.log']}})) +print(subprocess.check_output(['nvidia-smi','--query-gpu=index,utilization.gpu,memory.used','--format=csv,noheader'],text=True)) +'@ +$poll=$poll.Replace('__ROOT__',$RemoteRoot) +$payload=[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($poll.Replace("`r`n","`n"))) +while($true){ + $lines=@(& ssh -o BatchMode=yes -o ConnectTimeout=15 -o ServerAliveInterval=15 -o ServerAliveCountMax=2 $HostAlias "printf %s $payload | base64 -d | python") + $code=$LASTEXITCODE + Add-Content -LiteralPath (Join-Path $LogDirectory 'poll.log') -Value (@((Get-Date -Format o))+$lines) -Encoding UTF8 + $line=$lines | Where-Object {$_ -like 'SNAPSHOT_JSON=*'} | Select-Object -Last 1 + if($code -eq 0 -and $line){ + $snapshot=$line.Substring('SNAPSHOT_JSON='.Length) | ConvertFrom-Json + if($snapshot.failed){throw 'Benchmark supplement failed; instance retained.'} + if($snapshot.complete){ + $dest=Join-Path $LogDirectory 'completed_results';New-Item -ItemType Directory -Force -Path $dest | Out-Null + foreach($name in @('summary.json','backup.json','execution.json')){ + & scp -O "${HostAlias}:${RemoteRoot}/$name" (Join-Path $dest $name) + if($LASTEXITCODE -ne 0){throw "Recovery failed: $name"} + } + foreach($arm in @('off','on')){ + $folder=Join-Path $dest $arm;New-Item -ItemType Directory -Force -Path $folder | Out-Null + & scp -O "${HostAlias}:${RemoteRoot}/full/$arm/eval/*.json" $folder + if($LASTEXITCODE -ne 0){throw "Score recovery failed: $arm"} + & scp -O "${HostAlias}:${RemoteRoot}/full/$arm/restore_receipt.json" $folder + if($LASTEXITCODE -ne 0){throw "Restore receipt recovery failed: $arm"} + } + $receipt=Get-Content (Join-Path $dest 'backup.json') -Raw | ConvertFrom-Json + if(-not $receipt.verified -or -not $receipt.all_payload_hashes_match){throw 'Backup verification missing'} + Add-Content (Join-Path $LogDirectory 'poll.log') 'COMPLETE: scores recovered; instance retained by user request.' + break + } + } + Start-Sleep -Seconds $IntervalSeconds +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/monitor_qwen3_nofit_merge4.ps1 b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/monitor_qwen3_nofit_merge4.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..83d89b345dedf9c9a317369eade8051d1dbbfff0 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/monitor_qwen3_nofit_merge4.ps1 @@ -0,0 +1,52 @@ +param( + [string]$HostAlias='vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011', + [string]$RemoteRoot='/workspace/artifacts/qwen3-base-nofit-merge4-20261011', + [string]$LogDirectory='C:\Users\CELL\orca\artifacts\paper_revision\qwen3-base-nofit-merge4-55359231-20261011-monitor', + [int]$IntervalSeconds=300 +) +$ErrorActionPreference='Stop' +New-Item -ItemType Directory -Force -Path $LogDirectory | Out-Null +if ($RemoteRoot -notmatch '^/workspace/artifacts/[a-zA-Z0-9._-]+$') { throw 'Unsafe root' } +$taskPoll=@' +from pathlib import Path +import json,subprocess,re,datetime +r=Path('__ROOT__') +def tail(p): + return [re.sub(r'https?://\S+','[URL]',s)[-500:] for s in p.read_text(errors='replace').splitlines()[-3:]] if p.exists() else [] +arms={} +for arm in ('wa','wb','average','mka'): + a=r/'full'/arm + arms[arm]=dict(cycles=len(list((a/'run').glob('cycle_*/cycle_metadata.json'))), + smoke_cycles=len(list((r/'smoke'/arm/'run').glob('cycle_*/cycle_metadata.json'))), + markers=[x for x in ('PRUNE_COMPLETE','EVAL_COMPLETE','BACKUP_COMPLETE') if (a/x).exists()], + logs={str(p):tail(a/p) for p in map(Path,['logs/prune.log','logs/local_mse.log','logs/ppl.log','logs/eval14.log'])}) +state=dict(utc=datetime.datetime.now(datetime.timezone.utc).isoformat(), + failed=[p.name for p in r.glob('*FAILED')], complete=(r/'RECORDS_COMMITTED').exists(), + markers=[x for x in ('DEPS_READY','BASE_READY','DATA_READY','EVAL_READY','COMMON_TEACHER_READY','INPUTS_READY','COMPLETE') if (r/x).exists()], + logs={str(p):tail(r/p) for p in map(Path,['bootstrap.log','controller.log','data_prepare.log','local_mse_prepare.log'])},arms=arms) +print('SNAPSHOT_JSON='+json.dumps(state)) +print(subprocess.check_output(['nvidia-smi','--query-gpu=index,utilization.gpu,memory.used','--format=csv,noheader'],text=True)) +'@ +$taskPoll=$taskPoll.Replace('__ROOT__',$RemoteRoot) +$taskEncoded=[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($taskPoll.Replace("`r`n","`n"))) +while ($true) { + $taskPreference=$ErrorActionPreference + $ErrorActionPreference='Continue' + $taskLines=@(& ssh -o BatchMode=yes -o ConnectTimeout=15 -o ServerAliveInterval=15 -o ServerAliveCountMax=2 $HostAlias "printf %s $taskEncoded | base64 -d | python" 2>$null) + $ErrorActionPreference=$taskPreference + $taskLine=$taskLines | Where-Object { $_ -like 'SNAPSHOT_JSON=*' } | Select-Object -Last 1 + Add-Content -LiteralPath (Join-Path $LogDirectory 'poll.log') -Value (@((Get-Date -Format o))+$taskLines) -Encoding UTF8 + if ($taskLine) { + $taskState=$taskLine.Substring(14) | ConvertFrom-Json + if ($taskState.failed.Count -gt 0) { throw 'Experiment failed; retained. Inspect poll.log.' } + if ($taskState.complete) { + & powershell.exe -NoProfile -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot '../receive-project.ps1') -HostAlias $HostAlias + if ($LASTEXITCODE -ne 0) { throw 'Commit recovery failed' } + & git -C (Join-Path $PSScriptRoot '..') push origin "codex/$HostAlias" + if ($LASTEXITCODE -ne 0) { throw 'Local push failed' } + Add-Content -LiteralPath (Join-Path $LogDirectory 'poll.log') -Value 'COMPLETE: result commit recovered and pushed; instance retained.' + break + } + } + Start-Sleep -Seconds $IntervalSeconds +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/monitor_qwen3_nofit_transport_zs6.ps1 b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/monitor_qwen3_nofit_transport_zs6.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..0a73aecaba20c6034c5a38059fda24ad47b42b7b --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/monitor_qwen3_nofit_transport_zs6.ps1 @@ -0,0 +1,78 @@ +param( + [string]$HostAlias='vast-paper_revision-qwen3-base-nofit-transport-zs6-54974094-20261009', + [string]$LogDirectory='C:\Users\CELL\orca\artifacts\paper_revision\qwen3-base-nofit-transport-zs6-20261009-monitor', + [string]$RemoteRoot='/workspace/artifacts/qwen3-base-nofit-transport-zs6-20261009', + [ValidateRange(30,3600)][int]$IntervalSeconds=300, + [ValidateRange(1,40)][int]$RemovedBlocks=9, + [switch]$FullBench14 +) +$ErrorActionPreference='Stop' +New-Item -ItemType Directory -Force -Path $LogDirectory | Out-Null +$root=$RemoteRoot +if ($root -notmatch '^/workspace/artifacts/[A-Za-z0-9._-]+$') { throw 'Unsafe remote artifact root' } +$poll=@' +from pathlib import Path +import json,re,time,subprocess +r=Path('__REMOTE_ROOT__') +def tail(p): + return [re.sub(r'https?://\S+','[URL omitted]',s)[-500:] for s in p.read_text(errors='replace').splitlines()[-3:]] if p.exists() else [] +arms={} +for arm in ['off','on']: + d=r/'full'/arm;s=r/'smoke'/arm + arms[arm]={'smoke_verified':(s/'VERIFIED').exists(), + 'markers':[m for m in ['PRUNE_COMPLETE','EVAL_COMPLETE','BACKUP_COMPLETE'] if (d/m).exists()], + 'cycles':len(list((d/'run').glob('cycle_*/cycle_metadata.json'))), + 'logs':{phase:{n:tail(r/phase/arm/'logs'/n) for n in ['prune.log','zs6.log','ppl.log','eval14.log']} for phase in ['smoke','full']}} +print('SNAPSHOT_JSON='+json.dumps({'utc':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()), + 'failed':any((r/m).exists() for m in ['FAILED','BOOTSTRAP_FAILED','EVAL_FAILED','off_FAILED','on_FAILED']), + 'complete':(r/'COMPLETE').exists(),'arms':arms, + 'logs':{n:tail(r/n) for n in ['bootstrap.log','eval_prepare.log','controller.log','off_pipeline.log','on_pipeline.log']}})) +print(subprocess.check_output(['nvidia-smi','--query-gpu=index,utilization.gpu,memory.used','--format=csv,noheader'],text=True)) +'@ +$poll=$poll.Replace('__REMOTE_ROOT__',$root) +$payload=[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($poll.Replace("`r`n","`n"))) +while ($true) { + $lines=@(& ssh -o BatchMode=yes -o ConnectTimeout=15 -o ServerAliveInterval=15 -o ServerAliveCountMax=2 $HostAlias "printf %s $payload | base64 -d | python") + $code=$LASTEXITCODE + Add-Content -LiteralPath (Join-Path $LogDirectory 'poll.log') -Value (@((Get-Date -Format o))+$lines) -Encoding UTF8 + $line=$lines | Where-Object { $_ -like 'SNAPSHOT_JSON=*' } | Select-Object -Last 1 + if ($code -eq 0 -and $line) { + $snapshot=$line.Substring('SNAPSHOT_JSON='.Length) | ConvertFrom-Json + if ($snapshot.failed) { throw 'No-FiT transport control failed; inspect logs. Instance retained.' } + if ($snapshot.complete) { + $dest=Join-Path $LogDirectory 'completed_results' + New-Item -ItemType Directory -Force -Path $dest | Out-Null + & scp "${HostAlias}:${root}/summary.json" (Join-Path $dest 'summary.json') + if ($LASTEXITCODE -ne 0) { throw 'Summary recovery failed' } + foreach ($arm in @('off','on')) { + $folder=Join-Path $dest $arm + New-Item -ItemType Directory -Force -Path $folder | Out-Null + if ($FullBench14) { + & scp -O "${HostAlias}:${root}/full/${arm}/eval/*.json" $folder + } else { + & scp "${HostAlias}:${root}/full/${arm}/eval/zs6.json" (Join-Path $folder 'zs6.json') + } + if ($LASTEXITCODE -ne 0) { throw "Evaluation recovery failed: $arm" } + if ($rowHasPpl = ($snapshot.arms.$arm.logs.full.'ppl.log'.Count -gt 0)) { + & scp "${HostAlias}:${root}/full/${arm}/eval/final_ppl.json" (Join-Path $folder 'final_ppl.json') + if ($LASTEXITCODE -ne 0) { throw "PPL recovery failed: $arm" } + } + & scp -r "${HostAlias}:${root}/full/${arm}/audit" $folder + if ($LASTEXITCODE -ne 0) { throw "Audit recovery failed: $arm" } + & scp "${HostAlias}:${root}/${arm}_backup.json" (Join-Path $folder 'backup.json') + if ($LASTEXITCODE -ne 0) { throw "Backup receipt recovery failed: $arm" } + } + $rows=Get-Content -LiteralPath (Join-Path $dest 'summary.json') -Raw -Encoding UTF8 | ConvertFrom-Json + if ($FullBench14) { $rows=$rows.rows } + if ($rows.Count -ne 2) { throw 'Expected two completed arms' } + foreach ($row in $rows) { + if ($row.fit -or $row.removed_blocks -ne $RemovedBlocks -or -not $row.backup.verified -or -not $row.backup.all_payload_hashes_match) { + throw 'Protocol/backup mismatch' + } + } + Add-Content -LiteralPath (Join-Path $LogDirectory 'poll.log') -Value 'COMPLETE: both controls recovered; instance retained.' + break + } + } + Start-Sleep -Seconds $IntervalSeconds +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/nofit_merge_control.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/nofit_merge_control.py new file mode 100644 index 0000000000000000000000000000000000000000..40b2aa630419867f4a0eb965fe39a8267276e7d1 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/nofit_merge_control.py @@ -0,0 +1,37 @@ +"""Exact non-Fisher merge controls; never construct an optimizer or FiT U.""" +from __future__ import annotations + +import torch +from fusegpt.fixed_span_merge_init import MLP_NAMES, tensor_hash +from fusegpt.fit_ablation_mka import PROJECTION_WEIGHTS + + +def merge_in_place(a, transported_b, method, *, mka_lambda=None): + ratio = {"wa": 1., "wb": 0., "average": .5}.get(method, mka_lambda) + if method not in {"wa", "wb", "average", "mka"} or ratio is None or not 0 <= ratio <= 1: + raise ValueError((method, ratio)) + names = tuple(PROJECTION_WEIGHTS) if method == "mka" else MLP_NAMES + frozen = {n: tensor_hash(p) for n, p in a.named_parameters() if n not in names} + source = {n: tensor_hash(p) for n, p in transported_b.named_parameters()} + before = {n: tensor_hash(a.get_parameter(n)) for n in names} + with torch.no_grad(): + for name in names: + left = a.get_parameter(name) + right = transported_b.get_parameter(name) + assert left.shape == right.shape + # Match the corrected MKA / exact endpoint controls: FP32 mix, + # one cast to deployed BF16. No sigmoid endpoint clamping. + initial = left.detach().clone() if ratio == 1 else ( + right.detach().clone() if ratio == 0 else + torch.lerp(right.detach().float(), left.detach().float(), ratio).to(left.dtype)) + assert torch.isfinite(initial).all(), name + left.copy_(initial) + assert {n: tensor_hash(p) for n, p in a.named_parameters() if n not in names} == frozen + assert {n: tensor_hash(p) for n, p in transported_b.named_parameters()} == source + return {"initializer": method, "lambda_left": ratio, "lambda_right": 1 - ratio, + "projection_names": list(names), "scope": "attention_and_mlp" if method == "mka" else "mlp", + "frozen_anchor_sha256": frozen, "anchor_before_sha256": before, + "source_post_transport_sha256": source, + "anchor_after_sha256": {n: tensor_hash(a.get_parameter(n)) for n in names}, + "norm_policy": "keep_Wa_including_QK_norms", "source_mutated_by_merge": False, + "fisher_computed": False, "fit": False, "optimizer_steps": 0, "backward_calls": 0} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/offline_eval_policy.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/offline_eval_policy.py new file mode 100644 index 0000000000000000000000000000000000000000..44cb57cdff81cc7b2d738749364bfca9c156bd47 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/offline_eval_policy.py @@ -0,0 +1,26 @@ +"""Run-local offline policy, applied before importing Hugging Face libraries.""" +import json +import os +from pathlib import Path + + +def apply_run_offline_policy(argv): + output = None + for index, value in enumerate(argv): + if value == '--output' and index + 1 < len(argv): + output = argv[index + 1] + elif value.startswith('--output='): + output = value.split('=', 1)[1] + if output is None: + return None + policy_path = Path(output).parent.parent / 'offline_eval.json' + if not policy_path.is_file(): + return None + policy = json.loads(policy_path.read_text()) + cache = Path(policy['wikitext_cache']) + if not (cache / 'dataset_dict.json').is_file(): + raise FileNotFoundError(f'Offline WikiText cache missing: {cache}; no online fallback') + for key in ('HF_HUB_OFFLINE', 'HF_DATASETS_OFFLINE', 'TRANSFORMERS_OFFLINE'): + os.environ[key] = '1' + print('STRICT_OFFLINE_RUN_POLICY_APPLIED', flush=True) + return policy diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/package_qwen3_nofit_merge_backup.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/package_qwen3_nofit_merge_backup.py new file mode 100644 index 0000000000000000000000000000000000000000..81117956267ad424e29f5719bfdcbdce1f102307 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/package_qwen3_nofit_merge_backup.py @@ -0,0 +1,47 @@ +"""Retain every stage, not just final weights; enable previous-teacher MSE.""" +from pathlib import Path +import json +import shutil +import sys +out,backup,project,external,root=map(Path,sys.argv[1:]) +backup.mkdir(parents=True,exist_ok=True) +def link_immutable(src,dst): + src,dst=Path(src),Path(dst) + if dst.exists(): + assert src.samefile(dst), f'Refusing to overwrite non-linked file: {dst}' + else: + dst.hardlink_to(src) + return str(dst) +for folder in ('compact','audit','eval','logs'): + shutil.copytree(out/folder,backup/folder,dirs_exist_ok=True, + copy_function=link_immutable if folder in ('compact','audit') else shutil.copy2) +assert len(list((backup/'compact').glob('stage_*/*.pt')))==9 +for c in range(1,10): + d=backup/f'cycle_metadata/cycle_{c}';d.mkdir(parents=True,exist_ok=True) + shutil.copy2(out/f'run/cycle_{c}/cycle_metadata.json',d/'cycle_metadata.json') +shutil.copytree(root/'inputs',backup/'inputs',dirs_exist_ok=True) +# Dense activations are shared across the four arms. Exact inputs + original +# immutable teacher revision suffice to regenerate them; save protocol here. +shutil.copy2(root/'local_mse_reference/protocol.json',backup/'local_mse_protocol.json') +shutil.copy2(root/'local_mse_reference/evaluation_input_ids.pt',backup/'inputs/local_mse_evaluation_input_ids.pt') +shutil.copy2(root/'fisher_reference/local_mse.json',backup/'fisher_reference_local_mse.json') +shutil.copy2(root/'fisher_reference/restore_receipt.json',backup/'fisher_reference_restore_receipt.json') +code=backup/'code';code.mkdir(exist_ok=True) +for path in [*project.glob('scripts/*qwen3_nofit*'),project/'src/fusegpt/nofit_merge_control.py', + project/'src/fusegpt/fit_ablation_mka.py',project/'src/fusegpt/fixed_span_merge_init.py', + project/'src/fusegpt/abprune_compact.py',project/'src/fusegpt/abprune_stagewise_geometry.py', + project/'scripts/eval_llama31_fit_ablation_one.sh',project/'scripts/evaluate_abprune_compact_ppl.py', + project/'scripts/evaluate_hchc_wikitext2_ppl.py',project/'scripts/eval_opencompass_ppl_fast.py', + project/'scripts/eval_llama31_8b_12bench.py',project/'scripts/aggregate_transport_rho_fit_14bench.py', + project/'scripts/prepare_lmeval9_local_cache.py',project/'scripts/offline_eval_policy.py', + project/'exp_md/2026-10-11_qwen3-base-nofit-merge4.md', + *[external/'src'/n for n in ['fuse_layers.py','fuse_layers_distill.py','source_feature_transport.py','common_lm_data.py','dm500_rows.py']]]: + if path.is_file(): shutil.copy2(path,code/path.name) +import subprocess +(code/'project_commit.txt').write_text(subprocess.check_output(['git','-C',str(project),'rev-parse','HEAD'],text=True)) +receipt=dict(stages=list(range(1,10)),stage0_original_model='Qwen/Qwen3-8B-Base', + stage0_revision='49e3418fbbbca6ecbdf9608b4d22e5a407081db4', + previous_teacher_reconstruction='stage(k-1) compact anchor + immutable original unremoved layers', + original_input_ids_saved=True,all_transport_matrices_saved=True, + intermediate_weights_saved=True,fit=False) +(backup/'stage_retention.json').write_text(json.dumps(receipt,indent=2)+'\n') diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/prepare_lmeval9_local_cache.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/prepare_lmeval9_local_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..0c169eca6c555cbd1fdb7fd1dd9bd9a085806b09 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/prepare_lmeval9_local_cache.py @@ -0,0 +1,108 @@ +"""Materialize the exact harness datasets once, without changing task definitions.""" +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +import hashlib +import json +from pathlib import Path +import time + +import datasets +from lm_eval.tasks import TaskManager + + +def cache_key(path, name, kwargs): + return hashlib.sha256(json.dumps([path, name, kwargs], sort_keys=True).encode()).hexdigest() + + +def specs(): + manager = TaskManager() + names = ["piqa", "winogrande", "hellaswag", "arc_challenge", "arc_easy", "openbookqa", "sciq"] + entries = [manager.task_index[n] for n in names] + entries += [e for e in manager.task_index.values() + if e.yaml_path and e.yaml_path.parent.name == "default" + and (e.cfg or {}).get("dataset_path") == "cais/mmlu"] + entries += [e for e in manager.task_index.values() + if (e.cfg or {}).get("dataset_path") == "haonan-li/cmmlu"] + unique = {} + for entry in entries: + cfg = entry.cfg + path, name, kwargs = cfg["dataset_path"], cfg.get("dataset_name"), cfg.get("dataset_kwargs") or {} + unique[cache_key(path, name, kwargs)] = {"path": path, "name": name, "kwargs": kwargs} + assert len(unique) == 131, len(unique) + return unique + + +def materialize(root, key, spec, offline): + target = root / key + metadata = target / "source.json" + if metadata.is_file(): + return key, json.loads(metadata.read_text()) + start = time.monotonic() + try: + data = datasets.load_dataset(spec["path"], spec["name"], **spec["kwargs"]) + except Exception: + if offline: + return key, None + raise + data.save_to_disk(str(target)) + report = {**spec, "splits": {k: {"rows": len(v), "fingerprint": v._fingerprint} + for k, v in data.items()}} + metadata.write_text(json.dumps(report, indent=2) + "\n") + restored = datasets.load_from_disk(str(target)) + assert {k: len(v) for k, v in restored.items()} == {k: len(v) for k, v in data.items()} + assert all(restored[k].features == data[k].features for k in data) + assert all(restored[k].data.table.equals(data[k].data.table) for k in data) + print("READY", spec["path"], spec["name"], round(time.monotonic() - start, 2), flush=True) + return key, report + + +def install_local_loader(root): + manifest = json.loads((root / "manifest.json").read_text()) + assert len(manifest) == 131 + memo = {} + + def local_load(path, name=None, **kwargs): + key = cache_key(path, name, kwargs) + if key not in manifest: + raise RuntimeError(f"Unprepared dataset request: {path}/{name}; no online fallback") + if key not in memo: + memo[key] = datasets.load_from_disk(str(root / key)) + print("LOCAL_DATASET", path, name, flush=True) + return memo[key] + + datasets.load_dataset = local_load + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("root", type=Path) + parser.add_argument("--offline-export", action="store_true") + parser.add_argument("--validate-local", action="store_true", + help="Validate a complete transferred cache without Hub requests") + parser.add_argument("--workers", type=int, default=8) + args = parser.parse_args() + args.root.mkdir(parents=True, exist_ok=True) + sources = specs() + if args.validate_local: + manifest = json.loads((args.root / "manifest.json").read_text()) + assert set(manifest) == set(sources), "Task configuration mismatch" + for key, spec in sources.items(): + report = manifest[key] + assert all(report[k] == v for k, v in spec.items()) + assert json.loads((args.root / key / "source.json").read_text()) == report + data = datasets.load_from_disk(str(args.root / key)) + assert {k: len(v) for k, v in data.items()} == { + k: v["rows"] for k, v in report["splits"].items()} + print("VALIDATED_LOCAL", len(manifest), "NO_HUB_REQUESTS", flush=True) + raise SystemExit(0) + completed = {} + with ThreadPoolExecutor(max_workers=1 if args.offline_export else args.workers) as executor: + futures = [executor.submit(materialize, args.root, k, v, args.offline_export) + for k, v in sources.items()] + for future in as_completed(futures): + key, result = future.result() + if result is not None: + completed[key] = result + print("PREPARED", len(completed), "OF", len(sources), flush=True) + if len(completed) == len(sources): + (args.root / "manifest.json").write_text(json.dumps(completed, indent=2) + "\n") diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/prepare_qwen3_nofit_merge_inputs.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/prepare_qwen3_nofit_merge_inputs.py new file mode 100644 index 0000000000000000000000000000000000000000..e7bcf87b09f3509d8144c5d52238ea3f3f10f75f --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/prepare_qwen3_nofit_merge_inputs.py @@ -0,0 +1,35 @@ +"""Materialize the exact historical calibration once for four workers.""" +import argparse +import json +import os +from pathlib import Path +import sys + +p = argparse.ArgumentParser() +p.add_argument('--external-root', type=Path, required=True) +p.add_argument('--model', required=True) +p.add_argument('--output', type=Path, required=True) +a = p.parse_args() +sys.path.insert(0, str(a.external_root / 'src')) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) +import torch +from transformers import AutoTokenizer +from common_lm_data import SharedLMDataSpec, build_chunks +from fusegpt.fixed_span_merge_init import tensor_hash + +os.environ['ABPRUNE_DATASET_REVISION'] = 'b5f90f419b7489cdba26fdbc8c022fcb5562f968' +tokenizer = AutoTokenizer.from_pretrained(a.model, local_files_only=True) +chunks = build_chunks(SharedLMDataSpec(dataset='slimpajama', config=None, split='train', + text_field='text', seq_len=1024, num_sequences=64, seed=42), tokenizer) +ids = torch.stack(chunks).long() +assert ids.shape == (64, 1024) +assert tensor_hash(ids) == '85a84b9a68cf0856fd363186d3a273860886b1c17f876cae145fd092454aff55' +a.output.mkdir(parents=True, exist_ok=True) +torch.save({'input_ids': ids, 'attention_mask': torch.ones_like(ids)}, a.output / 'calib.pt') +manifest = dict(model='Qwen/Qwen3-8B-Base', model_revision='49e3418fbbbca6ecbdf9608b4d22e5a407081db4', + dataset='DKYoon/SlimPajama-6B', revision=os.environ['ABPRUNE_DATASET_REVISION'], split='train', + packing='historical nonstreaming dataset.shuffle(seed42); concatenate encoded text without separators or BOS', + samples=64, seq_len=1024, batch_size=2, seed=42, valid_tokens=65536, + input_ids_sha256=tensor_hash(ids), attention_mask_sha256=tensor_hash(torch.ones_like(ids))) +(a.output / 'manifest.json').write_text(json.dumps(manifest, indent=2)+'\n') +print('EXACT_HISTORICAL_CALIBRATION_VERIFIED', json.dumps(manifest), flush=True) diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/project_commit.txt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/project_commit.txt new file mode 100644 index 0000000000000000000000000000000000000000..a3bc53a9e815a696d1461e9e40829b62cdda9029 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/project_commit.txt @@ -0,0 +1 @@ +228a79f631d8ddf633f8f5e77677af7add92c2bc diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/qwen3_nofit_local_mse.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/qwen3_nofit_local_mse.py new file mode 100644 index 0000000000000000000000000000000000000000..e57e264f76c08e19212e5493a8203a03d2775fa7 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/qwen3_nofit_local_mse.py @@ -0,0 +1,101 @@ +"""Evaluation-only boundary MSE, all variants against the same dense teacher.""" +from __future__ import annotations +import argparse +import json +from pathlib import Path +import torch +from fusegpt.fixed_span_merge_init import tensor_hash +from evaluate_hchc_wikitext2_ppl import packed_wikitext_tokens, primary + +REVISION = '49e3418fbbbca6ecbdf9608b4d22e5a407081db4' + + +class BoundaryReached(Exception): + pass + + +@torch.no_grad() +def capture(model, ids, boundary, batch_size=2): + outputs, inputs = [], [] + current = {} + def pre(module, args, kwargs): + value = args[0] if args else kwargs['hidden_states'] + current['input'] = value.detach().cpu() + def post(module, args, output): + current['output'] = primary(output).detach().cpu() + raise BoundaryReached() + h1 = model.model.layers[12].register_forward_pre_hook(pre, with_kwargs=True) + h2 = model.model.layers[boundary].register_forward_hook(post) + try: + for start in range(0, len(ids), batch_size): + current.clear() + batch = ids[start:start+batch_size].to('cuda') + try: + model.model(input_ids=batch, attention_mask=torch.ones_like(batch), use_cache=False) + except BoundaryReached: + pass + else: + raise RuntimeError('Boundary hook did not run') + inputs.append(current['input']) + outputs.append(current['output']) + finally: + h1.remove(); h2.remove() + return torch.cat(inputs), torch.cat(outputs) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--model', required=True) + p.add_argument('--cache', type=Path, required=True) + p.add_argument('--wikitext-cache', type=Path) + p.add_argument('--prepare', action='store_true') + p.add_argument('--compact-stage', type=Path) + p.add_argument('--output', type=Path) + a = p.parse_args() + from transformers import AutoModelForCausalLM, AutoTokenizer + if a.prepare: + assert a.wikitext_cache and not a.compact_stage + tokenizer = AutoTokenizer.from_pretrained(a.model, local_files_only=True) + ids = packed_wikitext_tokens(tokenizer, 'test', dataset_cache=a.wikitext_cache)[:64*1024].reshape(64,1024) + model = AutoModelForCausalLM.from_pretrained(a.model, dtype=torch.bfloat16, + local_files_only=True, attn_implementation='eager').to('cuda').eval() + anchor_inputs, targets = capture(model, ids, 21) + a.cache.mkdir(parents=True, exist_ok=True) + torch.save(dict(input_ids=ids, anchor_inputs=anchor_inputs, targets=targets), a.cache/'dense_teacher.pt') + torch.save(dict(input_ids=ids, attention_mask=torch.ones_like(ids)), a.cache/'evaluation_input_ids.pt') + protocol = dict(model='Qwen/Qwen3-8B-Base', revision=REVISION, + teacher='common_original_dense', dense_input_layer=12, dense_output_layer=21, + student_output_layer=12, sequences=64, sequence_length=1024, valid_tokens=65536, + metric='sum float32 squared activation differences / valid tokens / hidden width', + dtype='bfloat16 eager; float32 differences; float64 aggregate', training=False, + input_ids_sha256=tensor_hash(ids), anchor_inputs_sha256=tensor_hash(anchor_inputs), + targets_sha256=tensor_hash(targets), dataset='WikiText2 test, first64 packed1024 windows, HCHC packing') + (a.cache/'protocol.json').write_text(json.dumps(protocol,indent=2)+'\n') + print('COMMON_DENSE_TEACHER_READY',json.dumps(protocol),flush=True) + return + assert a.compact_stage and a.output + from fusegpt.abprune_stagewise_geometry import load_compact_model + reference = torch.load(a.cache/'dense_teacher.pt',weights_only=True) + protocol = json.loads((a.cache/'protocol.json').read_text()) + for field in ('input_ids','anchor_inputs','targets'): + assert tensor_hash(reference[field]) == protocol[field+'_sha256'] + model = load_compact_model(a.model,a.compact_stage,cycle=9,anchor=12, + dtype=torch.bfloat16,device=torch.device('cuda'),attn='eager',revision=REVISION).eval() + inputs, outputs = capture(model,reference['input_ids'],12) + assert torch.equal(inputs,reference['anchor_inputs']), 'Upstream prefix changed; common MSE inputs not matched' + # Keep accumulation chunked; the cached tensors themselves remain BF16. + numerator = denominator = 0. + for output,target in zip(outputs,reference['targets'],strict=True): + numerator += (output.float()-target.float()).square().double().sum().item() + denominator += target.float().square().double().sum().item() + assert denominator > 0 and numerator >= 0 + report = dict(local_mse=numerator/outputs.numel(),local_nmse=numerator/denominator, + squared_error_sum=numerator,target_energy_sum=denominator,elements=outputs.numel(), + protocol=protocol,prefix_input_exact_match=True,compact_stage=str(a.compact_stage),fit=False) + a.output.parent.mkdir(parents=True,exist_ok=True) + a.output.write_text(json.dumps(report,indent=2)+'\n') + print('LOCAL_MSE_COMPLETE',json.dumps(report),flush=True) + + +if __name__ == '__main__': + main() diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/qwen3_nofit_transport_config.sh b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/qwen3_nofit_transport_config.sh new file mode 100644 index 0000000000000000000000000000000000000000..e6d6cd478789ccc3d49cc0439d15172fffde6dea --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/qwen3_nofit_transport_config.sh @@ -0,0 +1,27 @@ +# Shared defaults preserve the completed Base protocol. Instruct is explicit. +if [[ "${QWEN_NOFIT_VARIANT:-base}" == llama_base ]]; then + export NOFIT_MODEL_ID=meta-llama/Llama-3.1-8B + export NOFIT_MODEL_REVISION=d04e592bb4f6aa9cfee91e2e20afa771667e1d4b + export NOFIT_ANCHOR=21 NOFIT_INSTRUCTION_DATA=0 NOFIT_MEASURE_PPL=1 + export ABPRUNE_DATASET_REVISION=b5f90f419b7489cdba26fdbc8c022fcb5562f968 + export NOFIT_EXPECTED_CALIB_SHA256=fa4a6219f7a7b17364eef6b5b7771533b57a407f1d26db1d483bbd955814ce38 + export NOFIT_PROTOCOL=exp_md/2026-10-09_llama31-base-nofit-transport-bench14.md + export NOFIT_RUN_SLUG=20261009-llama31-base-nofit-transport-bench14 + export NOFIT_MODEL_TAG=llama31_8b_base +elif [[ "${QWEN_NOFIT_VARIANT:-base}" == instruct ]]; then + export NOFIT_MODEL_ID=Qwen/Qwen3-8B + export NOFIT_MODEL_REVISION=b968826d9c46dd6066d109eabc6255188de91218 + export NOFIT_ANCHOR=24 NOFIT_INSTRUCTION_DATA=1 NOFIT_MEASURE_PPL=1 + export ABPRUNE_DATASET_REVISION=ce9ed009ef3292bac33d3520afaf5ec804066120 + export NOFIT_EXPECTED_CALIB_SHA256='' + export NOFIT_PROTOCOL=exp_md/2026-10-09_qwen3-instruct-nofit-transport-zs6-ppl.md + export NOFIT_RUN_SLUG=20261009-qwen3-instruct-nofit-transport-zs6-ppl +else + export NOFIT_MODEL_ID=Qwen/Qwen3-8B-Base + export NOFIT_MODEL_REVISION=49e3418fbbbca6ecbdf9608b4d22e5a407081db4 + export NOFIT_ANCHOR=12 NOFIT_INSTRUCTION_DATA=0 NOFIT_MEASURE_PPL=0 + export ABPRUNE_DATASET_REVISION=b5f90f419b7489cdba26fdbc8c022fcb5562f968 + export NOFIT_EXPECTED_CALIB_SHA256=85a84b9a68cf0856fd363186d3a273860886b1c17f876cae145fd092454aff55 + export NOFIT_PROTOCOL=exp_md/2026-10-09_qwen3-base-nofit-transport-zs6.md + export NOFIT_RUN_SLUG=20261009-qwen3-base-nofit-transport-zs6 +fi diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/run_qwen3_nofit_merge_control.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/run_qwen3_nofit_merge_control.py new file mode 100644 index 0000000000000000000000000000000000000000..59b1cf689eca5bf21a9b56d14c7d4b29add055c5 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/run_qwen3_nofit_merge_control.py @@ -0,0 +1,134 @@ +"""Our fixed span + existing MLP transport, exact merge, no FiT / Fisher.""" +from __future__ import annotations +import argparse +import json +from pathlib import Path +import sys + + +def main(): + p = argparse.ArgumentParser(allow_abbrev=False) + p.add_argument('--external-root', type=Path, required=True) + p.add_argument('--inputs', type=Path, required=True) + p.add_argument('--audit-root', type=Path, required=True) + p.add_argument('--initializer', choices=['wa', 'wb', 'average', 'mka'], required=True) + opt, rest = p.parse_known_args() + sys.path.insert(0, str(opt.external_root / 'src')) + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) + import torch + import fuse_layers as base + import source_feature_transport as transport + from fusegpt.fixed_span_merge_init import tensor_hash + from fusegpt.nofit_merge_control import merge_in_place + from fusegpt.fit_ablation_come import valid_calibration_batches + opt.audit_root.mkdir(parents=True, exist_ok=True) + payload = torch.load(opt.inputs / 'calib.pt', weights_only=True) + manifest = json.loads((opt.inputs / 'manifest.json').read_text()) + assert tensor_hash(payload['input_ids']) == manifest['input_ids_sha256'] + assert manifest['input_ids_sha256'] == '85a84b9a68cf0856fd363186d3a273860886b1c17f876cae145fd092454aff55' + assert payload['input_ids'].shape == (64, 1024) + records = [] + original_parse = base.parse_args + + def parse(): + args = original_parse() + assert args.skip_distill and args.lora_epochs == 0 and args.source_feature_transport + assert args.skip_eval and args.no_head_permute_merge and args.no_head_permute_select + assert args.seed == 42 and args.batch_size == 2 and args.num_samples == 64 + assert args.norm_policy == 'copy_n1_n2' and args.num_progressive in (2, 9) + return args + + base.parse_args = parse + + def calibration(args, tokenizer): + d = torch.utils.data.TensorDataset(payload['input_ids'], payload['attention_mask']) + return [], list(payload['input_ids']), torch.utils.data.DataLoader(d, batch_size=2, shuffle=False) + + base.build_calibration_dataloader = calibration + + def select(args, model, layers, dataloader, cosine_loader, scores, start, excluded): + cycle = len(records) + 1 + assert len(layers) == 37 - cycle + ratio = {'wa': 1., 'wb': 0., 'average': .5}.get(opt.initializer) + record = dict(cycle=cycle, anchor=12, removed_original_layer=12+cycle, + selection_policy='fixed_our_span_13_22_anchor12', fit=False) + if opt.initializer == 'mka': + from fusegpt import fit_ablation_mka as mka + config = mka.GeometryConfig() + batches = valid_calibration_batches(dataloader) + assert len(batches) == 64 + features = mka.collect_features(model, batches, pooling=config.pooling) + manifolds = [mka.diffusion_embedding(x, config) for x in features] + del features + matrix, entropies = mka.npib_matrix([x[0] for x in manifolds], config) + ratio = mka.upstream_lambda(matrix, 12, 13, config.beta) + record.update(npib_matrix=matrix.tolist(), entropies=entropies, + geometry_config=vars(config), geometry_samples=64, + implementation='corrected_local_MKA_not_exact_paper_reproduction') + del batches, manifolds, matrix + record['lambda_left'] = ratio + records.append(record) + print('NOFIT_FIXED_SELECTION', cycle, opt.initializer, ratio, flush=True) + return 12, None, {'method': opt.initializer, 'fisher_computed': False} + + base.resolve_layer_idx = select + solve = transport.solve_transport_from_statistics + + def saved_solve(*args, **kwargs): + solution, stats = solve(*args, **kwargs) + torch.save(solution.detach().cpu(), opt.audit_root / f'cycle_{len(records):02d}_transport.pt') + records[-1]['transport_matrix_sha256'] = tensor_hash(solution) + return solution, stats + + transport.solve_transport_from_statistics = saved_solve + transport_fn = base.fit_and_apply_source_feature_transport + + def transported(**kwargs): + b = kwargs['layer_b'] + before = {n: tensor_hash(p) for n, p in b.named_parameters()} + result = transport_fn(**kwargs) + after = {n: tensor_hash(p) for n, p in b.named_parameters()} + records[-1]['transport'] = dict(stats=result, source_before_sha256=before, + source_after_sha256=after, changed=[n for n in before if before[n] != after[n]]) + assert set(records[-1]['transport']['changed']) <= {'mlp.gate_proj.weight', 'mlp.up_proj.weight'} + return result + + base.fit_and_apply_source_feature_transport = transported + base.compute_fisher = lambda *a, **kw: ([{}, {}], 0, [{}, {}]) + base.compute_same_anchor_contextual_fisher = lambda **kw: ([{}, {}], 0, [{}, {}], + {'computed': False, 'anchor_candidate_mean_nll': 0., 'transported_source_candidate_mean_nll': 0.}) + + def merge(a, b, *args, **kwargs): + record = records[-1] + assert 'transport' in record + audit = merge_in_place(a, b, opt.initializer, mka_lambda=record['lambda_left']) + assert audit['source_post_transport_sha256'] == record['transport']['source_after_sha256'] + record['merge'] = audit + (opt.audit_root / f'cycle_{len(records):02d}_merge.json').write_text(json.dumps(record, indent=2)+'\n') + return len(audit['projection_names']) + + base.merge_layers = merge + # Qwen3 includes attention Q/K RMSNorm. All norms remain the recipient's; + # do not let a generic norm policy silently modify them after our merge. + base.apply_norm_policy = lambda *a, **kw: None + def forbidden(*a, **kw): + raise AssertionError('FiT/gradient optimization is forbidden in this experiment') + base.distill_reparam_merge = forbidden + torch.Tensor.backward = forbidden + (opt.audit_root / 'data_manifest.json').write_text(json.dumps(manifest, indent=2)+'\n') + sys.argv = [sys.argv[0], *rest] + base.main() + output = Path(rest[rest.index('--output_dir')+1]) + for record in records: + path = output / f"cycle_{record['cycle']}" / 'cycle_metadata.json' + metadata = json.loads(path.read_text()) + metadata.update(fisher_computed=False, fit=False, nofit_merge_control=record) + if metadata.get('source_feature_transport'): + metadata['source_feature_transport']['fisher_order'] = 'Transport then non-Fisher exact merge; no FiT' + path.write_text(json.dumps(metadata, indent=2)+'\n') + (opt.audit_root / 'selection.json').write_text(json.dumps(records, indent=2)+'\n') + print('NOFIT_ALL_CYCLES_COMPLETE', len(records), flush=True) + + +if __name__ == '__main__': + main() diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/run_qwen3_nofit_transport_arm.sh b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/run_qwen3_nofit_transport_arm.sh new file mode 100644 index 0000000000000000000000000000000000000000..89dec3ff69e054616591cf9faa3ca004dfd5ec58 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/run_qwen3_nofit_transport_arm.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail +project="$1"; external="$2"; out="$3"; model="$4"; arm="$5"; gpu="$6"; cycles="${7:-9}" +source "$project/scripts/qwen3_nofit_transport_config.sh" +export CUDA_VISIBLE_DEVICES="$gpu" HF_HOME=/workspace/.hf_home HF_TOKEN_PATH=/root/.cache/huggingface/token +export HF_DATASETS_CACHE=/workspace/.hf_datasets36 HF_DATASETS_TRUST_REMOTE_CODE=1 +export HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 +export PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=4 +export PYTHONPATH="$external/src:$project/src:$project" +mkdir -p "$out/logs" "$out/audit" "$out/eval" +order=(); for ((i=0;i "$out/logs/prune.log" 2>&1 +python -m fusegpt.abprune_compact --run-root "$out/run" --output-root "$out/compact" \ + --model "$NOFIT_MODEL_ID" --base-revision "$NOFIT_MODEL_REVISION" \ + --run-id "${NOFIT_MODEL_TAG:-qwen3_8b_${QWEN_NOFIT_VARIANT:-base}}_nofit_transport_${arm}_20261009" --cycles "$cycles" --layer-order "${order[*]}" +touch "$out/PRUNE_COMPLETE" diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/run_qwen3_nofit_transport_control.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/run_qwen3_nofit_transport_control.py new file mode 100644 index 0000000000000000000000000000000000000000..dd0fbe2494c48efe5488cbaccdabe3573977b8b6 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/run_qwen3_nofit_transport_control.py @@ -0,0 +1,106 @@ +"""Matched contextual Fisher at a fixed span; only source transport differs.""" +import argparse +import hashlib +import json +from pathlib import Path +import sys + +p = argparse.ArgumentParser(add_help=False) +p.add_argument('--external-root', required=True) +p.add_argument('--audit-root', type=Path, required=True) +p.add_argument('--transport', choices=['on', 'off'], required=True) +p.add_argument('--anchor', type=int, default=12) +p.add_argument('--instruction-data', action='store_true') +p.add_argument('--expected-calib-sha256', default='85a84b9a68cf0856fd363186d3a273860886b1c17f876cae145fd092454aff55') +opt, rest = p.parse_known_args() +sys.path.insert(0, str(Path(opt.external_root) / 'src')) +import fuse_layers as base +if opt.instruction_data: + import os + os.environ['ABPRUNE_EXTERNAL_SRC'] = str(Path(opt.external_root) / 'src') + import run_external_abprune_instruct_order # installs the established first-turn record loaders +import fuse_layers_order as ordered + +opt.audit_root.mkdir(parents=True, exist_ok=True) +original_parse = base.parse_args +original_prepare = base.prepare_all_data +original_merge = base.distill_reparam_merge +original_contextual = base.compute_same_anchor_contextual_fisher +cycle = 0 + +def checked_parse(): + args = original_parse() + assert args.reparam_skip_train and not args.skip_distill + assert args.reparam_param_subset == 'mlp' and args.reparam_additive_u + assert args.reparam_rho_objective == 'none' and args.lora_epochs == 0 + assert bool(args.source_feature_transport) == (opt.transport == 'on') + return args + +def prepare(*args, **kwargs): + prepared = original_prepare(*args, **kwargs) + fingerprints = {} + for name in ('calib_loader', 'distill_loader'): + loader = getattr(prepared, name) + ids_hash, mask_hash = hashlib.sha256(), hashlib.sha256() + tokens = 0 + for item in loader.dataset: + ids, mask = item[:2] + ids_hash.update(ids.cpu().numpy().astype(' "$out/logs/ppl.log" 2>&1 + fi +) +eval_arm off 0 & off_pid=$! +eval_arm on 1 & on_pid=$! +failed=0; for pid in "$off_pid" "$on_pid"; do wait "$pid" || failed=1; done +((failed==0)) || exit 1 +python - "$root" "$project" <<'PY' +import json,math,shutil,sys,subprocess +from pathlib import Path +r=Path(sys.argv[1]); project=Path(sys.argv[2]); rows=json.loads((r/'summary.json').read_text()) +for row in rows: + result=json.loads((r/'full'/row['transport']/'eval/final_ppl.json').read_text()) + assert result['metrics']['tokens']==296815 and result['metrics']['sequences']==145 + assert result['decoder_layers']==27 and result['sequence_length']==2048 and result['batch_size']==4 + assert result['offline_policy'] and math.isfinite(result['metrics']['perplexity']) + row['ppl']=result['metrics']['perplexity']; row['ppl_protocol']=result['protocol'] + row['ppl_tokens']=result['metrics']['tokens']; row['ppl_sequence_length']=2048 +summary={'rows':rows,'historical_transport_on_fit':{'zs6':0.5528594177270905,'ppl':11.45046489901775, + 'record_id':'qwen3_base_transport_9block','anchor':12,'removed_interval':[13,22], + 'ppl_source_commit':'d9bc7529658ffc6f7c5af9e2bae0161c73cfc264', + 'ppl_source_path':'results/qwen3_8b_transport_llmstreamline_ppl_20261001/transport/9block/ppl.json'}, + 'code_commit':subprocess.check_output(['git','-C',str(project),'rev-parse','HEAD'],text=True).strip()} +(r/'summary_with_ppl.json').write_text(json.dumps(summary,indent=2)+'\n') +b=r/'backup/ppl_supplement'; b.mkdir(exist_ok=True) +shutil.copy2(r/'summary_with_ppl.json',b/'summary_with_ppl.json') +shutil.copy2(r/'wikitext2_cache_hashes.json',b/'wikitext2_cache_hashes.json') +for arm in ['off','on']: + d=b/arm;d.mkdir(exist_ok=True) + for source,name in [(r/'full'/arm/'eval/final_ppl.json','final_ppl.json'), + (r/'full'/arm/'logs/ppl.log','ppl.log'), + (r/'full'/arm/'eval/zs6.json','zs6.json')]: shutil.copy2(source,d/name) +for name in ['run_qwen3_nofit_transport_ppl.sh','evaluate_abprune_compact_ppl.py', + 'evaluate_hchc_wikitext2_ppl.py','offline_eval_policy.py']: + shutil.copy2(project/'scripts'/name,b/name) +shutil.copy2(project/'exp_md/2026-10-09_qwen3-base-nofit-transport-zs6.md',b/'protocol.md') +print(json.dumps({'ppl':{x['transport']:x['ppl'] for x in rows}}),flush=True) +PY +python "$project/scripts/write_tree_manifest.py" "$root/backup/ppl_supplement" +HF_HUB_OFFLINE=0 HF_DATASETS_OFFLINE=0 TRANSFORMERS_OFFLINE=0 \ +python "$project/scripts/upload_verified_hf_tree.py" "$root/backup/ppl_supplement" --repo-id LJYAI/paper_revision \ + --path-in-repo "runs/$alias/20261009-qwen3-base-nofit-transport-zs6/ppl_supplement" --verify-all-files \ + --commit-message "Qwen3 Base no-FiT Transport OFF ON WikiText PPL supplement" > "$root/ppl_backup.json" +touch "$root/PPL_COMPLETE" diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/source_feature_transport.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/source_feature_transport.py new file mode 100644 index 0000000000000000000000000000000000000000..242e2e52572627622decabcc62b7e24a1b7f0046 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/source_feature_transport.py @@ -0,0 +1,342 @@ +"""Stage-local source MLP transport into the anchor MLP input coordinate. + +The fitted map ``T`` satisfies ``X_anchor @ T ~= X_source``. For linear +input projections stored as ``W`` (PyTorch convention ``y = x @ W.T``), the +equivalent source projection in the anchor coordinate is ``W @ T.T``. + +This module intentionally transports only ``gate_proj`` and ``up_proj``. +``down_proj`` consumes the SwiGLU intermediate representation rather than the +decoder hidden coordinate and is therefore left unchanged. +""" + +from __future__ import annotations + +import math +from typing import Dict, List, Tuple + +import torch + + +class _StopAfterSource(RuntimeError): + """Abort a model forward once both MLP inputs have been captured.""" + + +def _capture_mlp_inputs(model, layer_a, layer_b, input_ids, attention_mask=None): + captured: Dict[str, torch.Tensor] = {} + + def capture_anchor(_module, inputs): + captured["anchor"] = inputs[0].detach() + + def capture_source(_module, inputs): + captured["source"] = inputs[0].detach() + raise _StopAfterSource() + + handles = ( + layer_a.mlp.register_forward_pre_hook(capture_anchor), + layer_b.mlp.register_forward_pre_hook(capture_source), + ) + try: + kwargs = {"input_ids": input_ids, "use_cache": False} + if attention_mask is not None: + kwargs["attention_mask"] = attention_mask + try: + model(**kwargs) + except _StopAfterSource: + pass + finally: + for handle in handles: + handle.remove() + + if set(captured) != {"anchor", "source"}: + raise RuntimeError( + "Failed to capture anchor/source MLP inputs; " + f"captured={sorted(captured)}" + ) + return captured["anchor"], captured["source"] + + +def solve_transport_from_statistics( + gram_sum: torch.Tensor, + cross_sum: torch.Tensor, + source_sq_sum: float, + token_count: int, + ridge_relative: float, +) -> Tuple[torch.Tensor, Dict[str, float]]: + """Solve ridge OLS and return its calibration-set reconstruction metrics.""" + if token_count <= 0: + raise ValueError("token_count must be positive") + if ridge_relative < 0.0: + raise ValueError("ridge_relative must be non-negative") + + gram = gram_sum / float(token_count) + cross = cross_sum / float(token_count) + target_sq = float(source_sq_sum) / float(token_count) + mean_diagonal = float(torch.diagonal(gram).mean().item()) + ridge = float(ridge_relative) * mean_diagonal + regularized = gram.clone() + regularized.diagonal().add_(ridge) + chol, info = torch.linalg.cholesky_ex(regularized) + if int(info.max().item()) != 0: + raise RuntimeError( + "Source-feature transport Cholesky failed: " + f"info={int(info.max().item())}" + ) + solution = torch.cholesky_solve(cross, chol) + + # Since (G + ridge I) T = C, tr(T' G T) can be computed without another + # O(d^3) multiply as - ridge ||T||^2. + dot = float((solution.double() * cross.double()).sum().item()) + solution_sq = float(solution.double().square().sum().item()) + predicted_sq = max(0.0, dot - ridge * solution_sq) + error_sq = max(0.0, target_sq + predicted_sq - 2.0 * dot) + denominator = math.sqrt(max(predicted_sq, 0.0) * max(target_sq, 0.0)) + identity = torch.eye( + solution.shape[0], dtype=solution.dtype, device=solution.device + ) + relative_identity_distance = float( + torch.linalg.vector_norm((solution - identity).float()).item() + / math.sqrt(float(solution.shape[0])) + ) + del identity + + stats = { + "tokens": int(token_count), + "ridge_relative": float(ridge_relative), + "ridge_absolute": ridge, + "gram_mean_diagonal": mean_diagonal, + "calibration_nmse": error_sq / target_sq if target_sq > 0.0 else float("nan"), + "calibration_cosine": dot / denominator if denominator > 0.0 else float("nan"), + "calibration_norm_ratio": ( + math.sqrt(predicted_sq / target_sq) if target_sq > 0.0 else float("nan") + ), + "transport_relative_identity_distance": relative_identity_distance, + } + return solution, stats + + +def _apply_transport_to_source_mlp(source_mlp, solution, row_chunk: int) -> None: + if row_chunk <= 0: + raise ValueError("row_chunk must be positive") + with torch.no_grad(): + for projection_name in ("gate_proj", "up_proj"): + projection = getattr(source_mlp, projection_name) + original = projection.weight.detach().clone() + if original.ndim != 2 or original.shape[1] != solution.shape[0]: + raise ValueError( + f"Unexpected {projection_name} shape {tuple(original.shape)} " + f"for transport {tuple(solution.shape)}" + ) + for start in range(0, original.shape[0], row_chunk): + end = min(start + row_chunk, original.shape[0]) + transported = original[start:end].float() @ solution.transpose(0, 1) + projection.weight[start:end].copy_(transported.to(projection.weight.dtype)) + del original + + +def fit_and_apply_source_feature_transport( + model, + layer_a, + layer_b, + dataloader, + device: str, + ridge_relative: float = 1e-3, + max_batches: int = 0, + row_chunk: int = 128, +) -> Dict[str, object]: + """Fit ``X_a T ~= X_b`` and rewrite source MLP input projections.""" + if dataloader is None: + raise ValueError("Source-feature transport requires a dataloader") + + previous_training = model.training + model.eval() + gram_sum = None + cross_sum = None + source_sq_sum = 0.0 + token_count = 0 + batches = 0 + try: + with torch.no_grad(): + for batch in dataloader: + if max_batches > 0 and batches >= max_batches: + break + input_ids = batch[0].to(device, non_blocking=True) + attention_mask = ( + batch[1].to(device, non_blocking=True) if len(batch) > 1 else None + ) + x_a, x_b = _capture_mlp_inputs( + model, layer_a, layer_b, input_ids, attention_mask + ) + x_a = x_a.reshape(-1, x_a.shape[-1]).float() + x_b = x_b.reshape(-1, x_b.shape[-1]).float() + if attention_mask is not None: + valid = attention_mask.reshape(-1).to(dtype=torch.bool) + if valid.numel() != x_a.shape[0]: + raise RuntimeError( + "Attention mask does not align with captured MLP inputs" + ) + x_a = x_a[valid] + x_b = x_b[valid] + if gram_sum is None: + width = x_a.shape[-1] + gram_sum = torch.zeros( + (width, width), dtype=torch.float32, device=x_a.device + ) + cross_sum = torch.zeros_like(gram_sum) + gram_sum.addmm_(x_a.transpose(0, 1), x_a) + cross_sum.addmm_(x_a.transpose(0, 1), x_b) + source_sq_sum += float(x_b.double().square().sum().item()) + token_count += int(x_a.shape[0]) + batches += 1 + finally: + model.train(previous_training) + + if gram_sum is None or cross_sum is None or batches == 0: + raise RuntimeError("No batches were processed for source-feature transport") + solution, stats = solve_transport_from_statistics( + gram_sum, + cross_sum, + source_sq_sum, + token_count, + ridge_relative, + ) + _apply_transport_to_source_mlp(layer_b.mlp, solution, row_chunk=row_chunk) + stats.update( + { + "enabled": True, + "batches": int(batches), + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T", + ], + "down_proj_transported": False, + } + ) + del solution, gram_sum, cross_sum + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return stats + + +def _drop_container_layer(container, index: int): + if isinstance(container, torch.nn.ModuleList): + return torch.nn.ModuleList( + [layer for layer_index, layer in enumerate(container) if layer_index != index] + ) + if isinstance(container, list): + return [layer for layer_index, layer in enumerate(container) if layer_index != index] + raise TypeError(f"Unsupported layer container: {type(container)}") + + +def compute_same_anchor_contextual_fisher( + model, + parent, + layer_attr: str, + container, + layer_idx: int, + layer_a, + layer_b, + dataloader, + device: str, + fisher_mode: str, +) -> Tuple[List[Dict[str, object]], int, List[Dict[str, int]], Dict[str, float]]: + """Measure both candidate MLPs in the same post-pruning anchor topology. + + The source MLP must already be feature-transported. Two virtual pruned + models are evaluated on every calibration batch: one uses the native + anchor MLP and one replaces it with the transported source MLP. The source + decoder block is absent in both arms, so the gradients describe the actual + deployment coordinate/topology rather than the source's old depth. + """ + if fisher_mode != "param": + raise ValueError("Same-anchor contextual Fisher currently requires fisher_mode=param") + + anchor_mlp = layer_a.mlp + source_mlp = layer_b.mlp + previous_container = container + pruned_container = _drop_container_layer(container, layer_idx + 1) + previous_training = model.training + + names_a = dict(layer_a.named_parameters()) + names_b = dict(layer_b.named_parameters()) + fisher_a: Dict[str, object] = {} + fisher_b: Dict[str, object] = {} + numels_a: Dict[str, int] = {} + numels_b: Dict[str, int] = {} + for name, parameter in names_a.items(): + if name.startswith("mlp."): + fisher_a[name] = torch.zeros_like(parameter, dtype=torch.float32, device="cpu") + else: + fisher_a[name] = 0.0 + numels_a[name] = int(parameter.numel()) + for name, parameter in names_b.items(): + if name.startswith("mlp."): + fisher_b[name] = torch.zeros_like(parameter, dtype=torch.float32, device="cpu") + else: + fisher_b[name] = 0.0 + numels_b[name] = int(parameter.numel()) + + for parameter in model.parameters(): + parameter.requires_grad_(False) + + anchor_loss_sum = 0.0 + source_loss_sum = 0.0 + num_batches = 0 + + def run_arm(candidate_mlp, fisher_target, input_ids, attention_mask): + layer_a.mlp = candidate_mlp + for parameter in candidate_mlp.parameters(): + parameter.requires_grad_(True) + model.zero_grad(set_to_none=True) + labels = input_ids + if attention_mask is not None: + labels = input_ids.masked_fill(attention_mask == 0, -100) + kwargs = {"input_ids": input_ids, "labels": labels, "use_cache": False} + if attention_mask is not None: + kwargs["attention_mask"] = attention_mask + output = model(**kwargs) + output.loss.backward() + for local_name, parameter in candidate_mlp.named_parameters(): + full_name = f"mlp.{local_name}" + if parameter.grad is not None: + fisher_target[full_name] += parameter.grad.detach().float().square().cpu() + loss = float(output.loss.detach().item()) + model.zero_grad(set_to_none=True) + for parameter in candidate_mlp.parameters(): + parameter.requires_grad_(False) + return loss + + try: + setattr(parent, layer_attr, pruned_container) + model.eval() + for batch in dataloader: + input_ids = batch[0].to(device, non_blocking=True) + attention_mask = ( + batch[1].to(device, non_blocking=True) if len(batch) > 1 else None + ) + anchor_loss_sum += run_arm( + anchor_mlp, fisher_a, input_ids, attention_mask + ) + source_loss_sum += run_arm( + source_mlp, fisher_b, input_ids, attention_mask + ) + num_batches += 1 + finally: + layer_a.mlp = anchor_mlp + setattr(parent, layer_attr, previous_container) + model.train(previous_training) + + if num_batches == 0: + raise RuntimeError("No batches were processed for same-anchor contextual Fisher") + stats = { + "num_batches": int(num_batches), + "anchor_candidate_mean_nll": anchor_loss_sum / float(num_batches), + "transported_source_candidate_mean_nll": source_loss_sum / float(num_batches), + "source_minus_anchor_mean_nll": ( + source_loss_sum - anchor_loss_sum + ) / float(num_batches), + "topology": "source block dropped; candidate MLP evaluated at anchor slot", + "shared_input_coordinate": True, + "scope": "mlp gate/up/down diagonal empirical Fisher", + } + return [fisher_a, fisher_b], num_batches, [numels_a, numels_b], stats diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/summarize_qwen3_nofit_merge.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/summarize_qwen3_nofit_merge.py new file mode 100644 index 0000000000000000000000000000000000000000..ecef42a2954e27709308155f731bbe03c906d9a0 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/summarize_qwen3_nofit_merge.py @@ -0,0 +1,41 @@ +from pathlib import Path +import json +import sys +root,project=map(Path,sys.argv[1:]); rows=[] +for arm in ('wa','wb','average','mka'): + out=root/'full'/arm + scores=json.loads((out/'eval/final_14bench.json').read_text())['task_scores'] + assert len(scores)==14 + ppl=json.loads((out/'eval/final_ppl.json').read_text())['metrics'] + assert ppl['tokens']==296815 and ppl['sequences']==145 + mse=json.loads((out/'eval/local_mse.json').read_text()) + backup=json.loads((root/f'{arm}_backup.json').read_text()) + assert backup['verified'] and backup['all_payload_hashes_match'] + rows.append(dict(model='Qwen/Qwen3-8B-Base',method=arm,removed_blocks=9, + interval=[13,22],anchor=12,transport=True,fit=False, + merge_scope='attention_and_mlp' if arm=='mka' else 'mlp', + ppl=ppl['perplexity'],local_mse=mse['local_mse'],local_nmse=mse['local_nmse'], + zs12=sum(v for k,v in scores.items() if k not in {'mmlu','cmmlu'})/12, + mmlu=scores['mmlu'],cmmlu=scores['cmmlu'],avg=sum(scores.values())/14, + avg_definition='unweighted Bench14 macro',task_scores=scores,backup=backup)) +baseline=json.loads((project/'results/qwen3_8b_base_nofit_bench14_20261009/summary.json').read_text()) +if isinstance(baseline,dict): baseline=baseline.get('rows',baseline.get('results',baseline)) +assert isinstance(baseline,list) +fisher=next(r for r in baseline if r['transport']=='on') +fmse=json.loads((root/'fisher_reference/local_mse.json').read_text()) +rows.append(dict(model='Qwen/Qwen3-8B-Base',method='fisher_existing',removed_blocks=9, + interval=[13,22],anchor=12,transport=True,fit=False,merge_scope='mlp', + ppl=fisher['ppl'],local_mse=fmse['local_mse'],local_nmse=fmse['local_nmse'], + zs12=fisher['zs12_avg'],mmlu=fisher['mmlu'],cmmlu=fisher['cmmlu'],avg=fisher['bench14_avg'], + task_scores=fisher['task_scores'],existing_scores_reused=True, + weight_restore=json.loads((root/'fisher_reference/restore_receipt.json').read_text()))) +report=dict(rows=rows,scope_caveat='MKA merges attention+MLP; four other controls are MLP-only. Not a same-scope coefficient-only comparison.', + local_mse_protocol=json.loads((root/'local_mse_reference/protocol.json').read_text())) +(root/'summary.json').write_text(json.dumps(report,indent=2)+'\n') +records=project/'results/qwen3_base_nofit_merge4_20261011';records.mkdir(parents=True,exist_ok=True) +(records/'summary.json').write_text(json.dumps(report,indent=2)+'\n') +import shutil +for arm in ('wa','wb','average','mka'): + d=records/arm;d.mkdir(exist_ok=True) + for path in (root/'full'/arm/'eval').glob('*.json'): shutil.copy2(path,d/path.name) + shutil.copy2(root/f'{arm}_backup.json',d/'backup.json') diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/verify_qwen3_nofit_merge.py b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/verify_qwen3_nofit_merge.py new file mode 100644 index 0000000000000000000000000000000000000000..321031a701a05eb9816738bf380d4e0a7b4a93f3 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/code/verify_qwen3_nofit_merge.py @@ -0,0 +1,23 @@ +"""Fail closed on mismatched topology, scope or accidental training.""" +from pathlib import Path +import json +import sys +r=Path(sys.argv[1]); arm=sys.argv[2]; cycles=int(sys.argv[3]) +for c in range(1,cycles+1): + m=json.loads((r/f'run/cycle_{c}/cycle_metadata.json').read_text()) + a=json.loads((r/f'audit/cycle_{c:02d}_merge.json').read_text()) + assert m['layer_merged']==12 and m['num_layers_after']==36-c + assert not m['fisher_computed'] and not m['fit'] and m['fisher_num_batches']==0 + assert m['source_feature_transport'] and not m['lora']['enabled'] + assert not (m.get('comm') or {}).get('enabled',False) + assert a['anchor']==12 and a['removed_original_layer']==12+c + assert a['merge']['initializer']==arm and not a['merge']['fit'] + assert a['merge']['optimizer_steps']==a['merge']['backward_calls']==0 + assert len(a['merge']['projection_names'])==(7 if arm=='mka' else 3) + assert a['merge']['source_post_transport_sha256']==a['transport']['source_after_sha256'] + assert a['transport']['stats']['batches']==32 and a['transport']['stats']['tokens']==65536 + assert (r/f'audit/cycle_{c:02d}_transport.pt').is_file() + meta=json.loads((r/f'compact/stage_{c:02d}/metadata.json').read_text()) + assert meta['anchor_index']==12 and meta['removed_original_layers']==list(range(13,13+c)) + assert len(list((r/f'compact/stage_{c:02d}').glob('*.pt')))==1 +print('NOFIT_MERGE_SCOPE_DATA_TOPOLOGY_VERIFIED',arm,cycles,flush=True) diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/manifest.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..9587d0102dc501b8ac956b6957c19768467fc483 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/manifest.json @@ -0,0 +1,335 @@ +{ + "format": "abprune_compact_manifest_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "stages": [ + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 1, + "removed_count": 1, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13 + ], + "source_cycle": "cycle_1", + "anchor_state_file": "anchor_layer.pt", + "sha256": "a1ac4eff65ee140fd4682b95835767941e8f5a8642facdcbee6c06d518bfac35" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 2, + "removed_count": 2, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14 + ], + "source_cycle": "cycle_2", + "anchor_state_file": "anchor_layer.pt", + "sha256": "f8201bf31937b7a4c5d843fdc48bc70c0aeab505a8e6170a18803e6a7ce60cda" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 3, + "removed_count": 3, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15 + ], + "source_cycle": "cycle_3", + "anchor_state_file": "anchor_layer.pt", + "sha256": "a114120d0594d9362d96686b8937d63e05cabcee907f134bcf76566ceb66c8fa" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 4, + "removed_count": 4, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16 + ], + "source_cycle": "cycle_4", + "anchor_state_file": "anchor_layer.pt", + "sha256": "6919d53d037f958c0bc281a2b8c292c51bc35bacc89b72cd8b6dde423f6967eb" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 5, + "removed_count": 5, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16, + 17 + ], + "source_cycle": "cycle_5", + "anchor_state_file": "anchor_layer.pt", + "sha256": "0cf2ae746ca7b97394b4076e8538c734de11781c3e47176d93ab264dd3cc73a7" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 6, + "removed_count": 6, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16, + 17, + 18 + ], + "source_cycle": "cycle_6", + "anchor_state_file": "anchor_layer.pt", + "sha256": "e41eef6ffeded49d5f41417946580cb54f372ba2d681922fb6b4c4b89258ecb9" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 7, + "removed_count": 7, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19 + ], + "source_cycle": "cycle_7", + "anchor_state_file": "anchor_layer.pt", + "sha256": "0e1a7fe2e430feafd208abdbc8aa428aa9ef114c9ad07e592c14c0bac25cb521" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 8, + "removed_count": 8, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ], + "source_cycle": "cycle_8", + "anchor_state_file": "anchor_layer.pt", + "sha256": "1ec9742a6d66e18d3084074943ac27a36cc11f5eac281dcc31115f646dc89834" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 9, + "removed_count": 9, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "source_cycle": "cycle_9", + "anchor_state_file": "anchor_layer.pt", + "sha256": "c76a5669993504adf2b28615fdfbb4451d8ca276ebdfb58b638aa1eb1c5cfe8c" + } + ] +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_01/anchor_layer.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_01/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..4197c2d95d61e316a1eee79bad5c8266dd5bf73a --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_01/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1ac4eff65ee140fd4682b95835767941e8f5a8642facdcbee6c06d518bfac35 +size 385897862 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_01/metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_01/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e1e56a4920b169bfac22efa98ac579c7a1f4f830 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_01/metadata.json @@ -0,0 +1,31 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 1, + "removed_count": 1, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13 + ], + "source_cycle": "cycle_1", + "anchor_state_file": "anchor_layer.pt", + "sha256": "a1ac4eff65ee140fd4682b95835767941e8f5a8642facdcbee6c06d518bfac35" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_02/anchor_layer.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_02/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..46bd7a7e42eabf8c15f6592fee679720421071d6 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_02/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8201bf31937b7a4c5d843fdc48bc70c0aeab505a8e6170a18803e6a7ce60cda +size 385897862 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_02/metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_02/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5f039d547136f62d61f8b70774a5bf717db7e488 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_02/metadata.json @@ -0,0 +1,32 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 2, + "removed_count": 2, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14 + ], + "source_cycle": "cycle_2", + "anchor_state_file": "anchor_layer.pt", + "sha256": "f8201bf31937b7a4c5d843fdc48bc70c0aeab505a8e6170a18803e6a7ce60cda" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_03/anchor_layer.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_03/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5858f6e7b682b99a7ec9c4a6c3a2a1015e31144 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_03/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a114120d0594d9362d96686b8937d63e05cabcee907f134bcf76566ceb66c8fa +size 385897862 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_03/metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_03/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d3d4552c90e209bd9529f96de55dd79be2b0bca9 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_03/metadata.json @@ -0,0 +1,33 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 3, + "removed_count": 3, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15 + ], + "source_cycle": "cycle_3", + "anchor_state_file": "anchor_layer.pt", + "sha256": "a114120d0594d9362d96686b8937d63e05cabcee907f134bcf76566ceb66c8fa" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_04/anchor_layer.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_04/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d98ce233402a01a0080031b501ad521d772533a --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_04/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6919d53d037f958c0bc281a2b8c292c51bc35bacc89b72cd8b6dde423f6967eb +size 385897862 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_04/metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_04/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0b2a732194a9e120756b5249cd3ea18966d0c0e4 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_04/metadata.json @@ -0,0 +1,34 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 4, + "removed_count": 4, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16 + ], + "source_cycle": "cycle_4", + "anchor_state_file": "anchor_layer.pt", + "sha256": "6919d53d037f958c0bc281a2b8c292c51bc35bacc89b72cd8b6dde423f6967eb" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_05/anchor_layer.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_05/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7db20c98eaf98d9e3bc8c77f98236ad94067843 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_05/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cf2ae746ca7b97394b4076e8538c734de11781c3e47176d93ab264dd3cc73a7 +size 385897862 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_05/metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_05/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9e514e2f95ed8dd059137112ef3391f52a32bf82 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_05/metadata.json @@ -0,0 +1,35 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 5, + "removed_count": 5, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16, + 17 + ], + "source_cycle": "cycle_5", + "anchor_state_file": "anchor_layer.pt", + "sha256": "0cf2ae746ca7b97394b4076e8538c734de11781c3e47176d93ab264dd3cc73a7" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_06/anchor_layer.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_06/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7fb8b6f1c693a9d87c45225144f2003e1d9b7c7 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_06/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e41eef6ffeded49d5f41417946580cb54f372ba2d681922fb6b4c4b89258ecb9 +size 385897862 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_06/metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_06/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..14b45fa66adb07746125d44e09a3ce50ce10c811 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_06/metadata.json @@ -0,0 +1,36 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 6, + "removed_count": 6, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16, + 17, + 18 + ], + "source_cycle": "cycle_6", + "anchor_state_file": "anchor_layer.pt", + "sha256": "e41eef6ffeded49d5f41417946580cb54f372ba2d681922fb6b4c4b89258ecb9" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_07/anchor_layer.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_07/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..e65de7213460f067bc7d2fbbf4d55fd6adae23b2 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_07/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e1a7fe2e430feafd208abdbc8aa428aa9ef114c9ad07e592c14c0bac25cb521 +size 385897862 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_07/metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_07/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1d453cf815388c97d0bc7857c012f96db97a2e86 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_07/metadata.json @@ -0,0 +1,37 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 7, + "removed_count": 7, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19 + ], + "source_cycle": "cycle_7", + "anchor_state_file": "anchor_layer.pt", + "sha256": "0e1a7fe2e430feafd208abdbc8aa428aa9ef114c9ad07e592c14c0bac25cb521" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_08/anchor_layer.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_08/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdf8b514682a43357b55b9b37abbc148bc7b91c6 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_08/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ec9742a6d66e18d3084074943ac27a36cc11f5eac281dcc31115f646dc89834 +size 385897862 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_08/metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_08/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f53065ebff4cf084359886aebff84cac2fdc6d1f --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_08/metadata.json @@ -0,0 +1,38 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 8, + "removed_count": 8, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ], + "source_cycle": "cycle_8", + "anchor_state_file": "anchor_layer.pt", + "sha256": "1ec9742a6d66e18d3084074943ac27a36cc11f5eac281dcc31115f646dc89834" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_09/anchor_layer.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_09/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..d90efff92e9a99a161b6af0de02c4007fb933dee --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_09/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c76a5669993504adf2b28615fdfbb4451d8ca276ebdfb58b638aa1eb1c5cfe8c +size 385897862 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_09/metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_09/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4692537370f109b903e48fdd0c86c4b0ce62f4b6 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/compact/stage_09/metadata.json @@ -0,0 +1,39 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-8B-Base", + "base_model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "run_id": "qwen3_base_nofit_merge_wb_20261011", + "target_prune_count": 9, + "stage": 9, + "removed_count": 9, + "anchor_index": 12, + "layer_order": [ + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12 + ], + "packaged_stage_range": [ + 1, + 9 + ], + "removed_original_layers": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ], + "source_cycle": "cycle_9", + "anchor_state_file": "anchor_layer.pt", + "sha256": "c76a5669993504adf2b28615fdfbb4451d8ca276ebdfb58b638aa1eb1c5cfe8c" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_1/cycle_metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_1/cycle_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..acee35ab983b52d8e323e046d154f7e8f403174e --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_1/cycle_metadata.json @@ -0,0 +1,276 @@ +{ + "cycle": 1, + "layer_merged": 12, + "num_layers_before": 36, + "num_layers_after": 35, + "fused_layer_state": "fused_layer.pt", + "profile_no_save": false, + "dwce_score": null, + "dwce_scores": null, + "dwce_meta": { + "method": "wb", + "fisher_computed": false + }, + "fisher_num_batches": 0, + "merge_method": "fisher", + "merged_params": 3, + "num_sequences": 64, + "teacher_source": "base_model", + "teacher_cycle": 0, + "eval": { + "datasets": [ + "wikitext2" + ], + "configs": [ + null + ], + "split": "test", + "num_samples": 0, + "seq_len": 1024, + "post_ppl": null, + "merge_only_ppl": null + }, + "comm": { + "enabled": false + }, + "distill": { + "enabled": false, + "method": "reparam", + "calib_samples": 256, + "inst_samples": 0, + "seq_len": 512, + "batch_size": 2, + "epochs": 1.0, + "lr": 0.0001, + "kl_weight": 0.01, + "kl_temp": 4.0, + "kl_chunk_size": 16384, + "hidden_mse_weight": 1.0, + "attn_mse_weight": 0.0, + "mlp_mse_weight": 0.0, + "reparam_eta": 0.01, + "reparam_gamma": 0.0001, + "reparam_attn_reg_scale": 1.0, + "reparam_mlp_reg_scale": 1.0, + "reparam_param_subset": "all", + "reparam_train_subset": "same", + "reparam_skip_train": false, + "reparam_stats": null, + "reparam_gate_summary": null, + "post_merge_nll_scale": null, + "merge_only_ppl": null, + "post_ppl": null, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "instruction_dataset": null, + "instruction_config": null, + "instruction_split": "train" + }, + "lora": { + "enabled": false, + "seq_len": 512, + "batch_size": 2, + "epochs": 0.0, + "rank": 8, + "alpha": 16.0, + "dropout": 0.0, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "respect_exclude_pairs": false, + "kl_enabled": false, + "kl_weight": 0.1, + "kl_temp": 4.0, + "post_ppl": null, + "lr": 0.0001, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "log_steps": 100, + "eval_every": 0, + "eval_max_batches": null + }, + "norm_policy": "copy_n1_n2", + "source_feature_transport": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.12798181447783583, + "calibration_cosine": 0.9338195359584089, + "calibration_norm_ratio": 0.9329591782339236, + "transport_relative_identity_distance": 0.4961368441581726, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "Transport then non-Fisher exact merge; no FiT" + }, + "same_anchor_contextual_fisher": { + "computed": false, + "anchor_candidate_mean_nll": 0.0, + "transported_source_candidate_mean_nll": 0.0 + }, + "full_model_saved": false, + "timing": { + "seconds": { + "selection": 8.189998334273696e-05, + "rebase": 0.0, + "source_feature_transport": 4.481229559984058, + "fisher": 5.188002251088619e-05, + "fit": 2.746852558979299, + "materialize_save_layer": 0.6389357629814185, + "metadata_write": 0.0004781170282512903, + "teacher_snapshot": 0.11497980495914817, + "cycle_cleanup": 0.23669262597104535, + "cycle_total": 8.220148556982167 + }, + "tracked_component_total_seconds": 8.219302209909074, + "cycle_total_seconds": 8.220148556982167, + "ratios_of_tracked_components": { + "selection": 9.96434748974183e-06, + "rebase": 0.0, + "source_feature_transport": 0.5452080292876385, + "fisher": 6.311974080760818e-06, + "fit": 0.3341953475889635, + "materialize_save_layer": 0.07773601051085902, + "metadata_write": 5.817002660820516e-05, + "teacher_snapshot": 0.013988998338633924, + "cycle_cleanup": 0.028797167925726355 + }, + "ratios_of_cycle_total": { + "selection": 9.963321559824047e-06, + "rebase": 0.0, + "source_feature_transport": 0.5451518946306289, + "fisher": 6.311324199465893e-06, + "fit": 0.33416093881249037, + "materialize_save_layer": 0.07772800680575397, + "metadata_write": 5.8164037418177715e-05, + "teacher_snapshot": 0.013987558030381908, + "cycle_cleanup": 0.028794202967293018 + } + }, + "fisher_computed": false, + "fit": false, + "nofit_merge_control": { + "cycle": 1, + "anchor": 12, + "removed_original_layer": 13, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "ea43767088fa2f8cc0b7ba197024635f7b8a7e19fdd34170b499f470c2461672", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.12798181447783583, + "calibration_cosine": 0.9338195359584089, + "calibration_norm_ratio": 0.9329591782339236, + "transport_relative_identity_distance": 0.4961368441581726, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "67540e2474547f1d622b738128696574720591fddabb8157d2f6812ecc3af308", + "self_attn.k_proj.weight": "efbe8b71b99ef90277849155ebc45e9aad5ffe746f2958c077c7c7130d783c7b", + "self_attn.v_proj.weight": "c53545e1ac9a6447efa83242434a0b28bcda5fe12ef3866e173dd13c3da45e38", + "self_attn.o_proj.weight": "8812afef76626e47fb5dffe3a77a0cb07a07e8a7f5cf3a97453368a1284b0ffa", + "self_attn.q_norm.weight": "79bbb659cb1c2af322ca85abf7d26b088c837bcd284e70efa95a6449f69b5673", + "self_attn.k_norm.weight": "e50a57833625792c16439de078eeb810df393e1a49ef7bbe43f5af6a93923adc", + "mlp.gate_proj.weight": "6bc0a4412e6639d40e3a7d5c9099a5a05f2573bd12744019550fc11da2c52c55", + "mlp.up_proj.weight": "8715dd4fa81b5d58a577a1b2fdfb24592b23775921014b36f832fe8dbf2bbf51", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b", + "input_layernorm.weight": "c419d161cc4932ecddf0ec98cbe5737b92688f0d5aa71a7e95d2006ca824ffca", + "post_attention_layernorm.weight": "ee6a9a96c2b4f35eb0807060229f10bc28080072ed9ecfa0a17e1aaa3cb4df40" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "67540e2474547f1d622b738128696574720591fddabb8157d2f6812ecc3af308", + "self_attn.k_proj.weight": "efbe8b71b99ef90277849155ebc45e9aad5ffe746f2958c077c7c7130d783c7b", + "self_attn.v_proj.weight": "c53545e1ac9a6447efa83242434a0b28bcda5fe12ef3866e173dd13c3da45e38", + "self_attn.o_proj.weight": "8812afef76626e47fb5dffe3a77a0cb07a07e8a7f5cf3a97453368a1284b0ffa", + "self_attn.q_norm.weight": "79bbb659cb1c2af322ca85abf7d26b088c837bcd284e70efa95a6449f69b5673", + "self_attn.k_norm.weight": "e50a57833625792c16439de078eeb810df393e1a49ef7bbe43f5af6a93923adc", + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b", + "input_layernorm.weight": "c419d161cc4932ecddf0ec98cbe5737b92688f0d5aa71a7e95d2006ca824ffca", + "post_attention_layernorm.weight": "ee6a9a96c2b4f35eb0807060229f10bc28080072ed9ecfa0a17e1aaa3cb4df40" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "3e138d6d890dfaeeeb88ae9b8a48c084aea69752230749e87f1586b975880ddd", + "mlp.up_proj.weight": "1f3dacc4c4a3af2a55d6273cfc2374077a2eda18cec7cf8237e05c54cc1b8ea5", + "mlp.down_proj.weight": "7a34a8a9bcc1a25c928397e3be5c409081b9de0ec283d13425482c0020e6db43" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "67540e2474547f1d622b738128696574720591fddabb8157d2f6812ecc3af308", + "self_attn.k_proj.weight": "efbe8b71b99ef90277849155ebc45e9aad5ffe746f2958c077c7c7130d783c7b", + "self_attn.v_proj.weight": "c53545e1ac9a6447efa83242434a0b28bcda5fe12ef3866e173dd13c3da45e38", + "self_attn.o_proj.weight": "8812afef76626e47fb5dffe3a77a0cb07a07e8a7f5cf3a97453368a1284b0ffa", + "self_attn.q_norm.weight": "79bbb659cb1c2af322ca85abf7d26b088c837bcd284e70efa95a6449f69b5673", + "self_attn.k_norm.weight": "e50a57833625792c16439de078eeb810df393e1a49ef7bbe43f5af6a93923adc", + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b", + "input_layernorm.weight": "c419d161cc4932ecddf0ec98cbe5737b92688f0d5aa71a7e95d2006ca824ffca", + "post_attention_layernorm.weight": "ee6a9a96c2b4f35eb0807060229f10bc28080072ed9ecfa0a17e1aaa3cb4df40" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_2/cycle_metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_2/cycle_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..adfeee868ee2f99fa381e1fcfc7193b8c4e0610a --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_2/cycle_metadata.json @@ -0,0 +1,276 @@ +{ + "cycle": 2, + "layer_merged": 12, + "num_layers_before": 35, + "num_layers_after": 34, + "fused_layer_state": "fused_layer.pt", + "profile_no_save": false, + "dwce_score": null, + "dwce_scores": null, + "dwce_meta": { + "method": "wb", + "fisher_computed": false + }, + "fisher_num_batches": 0, + "merge_method": "fisher", + "merged_params": 3, + "num_sequences": 64, + "teacher_source": "previous_cycle", + "teacher_cycle": 1, + "eval": { + "datasets": [ + "wikitext2" + ], + "configs": [ + null + ], + "split": "test", + "num_samples": 0, + "seq_len": 1024, + "post_ppl": null, + "merge_only_ppl": null + }, + "comm": { + "enabled": false + }, + "distill": { + "enabled": false, + "method": "reparam", + "calib_samples": 256, + "inst_samples": 0, + "seq_len": 512, + "batch_size": 2, + "epochs": 1.0, + "lr": 0.0001, + "kl_weight": 0.01, + "kl_temp": 4.0, + "kl_chunk_size": 16384, + "hidden_mse_weight": 1.0, + "attn_mse_weight": 0.0, + "mlp_mse_weight": 0.0, + "reparam_eta": 0.01, + "reparam_gamma": 0.0001, + "reparam_attn_reg_scale": 1.0, + "reparam_mlp_reg_scale": 1.0, + "reparam_param_subset": "all", + "reparam_train_subset": "same", + "reparam_skip_train": false, + "reparam_stats": null, + "reparam_gate_summary": null, + "post_merge_nll_scale": null, + "merge_only_ppl": null, + "post_ppl": null, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "instruction_dataset": null, + "instruction_config": null, + "instruction_split": "train" + }, + "lora": { + "enabled": false, + "seq_len": 512, + "batch_size": 2, + "epochs": 0.0, + "rank": 8, + "alpha": 16.0, + "dropout": 0.0, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "respect_exclude_pairs": false, + "kl_enabled": false, + "kl_weight": 0.1, + "kl_temp": 4.0, + "post_ppl": null, + "lr": 0.0001, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "log_steps": 100, + "eval_every": 0, + "eval_max_batches": null + }, + "norm_policy": "copy_n1_n2", + "source_feature_transport": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.10099672573787757, + "calibration_cosine": 0.9481582440752598, + "calibration_norm_ratio": 0.9472741933087646, + "transport_relative_identity_distance": 0.45524656772613525, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "Transport then non-Fisher exact merge; no FiT" + }, + "same_anchor_contextual_fisher": { + "computed": false, + "anchor_candidate_mean_nll": 0.0, + "transported_source_candidate_mean_nll": 0.0 + }, + "full_model_saved": false, + "timing": { + "seconds": { + "selection": 4.383001942187548e-05, + "rebase": 0.0, + "source_feature_transport": 4.08041910699103, + "fisher": 3.761000698432326e-05, + "fit": 2.7584136540535837, + "materialize_save_layer": 0.6302790559711866, + "metadata_write": 0.0005568969645537436, + "teacher_snapshot": 0.5216269000084139, + "cycle_cleanup": 0.16023769503226504, + "cycle_total": 8.152335364022292 + }, + "tracked_component_total_seconds": 8.15161474904744, + "cycle_total_seconds": 8.152335364022292, + "ratios_of_tracked_components": { + "selection": 5.376851184852333e-06, + "rebase": 0.0, + "source_feature_transport": 0.5005657446541926, + "fisher": 4.6138106549647965e-06, + "fit": 0.33838861857105296, + "materialize_save_layer": 0.07731953427323564, + "metadata_write": 6.83173802612323e-05, + "teacher_snapshot": 0.06399062223461541, + "cycle_cleanup": 0.019657172224802413 + }, + "ratios_of_cycle_total": { + "selection": 5.376375905155369e-06, + "rebase": 0.0, + "source_feature_transport": 0.5005214978027825, + "fisher": 4.6134028232330725e-06, + "fit": 0.3383587071536525, + "materialize_save_layer": 0.07731269971457753, + "metadata_write": 6.831134143615203e-05, + "teacher_snapshot": 0.06398496586762688, + "cycle_cleanup": 0.019655434654887054 + } + }, + "fisher_computed": false, + "fit": false, + "nofit_merge_control": { + "cycle": 2, + "anchor": 12, + "removed_original_layer": 14, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "2f2164ca8bffcf85ff9cfb9506308fa1340559aba7e44bda37d1e050e6e82d36", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.10099672573787757, + "calibration_cosine": 0.9481582440752598, + "calibration_norm_ratio": 0.9472741933087646, + "transport_relative_identity_distance": 0.45524656772613525, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "f57ad388d57bb515af2ac9324d86dc9e19c4cc44bc73879a7b409f9c07597c15", + "self_attn.k_proj.weight": "09cba09d72eea6b683087478aeaaf6d321423335a9eb57d47006adbc1f336c63", + "self_attn.v_proj.weight": "af4adee7b325a938beb530808104ab27a70afc0d494134b3de1f5ea8ef060773", + "self_attn.o_proj.weight": "6888fff18d668ea322341951eb6ab19a3fbbc4cf94b758ed0f8dba411a262d1f", + "self_attn.q_norm.weight": "e3e9ec56f12aa5ec410018bbac99e6953ad52f3b8b796d73c5bd0cc6545fb93b", + "self_attn.k_norm.weight": "a5bf1576b21d93da019094f1a61912669136a5d9e89e8d2301c88f6bf2def290", + "mlp.gate_proj.weight": "1d90e8cf937190e4027f0c7bc34bbabd5bb43c5f3d081537e0d866da8a3efe3f", + "mlp.up_proj.weight": "dc867f2d0718d761bb166f69bd21878837006daf9ae392c955a2e9578a15ad97", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d", + "input_layernorm.weight": "c21d58d80fa7aca24ff0dac67092f8204b3876926fa095bb7e6ceb3a16060706", + "post_attention_layernorm.weight": "1e599b9bd0ed7fdbcef34cff94651bf467bf923745a76a96d33a74ae551ad5d5" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "f57ad388d57bb515af2ac9324d86dc9e19c4cc44bc73879a7b409f9c07597c15", + "self_attn.k_proj.weight": "09cba09d72eea6b683087478aeaaf6d321423335a9eb57d47006adbc1f336c63", + "self_attn.v_proj.weight": "af4adee7b325a938beb530808104ab27a70afc0d494134b3de1f5ea8ef060773", + "self_attn.o_proj.weight": "6888fff18d668ea322341951eb6ab19a3fbbc4cf94b758ed0f8dba411a262d1f", + "self_attn.q_norm.weight": "e3e9ec56f12aa5ec410018bbac99e6953ad52f3b8b796d73c5bd0cc6545fb93b", + "self_attn.k_norm.weight": "a5bf1576b21d93da019094f1a61912669136a5d9e89e8d2301c88f6bf2def290", + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d", + "input_layernorm.weight": "c21d58d80fa7aca24ff0dac67092f8204b3876926fa095bb7e6ceb3a16060706", + "post_attention_layernorm.weight": "1e599b9bd0ed7fdbcef34cff94651bf467bf923745a76a96d33a74ae551ad5d5" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "655a66345118a70ec952e549407ec7f19222782a04bf0db39c5ee8372372a2bf", + "mlp.up_proj.weight": "cb86cdbce20a724154f50658e0c2cfc3ec3d63e3802388217035698470825d82", + "mlp.down_proj.weight": "032dc5aba56bd954e7498918ca13b8589c12587c97ad88560480e8d28c18883b" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "f57ad388d57bb515af2ac9324d86dc9e19c4cc44bc73879a7b409f9c07597c15", + "self_attn.k_proj.weight": "09cba09d72eea6b683087478aeaaf6d321423335a9eb57d47006adbc1f336c63", + "self_attn.v_proj.weight": "af4adee7b325a938beb530808104ab27a70afc0d494134b3de1f5ea8ef060773", + "self_attn.o_proj.weight": "6888fff18d668ea322341951eb6ab19a3fbbc4cf94b758ed0f8dba411a262d1f", + "self_attn.q_norm.weight": "e3e9ec56f12aa5ec410018bbac99e6953ad52f3b8b796d73c5bd0cc6545fb93b", + "self_attn.k_norm.weight": "a5bf1576b21d93da019094f1a61912669136a5d9e89e8d2301c88f6bf2def290", + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d", + "input_layernorm.weight": "c21d58d80fa7aca24ff0dac67092f8204b3876926fa095bb7e6ceb3a16060706", + "post_attention_layernorm.weight": "1e599b9bd0ed7fdbcef34cff94651bf467bf923745a76a96d33a74ae551ad5d5" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_3/cycle_metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_3/cycle_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6cdbc6233f493847f9509f8f13855d2a906252f7 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_3/cycle_metadata.json @@ -0,0 +1,276 @@ +{ + "cycle": 3, + "layer_merged": 12, + "num_layers_before": 34, + "num_layers_after": 33, + "fused_layer_state": "fused_layer.pt", + "profile_no_save": false, + "dwce_score": null, + "dwce_scores": null, + "dwce_meta": { + "method": "wb", + "fisher_computed": false + }, + "fisher_num_batches": 0, + "merge_method": "fisher", + "merged_params": 3, + "num_sequences": 64, + "teacher_source": "previous_cycle", + "teacher_cycle": 2, + "eval": { + "datasets": [ + "wikitext2" + ], + "configs": [ + null + ], + "split": "test", + "num_samples": 0, + "seq_len": 1024, + "post_ppl": null, + "merge_only_ppl": null + }, + "comm": { + "enabled": false + }, + "distill": { + "enabled": false, + "method": "reparam", + "calib_samples": 256, + "inst_samples": 0, + "seq_len": 512, + "batch_size": 2, + "epochs": 1.0, + "lr": 0.0001, + "kl_weight": 0.01, + "kl_temp": 4.0, + "kl_chunk_size": 16384, + "hidden_mse_weight": 1.0, + "attn_mse_weight": 0.0, + "mlp_mse_weight": 0.0, + "reparam_eta": 0.01, + "reparam_gamma": 0.0001, + "reparam_attn_reg_scale": 1.0, + "reparam_mlp_reg_scale": 1.0, + "reparam_param_subset": "all", + "reparam_train_subset": "same", + "reparam_skip_train": false, + "reparam_stats": null, + "reparam_gate_summary": null, + "post_merge_nll_scale": null, + "merge_only_ppl": null, + "post_ppl": null, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "instruction_dataset": null, + "instruction_config": null, + "instruction_split": "train" + }, + "lora": { + "enabled": false, + "seq_len": 512, + "batch_size": 2, + "epochs": 0.0, + "rank": 8, + "alpha": 16.0, + "dropout": 0.0, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "respect_exclude_pairs": false, + "kl_enabled": false, + "kl_weight": 0.1, + "kl_temp": 4.0, + "post_ppl": null, + "lr": 0.0001, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "log_steps": 100, + "eval_every": 0, + "eval_max_batches": null + }, + "norm_policy": "copy_n1_n2", + "source_feature_transport": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.11038890826345682, + "calibration_cosine": 0.9431923979113933, + "calibration_norm_ratio": 0.9422936536202322, + "transport_relative_identity_distance": 0.480193167924881, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "Transport then non-Fisher exact merge; no FiT" + }, + "same_anchor_contextual_fisher": { + "computed": false, + "anchor_candidate_mean_nll": 0.0, + "transported_source_candidate_mean_nll": 0.0 + }, + "full_model_saved": false, + "timing": { + "seconds": { + "selection": 4.0579994674772024e-05, + "rebase": 0.0, + "source_feature_transport": 4.0293032840127125, + "fisher": 3.4240016248077154e-05, + "fit": 2.726361538982019, + "materialize_save_layer": 0.6441661670105532, + "metadata_write": 0.0004462780198082328, + "teacher_snapshot": 0.5075800369959325, + "cycle_cleanup": 0.16246394597692415, + "cycle_total": 8.071012888045516 + }, + "tracked_component_total_seconds": 8.070396071008872, + "cycle_total_seconds": 8.071012888045516, + "ratios_of_tracked_components": { + "selection": 5.028253175893901e-06, + "rebase": 0.0, + "source_feature_transport": 0.4992695833711434, + "fisher": 4.242668630735102e-06, + "fit": 0.337822520108508, + "materialize_save_layer": 0.07981840808589045, + "metadata_write": 5.529815586268296e-05, + "teacher_snapshot": 0.06289406771735806, + "cycle_cleanup": 0.020130851639430715 + }, + "ratios_of_cycle_total": { + "selection": 5.02786889795178e-06, + "rebase": 0.0, + "source_feature_transport": 0.4992314273194591, + "fisher": 4.2423443901064995e-06, + "fit": 0.3377967024461334, + "materialize_save_layer": 0.07981230806416728, + "metadata_write": 5.5293929770480625e-05, + "teacher_snapshot": 0.06288926111711965, + "cycle_cleanup": 0.020129313164343934 + } + }, + "fisher_computed": false, + "fit": false, + "nofit_merge_control": { + "cycle": 3, + "anchor": 12, + "removed_original_layer": 15, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "c74bae7cb32fe1396e861248ba6bf41c2fd708fc354660e485db8f232fea727e", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.11038890826345682, + "calibration_cosine": 0.9431923979113933, + "calibration_norm_ratio": 0.9422936536202322, + "transport_relative_identity_distance": 0.480193167924881, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "6005f741b04aefb4b83452f774f7d013b010b6218020e382a98c92a45d8878ac", + "self_attn.k_proj.weight": "e4d7d1464c351cfcb69799eb3e3bf63cc597383cbfb0d07263236ba2d8eb3d5b", + "self_attn.v_proj.weight": "25063dfdce05fdcb1a48c5d71c3e0107d7a894b7fb6c0f7defa2e768c68ba3d7", + "self_attn.o_proj.weight": "e96bf888e3b0629075b2ee05288e147fd875119467d978c2269ecfeb3b0db4eb", + "self_attn.q_norm.weight": "d77168c19fe27a0d64a2611360be089b4435adc62444d0e82a986419be1d9b44", + "self_attn.k_norm.weight": "2cbb9d53874b3fe1d350a9c9031d64cd138a1dbac549529d07da2927106a10ff", + "mlp.gate_proj.weight": "3acdf3c7c3b8dd224f2837ef92c5e72e8bc7963baa021e9ba2a032da6084c86a", + "mlp.up_proj.weight": "6a0bdd43c6eab5e133991697799089710e9077591b56bcbde02440054dbfebbc", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea", + "input_layernorm.weight": "3f2e5d516a4c80f8d93f0221e5ecbc2e7e692f111118c7c629f210ee9d03419d", + "post_attention_layernorm.weight": "3a237234fa5182946c09967a08faec9d9060f4bd780bdd6627f6159edebe2f98" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "6005f741b04aefb4b83452f774f7d013b010b6218020e382a98c92a45d8878ac", + "self_attn.k_proj.weight": "e4d7d1464c351cfcb69799eb3e3bf63cc597383cbfb0d07263236ba2d8eb3d5b", + "self_attn.v_proj.weight": "25063dfdce05fdcb1a48c5d71c3e0107d7a894b7fb6c0f7defa2e768c68ba3d7", + "self_attn.o_proj.weight": "e96bf888e3b0629075b2ee05288e147fd875119467d978c2269ecfeb3b0db4eb", + "self_attn.q_norm.weight": "d77168c19fe27a0d64a2611360be089b4435adc62444d0e82a986419be1d9b44", + "self_attn.k_norm.weight": "2cbb9d53874b3fe1d350a9c9031d64cd138a1dbac549529d07da2927106a10ff", + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea", + "input_layernorm.weight": "3f2e5d516a4c80f8d93f0221e5ecbc2e7e692f111118c7c629f210ee9d03419d", + "post_attention_layernorm.weight": "3a237234fa5182946c09967a08faec9d9060f4bd780bdd6627f6159edebe2f98" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "f7c92c612a7c8544bac0f0123aff1cb5f922f5b1cf88006c20b8768643f37029", + "mlp.up_proj.weight": "4a3522038cb62630906e194bfc0a6e0d5b7f7e8a614b029b83f93145bc4c4a04", + "mlp.down_proj.weight": "0ac1fc28ab86d85f9cb5f40fbde4485ceac5d2d531a13f5f05d81910ebd8b48d" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "6005f741b04aefb4b83452f774f7d013b010b6218020e382a98c92a45d8878ac", + "self_attn.k_proj.weight": "e4d7d1464c351cfcb69799eb3e3bf63cc597383cbfb0d07263236ba2d8eb3d5b", + "self_attn.v_proj.weight": "25063dfdce05fdcb1a48c5d71c3e0107d7a894b7fb6c0f7defa2e768c68ba3d7", + "self_attn.o_proj.weight": "e96bf888e3b0629075b2ee05288e147fd875119467d978c2269ecfeb3b0db4eb", + "self_attn.q_norm.weight": "d77168c19fe27a0d64a2611360be089b4435adc62444d0e82a986419be1d9b44", + "self_attn.k_norm.weight": "2cbb9d53874b3fe1d350a9c9031d64cd138a1dbac549529d07da2927106a10ff", + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea", + "input_layernorm.weight": "3f2e5d516a4c80f8d93f0221e5ecbc2e7e692f111118c7c629f210ee9d03419d", + "post_attention_layernorm.weight": "3a237234fa5182946c09967a08faec9d9060f4bd780bdd6627f6159edebe2f98" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_4/cycle_metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_4/cycle_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e0013d1d5c2065caf4426daf3baf3dca94f320b0 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_4/cycle_metadata.json @@ -0,0 +1,276 @@ +{ + "cycle": 4, + "layer_merged": 12, + "num_layers_before": 33, + "num_layers_after": 32, + "fused_layer_state": "fused_layer.pt", + "profile_no_save": false, + "dwce_score": null, + "dwce_scores": null, + "dwce_meta": { + "method": "wb", + "fisher_computed": false + }, + "fisher_num_batches": 0, + "merge_method": "fisher", + "merged_params": 3, + "num_sequences": 64, + "teacher_source": "previous_cycle", + "teacher_cycle": 3, + "eval": { + "datasets": [ + "wikitext2" + ], + "configs": [ + null + ], + "split": "test", + "num_samples": 0, + "seq_len": 1024, + "post_ppl": null, + "merge_only_ppl": null + }, + "comm": { + "enabled": false + }, + "distill": { + "enabled": false, + "method": "reparam", + "calib_samples": 256, + "inst_samples": 0, + "seq_len": 512, + "batch_size": 2, + "epochs": 1.0, + "lr": 0.0001, + "kl_weight": 0.01, + "kl_temp": 4.0, + "kl_chunk_size": 16384, + "hidden_mse_weight": 1.0, + "attn_mse_weight": 0.0, + "mlp_mse_weight": 0.0, + "reparam_eta": 0.01, + "reparam_gamma": 0.0001, + "reparam_attn_reg_scale": 1.0, + "reparam_mlp_reg_scale": 1.0, + "reparam_param_subset": "all", + "reparam_train_subset": "same", + "reparam_skip_train": false, + "reparam_stats": null, + "reparam_gate_summary": null, + "post_merge_nll_scale": null, + "merge_only_ppl": null, + "post_ppl": null, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "instruction_dataset": null, + "instruction_config": null, + "instruction_split": "train" + }, + "lora": { + "enabled": false, + "seq_len": 512, + "batch_size": 2, + "epochs": 0.0, + "rank": 8, + "alpha": 16.0, + "dropout": 0.0, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "respect_exclude_pairs": false, + "kl_enabled": false, + "kl_weight": 0.1, + "kl_temp": 4.0, + "post_ppl": null, + "lr": 0.0001, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "log_steps": 100, + "eval_every": 0, + "eval_max_batches": null + }, + "norm_policy": "copy_n1_n2", + "source_feature_transport": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09866496995725485, + "calibration_cosine": 0.9493870783680597, + "calibration_norm_ratio": 0.9484957145281611, + "transport_relative_identity_distance": 0.46458062529563904, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "Transport then non-Fisher exact merge; no FiT" + }, + "same_anchor_contextual_fisher": { + "computed": false, + "anchor_candidate_mean_nll": 0.0, + "transported_source_candidate_mean_nll": 0.0 + }, + "full_model_saved": false, + "timing": { + "seconds": { + "selection": 4.357995931059122e-05, + "rebase": 0.0, + "source_feature_transport": 4.015303591964766, + "fisher": 4.6550005208700895e-05, + "fit": 2.7029757410055026, + "materialize_save_layer": 0.650786395999603, + "metadata_write": 0.00044925801921635866, + "teacher_snapshot": 0.48057447897735983, + "cycle_cleanup": 0.17874478601152077, + "cycle_total": 8.02957844804041 + }, + "tracked_component_total_seconds": 8.028924381942488, + "cycle_total_seconds": 8.02957844804041, + "ratios_of_tracked_components": { + "selection": 5.4278701900101405e-06, + "rebase": 0.0, + "source_feature_transport": 0.5001047962284232, + "fisher": 5.7977884700713495e-06, + "fit": 0.33665477620945716, + "materialize_save_layer": 0.0810552404084486, + "metadata_write": 5.5954944628294883e-05, + "teacher_snapshot": 0.059855399816468494, + "cycle_cleanup": 0.022262606733914204 + }, + "ratios_of_cycle_total": { + "selection": 5.427428051497118e-06, + "rebase": 0.0, + "source_feature_transport": 0.500064059146802, + "fisher": 5.797316199091529e-06, + "fit": 0.3366273532909008, + "materialize_save_layer": 0.08104863788439914, + "metadata_write": 5.5950386701309135e-05, + "teacher_snapshot": 0.05985052416975169, + "cycle_cleanup": 0.02226079328674381 + } + }, + "fisher_computed": false, + "fit": false, + "nofit_merge_control": { + "cycle": 4, + "anchor": 12, + "removed_original_layer": 16, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "49bbea5f7162f9479174a850b673f69481d7c2e8e01b01c5a110ec706d662fce", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09866496995725485, + "calibration_cosine": 0.9493870783680597, + "calibration_norm_ratio": 0.9484957145281611, + "transport_relative_identity_distance": 0.46458062529563904, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "a9874967bd53acfd480ed2bd5552fab4e1a06f610af242d411e176c279e0191a", + "self_attn.k_proj.weight": "0a285bc594b54a921f698937c566bce629295ff4befcebb203038dd0d97eb3f0", + "self_attn.v_proj.weight": "dd8847c5b10e200df29bf79b30321ec7f1d90427ab9b3d273e18a4cbd83e597d", + "self_attn.o_proj.weight": "e6afd2a9ba7c9e6f1911d773062b9d37442a01ead339c9aa9255ff20f146b196", + "self_attn.q_norm.weight": "48de60cd86dec981224231f275146bc8d17990394835783504bbf5f0e9c5bb1a", + "self_attn.k_norm.weight": "d6ef88be0fc007a2bc5e9f1a0a70b7eac493495b5b54b4f72582378340b9bb4d", + "mlp.gate_proj.weight": "171beb5701a83f117d0a59a14a78be2936a062c7aa5700ec39efd9c816af96db", + "mlp.up_proj.weight": "e04d543903d934aae9062d2034f71ac4d3fb1ac69bd5f31d365f404c764e535e", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1", + "input_layernorm.weight": "45d58615dedc0906944d998c52fed1d0d139338362abd51fc1b98ae12edec92a", + "post_attention_layernorm.weight": "af332049d823017194704880c92765ab0cf4a519c8819753d0539106ea951cef" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "a9874967bd53acfd480ed2bd5552fab4e1a06f610af242d411e176c279e0191a", + "self_attn.k_proj.weight": "0a285bc594b54a921f698937c566bce629295ff4befcebb203038dd0d97eb3f0", + "self_attn.v_proj.weight": "dd8847c5b10e200df29bf79b30321ec7f1d90427ab9b3d273e18a4cbd83e597d", + "self_attn.o_proj.weight": "e6afd2a9ba7c9e6f1911d773062b9d37442a01ead339c9aa9255ff20f146b196", + "self_attn.q_norm.weight": "48de60cd86dec981224231f275146bc8d17990394835783504bbf5f0e9c5bb1a", + "self_attn.k_norm.weight": "d6ef88be0fc007a2bc5e9f1a0a70b7eac493495b5b54b4f72582378340b9bb4d", + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1", + "input_layernorm.weight": "45d58615dedc0906944d998c52fed1d0d139338362abd51fc1b98ae12edec92a", + "post_attention_layernorm.weight": "af332049d823017194704880c92765ab0cf4a519c8819753d0539106ea951cef" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "8292c665701ba7b813c028746607fae674b14b68899c1b53243212237e3d7d2c", + "mlp.up_proj.weight": "17669eaa9229e48a498c1cc80e87be7605a037083184521ca6ab99b277bfa6d2", + "mlp.down_proj.weight": "1f66d7843032883982c0b98e73d9193144e181de07f5506b718d1cea3edb23ea" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "a9874967bd53acfd480ed2bd5552fab4e1a06f610af242d411e176c279e0191a", + "self_attn.k_proj.weight": "0a285bc594b54a921f698937c566bce629295ff4befcebb203038dd0d97eb3f0", + "self_attn.v_proj.weight": "dd8847c5b10e200df29bf79b30321ec7f1d90427ab9b3d273e18a4cbd83e597d", + "self_attn.o_proj.weight": "e6afd2a9ba7c9e6f1911d773062b9d37442a01ead339c9aa9255ff20f146b196", + "self_attn.q_norm.weight": "48de60cd86dec981224231f275146bc8d17990394835783504bbf5f0e9c5bb1a", + "self_attn.k_norm.weight": "d6ef88be0fc007a2bc5e9f1a0a70b7eac493495b5b54b4f72582378340b9bb4d", + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1", + "input_layernorm.weight": "45d58615dedc0906944d998c52fed1d0d139338362abd51fc1b98ae12edec92a", + "post_attention_layernorm.weight": "af332049d823017194704880c92765ab0cf4a519c8819753d0539106ea951cef" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_5/cycle_metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_5/cycle_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fb1affea5567e0fa8a7fa25e09d6977e095e0cd5 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_5/cycle_metadata.json @@ -0,0 +1,276 @@ +{ + "cycle": 5, + "layer_merged": 12, + "num_layers_before": 32, + "num_layers_after": 31, + "fused_layer_state": "fused_layer.pt", + "profile_no_save": false, + "dwce_score": null, + "dwce_scores": null, + "dwce_meta": { + "method": "wb", + "fisher_computed": false + }, + "fisher_num_batches": 0, + "merge_method": "fisher", + "merged_params": 3, + "num_sequences": 64, + "teacher_source": "previous_cycle", + "teacher_cycle": 4, + "eval": { + "datasets": [ + "wikitext2" + ], + "configs": [ + null + ], + "split": "test", + "num_samples": 0, + "seq_len": 1024, + "post_ppl": null, + "merge_only_ppl": null + }, + "comm": { + "enabled": false + }, + "distill": { + "enabled": false, + "method": "reparam", + "calib_samples": 256, + "inst_samples": 0, + "seq_len": 512, + "batch_size": 2, + "epochs": 1.0, + "lr": 0.0001, + "kl_weight": 0.01, + "kl_temp": 4.0, + "kl_chunk_size": 16384, + "hidden_mse_weight": 1.0, + "attn_mse_weight": 0.0, + "mlp_mse_weight": 0.0, + "reparam_eta": 0.01, + "reparam_gamma": 0.0001, + "reparam_attn_reg_scale": 1.0, + "reparam_mlp_reg_scale": 1.0, + "reparam_param_subset": "all", + "reparam_train_subset": "same", + "reparam_skip_train": false, + "reparam_stats": null, + "reparam_gate_summary": null, + "post_merge_nll_scale": null, + "merge_only_ppl": null, + "post_ppl": null, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "instruction_dataset": null, + "instruction_config": null, + "instruction_split": "train" + }, + "lora": { + "enabled": false, + "seq_len": 512, + "batch_size": 2, + "epochs": 0.0, + "rank": 8, + "alpha": 16.0, + "dropout": 0.0, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "respect_exclude_pairs": false, + "kl_enabled": false, + "kl_weight": 0.1, + "kl_temp": 4.0, + "post_ppl": null, + "lr": 0.0001, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "log_steps": 100, + "eval_every": 0, + "eval_max_batches": null + }, + "norm_policy": "copy_n1_n2", + "source_feature_transport": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09917833468174767, + "calibration_cosine": 0.9491166565617855, + "calibration_norm_ratio": 0.9482434757273703, + "transport_relative_identity_distance": 0.4882379472255707, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "Transport then non-Fisher exact merge; no FiT" + }, + "same_anchor_contextual_fisher": { + "computed": false, + "anchor_candidate_mean_nll": 0.0, + "transported_source_candidate_mean_nll": 0.0 + }, + "full_model_saved": false, + "timing": { + "seconds": { + "selection": 4.278897540643811e-05, + "rebase": 0.0, + "source_feature_transport": 4.069975997030269, + "fisher": 5.07200020365417e-05, + "fit": 2.698854011017829, + "materialize_save_layer": 0.6448209949885495, + "metadata_write": 0.00046714802738279104, + "teacher_snapshot": 0.5063103040447459, + "cycle_cleanup": 0.16808561800280586, + "cycle_total": 8.089288879011292 + }, + "tracked_component_total_seconds": 8.088607582089026, + "cycle_total_seconds": 8.089288879011292, + "ratios_of_tracked_components": { + "selection": 5.2900298317336725e-06, + "rebase": 0.0, + "source_feature_transport": 0.5031738721065667, + "fisher": 6.270547992568378e-06, + "fit": 0.333661138042353, + "materialize_save_layer": 0.07971965365415998, + "metadata_write": 5.7753825073331326e-05, + "teacher_snapshot": 0.06259548369807084, + "cycle_cleanup": 0.020780538095951834 + }, + "ratios_of_cycle_total": { + "selection": 5.28958429429559e-06, + "rebase": 0.0, + "source_feature_transport": 0.5031314937448147, + "fisher": 6.2700198738286274e-06, + "fit": 0.3336330363995722, + "materialize_save_layer": 0.07971293949726793, + "metadata_write": 5.774896092471949e-05, + "teacher_snapshot": 0.06259021177478698, + "cycle_cleanup": 0.020778787915329094 + } + }, + "fisher_computed": false, + "fit": false, + "nofit_merge_control": { + "cycle": 5, + "anchor": 12, + "removed_original_layer": 17, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "b968cc0681a32eab25ab339122b95bf8f1b90e3ced55b7638652d052fbcdbcc3", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09917833468174767, + "calibration_cosine": 0.9491166565617855, + "calibration_norm_ratio": 0.9482434757273703, + "transport_relative_identity_distance": 0.4882379472255707, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "93af1649dbcf27dcc906cff079a1448af3e0b78dc0ae7fcd0038e95d05f79fdc", + "self_attn.k_proj.weight": "919f1a2454b2957fa1f611232b1c0aaccdab718aed512b435923cf972cd52015", + "self_attn.v_proj.weight": "2a660e851663561147921fae640f303f3f599da61afc8d68545fed320cc9b63d", + "self_attn.o_proj.weight": "bfb11d528abdacfc054070ed7a55f90ddde74fb17704bc742ceffa065adcc717", + "self_attn.q_norm.weight": "a4538fa22891825dbfbac330127ff29569ac04b5ab0ee89e60db7e7bc64f26a3", + "self_attn.k_norm.weight": "f26654e4f51003513e7ff0cd2f23e99387481dc5c639b01444903ec9632ae685", + "mlp.gate_proj.weight": "1ec134d886ea6246b31de9cb6cbb4c451fddfb333fca1e90375f8659eb27d81c", + "mlp.up_proj.weight": "b7549bcdd3b5089f09aebb1847587e2049463294f52f4335b88b7e64888edd45", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f", + "input_layernorm.weight": "70025e492885453112e0108ebf4311dfd994db35383e90da14292277834fb949", + "post_attention_layernorm.weight": "41357e377104509608d219bc3f4d8a51dde14749a9a6e6b592f6d4f4594faa85" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "93af1649dbcf27dcc906cff079a1448af3e0b78dc0ae7fcd0038e95d05f79fdc", + "self_attn.k_proj.weight": "919f1a2454b2957fa1f611232b1c0aaccdab718aed512b435923cf972cd52015", + "self_attn.v_proj.weight": "2a660e851663561147921fae640f303f3f599da61afc8d68545fed320cc9b63d", + "self_attn.o_proj.weight": "bfb11d528abdacfc054070ed7a55f90ddde74fb17704bc742ceffa065adcc717", + "self_attn.q_norm.weight": "a4538fa22891825dbfbac330127ff29569ac04b5ab0ee89e60db7e7bc64f26a3", + "self_attn.k_norm.weight": "f26654e4f51003513e7ff0cd2f23e99387481dc5c639b01444903ec9632ae685", + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f", + "input_layernorm.weight": "70025e492885453112e0108ebf4311dfd994db35383e90da14292277834fb949", + "post_attention_layernorm.weight": "41357e377104509608d219bc3f4d8a51dde14749a9a6e6b592f6d4f4594faa85" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "debf2f81470433f78676cc9649af3a208c369737428e6fcf4918ef2f14edeb9b", + "mlp.up_proj.weight": "5c1a4ef2663bb8509c1736405a121d2cf0105b113be03629fdd9c8ebae51064a", + "mlp.down_proj.weight": "c297d40f8af2ed96c90ff2f616877b4aaeed4bc2918f0e3b56ea769203724ce1" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "93af1649dbcf27dcc906cff079a1448af3e0b78dc0ae7fcd0038e95d05f79fdc", + "self_attn.k_proj.weight": "919f1a2454b2957fa1f611232b1c0aaccdab718aed512b435923cf972cd52015", + "self_attn.v_proj.weight": "2a660e851663561147921fae640f303f3f599da61afc8d68545fed320cc9b63d", + "self_attn.o_proj.weight": "bfb11d528abdacfc054070ed7a55f90ddde74fb17704bc742ceffa065adcc717", + "self_attn.q_norm.weight": "a4538fa22891825dbfbac330127ff29569ac04b5ab0ee89e60db7e7bc64f26a3", + "self_attn.k_norm.weight": "f26654e4f51003513e7ff0cd2f23e99387481dc5c639b01444903ec9632ae685", + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f", + "input_layernorm.weight": "70025e492885453112e0108ebf4311dfd994db35383e90da14292277834fb949", + "post_attention_layernorm.weight": "41357e377104509608d219bc3f4d8a51dde14749a9a6e6b592f6d4f4594faa85" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_6/cycle_metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_6/cycle_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..797e449323e250a348e3accd1a0a92b6c7514c10 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_6/cycle_metadata.json @@ -0,0 +1,276 @@ +{ + "cycle": 6, + "layer_merged": 12, + "num_layers_before": 31, + "num_layers_after": 30, + "fused_layer_state": "fused_layer.pt", + "profile_no_save": false, + "dwce_score": null, + "dwce_scores": null, + "dwce_meta": { + "method": "wb", + "fisher_computed": false + }, + "fisher_num_batches": 0, + "merge_method": "fisher", + "merged_params": 3, + "num_sequences": 64, + "teacher_source": "previous_cycle", + "teacher_cycle": 5, + "eval": { + "datasets": [ + "wikitext2" + ], + "configs": [ + null + ], + "split": "test", + "num_samples": 0, + "seq_len": 1024, + "post_ppl": null, + "merge_only_ppl": null + }, + "comm": { + "enabled": false + }, + "distill": { + "enabled": false, + "method": "reparam", + "calib_samples": 256, + "inst_samples": 0, + "seq_len": 512, + "batch_size": 2, + "epochs": 1.0, + "lr": 0.0001, + "kl_weight": 0.01, + "kl_temp": 4.0, + "kl_chunk_size": 16384, + "hidden_mse_weight": 1.0, + "attn_mse_weight": 0.0, + "mlp_mse_weight": 0.0, + "reparam_eta": 0.01, + "reparam_gamma": 0.0001, + "reparam_attn_reg_scale": 1.0, + "reparam_mlp_reg_scale": 1.0, + "reparam_param_subset": "all", + "reparam_train_subset": "same", + "reparam_skip_train": false, + "reparam_stats": null, + "reparam_gate_summary": null, + "post_merge_nll_scale": null, + "merge_only_ppl": null, + "post_ppl": null, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "instruction_dataset": null, + "instruction_config": null, + "instruction_split": "train" + }, + "lora": { + "enabled": false, + "seq_len": 512, + "batch_size": 2, + "epochs": 0.0, + "rank": 8, + "alpha": 16.0, + "dropout": 0.0, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "respect_exclude_pairs": false, + "kl_enabled": false, + "kl_weight": 0.1, + "kl_temp": 4.0, + "post_ppl": null, + "lr": 0.0001, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "log_steps": 100, + "eval_every": 0, + "eval_max_batches": null + }, + "norm_policy": "copy_n1_n2", + "source_feature_transport": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09820423130187533, + "calibration_cosine": 0.9496297005441647, + "calibration_norm_ratio": 0.9487355766850942, + "transport_relative_identity_distance": 0.48003679513931274, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "Transport then non-Fisher exact merge; no FiT" + }, + "same_anchor_contextual_fisher": { + "computed": false, + "anchor_candidate_mean_nll": 0.0, + "transported_source_candidate_mean_nll": 0.0 + }, + "full_model_saved": false, + "timing": { + "seconds": { + "selection": 4.16599796153605e-05, + "rebase": 0.0, + "source_feature_transport": 4.028148988960311, + "fisher": 3.316003130748868e-05, + "fit": 2.703971947019454, + "materialize_save_layer": 0.6423682359745726, + "metadata_write": 0.0004288180498406291, + "teacher_snapshot": 0.41955157299526036, + "cycle_cleanup": 0.20684619998792186, + "cycle_total": 8.00199037999846 + }, + "tracked_component_total_seconds": 8.001390582998283, + "cycle_total_seconds": 8.00199037999846, + "ratios_of_tracked_components": { + "selection": 5.2065924270565555e-06, + "rebase": 0.0, + "source_feature_transport": 0.5034311157762382, + "fisher": 4.144283542156861e-06, + "fit": 0.3379377520658692, + "materialize_save_layer": 0.08028207463581465, + "metadata_write": 5.3592940551083846e-05, + "teacher_snapshot": 0.052434832251126766, + "cycle_cleanup": 0.025851281454430938 + }, + "ratios_of_cycle_total": { + "selection": 5.206202161838704e-06, + "rebase": 0.0, + "source_feature_transport": 0.5033933806055245, + "fisher": 4.143972903338464e-06, + "fit": 0.3379124216117808, + "materialize_save_layer": 0.08027605701454195, + "metadata_write": 5.358892343990941e-05, + "teacher_snapshot": 0.05243090194709046, + "cycle_cleanup": 0.025849343746394464 + } + }, + "fisher_computed": false, + "fit": false, + "nofit_merge_control": { + "cycle": 6, + "anchor": 12, + "removed_original_layer": 18, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "b1c313ea0a3aa1df29a0618ae2ef9e5bc49705d8fb26ca876cad9c127db74e9e", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.09820423130187533, + "calibration_cosine": 0.9496297005441647, + "calibration_norm_ratio": 0.9487355766850942, + "transport_relative_identity_distance": 0.48003679513931274, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "3d50a07d21693613e0d831209868828215164c78606ad76bb5339d0093e2b167", + "self_attn.k_proj.weight": "055922b3d6d8ae628f8a4761be8273b2b35bd979a899e599e03a4aebc9b31b15", + "self_attn.v_proj.weight": "35d50fc9ef07106a763c826a6bed55f5f8de393edeac002703ab161116d8dead", + "self_attn.o_proj.weight": "a211c679b699b735e0587b7e0bab5610d182a4c54868438be766544a8f36b4eb", + "self_attn.q_norm.weight": "41ed057917753085abc49146d705c39743047800a2ac65dba3e227fb675ad494", + "self_attn.k_norm.weight": "f01c939b42b6e7b3a03519eadb0a08a563897458717fc05d98b7c7d158621f02", + "mlp.gate_proj.weight": "a60ac64753890f45bc06d840148fad55807f022c4f14d6437d926bfe08d6e401", + "mlp.up_proj.weight": "f614342e9f77c5b73e0b93c34e9a6cbfb1692decfd132853765e9da75c5a2edc", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7", + "input_layernorm.weight": "2c7f7059994a701ae1bfd696bd109340fa7a22df9deee3bfd6c4afb890155523", + "post_attention_layernorm.weight": "d85cdb5cf77b8d6d42c74fbd6aff643661b72fcfbe4f343311c4d31db4afc67f" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "3d50a07d21693613e0d831209868828215164c78606ad76bb5339d0093e2b167", + "self_attn.k_proj.weight": "055922b3d6d8ae628f8a4761be8273b2b35bd979a899e599e03a4aebc9b31b15", + "self_attn.v_proj.weight": "35d50fc9ef07106a763c826a6bed55f5f8de393edeac002703ab161116d8dead", + "self_attn.o_proj.weight": "a211c679b699b735e0587b7e0bab5610d182a4c54868438be766544a8f36b4eb", + "self_attn.q_norm.weight": "41ed057917753085abc49146d705c39743047800a2ac65dba3e227fb675ad494", + "self_attn.k_norm.weight": "f01c939b42b6e7b3a03519eadb0a08a563897458717fc05d98b7c7d158621f02", + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7", + "input_layernorm.weight": "2c7f7059994a701ae1bfd696bd109340fa7a22df9deee3bfd6c4afb890155523", + "post_attention_layernorm.weight": "d85cdb5cf77b8d6d42c74fbd6aff643661b72fcfbe4f343311c4d31db4afc67f" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "98ca3a1f76218aabde9d2dfeb23e28d798f812a336974e190ebaea9076ad4554", + "mlp.up_proj.weight": "91405003b4d46ce3a325e27da1f747b7b268d7570d3f6dab4f714a73aae768e6", + "mlp.down_proj.weight": "2b6fe52a74f5b15de22764a2c9b51acdfc75070dac110580366d2dcd9d207e0f" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "3d50a07d21693613e0d831209868828215164c78606ad76bb5339d0093e2b167", + "self_attn.k_proj.weight": "055922b3d6d8ae628f8a4761be8273b2b35bd979a899e599e03a4aebc9b31b15", + "self_attn.v_proj.weight": "35d50fc9ef07106a763c826a6bed55f5f8de393edeac002703ab161116d8dead", + "self_attn.o_proj.weight": "a211c679b699b735e0587b7e0bab5610d182a4c54868438be766544a8f36b4eb", + "self_attn.q_norm.weight": "41ed057917753085abc49146d705c39743047800a2ac65dba3e227fb675ad494", + "self_attn.k_norm.weight": "f01c939b42b6e7b3a03519eadb0a08a563897458717fc05d98b7c7d158621f02", + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7", + "input_layernorm.weight": "2c7f7059994a701ae1bfd696bd109340fa7a22df9deee3bfd6c4afb890155523", + "post_attention_layernorm.weight": "d85cdb5cf77b8d6d42c74fbd6aff643661b72fcfbe4f343311c4d31db4afc67f" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_7/cycle_metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_7/cycle_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad0f59935803a456afb399a3b6049424694b363 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_7/cycle_metadata.json @@ -0,0 +1,276 @@ +{ + "cycle": 7, + "layer_merged": 12, + "num_layers_before": 30, + "num_layers_after": 29, + "fused_layer_state": "fused_layer.pt", + "profile_no_save": false, + "dwce_score": null, + "dwce_scores": null, + "dwce_meta": { + "method": "wb", + "fisher_computed": false + }, + "fisher_num_batches": 0, + "merge_method": "fisher", + "merged_params": 3, + "num_sequences": 64, + "teacher_source": "previous_cycle", + "teacher_cycle": 6, + "eval": { + "datasets": [ + "wikitext2" + ], + "configs": [ + null + ], + "split": "test", + "num_samples": 0, + "seq_len": 1024, + "post_ppl": null, + "merge_only_ppl": null + }, + "comm": { + "enabled": false + }, + "distill": { + "enabled": false, + "method": "reparam", + "calib_samples": 256, + "inst_samples": 0, + "seq_len": 512, + "batch_size": 2, + "epochs": 1.0, + "lr": 0.0001, + "kl_weight": 0.01, + "kl_temp": 4.0, + "kl_chunk_size": 16384, + "hidden_mse_weight": 1.0, + "attn_mse_weight": 0.0, + "mlp_mse_weight": 0.0, + "reparam_eta": 0.01, + "reparam_gamma": 0.0001, + "reparam_attn_reg_scale": 1.0, + "reparam_mlp_reg_scale": 1.0, + "reparam_param_subset": "all", + "reparam_train_subset": "same", + "reparam_skip_train": false, + "reparam_stats": null, + "reparam_gate_summary": null, + "post_merge_nll_scale": null, + "merge_only_ppl": null, + "post_ppl": null, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "instruction_dataset": null, + "instruction_config": null, + "instruction_split": "train" + }, + "lora": { + "enabled": false, + "seq_len": 512, + "batch_size": 2, + "epochs": 0.0, + "rank": 8, + "alpha": 16.0, + "dropout": 0.0, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "respect_exclude_pairs": false, + "kl_enabled": false, + "kl_weight": 0.1, + "kl_temp": 4.0, + "post_ppl": null, + "lr": 0.0001, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "log_steps": 100, + "eval_every": 0, + "eval_max_batches": null + }, + "norm_policy": "copy_n1_n2", + "source_feature_transport": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.103606409701293, + "calibration_cosine": 0.9467810247760247, + "calibration_norm_ratio": 0.9459333354270779, + "transport_relative_identity_distance": 0.5275545120239258, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "Transport then non-Fisher exact merge; no FiT" + }, + "same_anchor_contextual_fisher": { + "computed": false, + "anchor_candidate_mean_nll": 0.0, + "transported_source_candidate_mean_nll": 0.0 + }, + "full_model_saved": false, + "timing": { + "seconds": { + "selection": 0.000234988983720541, + "rebase": 0.0, + "source_feature_transport": 4.04696036799578, + "fisher": 3.9969978388398886e-05, + "fit": 2.732918707013596, + "materialize_save_layer": 0.6510923550231382, + "metadata_write": 0.00045227701775729656, + "teacher_snapshot": 0.44334248796803877, + "cycle_cleanup": 0.16380883898818865, + "cycle_total": 8.039527288987301 + }, + "tracked_component_total_seconds": 8.038849992968608, + "cycle_total_seconds": 8.039527288987301, + "ratios_of_tracked_components": { + "selection": 2.9231666709302988e-05, + "rebase": 0.0, + "source_feature_transport": 0.5034252873900571, + "fisher": 4.972101534841387e-06, + "fit": 0.33996388903935454, + "materialize_save_layer": 0.08099322111901992, + "metadata_write": 5.626140780744666e-05, + "teacher_snapshot": 0.055149988910829284, + "cycle_cleanup": 0.020377148364687535 + }, + "ratios_of_cycle_total": { + "selection": 2.9229204065571543e-05, + "rebase": 0.0, + "source_feature_transport": 0.5033828759483637, + "fisher": 4.971682656410723e-06, + "fit": 0.33993524852601725, + "materialize_save_layer": 0.08098639778423503, + "metadata_write": 5.625666802286178e-05, + "teacher_snapshot": 0.05514534275856465, + "cycle_cleanup": 0.020375431676508786 + } + }, + "fisher_computed": false, + "fit": false, + "nofit_merge_control": { + "cycle": 7, + "anchor": 12, + "removed_original_layer": 19, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "4b807a71a3f5c920f7bbe9842c8bc0c89de7ebf5b02893847374cb46eeebbaae", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.103606409701293, + "calibration_cosine": 0.9467810247760247, + "calibration_norm_ratio": 0.9459333354270779, + "transport_relative_identity_distance": 0.5275545120239258, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "d14bc25171528405c3db791fd017b4e0fe37d06a9b07a9878cb81c1f0eb98ae6", + "self_attn.k_proj.weight": "8911d7363eb8d7e35ff407ac925183a06d54d624a8d2bd0ff41db97423515a54", + "self_attn.v_proj.weight": "4781046a7d6224a77c3e34dda92a2ef5748eb777d9d554acf5621c4e5742be2a", + "self_attn.o_proj.weight": "90f47fc1bc507ddfad16299dfcecb78a6b4efac01e21d3a6402c6ef3eae56348", + "self_attn.q_norm.weight": "6915d87fcc2402518b382dee5f5929f5774006c5d82904db327d2f118d6a5c99", + "self_attn.k_norm.weight": "cb20faa93398541fe6b2fbda2a527ea3243337db753acebdfd045bf309cbc4ae", + "mlp.gate_proj.weight": "aaf6792caf14dc5a810bfa937c966c7989238e23b21080752c784428eedbf79c", + "mlp.up_proj.weight": "5a9cf857484fc35820ab3032e13383687631e54fc1a7c7ff0443ec36e962cf17", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6", + "input_layernorm.weight": "316472831e20dec19263b56944811a059efb241cc842d8ace4fce706c98a8b11", + "post_attention_layernorm.weight": "7a462c4c15d73900fe1275d32448251905112e9530f615fe6776622c416f0f52" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "d14bc25171528405c3db791fd017b4e0fe37d06a9b07a9878cb81c1f0eb98ae6", + "self_attn.k_proj.weight": "8911d7363eb8d7e35ff407ac925183a06d54d624a8d2bd0ff41db97423515a54", + "self_attn.v_proj.weight": "4781046a7d6224a77c3e34dda92a2ef5748eb777d9d554acf5621c4e5742be2a", + "self_attn.o_proj.weight": "90f47fc1bc507ddfad16299dfcecb78a6b4efac01e21d3a6402c6ef3eae56348", + "self_attn.q_norm.weight": "6915d87fcc2402518b382dee5f5929f5774006c5d82904db327d2f118d6a5c99", + "self_attn.k_norm.weight": "cb20faa93398541fe6b2fbda2a527ea3243337db753acebdfd045bf309cbc4ae", + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6", + "input_layernorm.weight": "316472831e20dec19263b56944811a059efb241cc842d8ace4fce706c98a8b11", + "post_attention_layernorm.weight": "7a462c4c15d73900fe1275d32448251905112e9530f615fe6776622c416f0f52" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "03eb425fe1ff0c465accdbd062fa76ec3b951f4c173d9f900657e922811eff2f", + "mlp.up_proj.weight": "f482573a958623f2511b60e6bfdb8ddf1b18a35942833a3cf2f6036131673c06", + "mlp.down_proj.weight": "b61bbddd0e5ee13e6679fb129a28885d063105955f6cb5176c39b0ac18fbfda7" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "d14bc25171528405c3db791fd017b4e0fe37d06a9b07a9878cb81c1f0eb98ae6", + "self_attn.k_proj.weight": "8911d7363eb8d7e35ff407ac925183a06d54d624a8d2bd0ff41db97423515a54", + "self_attn.v_proj.weight": "4781046a7d6224a77c3e34dda92a2ef5748eb777d9d554acf5621c4e5742be2a", + "self_attn.o_proj.weight": "90f47fc1bc507ddfad16299dfcecb78a6b4efac01e21d3a6402c6ef3eae56348", + "self_attn.q_norm.weight": "6915d87fcc2402518b382dee5f5929f5774006c5d82904db327d2f118d6a5c99", + "self_attn.k_norm.weight": "cb20faa93398541fe6b2fbda2a527ea3243337db753acebdfd045bf309cbc4ae", + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6", + "input_layernorm.weight": "316472831e20dec19263b56944811a059efb241cc842d8ace4fce706c98a8b11", + "post_attention_layernorm.weight": "7a462c4c15d73900fe1275d32448251905112e9530f615fe6776622c416f0f52" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_8/cycle_metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_8/cycle_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9df2b4a1e704b90368f39688541ac5bae7839a7d --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_8/cycle_metadata.json @@ -0,0 +1,276 @@ +{ + "cycle": 8, + "layer_merged": 12, + "num_layers_before": 29, + "num_layers_after": 28, + "fused_layer_state": "fused_layer.pt", + "profile_no_save": false, + "dwce_score": null, + "dwce_scores": null, + "dwce_meta": { + "method": "wb", + "fisher_computed": false + }, + "fisher_num_batches": 0, + "merge_method": "fisher", + "merged_params": 3, + "num_sequences": 64, + "teacher_source": "previous_cycle", + "teacher_cycle": 7, + "eval": { + "datasets": [ + "wikitext2" + ], + "configs": [ + null + ], + "split": "test", + "num_samples": 0, + "seq_len": 1024, + "post_ppl": null, + "merge_only_ppl": null + }, + "comm": { + "enabled": false + }, + "distill": { + "enabled": false, + "method": "reparam", + "calib_samples": 256, + "inst_samples": 0, + "seq_len": 512, + "batch_size": 2, + "epochs": 1.0, + "lr": 0.0001, + "kl_weight": 0.01, + "kl_temp": 4.0, + "kl_chunk_size": 16384, + "hidden_mse_weight": 1.0, + "attn_mse_weight": 0.0, + "mlp_mse_weight": 0.0, + "reparam_eta": 0.01, + "reparam_gamma": 0.0001, + "reparam_attn_reg_scale": 1.0, + "reparam_mlp_reg_scale": 1.0, + "reparam_param_subset": "all", + "reparam_train_subset": "same", + "reparam_skip_train": false, + "reparam_stats": null, + "reparam_gate_summary": null, + "post_merge_nll_scale": null, + "merge_only_ppl": null, + "post_ppl": null, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "instruction_dataset": null, + "instruction_config": null, + "instruction_split": "train" + }, + "lora": { + "enabled": false, + "seq_len": 512, + "batch_size": 2, + "epochs": 0.0, + "rank": 8, + "alpha": 16.0, + "dropout": 0.0, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "respect_exclude_pairs": false, + "kl_enabled": false, + "kl_weight": 0.1, + "kl_temp": 4.0, + "post_ppl": null, + "lr": 0.0001, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "log_steps": 100, + "eval_every": 0, + "eval_max_batches": null + }, + "norm_policy": "copy_n1_n2", + "source_feature_transport": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.10606533290714001, + "calibration_cosine": 0.9454815765850116, + "calibration_norm_ratio": 0.9446186925723196, + "transport_relative_identity_distance": 0.5891098976135254, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "Transport then non-Fisher exact merge; no FiT" + }, + "same_anchor_contextual_fisher": { + "computed": false, + "anchor_candidate_mean_nll": 0.0, + "transported_source_candidate_mean_nll": 0.0 + }, + "full_model_saved": false, + "timing": { + "seconds": { + "selection": 4.438002360984683e-05, + "rebase": 0.0, + "source_feature_transport": 4.048203532001935, + "fisher": 3.051001112908125e-05, + "fit": 2.705056401027832, + "materialize_save_layer": 0.6487652659998275, + "metadata_write": 0.00044149800669401884, + "teacher_snapshot": 0.4064185360330157, + "cycle_cleanup": 0.16872839501593262, + "cycle_total": 7.978279564995319 + }, + "tracked_component_total_seconds": 7.977688518119976, + "cycle_total_seconds": 7.978279564995319, + "ratios_of_tracked_components": { + "selection": 5.563017847719309e-06, + "rebase": 0.0, + "source_feature_transport": 0.5074406606383695, + "fisher": 3.824417443697244e-06, + "fit": 0.33907771591780655, + "materialize_save_layer": 0.08132246132777264, + "metadata_write": 5.534159496090008e-05, + "teacher_snapshot": 0.050944397629702445, + "cycle_cleanup": 0.021150035456096648 + }, + "ratios_of_cycle_total": { + "selection": 5.562605728253002e-06, + "rebase": 0.0, + "source_feature_transport": 0.5074030684213445, + "fisher": 3.824134123219227e-06, + "fit": 0.33905259636379004, + "materialize_save_layer": 0.08131643679751252, + "metadata_write": 5.533749514507993e-05, + "teacher_snapshot": 0.050940623567037685, + "cycle_cleanup": 0.021148468619253208 + } + }, + "fisher_computed": false, + "fit": false, + "nofit_merge_control": { + "cycle": 8, + "anchor": 12, + "removed_original_layer": 20, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "2c7b2775e51300a42d0d75c89a6c22bee3e7c935b89f0db3cf2b3fd9863b8fd6", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.10606533290714001, + "calibration_cosine": 0.9454815765850116, + "calibration_norm_ratio": 0.9446186925723196, + "transport_relative_identity_distance": 0.5891098976135254, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "6bdb1d9174c7bf6d06b12d5d55b3a7649f4f880714e3e2d5a6f90c1449ad1a50", + "self_attn.k_proj.weight": "e3a98bfec07bba4f0fc6aa01b510f13fce379d8a5fdce7a70ce0870da65a2e43", + "self_attn.v_proj.weight": "f250ec94ab67ea3d397297fa0c70e7e24c7f77ca609a3a83cc8f7e792e4bc608", + "self_attn.o_proj.weight": "18f0f0ec64b28cbc88f0937245cb010fd86dcf094de2d96d1bd2162b76020229", + "self_attn.q_norm.weight": "2172344fe2c1fda6f261ece920fbd07a4b03d2385d2c0848c497aae4f07e69b5", + "self_attn.k_norm.weight": "a2c8f7815e194c45bad87d1e291c9ff2ad3a36929b1201d024354ec501563a4e", + "mlp.gate_proj.weight": "8334c424ddf9edb282ccb34d6f90321e9530eff5e81b7881ba9b3d74417a9672", + "mlp.up_proj.weight": "db91586fd2907e8ecccb075f036e5c180641fa35f7cd899a3f81411caa9d10df", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd", + "input_layernorm.weight": "96fb39db2f64c716ae6891acb4e4e7f2a1276921a97d0d9a50d746727e4bbcd0", + "post_attention_layernorm.weight": "cd331d158252933122eae174cda361e7f3eb260b867b079b375650b368a4ae06" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "6bdb1d9174c7bf6d06b12d5d55b3a7649f4f880714e3e2d5a6f90c1449ad1a50", + "self_attn.k_proj.weight": "e3a98bfec07bba4f0fc6aa01b510f13fce379d8a5fdce7a70ce0870da65a2e43", + "self_attn.v_proj.weight": "f250ec94ab67ea3d397297fa0c70e7e24c7f77ca609a3a83cc8f7e792e4bc608", + "self_attn.o_proj.weight": "18f0f0ec64b28cbc88f0937245cb010fd86dcf094de2d96d1bd2162b76020229", + "self_attn.q_norm.weight": "2172344fe2c1fda6f261ece920fbd07a4b03d2385d2c0848c497aae4f07e69b5", + "self_attn.k_norm.weight": "a2c8f7815e194c45bad87d1e291c9ff2ad3a36929b1201d024354ec501563a4e", + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd", + "input_layernorm.weight": "96fb39db2f64c716ae6891acb4e4e7f2a1276921a97d0d9a50d746727e4bbcd0", + "post_attention_layernorm.weight": "cd331d158252933122eae174cda361e7f3eb260b867b079b375650b368a4ae06" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "06d454e81788e8cfd28f613250c2c97f44fd78227c38bd35429f19915cdaeb51", + "mlp.up_proj.weight": "2c58e52893c9f2de1310538074a6ec2700403d256286f410d59a25064527f6c2", + "mlp.down_proj.weight": "d65d2b30d50eb54acafa83fef24d9a0fa7d57b6cb6cc06a94e476c758f8812d6" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "6bdb1d9174c7bf6d06b12d5d55b3a7649f4f880714e3e2d5a6f90c1449ad1a50", + "self_attn.k_proj.weight": "e3a98bfec07bba4f0fc6aa01b510f13fce379d8a5fdce7a70ce0870da65a2e43", + "self_attn.v_proj.weight": "f250ec94ab67ea3d397297fa0c70e7e24c7f77ca609a3a83cc8f7e792e4bc608", + "self_attn.o_proj.weight": "18f0f0ec64b28cbc88f0937245cb010fd86dcf094de2d96d1bd2162b76020229", + "self_attn.q_norm.weight": "2172344fe2c1fda6f261ece920fbd07a4b03d2385d2c0848c497aae4f07e69b5", + "self_attn.k_norm.weight": "a2c8f7815e194c45bad87d1e291c9ff2ad3a36929b1201d024354ec501563a4e", + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd", + "input_layernorm.weight": "96fb39db2f64c716ae6891acb4e4e7f2a1276921a97d0d9a50d746727e4bbcd0", + "post_attention_layernorm.weight": "cd331d158252933122eae174cda361e7f3eb260b867b079b375650b368a4ae06" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_9/cycle_metadata.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_9/cycle_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..873f3c6b1af1b4762f40013c5dd2a7488b1d3dcd --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/cycle_metadata/cycle_9/cycle_metadata.json @@ -0,0 +1,276 @@ +{ + "cycle": 9, + "layer_merged": 12, + "num_layers_before": 28, + "num_layers_after": 27, + "fused_layer_state": "fused_layer.pt", + "profile_no_save": false, + "dwce_score": null, + "dwce_scores": null, + "dwce_meta": { + "method": "wb", + "fisher_computed": false + }, + "fisher_num_batches": 0, + "merge_method": "fisher", + "merged_params": 3, + "num_sequences": 64, + "teacher_source": "previous_cycle", + "teacher_cycle": 8, + "eval": { + "datasets": [ + "wikitext2" + ], + "configs": [ + null + ], + "split": "test", + "num_samples": 0, + "seq_len": 1024, + "post_ppl": null, + "merge_only_ppl": null + }, + "comm": { + "enabled": false + }, + "distill": { + "enabled": false, + "method": "reparam", + "calib_samples": 256, + "inst_samples": 0, + "seq_len": 512, + "batch_size": 2, + "epochs": 1.0, + "lr": 0.0001, + "kl_weight": 0.01, + "kl_temp": 4.0, + "kl_chunk_size": 16384, + "hidden_mse_weight": 1.0, + "attn_mse_weight": 0.0, + "mlp_mse_weight": 0.0, + "reparam_eta": 0.01, + "reparam_gamma": 0.0001, + "reparam_attn_reg_scale": 1.0, + "reparam_mlp_reg_scale": 1.0, + "reparam_param_subset": "all", + "reparam_train_subset": "same", + "reparam_skip_train": false, + "reparam_stats": null, + "reparam_gate_summary": null, + "post_merge_nll_scale": null, + "merge_only_ppl": null, + "post_ppl": null, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "instruction_dataset": null, + "instruction_config": null, + "instruction_split": "train" + }, + "lora": { + "enabled": false, + "seq_len": 512, + "batch_size": 2, + "epochs": 0.0, + "rank": 8, + "alpha": 16.0, + "dropout": 0.0, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj" + ], + "respect_exclude_pairs": false, + "kl_enabled": false, + "kl_weight": 0.1, + "kl_temp": 4.0, + "post_ppl": null, + "lr": 0.0001, + "weight_decay": 0.0, + "max_grad_norm": 1.0, + "grad_accum_steps": 1, + "log_steps": 100, + "eval_every": 0, + "eval_max_batches": null + }, + "norm_policy": "copy_n1_n2", + "source_feature_transport": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.13979247738453124, + "calibration_cosine": 0.927474114888108, + "calibration_norm_ratio": 0.9266308047370905, + "transport_relative_identity_distance": 0.6954833269119263, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "Transport then non-Fisher exact merge; no FiT" + }, + "same_anchor_contextual_fisher": { + "computed": false, + "anchor_candidate_mean_nll": 0.0, + "transported_source_candidate_mean_nll": 0.0 + }, + "full_model_saved": false, + "timing": { + "seconds": { + "selection": 3.864895552396774e-05, + "rebase": 0.0, + "source_feature_transport": 4.0580107040004805, + "fisher": 4.0510029066354036e-05, + "fit": 2.6979933549882844, + "materialize_save_layer": 0.6465955360326916, + "metadata_write": 0.00044771801913157105, + "teacher_snapshot": 0.4412130580167286, + "cycle_cleanup": 0.17686755600152537, + "cycle_total": 8.021857643965632 + }, + "tracked_component_total_seconds": 8.021207086043432, + "cycle_total_seconds": 8.021857643965632, + "ratios_of_tracked_components": { + "selection": 4.818346554250585e-06, + "rebase": 0.0, + "source_feature_transport": 0.5059102277837024, + "fisher": 5.05036569082474e-06, + "fit": 0.33635752400442087, + "materialize_save_layer": 0.08061075210958472, + "metadata_write": 5.5816788461998676e-05, + "teacher_snapshot": 0.05500581811239122, + "cycle_cleanup": 0.02204999248919375 + }, + "ratios_of_cycle_total": { + "selection": 4.817955795194279e-06, + "rebase": 0.0, + "source_feature_transport": 0.5058691993933702, + "fisher": 5.049956115442578e-06, + "fit": 0.3363302460269692, + "materialize_save_layer": 0.08060421472563617, + "metadata_write": 5.5812261822966005e-05, + "teacher_snapshot": 0.0550013572415645, + "cycle_cleanup": 0.022048204275299296 + } + }, + "fisher_computed": false, + "fit": false, + "nofit_merge_control": { + "cycle": 9, + "anchor": 12, + "removed_original_layer": 21, + "selection_policy": "fixed_our_span_13_22_anchor12", + "fit": false, + "lambda_left": 0.0, + "transport_matrix_sha256": "7ad34658a18ce5ffbab2f8af974892f191e7db80d7b5da8d19e347938544b107", + "transport": { + "stats": { + "tokens": 65536, + "ridge_relative": 0.001, + "ridge_absolute": 6.060115993022919e-05, + "gram_mean_diagonal": 0.06060115993022919, + "calibration_nmse": 0.13979247738453124, + "calibration_cosine": 0.927474114888108, + "calibration_norm_ratio": 0.9266308047370905, + "transport_relative_identity_distance": 0.6954833269119263, + "enabled": true, + "batches": 32, + "coordinate_fit": "X_anchor @ T ~= X_source", + "weight_rewrite": [ + "mlp.gate_proj.weight <- W_source @ T.T", + "mlp.up_proj.weight <- W_source @ T.T" + ], + "down_proj_transported": false, + "fisher_order": "source transport first; same-anchor post-pruning contextual Fisher second" + }, + "source_before_sha256": { + "self_attn.q_proj.weight": "1839858bb3153a7d2907bbc4837cab44494ea4f8f73db2dac5a973f1500537c0", + "self_attn.k_proj.weight": "34776759a30508b301e6eeb4c03e74afc6e315a0e4005b788772f03267fa6bad", + "self_attn.v_proj.weight": "25ed7a7bfff05b2a63a9489ed945f49ed624baa5ffa7f971c263e4d3aade1d04", + "self_attn.o_proj.weight": "a8c4e7b3c03654158c39ea33456a291809aee2893e9dde1fa2d530da216b313c", + "self_attn.q_norm.weight": "21aa357e3be95df8e0b87310db2cf759d951d8c9a706c9e1bee207de27cbb87f", + "self_attn.k_norm.weight": "0dfad6f8b52d3a2c224e7cd2b5c1fe87db29b927a7d7fb4161f231df6df2ac02", + "mlp.gate_proj.weight": "4d745750b386ff30e96c0fe5ac21229e0672de87a3c4d30b7a877c333f250622", + "mlp.up_proj.weight": "b0b002f9bd65d4a6b4b37b6cf06139deb86a51ebc96fc27b4ebcff016f3e450b", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a", + "input_layernorm.weight": "2974f6d5be715ca00edf542f4e51e5eb5ced2a292277c1974a92f5f6fec44871", + "post_attention_layernorm.weight": "a00da323dd47889f5a6238369523374a092cab49267ad69f4716d7b1c3778d5f" + }, + "source_after_sha256": { + "self_attn.q_proj.weight": "1839858bb3153a7d2907bbc4837cab44494ea4f8f73db2dac5a973f1500537c0", + "self_attn.k_proj.weight": "34776759a30508b301e6eeb4c03e74afc6e315a0e4005b788772f03267fa6bad", + "self_attn.v_proj.weight": "25ed7a7bfff05b2a63a9489ed945f49ed624baa5ffa7f971c263e4d3aade1d04", + "self_attn.o_proj.weight": "a8c4e7b3c03654158c39ea33456a291809aee2893e9dde1fa2d530da216b313c", + "self_attn.q_norm.weight": "21aa357e3be95df8e0b87310db2cf759d951d8c9a706c9e1bee207de27cbb87f", + "self_attn.k_norm.weight": "0dfad6f8b52d3a2c224e7cd2b5c1fe87db29b927a7d7fb4161f231df6df2ac02", + "mlp.gate_proj.weight": "6b132cf55ede8fb805c51a5b57c84cddd3ac2e84d4d94d13538a0f3eae146ba2", + "mlp.up_proj.weight": "d8ae064ab88056091a57a96a007959b9d01bb6c1f037cdab35d76d5f5694b3ae", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a", + "input_layernorm.weight": "2974f6d5be715ca00edf542f4e51e5eb5ced2a292277c1974a92f5f6fec44871", + "post_attention_layernorm.weight": "a00da323dd47889f5a6238369523374a092cab49267ad69f4716d7b1c3778d5f" + }, + "changed": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight" + ] + }, + "merge": { + "initializer": "wb", + "lambda_left": 0.0, + "lambda_right": 1.0, + "projection_names": [ + "mlp.gate_proj.weight", + "mlp.up_proj.weight", + "mlp.down_proj.weight" + ], + "scope": "mlp", + "frozen_anchor_sha256": { + "self_attn.q_proj.weight": "1b6c758076acef0ff0bbf34e9ee7be3808252f432a10b0015a99b07a232ddf79", + "self_attn.k_proj.weight": "8d6f656be7538039543561a8c96dca260b3db7bdcb26946d9dca9a10101bb58f", + "self_attn.v_proj.weight": "ae85148c6da3808090619c005e94364db614bdec475df91ba4e312cb026fa377", + "self_attn.o_proj.weight": "e3c5cc19864068b28a406400c10d2704664356085511c7e44b587a1eab278d92", + "self_attn.q_norm.weight": "70223d2ee9393af355e7fa894d500b5c5bf032873b6b27cba6bb1f1b971cb904", + "self_attn.k_norm.weight": "df93c74989ee20de00c5875567ad98be1eb5d115d51bacb1dc615d22b15f488e", + "input_layernorm.weight": "f65024195899b2e5822f33d9c7d7c70442aaa262dd8bb1f4847d5c70f18d09e3", + "post_attention_layernorm.weight": "017c0603fff83a0a0231a2f3b9ae54ebe7f1d495c8d7092da5a10a87fadda049" + }, + "anchor_before_sha256": { + "mlp.gate_proj.weight": "9f8d001778d2562d9f0deb7aa67ab4b374ee11bc56cf341dd3bbc2f89da978a9", + "mlp.up_proj.weight": "82d2d94c9a1082f87643386884771be030ad54381da7fbfda1931d886bff69bc", + "mlp.down_proj.weight": "4d5fefc568246a3abfa21725082f7fdf4dd62e7c580a1280aebe0ec60a2230cd" + }, + "source_post_transport_sha256": { + "self_attn.q_proj.weight": "1839858bb3153a7d2907bbc4837cab44494ea4f8f73db2dac5a973f1500537c0", + "self_attn.k_proj.weight": "34776759a30508b301e6eeb4c03e74afc6e315a0e4005b788772f03267fa6bad", + "self_attn.v_proj.weight": "25ed7a7bfff05b2a63a9489ed945f49ed624baa5ffa7f971c263e4d3aade1d04", + "self_attn.o_proj.weight": "a8c4e7b3c03654158c39ea33456a291809aee2893e9dde1fa2d530da216b313c", + "self_attn.q_norm.weight": "21aa357e3be95df8e0b87310db2cf759d951d8c9a706c9e1bee207de27cbb87f", + "self_attn.k_norm.weight": "0dfad6f8b52d3a2c224e7cd2b5c1fe87db29b927a7d7fb4161f231df6df2ac02", + "mlp.gate_proj.weight": "6b132cf55ede8fb805c51a5b57c84cddd3ac2e84d4d94d13538a0f3eae146ba2", + "mlp.up_proj.weight": "d8ae064ab88056091a57a96a007959b9d01bb6c1f037cdab35d76d5f5694b3ae", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a", + "input_layernorm.weight": "2974f6d5be715ca00edf542f4e51e5eb5ced2a292277c1974a92f5f6fec44871", + "post_attention_layernorm.weight": "a00da323dd47889f5a6238369523374a092cab49267ad69f4716d7b1c3778d5f" + }, + "anchor_after_sha256": { + "mlp.gate_proj.weight": "6b132cf55ede8fb805c51a5b57c84cddd3ac2e84d4d94d13538a0f3eae146ba2", + "mlp.up_proj.weight": "d8ae064ab88056091a57a96a007959b9d01bb6c1f037cdab35d76d5f5694b3ae", + "mlp.down_proj.weight": "279864f62a9b78fa49dde314b363ca78f22fda503fbf8e9f64437ea6644a084a" + }, + "norm_policy": "keep_Wa_including_QK_norms", + "source_mutated_by_merge": false, + "fisher_computed": false, + "fit": false, + "optimizer_steps": 0, + "backward_calls": 0 + } + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_14bench.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_14bench.json new file mode 100644 index 0000000000000000000000000000000000000000..a61904e8126fc4d66b288d0c0a86b5a742194bee --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_14bench.json @@ -0,0 +1,37 @@ +{ + "task_scores": { + "race_high": 0.2835906232132647, + "race_middle": 0.2736768802228412, + "c3": 0.5068493150684932, + "cmnli": 0.3301200882280859, + "chid": 0.7574257425742574, + "piqa": 0.704570184983678, + "winogrande": 0.5422257300710339, + "hellaswag": 0.5501892053375822, + "arc_challenge": 0.3302047781569966, + "arc_easy": 0.5791245791245792, + "openbookqa": 0.378, + "sciq": 0.783, + "mmlu": 0.2650619569861843, + "cmmlu": 0.2644620963564151 + }, + "macro_mean_14": 0.46775008430881515, + "tasks": [ + "race_high", + "race_middle", + "c3", + "cmnli", + "chid", + "piqa", + "winogrande", + "hellaswag", + "arc_challenge", + "arc_easy", + "openbookqa", + "sciq", + "mmlu", + "cmmlu" + ], + "opencompass_protocol": "OpenCompass 0.5.4 canonical PPL prompts; full-prompt mean CE; lowest PPL", + "lm_eval_num_fewshot": 0 +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_lmeval9.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_lmeval9.json new file mode 100644 index 0000000000000000000000000000000000000000..8db474cb40e87c7d814d448e2da21cf092863c96 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_lmeval9.json @@ -0,0 +1,1401 @@ +{ + "model": "/workspace/models/qwen3-8b-base-49e3418", + "revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "compact_stage": "/workspace/artifacts/qwen3-base-nofit-merge4-20261011/full/wb/compact/stage_09", + "cycle": 9, + "anchor": 12, + "tasks": [ + "piqa", + "winogrande", + "hellaswag", + "arc_challenge", + "arc_easy", + "openbookqa", + "sciq", + "mmlu", + "cmmlu" + ], + "num_fewshot": 0, + "batch_size": "auto:4", + "limit": null, + "dataset_cache": "/workspace/artifacts/qwen3-base-nofit-merge4-20261011/lmeval_cache", + "preferred_metrics": { + "piqa": "acc_norm,none", + "winogrande": "acc,none", + "hellaswag": "acc_norm,none", + "arc_challenge": "acc_norm,none", + "arc_easy": "acc_norm,none", + "openbookqa": "acc_norm,none", + "sciq": "acc_norm,none", + "mmlu": "acc,none", + "cmmlu": "acc,none" + }, + "task_scores": { + "piqa": 0.704570184983678, + "winogrande": 0.5422257300710339, + "hellaswag": 0.5501892053375822, + "arc_challenge": 0.3302047781569966, + "arc_easy": 0.5791245791245792, + "openbookqa": 0.378, + "sciq": 0.783, + "mmlu": 0.2650619569861843, + "cmmlu": 0.2644620963564151 + }, + "macro_mean_all_rows": 0.48853761455738554, + "lm_eval_results": { + "piqa": { + "name": "piqa", + "alias": "piqa", + "sample_len": 1838, + "acc,none": 0.7078346028291621, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.704570184983678, + "acc_norm_stderr,none": "N/A" + }, + "winogrande": { + "name": "winogrande", + "alias": "winogrande", + "sample_len": 1267, + "acc,none": 0.5422257300710339, + "acc_stderr,none": "N/A" + }, + "hellaswag": { + "name": "hellaswag", + "alias": "hellaswag", + "sample_len": 10042, + "acc,none": 0.41435968930491934, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.5501892053375822, + "acc_norm_stderr,none": "N/A" + }, + "arc_challenge": { + "name": "arc_challenge", + "alias": "arc_challenge", + "sample_len": 1172, + "acc,none": 0.2986348122866894, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.3302047781569966, + "acc_norm_stderr,none": "N/A" + }, + "arc_easy": { + "name": "arc_easy", + "alias": "arc_easy", + "sample_len": 2376, + "acc,none": 0.6191077441077442, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.5791245791245792, + "acc_norm_stderr,none": "N/A" + }, + "openbookqa": { + "name": "openbookqa", + "alias": "openbookqa", + "sample_len": 500, + "acc,none": 0.262, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.378, + "acc_norm_stderr,none": "N/A" + }, + "sciq": { + "name": "sciq", + "alias": "sciq", + "sample_len": 1000, + "acc,none": 0.829, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.783, + "acc_norm_stderr,none": "N/A" + }, + "mmlu_abstract_algebra": { + "name": "mmlu_abstract_algebra", + "alias": "abstract_algebra", + "sample_len": 100, + "acc,none": 0.29, + "acc_stderr,none": "N/A" + }, + "mmlu_anatomy": { + "name": "mmlu_anatomy", + "alias": "anatomy", + "sample_len": 135, + "acc,none": 0.2518518518518518, + "acc_stderr,none": "N/A" + }, + "mmlu_astronomy": { + "name": "mmlu_astronomy", + "alias": "astronomy", + "sample_len": 152, + "acc,none": 0.2894736842105263, + "acc_stderr,none": "N/A" + }, + "mmlu_college_biology": { + "name": "mmlu_college_biology", + "alias": "college_biology", + "sample_len": 144, + "acc,none": 0.22916666666666666, + "acc_stderr,none": "N/A" + }, + "mmlu_college_chemistry": { + "name": "mmlu_college_chemistry", + "alias": "college_chemistry", + "sample_len": 100, + "acc,none": 0.17, + "acc_stderr,none": "N/A" + }, + "mmlu_college_computer_science": { + "name": "mmlu_college_computer_science", + "alias": "college_computer_science", + "sample_len": 100, + "acc,none": 0.23, + "acc_stderr,none": "N/A" + }, + "mmlu_college_mathematics": { + "name": "mmlu_college_mathematics", + "alias": "college_mathematics", + "sample_len": 100, + "acc,none": 0.29, + "acc_stderr,none": "N/A" + }, + "mmlu_college_physics": { + "name": "mmlu_college_physics", + "alias": "college_physics", + "sample_len": 102, + "acc,none": 0.27450980392156865, + "acc_stderr,none": "N/A" + }, + "mmlu_computer_security": { + "name": "mmlu_computer_security", + "alias": "computer_security", + "sample_len": 100, + "acc,none": 0.24, + "acc_stderr,none": "N/A" + }, + "mmlu_conceptual_physics": { + "name": "mmlu_conceptual_physics", + "alias": "conceptual_physics", + "sample_len": 235, + "acc,none": 0.2425531914893617, + "acc_stderr,none": "N/A" + }, + "mmlu_electrical_engineering": { + "name": "mmlu_electrical_engineering", + "alias": "electrical_engineering", + "sample_len": 145, + "acc,none": 0.2827586206896552, + "acc_stderr,none": "N/A" + }, + "mmlu_elementary_mathematics": { + "name": "mmlu_elementary_mathematics", + "alias": "elementary_mathematics", + "sample_len": 378, + "acc,none": 0.2566137566137566, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_biology": { + "name": "mmlu_high_school_biology", + "alias": "high_school_biology", + "sample_len": 310, + "acc,none": 0.3064516129032258, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_chemistry": { + "name": "mmlu_high_school_chemistry", + "alias": "high_school_chemistry", + "sample_len": 203, + "acc,none": 0.2857142857142857, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_computer_science": { + "name": "mmlu_high_school_computer_science", + "alias": "high_school_computer_science", + "sample_len": 100, + "acc,none": 0.28, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_mathematics": { + "name": "mmlu_high_school_mathematics", + "alias": "high_school_mathematics", + "sample_len": 270, + "acc,none": 0.2777777777777778, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_physics": { + "name": "mmlu_high_school_physics", + "alias": "high_school_physics", + "sample_len": 151, + "acc,none": 0.26490066225165565, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_statistics": { + "name": "mmlu_high_school_statistics", + "alias": "high_school_statistics", + "sample_len": 216, + "acc,none": 0.19444444444444445, + "acc_stderr,none": "N/A" + }, + "mmlu_machine_learning": { + "name": "mmlu_machine_learning", + "alias": "machine_learning", + "sample_len": 112, + "acc,none": 0.32142857142857145, + "acc_stderr,none": "N/A" + }, + "mmlu_business_ethics": { + "name": "mmlu_business_ethics", + "alias": "business_ethics", + "sample_len": 100, + "acc,none": 0.23, + "acc_stderr,none": "N/A" + }, + "mmlu_clinical_knowledge": { + "name": "mmlu_clinical_knowledge", + "alias": "clinical_knowledge", + "sample_len": 265, + "acc,none": 0.25660377358490566, + "acc_stderr,none": "N/A" + }, + "mmlu_college_medicine": { + "name": "mmlu_college_medicine", + "alias": "college_medicine", + "sample_len": 173, + "acc,none": 0.2658959537572254, + "acc_stderr,none": "N/A" + }, + "mmlu_global_facts": { + "name": "mmlu_global_facts", + "alias": "global_facts", + "sample_len": 100, + "acc,none": 0.39, + "acc_stderr,none": "N/A" + }, + "mmlu_human_aging": { + "name": "mmlu_human_aging", + "alias": "human_aging", + "sample_len": 223, + "acc,none": 0.16591928251121077, + "acc_stderr,none": "N/A" + }, + "mmlu_management": { + "name": "mmlu_management", + "alias": "management", + "sample_len": 103, + "acc,none": 0.20388349514563106, + "acc_stderr,none": "N/A" + }, + "mmlu_marketing": { + "name": "mmlu_marketing", + "alias": "marketing", + "sample_len": 234, + "acc,none": 0.2606837606837607, + "acc_stderr,none": "N/A" + }, + "mmlu_medical_genetics": { + "name": "mmlu_medical_genetics", + "alias": "medical_genetics", + "sample_len": 100, + "acc,none": 0.26, + "acc_stderr,none": "N/A" + }, + "mmlu_miscellaneous": { + "name": "mmlu_miscellaneous", + "alias": "miscellaneous", + "sample_len": 783, + "acc,none": 0.30140485312899107, + "acc_stderr,none": "N/A" + }, + "mmlu_nutrition": { + "name": "mmlu_nutrition", + "alias": "nutrition", + "sample_len": 306, + "acc,none": 0.27450980392156865, + "acc_stderr,none": "N/A" + }, + "mmlu_professional_accounting": { + "name": "mmlu_professional_accounting", + "alias": "professional_accounting", + "sample_len": 282, + "acc,none": 0.30851063829787234, + "acc_stderr,none": "N/A" + }, + "mmlu_professional_medicine": { + "name": "mmlu_professional_medicine", + "alias": "professional_medicine", + "sample_len": 272, + "acc,none": 0.36764705882352944, + "acc_stderr,none": "N/A" + }, + "mmlu_virology": { + "name": "mmlu_virology", + "alias": "virology", + "sample_len": 166, + "acc,none": 0.1927710843373494, + "acc_stderr,none": "N/A" + }, + "mmlu_econometrics": { + "name": "mmlu_econometrics", + "alias": "econometrics", + "sample_len": 114, + "acc,none": 0.23684210526315788, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_geography": { + "name": "mmlu_high_school_geography", + "alias": "high_school_geography", + "sample_len": 198, + "acc,none": 0.32323232323232326, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_government_and_politics": { + "name": "mmlu_high_school_government_and_politics", + "alias": "high_school_government_and_politics", + "sample_len": 193, + "acc,none": 0.3316062176165803, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_macroeconomics": { + "name": "mmlu_high_school_macroeconomics", + "alias": "high_school_macroeconomics", + "sample_len": 390, + "acc,none": 0.2564102564102564, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_microeconomics": { + "name": "mmlu_high_school_microeconomics", + "alias": "high_school_microeconomics", + "sample_len": 238, + "acc,none": 0.21428571428571427, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_psychology": { + "name": "mmlu_high_school_psychology", + "alias": "high_school_psychology", + "sample_len": 545, + "acc,none": 0.26972477064220185, + "acc_stderr,none": "N/A" + }, + "mmlu_human_sexuality": { + "name": "mmlu_human_sexuality", + "alias": "human_sexuality", + "sample_len": 131, + "acc,none": 0.25190839694656486, + "acc_stderr,none": "N/A" + }, + "mmlu_professional_psychology": { + "name": "mmlu_professional_psychology", + "alias": "professional_psychology", + "sample_len": 612, + "acc,none": 0.272875816993464, + "acc_stderr,none": "N/A" + }, + "mmlu_public_relations": { + "name": "mmlu_public_relations", + "alias": "public_relations", + "sample_len": 110, + "acc,none": 0.23636363636363636, + "acc_stderr,none": "N/A" + }, + "mmlu_security_studies": { + "name": "mmlu_security_studies", + "alias": "security_studies", + "sample_len": 245, + "acc,none": 0.2816326530612245, + "acc_stderr,none": "N/A" + }, + "mmlu_sociology": { + "name": "mmlu_sociology", + "alias": "sociology", + "sample_len": 201, + "acc,none": 0.23880597014925373, + "acc_stderr,none": "N/A" + }, + "mmlu_us_foreign_policy": { + "name": "mmlu_us_foreign_policy", + "alias": "us_foreign_policy", + "sample_len": 100, + "acc,none": 0.23, + "acc_stderr,none": "N/A" + }, + "mmlu_formal_logic": { + "name": "mmlu_formal_logic", + "alias": "formal_logic", + "sample_len": 126, + "acc,none": 0.24603174603174602, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_european_history": { + "name": "mmlu_high_school_european_history", + "alias": "high_school_european_history", + "sample_len": 165, + "acc,none": 0.2727272727272727, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_us_history": { + "name": "mmlu_high_school_us_history", + "alias": "high_school_us_history", + "sample_len": 204, + "acc,none": 0.29411764705882354, + "acc_stderr,none": "N/A" + }, + "mmlu_high_school_world_history": { + "name": "mmlu_high_school_world_history", + "alias": "high_school_world_history", + "sample_len": 237, + "acc,none": 0.22362869198312235, + "acc_stderr,none": "N/A" + }, + "mmlu_international_law": { + "name": "mmlu_international_law", + "alias": "international_law", + "sample_len": 121, + "acc,none": 0.2644628099173554, + "acc_stderr,none": "N/A" + }, + "mmlu_jurisprudence": { + "name": "mmlu_jurisprudence", + "alias": "jurisprudence", + "sample_len": 108, + "acc,none": 0.3148148148148148, + "acc_stderr,none": "N/A" + }, + "mmlu_logical_fallacies": { + "name": "mmlu_logical_fallacies", + "alias": "logical_fallacies", + "sample_len": 163, + "acc,none": 0.2883435582822086, + "acc_stderr,none": "N/A" + }, + "mmlu_moral_disputes": { + "name": "mmlu_moral_disputes", + "alias": "moral_disputes", + "sample_len": 346, + "acc,none": 0.30346820809248554, + "acc_stderr,none": "N/A" + }, + "mmlu_moral_scenarios": { + "name": "mmlu_moral_scenarios", + "alias": "moral_scenarios", + "sample_len": 895, + "acc,none": 0.2424581005586592, + "acc_stderr,none": "N/A" + }, + "mmlu_philosophy": { + "name": "mmlu_philosophy", + "alias": "philosophy", + "sample_len": 311, + "acc,none": 0.2958199356913183, + "acc_stderr,none": "N/A" + }, + "mmlu_prehistory": { + "name": "mmlu_prehistory", + "alias": "prehistory", + "sample_len": 324, + "acc,none": 0.29012345679012347, + "acc_stderr,none": "N/A" + }, + "mmlu_professional_law": { + "name": "mmlu_professional_law", + "alias": "professional_law", + "sample_len": 1534, + "acc,none": 0.23598435462842243, + "acc_stderr,none": "N/A" + }, + "mmlu_world_religions": { + "name": "mmlu_world_religions", + "alias": "world_religions", + "sample_len": 171, + "acc,none": 0.23976608187134502, + "acc_stderr,none": "N/A" + }, + "cmmlu_agronomy": { + "name": "cmmlu_agronomy", + "alias": "cmmlu_agronomy", + "sample_len": 169, + "acc,none": 0.26627218934911245, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.26627218934911245, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_anatomy": { + "name": "cmmlu_anatomy", + "alias": "cmmlu_anatomy", + "sample_len": 148, + "acc,none": 0.22297297297297297, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.22297297297297297, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_ancient_chinese": { + "name": "cmmlu_ancient_chinese", + "alias": "cmmlu_ancient_chinese", + "sample_len": 164, + "acc,none": 0.27439024390243905, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.27439024390243905, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_arts": { + "name": "cmmlu_arts", + "alias": "cmmlu_arts", + "sample_len": 160, + "acc,none": 0.24375, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.24375, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_astronomy": { + "name": "cmmlu_astronomy", + "alias": "cmmlu_astronomy", + "sample_len": 165, + "acc,none": 0.23636363636363636, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.23636363636363636, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_business_ethics": { + "name": "cmmlu_business_ethics", + "alias": "cmmlu_business_ethics", + "sample_len": 209, + "acc,none": 0.3014354066985646, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.3014354066985646, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_chinese_civil_service_exam": { + "name": "cmmlu_chinese_civil_service_exam", + "alias": "cmmlu_chinese_civil_service_exam", + "sample_len": 160, + "acc,none": 0.275, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.275, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_chinese_driving_rule": { + "name": "cmmlu_chinese_driving_rule", + "alias": "cmmlu_chinese_driving_rule", + "sample_len": 131, + "acc,none": 0.2824427480916031, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2824427480916031, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_chinese_food_culture": { + "name": "cmmlu_chinese_food_culture", + "alias": "cmmlu_chinese_food_culture", + "sample_len": 136, + "acc,none": 0.2647058823529412, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2647058823529412, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_chinese_foreign_policy": { + "name": "cmmlu_chinese_foreign_policy", + "alias": "cmmlu_chinese_foreign_policy", + "sample_len": 107, + "acc,none": 0.2616822429906542, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2616822429906542, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_chinese_history": { + "name": "cmmlu_chinese_history", + "alias": "cmmlu_chinese_history", + "sample_len": 323, + "acc,none": 0.25386996904024767, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.25386996904024767, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_chinese_literature": { + "name": "cmmlu_chinese_literature", + "alias": "cmmlu_chinese_literature", + "sample_len": 204, + "acc,none": 0.27941176470588236, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.27941176470588236, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_chinese_teacher_qualification": { + "name": "cmmlu_chinese_teacher_qualification", + "alias": "cmmlu_chinese_teacher_qualification", + "sample_len": 179, + "acc,none": 0.3128491620111732, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.3128491620111732, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_clinical_knowledge": { + "name": "cmmlu_clinical_knowledge", + "alias": "cmmlu_clinical_knowledge", + "sample_len": 237, + "acc,none": 0.25738396624472576, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.25738396624472576, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_college_actuarial_science": { + "name": "cmmlu_college_actuarial_science", + "alias": "cmmlu_college_actuarial_science", + "sample_len": 106, + "acc,none": 0.29245283018867924, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.29245283018867924, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_college_education": { + "name": "cmmlu_college_education", + "alias": "cmmlu_college_education", + "sample_len": 107, + "acc,none": 0.2523364485981308, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2523364485981308, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_college_engineering_hydrology": { + "name": "cmmlu_college_engineering_hydrology", + "alias": "cmmlu_college_engineering_hydrology", + "sample_len": 106, + "acc,none": 0.2830188679245283, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2830188679245283, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_college_law": { + "name": "cmmlu_college_law", + "alias": "cmmlu_college_law", + "sample_len": 108, + "acc,none": 0.2222222222222222, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2222222222222222, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_college_mathematics": { + "name": "cmmlu_college_mathematics", + "alias": "cmmlu_college_mathematics", + "sample_len": 105, + "acc,none": 0.24761904761904763, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.24761904761904763, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_college_medical_statistics": { + "name": "cmmlu_college_medical_statistics", + "alias": "cmmlu_college_medical_statistics", + "sample_len": 106, + "acc,none": 0.27358490566037735, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.27358490566037735, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_college_medicine": { + "name": "cmmlu_college_medicine", + "alias": "cmmlu_college_medicine", + "sample_len": 273, + "acc,none": 0.21245421245421245, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.21245421245421245, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_computer_science": { + "name": "cmmlu_computer_science", + "alias": "cmmlu_computer_science", + "sample_len": 204, + "acc,none": 0.27941176470588236, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.27941176470588236, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_computer_security": { + "name": "cmmlu_computer_security", + "alias": "cmmlu_computer_security", + "sample_len": 171, + "acc,none": 0.2631578947368421, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2631578947368421, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_conceptual_physics": { + "name": "cmmlu_conceptual_physics", + "alias": "cmmlu_conceptual_physics", + "sample_len": 147, + "acc,none": 0.22448979591836735, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.22448979591836735, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_construction_project_management": { + "name": "cmmlu_construction_project_management", + "alias": "cmmlu_construction_project_management", + "sample_len": 139, + "acc,none": 0.28776978417266186, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.28776978417266186, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_economics": { + "name": "cmmlu_economics", + "alias": "cmmlu_economics", + "sample_len": 159, + "acc,none": 0.29559748427672955, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.29559748427672955, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_education": { + "name": "cmmlu_education", + "alias": "cmmlu_education", + "sample_len": 163, + "acc,none": 0.27607361963190186, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.27607361963190186, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_electrical_engineering": { + "name": "cmmlu_electrical_engineering", + "alias": "cmmlu_electrical_engineering", + "sample_len": 172, + "acc,none": 0.19186046511627908, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.19186046511627908, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_elementary_chinese": { + "name": "cmmlu_elementary_chinese", + "alias": "cmmlu_elementary_chinese", + "sample_len": 252, + "acc,none": 0.32936507936507936, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.32936507936507936, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_elementary_commonsense": { + "name": "cmmlu_elementary_commonsense", + "alias": "cmmlu_elementary_commonsense", + "sample_len": 198, + "acc,none": 0.2727272727272727, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2727272727272727, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_elementary_information_and_technology": { + "name": "cmmlu_elementary_information_and_technology", + "alias": "cmmlu_elementary_information_and_technology", + "sample_len": 238, + "acc,none": 0.2605042016806723, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2605042016806723, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_elementary_mathematics": { + "name": "cmmlu_elementary_mathematics", + "alias": "cmmlu_elementary_mathematics", + "sample_len": 230, + "acc,none": 0.2565217391304348, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2565217391304348, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_ethnology": { + "name": "cmmlu_ethnology", + "alias": "cmmlu_ethnology", + "sample_len": 135, + "acc,none": 0.3111111111111111, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.3111111111111111, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_food_science": { + "name": "cmmlu_food_science", + "alias": "cmmlu_food_science", + "sample_len": 143, + "acc,none": 0.24475524475524477, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.24475524475524477, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_genetics": { + "name": "cmmlu_genetics", + "alias": "cmmlu_genetics", + "sample_len": 176, + "acc,none": 0.2556818181818182, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2556818181818182, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_global_facts": { + "name": "cmmlu_global_facts", + "alias": "cmmlu_global_facts", + "sample_len": 149, + "acc,none": 0.22818791946308725, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.22818791946308725, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_high_school_biology": { + "name": "cmmlu_high_school_biology", + "alias": "cmmlu_high_school_biology", + "sample_len": 169, + "acc,none": 0.2485207100591716, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2485207100591716, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_high_school_chemistry": { + "name": "cmmlu_high_school_chemistry", + "alias": "cmmlu_high_school_chemistry", + "sample_len": 132, + "acc,none": 0.2196969696969697, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2196969696969697, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_high_school_geography": { + "name": "cmmlu_high_school_geography", + "alias": "cmmlu_high_school_geography", + "sample_len": 118, + "acc,none": 0.2627118644067797, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2627118644067797, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_high_school_mathematics": { + "name": "cmmlu_high_school_mathematics", + "alias": "cmmlu_high_school_mathematics", + "sample_len": 164, + "acc,none": 0.27439024390243905, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.27439024390243905, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_high_school_physics": { + "name": "cmmlu_high_school_physics", + "alias": "cmmlu_high_school_physics", + "sample_len": 110, + "acc,none": 0.2545454545454545, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2545454545454545, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_high_school_politics": { + "name": "cmmlu_high_school_politics", + "alias": "cmmlu_high_school_politics", + "sample_len": 143, + "acc,none": 0.3076923076923077, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.3076923076923077, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_human_sexuality": { + "name": "cmmlu_human_sexuality", + "alias": "cmmlu_human_sexuality", + "sample_len": 126, + "acc,none": 0.2619047619047619, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2619047619047619, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_international_law": { + "name": "cmmlu_international_law", + "alias": "cmmlu_international_law", + "sample_len": 185, + "acc,none": 0.1945945945945946, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.1945945945945946, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_journalism": { + "name": "cmmlu_journalism", + "alias": "cmmlu_journalism", + "sample_len": 172, + "acc,none": 0.23837209302325582, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.23837209302325582, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_jurisprudence": { + "name": "cmmlu_jurisprudence", + "alias": "cmmlu_jurisprudence", + "sample_len": 411, + "acc,none": 0.26520681265206814, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.26520681265206814, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_legal_and_moral_basis": { + "name": "cmmlu_legal_and_moral_basis", + "alias": "cmmlu_legal_and_moral_basis", + "sample_len": 214, + "acc,none": 0.2897196261682243, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2897196261682243, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_logical": { + "name": "cmmlu_logical", + "alias": "cmmlu_logical", + "sample_len": 123, + "acc,none": 0.24390243902439024, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.24390243902439024, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_machine_learning": { + "name": "cmmlu_machine_learning", + "alias": "cmmlu_machine_learning", + "sample_len": 122, + "acc,none": 0.21311475409836064, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.21311475409836064, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_management": { + "name": "cmmlu_management", + "alias": "cmmlu_management", + "sample_len": 210, + "acc,none": 0.2904761904761905, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2904761904761905, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_marketing": { + "name": "cmmlu_marketing", + "alias": "cmmlu_marketing", + "sample_len": 180, + "acc,none": 0.26666666666666666, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.26666666666666666, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_marxist_theory": { + "name": "cmmlu_marxist_theory", + "alias": "cmmlu_marxist_theory", + "sample_len": 189, + "acc,none": 0.3492063492063492, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.3492063492063492, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_modern_chinese": { + "name": "cmmlu_modern_chinese", + "alias": "cmmlu_modern_chinese", + "sample_len": 116, + "acc,none": 0.29310344827586204, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.29310344827586204, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_nutrition": { + "name": "cmmlu_nutrition", + "alias": "cmmlu_nutrition", + "sample_len": 145, + "acc,none": 0.3103448275862069, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.3103448275862069, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_philosophy": { + "name": "cmmlu_philosophy", + "alias": "cmmlu_philosophy", + "sample_len": 105, + "acc,none": 0.26666666666666666, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.26666666666666666, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_professional_accounting": { + "name": "cmmlu_professional_accounting", + "alias": "cmmlu_professional_accounting", + "sample_len": 175, + "acc,none": 0.2742857142857143, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2742857142857143, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_professional_law": { + "name": "cmmlu_professional_law", + "alias": "cmmlu_professional_law", + "sample_len": 211, + "acc,none": 0.27014218009478674, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.27014218009478674, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_professional_medicine": { + "name": "cmmlu_professional_medicine", + "alias": "cmmlu_professional_medicine", + "sample_len": 376, + "acc,none": 0.2047872340425532, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2047872340425532, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_professional_psychology": { + "name": "cmmlu_professional_psychology", + "alias": "cmmlu_professional_psychology", + "sample_len": 232, + "acc,none": 0.27586206896551724, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.27586206896551724, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_public_relations": { + "name": "cmmlu_public_relations", + "alias": "cmmlu_public_relations", + "sample_len": 174, + "acc,none": 0.3218390804597701, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.3218390804597701, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_security_study": { + "name": "cmmlu_security_study", + "alias": "cmmlu_security_study", + "sample_len": 135, + "acc,none": 0.22962962962962963, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.22962962962962963, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_sociology": { + "name": "cmmlu_sociology", + "alias": "cmmlu_sociology", + "sample_len": 226, + "acc,none": 0.27876106194690264, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.27876106194690264, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_sports_science": { + "name": "cmmlu_sports_science", + "alias": "cmmlu_sports_science", + "sample_len": 165, + "acc,none": 0.21818181818181817, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.21818181818181817, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_traditional_chinese_medicine": { + "name": "cmmlu_traditional_chinese_medicine", + "alias": "cmmlu_traditional_chinese_medicine", + "sample_len": 185, + "acc,none": 0.31351351351351353, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.31351351351351353, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_virology": { + "name": "cmmlu_virology", + "alias": "cmmlu_virology", + "sample_len": 169, + "acc,none": 0.27218934911242604, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.27218934911242604, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_world_history": { + "name": "cmmlu_world_history", + "alias": "cmmlu_world_history", + "sample_len": 161, + "acc,none": 0.2857142857142857, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2857142857142857, + "acc_norm_stderr,none": "N/A" + }, + "cmmlu_world_religions": { + "name": "cmmlu_world_religions", + "alias": "cmmlu_world_religions", + "sample_len": 160, + "acc,none": 0.24375, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.24375, + "acc_norm_stderr,none": "N/A" + }, + "mmlu_stem": { + "alias": "stem", + "name": "mmlu_stem", + "sample_len": 3153, + "acc,none": 0.26324135743736127, + "acc_stderr,none": "N/A", + "sample_count": { + "acc,none": 3153 + } + }, + "mmlu_other": { + "alias": "other", + "name": "mmlu_other", + "sample_len": 3107, + "acc,none": 0.27679433537174125, + "acc_stderr,none": "N/A", + "sample_count": { + "acc,none": 3107 + } + }, + "mmlu_social_sciences": { + "alias": "social sciences", + "name": "mmlu_social_sciences", + "sample_len": 3077, + "acc,none": 0.26616834579135523, + "acc_stderr,none": "N/A", + "sample_count": { + "acc,none": 3077 + } + }, + "mmlu_humanities": { + "alias": "humanities", + "name": "mmlu_humanities", + "sample_len": 4705, + "acc,none": 0.25781083953241235, + "acc_stderr,none": "N/A", + "sample_count": { + "acc,none": 4705 + } + }, + "mmlu": { + "alias": "mmlu", + "name": "mmlu", + "sample_len": 14042, + "acc,none": 0.2650619569861843, + "acc_stderr,none": "N/A", + "sample_count": { + "acc,none": 14042 + } + }, + "cmmlu": { + "alias": "cmmlu", + "name": "cmmlu", + "sample_len": 11582, + "acc,none": 0.2644620963564151, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2644620963564151, + "acc_norm_stderr,none": "N/A", + "sample_count": { + "acc,none": 11582, + "acc_norm,none": 11582 + } + } + }, + "lm_eval_groups": { + "mmlu_stem": { + "alias": "stem", + "name": "mmlu_stem", + "sample_len": 3153, + "acc,none": 0.26324135743736127, + "acc_stderr,none": "N/A", + "sample_count": { + "acc,none": 3153 + } + }, + "mmlu_other": { + "alias": "other", + "name": "mmlu_other", + "sample_len": 3107, + "acc,none": 0.27679433537174125, + "acc_stderr,none": "N/A", + "sample_count": { + "acc,none": 3107 + } + }, + "mmlu_social_sciences": { + "alias": "social sciences", + "name": "mmlu_social_sciences", + "sample_len": 3077, + "acc,none": 0.26616834579135523, + "acc_stderr,none": "N/A", + "sample_count": { + "acc,none": 3077 + } + }, + "mmlu_humanities": { + "alias": "humanities", + "name": "mmlu_humanities", + "sample_len": 4705, + "acc,none": 0.25781083953241235, + "acc_stderr,none": "N/A", + "sample_count": { + "acc,none": 4705 + } + }, + "mmlu": { + "alias": "mmlu", + "name": "mmlu", + "sample_len": 14042, + "acc,none": 0.2650619569861843, + "acc_stderr,none": "N/A", + "sample_count": { + "acc,none": 14042 + } + }, + "cmmlu": { + "alias": "cmmlu", + "name": "cmmlu", + "sample_len": 11582, + "acc,none": 0.2644620963564151, + "acc_stderr,none": "N/A", + "acc_norm,none": 0.2644620963564151, + "acc_norm_stderr,none": "N/A", + "sample_count": { + "acc,none": 11582, + "acc_norm,none": 11582 + } + } + }, + "lm_eval_versions": { + "arc_challenge": 1.0, + "arc_easy": 1.0, + "cmmlu": "1.0", + "cmmlu_agronomy": 1.0, + "cmmlu_anatomy": 1.0, + "cmmlu_ancient_chinese": 1.0, + "cmmlu_arts": 1.0, + "cmmlu_astronomy": 1.0, + "cmmlu_business_ethics": 1.0, + "cmmlu_chinese_civil_service_exam": 1.0, + "cmmlu_chinese_driving_rule": 1.0, + "cmmlu_chinese_food_culture": 1.0, + "cmmlu_chinese_foreign_policy": 1.0, + "cmmlu_chinese_history": 1.0, + "cmmlu_chinese_literature": 1.0, + "cmmlu_chinese_teacher_qualification": 1.0, + "cmmlu_clinical_knowledge": 1.0, + "cmmlu_college_actuarial_science": 1.0, + "cmmlu_college_education": 1.0, + "cmmlu_college_engineering_hydrology": 1.0, + "cmmlu_college_law": 1.0, + "cmmlu_college_mathematics": 1.0, + "cmmlu_college_medical_statistics": 1.0, + "cmmlu_college_medicine": 1.0, + "cmmlu_computer_science": 1.0, + "cmmlu_computer_security": 1.0, + "cmmlu_conceptual_physics": 1.0, + "cmmlu_construction_project_management": 1.0, + "cmmlu_economics": 1.0, + "cmmlu_education": 1.0, + "cmmlu_electrical_engineering": 1.0, + "cmmlu_elementary_chinese": 1.0, + "cmmlu_elementary_commonsense": 1.0, + "cmmlu_elementary_information_and_technology": 1.0, + "cmmlu_elementary_mathematics": 1.0, + "cmmlu_ethnology": 1.0, + "cmmlu_food_science": 1.0, + "cmmlu_genetics": 1.0, + "cmmlu_global_facts": 1.0, + "cmmlu_high_school_biology": 1.0, + "cmmlu_high_school_chemistry": 1.0, + "cmmlu_high_school_geography": 1.0, + "cmmlu_high_school_mathematics": 1.0, + "cmmlu_high_school_physics": 1.0, + "cmmlu_high_school_politics": 1.0, + "cmmlu_human_sexuality": 1.0, + "cmmlu_international_law": 1.0, + "cmmlu_journalism": 1.0, + "cmmlu_jurisprudence": 1.0, + "cmmlu_legal_and_moral_basis": 1.0, + "cmmlu_logical": 1.0, + "cmmlu_machine_learning": 1.0, + "cmmlu_management": 1.0, + "cmmlu_marketing": 1.0, + "cmmlu_marxist_theory": 1.0, + "cmmlu_modern_chinese": 1.0, + "cmmlu_nutrition": 1.0, + "cmmlu_philosophy": 1.0, + "cmmlu_professional_accounting": 1.0, + "cmmlu_professional_law": 1.0, + "cmmlu_professional_medicine": 1.0, + "cmmlu_professional_psychology": 1.0, + "cmmlu_public_relations": 1.0, + "cmmlu_security_study": 1.0, + "cmmlu_sociology": 1.0, + "cmmlu_sports_science": 1.0, + "cmmlu_traditional_chinese_medicine": 1.0, + "cmmlu_virology": 1.0, + "cmmlu_world_history": 1.0, + "cmmlu_world_religions": 1.0, + "hellaswag": 1.0, + "mmlu": "2", + "mmlu_abstract_algebra": 1.0, + "mmlu_anatomy": 1.0, + "mmlu_astronomy": 1.0, + "mmlu_business_ethics": 1.0, + "mmlu_clinical_knowledge": 1.0, + "mmlu_college_biology": 1.0, + "mmlu_college_chemistry": 1.0, + "mmlu_college_computer_science": 1.0, + "mmlu_college_mathematics": 1.0, + "mmlu_college_medicine": 1.0, + "mmlu_college_physics": 1.0, + "mmlu_computer_security": 1.0, + "mmlu_conceptual_physics": 1.0, + "mmlu_econometrics": 1.0, + "mmlu_electrical_engineering": 1.0, + "mmlu_elementary_mathematics": 1.0, + "mmlu_formal_logic": 1.0, + "mmlu_global_facts": 1.0, + "mmlu_high_school_biology": 1.0, + "mmlu_high_school_chemistry": 1.0, + "mmlu_high_school_computer_science": 1.0, + "mmlu_high_school_european_history": 1.0, + "mmlu_high_school_geography": 1.0, + "mmlu_high_school_government_and_politics": 1.0, + "mmlu_high_school_macroeconomics": 1.0, + "mmlu_high_school_mathematics": 1.0, + "mmlu_high_school_microeconomics": 1.0, + "mmlu_high_school_physics": 1.0, + "mmlu_high_school_psychology": 1.0, + "mmlu_high_school_statistics": 1.0, + "mmlu_high_school_us_history": 1.0, + "mmlu_high_school_world_history": 1.0, + "mmlu_human_aging": 1.0, + "mmlu_human_sexuality": 1.0, + "mmlu_humanities": "2", + "mmlu_international_law": 1.0, + "mmlu_jurisprudence": 1.0, + "mmlu_logical_fallacies": 1.0, + "mmlu_machine_learning": 1.0, + "mmlu_management": 1.0, + "mmlu_marketing": 1.0, + "mmlu_medical_genetics": 1.0, + "mmlu_miscellaneous": 1.0, + "mmlu_moral_disputes": 1.0, + "mmlu_moral_scenarios": 1.0, + "mmlu_nutrition": 1.0, + "mmlu_other": "2", + "mmlu_philosophy": 1.0, + "mmlu_prehistory": 1.0, + "mmlu_professional_accounting": 1.0, + "mmlu_professional_law": 1.0, + "mmlu_professional_medicine": 1.0, + "mmlu_professional_psychology": 1.0, + "mmlu_public_relations": 1.0, + "mmlu_security_studies": 1.0, + "mmlu_social_sciences": "2", + "mmlu_sociology": 1.0, + "mmlu_stem": "2", + "mmlu_us_foreign_policy": 1.0, + "mmlu_virology": 1.0, + "mmlu_world_religions": 1.0, + "openbookqa": 1.0, + "piqa": 1.0, + "sciq": 1.0, + "winogrande": 1.0 + }, + "lm_eval_config": { + "model": "/workspace/models/qwen3-8b-base-49e3418", + "model_args": null, + "model_num_parameters": 6454217472, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto:4", + "batch_sizes": [ + 28, + 64, + 64, + 64 + ], + "device": null, + "use_cache": null, + "limit": null, + "bootstrap_iters": 0, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_opencompass5.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_opencompass5.json new file mode 100644 index 0000000000000000000000000000000000000000..1763b564ca7a55bd20e5908a4eceb437d5b557b4 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_opencompass5.json @@ -0,0 +1,44 @@ +{ + "model": "/workspace/models/qwen3-8b-base-49e3418", + "revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "compact_stage": "/workspace/artifacts/qwen3-base-nofit-merge4-20261011/full/wb/compact/stage_09", + "cycle": 9, + "anchor": 12, + "tokenizer": "/workspace/models/qwen3-8b-base-49e3418", + "protocol": "OpenCompass 0.5.4 canonical PPL prompts; full-prompt mean CE; lowest PPL", + "max_batch": 32, + "token_budget": 8192, + "limit": null, + "results": { + "race-middle": { + "accuracy": 0.2736768802228412, + "correct": 393, + "samples": 1436, + "choices": 5744 + }, + "race-high": { + "accuracy": 0.2835906232132647, + "correct": 992, + "samples": 3498, + "choices": 13992 + }, + "C3": { + "accuracy": 0.5068493150684932, + "correct": 925, + "samples": 1825, + "choices": 7300 + }, + "cmnli": { + "accuracy": 0.3301200882280859, + "correct": 4041, + "samples": 12241, + "choices": 36723 + }, + "chid-dev": { + "accuracy": 0.7574257425742574, + "correct": 153, + "samples": 202, + "choices": 1414 + } + } +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_ppl.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_ppl.json new file mode 100644 index 0000000000000000000000000000000000000000..c1e1542cdf91784b830ef6e3ac84ffbc9c3d172f --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/final_ppl.json @@ -0,0 +1,40 @@ +{ + "model": "/workspace/models/qwen3-8b-base-49e3418", + "revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "compact_stage": "/workspace/artifacts/qwen3-base-nofit-merge4-20261011/full/wb/compact/stage_09", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "offline_policy": { + "wikitext_cache": "/workspace/artifacts/qwen3-base-nofit-merge4-20261011/wikitext2_cache", + "dataset": "Salesforce/wikitext", + "config": "wikitext-2-raw-v1", + "splits": { + "test": 4358, + "train": 36718, + "validation": 3760 + }, + "fingerprints": { + "test": "a46124b21ac53738", + "train": "5d4fb603254a7a5b", + "validation": "51d84d52d095f90d" + } + }, + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 1, + "dtype": "bfloat16", + "metrics": { + "nll": 2.8299697432024726, + "perplexity": 16.944948116920326, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 12, + "cycles": 9, + "original_layers": 36, + "decoder_layers": 27, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/local_mse.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/local_mse.json new file mode 100644 index 0000000000000000000000000000000000000000..5d754238f897f35d6fb55da605ee4e5deeaaf4e0 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/eval/local_mse.json @@ -0,0 +1,28 @@ +{ + "local_mse": 30.29728989404966, + "local_nmse": 0.552240670480525, + "squared_error_sum": 8132866828.273412, + "target_energy_sum": 14727033452.99922, + "elements": 268435456, + "protocol": { + "model": "Qwen/Qwen3-8B-Base", + "revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "teacher": "common_original_dense", + "dense_input_layer": 12, + "dense_output_layer": 21, + "student_output_layer": 12, + "sequences": 64, + "sequence_length": 1024, + "valid_tokens": 65536, + "metric": "sum float32 squared activation differences / valid tokens / hidden width", + "dtype": "bfloat16 eager; float32 differences; float64 aggregate", + "training": false, + "input_ids_sha256": "3f1977a32c7a533356dbc14d142c64b470e67dc9942344ff2942e3b1f8b9e2a1", + "anchor_inputs_sha256": "57378a13b898fd63fc40d2c0ccb209ec5d5b636026accc93521b928669200250", + "targets_sha256": "8c246a6cca66306de3cd208e8ec8ef5df8eb9d3e3976f16ad34329cb1216ab40", + "dataset": "WikiText2 test, first64 packed1024 windows, HCHC packing" + }, + "prefix_input_exact_match": true, + "compact_stage": "/workspace/artifacts/qwen3-base-nofit-merge4-20261011/full/wb/compact/stage_09", + "fit": false +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/fisher_reference_local_mse.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/fisher_reference_local_mse.json new file mode 100644 index 0000000000000000000000000000000000000000..1832919bd8ac5aab1ef2f93677c669ebf39dba82 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/fisher_reference_local_mse.json @@ -0,0 +1,28 @@ +{ + "local_mse": 30.159363496315123, + "local_nmse": 0.5497266315473979, + "squared_error_sum": 8095842492.803104, + "target_energy_sum": 14727033452.99922, + "elements": 268435456, + "protocol": { + "model": "Qwen/Qwen3-8B-Base", + "revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "teacher": "common_original_dense", + "dense_input_layer": 12, + "dense_output_layer": 21, + "student_output_layer": 12, + "sequences": 64, + "sequence_length": 1024, + "valid_tokens": 65536, + "metric": "sum float32 squared activation differences / valid tokens / hidden width", + "dtype": "bfloat16 eager; float32 differences; float64 aggregate", + "training": false, + "input_ids_sha256": "3f1977a32c7a533356dbc14d142c64b470e67dc9942344ff2942e3b1f8b9e2a1", + "anchor_inputs_sha256": "57378a13b898fd63fc40d2c0ccb209ec5d5b636026accc93521b928669200250", + "targets_sha256": "8c246a6cca66306de3cd208e8ec8ef5df8eb9d3e3976f16ad34329cb1216ab40", + "dataset": "WikiText2 test, first64 packed1024 windows, HCHC packing" + }, + "prefix_input_exact_match": true, + "compact_stage": "/workspace/artifacts/qwen3-base-nofit-merge4-20261011/fisher_reference/compact/stage_09", + "fit": false +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/fisher_reference_restore_receipt.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/fisher_reference_restore_receipt.json new file mode 100644 index 0000000000000000000000000000000000000000..b2357d3d5ed42b07fe02c9eee721578ebde832ae --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/fisher_reference_restore_receipt.json @@ -0,0 +1,18 @@ +{ + "repo": "LJYAI/paper_revision", + "path": "runs/vast-paper_revision-qwen3-base-nofit-transport-zs6-54974094-20261009/20261009-qwen3-base-nofit-transport-zs6/on", + "revision": "b2a437466b134a6311d3ab625b0b37f9aa1a96d6", + "files": [ + { + "path": "compact/stage_09/anchor_layer.pt", + "size": 385897862, + "sha256": "c02c1a2d0a0031ff0ea1ce99c5138e4099daa415d002c49cd4bec20b31294166" + }, + { + "path": "compact/stage_09/metadata.json", + "size": 681, + "sha256": "36841f8a643f275a86d3fe9d2c1517fda0e3da6dfacc5ab12dbd6070f1699c55" + } + ], + "verified": true +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/inputs/calib.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/inputs/calib.pt new file mode 100644 index 0000000000000000000000000000000000000000..42c3d6f5d034b2acb2da3240dfbf78abb83e124a --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/inputs/calib.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbc1b1fb69a38455c4b8e0f12dade2f81bf883463f3e9fdae5718136508f0a43 +size 1050325 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/inputs/local_mse_evaluation_input_ids.pt b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/inputs/local_mse_evaluation_input_ids.pt new file mode 100644 index 0000000000000000000000000000000000000000..fda57995ed5efacef3c28ed665fd46140a6e84bf --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/inputs/local_mse_evaluation_input_ids.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0ca0b84136692de50e29c6d46197b16f16af642d6b30bb289b0c65f41db61ed +size 2917773 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/inputs/manifest.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/inputs/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..460dfdbcb9b802235868b5d0ac3803cf9a63ffc1 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/inputs/manifest.json @@ -0,0 +1,15 @@ +{ + "model": "Qwen/Qwen3-8B-Base", + "model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "dataset": "DKYoon/SlimPajama-6B", + "revision": "b5f90f419b7489cdba26fdbc8c022fcb5562f968", + "split": "train", + "packing": "historical nonstreaming dataset.shuffle(seed42); concatenate encoded text without separators or BOS", + "samples": 64, + "seq_len": 1024, + "batch_size": 2, + "seed": 42, + "valid_tokens": 65536, + "input_ids_sha256": "85a84b9a68cf0856fd363186d3a273860886b1c17f876cae145fd092454aff55", + "attention_mask_sha256": "b1476de1907209012e63fa502715fd97db26e595352faa808d8c10433a1472a4" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/local_mse_protocol.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/local_mse_protocol.json new file mode 100644 index 0000000000000000000000000000000000000000..ee0c10ff7da7678e030c3857a796852a4dc8bf90 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/local_mse_protocol.json @@ -0,0 +1,18 @@ +{ + "model": "Qwen/Qwen3-8B-Base", + "revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "teacher": "common_original_dense", + "dense_input_layer": 12, + "dense_output_layer": 21, + "student_output_layer": 12, + "sequences": 64, + "sequence_length": 1024, + "valid_tokens": 65536, + "metric": "sum float32 squared activation differences / valid tokens / hidden width", + "dtype": "bfloat16 eager; float32 differences; float64 aggregate", + "training": false, + "input_ids_sha256": "3f1977a32c7a533356dbc14d142c64b470e67dc9942344ff2942e3b1f8b9e2a1", + "anchor_inputs_sha256": "57378a13b898fd63fc40d2c0ccb209ec5d5b636026accc93521b928669200250", + "targets_sha256": "8c246a6cca66306de3cd208e8ec8ef5df8eb9d3e3976f16ad34329cb1216ab40", + "dataset": "WikiText2 test, first64 packed1024 windows, HCHC packing" +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/logs/eval14.log b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/logs/eval14.log new file mode 100644 index 0000000000000000000000000000000000000000..fe9b378bf5685a0901713201c3c11d2b946b8b1c --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/logs/eval14.log @@ -0,0 +1,496 @@ +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/399 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-8b-base-49e3418", + "revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "compact_stage": "/workspace/artifacts/qwen3-base-nofit-merge4-20261011/full/wb/compact/stage_09", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "offline_policy": { + "wikitext_cache": "/workspace/artifacts/qwen3-base-nofit-merge4-20261011/wikitext2_cache", + "dataset": "Salesforce/wikitext", + "config": "wikitext-2-raw-v1", + "splits": { + "test": 4358, + "train": 36718, + "validation": 3760 + }, + "fingerprints": { + "test": "a46124b21ac53738", + "train": "5d4fb603254a7a5b", + "validation": "51d84d52d095f90d" + } + }, + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 1, + "dtype": "bfloat16", + "metrics": { + "nll": 2.8299697432024726, + "perplexity": 16.944948116920326, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 12, + "cycles": 9, + "original_layers": 36, + "decoder_layers": 27, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/logs/prune.log b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/logs/prune.log new file mode 100644 index 0000000000000000000000000000000000000000..5f4fc4705941b1a78232b020096b79744076ae59 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/logs/prune.log @@ -0,0 +1,187 @@ + Loading weights: 0%| | 0/399 [00:00= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:304.) + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) +/opt/conda/lib/python3.11/site-packages/torch/nn/modules/linear.py:134: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:304.) + return F.linear(input, self.weight, self.bias) +/workspace/external/qwen3-base-nofit-merge4/src/source_feature_transport.py:185: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:304.) + gram_sum.addmm_(x_a.transpose(0, 1), x_a) +/workspace/external/qwen3-base-nofit-merge4/src/source_feature_transport.py:186: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:304.) + cross_sum.addmm_(x_a.transpose(0, 1), x_b) +/workspace/external/qwen3-base-nofit-merge4/src/source_feature_transport.py:131: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:304.) + transported = original[start:end].float() @ solution.transpose(0, 1) +[source-feature-transport] cycle=1 batches=32 tokens=65536 nmse=0.127982 cosine=0.933820 norm_ratio=0.932959 +[source-feature-transport] contextual-fisher cycle=1 anchor_nll=0.000000 source_nll=0.000000 +[timing] Cycle 1 component wall-clock: +[timing] selection: 0.000s (0.00% of cycle) +[timing] rebase: 0.000s (0.00% of cycle) +[timing] fisher: 0.000s (0.00% of cycle) +[timing] fit: 2.747s (33.42% of cycle) +[timing] materialize_save_layer: 0.639s (7.77% of cycle) +[timing] metadata_write: 0.000s (0.01% of cycle) +[timing] teacher_snapshot: 0.115s (1.40% of cycle) +[timing] cycle_cleanup: 0.237s (2.88% of cycle) +[timing] cycle_total: 8.220s +[timing] tracked_components_total: 8.219s +[progressive] Cycle 2/9 +NOFIT_FIXED_SELECTION 2 wb 0.0 +[fuse] Head permutation disabled; merging with original head order. +[source-feature-transport] cycle=2 batches=32 tokens=65536 nmse=0.100997 cosine=0.948158 norm_ratio=0.947274 +[source-feature-transport] contextual-fisher cycle=2 anchor_nll=0.000000 source_nll=0.000000 +[timing] Cycle 2 component wall-clock: +[timing] selection: 0.000s (0.00% of cycle) +[timing] rebase: 0.000s (0.00% of cycle) +[timing] fisher: 0.000s (0.00% of cycle) +[timing] fit: 2.758s (33.84% of cycle) +[timing] materialize_save_layer: 0.630s (7.73% of cycle) +[timing] metadata_write: 0.001s (0.01% of cycle) +[timing] teacher_snapshot: 0.522s (6.40% of cycle) +[timing] cycle_cleanup: 0.160s (1.97% of cycle) +[timing] cycle_total: 8.152s +[timing] tracked_components_total: 8.152s +[progressive] Cycle 3/9 +NOFIT_FIXED_SELECTION 3 wb 0.0 +[fuse] Head permutation disabled; merging with original head order. +[source-feature-transport] cycle=3 batches=32 tokens=65536 nmse=0.110389 cosine=0.943192 norm_ratio=0.942294 +[source-feature-transport] contextual-fisher cycle=3 anchor_nll=0.000000 source_nll=0.000000 +[timing] Cycle 3 component wall-clock: +[timing] selection: 0.000s (0.00% of cycle) +[timing] rebase: 0.000s (0.00% of cycle) +[timing] fisher: 0.000s (0.00% of cycle) +[timing] fit: 2.726s (33.78% of cycle) +[timing] materialize_save_layer: 0.644s (7.98% of cycle) +[timing] metadata_write: 0.000s (0.01% of cycle) +[timing] teacher_snapshot: 0.508s (6.29% of cycle) +[timing] cycle_cleanup: 0.162s (2.01% of cycle) +[timing] cycle_total: 8.071s +[timing] tracked_components_total: 8.070s +[progressive] Cycle 4/9 +NOFIT_FIXED_SELECTION 4 wb 0.0 +[fuse] Head permutation disabled; merging with original head order. +[source-feature-transport] cycle=4 batches=32 tokens=65536 nmse=0.098665 cosine=0.949387 norm_ratio=0.948496 +[source-feature-transport] contextual-fisher cycle=4 anchor_nll=0.000000 source_nll=0.000000 +[timing] Cycle 4 component wall-clock: +[timing] selection: 0.000s (0.00% of cycle) +[timing] rebase: 0.000s (0.00% of cycle) +[timing] fisher: 0.000s (0.00% of cycle) +[timing] fit: 2.703s (33.66% of cycle) +[timing] materialize_save_layer: 0.651s (8.10% of cycle) +[timing] metadata_write: 0.000s (0.01% of cycle) +[timing] teacher_snapshot: 0.481s (5.99% of cycle) +[timing] cycle_cleanup: 0.179s (2.23% of cycle) +[timing] cycle_total: 8.030s +[timing] tracked_components_total: 8.029s +[progressive] Cycle 5/9 +NOFIT_FIXED_SELECTION 5 wb 0.0 +[fuse] Head permutation disabled; merging with original head order. +[source-feature-transport] cycle=5 batches=32 tokens=65536 nmse=0.099178 cosine=0.949117 norm_ratio=0.948243 +[source-feature-transport] contextual-fisher cycle=5 anchor_nll=0.000000 source_nll=0.000000 +[timing] Cycle 5 component wall-clock: +[timing] selection: 0.000s (0.00% of cycle) +[timing] rebase: 0.000s (0.00% of cycle) +[timing] fisher: 0.000s (0.00% of cycle) +[timing] fit: 2.699s (33.36% of cycle) +[timing] materialize_save_layer: 0.645s (7.97% of cycle) +[timing] metadata_write: 0.000s (0.01% of cycle) +[timing] teacher_snapshot: 0.506s (6.26% of cycle) +[timing] cycle_cleanup: 0.168s (2.08% of cycle) +[timing] cycle_total: 8.089s +[timing] tracked_components_total: 8.089s +[progressive] Cycle 6/9 +NOFIT_FIXED_SELECTION 6 wb 0.0 +[fuse] Head permutation disabled; merging with original head order. +[source-feature-transport] cycle=6 batches=32 tokens=65536 nmse=0.098204 cosine=0.949630 norm_ratio=0.948736 +[source-feature-transport] contextual-fisher cycle=6 anchor_nll=0.000000 source_nll=0.000000 +[timing] Cycle 6 component wall-clock: +[timing] selection: 0.000s (0.00% of cycle) +[timing] rebase: 0.000s (0.00% of cycle) +[timing] fisher: 0.000s (0.00% of cycle) +[timing] fit: 2.704s (33.79% of cycle) +[timing] materialize_save_layer: 0.642s (8.03% of cycle) +[timing] metadata_write: 0.000s (0.01% of cycle) +[timing] teacher_snapshot: 0.420s (5.24% of cycle) +[timing] cycle_cleanup: 0.207s (2.58% of cycle) +[timing] cycle_total: 8.002s +[timing] tracked_components_total: 8.001s +[progressive] Cycle 7/9 +NOFIT_FIXED_SELECTION 7 wb 0.0 +[fuse] Head permutation disabled; merging with original head order. +[source-feature-transport] cycle=7 batches=32 tokens=65536 nmse=0.103606 cosine=0.946781 norm_ratio=0.945933 +[source-feature-transport] contextual-fisher cycle=7 anchor_nll=0.000000 source_nll=0.000000 +[timing] Cycle 7 component wall-clock: +[timing] selection: 0.000s (0.00% of cycle) +[timing] rebase: 0.000s (0.00% of cycle) +[timing] fisher: 0.000s (0.00% of cycle) +[timing] fit: 2.733s (33.99% of cycle) +[timing] materialize_save_layer: 0.651s (8.10% of cycle) +[timing] metadata_write: 0.000s (0.01% of cycle) +[timing] teacher_snapshot: 0.443s (5.51% of cycle) +[timing] cycle_cleanup: 0.164s (2.04% of cycle) +[timing] cycle_total: 8.040s +[timing] tracked_components_total: 8.039s +[progressive] Cycle 8/9 +NOFIT_FIXED_SELECTION 8 wb 0.0 +[fuse] Head permutation disabled; merging with original head order. +[source-feature-transport] cycle=8 batches=32 tokens=65536 nmse=0.106065 cosine=0.945482 norm_ratio=0.944619 +[source-feature-transport] contextual-fisher cycle=8 anchor_nll=0.000000 source_nll=0.000000 +[timing] Cycle 8 component wall-clock: +[timing] selection: 0.000s (0.00% of cycle) +[timing] rebase: 0.000s (0.00% of cycle) +[timing] fisher: 0.000s (0.00% of cycle) +[timing] fit: 2.705s (33.91% of cycle) +[timing] materialize_save_layer: 0.649s (8.13% of cycle) +[timing] metadata_write: 0.000s (0.01% of cycle) +[timing] teacher_snapshot: 0.406s (5.09% of cycle) +[timing] cycle_cleanup: 0.169s (2.11% of cycle) +[timing] cycle_total: 7.978s +[timing] tracked_components_total: 7.978s +[progressive] Cycle 9/9 +NOFIT_FIXED_SELECTION 9 wb 0.0 +[fuse] Head permutation disabled; merging with original head order. +[source-feature-transport] cycle=9 batches=32 tokens=65536 nmse=0.139792 cosine=0.927474 norm_ratio=0.926631 +[source-feature-transport] contextual-fisher cycle=9 anchor_nll=0.000000 source_nll=0.000000 +[timing] Cycle 9 component wall-clock: +[timing] selection: 0.000s (0.00% of cycle) +[timing] rebase: 0.000s (0.00% of cycle) +[timing] fisher: 0.000s (0.00% of cycle) +[timing] fit: 2.698s (33.63% of cycle) +[timing] materialize_save_layer: 0.647s (8.06% of cycle) +[timing] metadata_write: 0.000s (0.01% of cycle) +[timing] teacher_snapshot: 0.441s (5.50% of cycle) +[timing] cycle_cleanup: 0.177s (2.20% of cycle) +[timing] cycle_total: 8.022s +[timing] tracked_components_total: 8.021s +[save] Skipping final pre-LoRA weight save (compact_only). +[save] Skipping final model weight save (compact_only). +[progressive] Completed 9 cycles. Compact weights written to /workspace/artifacts/qwen3-base-nofit-merge4-20261011/full/wb/run; final dense model save skipped. +NOFIT_ALL_CYCLES_COMPLETE 9 diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/manifest.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d967c79e721f3cbafa69cec05a7c7affed0b7795 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/manifest.json @@ -0,0 +1,517 @@ +{ + "schema": "sha256-tree-v1", + "files": [ + { + "path": "audit/cycle_01_merge.json", + "size": 6465, + "sha256": "0832ec37ec0efb80bf9c464b7ea134d3089eeeda2ba0abb06b64685ce50d2e30" + }, + { + "path": "audit/cycle_01_transport.pt", + "size": 67110518, + "sha256": "ce29905e20c096a8d0a555134ba437d93e2e81d3e91cbfdff0b3e62513583bbb" + }, + { + "path": "audit/cycle_02_merge.json", + "size": 6466, + "sha256": "3081953ee7571c5b47f3e71cc4910afbddcfbdec18c7da8a0e43e96b6391ba88" + }, + { + "path": "audit/cycle_02_transport.pt", + "size": 67110518, + "sha256": "771567c95fea3376299e7efde066b56970e55b94ec143fe51c9236dce5599fc0" + }, + { + "path": "audit/cycle_03_merge.json", + "size": 6464, + "sha256": "870eb0acd5d813a14a831b894cb1d74974611ae5bd60a1b864b0ee96508c36e4" + }, + { + "path": "audit/cycle_03_transport.pt", + "size": 67110518, + "sha256": "092a56f0df3e4eb2717935143213ae33bb0fc618abe15caefafdc39ff542cc90" + }, + { + "path": "audit/cycle_04_merge.json", + "size": 6466, + "sha256": "fb1bd9c06b9ba08d9a738727ca37f12b8975984b312a57be688efebd3536847e" + }, + { + "path": "audit/cycle_04_transport.pt", + "size": 67110518, + "sha256": "030b90624e604f764c93dc04a03b3b58095bbc3d2b1a21e22226d92b950f95d9" + }, + { + "path": "audit/cycle_05_merge.json", + "size": 6465, + "sha256": "dbcba457aaadfe55407ce1c3bf1de13e8c956fba6dc94d146c82588adc3615c6" + }, + { + "path": "audit/cycle_05_transport.pt", + "size": 67110518, + "sha256": "c6889d5ef721bde145df816af6b64c40e1f943e4ddd06411a06a3196a78d20c2" + }, + { + "path": "audit/cycle_06_merge.json", + "size": 6466, + "sha256": "e4283bc2b6bfc00c72c2486b20a4cb083de7a62c9624c9a2a00d3637cb89a65d" + }, + { + "path": "audit/cycle_06_transport.pt", + "size": 67110518, + "sha256": "b1400002d389e07d4a0bb71829c43f6aa137b91935e5ef07a925fea5f5040cee" + }, + { + "path": "audit/cycle_07_merge.json", + "size": 6463, + "sha256": "814ed5282f0126ba9204842c3a4b3ba65a94890458edfc08b882ac899c2e8b31" + }, + { + "path": "audit/cycle_07_transport.pt", + "size": 67110518, + "sha256": "d501ae2b2629df66facd8e4c516ff89d4312221d3383c886067b857ce6f86764" + }, + { + "path": "audit/cycle_08_merge.json", + "size": 6465, + "sha256": "a0480b31dcce080504e4445ec42f3abc68788c10cebf670f303d663858807091" + }, + { + "path": "audit/cycle_08_transport.pt", + "size": 67110518, + "sha256": "43deee7cc5037a42f1952463af7065212e8cfabff7be44e11268becab95c3bb0" + }, + { + "path": "audit/cycle_09_merge.json", + "size": 6464, + "sha256": "67d7c08fe6f5618310e3fde178f6db656e674e672ce2e2e2955f1a67acf54496" + }, + { + "path": "audit/cycle_09_transport.pt", + "size": 67110518, + "sha256": "7f785d329bfa8289c73722d71a8d82628c2c22f8a25f05a7201cc3f792ec3481" + }, + { + "path": "audit/data_manifest.json", + "size": 610, + "sha256": "351ffe83205ea1f03e9f6c3fb619df647d24e0b319e788df8e82f807cac830b9" + }, + { + "path": "audit/selection.json", + "size": 60176, + "sha256": "74803d5ac6e927edd843bb92339d950800f93ba7350b7a670cde0f2d6a8833d6" + }, + { + "path": "code/2026-10-11_qwen3-base-nofit-merge4.md", + "size": 8800, + "sha256": "aee63dae5b4ccdbd00d3df36aaea77831148baaea7b524d37a93e05bacac6ce4" + }, + { + "path": "code/abprune_compact.py", + "size": 12360, + "sha256": "34f00dc3b96ea3c3dd6dafada682352e2b332e6b1c79fb2033e13f9585254ff3" + }, + { + "path": "code/abprune_stagewise_geometry.py", + "size": 11292, + "sha256": "b616899132d69bff0056a8573f49f62cbe0f0ad99a6e75aa0ec7f2b8a47c9fe0" + }, + { + "path": "code/aggregate_transport_rho_fit_14bench.py", + "size": 1892, + "sha256": "551c83e8e313bfaecd6d052c3a0e8e45142c21b8124acf86bc983cfb7ed7f457" + }, + { + "path": "code/bootstrap_qwen3_nofit_bench8.sh", + "size": 3393, + "sha256": "14fef8b8339f11a79fd0626d796ebab929500a42c5bb960effaa5f14e74a1145" + }, + { + "path": "code/bootstrap_qwen3_nofit_merge4.sh", + "size": 3592, + "sha256": "9f62603ccf9ef05d47b484d20f39c011b61144bac97157c1a4cdf1e71d243ce4" + }, + { + "path": "code/common_lm_data.py", + "size": 20467, + "sha256": "bca8aae6c3a32a39f8461fd100a3b004e87603b1d1f454a0a8557bed08696e3b" + }, + { + "path": "code/dm500_rows.py", + "size": 4712, + "sha256": "2aac800856d3602aab0887550562e30eed706fc39574fa771ada81152a9cc034" + }, + { + "path": "code/eval_llama31_8b_12bench.py", + "size": 6694, + "sha256": "94f6ca3e633f10b5cdef756c1e88e32cdf1615a8f17d078918c0b412a445fb37" + }, + { + "path": "code/eval_llama31_fit_ablation_one.sh", + "size": 3118, + "sha256": "ae03668f8f4aae078813858b17aada4d0cc828a848d4321036736bf12ca665dd" + }, + { + "path": "code/eval_opencompass_ppl_fast.py", + "size": 9619, + "sha256": "e1bf9a9c3e658d3205ec6210c87eb2d07d55f428e6dec055d8160251d4e0ba59" + }, + { + "path": "code/evaluate_abprune_compact_ppl.py", + "size": 7038, + "sha256": "916db0545ee18ba39dbbb0af9ce4f379a1ddef0b54d0477caf402ee24421a050" + }, + { + "path": "code/evaluate_hchc_wikitext2_ppl.py", + "size": 5422, + "sha256": "a1c1c0bf8422ddc70cc5e9c0049830b9aa594742fdeffa110de7b3dcf30c1dd3" + }, + { + "path": "code/fit_ablation_mka.py", + "size": 19854, + "sha256": "05679b074c4e8dda5f902598c0115e538128da42dfe75c5402f4adf84262da03" + }, + { + "path": "code/fixed_span_merge_init.py", + "size": 2940, + "sha256": "c12a7d203b20231066525c076a789b9f91aabf569d981202c018597360c261ed" + }, + { + "path": "code/fuse_layers.py", + "size": 131351, + "sha256": "d2bca0ca9b5bc42da7a29cd5a14f860821486625cebd9da60d01521f07a3f6dc" + }, + { + "path": "code/fuse_layers_distill.py", + "size": 148110, + "sha256": "d0f2ffcb021e486b9462b45fd7507d492a47ffa5f803768a9bc856721e4be16c" + }, + { + "path": "code/launch_qwen3_nofit_bench8.sh", + "size": 4599, + "sha256": "7e39f742327bb6e6b8ec167fec02dc53b8d195ff3f0f2a3a7f76b25a29832eda" + }, + { + "path": "code/launch_qwen3_nofit_merge4.sh", + "size": 4432, + "sha256": "45a8eed6aba7b6dee6e562c72a0af19de2240a5513cd000fa609794b02e1df27" + }, + { + "path": "code/launch_qwen3_nofit_transport_zs6.sh", + "size": 6783, + "sha256": "8e24c31cb23b2a768f0f109d17ce10673354f3584f3785654ea1acf1aedd78a2" + }, + { + "path": "code/monitor_qwen3_nofit_bench8.ps1", + "size": 3431, + "sha256": "c3cb4022cd1d1be1352f9725ca2f831cfe1419f33f61aac060e9ef9101955423" + }, + { + "path": "code/monitor_qwen3_nofit_merge4.ps1", + "size": 3198, + "sha256": "8a712ab748d3ae4d8823581e7554cede61b2109af5f75e5f9e6c44c2e1b40106" + }, + { + "path": "code/monitor_qwen3_nofit_transport_zs6.ps1", + "size": 4417, + "sha256": "c3c23490b9773b0904ace353bf6d83d9037c38b5402274b3ab0a5d881fb6db71" + }, + { + "path": "code/nofit_merge_control.py", + "size": 2237, + "sha256": "8ab597afa87424fb699934aa64eb9a740d5172003501776f399f0026261eafd7" + }, + { + "path": "code/offline_eval_policy.py", + "size": 1001, + "sha256": "6d4b2b315573cc308ada19e24b81166d10a1dd4e2bcf0a1f3799229a6d58135a" + }, + { + "path": "code/package_qwen3_nofit_merge_backup.py", + "size": 3039, + "sha256": "5be38cb201181c0856cc96e05b52cd0ed1bfa5193ee699834fbeb65cb8389e77" + }, + { + "path": "code/prepare_lmeval9_local_cache.py", + "size": 4706, + "sha256": "b9d7cfa46fa42cc017f53c5fffa34832756246377c3586484c0c54a53a00de8b" + }, + { + "path": "code/prepare_qwen3_nofit_merge_inputs.py", + "size": 1899, + "sha256": "f3938f4c2381449e23aa3ce9ff85b7703618bd5d53d96bbc4c5f6caa535ec197" + }, + { + "path": "code/project_commit.txt", + "size": 41, + "sha256": "8906e06db63748514b9aa3c9d752ac3de3827edeaa522e7da814e57fe7f83042" + }, + { + "path": "code/qwen3_nofit_local_mse.py", + "size": 5164, + "sha256": "db4bae16b8a924cfc8e11c240024678bb8a38f632f6a03b301c77461acc709eb" + }, + { + "path": "code/qwen3_nofit_transport_config.sh", + "size": 1692, + "sha256": "e7e011c251f1b77b66d123a14b91631b00a2bb7d0e09ac99de4c41cd1c1bd17b" + }, + { + "path": "code/run_qwen3_nofit_merge_control.py", + "size": 6831, + "sha256": "16f2c0ed207b261ecca8116795543fa7e2f65e3880f84fc7986ef9a450711030" + }, + { + "path": "code/run_qwen3_nofit_transport_arm.sh", + "size": 2519, + "sha256": "5d47c86936ec57c442a57184158ee7082668fc3c091eba80599c6b60d44d4903" + }, + { + "path": "code/run_qwen3_nofit_transport_control.py", + "size": 4714, + "sha256": "78d96f3ccdc1226506fdf93e9c4da0ec0dd9b1aff6336a99921b60d5d1a36fa6" + }, + { + "path": "code/run_qwen3_nofit_transport_ppl.sh", + "size": 4514, + "sha256": "cc050f998634b415ceeafca4949391852960f78fde2efe49157cbf776f63c871" + }, + { + "path": "code/source_feature_transport.py", + "size": 13200, + "sha256": "75809bdbbc9bf73f07adaf1f0c816fc9abd1d9b7c0b8a0d561c96cb0313e243f" + }, + { + "path": "code/summarize_qwen3_nofit_merge.py", + "size": 2603, + "sha256": "a43ec0886823c1dc616718f33760b116f3b8e96f0434a518a31c54bf5de35479" + }, + { + "path": "code/verify_qwen3_nofit_merge.py", + "size": 1416, + "sha256": "1bf3cce4ea287c78168d578a8763c0d12779ea591cd338e9266e893d3f4f51ca" + }, + { + "path": "compact/manifest.json", + "size": 7430, + "sha256": "d9e20b5c3c189263c071ce95fdaa51844ee7aa1f418f207a9baff16cef9205f5" + }, + { + "path": "compact/stage_01/anchor_layer.pt", + "size": 385897862, + "sha256": "a1ac4eff65ee140fd4682b95835767941e8f5a8642facdcbee6c06d518bfac35" + }, + { + "path": "compact/stage_01/metadata.json", + "size": 615, + "sha256": "fc480104e736b68c97399e68f74afe79e494bfc8481b3c9446a049a782b2420c" + }, + { + "path": "compact/stage_02/anchor_layer.pt", + "size": 385897862, + "sha256": "f8201bf31937b7a4c5d843fdc48bc70c0aeab505a8e6170a18803e6a7ce60cda" + }, + { + "path": "compact/stage_02/metadata.json", + "size": 623, + "sha256": "ff794ec68d7c35397a7a888e1c4f62bce5029cad4de26fb56f14c5f3876dde55" + }, + { + "path": "compact/stage_03/anchor_layer.pt", + "size": 385897862, + "sha256": "a114120d0594d9362d96686b8937d63e05cabcee907f134bcf76566ceb66c8fa" + }, + { + "path": "compact/stage_03/metadata.json", + "size": 631, + "sha256": "16ef935dbe430d25aa5ec47bcd46e75b43e50f7e0893498af1255684f2e5ca6c" + }, + { + "path": "compact/stage_04/anchor_layer.pt", + "size": 385897862, + "sha256": "6919d53d037f958c0bc281a2b8c292c51bc35bacc89b72cd8b6dde423f6967eb" + }, + { + "path": "compact/stage_04/metadata.json", + "size": 639, + "sha256": "07925db35d316b831d563c80569df9a266e2faab3430ea75fd1c3a84ac029eca" + }, + { + "path": "compact/stage_05/anchor_layer.pt", + "size": 385897862, + "sha256": "0cf2ae746ca7b97394b4076e8538c734de11781c3e47176d93ab264dd3cc73a7" + }, + { + "path": "compact/stage_05/metadata.json", + "size": 647, + "sha256": "fb52280a7354c99bbc6d1d1a087c0e4ffc704a0e830fcf076d2f72dbbd02004c" + }, + { + "path": "compact/stage_06/anchor_layer.pt", + "size": 385897862, + "sha256": "e41eef6ffeded49d5f41417946580cb54f372ba2d681922fb6b4c4b89258ecb9" + }, + { + "path": "compact/stage_06/metadata.json", + "size": 655, + "sha256": "c1f2745eec8f11c1353b8b876a9c3297eac0dae8f22c415d665b4fd66907146c" + }, + { + "path": "compact/stage_07/anchor_layer.pt", + "size": 385897862, + "sha256": "0e1a7fe2e430feafd208abdbc8aa428aa9ef114c9ad07e592c14c0bac25cb521" + }, + { + "path": "compact/stage_07/metadata.json", + "size": 663, + "sha256": "0155745132a3fa38969a684b4ea503a76af55c7a5e848877871828459d22b94c" + }, + { + "path": "compact/stage_08/anchor_layer.pt", + "size": 385897862, + "sha256": "1ec9742a6d66e18d3084074943ac27a36cc11f5eac281dcc31115f646dc89834" + }, + { + "path": "compact/stage_08/metadata.json", + "size": 671, + "sha256": "c314bd47be1bdf64542a8b025d4a47b7b1a9e01e7c17e862de703e81e8bb0df7" + }, + { + "path": "compact/stage_09/anchor_layer.pt", + "size": 385897862, + "sha256": "c76a5669993504adf2b28615fdfbb4451d8ca276ebdfb58b638aa1eb1c5cfe8c" + }, + { + "path": "compact/stage_09/metadata.json", + "size": 679, + "sha256": "af9b23f7db8f56d1e927c86144a234a261f005cc338a191c41c8ef2e85a16054" + }, + { + "path": "cycle_metadata/cycle_1/cycle_metadata.json", + "size": 11247, + "sha256": "34213531e6e091c4ef4f8fcf49941186c99b1e0a874cb3d493bbc64f6d817031" + }, + { + "path": "cycle_metadata/cycle_2/cycle_metadata.json", + "size": 11250, + "sha256": "e7071baa32ec588659849257a080d0b53d4af8919ed32995a7f5046d9badb0f1" + }, + { + "path": "cycle_metadata/cycle_3/cycle_metadata.json", + "size": 11248, + "sha256": "366077e6d56e77c5d8a591c998fcbbc0c0f6df806653f1c62bd3776bbe7524bf" + }, + { + "path": "cycle_metadata/cycle_4/cycle_metadata.json", + "size": 11253, + "sha256": "738e9736886e1b010daf179b06b32ec2d887e8b08851b8c00171c629b6a26b5b" + }, + { + "path": "cycle_metadata/cycle_5/cycle_metadata.json", + "size": 11248, + "sha256": "af5f9189a0b52be439fb773d10f593834d1a88f68d7fc6420b26e94e5bc381db" + }, + { + "path": "cycle_metadata/cycle_6/cycle_metadata.json", + "size": 11250, + "sha256": "889721d83833a8ebde6d85e61963fd0001b9985e7b063098eeb55b2043a28649" + }, + { + "path": "cycle_metadata/cycle_7/cycle_metadata.json", + "size": 11249, + "sha256": "d0e014a82b20815dae0abe524b56a7fd78864fa70775d687c5f41c5c0459eeb6" + }, + { + "path": "cycle_metadata/cycle_8/cycle_metadata.json", + "size": 11252, + "sha256": "7828538a52807998a6c5c6ccb8327d497c06ba94712e169ed06c0d48ab43f8c5" + }, + { + "path": "cycle_metadata/cycle_9/cycle_metadata.json", + "size": 11249, + "sha256": "09ffaf9655724bda1334d2f135891c5a3c49f022eddea1d623363c7153eadf76" + }, + { + "path": "eval/final_14bench.json", + "size": 886, + "sha256": "75da76f9c2c1978705aae452a7541027f6dd677186cfb56e0bcfff1b9e699d15" + }, + { + "path": "eval/final_lmeval9.json", + "size": 42915, + "sha256": "7f5cdf758683fe658a6fa183de04bf21eee9fa49468366cbb992a86cb4a39c7f" + }, + { + "path": "eval/final_opencompass5.json", + "size": 1121, + "sha256": "69eb8cd2d16c85b4dd814b51aff5c9cbfb83bf1f3a2ba058dfdfedc498b951d6" + }, + { + "path": "eval/final_ppl.json", + "size": 1274, + "sha256": "919f92b795cf0bd9daab904f8568ff52e054939a161670b4ea2088346949aeda" + }, + { + "path": "eval/local_mse.json", + "size": 1194, + "sha256": "2e0d46a6994e0b110cfe77360139135214e782aca41b5939c7b65e2db5d760f8" + }, + { + "path": "fisher_reference_local_mse.json", + "size": 1205, + "sha256": "77f5a28e3f1c9859c748a8c279f9eca514902b4410aa990875c6abace2bde652" + }, + { + "path": "fisher_reference_restore_receipt.json", + "size": 596, + "sha256": "64cc521715a1ab9ffbbfa89b9a826b6af9af4d8d483345499528acdd258b61e9" + }, + { + "path": "inputs/calib.pt", + "size": 1050325, + "sha256": "cbc1b1fb69a38455c4b8e0f12dade2f81bf883463f3e9fdae5718136508f0a43" + }, + { + "path": "inputs/local_mse_evaluation_input_ids.pt", + "size": 2917773, + "sha256": "f0ca0b84136692de50e29c6d46197b16f16af642d6b30bb289b0c65f41db61ed" + }, + { + "path": "inputs/manifest.json", + "size": 610, + "sha256": "351ffe83205ea1f03e9f6c3fb619df647d24e0b319e788df8e82f807cac830b9" + }, + { + "path": "local_mse_protocol.json", + "size": 811, + "sha256": "d53120bd3b3b7c77247c9e35fc43e0ff829bdf4327ace808b4155811aab3a183" + }, + { + "path": "logs/eval14.log", + "size": 294531, + "sha256": "198aeaf1d87e8fb8737d9c6d4fe32af76018e9948828fb1644c833f3cfa1eacc" + }, + { + "path": "logs/local_mse.log", + "size": 1337, + "sha256": "f82772658f8f7210900ef6c35265a2ac07d86dafbf990548920f02418547ffb3" + }, + { + "path": "logs/ppl.log", + "size": 1723, + "sha256": "e3f5de51bb9edd0b0a52cc50be84c33f15d2a9d81505d1821ff92de500c61f5e" + }, + { + "path": "logs/prune.log", + "size": 12799, + "sha256": "5cbe505da82dd087b5cdede87775f454cf2a43ce4058093e195af33d1685856f" + }, + { + "path": "stage_retention.json", + "size": 432, + "sha256": "3280a5aabc0915ce6965715203e1a961c7eb0c704fe652b3c97bc5228fe8acb6" + } + ], + "total_files": 102, + "total_bytes": 4082121511 +} diff --git a/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/stage_retention.json b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/stage_retention.json new file mode 100644 index 0000000000000000000000000000000000000000..6c43d4ede12d53f47b49b2079845f2a4def38df2 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-nofit-merge4-55359231-20261011/20261011-qwen3-base-nofit-merge4/wb/stage_retention.json @@ -0,0 +1,20 @@ +{ + "stages": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ], + "stage0_original_model": "Qwen/Qwen3-8B-Base", + "stage0_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "previous_teacher_reconstruction": "stage(k-1) compact anchor + immutable original unremoved layers", + "original_input_ids_saved": true, + "all_transport_matrices_saved": true, + "intermediate_weights_saved": true, + "fit": false +}