diff --git a/.gitattributes b/.gitattributes index 9012cb94702c275e061276af556d703b8e9390c8..14da1e523d19bbd8474689686fbff827f00a00b7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -240,3 +240,4 @@ runs/vast-paper_revision-qwen3-base-race9-54818226-20261008/20261008-qwen3-base- runs/vast-paper_revision-qwen3-base-race9-54818226-20261008/20261008-qwen3-base-race-lmeval9/transport/race_samples.json filter=lfs diff=lfs merge=lfs -text runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/come/model/tokenizer.json filter=lfs diff=lfs merge=lfs -text runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/mka/model/tokenizer.json filter=lfs diff=lfs merge=lfs -text +runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/abprune_compact.py b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/abprune_compact.py new file mode 100644 index 0000000000000000000000000000000000000000..ffabddabd8ced8b87666f63fb9f99fae8b36cb38 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/abprune_compact.py @@ -0,0 +1,284 @@ +"""Package and reconstruct compact ABPrune stage checkpoints. + +ABPrune already writes ``fused_layer.pt`` for every cycle. This utility makes +those files portable by pairing them with an explicit, immutable stage record; +the dense base model plus that record is sufficient to reconstruct a student. +""" +from __future__ import annotations + +import argparse, hashlib, json, shutil +from pathlib import Path +from typing import Any + +import torch + + +def copy_stage_state(src: Path, dst: Path, *, hard_link: bool = False) -> None: + """Optionally share immutable stage bytes within one experiment filesystem.""" + if not hard_link: + shutil.copy2(src, dst) + elif dst.exists(): + if not src.samefile(dst): + raise FileExistsError(f"Refusing to replace existing non-linked stage: {dst}") + else: + # Fail instead of silently consuming space if cross-device linking is + # unavailable. The caller explicitly selected the storage strategy. + dst.hardlink_to(src) + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(8 * 1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def post_merge_scale(source_cycle: Path) -> dict | None: + """Return the deployable post-merge scale stored for a cycle, if present.""" + metadata_path = source_cycle / "cycle_metadata.json" + if not metadata_path.is_file(): + return None + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + distill = metadata.get("distill", {}) + scale = distill.get("post_merge_nll_scale") or {} + source = "post_merge_nll" + if not scale.get("enabled"): + scale = (distill.get("reparam_stats") or {}).get("residual_scale") or {} + source = "scale_aware_fit" + if not scale.get("enabled"): + return None + gamma = scale["gamma"] + cumulative_gamma = scale["cumulative_gamma"] + if not isinstance(gamma, list): + gamma = float(gamma) + else: + gamma = [float(value) for value in gamma] + if not isinstance(cumulative_gamma, list): + cumulative_gamma = float(cumulative_gamma) + else: + cumulative_gamma = [float(value) for value in cumulative_gamma] + result = { + "enabled": True, + "mode": scale.get("mode", "residual"), + "scale_parameterization": scale.get("scale_parameterization", "scalar"), + "gamma": gamma, + "cumulative_gamma": cumulative_gamma, + "source": source, + } + for field in ( + "optimization_mode", + "gamma_gradient_sources", + "theta_gradient_sources", + "nll_used", + ): + if field in scale: + result[field] = scale[field] + if "cumulative_gamma_statistics" in scale: + result["cumulative_gamma_statistics"] = scale["cumulative_gamma_statistics"] + if "raw" in scale and "ppl" in scale["raw"]: + result["calibration_raw_ppl"] = float(scale["raw"]["ppl"]) + if "fitted" in scale and "ppl" in scale["fitted"]: + result["calibration_fitted_ppl"] = float(scale["fitted"]["ppl"]) + return result + + +def read_compact_metadata(compact_stage: Path) -> dict[str, Any]: + return json.loads((compact_stage / "metadata.json").read_text(encoding="utf-8")) + + +def compact_scale_targets(metadata: dict[str, Any]) -> list[dict[str, Any]]: + """Return residual scales and current student indices for a compact stage.""" + if metadata.get("format") == "abprune_compact_stage_v2": + active = [int(value) for value in metadata["active_original_layers"]] + current_index = {original: index for index, original in enumerate(active)} + targets = [] + for state in metadata.get("retained_layer_states", []): + scale = state.get("post_merge_scale") or {} + if scale.get("enabled") and scale.get("mode") == "residual": + original = int(state["original_layer"]) + targets.append( + { + "original_layer": original, + "student_index": current_index[original], + "scale": scale, + } + ) + return targets + + scale = metadata.get("post_merge_scale") or {} + if scale.get("enabled") and scale.get("mode") == "residual": + return [ + { + "original_layer": int(metadata["anchor_index"]), + "student_index": int(metadata["anchor_index"]), + "scale": scale, + } + ] + return [] + + +def _package_topology_aware(args, source: Path, out: Path, order: list[int]) -> None: + """Package cycles whose recipients may switch between pair boundaries.""" + if int(getattr(args, "start_cycle", 1)) != 1: + raise ValueError("topology-aware compact packaging must start at cycle 1") + + removed: set[int] = set() + latest_states: dict[int, dict[str, Any]] = {} + stages: list[dict[str, Any]] = [] + original_layer_count: int | None = None + + for cycle in range(1, args.cycles + 1): + source_cycle = source / f"cycle_{cycle}" + src = source_cycle / "fused_layer.pt" + if not src.is_file(): + raise FileNotFoundError(src) + cycle_metadata_path = source_cycle / "cycle_metadata.json" + if not cycle_metadata_path.is_file(): + raise FileNotFoundError(cycle_metadata_path) + cycle_metadata = json.loads(cycle_metadata_path.read_text(encoding="utf-8")) + recipient = int(cycle_metadata["recipient_original_layer"]) + removed_original = int(cycle_metadata["removed_original_layer"]) + recipient_side = str(cycle_metadata["recipient_side"]) + if recipient_side not in {"left", "right"}: + raise ValueError(f"invalid recipient_side at cycle {cycle}: {recipient_side}") + if original_layer_count is None: + original_layer_count = int(cycle_metadata["num_layers_before"]) + if removed_original in removed: + raise ValueError(f"original layer {removed_original} removed more than once") + if recipient in removed: + raise ValueError(f"cycle {cycle} selects removed recipient {recipient}") + + removed.add(removed_original) + latest_states.pop(removed_original, None) + latest_states[recipient] = { + "original_layer": recipient, + "source_cycle": cycle, + "recipient_side": recipient_side, + "source_state": src, + "post_merge_scale": post_merge_scale(source_cycle), + } + + stage_dir = out / f"stage_{cycle:02d}" + stage_dir.mkdir(exist_ok=True) + retained_states = [] + for original, record in sorted(latest_states.items()): + state_name = f"layer_original_{original:04d}.pt" + state_path = stage_dir / state_name + copy_stage_state(record["source_state"], state_path, hard_link=bool(getattr(args, "hard_link", False))) + packaged = { + "original_layer": original, + "state_file": state_name, + "sha256": sha256(state_path), + "source_cycle": int(record["source_cycle"]), + "recipient_side": record["recipient_side"], + } + if record["post_merge_scale"] is not None: + packaged["post_merge_scale"] = record["post_merge_scale"] + retained_states.append(packaged) + + assert original_layer_count is not None + active = [ + original + for original in range(original_layer_count) + if original not in removed + ] + meta = { + "format": "abprune_compact_stage_v2", + "model": args.model, + "base_model_revision": args.base_revision, + "run_id": args.run_id, + "target_prune_count": args.cycles, + "stage": cycle, + "removed_count": len(removed), + "original_layer_count": original_layer_count, + "layer_order": order, + "latest_pair_index": order[cycle - 1], + "latest_recipient_side": recipient_side, + "latest_recipient_original_layer": recipient, + "latest_removed_original_layer": removed_original, + "removed_original_layers": sorted(removed), + "active_original_layers": active, + "retained_layer_states": retained_states, + "source_cycle": f"cycle_{cycle}", + } + (stage_dir / "metadata.json").write_text( + json.dumps(meta, indent=2) + "\n", encoding="utf-8" + ) + stages.append(meta) + + manifest = { + "format": "abprune_compact_manifest_v2", + "model": args.model, + "base_model_revision": args.base_revision, + "run_id": args.run_id, + "target_prune_count": args.cycles, + "original_layer_count": original_layer_count, + "layer_order": order, + "stages": stages, + } + (out / "manifest.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" + ) + + +def package(args): + source = Path(args.run_root); out = Path(args.output_root); out.mkdir(parents=True, exist_ok=True) + order = [int(x) for x in args.layer_order.replace(',', ' ').split()] + start_cycle = int(getattr(args, "start_cycle", 1)) + expected_order = args.cycles - start_cycle + 1 + if not 1 <= start_cycle <= args.cycles: + raise ValueError("start_cycle must be between 1 and cycles") + if len(order) != expected_order: + raise ValueError("layer_order length must equal cycles - start_cycle + 1") + first_metadata_path = source / f"cycle_{start_cycle}" / "cycle_metadata.json" + if first_metadata_path.is_file(): + first_metadata = json.loads(first_metadata_path.read_text(encoding="utf-8")) + topology_fields = { + "recipient_side", + "recipient_original_layer", + "removed_original_layer", + } + if topology_fields.issubset(first_metadata): + _package_topology_aware(args, source, out, order) + return + stages = [] + for cycle in range(start_cycle, args.cycles + 1): + order_index = cycle - start_cycle + source_cycle = source / f"cycle_{cycle}" + src = source_cycle / "fused_layer.pt" + if not src.exists(): raise FileNotFoundError(src) + dst_dir = out / f"stage_{cycle:02d}"; dst_dir.mkdir(exist_ok=True) + dst = dst_dir / "anchor_layer.pt"; copy_stage_state(src, dst, hard_link=bool(getattr(args, "hard_link", False))) + meta = { + "format": "abprune_compact_stage_v1", "model": args.model, + "base_model_revision": args.base_revision, "run_id": args.run_id, + "target_prune_count": args.cycles, "stage": cycle, + "removed_count": cycle, "anchor_index": order[order_index], + "layer_order": order, + "packaged_stage_range": [start_cycle, args.cycles], + "removed_original_layers": list(range(order[order_index] + 1, order[order_index] + cycle + 1)), + "source_cycle": f"cycle_{cycle}", "anchor_state_file": "anchor_layer.pt", + "sha256": sha256(dst), + } + scale = post_merge_scale(source_cycle) + if scale is not None: + meta["post_merge_scale"] = scale + (dst_dir / "metadata.json").write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8") + stages.append(meta) + manifest = {"format": "abprune_compact_manifest_v1", "model": args.model, + "base_model_revision": args.base_revision, "run_id": args.run_id, + "target_prune_count": args.cycles, "layer_order": order, "stages": stages} + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + +def main(): + p = argparse.ArgumentParser(); p.add_argument("--run-root", required=True); p.add_argument("--output-root", required=True) + p.add_argument("--model", required=True); p.add_argument("--base-revision", required=True); p.add_argument("--run-id", required=True) + p.add_argument("--cycles", type=int, required=True); p.add_argument("--layer-order", required=True) + p.add_argument("--start-cycle", type=int, default=1) + p.add_argument("--hard-link", action="store_true", help="Hard-link immutable stage weights on the same filesystem") + package(p.parse_args()) + + +if __name__ == "__main__": main() diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/bootstrap_qwen3_base_half2.sh b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/bootstrap_qwen3_base_half2.sh new file mode 100644 index 0000000000000000000000000000000000000000..d297de1d0086e1775d758df0658db06eb833a495 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/bootstrap_qwen3_base_half2.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail +project="$1"; external="$2"; root="$3"; model="$4" +export HF_HOME=/workspace/.hf_home HF_TOKEN_PATH=/root/.cache/huggingface/token +export HF_DATASETS_CACHE=/workspace/.hf_datasets36 HF_DATASETS_TRUST_REMOTE_CODE=1 +export PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false OMP_NUM_THREADS=4 +export PYTHONPATH="$project/src:$external/src:$project" +mkdir -p "$root" +trap 'rc=$?; if ((rc)); then touch "$root/BOOTSTRAP_FAILED"; fi' EXIT +bash "$project/scripts/bootstrap_qwen3_8b_mergeinit_fit2.sh" "$project" "$external" "$root" "$model" +python -m pip install peft einops threadpoolctl +touch "$model/.model_download_complete" +# Original selector and ReplaceMe are isolated from the patched training engine. +legacy="${external}-legacy" +if [[ ! -d "$legacy/.git" ]]; then + git clone "$project/vendor/abprune_experiment_base_39e.bundle" "$legacy" + git -C "$legacy" remote remove origin +fi +git -C "$legacy" apply "$project/patches/replaceme_llama_instruction_mask_bf16.patch" +git -C "$legacy" apply "$project/patches/replaceme_qwen_layer_types.patch" +python - "$root" <<'PY' +import importlib.metadata as m, json, platform, sys +from pathlib import Path +import torch +r=Path(sys.argv[1]) +e={'python':platform.python_version(),'torch':torch.__version__,'cuda':torch.version.cuda, + 'packages':{n:m.version(n) for n in ['transformers','datasets','huggingface_hub','lm_eval','accelerate','scipy','scikit-learn','peft']}} +(r/'environment.json').write_text(json.dumps(e,indent=2)+'\n') +print('ENVIRONMENT',json.dumps(e),flush=True) +PY +while [[ ! -f "$root/EVAL_ASSETS_READY" ]]; do sleep 3; done +python - "$root" <<'PY' +import hashlib,sys +from pathlib import Path +r=Path(sys.argv[1]) +for n,h in [('opencompass5-data.tar.gz','b49605cc272cee94ddc177d2350e06d70f0f610926a2abf20c8ccd35ad8cd353'),('lmeval-cache-131.tar.gz','e5a319ed00b52181df3d80e47e4939e78fddd393803c29a9bcb63c2dd8a6d468')]: + assert hashlib.sha256((r/n).read_bytes()).hexdigest()==h,n +PY +mkdir -p "$root/eval_data" +tar -xzf "$root/opencompass5-data.tar.gz" -C "$root/eval_data" +tar -xzf "$root/lmeval-cache-131.tar.gz" -C "$root" +HF_HUB_OFFLINE=1 HF_DATASETS_OFFLINE=1 python "$project/scripts/prepare_lmeval9_local_cache.py" "$root/lmeval_cache" --validate-local +python - <<'PY' +from datasets import load_dataset +load_dataset('Salesforce/wikitext','wikitext-2-raw-v1') +PY +python "$project/scripts/prepare_offline_ppl_cache.py" "$root" +touch "$root/EVAL_READY" "$root/HALF_READY" diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/come.py b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/come.py new file mode 100644 index 0000000000000000000000000000000000000000..22d1937fdbf534bb2c3e37af3357627040ef4372 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/come.py @@ -0,0 +1,612 @@ +"""Native Hugging Face implementation of CoMe structured layer pruning. + +This module implements the pruning-only portion of CoMe for decoder-only +models whose blocks use Llama-style attention and gated MLP projections. It +intentionally does not implement post-pruning distillation. +""" + +from __future__ import annotations + +import json +import math +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterable, Sequence + +import torch +from torch import Tensor, nn +from tqdm.auto import tqdm + + +@dataclass +class CalibrationScores: + """Scores collected from one pass over the calibration sequences.""" + + block_importance: list[float] + pair_similarity: list[float] + attention_groups: list[Tensor] + mlp_neurons: list[Tensor] + + +@dataclass +class MergeRecord: + step: int + left_index: int + source_layers: list[list[int]] + retention_ratios: list[float] + retained_attention_groups: list[int] + retained_mlp_neurons: list[int] + pair_score: float + + +def allocate_retained_channels(total: int, ratios: Sequence[float]) -> list[int]: + """Allocate exactly ``total`` channels, putting rounding slack in the max arm.""" + + if total < 1: + raise ValueError("total must be positive") + if not ratios or any(value < 0 for value in ratios): + raise ValueError("ratios must be a non-empty sequence of non-negative values") + ratio_sum = float(sum(ratios)) + if ratio_sum <= 0: + normalized = [1.0 / len(ratios)] * len(ratios) + else: + normalized = [float(value) / ratio_sum for value in ratios] + + largest = max(range(len(normalized)), key=normalized.__getitem__) + counts = [0] * len(normalized) + for index, ratio in enumerate(normalized): + if index != largest: + counts[index] = math.floor(total * ratio) + counts[largest] = total - sum(counts) + return counts + + +def gqa_projection_indices( + groups: Tensor, + *, + num_attention_heads: int, + num_key_value_heads: int, + head_dim: int, +) -> tuple[Tensor, Tensor]: + """Return Q and KV row indices for complete GQA groups. + + Hugging Face stores query heads contiguously by KV group. Expanding a KV + mask with ``repeat`` mixes these groups for GQA models; this explicit + mapping keeps all query heads belonging to each selected KV head together. + """ + + if num_attention_heads % num_key_value_heads: + raise ValueError("num_attention_heads must be divisible by num_key_value_heads") + queries_per_kv = num_attention_heads // num_key_value_heads + q_parts: list[Tensor] = [] + kv_parts: list[Tensor] = [] + for group in groups.to(dtype=torch.long, device="cpu").tolist(): + if group < 0 or group >= num_key_value_heads: + raise IndexError(f"KV group {group} is outside [0, {num_key_value_heads})") + q_start = group * queries_per_kv * head_dim + kv_start = group * head_dim + q_parts.append(torch.arange(q_start, q_start + queries_per_kv * head_dim)) + kv_parts.append(torch.arange(kv_start, kv_start + head_dim)) + empty = torch.empty(0, dtype=torch.long) + return ( + torch.cat(q_parts) if q_parts else empty, + torch.cat(kv_parts) if kv_parts else empty, + ) + + +def _copy_tensor(target: Tensor, value: Tensor) -> None: + with torch.no_grad(): + target.copy_(value.to(device=target.device, dtype=target.dtype)) + + +def _selected_indices(scores: Tensor, count: int) -> Tensor: + if count == 0: + return torch.empty(0, dtype=torch.long) + if count > scores.numel(): + raise ValueError(f"cannot select {count} values from {scores.numel()} scores") + # Sorting the selected indices preserves the original within-layer order. + return torch.sort(torch.topk(scores.float(), count, largest=True).indices).values.cpu() + + +def _cat_selected( + modules: Sequence[nn.Linear], + indices: Sequence[Tensor], + *, + dimension: int, +) -> Tensor: + chunks = [] + for module, selected in zip(modules, indices, strict=True): + chunks.append(torch.index_select(module.weight.detach(), dimension, selected.to(module.weight.device))) + return torch.cat(chunks, dim=dimension) + + +def _weighted_optional_bias(modules: Sequence[nn.Linear], ratios: Sequence[float]) -> Tensor | None: + biases = [module.bias for module in modules] + if all(bias is None for bias in biases): + return None + if any(bias is None for bias in biases): + raise ValueError("projection bias presence differs across the layers being merged") + assert all(bias is not None for bias in biases) + result = torch.zeros_like(biases[0]) + for bias, ratio in zip(biases, ratios, strict=True): + result.add_(bias.detach().to(result.device), alpha=float(ratio)) + return result + + +def merge_attention( + layers: Sequence[nn.Module], + ratios: Sequence[float], + group_scores: Sequence[Tensor], +) -> list[int]: + """Merge attention projections in-place into the first decoder layer.""" + + attentions = [layer.self_attn for layer in layers] + first = attentions[0] + q_heads = int(first.config.num_attention_heads) + kv_heads = int(first.config.num_key_value_heads) + head_dim = int(getattr(first, "head_dim", first.q_proj.out_features // q_heads)) + counts = allocate_retained_channels(kv_heads, ratios) + groups = [_selected_indices(score, count) for score, count in zip(group_scores, counts, strict=True)] + q_indices: list[Tensor] = [] + kv_indices: list[Tensor] = [] + for selected in groups: + q_index, kv_index = gqa_projection_indices( + selected, + num_attention_heads=q_heads, + num_key_value_heads=kv_heads, + head_dim=head_dim, + ) + q_indices.append(q_index) + kv_indices.append(kv_index) + + q_modules = [attention.q_proj for attention in attentions] + k_modules = [attention.k_proj for attention in attentions] + v_modules = [attention.v_proj for attention in attentions] + o_modules = [attention.o_proj for attention in attentions] + q_weight = _cat_selected(q_modules, q_indices, dimension=0) + k_weight = _cat_selected(k_modules, kv_indices, dimension=0) + v_weight = _cat_selected(v_modules, kv_indices, dimension=0) + o_weight = _cat_selected(o_modules, q_indices, dimension=1) + + _copy_tensor(first.q_proj.weight, q_weight) + _copy_tensor(first.k_proj.weight, k_weight) + _copy_tensor(first.v_proj.weight, v_weight) + _copy_tensor(first.o_proj.weight, o_weight) + for destination, source_modules, selected in ( + (first.q_proj, q_modules, q_indices), + (first.k_proj, k_modules, kv_indices), + (first.v_proj, v_modules, kv_indices), + ): + biases = [module.bias for module in source_modules] + if not all(bias is None for bias in biases): + if any(bias is None for bias in biases): + raise ValueError("projection bias presence differs across merged layers") + bias_chunks = [ + torch.index_select(module.bias.detach(), 0, index.to(module.bias.device)) + for module, index in zip(source_modules, selected, strict=True) + ] + assert destination.bias is not None + _copy_tensor(destination.bias, torch.cat(bias_chunks)) + merged_o_bias = _weighted_optional_bias(o_modules, ratios) + if merged_o_bias is not None: + assert first.o_proj.bias is not None + _copy_tensor(first.o_proj.bias, merged_o_bias) + + # Qwen3 applies a shared per-head RMSNorm after Q/K projection. It cannot + # be sliced by group, so use the same convex merge as other shared values. + for norm_name in ("q_norm", "k_norm"): + norms = [getattr(attention, norm_name, None) for attention in attentions] + if all(norm is None for norm in norms): + continue + if any(norm is None for norm in norms): + raise ValueError(f"{norm_name} presence differs across merged layers") + destination = norms[0] + merged = torch.zeros_like(destination.weight) + for norm, ratio in zip(norms, ratios, strict=True): + merged.add_(norm.weight.detach().to(merged.device), alpha=float(ratio)) + _copy_tensor(destination.weight, merged) + + return counts + + +def merge_mlp( + layers: Sequence[nn.Module], + ratios: Sequence[float], + neuron_scores: Sequence[Tensor], +) -> list[int]: + """Merge gated MLP projections in-place into the first decoder layer.""" + + mlps = [layer.mlp for layer in layers] + intermediate_size = int(mlps[0].gate_proj.out_features) + counts = allocate_retained_channels(intermediate_size, ratios) + selected = [_selected_indices(score, count) for score, count in zip(neuron_scores, counts, strict=True)] + + for name in ("gate_proj", "up_proj"): + modules = [getattr(mlp, name) for mlp in mlps] + merged = _cat_selected(modules, selected, dimension=0) + destination = getattr(mlps[0], name) + _copy_tensor(destination.weight, merged) + biases = [module.bias for module in modules] + if not all(bias is None for bias in biases): + if any(bias is None for bias in biases): + raise ValueError(f"{name} bias presence differs across merged layers") + bias_chunks = [ + torch.index_select(module.bias.detach(), 0, index.to(module.bias.device)) + for module, index in zip(modules, selected, strict=True) + ] + assert destination.bias is not None + _copy_tensor(destination.bias, torch.cat(bias_chunks)) + + down_modules = [mlp.down_proj for mlp in mlps] + _copy_tensor(mlps[0].down_proj.weight, _cat_selected(down_modules, selected, dimension=1)) + merged_bias = _weighted_optional_bias(down_modules, ratios) + if merged_bias is not None: + assert mlps[0].down_proj.bias is not None + _copy_tensor(mlps[0].down_proj.bias, merged_bias) + return counts + + +def merge_norms(layers: Sequence[nn.Module]) -> None: + """Average the two residual-stream norms into the destination block.""" + + for name in ("input_layernorm", "post_attention_layernorm"): + norms = [getattr(layer, name) for layer in layers] + destination = norms[0] + _copy_tensor(destination.weight, torch.stack([norm.weight.detach().to(destination.weight.device) for norm in norms]).mean(0)) + destination_bias = getattr(destination, "bias", None) + if destination_bias is not None: + source_biases = [getattr(norm, "bias", None) for norm in norms] + if any(bias is None for bias in source_biases): + raise ValueError(f"{name} bias presence differs across merged layers") + _copy_tensor( + destination_bias, + torch.stack([bias.detach().to(destination_bias.device) for bias in source_biases]).mean(0), + ) + + +def compute_retention_ratios( + importance: Sequence[float], + *, + power: float, + minimum: float, +) -> list[float]: + if power < 0: + raise ValueError("power must be non-negative") + if not 0 <= minimum <= 1: + raise ValueError("minimum must be in [0, 1]") + values = [max(float(value), 0.0) ** power for value in importance] + total = sum(values) + ratios = [value / total for value in values] if total > 0 else [1 / len(values)] * len(values) + largest = max(range(len(ratios)), key=ratios.__getitem__) + if ratios[largest] < minimum: + if len(ratios) == 1: + return [1.0] + remainder = sum(ratios) - ratios[largest] + scale = (1.0 - minimum) / remainder if remainder > 0 else 0.0 + ratios = [ratio * scale for ratio in ratios] + ratios[largest] = minimum + return ratios + + +class _ProjectionScoreCollector: + def __init__(self, modules: Sequence[nn.Linear]): + self.scores = [torch.zeros(module.in_features, dtype=torch.float64) for module in modules] + self._handles = [] + for index, module in enumerate(modules): + self._handles.append(module.register_forward_pre_hook(self._hook(index))) + + def _hook(self, index: int): + def collect(module: nn.Linear, inputs: tuple[Tensor, ...]) -> None: + activation = inputs[0].detach().float().abs() + reduce_dims = tuple(range(activation.ndim - 1)) + activation_sum = activation.sum(dim=reduce_dims) + weight_l1 = module.weight.detach().float().abs().sum(dim=0) + self.scores[index].add_((activation_sum * weight_l1).double().cpu()) + + return collect + + def close(self) -> None: + for handle in self._handles: + handle.remove() + + +def _angular_distance(left: Tensor, right: Tensor, *, last_token_only: bool) -> Tensor: + if last_token_only: + left = left[:, -1:, :] + right = right[:, -1:, :] + cosine = torch.nn.functional.cosine_similarity(left.float(), right.float(), dim=-1) + if last_token_only: + return torch.acos(cosine.clamp(0, 1)) / math.pi + return 1 - cosine + + +@torch.inference_mode() +def collect_calibration_scores( + model: nn.Module, + batches: Iterable[dict[str, Tensor]], + *, + sbi_metric: str = "angular_last", +) -> CalibrationScores: + """Collect CoMe block, adjacent-pair, attention, and MLP scores.""" + + if sbi_metric not in {"angular_last", "cosine_all"}: + raise ValueError("sbi_metric must be 'angular_last' or 'cosine_all'") + backbone = model.model + layers = list(backbone.layers) + attention = _ProjectionScoreCollector([layer.self_attn.o_proj for layer in layers]) + mlp = _ProjectionScoreCollector([layer.mlp.down_proj for layer in layers]) + block_sums = torch.zeros(len(layers), dtype=torch.float64) + pair_sums = torch.zeros(len(layers) - 1, dtype=torch.float64) + block_count = 0 + pair_count = 0 + input_device = model.get_input_embeddings().weight.device + try: + for batch in tqdm(batches, desc=f"CoMe calibration ({len(layers)} layers)"): + prepared = {key: value.to(input_device) for key, value in batch.items()} + outputs = backbone( + **prepared, + use_cache=False, + output_hidden_states=True, + return_dict=True, + ) + hidden = outputs.hidden_states + for index in range(len(layers)): + distance = _angular_distance(hidden[index], hidden[index + 1], last_token_only=False) + block_sums[index] += distance.double().sum().cpu() + batch_token_count = hidden[0].shape[0] * hidden[0].shape[1] + block_count += batch_token_count + for index in range(len(layers) - 1): + distance = _angular_distance( + hidden[index], + hidden[index + 2], + last_token_only=sbi_metric == "angular_last", + ) + pair_sums[index] += distance.double().sum().cpu() + pair_count += hidden[0].shape[0] if sbi_metric == "angular_last" else batch_token_count + finally: + attention.close() + mlp.close() + + if block_count == 0: + raise ValueError("calibration batches are empty") + attention_groups: list[Tensor] = [] + for layer, score in zip(layers, attention.scores, strict=True): + attn = layer.self_attn + q_heads = int(attn.config.num_attention_heads) + kv_heads = int(attn.config.num_key_value_heads) + head_dim = int(getattr(attn, "head_dim", attn.q_proj.out_features // q_heads)) + queries_per_kv = q_heads // kv_heads + expected = kv_heads * queries_per_kv * head_dim + if score.numel() != expected: + raise ValueError(f"unexpected attention input width {score.numel()} (expected {expected})") + attention_groups.append(score.reshape(kv_heads, queries_per_kv, head_dim).mean((1, 2))) + return CalibrationScores( + block_importance=(block_sums / block_count).tolist(), + pair_similarity=(pair_sums / max(pair_count, 1)).tolist(), + attention_groups=attention_groups, + mlp_neurons=mlp.scores, + ) + + +def _choose_pair(pair_scores: Sequence[float], lineage: Sequence[Sequence[int]], allow_repeat: bool) -> int: + candidates = [] + for index, score in enumerate(pair_scores): + repeated = len(lineage[index]) > 1 or len(lineage[index + 1]) > 1 + if allow_repeat or not repeated: + candidates.append((float(score), index)) + if not candidates: + raise RuntimeError( + "no legal adjacent pair remains; use --allow-repeat or choose a target at least ceil(original_layers / 2)" + ) + return min(candidates)[1] + + +def _update_config_after_delete(model: nn.Module, deleted_index: int) -> None: + model.config.num_hidden_layers = len(model.model.layers) + if hasattr(model.config, "layer_types") and model.config.layer_types is not None: + layer_types = list(model.config.layer_types) + if deleted_index < len(layer_types): + del layer_types[deleted_index] + model.config.layer_types = layer_types + for index, layer in enumerate(model.model.layers): + if hasattr(layer.self_attn, "layer_idx"): + layer.self_attn.layer_idx = index + + +@torch.inference_mode() +def progressive_prune( + model: nn.Module, + calibration_batches: Sequence[dict[str, Tensor]], + *, + target_layers: int, + power: float = 1.0, + minimum_retention: float = 0.0, + allow_repeat: bool = False, + sbi_metric: str = "angular_last", +) -> list[MergeRecord]: + """Progressively merge adjacent layer pairs until ``target_layers`` remain.""" + + original_layers = len(model.model.layers) + if not 1 <= target_layers <= original_layers: + raise ValueError(f"target_layers must be in [1, {original_layers}]") + if not allow_repeat and target_layers < math.ceil(original_layers / 2): + raise ValueError("without repeated merges, target_layers cannot be below ceil(original_layers / 2)") + lineage: list[list[int]] = [[index] for index in range(original_layers)] + records: list[MergeRecord] = [] + model.eval() + model.config.use_cache = False + + while len(model.model.layers) > target_layers: + scores = collect_calibration_scores(model, calibration_batches, sbi_metric=sbi_metric) + left = _choose_pair(scores.pair_similarity, lineage, allow_repeat) + indices = [left, left + 1] + leaf_count = sum(len(lineage[index]) for index in indices) + adjusted_minimum = minimum_retention + if allow_repeat and minimum_retention > 0 and leaf_count > 2: + adjusted_minimum = minimum_retention ** (1 / (leaf_count - 1)) + ratios = compute_retention_ratios( + [scores.block_importance[index] for index in indices], + power=power, + minimum=adjusted_minimum, + ) + print( + f"CoMe step {len(records) + 1}: merge current layers {left}/{left + 1} " + f"(sources {lineage[left]} + {lineage[left + 1]}, ratios={ratios})", + flush=True, + ) + merge_layers = [model.model.layers[index] for index in indices] + attention_counts = merge_attention( + merge_layers, + ratios, + [scores.attention_groups[index] for index in indices], + ) + mlp_counts = merge_mlp( + merge_layers, + ratios, + [scores.mlp_neurons[index] for index in indices], + ) + merge_norms(merge_layers) + records.append( + MergeRecord( + step=len(records) + 1, + left_index=left, + source_layers=[list(lineage[index]) for index in indices], + retention_ratios=ratios, + retained_attention_groups=attention_counts, + retained_mlp_neurons=mlp_counts, + pair_score=float(scores.pair_similarity[left]), + ) + ) + del model.model.layers[left + 1] + lineage[left] = lineage[left] + lineage[left + 1] + del lineage[left + 1] + _update_config_after_delete(model, left + 1) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return records + + +def build_wikitext2_calibration( + tokenizer, + *, + samples: int, + sequence_length: int, + seed: int, +) -> list[dict[str, Tensor]]: + """Reproduce CoMe's shuffled WikiText-2 prefix calibration stream. + + The upstream ``nsamples`` argument counts shuffled dataset rows, not 2,048 + token windows. Full windows are carved from their concatenated token stream + and the short tail is discarded. + """ + + from datasets import load_dataset + + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train").shuffle(seed=seed) + selected = dataset.select(range(min(samples, len(dataset)))) + text = "\n\n".join(selected["text"]) + token_ids = tokenizer(text, return_tensors="pt").input_ids[0] + windows = token_ids.numel() // sequence_length + if windows == 0: + raise ValueError("selected WikiText-2 rows contain fewer tokens than sequence_length") + batches = [] + for index in range(windows): + start = index * sequence_length + input_ids = token_ids[start : start + sequence_length].unsqueeze(0) + batches.append({"input_ids": input_ids, "attention_mask": torch.ones_like(input_ids)}) + return batches + + +def build_slimpajama_calibration( + tokenizer, + *, + rows: int, + sequence_length: int, + seed: int, + shuffle_buffer: int = 10_000, +) -> list[dict[str, Tensor]]: + """Pack full windows after concatenating shuffled SlimPajama rows. + + This mirrors upstream CoMe's WikiText loader: ``nsamples`` counts dataset + rows, not model-forward windows. + """ + + from datasets import load_dataset + + if rows < 1: + raise ValueError("rows must be positive") + dataset = load_dataset( + "DKYoon/SlimPajama-6B", + split="train", + streaming=True, + ).shuffle(seed=seed, buffer_size=shuffle_buffer) + token_buffer: list[int] = [] + selected_rows = 0 + for row in dataset: + text = row.get("text") + selected_rows += 1 + if isinstance(text, str): + token_buffer.extend(tokenizer.encode(text, add_special_tokens=False)) + if selected_rows == rows: + break + if selected_rows != rows: + raise RuntimeError(f"could only read {selected_rows}/{rows} SlimPajama rows") + windows = len(token_buffer) // sequence_length + if windows == 0: + raise ValueError("selected SlimPajama rows contain fewer tokens than sequence_length") + batches: list[dict[str, Tensor]] = [] + for index in range(windows): + start = index * sequence_length + input_ids = torch.tensor(token_buffer[start : start + sequence_length], dtype=torch.long).unsqueeze(0) + batches.append({"input_ids": input_ids, "attention_mask": torch.ones_like(input_ids)}) + return batches + + +def batch_calibration( + samples: Sequence[dict[str, Tensor]], batch_size: int +) -> list[dict[str, Tensor]]: + if batch_size < 1: + raise ValueError("batch_size must be positive") + batches = [] + for start in range(0, len(samples), batch_size): + chunk = samples[start : start + batch_size] + batches.append({key: torch.cat([item[key] for item in chunk], dim=0) for key in chunk[0]}) + return batches + + +def save_merge_metadata( + output_dir: str | Path, + *, + source_model: str, + records: Sequence[MergeRecord], + arguments: dict, +) -> None: + path = Path(output_dir) / "come_pruning.json" + payload = { + "method": "CoMe structured pruning (no distillation)", + "source_model": source_model, + "arguments": arguments, + "merges": [asdict(record) for record in records], + } + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def enable_torch24_dtensor_save_compat() -> None: + """Expose PyTorch 2.4's private DTensor where Transformers 4.52 expects it.""" + + try: + import torch.distributed.tensor as public_tensor + except ImportError: + return + if hasattr(public_tensor, "DTensor"): + return + try: + from torch.distributed._tensor import DTensor + except ImportError: + return + public_tensor.DTensor = DTensor + # Transformers 4.52 imports this name only when it detects torch >= 2.5, + # but save_pretrained references it unconditionally on torch 2.4. + import transformers.modeling_utils as modeling_utils + + modeling_utils.DTensor = DTensor diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/eval_llama31_8b_12bench.py b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/eval_llama31_8b_12bench.py new file mode 100644 index 0000000000000000000000000000000000000000..8bbc8713a4d0f04209bec7ee454a24744f9d3bb8 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/eval_llama31_8b_12bench.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Evaluate the lm-eval portion of the requested Llama benchmark suite. + +RACE-H/M, C3, CMNLI, and CHID are evaluated with their canonical OpenCompass +PPL task definitions by the companion launcher. This script handles the +remaining eight tasks plus full zero-shot MMLU and CMMLU. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + + +TASKS = ( + "piqa", + "winogrande", + "hellaswag", + "arc_challenge", + "arc_easy", + "openbookqa", + "sciq", + "mmlu", + "cmmlu", +) + +# Use normalized accuracy where the standard harness exposes it. The +# extraction code falls back to plain accuracy for tasks/versions without it. +PREFERRED_METRICS = { + "piqa": "acc_norm,none", + "hellaswag": "acc_norm,none", + "arc_challenge": "acc_norm,none", + "arc_easy": "acc_norm,none", + "openbookqa": "acc_norm,none", + "sciq": "acc_norm,none", +} + + +def _score(task: str, result: dict[str, Any]) -> tuple[str, float]: + preferred = PREFERRED_METRICS.get(task, "acc,none") + candidates = (preferred, "acc_norm,none", "acc,none") + for metric in candidates: + value = result.get(metric) + if value is not None: + return metric, float(value) + available = sorted(key for key in result if "," in key and not key.endswith("_stderr,none")) + raise KeyError(f"No supported accuracy metric for {task}; available={available}") + + +def _load_model(args: argparse.Namespace): + if args.compact_stage is None: + model = AutoModelForCausalLM.from_pretrained( + args.model, + revision=args.revision, + torch_dtype=torch.bfloat16, + attn_implementation="eager", + local_files_only=args.local_files_only, + ).cuda().eval() + tokenizer = AutoTokenizer.from_pretrained( + args.model, + revision=args.revision, + local_files_only=args.local_files_only, + ) + return model, tokenizer + + if args.cycle is None or args.anchor is None: + raise ValueError("--compact-stage requires --cycle and --anchor") + from fusegpt.abprune_stagewise_geometry import load_compact_model + + model = load_compact_model( + args.model, + args.compact_stage, + cycle=args.cycle, + anchor=args.anchor, + dtype=torch.bfloat16, + device=torch.device("cuda"), + attn="eager", + revision=args.revision, + ) + tokenizer = AutoTokenizer.from_pretrained( + args.model, revision=args.revision, local_files_only=args.local_files_only) + return model, tokenizer + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", default=None) + parser.add_argument("--local-files-only", action="store_true") + parser.add_argument("--compact-stage", type=Path, default=None) + parser.add_argument("--cycle", type=int, default=None) + parser.add_argument("--anchor", type=int, default=None) + parser.add_argument("--batch-size", default="auto:4") + parser.add_argument("--limit", type=float, default=None) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--dataset-cache", type=Path, default=None) + parser.add_argument("--tasks", nargs="+", choices=TASKS, default=list(TASKS), + help="Evaluate a subset without repeating already scored tasks") + args = parser.parse_args() + if len(set(args.tasks)) != len(args.tasks): + parser.error("--tasks must not contain duplicates") + + # CMMLU is still distributed through a Hugging Face dataset loading + # script. Opt in explicitly so non-interactive experiment jobs do not + # stop at the datasets trust prompt. + os.environ.setdefault("HF_DATASETS_TRUST_REMOTE_CODE", "1") + + from lm_eval import evaluator + from lm_eval.models.huggingface import HFLM + + if args.dataset_cache is not None: + from prepare_lmeval9_local_cache import install_local_loader + install_local_loader(args.dataset_cache) + + model, tokenizer = _load_model(args) + model.config.use_cache = False + if getattr(model, "generation_config", None) is not None: + model.generation_config.use_cache = False + for layer_idx, layer in enumerate(model.model.layers): + if hasattr(layer, "self_attn") and hasattr(layer.self_attn, "layer_idx"): + layer.self_attn.layer_idx = layer_idx + + lm = HFLM( + pretrained=model, + tokenizer=tokenizer, + backend="causal", + batch_size=args.batch_size, + ) + raw = evaluator.simple_evaluate( + model=lm, + tasks=args.tasks, + num_fewshot=0, + batch_size=args.batch_size, + random_seed=0, + numpy_random_seed=1234, + torch_random_seed=1234, + fewshot_random_seed=1234, + bootstrap_iters=0, + limit=args.limit, + ) + + scores: dict[str, float] = {} + metrics: dict[str, str] = {} + for task in args.tasks: + source = raw.get("groups", {}) if task in {"mmlu", "cmmlu"} else raw["results"] + if task not in source: + raise KeyError( + f"Requested task {task!r} missing from lm-eval results: " + f"results={sorted(raw['results'])}, groups={sorted(raw.get('groups', {}))}" + ) + metric, value = _score(task, source[task]) + metrics[task] = metric + scores[task] = value + + report = { + "model": args.model, + "revision": args.revision, + "compact_stage": str(args.compact_stage) if args.compact_stage else None, + "cycle": args.cycle, + "anchor": args.anchor, + "tasks": args.tasks, + "num_fewshot": 0, + "batch_size": args.batch_size, + "limit": args.limit, + "dataset_cache": str(args.dataset_cache) if args.dataset_cache else None, + "preferred_metrics": metrics, + "task_scores": scores, + "macro_mean_all_rows": sum(scores.values()) / len(scores), + "lm_eval_results": raw["results"], + "lm_eval_groups": raw.get("groups"), + "lm_eval_versions": raw.get("versions"), + "lm_eval_config": raw.get("config"), + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(report, indent=2, ensure_ascii=False, default=str) + "\n", + encoding="utf-8", + ) + print( + json.dumps( + {"task_scores": scores, "macro_mean_all_rows": report["macro_mean_all_rows"]}, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/eval_opencompass_ppl_fast.py b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/eval_opencompass_ppl_fast.py new file mode 100644 index 0000000000000000000000000000000000000000..e32bb5fdc36375a58debd2042f8a60c706ab3ef6 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/eval_opencompass_ppl_fast.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Fast, equivalent evaluator for selected OpenCompass zero-shot PPL tasks. + +The prompts, labels, split choices, full-prompt mean-CE scoring, and lowest-PPL +decision rule mirror OpenCompass 0.5.4's canonical configs for RACE, C3, +CMNLI, and CHID. Unlike the stock runner, this evaluator disables KV cache +and length-batches all choices, which matters substantially on 40 GB GPUs. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Iterable + +import torch +import torch.nn.functional as F +from transformers import AutoModelForCausalLM, AutoTokenizer + + +def read_jsonl(path: Path) -> Iterable[dict]: + with path.open("r", encoding="utf-8") as stream: + for line in stream: + yield json.loads(line) + + +def race(path: Path) -> tuple[list[list[str]], list[int]]: + prompts, golds = [], [] + for row in read_jsonl(path): + human = ( + "Read the article, and answer the question by replying A, B, C or D.\n\n" + f"Article:\n{row['article']}\n\nQ: {row['question']}\n\n" + f"A. {row['options'][0]}\nB. {row['options'][1]}\n" + f"C. {row['options'][2]}\nD. {row['options'][3]}" + ) + prompts.append([human + "\n" + f"A: {answer}" for answer in "ABCD"]) + golds.append("ABCD".index(row["answer"])) + return prompts, golds + + +def c3(path: Path) -> tuple[list[list[str]], list[int]]: + prompts, golds = [], [] + rows = json.loads(path.read_text(encoding="utf-8")) + for row in rows: + content = " ".join("".join(paragraph) for paragraph in row[0]) + for question in row[1]: + choices = list(question["choice"]) + golds.append(choices.index(question["answer"])) + while len(choices) < 4: + choices.append(choices[0]) + human = f"文章：{content}\n问题：{question['question']}" + prompts.append([human + "\n" + f"答案：{choice}" for choice in choices[:4]]) + return prompts, golds + + +def cmnli(path: Path) -> tuple[list[list[str]], list[int]]: + labels = ("contradiction", "entailment", "neutral") + verbalizers = ("矛盾", "蕴含", "无关") + prompts, golds = [], [] + for row in read_jsonl(path): + if row["label"] == "-": + continue + human = ( + f"语句一：“{row['sentence1']}”\n语句二：“{row['sentence2']}”\n" + "请问这两句话是什么关系？" + ) + prompts.append([human + "\n" + word for word in verbalizers]) + golds.append(labels.index(row["label"])) + return prompts, golds + + +def chid(path: Path) -> tuple[list[list[str]], list[int]]: + prompts, golds = [], [] + for row in read_jsonl(path): + candidates = [row["content"].replace("#idiom#", candidate) for candidate in row["candidates"]] + prompts.append([ + f"以下句子是否通顺？\n{content}\n这个句子是通顺的。" for content in candidates + ]) + golds.append(int(row["answer"])) + return prompts, golds + + +def score_task(model, tokenizer, choices: list[list[str]], golds: list[int], max_batch: int, + token_budget: int) -> dict: + flat = [(sample_idx, choice_idx, prompt) + for sample_idx, row in enumerate(choices) + for choice_idx, prompt in enumerate(row)] + encoded = tokenizer( + [item[2] for item in flat], + add_special_tokens=True, + padding=False, + truncation=True, + max_length=model.config.max_position_embeddings, + )["input_ids"] + order = sorted(range(len(flat)), key=lambda idx: len(encoded[idx]), reverse=True) + losses = [0.0] * len(flat) + cursor = 0 + while cursor < len(order): + longest = len(encoded[order[cursor]]) + batch_size = min(max_batch, max(1, token_budget // longest), len(order) - cursor) + while True: + batch_indices = order[cursor:cursor + batch_size] + max_len = max(len(encoded[idx]) for idx in batch_indices) + input_ids = torch.full( + (batch_size, max_len), tokenizer.pad_token_id, dtype=torch.long, device="cuda" + ) + attention = torch.zeros((batch_size, max_len), dtype=torch.long, device="cuda") + for row_idx, flat_idx in enumerate(batch_indices): + values = torch.tensor(encoded[flat_idx], dtype=torch.long, device="cuda") + input_ids[row_idx, : len(values)] = values + attention[row_idx, : len(values)] = 1 + try: + with torch.inference_mode(): + logits = model(input_ids=input_ids, attention_mask=attention, use_cache=False).logits + token_loss = F.cross_entropy( + logits[:, :-1].float().reshape(-1, logits.shape[-1]), + input_ids[:, 1:].reshape(-1), + reduction="none", + ).view(batch_size, -1) + # OpenCompass divides by the number of non-padding input + # tokens (including BOS), while scoring shifted labels. + sequence_loss = ( + token_loss * attention[:, 1:] + ).sum(dim=1) / attention.sum(dim=1) + break + except torch.OutOfMemoryError: + del input_ids, attention + torch.cuda.empty_cache() + if batch_size == 1: + raise + batch_size = max(1, batch_size // 2) + for flat_idx, value in zip(batch_indices, sequence_loss.tolist()): + losses[flat_idx] = value + cursor += batch_size + if cursor % 1000 < batch_size or cursor == len(order): + print(f"choices={cursor}/{len(order)}", flush=True) + + score_rows = [[0.0] * len(row) for row in choices] + for flat_idx, (sample_idx, choice_idx, _) in enumerate(flat): + score_rows[sample_idx][choice_idx] = losses[flat_idx] + predictions = [min(range(len(row)), key=row.__getitem__) for row in score_rows] + correct = sum(prediction == gold for prediction, gold in zip(predictions, golds)) + return { + "accuracy": correct / len(golds), + "correct": correct, + "samples": len(golds), + "choices": len(flat), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", default=None) + parser.add_argument("--tokenizer", required=True) + parser.add_argument("--compact-stage", type=Path, default=None) + parser.add_argument("--cycle", type=int, default=None) + parser.add_argument("--anchor", type=int, default=None) + parser.add_argument("--data-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--max-batch", type=int, default=32) + parser.add_argument("--token-budget", type=int, default=8192) + parser.add_argument("--limit", type=int, default=None) + args = parser.parse_args() + + tokenizer = AutoTokenizer.from_pretrained(args.tokenizer, local_files_only=True) + if tokenizer.pad_token_id is None: + tokenizer.pad_token_id = tokenizer.eos_token_id + tokenizer.padding_side = "right" + if args.compact_stage is None: + model = AutoModelForCausalLM.from_pretrained( + args.model, + revision=args.revision, + torch_dtype=torch.bfloat16, + attn_implementation="sdpa", + local_files_only=True, + ).cuda().eval() + else: + if args.cycle is None or args.anchor is None: + raise ValueError("--compact-stage requires --cycle and --anchor") + from fusegpt.abprune_stagewise_geometry import load_compact_model + + model = load_compact_model( + args.model, + args.compact_stage, + cycle=args.cycle, + anchor=args.anchor, + dtype=torch.bfloat16, + device=torch.device("cuda"), + attn="sdpa", + revision=args.revision, + ) + model.config.use_cache = False + + datasets = { + "race-middle": race(args.data_root / "race/test/middle.jsonl"), + "race-high": race(args.data_root / "race/test/high.jsonl"), + "C3": c3(args.data_root / "CLUE/C3/dev_0.json"), + "cmnli": cmnli(args.data_root / "CLUE/cmnli/cmnli_public/dev.json"), + "chid-dev": chid(args.data_root / "FewCLUE/chid/dev_few_all.json"), + } + results = {} + for name, (choices, golds) in datasets.items(): + if args.limit is not None: + choices, golds = choices[: args.limit], golds[: args.limit] + print(f"task={name} samples={len(golds)}", flush=True) + results[name] = score_task( + model, tokenizer, choices, golds, args.max_batch, args.token_budget + ) + print(json.dumps({name: results[name]}, ensure_ascii=False), flush=True) + + report = { + "model": args.model, + "revision": args.revision, + "compact_stage": str(args.compact_stage) if args.compact_stage else None, + "cycle": args.cycle, + "anchor": args.anchor, + "tokenizer": args.tokenizer, + "protocol": "OpenCompass 0.5.4 canonical PPL prompts; full-prompt mean CE; lowest PPL", + "max_batch": args.max_batch, + "token_budget": args.token_budget, + "limit": args.limit, + "results": results, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/evaluate_hchc_wikitext2_ppl.py b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/evaluate_hchc_wikitext2_ppl.py new file mode 100644 index 0000000000000000000000000000000000000000..28ea302a1e40246a770b97e6ee6fd9198cf57c28 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/evaluate_hchc_wikitext2_ppl.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Evaluate a saved causal LM with the HCHC WikiText-2 PPL protocol.""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + +import torch +from datasets import load_dataset, load_from_disk + + +def primary(output: Any) -> torch.Tensor: + if torch.is_tensor(output): + return output + if isinstance(output, (tuple, list)) and output and torch.is_tensor(output[0]): + return output[0] + raise TypeError(f"Unsupported decoder-layer output type: {type(output)!r}") + + +def replace_primary(output: Any, hidden: torch.Tensor) -> Any: + if torch.is_tensor(output): + return hidden + if isinstance(output, tuple): + return (hidden, *output[1:]) + if isinstance(output, list): + return [hidden, *output[1:]] + raise TypeError(f"Unsupported decoder-layer output type: {type(output)!r}") + + +def register_residual_scale_hook(layer, gamma: float): + """Install h_out = h_in + gamma * (block(h_in) - h_in).""" + scale = torch.tensor(gamma, device=next(layer.parameters()).device) + + def hook(_module, inputs, output): + if not inputs or not torch.is_tensor(inputs[0]): + raise RuntimeError("Could not capture decoder-layer residual input") + layer_input = inputs[0] + corrected = layer_input + scale * (primary(output) - layer_input) + return replace_primary(output, corrected) + + return layer.register_forward_hook(hook) + + +def packed_wikitext_tokens(tokenizer, split: str, *, dataset_cache: Path | None = None) -> torch.Tensor: + """Reproduce the canonical HCHC WikiText-2 packing protocol.""" + if dataset_cache is not None: + dataset = load_from_disk(str(dataset_cache))[split] + else: + dataset = load_dataset( + "Salesforce/wikitext", "wikitext-2-raw-v1", split=split + ) + text = "\n\n".join(row for row in dataset["text"] if row.strip()) + return tokenizer( + text, add_special_tokens=False, return_tensors="pt" + ).input_ids[0] + + +@torch.no_grad() +def evaluate_perplexity( + model, + tokens: torch.Tensor, + sequence_length: int, + batch_size: int, +) -> dict[str, float | int]: + num_sequences = tokens.numel() // sequence_length + ids = tokens[: num_sequences * sequence_length].view(num_sequences, sequence_length) + total_nll = 0.0 + total_predicted_tokens = 0 + model.eval() + for start in range(0, num_sequences, batch_size): + batch = ids[start : start + batch_size].to("cuda") + output = model(input_ids=batch, labels=batch, use_cache=False) + predicted = batch.shape[0] * (batch.shape[1] - 1) + total_nll += output.loss.float().item() * predicted + total_predicted_tokens += predicted + mean_nll = total_nll / total_predicted_tokens + return { + "nll": mean_nll, + "perplexity": math.exp(mean_nll), + "tokens": total_predicted_tokens, + "sequences": num_sequences, + "source_tokens": int(tokens.numel()), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--sequence-length", type=int, default=2048) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--residual-anchor", type=int) + parser.add_argument("--residual-gamma", type=float) + args = parser.parse_args() + + if (args.residual_anchor is None) != (args.residual_gamma is None): + parser.error("--residual-anchor and --residual-gamma must be supplied together") + + from transformers import AutoModelForCausalLM, AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True) + model = AutoModelForCausalLM.from_pretrained( + args.model, + torch_dtype=torch.bfloat16, + local_files_only=True, + low_cpu_mem_usage=True, + attn_implementation="sdpa", + ).to("cuda") + model.config.use_cache = False + + handle = None + if args.residual_gamma is not None: + if args.residual_anchor >= len(model.model.layers): + raise ValueError( + f"Residual anchor {args.residual_anchor} is outside decoder depth " + f"{len(model.model.layers)}" + ) + handle = register_residual_scale_hook( + model.model.layers[args.residual_anchor], args.residual_gamma + ) + + tokens = packed_wikitext_tokens(tokenizer, "test") + try: + metrics = evaluate_perplexity( + model, tokens, args.sequence_length, args.batch_size + ) + finally: + if handle is not None: + handle.remove() + + report = { + "model": str(args.model), + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: non-empty rows joined with double newline before one-shot tokenization", + "sequence_length": args.sequence_length, + "batch_size": args.batch_size, + "dtype": "bfloat16", + "residual_anchor": args.residual_anchor, + "residual_gamma": args.residual_gamma, + "metrics": metrics, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/mka.py b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/mka.py new file mode 100644 index 0000000000000000000000000000000000000000..a06594bd33dfbea70c54093a43d7cb65e8abb9a1 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/mka.py @@ -0,0 +1,505 @@ +"""Corrected MKA variant: adjacent NPIB selection, upstream lambda, real fusion. + +Not an exact reproduction of the paper: use sequence-level pooled activations, +recompute geometry after every merge, and resolve NPIB ties back-to-front. +The seven projection weights are fused; residual and Q/K norms stay at anchor. +""" + +from __future__ import annotations + +import hashlib +import math +import random +import time +from collections.abc import Callable, Iterable, Sequence +from dataclasses import asdict, dataclass + +import numpy as np +import torch +from scipy.linalg import eigh +from scipy.spatial.distance import pdist, squareform +from sklearn.feature_selection import mutual_info_regression +from sklearn.neighbors import NearestNeighbors +from torch import Tensor, nn + +PROJECTION_WEIGHTS = ( + "mlp.down_proj.weight", + "mlp.up_proj.weight", + "mlp.gate_proj.weight", + "self_attn.k_proj.weight", + "self_attn.o_proj.weight", + "self_attn.q_proj.weight", + "self_attn.v_proj.weight", +) +UPSTREAM = "https://github.com/SempraETY/Pruning-via-Merging" +UPSTREAM_COMMIT = "3b60b002a0963266e2f4ca78f133b4e091bc607a" + + +@dataclass(frozen=True) +class GeometryConfig: + sigma: float = 8.0 + alpha: float = 0.5 + dimensions: int = 2 + mi_neighbors: int = 5 + entropy_neighbors: int = 50 + beta: float = 1.0 + seed: int = 0 + pooling: str = "mean" + + def validate(self) -> None: + if not math.isfinite(self.sigma) or self.sigma <= 0: + raise ValueError("sigma must be positive and finite") + if not math.isfinite(self.alpha) or not 0 <= self.alpha <= 1: + raise ValueError("alpha must be finite and in [0, 1]") + if self.dimensions < 1 or min(self.mi_neighbors, self.entropy_neighbors) < 1: + raise ValueError("dimensions and neighbor counts must be positive") + if not math.isfinite(self.beta) or self.beta < 0: + raise ValueError("beta must be non-negative and finite") + if self.pooling not in {"mean", "last"}: + raise ValueError("pooling must be 'mean' or 'last'") + + +def sample_slimpajama( + tokenizer, + *, + samples: int = 100, + sequence_length: int = 512, + seed: int = 0, + revision: str = "b5f90f419b7489cdba26fdbc8c022fcb5562f968", + shuffle_buffer: int = 10000, + max_documents: int = 100000, + documents: Iterable[dict] | None = None, +) -> tuple[list[dict[str, Tensor]], dict]: + """Exactly one random fixed-length, no-BOS window from each distinct document. + + Unlike CoMe's row-packing loader, ``samples`` is the number of sequences. + Persist returned IDs and provenance; do not sample again between stages. + """ + if min(samples, sequence_length, shuffle_buffer, max_documents) < 1: + raise ValueError("sampling counts, length and buffer must be positive") + if documents is None: + from datasets import load_dataset + + documents = load_dataset( + "DKYoon/SlimPajama-6B", revision=revision, split="train", streaming=True + ).shuffle(seed=seed, buffer_size=shuffle_buffer) + rng = random.Random(seed) + batches, records, seen = [], [], set() + iterator = iter(documents) + try: + for row_index, row in enumerate(iterator): + if row_index >= max_documents: + break + text = row.get("text") + if not isinstance(text, str) or not text.strip(): + continue + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + if digest in seen: + continue + seen.add(digest) + ids = tokenizer.encode(text, add_special_tokens=False) + if len(ids) < sequence_length: + continue + start = rng.randrange(len(ids) - sequence_length + 1) + tensor = torch.tensor( + ids[start : start + sequence_length], dtype=torch.long + )[None, :] + batches.append( + {"input_ids": tensor, "attention_mask": torch.ones_like(tensor)} + ) + records.append( + { + "stream_row": row_index, + "document_sha256": digest, + "document_tokens": len(ids), + "window_start": start, + } + ) + if len(batches) == samples: + break + finally: + close = getattr(iterator, "close", None) + if close is not None: + close() + if len(batches) != samples: + raise RuntimeError( + f"Only found {len(batches)}/{samples} distinct sufficiently long documents" + ) + checksum = hashlib.sha256() + for batch in batches: + checksum.update(batch["input_ids"].numpy().astype(" list[np.ndarray]: + """One backbone forward per batch; block hooks avoid hidden-state off-by-one. + + Collect outputs *before* the final backbone norm, including the final block. + Pool only valid tokens; activations are CPU float32 after pooling. + """ + if pooling not in {"mean", "last"} or not batches: + raise ValueError("valid pooling and nonempty calibration batches are required") + layers = model.model.layers + features: list[list[Tensor]] = [[] for _ in layers] + current_mask: Tensor | None = None + + def hook(index): + def capture(module, inputs, output): + hidden = output[0] if isinstance(output, (tuple, list)) else output + mask = current_mask.to(hidden.device, dtype=torch.bool) + if hidden.shape[:2] != mask.shape or (mask.sum(-1) == 0).any(): + raise ValueError("block output and valid-token mask do not match") + if pooling == "mean": + value = (hidden.float() * mask[:, :, None]).sum(1) / mask.sum(1)[ + :, None + ] + else: + positions = torch.arange(mask.shape[1], device=mask.device)[None, :] + last = positions.expand_as(mask).masked_fill(~mask, -1).max(1).values + value = hidden[ + torch.arange(hidden.shape[0], device=hidden.device), last + ].float() + if not torch.isfinite(value).all(): + raise FloatingPointError(f"nonfinite activation in block {index}") + features[index].append(value.cpu()) + + return capture + + handles = [layer.register_forward_hook(hook(i)) for i, layer in enumerate(layers)] + was_training = model.training + model.eval() + try: + device = model.get_input_embeddings().weight.device + for batch in batches: + prepared = {key: value.to(device) for key, value in batch.items()} + current_mask = prepared.get( + "attention_mask", torch.ones_like(prepared["input_ids"]) + ) + model.model(**prepared, use_cache=False, return_dict=True) + finally: + for handle in handles: + handle.remove() + model.train(was_training) + expected = sum(batch["input_ids"].shape[0] for batch in batches) + arrays = [torch.cat(parts, dim=0).numpy() for parts in features] + if any(array.shape[0] != expected for array in arrays): + raise RuntimeError("not all calibration examples reached each decoder block") + return arrays + + +def diffusion_embedding( + features: np.ndarray, config: GeometryConfig +) -> tuple[np.ndarray, dict]: + """Correctly shaped sample-by-sample diffusion kernel, following upstream SVD geometry.""" + config.validate() + x = np.asarray(features, dtype=np.float64) + if x.ndim != 2 or not np.isfinite(x).all() or len(x) <= config.dimensions + 1: + raise ValueError( + "features must be finite [samples, features], samples > dimensions + 1" + ) + distance = squareform(pdist(x, metric="euclidean")) + if distance.max() <= np.finfo(float).eps: + raise ValueError( + "constant calibration features: cannot estimate manifold geometry" + ) + kernel = np.exp(-np.sqrt(distance / config.sigma)) + p = kernel.sum(0) + normalized = kernel / np.outer(p, p) ** config.alpha + v = np.sqrt(normalized.sum(0)) + operator = normalized / np.outer(v, v) + condition = float(np.linalg.cond(operator)) + regularization = 1e-6 if not math.isfinite(condition) or condition > 1e12 else 0.0 + if regularization: + operator += regularization * np.eye(len(x)) + values, vectors = eigh( + operator, subset_by_index=(len(x) - config.dimensions - 1, len(x) - 1) + ) + values, vectors = values[::-1], vectors[:, ::-1] + # Fix arbitrary eigenvector signs; the leading vector of this positive kernel is positive. + for dim in range(vectors.shape[1]): + if vectors[np.argmax(np.abs(vectors[:, dim])), dim] < 0: + vectors[:, dim] *= -1 + if np.any(np.abs(vectors[:, 0]) < 1e-12): + raise FloatingPointError( + "diffusion normalization has a near-zero leading eigenvector" + ) + embedding = vectors[:, 1:] / vectors[:, :1] + if not np.isfinite(embedding).all(): + raise FloatingPointError("nonfinite diffusion embedding") + # Upstream uses per-axis rank normalization before its KNN estimators. + embedding = np.argsort( + np.argsort(embedding, axis=0, kind="stable"), axis=0, kind="stable" + ) / len(x) + return embedding, { + "samples": len(x), + "condition_number": condition if math.isfinite(condition) else None, + "regularization": regularization, + "eigenvalues": values.tolist(), + } + + +def entropy_knn(x: np.ndarray, neighbors: int) -> float: + """Upstream differential-entropy estimator, evaluated in log space.""" + n, dimensions = x.shape + k = min(neighbors, n - 1) + distances = NearestNeighbors(n_neighbors=k + 1).fit(x).kneighbors(x)[0][:, -1] + if np.any(distances <= 0) or not np.isfinite(distances).all(): + raise FloatingPointError( + "duplicate/degenerate manifold points in entropy estimator" + ) + return float(math.log(n / k) + dimensions * np.log(distances).mean()) + + +def npib_matrix( + embeddings: Sequence[np.ndarray], config: GeometryConfig +) -> tuple[np.ndarray, list[float]]: + """Match upstream axis-wise MI average and symmetric sqrt(H_i*H_j) normalization. + + Differential entropies can be negative. Preserve upstream's formula when + their product is positive; fail clearly for zero/negative products or NaNs. + This estimator is a heuristic, not a guaranteed [0, 1] similarity. + """ + config.validate() + if not embeddings or len({x.shape for x in embeddings}) != 1: + raise ValueError("embeddings must have identical sample/coordinate shapes") + n = len(embeddings[0]) + if n < 3 or any(not np.isfinite(x).all() for x in embeddings): + raise ValueError("need at least three finite manifold points") + entropies = [entropy_knn(x, config.entropy_neighbors) for x in embeddings] + scores = np.empty((len(embeddings), len(embeddings)), dtype=np.float64) + for i, left in enumerate(embeddings): + for j in range(i, len(embeddings)): + right = embeddings[j] + product = entropies[i] * entropies[j] + if not math.isfinite(product) or product <= 1e-12: + raise FloatingPointError( + f"invalid entropy product at ({i}, {j}): {entropies[i]}, {entropies[j]}" + ) + mi = np.mean( + [ + np.mean( + mutual_info_regression( + left, + right[:, dim], + n_neighbors=min(config.mi_neighbors, n - 1), + random_state=config.seed, + ) + ) + for dim in range(right.shape[1]) + ] + ) + scores[i, j] = scores[j, i] = mi / math.sqrt(product) + if not np.isfinite(scores).all(): + raise FloatingPointError("NPIB contains nonfinite scores") + return scores, entropies + + +def upstream_lambda( + scores: np.ndarray, left: int, right: int, beta: float = 1.0 +) -> float: + """Exactly upstream's sigmoid(beta * row_mean_i / (row_mean_i + row_mean_j)).""" + means = np.mean(scores[[left, right]], axis=1) + total = float(means.sum()) + if not np.isfinite(means).all() or (means < 0).any() or total <= 1e-12: + raise FloatingPointError( + "cannot compute upstream lambda from nonpositive/nonfinite row means" + ) + if not math.isfinite(beta) or beta < 0: + raise ValueError("beta must be non-negative and finite") + value = beta * float(means[0]) / total + return 1.0 / (1.0 + math.exp(-value)) + + +def select_pair( + scores: np.ndarray, + *, + selection: str = "npib", + protected_first: int = 0, + protected_last: int = 0, +) -> int: + """Adjacent pairs only; NPIB descending, and deeper pair wins exact ties.""" + n = len(scores) + if scores.shape != (n, n) or not np.isfinite(scores).all(): + raise ValueError("scores must be a finite square matrix") + if min(protected_first, protected_last) < 0: + raise ValueError("protected layer counts must be non-negative") + candidates = range(protected_first, n - protected_last - 1) + if not candidates: + raise ValueError("no eligible adjacent pair remains") + if selection == "back-to-front": + return candidates[-1] + if selection != "npib": + raise ValueError("selection must be 'npib' or 'back-to-front'") + return max(candidates, key=lambda i: (float(scores[i, i + 1]), i)) + + +@torch.inference_mode() +def merge_pair(model: nn.Module, left: int, ratio: float) -> int: + """FP32 weighted sum copied into real parameters, then delete the right block.""" + layers = model.model.layers + if ( + not 0 <= left < len(layers) - 1 + or not math.isfinite(ratio) + or not 0 <= ratio <= 1 + ): + raise ValueError("invalid adjacent pair or merge ratio") + destination, source = layers[left], layers[left + 1] + pairs = [] + for name in PROJECTION_WEIGHTS: + pairs.append((destination.get_parameter(name), source.get_parameter(name))) + # Qwen2.5 has Q/K/V biases. Fuse them with the same coefficient; + # Llama and Qwen3 (bias-free projections) retain their existing behavior. + module_name = name.rsplit('.', 1)[0] + a_bias = getattr(destination.get_submodule(module_name), 'bias', None) + b_bias = getattr(source.get_submodule(module_name), 'bias', None) + if (a_bias is None) != (b_bias is None): + raise ValueError('projection bias presence differs between merge layers') + if a_bias is not None: + pairs.append((a_bias, b_bias)) + # Validate all shapes/values/config before any model mutation. + if any(a.shape != b.shape for a, b in pairs): + raise ValueError("projection shapes differ between merge layers") + if any(not torch.isfinite(p).all() for pair in pairs for p in pair): + raise FloatingPointError("nonfinite source weight") + configs = [model.config, model.model.config] + for config in configs: + types = getattr(config, "layer_types", None) + if types is not None and len(types) != len(layers): + raise ValueError("config.layer_types length does not match decoder depth") + changed = 0 + for target, other in pairs: + merged = torch.lerp( + other.detach().to(target.device, dtype=torch.float32), target.float(), ratio + ) + merged = merged.to(target.dtype) + if not torch.isfinite(merged).all(): + raise FloatingPointError("merge overflow after conversion to model dtype") + changed += int(not torch.equal(target, merged)) + target.copy_(merged) + del layers[left + 1] + handled = set() + for config in configs: + if id(config) in handled: + continue + handled.add(id(config)) + config.num_hidden_layers = len(layers) + if getattr(config, "layer_types", None) is not None: + config.layer_types = list(config.layer_types[: left + 1]) + list( + config.layer_types[left + 2 :] + ) + for index, layer in enumerate(layers): + if hasattr(layer.self_attn, "layer_idx"): + layer.self_attn.layer_idx = index + return changed + + +def progressive_prune( + model: nn.Module, + batches: Sequence[dict[str, Tensor]], + *, + target_layers: int, + geometry: GeometryConfig | None = None, + selection: str = "npib", + protected_first: int = 0, + protected_last: int = 0, + on_stage: Callable[[nn.Module, dict], None] | None = None, + transport: bool = False, + transport_ridge: float = 1e-3, + transport_fit_fraction: float = 0.8, + initial_lineage: Sequence[Sequence[int]] | None = None, +) -> list[dict]: + """Recompute activations/manifolds/NPIB on fixed data after every accepted merge. + + No threshold or forced fallbacks; stop at the requested exact depth. Repeated + merging of a previously merged block is allowed and recorded in its lineage. + """ + geometry = geometry or GeometryConfig() + geometry.validate() + depth = len(model.model.layers) + if min(protected_first, protected_last) < 0 or not 1 <= target_layers <= depth: + raise ValueError("invalid target depth or protected counts") + if target_layers < protected_first + protected_last + 1: + raise ValueError("target depth would remove protected layers") + if selection not in {"npib", "back-to-front"}: + raise ValueError("invalid selection mode") + lineage = ([list(group) for group in initial_lineage] if initial_lineage is not None + else [[i] for i in range(depth)]) + flattened = [index for group in lineage for index in group] + if (len(lineage) != depth or any(not group for group in lineage) + or any(type(index) is not int for index in flattened) + or flattened != list(range(len(flattened)))): + raise ValueError("initial lineage must partition original layers in order") + stage_offset = len(flattened) - depth + records = [] + model.eval() + while len(model.model.layers) > target_layers: + started = time.monotonic() + step = stage_offset + len(records) + 1 + print( + f"MKA stage {step}: collecting features for {len(lineage)} layers", + flush=True, + ) + features = collect_features(model, batches, pooling=geometry.pooling) + results = [diffusion_embedding(x, geometry) for x in features] + del features + scores, entropies = npib_matrix([x[0] for x in results], geometry) + left = select_pair( + scores, + selection=selection, + protected_first=protected_first, + protected_last=protected_last, + ) + ratio = upstream_lambda(scores, left, left + 1, geometry.beta) + record = { + "step": step, + "selection": selection, + "left_index": left, + "source_layers": [lineage[left][:], lineage[left + 1][:]], + "pair_npib": float(scores[left, left + 1]), + "lambda_left": ratio, + "lambda_right": 1 - ratio, + "row_means": scores.mean(1).tolist(), + "npib_matrix": scores.tolist(), + "entropies": entropies, + "geometry": asdict(geometry), + "manifolds": [x[1] for x in results], + } + if transport: + from .mka_transport import transport_merge_pair + changed, diagnostic = transport_merge_pair( + model, batches, left, ratio, ridge_relative=transport_ridge, + fit_fraction=transport_fit_fraction, + ) + record["changed_parameter_tensors"] = changed + record["transport"] = diagnostic + else: + record["changed_parameter_tensors"] = merge_pair(model, left, ratio) + lineage[left] += lineage.pop(left + 1) + record["lineage_after"] = [x[:] for x in lineage] + record["remaining_layers"] = len(lineage) + record["elapsed_seconds"] = time.monotonic() - started + records.append(record) + print( + f"MKA stage {step}: merge {left}/{left + 1}, NPIB={record['pair_npib']:.6g}, " + f"lambda={ratio:.6g}, remaining={len(lineage)}", + flush=True, + ) + if on_stage is not None: + on_stage(model, record) + return records diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/mka_transport.py b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/mka_transport.py new file mode 100644 index 0000000000000000000000000000000000000000..1b7039ae377d2720b907087e234fa48549948e07 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/mka_transport.py @@ -0,0 +1,232 @@ +"""Token-level ridge input transport before MKA's original scalar merge. + +Attention Q/K/V and MLP gate/up use separate maps. MLP deployment inputs are +measured with the provisional merged attention; source targets come from the +unmodified current model. Output projections and anchor norms are not mapped. +""" +from __future__ import annotations + +import math +from contextlib import contextmanager + +import torch + +ATTENTION = tuple(f"self_attn.{name}_proj.weight" for name in ("q", "k", "v", "o")) +INPUTS = ATTENTION[:3] + ("mlp.gate_proj.weight", "mlp.up_proj.weight") + + +class _Captured(RuntimeError): + pass + + +@contextmanager +def capture_inputs(model, modules, stop): + captured = {} + handles = [] + + def hook(name): + def capture(_module, args, kwargs): + value = args[0] if args else kwargs["hidden_states"] + captured[name] = value.detach() + if name == stop: + raise _Captured() + return capture + + try: + for name, module in modules.items(): + handles.append(module.register_forward_pre_hook(hook(name), with_kwargs=True)) + yield captured + finally: + for handle in handles: + handle.remove() + + +def forward_until_capture(model, batch): + device = model.get_input_embeddings().weight.device + try: + model.model(**{k: v.to(device) for k, v in batch.items()}, use_cache=False, + return_dict=True) + except _Captured: + pass + + +def valid_tokens(value, batch): + mask = batch.get("attention_mask", torch.ones_like(batch["input_ids"])) + if value.shape[:2] != mask.shape: + raise ValueError("transport activation/mask shape mismatch") + result = value[mask.to(value.device, dtype=torch.bool)] + if not len(result) or not torch.isfinite(result).all(): + raise FloatingPointError("empty/nonfinite transport inputs") + return result + + +class Statistics: + def __init__(self): + self.gram = self.cross = None + self.target_sq = 0.0 + self.tokens = 0 + + def add(self, x, y): + x, y = x.float(), y.to(x.device).float() + if x.shape != y.shape or x.ndim != 2: + raise ValueError("transport requires paired equal-width token inputs") + if self.gram is None: + self.gram = torch.zeros((x.shape[1], x.shape[1]), device=x.device) + self.cross = torch.zeros_like(self.gram) + self.gram.addmm_(x.T, x) + self.cross.addmm_(x.T, y) + self.target_sq += float(y.double().square().sum()) + self.tokens += len(x) + + def score(self, mapping): + # CPU FP64 avoids TF32 and cancellation in the diagnostic quadratic. + gram = self.gram.double().cpu() / self.tokens + cross = self.cross.double().cpu() / self.tokens + t = mapping.double().cpu() + target = self.target_sq / self.tokens + dot = float((t * cross).sum()) + prediction = float((t * (gram @ t)).sum()) + error = max(0.0, prediction + target - 2 * dot) + return {"tokens": self.tokens, "nmse": error / target, + "cosine": dot / math.sqrt(max(prediction * target, 1e-30)), + "norm_ratio": math.sqrt(max(prediction / target, 0.0))} + + def solve(self, ridge_relative): + gram = self.gram.double().cpu() / self.tokens + cross = self.cross.double().cpu() / self.tokens + ridge = float(gram.diagonal().mean()) * ridge_relative + gram.diagonal().add_(ridge) + chol, info = torch.linalg.cholesky_ex(gram) + if int(info.max()) != 0: + raise FloatingPointError("transport regularized Gram is not positive definite") + mapping = torch.cholesky_solve(cross, chol).float().to(self.gram.device) + if not torch.isfinite(mapping).all(): + raise FloatingPointError("nonfinite transport mapping") + return mapping, {"ridge_relative": ridge_relative, "ridge_absolute": ridge, + "relative_identity_distance": float( + (mapping - torch.eye(len(mapping), device=mapping.device)).norm() + / math.sqrt(len(mapping))), "fit": self.score(mapping)} + + +def transport_weight(weight, mapping, chunk=256): + result = torch.empty_like(weight) + for start in range(0, len(weight), chunk): + result[start:start + chunk] = ( + weight[start:start + chunk].float() @ mapping.T + ).to(weight.dtype) + if not torch.isfinite(result).all(): + raise FloatingPointError("nonfinite transported weight") + return result + + +@torch.inference_mode() +def transport_merge_pair(model, batches, left, ratio, *, ridge_relative=1e-3, + fit_fraction=0.8, observer=None): + """Fit on whole examples, validate on disjoint examples, then merge once. + +Temporary attention changes are restored even if fitting raises. Final source +rewrites are restored if merging fails before layer deletion. No optimizer, +Fisher estimate, output-target regression, or benchmark selection is involved. +""" + from .mka import PROJECTION_WEIGHTS, merge_pair + + if not math.isfinite(ridge_relative) or ridge_relative <= 0: + raise ValueError("transport ridge must be positive and finite") + if not 0 < fit_fraction < 1 or not batches: + raise ValueError("transport requires calibration and a proper fit split") + if not 0 <= left < len(model.model.layers) - 1 or not 0 <= ratio <= 1: + raise ValueError("invalid merge pair/ratio") + anchor, source = model.model.layers[left:left + 2] + pairs = [(anchor.get_parameter(n), source.get_parameter(n)) for n in PROJECTION_WEIGHTS] + if any(a.shape != b.shape for a, b in pairs): + raise ValueError("projection shapes differ") + if any(not torch.isfinite(p).all() for pair in pairs for p in pair): + raise FloatingPointError("nonfinite source projection") + # The experiment runner uses batch=1; generic batches must not straddle + # the document split, so split by batches and record exact example counts. + split = int(len(batches) * fit_fraction) + if not 0 < split < len(batches): + raise ValueError("transport split needs at least two calibration batches") + attn_stats = [Statistics(), Statistics()] + mlp_targets = [] + native = {"anchor_attn": anchor.self_attn, "source_attn": source.self_attn, + "source_mlp": source.mlp} + was_training = model.training + old_tf32 = torch.backends.cuda.matmul.allow_tf32 + model.eval() + torch.backends.cuda.matmul.allow_tf32 = False + anchor_backup = {} + source_backup = {} + try: + for index, batch in enumerate(batches): + with capture_inputs(model, native, "source_mlp") as captured: + forward_until_capture(model, batch) + if set(captured) != set(native): + raise RuntimeError("incomplete native input capture") + attn_stats[int(index >= split)].add( + valid_tokens(captured["anchor_attn"], batch), + valid_tokens(captured["source_attn"], batch)) + mlp_targets.append(valid_tokens(captured["source_mlp"], batch).cpu()) + t_attn, attn_diagnostic = attn_stats[0].solve(ridge_relative) + attn_diagnostic["heldout"] = attn_stats[1].score(t_attn) + del attn_stats + # Provisional attention equals the actual attention deployed at the + # anchor after this merge, including raw o_proj and projection biases. + for name in ATTENTION: + target = anchor.get_parameter(name) + original_source = source.get_parameter(name) + anchor_backup[name] = target.detach().cpu().clone() + candidate = (transport_weight(original_source, t_attn) + if name in INPUTS else original_source) + target.copy_(torch.lerp(candidate.float(), target.float(), ratio).to(target.dtype)) + bias_name = name.replace(".weight", ".bias") + a_bias = getattr(anchor.get_submodule(name.rsplit(".", 1)[0]), "bias", None) + b_bias = getattr(source.get_submodule(name.rsplit(".", 1)[0]), "bias", None) + if (a_bias is None) != (b_bias is None): + raise ValueError("projection bias presence differs") + if a_bias is not None: + anchor_backup[bias_name] = a_bias.detach().cpu().clone() + a_bias.copy_(torch.lerp(b_bias.float(), a_bias.float(), ratio).to(a_bias.dtype)) + mlp_stats = [Statistics(), Statistics()] + for index, (batch, target) in enumerate(zip(batches, mlp_targets)): + with capture_inputs(model, {"deployed_mlp": anchor.mlp}, "deployed_mlp") as captured: + forward_until_capture(model, batch) + if "deployed_mlp" not in captured: + raise RuntimeError("incomplete deployed MLP capture") + mlp_stats[int(index >= split)].add( + valid_tokens(captured["deployed_mlp"], batch), target) + t_mlp, mlp_diagnostic = mlp_stats[0].solve(ridge_relative) + mlp_diagnostic["heldout"] = mlp_stats[1].score(t_mlp) + del mlp_stats, mlp_targets + for name, value in anchor_backup.items(): + anchor.get_parameter(name).copy_(value) + anchor_backup.clear() + # Optional diagnostic observes native weights and the fitted maps before + # any source rewrite. It must not mutate model parameters or the maps. + if observer is not None: + observer(anchor, source, {"attention": t_attn, "mlp": t_mlp}) + # Rewrite only source input projections. Then reuse original MKA's + # seven-projection merge, norms policy, layer deletion and reindexing. + for name in INPUTS: + parameter = source.get_parameter(name) + source_backup[name] = parameter.detach().cpu().clone() + parameter.copy_(transport_weight(parameter, t_attn if name.startswith("self_attn") else t_mlp)) + changed = merge_pair(model, left, ratio) + source_backup.clear() + return changed, {"enabled": True, "projections": list(INPUTS), + "fit_batches": split, "heldout_batches": len(batches) - split, + "fit_examples": sum(b["input_ids"].shape[0] for b in batches[:split]), + "heldout_examples": sum(b["input_ids"].shape[0] for b in batches[split:]), + "attention": attn_diagnostic, "mlp": mlp_diagnostic, + "mlp_input_policy": "provisional merged attention with retained anchor norms", + "source_target_policy": "native current-model inputs before any stage mutation", + "coordinate_fit": "X_deployed @ T ~= X_native_source", + "weight_rewrite": "W_source <- W_source @ T.T", + "output_projections_transported": False} + finally: + for name, value in anchor_backup.items(): + anchor.get_parameter(name).copy_(value) + for name, value in source_backup.items(): + source.get_parameter(name).copy_(value) + torch.backends.cuda.matmul.allow_tf32 = old_tf32 + model.train(was_training) diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/replaceme_llama_instruction_mask_bf16.patch b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/replaceme_llama_instruction_mask_bf16.patch new file mode 100644 index 0000000000000000000000000000000000000000..19c147c1317dfe56ab1ded23a1bda42bde84ba21 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/replaceme_llama_instruction_mask_bf16.patch @@ -0,0 +1,46 @@ +diff --git a/compare_model/ReplaceMe/ReplaceMe/lstsq.py b/compare_model/ReplaceMe/ReplaceMe/lstsq.py +--- a/compare_model/ReplaceMe/ReplaceMe/lstsq.py ++++ b/compare_model/ReplaceMe/ReplaceMe/lstsq.py +@@ -92,6 +92,7 @@ def lstsq( + quantization_config=quantization_config, + output_hidden_states=True, + token=token, ++ dtype=torch.bfloat16, + ) + tokenizer = AutoTokenizer.from_pretrained(model_path) + hidden_size = model.config.hidden_size +@@ -173,19 +174,22 @@ def lstsq( + with torch.no_grad(): + outputs = model(**inputs) + ++ valid_tokens = inputs["attention_mask"].reshape(-1).to(dtype=torch.bool) + hidden_states = outputs.hidden_states[1:] + + # Get relevant hidden states + hidden_states_mlp = [ +- mlp_activations[f'layer_{i}_mlp'].view(-1, hidden_size).to(torch.float64) ++ mlp_activations[f'layer_{i}_mlp'] ++ .view(-1, hidden_size)[valid_tokens] ++ .to(torch.float64) + for i in range(model.config.num_hidden_layers) if i+1 in start_ids + ] + hidden_states_i = [ +- hidden_states[i].view(-1, hidden_size).to(torch.float64) ++ hidden_states[i].view(-1, hidden_size)[valid_tokens].to(torch.float64) + for i in range(model.config.num_hidden_layers) if i+1 in start_ids + ] + hidden_states_n = [ +- hidden_states[i].view(-1, hidden_size).to(torch.float64) ++ hidden_states[i].view(-1, hidden_size)[valid_tokens].to(torch.float64) + for i in range(model.config.num_hidden_layers) if i+1 in end_ids + ] + +@@ -231,7 +234,7 @@ def lstsq( + model = AutoModelForCausalLM.from_pretrained( + model_path, + device_map='cpu', +- torch_dtype=torch.bfloat16, ++ dtype=torch.bfloat16, + ) + + # Apply transformations diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/replaceme_qwen_layer_types.patch b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/replaceme_qwen_layer_types.patch new file mode 100644 index 0000000000000000000000000000000000000000..de23545dfe7ec389c8b7abdba04e18a6ded3bcbf --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/replaceme_qwen_layer_types.patch @@ -0,0 +1,25 @@ +diff --git a/compare_model/ReplaceMe/ReplaceMe/utils.py b/compare_model/ReplaceMe/ReplaceMe/utils.py +--- a/compare_model/ReplaceMe/ReplaceMe/utils.py ++++ b/compare_model/ReplaceMe/ReplaceMe/utils.py +@@ -80,11 +80,16 @@ def angular_distance(x_l: torch.Tensor, x_l_plus_n: torch.Tensor) -> torch.Tens + + def truncate_model(model: nn.Module, start_layer: int, end_layer: int) -> nn.Module: + """Truncate model by removing specified layers.""" +- model.config.num_hidden_layers -= (end_layer - start_layer) +- model.model.layers = nn.ModuleList([ +- layer for idx, layer in enumerate(model.model.layers) +- if idx < start_layer or idx >= end_layer +- ]) ++ layers = model.model.layers ++ kept_indices = [ ++ idx for idx in range(len(layers)) ++ if idx < start_layer or idx >= end_layer ++ ] ++ model.config.num_hidden_layers = len(kept_indices) ++ model.model.layers = nn.ModuleList([layers[idx] for idx in kept_indices]) ++ layer_types = getattr(model.config, "layer_types", None) ++ if isinstance(layer_types, (list, tuple)) and len(layer_types) == len(layers): ++ model.config.layer_types = [layer_types[idx] for idx in kept_indices] + return model + + diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_come_pruning.py b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_come_pruning.py new file mode 100644 index 0000000000000000000000000000000000000000..d3306caf1e6882f66f9457a498eac4215de2040b --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_come_pruning.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +"""Prune a native Hugging Face Qwen3 or Llama 3 model with CoMe.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +from fusegpt.come import ( + batch_calibration, + build_slimpajama_calibration, + build_wikitext2_calibration, + enable_torch24_dtensor_save_compat, + progressive_prune, + save_merge_metadata, +) + + +DTYPES = { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float32": torch.float32, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True, help="Hugging Face model id or local directory") + parser.add_argument("--output-dir", required=True) + parser.add_argument("--target-layers", required=True, type=int) + parser.add_argument("--calibration-dataset", choices=("slimpajama", "wikitext2"), default="slimpajama") + parser.add_argument("--calibration-rows", type=int, default=256) + parser.add_argument("--sequence-length", type=int, default=2048) + parser.add_argument("--batch-size", type=int, default=1) + parser.add_argument("--seed", type=int, default=10) + parser.add_argument("--power", type=float, default=1.0, help="Block-importance exponent") + parser.add_argument("--minimum-retention", type=float, default=0.0) + parser.add_argument("--allow-repeat", action="store_true", help="Allow a merged block to be merged again") + parser.add_argument("--sbi-metric", choices=("angular_last", "cosine_all"), default="angular_last") + parser.add_argument("--dtype", choices=tuple(DTYPES), default="bfloat16") + parser.add_argument("--device-map", default="auto") + parser.add_argument("--attn-implementation", choices=("eager", "sdpa", "flash_attention_2"), default="sdpa") + parser.add_argument("--trust-remote-code", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + output_dir = Path(args.output_dir) + if (output_dir / "config.json").exists(): + raise FileExistsError(f"Refusing to overwrite an existing checkpoint: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=args.trust_remote_code) + model = AutoModelForCausalLM.from_pretrained( + args.model, + torch_dtype=DTYPES[args.dtype], + device_map=args.device_map, + low_cpu_mem_usage=True, + trust_remote_code=args.trust_remote_code, + attn_implementation=args.attn_implementation, + ) + architecture = model.config.model_type + if architecture not in {"llama", "qwen3"}: + raise ValueError(f"unsupported model_type={architecture!r}; expected 'llama' or 'qwen3'") + + if args.calibration_dataset == "slimpajama": + samples = build_slimpajama_calibration( + tokenizer, + rows=args.calibration_rows, + sequence_length=args.sequence_length, + seed=args.seed, + ) + else: + samples = build_wikitext2_calibration( + tokenizer, + samples=args.calibration_rows, + sequence_length=args.sequence_length, + seed=args.seed, + ) + print( + f"Calibration: {args.calibration_rows} {args.calibration_dataset} rows -> " + f"{len(samples)} windows x {args.sequence_length} tokens", + flush=True, + ) + batches = batch_calibration(samples, args.batch_size) + records = progressive_prune( + model, + batches, + target_layers=args.target_layers, + power=args.power, + minimum_retention=args.minimum_retention, + allow_repeat=args.allow_repeat, + sbi_metric=args.sbi_metric, + ) + model.config.use_cache = True + enable_torch24_dtensor_save_compat() + model.save_pretrained(output_dir, safe_serialization=True) + tokenizer.save_pretrained(output_dir) + metadata_arguments = dict(vars(args)) + metadata_arguments["packed_calibration_windows"] = len(samples) + metadata_arguments["packed_calibration_tokens"] = len(samples) * args.sequence_length + save_merge_metadata( + output_dir, + source_model=args.model, + records=records, + arguments=metadata_arguments, + ) + print(f"Saved {len(model.model.layers)}-layer model to {output_dir}") + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_mka_pruning.py b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_mka_pruning.py new file mode 100644 index 0000000000000000000000000000000000000000..a1a3aa42eca397d37ca857f911cf9b3a3c87abdd --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_mka_pruning.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python +"""Run corrected, training-free MKA on native Hugging Face Llama/Qwen models.""" + +from __future__ import annotations + +import argparse +import gc +import hashlib +import json +import platform +import shutil +import subprocess +import sys +from dataclasses import asdict +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument( + "--revision", help="Source model/tokenizer revision (pin for real experiments)" + ) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--resume-checkpoint", help="Continue a completed native MKA checkpoint, retaining original lineage") + depth = parser.add_mutually_exclusive_group(required=True) + depth.add_argument("--remove-blocks", type=int) + depth.add_argument("--target-layers", type=int) + parser.add_argument( + "--selection", + choices=("npib", "back-to-front"), + default="npib", + help="NPIB maximum adjacent pair; exact ties prefer deeper pair", + ) + parser.add_argument("--calibration-samples", type=int, default=100) + parser.add_argument("--sequence-length", type=int, default=512) + parser.add_argument( + "--calibration-ids", + help="Replay this runner's calibration_ids.pt instead of resampling", + ) + parser.add_argument( + "--calibration-revision", default="b5f90f419b7489cdba26fdbc8c022fcb5562f968" + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--batch-size", type=int, default=1) + parser.add_argument("--pooling", choices=("mean", "last"), default="mean") + parser.add_argument("--sigma", type=float, default=8.0) + parser.add_argument("--alpha", type=float, default=0.5) + parser.add_argument("--manifold-dimensions", type=int, default=2) + parser.add_argument("--mi-neighbors", type=int, default=5) + parser.add_argument("--entropy-neighbors", type=int, default=50) + parser.add_argument("--beta", type=float, default=1.0) + parser.add_argument("--protected-first", type=int, default=0) + parser.add_argument("--protected-last", type=int, default=0) + parser.add_argument("--save-at-removals", type=int, nargs="*", default=[]) + parser.add_argument( + "--dtype", choices=("bfloat16", "float16", "float32"), default="bfloat16" + ) + parser.add_argument("--device-map", default="auto") + parser.add_argument( + "--attn-implementation", + choices=("eager", "sdpa", "flash_attention_2"), + default="sdpa", + ) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument( + "--verify-reload", action=argparse.BooleanOptionalAction, default=True + ) + parser.add_argument("--cpu-threads", type=int, default=4) + parser.add_argument("--transport", action="store_true", + help="Ridge input transport of source Q/K/V and gate/up before MKA merge") + parser.add_argument("--transport-ridge", type=float, default=1e-3) + parser.add_argument("--transport-fit-fraction", type=float, default=0.8) + return parser.parse_args(argv) + + +def write_json(path: Path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, ensure_ascii=False, allow_nan=False) + "\n", + encoding="utf-8", + ) + + +def replay_calibration(path: str, tokenizer, args): + import torch + + bundle = torch.load(path, map_location="cpu", weights_only=True) + spec = bundle["metadata"] + chunks = bundle["input_ids"] + if ( + spec["tokenizer"] != args.model + or spec["tokenizer_revision"] != args.revision + or spec["tokenizer_vocab_sha256"] != vocabulary_checksum(tokenizer) + ): + raise ValueError( + "calibration cache tokenizer identity/revision/vocabulary differs" + ) + if len(chunks) != args.calibration_samples or any( + x.dtype != torch.long or tuple(x.shape) != (args.sequence_length,) + for x in chunks + ): + raise ValueError( + "calibration cache sample count, dtype or sequence length differs" + ) + checksum = hashlib.sha256() + for chunk in chunks: + checksum.update(chunk.numpy().astype("= len(tokenizer)).any() for x in chunks): + raise ValueError("calibration cache contains invalid token IDs") + return [ + {"input_ids": chunk[None, :], "attention_mask": torch.ones_like(chunk[None, :])} + for chunk in chunks + ], spec + + +def vocabulary_checksum(tokenizer): + serialized = json.dumps(tokenizer.get_vocab(), sort_keys=True, ensure_ascii=False) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +def read_resume_protocol(checkpoint, args, geometry, calibration): + path = Path(checkpoint) / "mka_pruning.json" + protocol = json.loads(path.read_text(encoding="utf-8")) + if (protocol.get("status") not in {"complete", "checkpoint_complete"} + or protocol["source_model"] != args.model + or protocol["resolved_model_revision"] != args.revision + or protocol["geometry"] != asdict(geometry) + or protocol["calibration"] != calibration): + raise ValueError("resume checkpoint source, geometry or calibration mismatch") + expected_transport = {"enabled": args.transport, "ridge_relative": args.transport_ridge, + "fit_fraction": args.transport_fit_fraction, + "scope": "attention q/k/v and MLP gate/up; separate maps"} + if protocol.get("transport") != expected_transport: + raise ValueError("resume checkpoint transport settings mismatch") + for key in ("selection", "protected_first", "protected_last"): + if protocol["arguments"][key] != getattr(args, key): + raise ValueError(f"resume checkpoint {key} mismatch") + merges = protocol["merges"] + if (not merges or [row["step"] for row in merges] != list(range(1, len(merges) + 1)) + or protocol["original_depth"] - protocol["actual_depth"] != len(merges) + or merges[-1]["remaining_layers"] != protocol["actual_depth"] + or len(merges[-1]["lineage_after"]) != protocol["actual_depth"] + or [i for group in merges[-1]["lineage_after"] for i in group] != list(range(protocol["original_depth"]))): + raise ValueError("resume checkpoint merge history/depth mismatch") + return protocol, hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(argv=None): + args = parse_args(argv) + import numpy as np + import scipy + import sklearn + import torch + import transformers + from threadpoolctl import threadpool_limits + from transformers import AutoModelForCausalLM, AutoTokenizer + + from fusegpt.come import batch_calibration, enable_torch24_dtensor_save_compat + from fusegpt.mka import ( + UPSTREAM, + UPSTREAM_COMMIT, + GeometryConfig, + progressive_prune, + sample_slimpajama, + ) + + geometry = GeometryConfig( + sigma=args.sigma, + alpha=args.alpha, + dimensions=args.manifold_dimensions, + mi_neighbors=args.mi_neighbors, + entropy_neighbors=args.entropy_neighbors, + beta=args.beta, + seed=args.seed, + pooling=args.pooling, + ) + geometry.validate() + if min(args.batch_size, args.cpu_threads, args.sequence_length) < 1: + raise ValueError("batch size, CPU threads and sequence length must be positive") + if args.calibration_samples <= args.manifold_dimensions + 1: + raise ValueError("calibration sample count must exceed manifold dimensions + 1") + torch.set_num_threads(args.cpu_threads) + torch.manual_seed(args.seed) + output = Path(args.output_dir) + output.mkdir(parents=True, exist_ok=True) + if any(output.iterdir()): + raise FileExistsError( + "output directory must be empty; refusing to overwrite an earlier run" + ) + tokenizer = AutoTokenizer.from_pretrained( + args.model, revision=args.revision, trust_remote_code=args.trust_remote_code + ) + if args.calibration_ids: + samples, spec = replay_calibration(args.calibration_ids, tokenizer, args) + else: + samples, spec = sample_slimpajama( + tokenizer, + samples=args.calibration_samples, + sequence_length=args.sequence_length, + seed=args.seed, + revision=args.calibration_revision, + ) + spec.update( + { + "tokenizer": args.model, + "tokenizer_revision": args.revision, + "tokenizer_vocab_sha256": vocabulary_checksum(tokenizer), + } + ) + torch.save( + {"input_ids": [x["input_ids"][0] for x in samples], "metadata": spec}, + output / "calibration_ids.pt", + ) + write_json(output / "calibration.json", spec) + batches = batch_calibration(samples, args.batch_size) + previous, parent_hash = (read_resume_protocol(args.resume_checkpoint, args, geometry, spec) + if args.resume_checkpoint else (None, None)) + model = AutoModelForCausalLM.from_pretrained( + args.resume_checkpoint or args.model, + revision=None if previous else args.revision, + torch_dtype=getattr(torch, args.dtype), + device_map=args.device_map, + low_cpu_mem_usage=True, + attn_implementation=args.attn_implementation, + trust_remote_code=args.trust_remote_code, + ) + if model.config.model_type not in {"llama", "qwen3", "qwen2"}: + raise ValueError( + "only native Llama, Qwen3 and Qwen2 decoder architectures are supported" + ) + current_depth = len(model.model.layers) + original_depth = previous["original_depth"] if previous else current_depth + if previous and current_depth != previous["actual_depth"]: + raise ValueError("resume checkpoint config depth differs from merge history") + if args.remove_blocks is not None and not 1 <= args.remove_blocks < original_depth: + raise ValueError("remove-blocks must be positive and less than original depth") + target = ( + args.target_layers + if args.target_layers is not None + else original_depth - args.remove_blocks + ) + if not 1 <= target < current_depth: + raise ValueError( + "target must remove at least one block and retain at least one" + ) + if any(not original_depth - current_depth < x <= original_depth - target for x in args.save_at_removals): + raise ValueError( + "save-at-removals must fall within the requested pruning trajectory" + ) + if ( + spec["samples"] != args.calibration_samples + or args.sequence_length != spec["sequence_length"] + ): + raise ValueError("calibration provenance does not match requested counts") + resolved_revision = previous["resolved_model_revision"] if previous else getattr(model.config, "_commit_hash", None) + git = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=False, + ) + metadata = { + "method": ("MKA corrected + input transport (no gradient training)" if args.transport + else "MKA corrected variant (training-free)"), + "upstream": UPSTREAM, + "upstream_commit": UPSTREAM_COMMIT, + "source_model": args.model, + "resolved_model_revision": resolved_revision, + "git_commit": git.stdout.strip(), + "arguments": vars(args), + "geometry": asdict(geometry), + "calibration": spec, + "original_depth": original_depth, + "target_depth": target, + "allow_repeat": True, + "transport": {"enabled": args.transport, "ridge_relative": args.transport_ridge, + "fit_fraction": args.transport_fit_fraction, + "scope": "attention q/k/v and MLP gate/up; separate maps"}, + "norm_policy": "retain left residual norms and Q/K norms; seven projection weights and their existing biases", + "implementation_differences": [ + "sequence-level masked pooling", + ( + "adjacent maximum NPIB selection with deeper exact-tie priority" + if args.selection == "npib" + else "fixed back-to-front adjacent merge order" + ), + "recompute current-model geometry every stage", + "actual parameter copy_", + "independent SlimPajama train calibration; no MMLU test inputs", + ], + "environment": { + "python": platform.python_version(), + "torch": torch.__version__, + "transformers": transformers.__version__, + "numpy": np.__version__, + "scipy": scipy.__version__, + "sklearn": sklearn.__version__, + }, + "status": "running", + "merges": previous["merges"][:] if previous else [], + } + if previous: + metadata["resume_provenance"] = {"checkpoint": str(Path(args.resume_checkpoint).resolve()), + "parent_protocol_sha256": parent_hash, + "parent_git_commit": previous["git_commit"], + "previous_merges": len(previous["merges"]), + "starting_depth": current_depth} + for record in previous["merges"]: + write_json(output / "stages" / f"stage_{record['step']:04d}.json", record) + write_json(output / "mka_pruning.json", metadata) + enable_torch24_dtensor_save_compat() + + def save_checkpoint(current, directory): + current.save_pretrained(directory, safe_serialization=True) + tokenizer.save_pretrained(directory) + + def journal(current, record): + metadata["merges"].append(record) + write_json(output / "stages" / f"stage_{record['step']:04d}.json", record) + write_json(output / "mka_pruning.json", metadata) + removed = original_depth - record["remaining_layers"] + if removed in args.save_at_removals: + directory = output / f"removed_{removed:02d}" + save_checkpoint(current, directory) + for name in ("calibration.json", "calibration_ids.pt"): + shutil.copy2(output / name, directory / name) + write_json( + directory / "mka_pruning.json", + { + **metadata, + "status": "checkpoint_complete", + "actual_depth": record["remaining_layers"], + "requested_final_depth": target, + "target_depth": record["remaining_layers"], + }, + ) + + use_cache = model.config.use_cache + try: + with threadpool_limits(limits=args.cpu_threads): + progressive_prune( + model, + batches, + target_layers=target, + geometry=geometry, + selection=args.selection, + protected_first=args.protected_first, + protected_last=args.protected_last, + on_stage=journal, + transport=args.transport, + transport_ridge=args.transport_ridge, + transport_fit_fraction=args.transport_fit_fraction, + initial_lineage=previous["merges"][-1]["lineage_after"] if previous else None, + ) + model.config.use_cache = use_cache + save_checkpoint(model, output) + metadata["actual_depth"] = len(model.model.layers) + if args.verify_reload: + probe = { + key: value[:, : min(args.sequence_length, 16)] + for key, value in samples[0].items() + } + + def probe_logits(current): + current.eval() + device = current.get_input_embeddings().weight.device + with torch.inference_mode(): + logits = ( + current( + **{k: v.to(device) for k, v in probe.items()}, + use_cache=False, + ) + .logits[:, -1] + .float() + .cpu() + ) + if not torch.isfinite(logits).all(): + raise FloatingPointError("nonfinite reload probe logits") + return logits + + expected = probe_logits(model) + del model + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + restored = AutoModelForCausalLM.from_pretrained( + output, + torch_dtype=getattr(torch, args.dtype), + device_map=args.device_map, + attn_implementation=args.attn_implementation, + trust_remote_code=args.trust_remote_code, + ) + actual = probe_logits(restored) + error = float((expected - actual).abs().max()) + tolerance = 0.05 if args.dtype == "bfloat16" else 0.005 + if len(restored.model.layers) != target or not torch.allclose( + expected, actual, atol=tolerance, rtol=tolerance + ): + raise AssertionError( + f"save/reload probe mismatch: max_abs_error={error}" + ) + metadata["reload_verification"] = { + "passed": True, + "max_abs_logit_error": error, + "atol": tolerance, + "rtol": tolerance, + "probe_tokens": probe["input_ids"].shape[1], + } + else: + metadata["reload_verification"] = { + "passed": None, + "reason": "disabled by CLI", + } + metadata["status"] = "complete" + except Exception as error: + metadata["status"] = "failed" + metadata["failure"] = {"type": type(error).__name__, "message": str(error)} + write_json(output / "mka_pruning.json", metadata) + raise + write_json(output / "mka_pruning.json", metadata) + print(f"Saved {target}-layer MKA model to {output}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_qwen3_8b_base_replaceme_depth.sh b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_qwen3_8b_base_replaceme_depth.sh new file mode 100644 index 0000000000000000000000000000000000000000..368d66332b676090f322b90d025d351f1e8542a3 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_qwen3_8b_base_replaceme_depth.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 7 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +repo="$(cd "$1" && pwd)" +project="$(cd "$2" && pwd)" +root="$(mkdir -p "$3" && cd "$3" && pwd)" +model="$(cd "$4" && pwd)" +blocks="$5" +removed_start="$6" +gpu="$7" +python_bin="${PYTHON_BIN:-python}" +bf16_patch="$project/patches/replaceme_llama_instruction_mask_bf16.patch" +qwen_patch="$project/patches/replaceme_qwen_layer_types.patch" +workdir="$repo/compare_model/ReplaceMe" +staged_model="$root/model_staged_ReplaceMe_lstsq_1" +final_model="$root/model" + +[[ "$blocks" =~ ^(5|9|14|18)$ ]] +[[ "$removed_start" =~ ^[0-9]+$ ]] +removed_end=$((removed_start + blocks)) +(( removed_start >= 1 && removed_end <= 36 )) + +export CUDA_VISIBLE_DEVICES="$gpu" +export HF_HOME=/workspace/.hf_home +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +mkdir -p "$root"/{logs,metadata,eval} + +test -f "$model/.model_download_complete" +test -f "$model/config.json" + +( + flock 9 + for patch in "$bf16_patch" "$qwen_patch"; do + if git -C "$repo" apply --check "$patch"; then + git -C "$repo" apply "$patch" + elif git -C "$repo" apply --reverse --check "$patch"; then + : + else + echo "RePlaceMe patch does not apply cleanly: $patch" >&2 + exit 1 + fi + done +) 9>"$repo/.replaceme_patch.lock" + +"$python_bin" -m py_compile \ + "$repo/compare_model/ReplaceMe/ReplaceMe/lstsq.py" \ + "$repo/compare_model/ReplaceMe/ReplaceMe/utils.py" \ + "$repo/compare_model/ReplaceMe/ReplaceMe/ReplaceMe_pipeline.py" +git -C "$repo" diff --binary > "$root/metadata/replaceme_source_patch.diff" +git -C "$repo" rev-parse HEAD > "$root/metadata/replaceme_source_commit.txt" + +"$python_bin" - "$root" "$model" "$blocks" "$removed_start" "$removed_end" <<'PY' +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +model = sys.argv[2] +blocks = int(sys.argv[3]) +removed_start = int(sys.argv[4]) +removed_end = int(sys.argv[5]) +(root / "selected_blocks.json").write_text( + json.dumps({"replaceme_blocks": [[removed_start, removed_end]]}, indent=2) + "\n" +) +config = { + "method": "lstsq", + "model_path": model, + "dataset": "DKYoon/SlimPajama-6B", + "dataset_column": "text", + "batch_size": 8, + "max_length": 1024, + "dataset_size": 10000, + "dataset_subset": "train", + "use_4bit": False, + "save_path": str(root / "model_staged"), + "save_transform_only": True, + "diag": False, + "alpha": 0.0, + "distances_path": None, + "num_A": 1, + "layers_to_skip": blocks, + "merge_consecutive": True, + "selected_blocks_path": str(root / "selected_blocks.json"), + "seed": 42, + "tasks": None, +} +(root / "config.json").write_text(json.dumps(config, indent=2) + "\n") +protocol = { + "method": "ReplaceMe streaming closed-form LSTSQ", + "model": "Qwen/Qwen3-8B-Base", + "model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "selected_span_source": "SlimPajama-500 original RePlaceMe angular-distance selector", + "removed_interval": [removed_start, removed_end], + "removed_original_layers": list(range(removed_start, removed_end)), + "left_boundary_layer": removed_start - 1, + "calibration": { + "dataset": "DKYoon/SlimPajama-6B", + "records": 10000, + "sequence_length": 1024, + "batch_size": 8, + "valid_tokens_only": True, + "dtype": "bfloat16", + "seed": 42, + }, + "fit": {"solver": "closed_form_lstsq", "ridge_alpha": 0.0}, +} +(root / "protocol.json").write_text(json.dumps(protocol, indent=2) + "\n") +PY + +if [[ ! -f "$root/TRAIN_COMPLETE" ]]; then + if [[ -e "$final_model" ]]; then + echo "Refusing to overwrite existing final model: $final_model" >&2 + exit 2 + fi + ( + cd "$workdir" + PYTHONPATH="$workdir:$repo:$project/src${PYTHONPATH:+:$PYTHONPATH}" \ + "$python_bin" -u run_replaceme.py --config "$root/config.json" + ) > "$root/logs/train.log" 2>&1 + test -f "$staged_model/config.json" + mv -- "$staged_model" "$final_model" + touch "$root/TRAIN_COMPLETE" +fi + +touch "$root/MODEL_READY_NO_EVAL" +echo "qwen3_8b_base_replaceme_${blocks}block=model_ready_evaluation_deferred" diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_qwen3_8b_native_span_transport_additive_u.sh b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_qwen3_8b_native_span_transport_additive_u.sh new file mode 100644 index 0000000000000000000000000000000000000000..cdd7a363ba2449b94f078c5593f48aeb204611d3 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_qwen3_8b_native_span_transport_additive_u.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 7 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +repo="$(cd "$1" && pwd)" +project="$(cd "$2" && pwd)" +root="$(mkdir -p "$3" && cd "$3" && pwd)" +model="$(cd "$4" && pwd)" +anchor="$5" +cycles="$6" +gpu="$7" +fit_tokens="${ABPRUNE_FIT_TARGET_TOKENS:-524288}" +calib_sequences="${ABPRUNE_CALIB_SEQUENCES:-64}" + +model_id=Qwen/Qwen3-8B-Base +revision=49e3418fbbbca6ecbdf9608b4d22e5a407081db4 +num_layers=36 +removed_start=$((anchor + 1)) +removed_end=$((removed_start + cycles)) +layer_order=() +for _ in $(seq 1 "$cycles"); do layer_order+=("$anchor"); done + +export HF_HOME=/workspace/.hf_home +export HF_TOKEN_PATH=/root/.cache/huggingface/token +export HF_XET_HIGH_PERFORMANCE=1 +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export PYTHONPATH="$repo/src:$project/src${PYTHONPATH:+:$PYTHONPATH}" +mkdir -p "$root/logs" "$root/run" "$root/compact" "$root/eval" "$root/metadata" + +python - "$root/protocol.json" "$anchor" "$cycles" "$removed_start" "$removed_end" "$fit_tokens" "$calib_sequences" <<'PY' +import json +import sys +from pathlib import Path + +output, anchor, cycles, removed_start, removed_end, fit_tokens, calib_sequences = sys.argv[1:] +anchor = int(anchor) +cycles = int(cycles) +removed_start = int(removed_start) +removed_end = int(removed_end) +Path(output).write_text(json.dumps({ + "experiment": f"qwen3_8b_native_cosine_transport_additive_u_{cycles}block_v1", + "model": "Qwen/Qwen3-8B-Base", + "revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "selection": { + "method": "native block-output-aligned contiguous cosine", + "dataset": "DKYoon/SlimPajama-6B", + "sequences": 500, + "sequence_length": 1024, + "seed": 20260910, + "anchor": anchor, + "removed_interval": [removed_start, removed_end], + }, + "anchor": anchor, + "removed_original_layers": list(range(removed_start, removed_end)), + "cycles": cycles, + "merge": { + "fisher": "same-anchor post-pruning contextual diagonal parameter Fisher", + "fisher_eps": 0.0, + "source_feature_transport": "X_anchor @ T ~= X_source; gate/up W_source <- W_source @ T.T", + "transport_ridge_relative": 1e-3, + "fit_parameterization": "W = W_F + U_add; U_add initialized at zero; Fisher gates frozen", + "rho_scaling": False, + }, + "training": { + "dataset": "DKYoon/SlimPajama-6B", + "sequence_length": 1024, + "target_tokens_per_stage": int(fit_tokens), + "calibration_sequences": int(calib_sequences), + "distill_epochs_per_stage": 1.0, + "distill_kl_weight": 0.02, + "hidden_mse_weight": 1.0, + "mlp_mse_weight": 1.0, + "parameter_subset": "mlp", + }, +}, indent=2) + "\n", encoding="utf-8") +PY + +python -m py_compile \ + "$repo/src/fuse_layers.py" \ + "$repo/src/fuse_layers_distill.py" \ + "$repo/src/source_feature_transport.py" +PYTHONPATH="$repo/src" python "$project/scripts/test_abprune_additive_u.py" + +if [[ ! -f "$root/TRAIN_COMPLETE" ]]; then + ( + export CUDA_VISIBLE_DEVICES="$gpu" + export DATASET=slimpajama DATASET_CONFIG=none + export NUM_PROGRESSIVE="$cycles" SEQ_LEN=1024 DISTILL_SEQ_LEN=1024 + export TARGET_TOKENS="$fit_tokens" CALIB_SEQUENCES="$calib_sequences" + export BATCH_SIZE=2 DISTILL_BATCH_SIZE=1 DISTILL_EPOCHS=1.0 + export DISTILL_KL_WEIGHT=0.02 LORA_EPOCHS=0 EVAL_NUM_SAMPLES=0 + export SKIP_FINAL_MODEL_SAVE=1 COMM_ENABLED=0 HEAD_PERMUTE=0 + export REPARAM_PARAM_SUBSET=mlp REPARAM_FISHER_INIT_SUBSET=same + export REPARAM_TRAIN_SUBSET=same REPARAM_RHO_OBJECTIVE=none + export DISTILL_HIDDEN_MSE_WEIGHT=1.0 DISTILL_ATTN_MSE_WEIGHT=0.0 + export DISTILL_MLP_MSE_WEIGHT=1.0 + export DTYPE=bfloat16 EVAL_DTYPE=bfloat16 DISABLE_TQDM=1 + unset SAVE_FULL_MODEL_CYCLES + + bash "$repo/script/run_fuse_layers_order.sh" "$model" "$root/run" \ + --layer_order "${layer_order[@]}" \ + --eps 0 --skip_eval --distill_log_steps 100 --seed 42 \ + --distill_lr 1e-4 --distill_weight_decay 0 --distill_max_grad_norm 1 --distill_grad_accum_steps 1 \ + --source_feature_transport \ + --source_feature_transport_ridge_relative 1e-3 \ + --reparam_additive_u + ) >> "$root/logs/train.log" 2>&1 + touch "$root/TRAIN_COMPLETE" +fi + +if [[ ! -f "$root/COMPACT_COMPLETE" ]]; then + python -m fusegpt.abprune_compact \ + --run-root "$root/run" --output-root "$root/compact" \ + --model "$model_id" --base-revision "$revision" \ + --run-id "qwen3_8b_native_cosine_transport_additive_u_${cycles}block_v1" \ + --cycles "$cycles" --layer-order "${layer_order[*]}" + for stage in $(seq 1 "$cycles"); do + padded="$(printf '%02d' "$stage")" + cp "$root/run/cycle_${stage}/cycle_metadata.json" \ + "$root/metadata/cycle_${padded}.json" + done + touch "$root/COMPACT_COMPLETE" +fi + +if [[ -f "$root/DEFER_EVAL" || "${DEFER_EVAL:-0}" == "1" ]]; then + touch "$root/MODEL_READY_NO_EVAL" + echo "qwen3_8b_native_transport_${cycles}block=model_ready_evaluation_deferred" + exit 0 +fi + +for stage in $(seq 1 "$cycles"); do + padded="$(printf '%02d' "$stage")" + if [[ ! -f "$root/eval/stage_${padded}_ppl.json" ]]; then + CUDA_VISIBLE_DEVICES="$gpu" python "$project/scripts/evaluate_abprune_compact_ppl.py" \ + --model "$model" --revision "$revision" \ + --compact-stage "$root/compact/stage_${padded}" \ + --anchor "$anchor" --cycles "$stage" --original-layers "$num_layers" \ + --seq-len 2048 --batch-size 4 \ + --output "$root/eval/stage_${padded}_ppl.json" \ + >> "$root/logs/eval.log" 2>&1 + fi +done + +final_padded="$(printf '%02d' "$cycles")" +if [[ ! -f "$root/eval/stage_${final_padded}_zs6.json" ]]; then + CUDA_VISIBLE_DEVICES="$gpu" python "$project/scripts/eval_abprune_compact_zeroshot.py" \ + --model "$model" --revision "$revision" \ + --compact-stage "$root/compact/stage_${final_padded}" \ + --cycle "$cycles" --anchor "$anchor" --batch-size auto:4 \ + --output "$root/eval/stage_${final_padded}_zs6.json" \ + >> "$root/logs/eval.log" 2>&1 +fi + +python "$project/scripts/summarize_qwen3_8b_native_span_transport.py" "$root" +touch "$root/COMPLETE" +echo "qwen3_8b_native_transport_${cycles}block=complete" diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_qwen3_base_half2.py b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_qwen3_base_half2.py new file mode 100644 index 0000000000000000000000000000000000000000..89fbeb5be0c6407717500b8efc380d6e694a1470 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/code/run_qwen3_base_half2.py @@ -0,0 +1,254 @@ +"""Two GPU lanes: native CoMe/MKA/ReplaceMe and Fisher+Transport+FiT ABPrune.""" +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor +import datetime +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +import traceback + +REV='49e3418fbbbca6ecbdf9608b4d22e5a407081db4' +METHODS=('come','mka','replaceme','abprune') + + +def read(path): + return json.loads(Path(path).read_text()) + + +def write(path, value): + path=Path(path); path.parent.mkdir(parents=True,exist_ok=True) + tmp=path.with_suffix(path.suffix+'.tmp') + tmp.write_text(json.dumps(value,indent=2)+'\n'); tmp.replace(path) + + +def main(): + p=argparse.ArgumentParser() + for name in ('project','external','root','model','session'): + p.add_argument('--'+name,required=True) + p.add_argument('--lane',choices=('all','baselines','abprune'),default='all') + a=p.parse_args() + project,external,root,model=map(Path,(a.project,a.external,a.root,a.model)) + assert root.resolve().is_relative_to(Path('/workspace/artifacts')) + assert subprocess.check_output(['git','-C',str(project),'branch','--show-current'],text=True).strip()=='codex/'+a.session + root.mkdir(parents=True,exist_ok=True) + while not (root/'HALF_READY').exists(): + if (root/'BOOTSTRAP_FAILED').exists(): + raise RuntimeError('Bootstrap failed; inspect bootstrap.log') + time.sleep(3) + assert read(model/'config.json')['num_hidden_layers']==36 + env=os.environ.copy() + env.update(HF_HOME='/workspace/.hf_home',HF_TOKEN_PATH='/root/.cache/huggingface/token', + HF_DATASETS_CACHE='/workspace/.hf_datasets36',HF_DATASETS_TRUST_REMOTE_CODE='1', + PYTHONUNBUFFERED='1',TOKENIZERS_PARALLELISM='false',OMP_NUM_THREADS='4', + PYTHONPATH=f'{project}/src:{external}/src:{project}',ABPRUNE_DATASET_REVISION='b5f90f419b7489cdba26fdbc8c022fcb5562f968') + commit=subprocess.check_output(['git','-C',str(project),'rev-parse','HEAD'],text=True).strip() + previous_execution=read(root/'execution.json') if (root/'execution.json').exists() else None + write(root/'execution.json',dict(commit=commit,session=a.session,started=datetime.datetime.now(datetime.timezone.utc).isoformat(), + gpu0=['come','mka','replaceme'],gpu1=['abprune'],removed_blocks=18,original_layers=36,auto_return=False, + lane=a.lane,previous_execution=previous_execution)) + + def run(cmd,log,gpu=None,offline=False,extra=None): + local=env.copy() + if gpu is not None: local['CUDA_VISIBLE_DEVICES']=str(gpu) + if offline: + local.update(HF_HUB_OFFLINE='1',HF_DATASETS_OFFLINE='1',TRANSFORMERS_OFFLINE='1') + if extra: local.update({k:str(v) for k,v in extra.items()}) + log=Path(log); log.parent.mkdir(parents=True,exist_ok=True) + with log.open('a') as stream: + stream.write(json.dumps({'utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'command':list(map(str,cmd))})+'\n'); stream.flush() + subprocess.run(list(map(str,cmd)),env=local,cwd=project,stdout=stream,stderr=subprocess.STDOUT,check=True) + + def phase(method,text): + write(root/'full'/method/'status.json',dict(phase=text,updated=datetime.datetime.now(datetime.timezone.utc).isoformat())) + + def select(): + s=root/'selection'; s.mkdir(exist_ok=True) + if not (s/'selection.json').exists(): + run(['bash',str(external)+'-legacy/run_compare_layer_selection_metrics.sh'],s/'selection.log',1, + extra=dict(MODEL=model,PRUNE_LAYERS=18,DATASET='slimpajama',NUM_SEQUENCES=500,SEQ_LEN=1024, + BATCH_SIZE=2,DTYPE='bfloat16',DEVICE='cuda',SEED=42, + CHUNK_CACHE_PATH=s/'slimpajama500_seed42.pt',OUTPUT_JSON=s/'selection.json')) + rows=read(s/'selection.json')['selections'] + chosen={x['method']:x for x in rows if x['method'] in ('fuse_region_cosine','replaceme')} + assert set(chosen)=={'fuse_region_cosine','replaceme'} + for x in chosen.values(): + lo=int(x['removed_start']) + # The unmodified 39e selector emits an inclusive removed_end. + hi=int(x['removed_end'])+1 if 'removed_interval_half_open' not in x else x['removed_interval_half_open'][1] + assert hi-lo==18 and 1<=lo 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/artifacts/qwen3-base-half2-20261012/full/replaceme/model", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: non-empty rows joined with double newline before one-shot tokenization", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "residual_anchor": null, + "residual_gamma": null, + "metrics": { + "nll": 5.628321035977067, + "perplexity": 278.1946465063952, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + } +} diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/logs/pruning.log b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/logs/pruning.log new file mode 100644 index 0000000000000000000000000000000000000000..79315ed30d29f64c4a1161e0a48169f25b8d5c05 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/logs/pruning.log @@ -0,0 +1,6 @@ +{"utc": "2026-10-12T14:50:19.049172+00:00", "command": ["bash", "/workspace/paper_revision/sessions/vast-paper_revision-qwen3-base-half2-55563310-20261012/scripts/run_qwen3_8b_base_replaceme_depth.sh", "/workspace/external/qwen3-base-half2-legacy", "/workspace/paper_revision/sessions/vast-paper_revision-qwen3-base-half2-55563310-20261012", "/workspace/artifacts/qwen3-base-half2-20261012/full/replaceme", "/workspace/models/qwen3-8b-base-49e3418", "18", "14", "0"]} +error: patch failed: compare_model/ReplaceMe/ReplaceMe/lstsq.py:92 +error: compare_model/ReplaceMe/ReplaceMe/lstsq.py: patch does not apply +error: patch failed: compare_model/ReplaceMe/ReplaceMe/utils.py:80 +error: compare_model/ReplaceMe/ReplaceMe/utils.py: patch does not apply +qwen3_8b_base_replaceme_18block=model_ready_evaluation_deferred diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/logs/train.log b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/logs/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8c57e2cdda14fc5929876b257052a5ab4123b7ef --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/logs/train.log @@ -0,0 +1,20 @@ +Seeded with seed: 42 +Seeded with seed: 42 +Seeded with seed: 42 +Seeded with seed: 42 +Seeded with seed: 42 +Seeded with seed: 42 +2026-10-12 14:50:21 [INFO] Using shared selected span; skipping ReplaceMe distance profiling +[transformers] The following generation flags are not valid and may be ignored: ['output_hidden_states']. Set `TRANSFORMERS_VERBOSITY=info` for more details. + Loading weights: 0%| | 0/399 [00:00system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/config.json b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1b9e65d7fb5936461ef1dae03163d30e5162e5e7 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/config.json @@ -0,0 +1,53 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151643, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 18, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.15.1", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/generation_config.json b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..84be359b05aa40afa929fa149a2b6366797d1132 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/generation_config.json @@ -0,0 +1,7 @@ +{ + "bos_token_id": 151643, + "do_sample": false, + "eos_token_id": 151643, + "max_new_tokens": 2048, + "transformers_version": "5.15.1" +} diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/model.safetensors b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fea32a3ef18ff8eb8e1d0be3d891ad362bec4c1d --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27fd55d4981905d0dc56c497562a61d733b63d8830f1ddc75982bdd70b31045a +size 9435422336 diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/tokenizer.json b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/tokenizer_config.json b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1127e125def6d7caf411d32f791f9d3d20ccd991 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/model/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/offline_eval.json b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/offline_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..1c11908f0422b8275108fcb9d5f81684a80e3ff4 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/offline_eval.json @@ -0,0 +1,15 @@ +{ + "wikitext_cache": "/workspace/artifacts/qwen3-base-half2-20261012/wikitext2_cache", + "dataset": "Salesforce/wikitext", + "config": "wikitext-2-raw-v1", + "splits": { + "test": 4358, + "train": 36718, + "validation": 3760 + }, + "fingerprints": { + "test": "a46124b21ac53738", + "train": "5d4fb603254a7a5b", + "validation": "51d84d52d095f90d" + } +} diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/protocol.json b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/protocol.json new file mode 100644 index 0000000000000000000000000000000000000000..999799764ef6426ee15dc64ed7832f51d03437b1 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/protocol.json @@ -0,0 +1,78 @@ +{ + "method": "ReplaceMe streaming closed-form LSTSQ", + "model": "Qwen/Qwen3-8B-Base", + "model_revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "selected_span_source": "SlimPajama-500 original RePlaceMe angular-distance selector", + "removed_interval": [ + 14, + 32 + ], + "removed_original_layers": [ + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "left_boundary_layer": 13, + "calibration": { + "dataset": "DKYoon/SlimPajama-6B", + "records": 10000, + "sequence_length": 1024, + "batch_size": 8, + "valid_tokens_only": true, + "dtype": "bfloat16", + "seed": 42 + }, + "fit": { + "solver": "closed_form_lstsq", + "ridge_alpha": 0.0 + }, + "selection": { + "method": "replaceme", + "score_direction": "min", + "removed_start": 14, + "removed_end": 31, + "compare_left": 13, + "compare_right": 31, + "selected_score": 0.31078056192398074, + "inferred_pruned_layers_zero_based": [ + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "notes": "Unified span score: compare last-token hidden states of the first and last removed layers.", + "removed_interval_half_open": [ + 14, + 32 + ] + } +} diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/protocol.md b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/protocol.md new file mode 100644 index 0000000000000000000000000000000000000000..c0bb9b664360ef69dc05622968bc62609fc11ff8 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/protocol.md @@ -0,0 +1,37 @@ +# Qwen3 Base 50 percent pruning with four methods + +Prune Qwen3 8B Base from 36 to 18 decoder blocks using native CoMe, corrected local MKA, ReplaceMe and Fisher plus Transport plus FiT ABPrune. GPU0 runs CoMe, MKA and ReplaceMe sequentially, advancing each immediately to WikiText2 PPL and Bench14. GPU1 selects the new 18-block spans, then runs ABPrune and its evaluations independently. This is a full-method comparison, not the earlier no-FiT merge-coefficient ablation. + +## Resources and provenance + +Instance `55563310`, offer `52960990`, machine `101241`, Lithuania: two RTX PRO 6000 Max-Q 96 GB GPUs, reported RAM 515109 MiB, 64 effective CPU cores, 220 GB disk. Listed total USD 2.7277777777777774 per hour with disk; traffic is additional. Offer reliability 0.9986672 and download/upload 2185.1/2158.0 Mbps. Session alias `vast-paper_revision-qwen3-base-half2-55563310-20261012`; branch `codex/vast-paper_revision-qwen3-base-half2-55563310-20261012`. + +Remote project `/workspace/paper_revision/sessions/vast-paper_revision-qwen3-base-half2-55563310-20261012`, artifacts `/workspace/artifacts/qwen3-base-half2-20261012`, patched ABPrune engine `/workspace/external/qwen3-base-half2`, original selector and ReplaceMe engine in the same prefix with suffix `-legacy`. Source code moves by Git bundle; the server receives no Git hosting credentials. Model `Qwen/Qwen3-8B-Base` is pinned to `49e3418fbbbca6ecbdf9608b4d22e5a407081db4`. + +Docker image `pytorch/pytorch:2.9.1-cuda12.8-cudnn9-devel`. Bootstrap pins transformers 5.15.1, datasets 3.6.0, Hugging Face Hub 1.33.0 and lm_eval 0.4.12. Exact Python, torch, CUDA and dependency versions are captured in `environment.json`, rather than inferred from the image. BF16 model and evaluation; FP32 transport solve. + +The ABPrune engine starts from vendor bundle commit `39e59afe09d0124e9bd95cb2ea1b52e84f5202a9`, plus immutable historical source overlay `61734e7b1a77fd279fe1e2584f7a0fdc26c175c2` and existing loader/mask compatibility patches. Native corrected MKA files were imported unchanged from committed branch `leejunyoung8631/rr3`, file-history commit `9c3fcb4d77ce1c91ffe8a36df4d96c0c7d87bf40`. This is the same corrected variant used by prior MKA baseline campaigns, not an exact paper reproduction and not the fixed-span MKA initialization ablation. + +## Data and method settings + +All calibration uses SlimPajama, not SlimOrca. ABPrune and ReplaceMe selectors share 500 packed 1024-token SlimPajama samples, seed42, batch2 and one chunk cache. Compute width18 from the original selection engine; do not extend the old width9 interval manually. Record `fuse_region_cosine` and `replaceme` selections separately. The dataset revision is `b5f90f419b7489cdba26fdbc8c022fcb5562f968`. + +ABPrune progressively absorbs the selected 18 original blocks into one fixed anchor. Same-anchor contextual parameter Fisher, epsilon0, source feature Transport ON with ridge relative 0.001, additive-U FiT on MLP only. Calibration64 x1024, batch2. Each stage fits524288 tokens, or512 x1024 sequences, batch1, one epoch, LR1e-4, weight decay0, no LoRA/head permutation. Joint loss keeps hidden MSE1, MLP MSE1 and KL0.02. A two-stage smoke uses4 calibration sequences and20 fitting sequences before the full18 stages. + +CoMe uses its native stagewise angular-last selector and channel coefficients:256 SlimPajama source rows packed into2048-token windows, seed10, batch1, power1, minimum retention0, allow-repeat, no Transport or FiT. MKA uses100 distinct-document SlimPajama windows x512, seed0, sequence mean pooling, current-model NPIB recomputed per stage, sigma8, alpha0.5, diffusion dimensions2, MI neighbors5, entropy neighbors50 and beta1. It merges all seven attention and MLP projections, retains anchor norms and allows repeated merges; no Transport or FiT. Preserve actual lineage in each native checkpoint. + +ReplaceMe uses its separately selected width18 interval and existing closed-form streaming least squares. SlimPajama10000 records, max length1024, batch8, seed42, BF16, valid-token masks, alpha0 and no4bit. Existing BF16 and Qwen layer-type compatibility patches are unchanged; the runner accepts width18 in addition to earlier5/9/14 widths. + +## Evaluation and retention + +Use full WikiText2 test HCHC packing,2048-token windows, BF16 eager, batch4, expected145 windows and296815 predicted tokens. Bench14 preserves established legacy OpenCompass-five full-prompt choice-label likelihood and lm-eval-nine zero-shot protocols: RACE-H, RACE-M, C3, CMNLI, CHID, PIQA, WinoGrande, HellaSwag, ARC-C, ARC-E, OpenBookQA, SciQ, MMLU and CMMLU. ZS12 excludes MMLU/CMMLU; Bench14 macro is the unweighted14-task mean. This is not the newer answer-sentence-only RACE experiment. + +Transfer hash-pinned local OpenCompass-five and131-configuration lm_eval dataset archives before evaluation. Warm and export WikiText once during preparation, then run all evaluations with strict offline datasets and local models. Missing caches fail explicitly without online fallback. + +Controller command: `python scripts/run_qwen3_base_half2.py --project PROJECT --external EXTERNAL --root ROOT --model MODEL --session SESSION`. Bootstrap command: `bash scripts/bootstrap_qwen3_base_half2.sh PROJECT EXTERNAL ROOT MODEL`. Execution commit, exact commands, stage logs, method protocols, selections and metrics are saved by the controller. + +Save native full checkpoints and the final compact ABPrune anchor with base-model pin. Back up each method with evaluation records, selectors, code and environment to private `LJYAI/paper_revision/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/METHOD`, verify all file hashes and record immutable revisions. Final results go to `results/qwen3_base_half18_20261012/`, followed by a remote commit and local bundle recovery/push. Do not return this new instance without further user instruction. + +## Current status + +Instance is running, SSH registration and host setup are in progress. No 50 percent result is complete at this checkpoint. Source branch is clean before new task files; no unrelated user changes are overwritten. The writing skill keeps the full-method comparison and the earlier no-FiT ablation distinct in the experiment record. diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/COMPLETE b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/COMPLETE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/selection.json b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/selection.json new file mode 100644 index 0000000000000000000000000000000000000000..d7e216ba00f53b5d13c9ec208c32d137df4bbdbc --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/selection.json @@ -0,0 +1,2119 @@ +{ + "num_hidden_layers": 36, + "num_batches": 250, + "num_examples": 500, + "selections": [ + { + "method": "llm_streamline", + "score_direction": "max", + "removed_start": 15, + "removed_end": 32, + "compare_left": 14, + "compare_right": 32, + "selected_score": 0.5545816756486893, + "inferred_pruned_layers_zero_based": [ + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32 + ], + "notes": "Unified span score: compare hidden_states[r-1] and hidden_states[r+w-1]." + }, + { + "method": "replaceme", + "score_direction": "min", + "removed_start": 14, + "removed_end": 31, + "compare_left": 13, + "compare_right": 31, + "selected_score": 0.31078056192398074, + "inferred_pruned_layers_zero_based": [ + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "notes": "Unified span score: compare last-token hidden states of the first and last removed layers." + }, + { + "method": "fuse_region_cosine", + "score_direction": "min", + "removed_start": 15, + "removed_end": 32, + "compare_left": 14, + "compare_right": 32, + "selected_score": 0.44541832387447355, + "inferred_pruned_layers_zero_based": [ + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32 + ], + "notes": "Unified span score: compare hidden_states[r-1] and hidden_states[r+w-1] using token-level 1-cosine." + }, + { + "method": "fuse_region_cosine_mlp_pre_residual", + "score_direction": "min", + "removed_start": 7, + "removed_end": 24, + "compare_left": 6, + "compare_right": 24, + "selected_score": 0.8522758095264434, + "inferred_pruned_layers_zero_based": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24 + ], + "notes": "Unified span score: compare pre-residual MLP outputs of the left boundary layer and the last removed layer." + } + ], + "per_start": { + "llm_streamline": [ + { + "removed_start": 1, + "removed_end": 18, + "compare_left": 0, + "compare_right": 18, + "score": 0.4001441835165024, + "inferred_pruned_layers_zero_based": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + { + "removed_start": 2, + "removed_end": 19, + "compare_left": 1, + "compare_right": 19, + "score": 0.41873254787921904, + "inferred_pruned_layers_zero_based": [ + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "removed_start": 3, + "removed_end": 20, + "compare_left": 2, + "compare_right": 20, + "score": 0.4322152615785599, + "inferred_pruned_layers_zero_based": [ + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ] + }, + { + "removed_start": 4, + "removed_end": 21, + "compare_left": 3, + "compare_right": 21, + "score": 0.458461238026619, + "inferred_pruned_layers_zero_based": [ + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ] + }, + { + "removed_start": 5, + "removed_end": 22, + "compare_left": 4, + "compare_right": 22, + "score": 0.49698565900325775, + "inferred_pruned_layers_zero_based": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ] + }, + { + "removed_start": 6, + "removed_end": 23, + "compare_left": 5, + "compare_right": 23, + "score": 0.513720078587532, + "inferred_pruned_layers_zero_based": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + }, + { + "removed_start": 7, + "removed_end": 24, + "compare_left": 6, + "compare_right": 24, + "score": 0.5292018551826477, + "inferred_pruned_layers_zero_based": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "removed_start": 8, + "removed_end": 25, + "compare_left": 7, + "compare_right": 25, + "score": 0.5275396839380264, + "inferred_pruned_layers_zero_based": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25 + ] + }, + { + "removed_start": 9, + "removed_end": 26, + "compare_left": 8, + "compare_right": 26, + "score": 0.51713119494915, + "inferred_pruned_layers_zero_based": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26 + ] + }, + { + "removed_start": 10, + "removed_end": 27, + "compare_left": 9, + "compare_right": 27, + "score": 0.5140425094366073, + "inferred_pruned_layers_zero_based": [ + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27 + ] + }, + { + "removed_start": 11, + "removed_end": 28, + "compare_left": 10, + "compare_right": 28, + "score": 0.5293747123479843, + "inferred_pruned_layers_zero_based": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28 + ] + }, + { + "removed_start": 12, + "removed_end": 29, + "compare_left": 11, + "compare_right": 29, + "score": 0.5324888409376144, + "inferred_pruned_layers_zero_based": [ + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "removed_start": 13, + "removed_end": 30, + "compare_left": 12, + "compare_right": 30, + "score": 0.552883662700653, + "inferred_pruned_layers_zero_based": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30 + ] + }, + { + "removed_start": 14, + "removed_end": 31, + "compare_left": 13, + "compare_right": 31, + "score": 0.5463030400276184, + "inferred_pruned_layers_zero_based": [ + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + { + "removed_start": 15, + "removed_end": 32, + "compare_left": 14, + "compare_right": 32, + "score": 0.5545816756486893, + "inferred_pruned_layers_zero_based": [ + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32 + ] + }, + { + "removed_start": 16, + "removed_end": 33, + "compare_left": 15, + "compare_right": 33, + "score": 0.5435223225355148, + "inferred_pruned_layers_zero_based": [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33 + ] + }, + { + "removed_start": 17, + "removed_end": 34, + "compare_left": 16, + "compare_right": 34, + "score": 0.5471235717535019, + "inferred_pruned_layers_zero_based": [ + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "removed_start": 18, + "removed_end": 35, + "compare_left": 17, + "compare_right": 35, + "score": 0.5483626971244812, + "inferred_pruned_layers_zero_based": [ + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35 + ] + } + ], + "replaceme": [ + { + "removed_start": 1, + "removed_end": 18, + "compare_left": 0, + "compare_right": 18, + "score": 0.3623302321434021, + "inferred_pruned_layers_zero_based": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + { + "removed_start": 2, + "removed_end": 19, + "compare_left": 1, + "compare_right": 19, + "score": 0.3579166704416275, + "inferred_pruned_layers_zero_based": [ + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "removed_start": 3, + "removed_end": 20, + "compare_left": 2, + "compare_right": 20, + "score": 0.3477127702236176, + "inferred_pruned_layers_zero_based": [ + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ] + }, + { + "removed_start": 4, + "removed_end": 21, + "compare_left": 3, + "compare_right": 21, + "score": 0.33255866980552673, + "inferred_pruned_layers_zero_based": [ + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ] + }, + { + "removed_start": 5, + "removed_end": 22, + "compare_left": 4, + "compare_right": 22, + "score": 0.32646954023838043, + "inferred_pruned_layers_zero_based": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ] + }, + { + "removed_start": 6, + "removed_end": 23, + "compare_left": 5, + "compare_right": 23, + "score": 0.3202547243833542, + "inferred_pruned_layers_zero_based": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + }, + { + "removed_start": 7, + "removed_end": 24, + "compare_left": 6, + "compare_right": 24, + "score": 0.32113656747341157, + "inferred_pruned_layers_zero_based": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "removed_start": 8, + "removed_end": 25, + "compare_left": 7, + "compare_right": 25, + "score": 0.32462449729442594, + "inferred_pruned_layers_zero_based": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25 + ] + }, + { + "removed_start": 9, + "removed_end": 26, + "compare_left": 8, + "compare_right": 26, + "score": 0.3259439049959183, + "inferred_pruned_layers_zero_based": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26 + ] + }, + { + "removed_start": 10, + "removed_end": 27, + "compare_left": 9, + "compare_right": 27, + "score": 0.3204443471431732, + "inferred_pruned_layers_zero_based": [ + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27 + ] + }, + { + "removed_start": 11, + "removed_end": 28, + "compare_left": 10, + "compare_right": 28, + "score": 0.31944401013851165, + "inferred_pruned_layers_zero_based": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28 + ] + }, + { + "removed_start": 12, + "removed_end": 29, + "compare_left": 11, + "compare_right": 29, + "score": 0.3113690857887268, + "inferred_pruned_layers_zero_based": [ + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "removed_start": 13, + "removed_end": 30, + "compare_left": 12, + "compare_right": 30, + "score": 0.3139335271120071, + "inferred_pruned_layers_zero_based": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30 + ] + }, + { + "removed_start": 14, + "removed_end": 31, + "compare_left": 13, + "compare_right": 31, + "score": 0.31078056192398074, + "inferred_pruned_layers_zero_based": [ + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + { + "removed_start": 15, + "removed_end": 32, + "compare_left": 14, + "compare_right": 32, + "score": 0.315525742650032, + "inferred_pruned_layers_zero_based": [ + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32 + ] + }, + { + "removed_start": 16, + "removed_end": 33, + "compare_left": 15, + "compare_right": 33, + "score": 0.31517581713199616, + "inferred_pruned_layers_zero_based": [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33 + ] + }, + { + "removed_start": 17, + "removed_end": 34, + "compare_left": 16, + "compare_right": 34, + "score": 0.3145262675285339, + "inferred_pruned_layers_zero_based": [ + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "removed_start": 18, + "removed_end": 35, + "compare_left": 17, + "compare_right": 35, + "score": 0.49567853558063507, + "inferred_pruned_layers_zero_based": [ + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35 + ] + } + ], + "fuse_region_cosine": [ + { + "removed_start": 1, + "removed_end": 18, + "compare_left": 0, + "compare_right": 18, + "score": 0.5998558185100555, + "inferred_pruned_layers_zero_based": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + { + "removed_start": 2, + "removed_end": 19, + "compare_left": 1, + "compare_right": 19, + "score": 0.5812674527168273, + "inferred_pruned_layers_zero_based": [ + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "removed_start": 3, + "removed_end": 20, + "compare_left": 2, + "compare_right": 20, + "score": 0.5677847363948823, + "inferred_pruned_layers_zero_based": [ + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ] + }, + { + "removed_start": 4, + "removed_end": 21, + "compare_left": 3, + "compare_right": 21, + "score": 0.541538762331009, + "inferred_pruned_layers_zero_based": [ + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ] + }, + { + "removed_start": 5, + "removed_end": 22, + "compare_left": 4, + "compare_right": 22, + "score": 0.5030143401622772, + "inferred_pruned_layers_zero_based": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ] + }, + { + "removed_start": 6, + "removed_end": 23, + "compare_left": 5, + "compare_right": 23, + "score": 0.486279922246933, + "inferred_pruned_layers_zero_based": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + }, + { + "removed_start": 7, + "removed_end": 24, + "compare_left": 6, + "compare_right": 24, + "score": 0.4707981493473053, + "inferred_pruned_layers_zero_based": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "removed_start": 8, + "removed_end": 25, + "compare_left": 7, + "compare_right": 25, + "score": 0.4724603159427643, + "inferred_pruned_layers_zero_based": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25 + ] + }, + { + "removed_start": 9, + "removed_end": 26, + "compare_left": 8, + "compare_right": 26, + "score": 0.48286880588531494, + "inferred_pruned_layers_zero_based": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26 + ] + }, + { + "removed_start": 10, + "removed_end": 27, + "compare_left": 9, + "compare_right": 27, + "score": 0.48595749270915983, + "inferred_pruned_layers_zero_based": [ + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27 + ] + }, + { + "removed_start": 11, + "removed_end": 28, + "compare_left": 10, + "compare_right": 28, + "score": 0.47062529051303864, + "inferred_pruned_layers_zero_based": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28 + ] + }, + { + "removed_start": 12, + "removed_end": 29, + "compare_left": 11, + "compare_right": 29, + "score": 0.46751115810871124, + "inferred_pruned_layers_zero_based": [ + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "removed_start": 13, + "removed_end": 30, + "compare_left": 12, + "compare_right": 30, + "score": 0.4471163386106491, + "inferred_pruned_layers_zero_based": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30 + ] + }, + { + "removed_start": 14, + "removed_end": 31, + "compare_left": 13, + "compare_right": 31, + "score": 0.453696959733963, + "inferred_pruned_layers_zero_based": [ + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + { + "removed_start": 15, + "removed_end": 32, + "compare_left": 14, + "compare_right": 32, + "score": 0.44541832387447355, + "inferred_pruned_layers_zero_based": [ + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32 + ] + }, + { + "removed_start": 16, + "removed_end": 33, + "compare_left": 15, + "compare_right": 33, + "score": 0.4564776759147644, + "inferred_pruned_layers_zero_based": [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33 + ] + }, + { + "removed_start": 17, + "removed_end": 34, + "compare_left": 16, + "compare_right": 34, + "score": 0.45287642765045166, + "inferred_pruned_layers_zero_based": [ + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "removed_start": 18, + "removed_end": 35, + "compare_left": 17, + "compare_right": 35, + "score": 0.4516373028755188, + "inferred_pruned_layers_zero_based": [ + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35 + ] + } + ], + "fuse_region_cosine_mlp_pre_residual": [ + { + "removed_start": 1, + "removed_end": 18, + "compare_left": 0, + "compare_right": 18, + "score": 0.998951346874237, + "inferred_pruned_layers_zero_based": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] + }, + { + "removed_start": 2, + "removed_end": 19, + "compare_left": 1, + "compare_right": 19, + "score": 0.9171930348873139, + "inferred_pruned_layers_zero_based": [ + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19 + ] + }, + { + "removed_start": 3, + "removed_end": 20, + "compare_left": 2, + "compare_right": 20, + "score": 0.968644463300705, + "inferred_pruned_layers_zero_based": [ + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ] + }, + { + "removed_start": 4, + "removed_end": 21, + "compare_left": 3, + "compare_right": 21, + "score": 0.9799396152496338, + "inferred_pruned_layers_zero_based": [ + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21 + ] + }, + { + "removed_start": 5, + "removed_end": 22, + "compare_left": 4, + "compare_right": 22, + "score": 0.9321213603019715, + "inferred_pruned_layers_zero_based": [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ] + }, + { + "removed_start": 6, + "removed_end": 23, + "compare_left": 5, + "compare_right": 23, + "score": 0.8878195629119873, + "inferred_pruned_layers_zero_based": [ + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + }, + { + "removed_start": 7, + "removed_end": 24, + "compare_left": 6, + "compare_right": 24, + "score": 0.8522758095264434, + "inferred_pruned_layers_zero_based": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24 + ] + }, + { + "removed_start": 8, + "removed_end": 25, + "compare_left": 7, + "compare_right": 25, + "score": 0.967814395904541, + "inferred_pruned_layers_zero_based": [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25 + ] + }, + { + "removed_start": 9, + "removed_end": 26, + "compare_left": 8, + "compare_right": 26, + "score": 0.9738603377342224, + "inferred_pruned_layers_zero_based": [ + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26 + ] + }, + { + "removed_start": 10, + "removed_end": 27, + "compare_left": 9, + "compare_right": 27, + "score": 0.9975382511615754, + "inferred_pruned_layers_zero_based": [ + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27 + ] + }, + { + "removed_start": 11, + "removed_end": 28, + "compare_left": 10, + "compare_right": 28, + "score": 0.9783859169483184, + "inferred_pruned_layers_zero_based": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28 + ] + }, + { + "removed_start": 12, + "removed_end": 29, + "compare_left": 11, + "compare_right": 29, + "score": 0.9918379864692688, + "inferred_pruned_layers_zero_based": [ + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29 + ] + }, + { + "removed_start": 13, + "removed_end": 30, + "compare_left": 12, + "compare_right": 30, + "score": 0.956602737903595, + "inferred_pruned_layers_zero_based": [ + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30 + ] + }, + { + "removed_start": 14, + "removed_end": 31, + "compare_left": 13, + "compare_right": 31, + "score": 1.0131061437129973, + "inferred_pruned_layers_zero_based": [ + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + }, + { + "removed_start": 15, + "removed_end": 32, + "compare_left": 14, + "compare_right": 32, + "score": 0.9944792635440827, + "inferred_pruned_layers_zero_based": [ + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32 + ] + }, + { + "removed_start": 16, + "removed_end": 33, + "compare_left": 15, + "compare_right": 33, + "score": 1.0215474195480347, + "inferred_pruned_layers_zero_based": [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33 + ] + }, + { + "removed_start": 17, + "removed_end": 34, + "compare_left": 16, + "compare_right": 34, + "score": 1.0073524992465972, + "inferred_pruned_layers_zero_based": [ + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "removed_start": 18, + "removed_end": 35, + "compare_left": 17, + "compare_right": 35, + "score": 1.0098007888793945, + "inferred_pruned_layers_zero_based": [ + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35 + ] + } + ] + }, + "config": { + "model": "/workspace/models/qwen3-8b-base-49e3418", + "dataset": "slimpajama", + "dataset_config": null, + "split": "train", + "text_field": "text", + "instruction_format": "slimorca", + "seq_len": 1024, + "target_tokens": 500000, + "num_sequences": 500, + "batch_size": 2, + "prune_layers": 18, + "max_batches": 0, + "device": "cuda", + "dtype": "bfloat16", + "trust_remote_code": false, + "data_spec": { + "dataset": "slimpajama", + "config": null, + "split": "train", + "text_field": "text", + "num_samples": 0, + "seq_len": 1024, + "num_sequences": 500, + "target_tokens": 500000, + "batch_size": 2, + "shuffle": false, + "num_workers": 0, + "seed": 42, + "add_bos": false + } + }, + "data": { + "actual_num_chunks": 500, + "seq_len": 1024, + "actual_total_tokens": 512000, + "chunk_cache_path": "/workspace/artifacts/qwen3-base-half2-20261012/selection/slimpajama500_seed42.pt" + } +} \ No newline at end of file diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/selection.log b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/selection.log new file mode 100644 index 0000000000000000000000000000000000000000..21533073069733a1833e2e747689371e85cde285 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/selection.log @@ -0,0 +1,50 @@ +{"utc": "2026-10-12T14:08:54.494731+00:00", "command": ["bash", "/workspace/external/qwen3-base-half2-legacy/run_compare_layer_selection_metrics.sh"]} +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[transformers] The following generation flags are not valid and may be ignored: ['output_hidden_states']. Set `TRANSFORMERS_VERBOSITY=info` for more details. + Loading weights: 0%| | 0/399 [00:00 32 pruned_layers=[15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] + replaceme score=3.107806e-01 removed=14..31 compare=13 -> 31 pruned_layers=[14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31] +fuse_region_cosine score=4.454183e-01 removed=15..32 compare=14 -> 32 pruned_layers=[15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32] +fuse_region_cosine_mlp_pre_residual score=8.522758e-01 removed=7..24 compare=6 -> 24 pruned_layers=[7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24] + +Top starts per method +===================== +llm_streamline + removed_start=15 score=5.545817e-01 pruned=[15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32] + removed_start=13 score=5.528837e-01 pruned=[13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30] + removed_start=18 score=5.483627e-01 pruned=[18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35] + removed_start=17 score=5.471236e-01 pruned=[17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34] + removed_start=14 score=5.463030e-01 pruned=[14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] +replaceme + removed_start=14 score=3.107806e-01 pruned=[14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] + removed_start=12 score=3.113691e-01 pruned=[12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] + removed_start=13 score=3.139335e-01 pruned=[13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30] + removed_start=17 score=3.145263e-01 pruned=[17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34] + removed_start=16 score=3.151758e-01 pruned=[16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33] +fuse_region_cosine + removed_start=15 score=4.454183e-01 pruned=[15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32] + removed_start=13 score=4.471163e-01 pruned=[13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30] + removed_start=18 score=4.516373e-01 pruned=[18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35] + removed_start=17 score=4.528764e-01 pruned=[17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34] + removed_start=14 score=4.536970e-01 pruned=[14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] +fuse_region_cosine_mlp_pre_residual + removed_start= 7 score=8.522758e-01 pruned=[7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + removed_start= 6 score=8.878196e-01 pruned=[6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23] + removed_start= 2 score=9.171930e-01 pruned=[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] + removed_start= 5 score=9.321214e-01 pruned=[5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + removed_start=13 score=9.566027e-01 pruned=[13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30] + +Wrote /workspace/artifacts/qwen3-base-half2-20261012/selection/selection.json +output_json=/workspace/artifacts/qwen3-base-half2-20261012/selection/selection.json +chunk_cache_path=/workspace/artifacts/qwen3-base-half2-20261012/selection/slimpajama500_seed42.pt diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/slimpajama500_seed42.pt b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/slimpajama500_seed42.pt new file mode 100644 index 0000000000000000000000000000000000000000..16440ee7f0de01d739026489a2a9226f2daf8e22 --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/slimpajama500_seed42.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f3a56be04acac5ca8ad09dd10e02280659326af9c065f6c57a86d27e69ff997 +size 4235401 diff --git a/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/summary.json b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c75e3e9b1b54076a21ed473fb2c77a82cb7c4d4d --- /dev/null +++ b/runs/vast-paper_revision-qwen3-base-half2-55563310-20261012/20261012-qwen3-base-half18/replaceme/selection/summary.json @@ -0,0 +1,77 @@ +{ + "model": "Qwen/Qwen3-8B-Base", + "revision": "49e3418fbbbca6ecbdf9608b4d22e5a407081db4", + "sequences": 500, + "sequence_length": 1024, + "seed": 42, + "selections": { + "replaceme": { + "method": "replaceme", + "score_direction": "min", + "removed_start": 14, + "removed_end": 31, + "compare_left": 13, + "compare_right": 31, + "selected_score": 0.31078056192398074, + "inferred_pruned_layers_zero_based": [ + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "notes": "Unified span score: compare last-token hidden states of the first and last removed layers.", + "removed_interval_half_open": [ + 14, + 32 + ] + }, + "fuse_region_cosine": { + "method": "fuse_region_cosine", + "score_direction": "min", + "removed_start": 15, + "removed_end": 32, + "compare_left": 14, + "compare_right": 32, + "selected_score": 0.44541832387447355, + "inferred_pruned_layers_zero_based": [ + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32 + ], + "notes": "Unified span score: compare hidden_states[r-1] and hidden_states[r+w-1] using token-level 1-cosine.", + "removed_interval_half_open": [ + 15, + 33 + ] + } + } +}