File size: 4,014 Bytes
4a382cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | #!/usr/bin/env python3
"""
Patch IQuestLab/IQuest-Coder models for transformers 5.x compatibility.
Fixes the meta-device RoPE bug where accelerate zeros out inv_freq during
model initialization, causing the model to produce only newlines/garbage.
Usage:
python patch_iquestcoder.py
This will find and patch all cached IQuest-Coder modeling files automatically.
Run this AFTER downloading the model (e.g. after a failed heretic run or
after running `huggingface-cli download IQuestLab/IQuest-Coder-V1-40B-Instruct`).
"""
import glob
import os
import re
import sys
# Pattern to find the forward method that needs patching
ORIGINAL_PATTERN = re.compile(
r'( @torch\.no_grad\(\)\n'
r' @dynamic_rope_update\n'
r' def forward\(self, x: torch\.Tensor, position_ids: torch\.Tensor\)'
r' -> Tuple\[torch\.Tensor, torch\.Tensor\]:\n)'
r'( inv_freq_expanded = self\.inv_freq\[None, :, None\]\.float\(\)\.expand\(position_ids\.shape\[0\], -1, 1\)\.to\(x\.device\))'
)
REPLACEMENT = (
r'\1'
r' # Lazy recompute: accelerate meta-device init leaves inv_freq as zeros\n'
r' if self.inv_freq is not None and self.inv_freq.numel() > 0 and (self.inv_freq == 0).all():\n'
r' inv_freq, self.attention_scaling = self.rope_init_fn(self.config, None)\n'
r' self.inv_freq = inv_freq.to(device=x.device, dtype=self.inv_freq.dtype)\n'
r' self.original_inv_freq = self.inv_freq\n'
r'\2'
)
# Check string to see if already patched
PATCH_MARKER = "Lazy recompute: accelerate meta-device init"
# Search locations for cached model files
SEARCH_PATHS = [
os.path.expanduser("~/.cache/huggingface/hub/models--IQuestLab--*/**/modeling_iquestcoder.py"),
"/llm/huggingface/modules/transformers_modules/IQuestLab/**/modeling_iquestcoder.py",
# Common alternate HF cache locations
"/data/huggingface/**/modeling_iquestcoder.py",
"/scratch/**/modeling_iquestcoder.py",
]
def find_model_files():
"""Find all cached IQuest-Coder modeling files."""
found = []
# Also check HF_HOME / TRANSFORMERS_CACHE env vars
for env_var in ["HF_HOME", "TRANSFORMERS_CACHE", "HUGGINGFACE_HUB_CACHE"]:
val = os.environ.get(env_var)
if val:
SEARCH_PATHS.append(os.path.join(val, "**/modeling_iquestcoder.py"))
for pattern in SEARCH_PATHS:
found.extend(glob.glob(pattern, recursive=True))
# Deduplicate (resolve symlinks)
seen = set()
unique = []
for f in found:
real = os.path.realpath(f)
if real not in seen:
seen.add(real)
unique.append(f)
return unique
def patch_file(filepath):
"""Apply the RoPE lazy-recompute patch to a modeling file."""
with open(filepath, "r", encoding="utf-8") as f:
content = f.read()
if PATCH_MARKER in content:
print(f" SKIP (already patched): {filepath}")
return False
new_content, count = ORIGINAL_PATTERN.subn(REPLACEMENT, content)
if count == 0:
print(f" WARN (pattern not found — may need manual patching): {filepath}")
return False
with open(filepath, "w", encoding="utf-8") as f:
f.write(new_content)
print(f" OK (patched {count} location(s)): {filepath}")
return True
def main():
print("IQuest-Coder RoPE patch for transformers 5.x")
print("=" * 50)
print()
files = find_model_files()
if not files:
print("No IQuest-Coder model files found in cache.")
print("Download the model first, then re-run this script.")
print()
print("Searched:")
for p in SEARCH_PATHS:
print(f" {p}")
sys.exit(1)
print(f"Found {len(files)} file(s):\n")
patched = 0
for f in files:
if patch_file(f):
patched += 1
print()
if patched:
print(f"Done — patched {patched} file(s).")
else:
print("No files needed patching.")
if __name__ == "__main__":
main()
|