Update copy_weights.py
Browse files- copy_weights.py +45 -1
copy_weights.py
CHANGED
|
@@ -6,7 +6,51 @@ import os
|
|
| 6 |
print("๐ Copying embeddings and lm_head...")
|
| 7 |
|
| 8 |
old_model_path = "."
|
| 9 |
-
new_tokenizer_path = "./
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
save_path = "./JiRack_init_new_vocab"
|
| 11 |
|
| 12 |
os.makedirs(save_path, exist_ok=True)
|
|
|
|
| 6 |
print("๐ Copying embeddings and lm_head...")
|
| 7 |
|
| 8 |
old_model_path = "."
|
| 9 |
+
new_tokenizer_path = "./jirack_code_tokenimport torch
|
| 10 |
+
from JiRackTernary_new import JiRackConfig, JiRackTernary1B
|
| 11 |
+
from transformers import AutoTokenizer
|
| 12 |
+
import os
|
| 13 |
+
|
| 14 |
+
print("๐ Copying embeddings and lm_head...")
|
| 15 |
+
|
| 16 |
+
old_model_path = "." ## curent model
|
| 17 |
+
new_tokenizer_path = "./jirack_code_tokenizer" ## new JiRack tokenizer
|
| 18 |
+
save_path = "./JiRack_init_model_with_new_vocab" ## new safe bank model
|
| 19 |
+
|
| 20 |
+
os.makedirs(save_path, exist_ok=True)
|
| 21 |
+
|
| 22 |
+
# Load new tokenizer
|
| 23 |
+
tokenizer = AutoTokenizer.from_pretrained(new_tokenizer_path)
|
| 24 |
+
new_vocab_size = len(tokenizer)
|
| 25 |
+
|
| 26 |
+
# Create new model with updated vocab size
|
| 27 |
+
config = JiRackConfig()
|
| 28 |
+
model = JiRackTernary1B(config)
|
| 29 |
+
|
| 30 |
+
# Load old model weights
|
| 31 |
+
old_state = torch.load(f"{old_model_path}/pytorch_model.bin", map_location="cpu")
|
| 32 |
+
|
| 33 |
+
old_vocab_size = 128256
|
| 34 |
+
|
| 35 |
+
with torch.no_grad():
|
| 36 |
+
# Copy old weights
|
| 37 |
+
model.token_emb.weight[:old_vocab_size] = old_state['token_emb.weight'][:old_vocab_size]
|
| 38 |
+
model.lm_head.weight[:old_vocab_size] = old_state['lm_head.weight'][:old_vocab_size]
|
| 39 |
+
|
| 40 |
+
# Initialize the new 3 tokens (FIM markers) with mean value
|
| 41 |
+
mean_emb = old_state['token_emb.weight'].mean(dim=0)
|
| 42 |
+
model.token_emb.weight[old_vocab_size:] = mean_emb
|
| 43 |
+
model.lm_head.weight[old_vocab_size:] = mean_emb
|
| 44 |
+
|
| 45 |
+
print(f"โ
Copied {old_vocab_size} tokens")
|
| 46 |
+
print(f"โ
Initialized {new_vocab_size - old_vocab_size} new tokens")
|
| 47 |
+
|
| 48 |
+
# Save
|
| 49 |
+
torch.save(model.state_dict(), f"{save_path}/pytorch_model.bin")
|
| 50 |
+
tokenizer.save_pretrained(save_path)
|
| 51 |
+
|
| 52 |
+
print(f"\n๐ Done! New model saved to: {save_path}")
|
| 53 |
+
print("Use this folder as the starting weights for training from scratch.")izer_fixed"
|
| 54 |
save_path = "./JiRack_init_new_vocab"
|
| 55 |
|
| 56 |
os.makedirs(save_path, exist_ok=True)
|