kgrabko commited on
Commit
15293b8
ยท
verified ยท
1 Parent(s): f510091

Update copy_weights.py

Browse files
Files changed (1) hide show
  1. copy_weights.py +45 -1
copy_weights.py CHANGED
@@ -6,7 +6,51 @@ import os
6
  print("๐Ÿš€ Copying embeddings and lm_head...")
7
 
8
  old_model_path = "."
9
- new_tokenizer_path = "./jirack_code_tokenizer_fixed"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  save_path = "./JiRack_init_new_vocab"
11
 
12
  os.makedirs(save_path, exist_ok=True)
 
6
  print("๐Ÿš€ Copying embeddings and lm_head...")
7
 
8
  old_model_path = "."
9
+ new_tokenizer_path = "./jirack_code_tokenimport torch
10
+ from JiRackTernary_new import JiRackConfig, JiRackTernary1B
11
+ from transformers import AutoTokenizer
12
+ import os
13
+
14
+ print("๐Ÿš€ Copying embeddings and lm_head...")
15
+
16
+ old_model_path = "." ## curent model
17
+ new_tokenizer_path = "./jirack_code_tokenizer" ## new JiRack tokenizer
18
+ save_path = "./JiRack_init_model_with_new_vocab" ## new safe bank model
19
+
20
+ os.makedirs(save_path, exist_ok=True)
21
+
22
+ # Load new tokenizer
23
+ tokenizer = AutoTokenizer.from_pretrained(new_tokenizer_path)
24
+ new_vocab_size = len(tokenizer)
25
+
26
+ # Create new model with updated vocab size
27
+ config = JiRackConfig()
28
+ model = JiRackTernary1B(config)
29
+
30
+ # Load old model weights
31
+ old_state = torch.load(f"{old_model_path}/pytorch_model.bin", map_location="cpu")
32
+
33
+ old_vocab_size = 128256
34
+
35
+ with torch.no_grad():
36
+ # Copy old weights
37
+ model.token_emb.weight[:old_vocab_size] = old_state['token_emb.weight'][:old_vocab_size]
38
+ model.lm_head.weight[:old_vocab_size] = old_state['lm_head.weight'][:old_vocab_size]
39
+
40
+ # Initialize the new 3 tokens (FIM markers) with mean value
41
+ mean_emb = old_state['token_emb.weight'].mean(dim=0)
42
+ model.token_emb.weight[old_vocab_size:] = mean_emb
43
+ model.lm_head.weight[old_vocab_size:] = mean_emb
44
+
45
+ print(f"โœ… Copied {old_vocab_size} tokens")
46
+ print(f"โœ… Initialized {new_vocab_size - old_vocab_size} new tokens")
47
+
48
+ # Save
49
+ torch.save(model.state_dict(), f"{save_path}/pytorch_model.bin")
50
+ tokenizer.save_pretrained(save_path)
51
+
52
+ print(f"\n๐ŸŽ‰ Done! New model saved to: {save_path}")
53
+ print("Use this folder as the starting weights for training from scratch.")izer_fixed"
54
  save_path = "./JiRack_init_new_vocab"
55
 
56
  os.makedirs(save_path, exist_ok=True)