teszenofficial commited on
Commit
9f85f6d
·
verified ·
1 Parent(s): 8891ab9

Delete tokenizer.py

Browse files
Files changed (1) hide show
  1. tokenizer.py +0 -131
tokenizer.py DELETED
@@ -1,131 +0,0 @@
1
- import sentencepiece as spm
2
- import os
3
- import json
4
-
5
-
6
- class MTPTokenizer:
7
- """Tokenizer using SentencePiece BPE - Optimizado para formato instruction-response"""
8
-
9
- def __init__(self, model_path=None):
10
- self.sp = None
11
- self.model_path = model_path
12
-
13
- if model_path and os.path.exists(model_path):
14
- self.load(model_path)
15
-
16
- def train(self, corpus_path, vocab_size=4000, model_prefix='mtp_tokenizer'):
17
- """Train SentencePiece BPE tokenizer en corpus con formato JSONL"""
18
-
19
- # Extraer texto de corpus JSONL
20
- texts = []
21
- print(f" → Procesando corpus para entrenar tokenizer...")
22
-
23
- with open(corpus_path, 'r', encoding='utf-8') as f:
24
- for line_num, line in enumerate(f, 1):
25
- line = line.strip()
26
- if not line:
27
- continue
28
-
29
- try:
30
- data = json.loads(line)
31
-
32
- # Agregar todos los campos de texto disponibles
33
- if 'instruction' in data:
34
- texts.append(data['instruction'].strip())
35
- if 'context' in data and data['context'].strip():
36
- texts.append(data['context'].strip())
37
- if 'response' in data:
38
- texts.append(data['response'].strip())
39
-
40
- except json.JSONDecodeError:
41
- continue
42
-
43
- # Filtrar textos vacíos
44
- texts = [t for t in texts if t and t.strip()]
45
-
46
- if not texts:
47
- raise ValueError("No se encontraron textos válidos en el corpus")
48
-
49
- # Guardar archivo temporal
50
- temp_file = 'temp_corpus.txt'
51
- with open(temp_file, 'w', encoding='utf-8') as f:
52
- for text in texts:
53
- f.write(text + '\n')
54
-
55
- # Estadísticas
56
- total_chars = sum(len(text) for text in texts)
57
- max_vocab = min(vocab_size, max(256, int(total_chars * 0.15))) # Heurística mejorada
58
-
59
- print(f" → Corpus stats: {len(texts)} textos, {total_chars} caracteres")
60
- print(f" → Vocabulario ajustado: {max_vocab} (solicitado: {vocab_size})")
61
-
62
- # Parámetros optimizados para Q&A
63
- spm.SentencePieceTrainer.train(
64
- input=temp_file,
65
- model_prefix=model_prefix,
66
- vocab_size=max_vocab,
67
- model_type='bpe',
68
- pad_id=0,
69
- unk_id=1,
70
- bos_id=2,
71
- eos_id=3,
72
- character_coverage=1.0,
73
- normalization_rule_name='identity', # No normalizar para mantener formato
74
- num_threads=4,
75
- split_digits=True,
76
- allow_whitespace_only_pieces=False,
77
- byte_fallback=False,
78
- max_sentencepiece_length=16,
79
- add_dummy_prefix=False, # Importante para mantener inicio de textos
80
- remove_extra_whitespaces=False # Mantener formato exacto
81
- )
82
-
83
- # Limpiar
84
- os.remove(temp_file)
85
-
86
- # Cargar modelo entrenado
87
- self.model_path = f"{model_prefix}.model"
88
- self.load(self.model_path)
89
-
90
- print(f"✓ Tokenizer entrenado: {self.vocab_size()} tokens")
91
- print(f"✓ Modelo guardado: {self.model_path}")
92
-
93
- def load(self, model_path):
94
- """Load trained tokenizer"""
95
- self.sp = spm.SentencePieceProcessor()
96
- self.sp.load(model_path)
97
- self.model_path = model_path
98
-
99
- def encode(self, text):
100
- """Encode text to token IDs"""
101
- if self.sp is None:
102
- raise ValueError("Tokenizer not loaded. Train or load a model first.")
103
- return self.sp.encode_as_ids(text)
104
-
105
- def decode(self, ids):
106
- """Decode token IDs to text"""
107
- if self.sp is None:
108
- raise ValueError("Tokenizer not loaded. Train or load a model first.")
109
- return self.sp.decode_ids(ids)
110
-
111
- def vocab_size(self):
112
- """Get vocabulary size"""
113
- if self.sp is None:
114
- return 0
115
- return self.sp.get_piece_size()
116
-
117
- def bos_id(self):
118
- """Beginning of sentence token ID"""
119
- return self.sp.bos_id()
120
-
121
- def eos_id(self):
122
- """End of sentence token ID"""
123
- return self.sp.eos_id()
124
-
125
- def pad_id(self):
126
- """Padding token ID"""
127
- return self.sp.pad_id()
128
-
129
- def unk_id(self):
130
- """Unknown token ID"""
131
- return self.sp.unk_id()