Dhia-GB commited on
Commit
da2d6ec
·
verified ·
1 Parent(s): 737a911

Chess Challenge submission by Dhia-GB

Browse files
Files changed (7) hide show
  1. README.md +26 -0
  2. config.json +24 -0
  3. model.safetensors +3 -0
  4. special_tokens_map.json +6 -0
  5. tokenizer.py +609 -0
  6. tokenizer_config.json +50 -0
  7. vocab.json +87 -0
README.md ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ tags:
4
+ - chess
5
+ - llm-course
6
+ - chess-challenge
7
+ license: mit
8
+ ---
9
+
10
+ # good_but_inefficient
11
+
12
+ Chess model submitted to the LLM Course Chess Challenge.
13
+
14
+ ## Submission Info
15
+
16
+ - **Submitted by**: [Dhia-GB](https://huggingface.co/Dhia-GB)
17
+ - **Parameters**: 832,896
18
+ - **Organization**: LLM-course
19
+
20
+ ## Model Details
21
+
22
+ - **Architecture**: Chess Transformer (GPT-style)
23
+ - **Vocab size**: 85
24
+ - **Embedding dim**: 128
25
+ - **Layers**: 5
26
+ - **Heads**: 4
config.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "../my_model_v2/checkpoint-1856/",
3
+ "architectures": [
4
+ "ChessForCausalLM"
5
+ ],
6
+ "bos_token_id": 1,
7
+ "dropout": 0.1,
8
+ "eos_token_id": 2,
9
+ "layer_norm_epsilon": 1e-06,
10
+ "model_type": "chess_transformer",
11
+ "n_ctx": 512,
12
+ "n_embd": 128,
13
+ "n_head": 4,
14
+ "n_inner": 384,
15
+ "n_layer": 5,
16
+ "pad_token_id": 0,
17
+ "tie_weights": true,
18
+ "torch_dtype": "float32",
19
+ "transformers_version": "4.48.2",
20
+ "use_rmsnorm": false,
21
+ "use_rope": true,
22
+ "use_swiglu": true,
23
+ "vocab_size": 85
24
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1e83ac7e86cca4090a70337acf67cca4d4d7df6ab02d6a88c0e69481d6966324
3
+ size 3336560
special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "[BOS]",
3
+ "eos_token": "[EOS]",
4
+ "pad_token": "[PAD]",
5
+ "unk_token": "[UNK]"
6
+ }
tokenizer.py ADDED
@@ -0,0 +1,609 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Custom Chess Tokenizer for the Chess Challenge.
3
+
4
+ This tokenizer treats each move as a single token using the extended UCI notation
5
+ from the Lichess dataset (e.g., WPe2e4, BNg8f6).
6
+
7
+ The dataset format uses:
8
+ - W/B prefix for White/Black
9
+ - Piece letter: P=Pawn, N=Knight, B=Bishop, R=Rook, Q=Queen, K=King
10
+ - Source and destination squares (e.g., e2e4)
11
+ - Special suffixes: (x)=capture, (+)=check, (+*)=checkmate, (o)/(O)=castling
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import os
18
+ from pathlib import Path
19
+ from typing import Dict, List, Optional
20
+
21
+ from transformers import PreTrainedTokenizer
22
+
23
+
24
+ class ChessTokenizer_v0(PreTrainedTokenizer):
25
+ """
26
+ A custom tokenizer for chess moves using extended UCI notation.
27
+
28
+ This tokenizer maps each possible chess move to a unique token ID.
29
+ The vocabulary is built from the training dataset to ensure all moves
30
+ encountered during training have a corresponding token.
31
+
32
+ Example:
33
+ >>> tokenizer = ChessTokenizer_v0()
34
+ >>> tokenizer.encode("WPe2e4 BPe7e5")
35
+ [1, 42, 87, 2] # [BOS, e2e4, e7e5, EOS]
36
+ """
37
+
38
+ model_input_names = ["input_ids", "attention_mask"]
39
+ vocab_files_names = {"vocab_file": "vocab.json"}
40
+
41
+ # Special tokens
42
+ PAD_TOKEN = "[PAD]"
43
+ BOS_TOKEN = "[BOS]"
44
+ EOS_TOKEN = "[EOS]"
45
+ UNK_TOKEN = "[UNK]"
46
+
47
+ def __init__(
48
+ self,
49
+ vocab_file: Optional[str] = None,
50
+ vocab: Optional[Dict[str, int]] = None,
51
+ **kwargs,
52
+ ):
53
+ """
54
+ Initialize the chess tokenizer.
55
+
56
+ Args:
57
+ vocab_file: Path to a JSON file containing the vocabulary mapping.
58
+ vocab: Dictionary mapping tokens to IDs (alternative to vocab_file).
59
+ **kwargs: Additional arguments passed to PreTrainedTokenizer.
60
+ """
61
+ # Initialize special tokens
62
+ self._pad_token = self.PAD_TOKEN
63
+ self._bos_token = self.BOS_TOKEN
64
+ self._eos_token = self.EOS_TOKEN
65
+ self._unk_token = self.UNK_TOKEN
66
+
67
+ # Remove any duplicate special-token entries passed through kwargs
68
+ # to avoid "multiple values for keyword" errors when loading from disk.
69
+ kwargs.pop("pad_token", None)
70
+ kwargs.pop("bos_token", None)
71
+ kwargs.pop("eos_token", None)
72
+ kwargs.pop("unk_token", None)
73
+
74
+ # Load or create vocabulary
75
+ if vocab is not None:
76
+ self._vocab = vocab
77
+ elif vocab_file is not None and os.path.exists(vocab_file):
78
+ with open(vocab_file, "r", encoding="utf-8") as f:
79
+ self._vocab = json.load(f)
80
+ else:
81
+ # Create a minimal vocabulary with just special tokens
82
+ # The full vocabulary should be built from the dataset
83
+ self._vocab = self._create_default_vocab()
84
+
85
+ # Create reverse mapping
86
+ self._ids_to_tokens = {v: k for k, v in self._vocab.items()}
87
+
88
+ # Call parent init AFTER setting up vocab
89
+ super().__init__(
90
+ pad_token=self._pad_token,
91
+ bos_token=self._bos_token,
92
+ eos_token=self._eos_token,
93
+ unk_token=self._unk_token,
94
+ **kwargs,
95
+ )
96
+
97
+ def _create_default_vocab(self) -> Dict[str, int]:
98
+ """
99
+ Create a minimal default vocabulary with just special tokens.
100
+
101
+ For the full vocabulary, use `build_vocab_from_dataset()`.
102
+ This minimal vocab is just a placeholder - you should build from data.
103
+ """
104
+ special_tokens = [self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN]
105
+ vocab = {token: idx for idx, token in enumerate(special_tokens)}
106
+ return vocab
107
+
108
+ @classmethod
109
+ def build_vocab_from_iterator(
110
+ cls,
111
+ iterator,
112
+ min_frequency: int = 1,
113
+ ) -> "ChessTokenizer_v0":
114
+ """
115
+ Build a tokenizer vocabulary from an iterator of game strings.
116
+
117
+ Args:
118
+ iterator: An iterator yielding game strings (space-separated moves).
119
+ min_frequency: Minimum frequency for a token to be included.
120
+
121
+ Returns:
122
+ A ChessTokenizer_v0 with the built vocabulary.
123
+ """
124
+ from collections import Counter
125
+
126
+ token_counts = Counter()
127
+
128
+ for game in iterator:
129
+ moves = game.strip().split()
130
+ token_counts.update(moves)
131
+
132
+ # Filter by frequency
133
+ tokens = [
134
+ token for token, count in token_counts.items()
135
+ if count >= min_frequency
136
+ ]
137
+
138
+ # Sort for reproducibility
139
+ tokens = sorted(tokens)
140
+
141
+ # Build vocabulary
142
+ special_tokens = [cls.PAD_TOKEN, cls.BOS_TOKEN, cls.EOS_TOKEN, cls.UNK_TOKEN]
143
+ vocab = {token: idx for idx, token in enumerate(special_tokens + tokens)}
144
+
145
+ return cls(vocab=vocab)
146
+
147
+ @classmethod
148
+ def build_vocab_from_dataset(
149
+ cls,
150
+ dataset_name: str = "dlouapre/lichess_2025-01_1M",
151
+ split: str = "train",
152
+ column: str = "text",
153
+ min_frequency: int = 500,
154
+ max_samples: Optional[int] = 100000,
155
+ ) -> "ChessTokenizer_v0":
156
+ """
157
+ Build a tokenizer vocabulary from a Hugging Face dataset.
158
+
159
+ Args:
160
+ dataset_name: Name of the dataset on Hugging Face Hub.
161
+ split: Dataset split to use.
162
+ column: Column containing the game strings.
163
+ min_frequency: Minimum frequency for a token to be included (default: 500).
164
+ max_samples: Maximum number of samples to process (default: 100k).
165
+
166
+ Returns:
167
+ A ChessTokenizer_v0 with the built vocabulary.
168
+ """
169
+ from datasets import load_dataset
170
+
171
+ dataset = load_dataset(dataset_name, split=split)
172
+
173
+ if max_samples is not None:
174
+ dataset = dataset.select(range(min(max_samples, len(dataset))))
175
+
176
+ def game_iterator():
177
+ for example in dataset:
178
+ yield example[column]
179
+
180
+ return cls.build_vocab_from_iterator(game_iterator(), min_frequency=min_frequency)
181
+
182
+ @property
183
+ def vocab_size(self) -> int:
184
+ """Return the size of the vocabulary."""
185
+ return len(self._vocab)
186
+
187
+ def get_vocab(self) -> Dict[str, int]:
188
+ """Return the vocabulary as a dictionary."""
189
+ return dict(self._vocab)
190
+
191
+ def _tokenize(self, text: str) -> List[str]:
192
+ """
193
+ Tokenize a string of moves into a list of tokens.
194
+
195
+ Args:
196
+ text: A string of space-separated moves.
197
+
198
+ Returns:
199
+ List of move tokens.
200
+ """
201
+ return text.strip().split()
202
+
203
+ def _convert_token_to_id(self, token: str) -> int:
204
+ """Convert a token to its ID."""
205
+ return self._vocab.get(token, self._vocab.get(self.UNK_TOKEN, 0))
206
+
207
+ def _convert_id_to_token(self, index: int) -> str:
208
+ """Convert an ID to its token."""
209
+ return self._ids_to_tokens.get(index, self.UNK_TOKEN)
210
+
211
+ def convert_tokens_to_string(self, tokens: List[str]) -> str:
212
+ """Convert a list of tokens back to a string."""
213
+ # Filter out special tokens for cleaner output
214
+ special = {self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN}
215
+ return " ".join(t for t in tokens if t not in special)
216
+
217
+ def save_vocabulary(
218
+ self,
219
+ save_directory: str,
220
+ filename_prefix: Optional[str] = None,
221
+ ) -> tuple:
222
+ """
223
+ Save the vocabulary to a JSON file.
224
+
225
+ Args:
226
+ save_directory: Directory to save the vocabulary.
227
+ filename_prefix: Optional prefix for the filename.
228
+
229
+ Returns:
230
+ Tuple containing the path to the saved vocabulary file.
231
+ """
232
+ if not os.path.isdir(save_directory):
233
+ os.makedirs(save_directory, exist_ok=True)
234
+
235
+ vocab_file = os.path.join(
236
+ save_directory,
237
+ (filename_prefix + "-" if filename_prefix else "") + "vocab.json",
238
+ )
239
+
240
+ with open(vocab_file, "w", encoding="utf-8") as f:
241
+ json.dump(self._vocab, f, ensure_ascii=False, indent=2)
242
+
243
+ return (vocab_file,)
244
+
245
+
246
+ def count_vocab_from_dataset(
247
+ dataset_name: str = "dlouapre/lichess_2025-01_1M",
248
+ split: str = "train",
249
+ column: str = "text",
250
+ max_samples: Optional[int] = 10000,
251
+ ) -> Dict[str, int]:
252
+ """
253
+ Count token frequencies in a dataset (useful for vocabulary analysis).
254
+
255
+ Args:
256
+ dataset_name: Name of the dataset on Hugging Face Hub.
257
+ split: Dataset split to use.
258
+ column: Column containing the game strings.
259
+ max_samples: Maximum number of samples to process.
260
+
261
+ Returns:
262
+ Dictionary mapping tokens to their frequencies.
263
+ """
264
+ from collections import Counter
265
+ from datasets import load_dataset
266
+
267
+ dataset = load_dataset(dataset_name, split=split)
268
+
269
+ if max_samples is not None:
270
+ dataset = dataset.select(range(min(max_samples, len(dataset))))
271
+
272
+ token_counts = Counter()
273
+
274
+ for example in dataset:
275
+ moves = example[column].strip().split()
276
+ token_counts.update(moves)
277
+
278
+ return dict(token_counts)
279
+
280
+
281
+ # ============================================================================
282
+ # V1 IMPROVEMENTS: Sub-word tokenizer that decomposes moves into components
283
+ # ============================================================================
284
+
285
+ import re
286
+
287
+ # Regex to parse extended UCI move format: WPe2e4(x)(+) etc.
288
+ MOVE_PATTERN = re.compile(
289
+ r"^(?P<side>[WB])"
290
+ r"(?P<piece>[PNBRQK])"
291
+ r"(?P<src>[a-h][1-8])"
292
+ r"(?P<dst>[a-h][1-8])"
293
+ r"(?P<suffix>.*)$"
294
+ )
295
+
296
+
297
+ class ChessTokenizer(PreTrainedTokenizer):
298
+ """
299
+ Sub-word chess tokenizer that decomposes moves into components.
300
+
301
+ Instead of treating each move as a single token (requiring ~1500 tokens),
302
+ this tokenizer breaks moves into:
303
+ - Side: [W], [B]
304
+ - Piece: [P], [N], [B], [R], [Q], [K]
305
+ - Source square: [a1] through [h8]
306
+ - Destination square: [a1] through [h8]
307
+ - Optional suffixes: [x] (capture), [+] (check), [#] (checkmate),
308
+ [O-O], [O-O-O], [=Q], [=R], [=B], [=N]
309
+
310
+ Total vocabulary: ~90 tokens (vs ~1500 for whole-move tokenizer)
311
+
312
+ Trade-off: Each move becomes 4-6 tokens instead of 1, but:
313
+ - Saves ~100-200K embedding parameters
314
+ - Model learns piece/square patterns independently
315
+ - Zero OOV - can represent any legal move
316
+
317
+ Example:
318
+ "WPe2e4" -> ["[W]", "[P]", "[e2]", "[e4]"]
319
+ "BNg8f6(x)(+)" -> ["[B]", "[N]", "[g8]", "[f6]", "[x]", "[+]"]
320
+ """
321
+
322
+ model_input_names = ["input_ids", "attention_mask"]
323
+ vocab_files_names = {"vocab_file": "vocab.json"}
324
+
325
+ # Special tokens
326
+ PAD_TOKEN = "[PAD]"
327
+ BOS_TOKEN = "[BOS]"
328
+ EOS_TOKEN = "[EOS]"
329
+ UNK_TOKEN = "[UNK]"
330
+
331
+ def __init__(
332
+ self,
333
+ vocab_file: Optional[str] = None,
334
+ vocab: Optional[Dict[str, int]] = None,
335
+ **kwargs,
336
+ ):
337
+ self._pad_token = self.PAD_TOKEN
338
+ self._bos_token = self.BOS_TOKEN
339
+ self._eos_token = self.EOS_TOKEN
340
+ self._unk_token = self.UNK_TOKEN
341
+
342
+ kwargs.pop("pad_token", None)
343
+ kwargs.pop("bos_token", None)
344
+ kwargs.pop("eos_token", None)
345
+ kwargs.pop("unk_token", None)
346
+
347
+ if vocab is not None:
348
+ self._vocab = vocab
349
+ elif vocab_file is not None and os.path.exists(vocab_file):
350
+ with open(vocab_file, "r", encoding="utf-8") as f:
351
+ self._vocab = json.load(f)
352
+ else:
353
+ self._vocab = self._create_default_vocab()
354
+
355
+ self._ids_to_tokens = {v: k for k, v in self._vocab.items()}
356
+
357
+ super().__init__(
358
+ pad_token=self._pad_token,
359
+ bos_token=self._bos_token,
360
+ eos_token=self._eos_token,
361
+ unk_token=self._unk_token,
362
+ **kwargs,
363
+ )
364
+
365
+ def _create_default_vocab(self) -> Dict[str, int]:
366
+ """
367
+ Create the fixed sub-word vocabulary.
368
+
369
+ This vocabulary is complete - no need to build from data.
370
+ """
371
+ vocab_list = []
372
+
373
+ # 1. Special tokens (4)
374
+ vocab_list.extend([self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN])
375
+
376
+ # 2. Side tokens (2)
377
+ vocab_list.extend(["[W]", "[B]"])
378
+
379
+ # 3. Piece tokens (6)
380
+ vocab_list.extend(["[P]", "[N]", "[Bi]", "[R]", "[Q]", "[K]"])
381
+
382
+ # 4. Square tokens (64)
383
+ for rank in "12345678":
384
+ for file in "abcdefgh":
385
+ vocab_list.append(f"[{file}{rank}]")
386
+
387
+ # 5. Suffix tokens
388
+ vocab_list.extend([
389
+ "[x]", # capture
390
+ "[+]", # check
391
+ "[#]", # checkmate
392
+ "[O-O]", # kingside castling
393
+ "[O-O-O]", # queenside castling
394
+ "[=Q]", # promotion to queen
395
+ "[=R]", # promotion to rook
396
+ "[=B]", # promotion to bishop
397
+ "[=N]", # promotion to knight
398
+ ])
399
+
400
+ return {token: idx for idx, token in enumerate(vocab_list)}
401
+
402
+ @property
403
+ def vocab_size(self) -> int:
404
+ return len(self._vocab)
405
+
406
+ def get_vocab(self) -> Dict[str, int]:
407
+ return dict(self._vocab)
408
+
409
+ def _tokenize(self, text: str) -> List[str]:
410
+ """
411
+ Tokenize a string of moves into sub-word tokens.
412
+
413
+ Args:
414
+ text: A string of space-separated moves (e.g., "WPe2e4 BPe7e5")
415
+
416
+ Returns:
417
+ List of sub-word tokens
418
+ """
419
+ tokens = []
420
+ moves = text.strip().split()
421
+
422
+ for move in moves:
423
+ tokens.extend(self._tokenize_move(move))
424
+
425
+ return tokens
426
+
427
+ def _tokenize_move(self, move: str) -> List[str]:
428
+ """Parse a single move into component tokens."""
429
+ # Handle castling first
430
+ if "O-O-O" in move or "o-o-o" in move:
431
+ side = "[W]" if move.startswith("W") else "[B]"
432
+ return [side, "[O-O-O]"]
433
+
434
+ if "O-O" in move or "o-o" in move:
435
+ side = "[W]" if move.startswith("W") else "[B]"
436
+ return [side, "[O-O]"]
437
+
438
+ # Parse regular move
439
+ match = MOVE_PATTERN.match(move)
440
+ if not match:
441
+ return [self.UNK_TOKEN]
442
+
443
+ tokens = []
444
+
445
+ # Side
446
+ side = match.group("side")
447
+ tokens.append(f"[{side}]")
448
+
449
+ # Piece (use [Bi] for bishop to avoid confusion with [B] for black)
450
+ piece = match.group("piece")
451
+ if piece == "B":
452
+ tokens.append("[Bi]")
453
+ else:
454
+ tokens.append(f"[{piece}]")
455
+
456
+ # Source and destination squares
457
+ tokens.append(f"[{match.group('src')}]")
458
+ tokens.append(f"[{match.group('dst')}]")
459
+
460
+ # Parse suffix for capture, check, checkmate, promotion
461
+ suffix = match.group("suffix") or ""
462
+
463
+ if "x" in suffix:
464
+ tokens.append("[x]")
465
+
466
+ # Checkmate before check (since checkmate contains +)
467
+ if "*" in suffix or "#" in suffix:
468
+ tokens.append("[#]")
469
+ elif "+" in suffix:
470
+ tokens.append("[+]")
471
+
472
+ # Promotion
473
+ if "=" in suffix:
474
+ idx = suffix.find("=")
475
+ if idx + 1 < len(suffix):
476
+ promo_piece = suffix[idx + 1].upper()
477
+ if promo_piece in "QRBN":
478
+ tokens.append(f"[={promo_piece}]")
479
+
480
+ return tokens
481
+
482
+ def _convert_token_to_id(self, token: str) -> int:
483
+ return self._vocab.get(token, self._vocab.get(self.UNK_TOKEN, 0))
484
+
485
+ def _convert_id_to_token(self, index: int) -> str:
486
+ return self._ids_to_tokens.get(index, self.UNK_TOKEN)
487
+
488
+ def convert_tokens_to_string(self, tokens: List[str]) -> str:
489
+ """
490
+ Convert tokens back to a readable string.
491
+
492
+ This reconstructs moves from their component tokens.
493
+ """
494
+ special = {self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN}
495
+
496
+ # Filter special tokens
497
+ filtered = [t for t in tokens if t not in special]
498
+
499
+ # Simple approach: just join with spaces
500
+ # A more sophisticated approach would reconstruct full moves
501
+ return " ".join(filtered)
502
+
503
+ def decode_to_moves(self, token_ids: List[int]) -> List[str]:
504
+ """
505
+ Decode token IDs back to chess moves.
506
+
507
+ Returns a list of reconstructed moves like ["WPe2e4", "BPe7e5"].
508
+ """
509
+ tokens = [self._convert_id_to_token(tid) for tid in token_ids]
510
+ special = {self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN}
511
+
512
+ moves = []
513
+ current_move = []
514
+
515
+ for token in tokens:
516
+ if token in special:
517
+ continue
518
+
519
+ # Start new move on side token
520
+ if token in ("[W]", "[B]"):
521
+ if current_move:
522
+ moves.append(self._reconstruct_move(current_move))
523
+ current_move = [token]
524
+ else:
525
+ current_move.append(token)
526
+
527
+ # Don't forget last move
528
+ if current_move:
529
+ moves.append(self._reconstruct_move(current_move))
530
+
531
+ return moves
532
+
533
+ def _reconstruct_move(self, tokens: List[str]) -> str:
534
+ """Reconstruct a move string from component tokens."""
535
+ if not tokens:
536
+ return ""
537
+
538
+ # Handle castling
539
+ if "[O-O-O]" in tokens:
540
+ side = "W" if "[W]" in tokens else "B"
541
+ return f"{side}KO-O-O"
542
+ if "[O-O]" in tokens:
543
+ side = "W" if "[W]" in tokens else "B"
544
+ return f"{side}KO-O"
545
+
546
+ move = ""
547
+
548
+ for token in tokens:
549
+ # Strip brackets
550
+ inner = token[1:-1] if token.startswith("[") and token.endswith("]") else token
551
+
552
+ if inner in ("W", "B"):
553
+ move += inner
554
+ elif inner == "Bi":
555
+ move += "B" # Bishop
556
+ elif inner in "PNRQK":
557
+ move += inner
558
+ elif len(inner) == 2 and inner[0] in "abcdefgh" and inner[1] in "12345678":
559
+ move += inner
560
+ elif inner == "x":
561
+ move += "(x)"
562
+ elif inner == "+":
563
+ move += "(+)"
564
+ elif inner == "#":
565
+ move += "(+*)"
566
+ elif inner.startswith("="):
567
+ move += f"({inner})"
568
+
569
+ return move
570
+
571
+ def save_vocabulary(
572
+ self,
573
+ save_directory: str,
574
+ filename_prefix: Optional[str] = None,
575
+ ) -> tuple:
576
+ if not os.path.isdir(save_directory):
577
+ os.makedirs(save_directory, exist_ok=True)
578
+
579
+ vocab_file = os.path.join(
580
+ save_directory,
581
+ (filename_prefix + "-" if filename_prefix else "") + "vocab.json",
582
+ )
583
+
584
+ with open(vocab_file, "w", encoding="utf-8") as f:
585
+ json.dump(self._vocab, f, ensure_ascii=False, indent=2)
586
+
587
+ return (vocab_file,)
588
+
589
+ def get_vocab_stats(self) -> Dict[str, int]:
590
+ """Get statistics about vocabulary composition."""
591
+ return {
592
+ "special": 4,
593
+ "sides": 2,
594
+ "pieces": 6,
595
+ "squares": 64,
596
+ "suffixes": 9,
597
+ "total": self.vocab_size,
598
+ }
599
+
600
+ # For compatibility - no need to build vocab from data anymore
601
+ @classmethod
602
+ def build_vocab_from_dataset(cls, **kwargs) -> "ChessTokenizer":
603
+ """Return a tokenizer with the fixed vocabulary (no data needed)."""
604
+ return cls()
605
+
606
+ @classmethod
607
+ def build_vocab_from_iterator(cls, iterator, **kwargs) -> "ChessTokenizer":
608
+ """Return a tokenizer with the fixed vocabulary (no data needed)."""
609
+ return cls()
tokenizer_config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "[PAD]",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "[BOS]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "[EOS]",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "[UNK]",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ }
35
+ },
36
+ "auto_map": {
37
+ "AutoTokenizer": [
38
+ "tokenizer.ChessTokenizer",
39
+ null
40
+ ]
41
+ },
42
+ "bos_token": "[BOS]",
43
+ "clean_up_tokenization_spaces": false,
44
+ "eos_token": "[EOS]",
45
+ "extra_special_tokens": {},
46
+ "model_max_length": 1000000000000000019884624838656,
47
+ "pad_token": "[PAD]",
48
+ "tokenizer_class": "ChessTokenizer",
49
+ "unk_token": "[UNK]"
50
+ }
vocab.json ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "[PAD]": 0,
3
+ "[BOS]": 1,
4
+ "[EOS]": 2,
5
+ "[UNK]": 3,
6
+ "[W]": 4,
7
+ "[B]": 5,
8
+ "[P]": 6,
9
+ "[N]": 7,
10
+ "[Bi]": 8,
11
+ "[R]": 9,
12
+ "[Q]": 10,
13
+ "[K]": 11,
14
+ "[a1]": 12,
15
+ "[b1]": 13,
16
+ "[c1]": 14,
17
+ "[d1]": 15,
18
+ "[e1]": 16,
19
+ "[f1]": 17,
20
+ "[g1]": 18,
21
+ "[h1]": 19,
22
+ "[a2]": 20,
23
+ "[b2]": 21,
24
+ "[c2]": 22,
25
+ "[d2]": 23,
26
+ "[e2]": 24,
27
+ "[f2]": 25,
28
+ "[g2]": 26,
29
+ "[h2]": 27,
30
+ "[a3]": 28,
31
+ "[b3]": 29,
32
+ "[c3]": 30,
33
+ "[d3]": 31,
34
+ "[e3]": 32,
35
+ "[f3]": 33,
36
+ "[g3]": 34,
37
+ "[h3]": 35,
38
+ "[a4]": 36,
39
+ "[b4]": 37,
40
+ "[c4]": 38,
41
+ "[d4]": 39,
42
+ "[e4]": 40,
43
+ "[f4]": 41,
44
+ "[g4]": 42,
45
+ "[h4]": 43,
46
+ "[a5]": 44,
47
+ "[b5]": 45,
48
+ "[c5]": 46,
49
+ "[d5]": 47,
50
+ "[e5]": 48,
51
+ "[f5]": 49,
52
+ "[g5]": 50,
53
+ "[h5]": 51,
54
+ "[a6]": 52,
55
+ "[b6]": 53,
56
+ "[c6]": 54,
57
+ "[d6]": 55,
58
+ "[e6]": 56,
59
+ "[f6]": 57,
60
+ "[g6]": 58,
61
+ "[h6]": 59,
62
+ "[a7]": 60,
63
+ "[b7]": 61,
64
+ "[c7]": 62,
65
+ "[d7]": 63,
66
+ "[e7]": 64,
67
+ "[f7]": 65,
68
+ "[g7]": 66,
69
+ "[h7]": 67,
70
+ "[a8]": 68,
71
+ "[b8]": 69,
72
+ "[c8]": 70,
73
+ "[d8]": 71,
74
+ "[e8]": 72,
75
+ "[f8]": 73,
76
+ "[g8]": 74,
77
+ "[h8]": 75,
78
+ "[x]": 76,
79
+ "[+]": 77,
80
+ "[#]": 78,
81
+ "[O-O]": 79,
82
+ "[O-O-O]": 80,
83
+ "[=Q]": 81,
84
+ "[=R]": 82,
85
+ "[=B]": 83,
86
+ "[=N]": 84
87
+ }