Chess Challenge submission by MDaytek

README.md

@@ -0,0 +1,31 @@
+---
+library_name: transformers
+tags:
+- chess
+- llm-course
+- chess-challenge
+license: mit
+---
+
+# chess-MDaytek
+
+Chess model submitted to the LLM Course Chess Challenge.
+
+## Submission Info
+
+- **Submitted by**: [MDaytek](https://huggingface.co/MDaytek)
+- **Parameters**: 749,856
+- **Organization**: LLM-course
+
+## Usage
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model = AutoModelForCausalLM.from_pretrained("LLM-course/chess-MDaytek", trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained("LLM-course/chess-MDaytek", trust_remote_code=True)
+```
+
+## Evaluation
+
+This model is evaluated at the [Chess Challenge Arena](https://huggingface.co/spaces/LLM-course/Chess1MChallenge).

config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "ChessForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "model.ChessConfig",
+    "AutoModelForCausalLM": "model.ChessForCausalLM"
+  },
+  "bos_token_id": 1,
+  "dropout": 0.15,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "chess_transformer",
+  "n_ctx": 256,
+  "n_embd": 96,
+  "n_head": 8,
+  "n_inner": 288,
+  "n_layer": 8,
+  "pad_token_id": 0,
+  "tie_weights": true,
+  "transformers_version": "4.57.6",
+  "vocab_size": 33
+}

model.py

@@ -0,0 +1,159 @@
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import PretrainedConfig, PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+def rotate_half(x):
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+def apply_rope(q, k):
+    dim = q.shape[-1]
+    device = q.device
+    seq_len = q.shape[-2]
+    theta = 1.0 / (10000 ** (torch.arange(0, dim, 2, device=device).float() / dim))
+    pos = torch.arange(seq_len, device=device).float()
+    freqs = torch.einsum('i,j->ij', pos, theta)
+    emb = torch.cat((freqs, freqs), dim=-1)
+    cos = emb.cos()[None, None, :, :]
+    sin = emb.sin()[None, None, :, :]
+    q = (q * cos) + (rotate_half(q) * sin)
+    k = (k * cos) + (rotate_half(k) * sin)
+    return q, k
+
+
+class ChessConfig(PretrainedConfig):
+    model_type = "chess_transformer"
+    def __init__(self, vocab_size=1200, n_embd=104, n_layer=8, n_head=8, n_ctx=256, n_inner=None, dropout=0.15, layer_norm_epsilon=1e-5, tie_weights=True, pad_token_id=0, bos_token_id=1, eos_token_id=2, **kwargs):
+        super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+        self.vocab_size = vocab_size
+        self.n_embd = n_embd
+        self.n_layer = n_layer
+        self.n_head = n_head
+        self.n_ctx = n_ctx
+        self.n_inner = n_inner if n_inner is not None else int(2.5 * n_embd)
+        self.dropout = dropout
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.tie_weights = tie_weights
+        self.tie_word_embeddings = True
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        assert config.n_embd % config.n_head == 0
+        self.n_head = config.n_head
+        self.n_embd = config.n_embd
+        self.head_dim = config.n_embd // config.n_head
+        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
+        self.c_proj = nn.Linear(config.n_embd, config.n_embd)
+        self.dropout = nn.Dropout(config.dropout)
+        self.register_buffer("bias", torch.tril(torch.ones(config.n_ctx, config.n_ctx)).view(1, 1, config.n_ctx, config.n_ctx), persistent=False)
+    def forward(self, x, attention_mask=None):
+        batch_size, seq_len, _ = x.size()
+        qkv = self.c_attn(x)
+        q, k, v = qkv.split(self.n_embd, dim=2)
+        q = q.view(batch_size, seq_len, self.n_head, self.head_dim).transpose(1, 2)
+        k = k.view(batch_size, seq_len, self.n_head, self.head_dim).transpose(1, 2)
+        v = v.view(batch_size, seq_len, self.n_head, self.head_dim).transpose(1, 2)
+        q, k = apply_rope(q, k)
+        attn_weights = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.head_dim)
+        causal_mask = self.bias[:, :, :seq_len, :seq_len]
+        attn_weights = attn_weights.masked_fill(causal_mask == 0, float("-inf"))
+        if attention_mask is not None:
+            attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
+            attn_weights = attn_weights.masked_fill(attention_mask == 0, float("-inf"))
+        attn_weights = F.softmax(attn_weights, dim=-1)
+        attn_weights = self.dropout(attn_weights)
+        attn_output = torch.matmul(attn_weights, v)
+        attn_output = attn_output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.n_embd)
+        attn_output = self.c_proj(attn_output)
+        return attn_output
+
+class FeedForward(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.c_fc = nn.Linear(config.n_embd, config.n_inner)
+        self.c_proj = nn.Linear(config.n_inner, config.n_embd)
+        self.dropout = nn.Dropout(config.dropout)
+    def forward(self, x):
+        x = self.c_fc(x)
+        x = F.gelu(x)
+        x = self.c_proj(x)
+        x = self.dropout(x)
+        return x
+
+class TransformerBlock(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
+        self.attn = MultiHeadAttention(config)
+        self.ln_2 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
+        self.mlp = FeedForward(config)
+    def forward(self, x, attention_mask=None):
+        x = x + self.attn(self.ln_1(x), attention_mask=attention_mask)
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+class ChessForCausalLM(PreTrainedModel):
+    config_class = ChessConfig
+    base_model_prefix = "transformer"
+    supports_gradient_checkpointing = True
+    keys_to_ignore_on_load_missing = ["lm_head.weight"]
+    def __init__(self, config):
+        super().__init__(config)
+        self.wte = nn.Embedding(config.vocab_size, config.n_embd)
+        self.drop = nn.Dropout(config.dropout)
+        self.h = nn.ModuleList([TransformerBlock(config) for _ in range(config.n_layer)])
+        self.ln_f = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+        if config.tie_weights:
+            self._tied_weights_keys = ["lm_head.weight"]
+        self.post_init()
+        if config.tie_weights:
+            self.tie_weights()
+    def get_input_embeddings(self):
+        return self.wte
+    def set_input_embeddings(self, new_embeddings):
+        self.wte = new_embeddings
+        if getattr(self.config, "tie_weights", False):
+            self.tie_weights()
+    def get_output_embeddings(self):
+        return self.lm_head
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+    def tie_weights(self):
+        if getattr(self.config, "tie_weights", False) or getattr(self.config, "tie_word_embeddings", False):
+            self._tie_or_clone_weights(self.lm_head, self.wte)
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                torch.nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+        elif isinstance(module, nn.LayerNorm):
+            torch.nn.init.ones_(module.weight)
+            torch.nn.init.zeros_(module.bias)
+    def forward(self, input_ids, attention_mask=None, position_ids=None, labels=None, return_dict=None, **kwargs):
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        batch_size, seq_len = input_ids.size()
+        token_embeds = self.wte(input_ids)
+        hidden_states = self.drop(token_embeds)
+        for block in self.h:
+            hidden_states = block(hidden_states, attention_mask=attention_mask)
+        hidden_states = self.ln_f(hidden_states)
+        logits = self.lm_head(hidden_states)
+        loss = None
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = nn.CrossEntropyLoss(ignore_index=-100)
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+        if not return_dict:
+            output = (logits,)
+            return ((loss,) + output) if loss is not None else output
+        return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=None, hidden_states=None, attentions=None)
+

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:605ee3e104ab4399da70b315787459ee6001bbabdea4c4737f2022db410c208a
+size 3007720

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "[BOS]",
+  "eos_token": "[EOS]",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

tokenizer.py

@@ -0,0 +1,55 @@
+import json
+import os
+from transformers import PreTrainedTokenizer
+
+
+class ChessTokenizer(PreTrainedTokenizer):
+    model_input_names = ["input_ids", "attention_mask"]
+    vocab_files_names = {"vocab_file": "vocab.json"}
+    PAD_TOKEN = "[PAD]"
+    BOS_TOKEN = "[BOS]"
+    EOS_TOKEN = "[EOS]"
+    UNK_TOKEN = "[UNK]"
+    def __init__(self, vocab_file=None, vocab=None, **kwargs):
+        self._pad_token = self.PAD_TOKEN
+        self._bos_token = self.BOS_TOKEN
+        self._eos_token = self.EOS_TOKEN
+        self._unk_token = self.UNK_TOKEN
+        kwargs.pop("pad_token", None)
+        kwargs.pop("bos_token", None)
+        kwargs.pop("eos_token", None)
+        kwargs.pop("unk_token", None)
+        if vocab is not None:
+            self._vocab = vocab
+        elif vocab_file is not None and os.path.exists(vocab_file):
+            with open(vocab_file, "r", encoding="utf-8") as f:
+                self._vocab = json.load(f)
+        else:
+            self._vocab = self._create_default_vocab()
+        self._ids_to_tokens = {v: k for k, v in self._vocab.items()}
+        super().__init__(pad_token=self._pad_token, bos_token=self._bos_token, eos_token=self._eos_token, unk_token=self._unk_token, **kwargs)
+    def _create_default_vocab(self):
+        special_tokens = [self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN]
+        return {token: idx for idx, token in enumerate(special_tokens)}
+    @property
+    def vocab_size(self):
+        return len(self._vocab)
+    def get_vocab(self):
+        return dict(self._vocab)
+    def _tokenize(self, text):
+        return text.strip().split()
+    def _convert_token_to_id(self, token):
+        return self._vocab.get(token, self._vocab.get(self.UNK_TOKEN, 0))
+    def _convert_id_to_token(self, index):
+        return self._ids_to_tokens.get(index, self.UNK_TOKEN)
+    def convert_tokens_to_string(self, tokens):
+        special = {self.PAD_TOKEN, self.BOS_TOKEN, self.EOS_TOKEN, self.UNK_TOKEN}
+        return " ".join(t for t in tokens if t not in special)
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory, exist_ok=True)
+        vocab_file = os.path.join(save_directory, (filename_prefix + "-" if filename_prefix else "") + "vocab.json")
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            json.dump(self._vocab, f, ensure_ascii=False, indent=2)
+        return (vocab_file,)
+

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenizer.ChessTokenizer",
+      null
+    ]
+  },
+  "tokenizer_class": "ChessTokenizer",
+  "bos_token": "[BOS]",
+  "eos_token": "[EOS]",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

vocab.json

@@ -0,0 +1,35 @@
+{
+  "[PAD]": 0,
+  "[BOS]": 1,
+  "[EOS]": 2,
+  "[UNK]": 3,
+  "WNb1c3": 4,
+  "BPe7e5": 5,
+  "WPe2e4": 6,
+  "BBf8c5": 7,
+  "WQd1h5": 8,
+  "BNg8f6": 9,
+  "WQh5e5(x+)": 10,
+  "BBc5e7": 11,
+  "WPd2d3": 12,
+  "BPd7d6": 13,
+  "WQe5f4": 14,
+  "BNf6h5": 15,
+  "WQf4f3": 16,
+  "BNh5f6": 17,
+  "WQf3g3": 18,
+  "BNf6g4": 19,
+  "WBf1e2": 20,
+  "BPh7h5": 21,
+  "WBe2g4(x)": 22,
+  "BBc8g4(x)": 23,
+  "WPf2f3": 24,
+  "BPh5h4": 25,
+  "WQg3g4(x)": 26,
+  "BPg7g6": 27,
+  "WNc3d5": 28,
+  "BBe7f8": 29,
+  "WBc1g5": 30,
+  "BQd8d7": 31,
+  "WNd5f6(+)": 32
+}