Chess Challenge submission by iliasslasri

README.md

@@ -14,13 +14,13 @@ Chess model submitted to the LLM Course Chess Challenge.
 ## Submission Info
 
 - **Submitted by**: [iliasslasri](https://huggingface.co/iliasslasri)
-- **Parameters**: 997,136
+- **Parameters**: 980,720
 - **Organization**: LLM-course
 
 ## Model Details
 
 - **Architecture**: Chess Transformer (GPT-style)
 - **Vocab size**: 75
-- **Embedding dim**: 96
+- **Embedding dim**: 92
 - **Layers**: 11
-- **Heads**: 8
+- **Heads**: 4

config.json

@@ -1,9 +1,9 @@
 {
-  "_name_or_path": "./gqa_1_ft/checkpoint-719934/",
+  "_name_or_path": "./11_4_92_ft_ft_ft/checkpoint-475008/",
   "architectures": [
     "ChessForCausalLM"
   ],
-  "attn": "GQA",
+  "attn": "MHA",
   "auto_map": {
     "AutoConfig": "model.ChessConfig",
     "AutoModelForCausalLM": "model.ChessForCausalLM"
@@ -14,11 +14,11 @@
   "layer_norm_epsilon": 1e-05,
   "model_type": "chess_transformer",
   "n_ctx": 256,
-  "n_embd": 96,
-  "n_head": 8,
-  "n_inner": 304,
+  "n_embd": 92,
+  "n_head": 4,
+  "n_inner": 276,
   "n_layer": 11,
-  "num_groups": 4,
+  "num_groups": 2,
   "pad_token_id": 0,
   "tie_weights": false,
   "tie_word_embeddings": false,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b7e13ee34b41230e39c989ce65f47c313da964b2d1fecb0c27bd1feccde1890
-size 4003888
+oid sha256:0f7244a5c854e9c9684f98b1b63970ad82899c0545f1fb1b105ce1ae2e8f76a8
+size 3934384