Chess Challenge submission by iliasslasri

README.md

@@ -14,13 +14,13 @@ Chess model submitted to the LLM Course Chess Challenge.
 ## Submission Info
 
 - **Submitted by**: [iliasslasri](https://huggingface.co/iliasslasri)
-- **Parameters**: 980,720
+- **Parameters**: 997,136
 - **Organization**: LLM-course
 
 ## Model Details
 
 - **Architecture**: Chess Transformer (GPT-style)
 - **Vocab size**: 75
-- **Embedding dim**: 92
+- **Embedding dim**: 96
 - **Layers**: 11
-- **Heads**: 4
+- **Heads**: 8

config.json

@@ -1,9 +1,9 @@
 {
-  "_name_or_path": "./11_4_92_ft_ft_ft/checkpoint-411921/",
+  "_name_or_path": "./gqa_1/checkpoint-341412/",
   "architectures": [
     "ChessForCausalLM"
   ],
-  "attn": "MHA",
+  "attn": "GQA",
   "auto_map": {
     "AutoConfig": "model.ChessConfig",
     "AutoModelForCausalLM": "model.ChessForCausalLM"
@@ -14,11 +14,11 @@
   "layer_norm_epsilon": 1e-05,
   "model_type": "chess_transformer",
   "n_ctx": 256,
-  "n_embd": 92,
-  "n_head": 4,
-  "n_inner": 276,
+  "n_embd": 96,
+  "n_head": 8,
+  "n_inner": 304,
   "n_layer": 11,
-  "num_groups": 2,
+  "num_groups": 4,
   "pad_token_id": 0,
   "tie_weights": false,
   "tie_word_embeddings": false,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:acf4e3c08c84faca194f0b7f0b9053b222458bdf7a7c516fc49d225aa7ae4a33
-size 3934384
+oid sha256:f569dc1198e833f8e1923bcf79f61d5b50dc6d343527f363fe4f210e3ff45ab5
+size 4003888