Update README.md

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+gpt-p_CHARS_CHAT_vocab_size=33n_embed=384context_size=256n_layer=6n_head=6dropout=0.2 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,32 @@
----
-license: cc0-1.0
----
+---
+license: cc0-1.0
+datasets:
+- Lichess/standard-chess-games
+pipeline_tag: text2text-generation
+tags:
+- chess
+---
+# Model card for chessPT
+A pretrained Decoder only transformer model for chess move prediction.
+
+## Intended use
+Predict new moves in a chess game based on PGN tokens.
+
+## Implementation
+The model implementation is based on Andrej Karpathy's [nanoGPT](https://github.com/karpathy/nanoGPT) following the webseries "Zero to Hero" on [youtube](https://www.youtube.com/playlist?list=PLAqhIrjkxbuWI23v9cThsA9GvCAUhRvKZ).
+
+## Training
+You can find the training script in the repositories files under `train.py`.
+This also contains the used parameters
+```python
+context_size = 256
+batch_size = 128
+max_iters = 30_000
+learning_rate = 3e-5
+eval_interval = 100
+eval_iters = 20
+n_embed = 384
+n_layer = 6
+n_head = 6
+dropout = 0.2
+```

gpt-p_CHARS_CHAT_vocab_size=33n_embed=384context_size=256n_layer=6n_head=6dropout=0.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfda5c18a6a7dcc83b73857034e48b26b420ca8be96c03c7f50097622d78a298
+size 52603542

train.py

@@ -0,0 +1,129 @@
+import argparse
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+from gpt_p.model import DecoderTransformer
+from datasets import load_dataset
+
+
+torch.manual_seed(420) # 1337
+
+base_name = 'gpt-p_CHARS_CHAT_'
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+context_size = 256 # how many tokens to consider while generating the next
+batch_size = 128 # how many independent sequences will we process in parallel
+max_iters = 30_000
+learning_rate = 3e-5
+eval_interval = 100
+eval_iters = 20 # number evaluation iterations
+n_embed = 384 # embedding size
+n_layer = 6 # number of transformer layers
+n_head = 6
+dropout = 0.2 # dropout factor
+
+dataset = load_dataset('Lichess/standard-chess-games', split='train')
+content = '\n'.join(list(filter(lambda x: 'eval' not in x, dataset['movetext'])))
+
+## BUILD DATA SET ##
+book = content
+characters = sorted(list(set(book)))
+vocab_size = len(characters)
+
+# convert 
+stoi = {ch: idx for idx, ch in enumerate(characters)}
+itos = {idx: ch for idx, ch in enumerate(characters)}
+
+encode = lambda s: [stoi[c] for c in s]
+decode = lambda i: ''.join([itos[x] for x in i])
+
+
+data = torch.tensor(encode(book), dtype=torch.long)
+n = int(0.9 * len(data))
+train_data = data[:n]
+val_data = data[n:]
+
+
+def get_batch(split):
+    data = train_data if split == 'train' else val_data
+    idx = torch.randint(len(data) - context_size, (batch_size,))
+    x = torch.stack([data[i:i+context_size] for i in idx])
+    y = torch.stack([data[i+1:i+context_size+1] for i in idx])
+    return x.to(device), y.to(device)
+
+## END BUILD DATA SET ##
+## MODEL DEFINITION ##
+
+def print_sample(input_value=None):
+    if input_value is None:
+        input_value = torch.zeros((1,1), dtype=torch.long, device=device)
+    print('Validation sample:')
+    sample = decode(model.generate(input_value, max_new_tokens=250, context_size=context_size)[0].tolist())
+    if '<E>' in sample:
+        sample = sample[:sample.find('<E>') + 3]
+    print(sample)
+
+
+@torch.no_grad()
+def estimate_loss():
+    out = {}
+    model.eval()
+    for split in ['train', 'val']:
+        losses = torch.zeros(eval_iters)
+        for k in range(eval_iters):
+            X, Y = get_batch(split)
+            logits, loss = model(X, Y)
+            losses[k] = loss.item()
+        out[split] = losses.mean()
+
+    input_string = '1. e4 g6'
+    print_sample(torch.tensor(encode(input_string), dtype=torch.long, device=device).view((1, len(input_string))))
+    model.train()
+    return out
+
+
+if __name__ == "__main__":
+    args = argparse.ArgumentParser()
+    args.add_argument('--load', '-l', action='store_true', default=False, help='Load model state.')
+    args.add_argument('--inference', '-i', action='store_true', default=False, help='Run only inference')
+    
+    args = args.parse_args()
+
+    params = {'vocab_size': vocab_size, 'n_embed': n_embed, 'context_size': context_size, 'n_layer': n_layer, 'n_head': n_head, 'dropout': dropout}
+    if args.load:
+        m = DecoderTransformer(vocab_size, n_embed, context_size, n_layer, n_head, dropout)
+        m.load_state_dict(torch.load(f'./models/{base_name}' + ''.join(f'{key}={v}' for key, v in params.items())))
+    else:
+        m = DecoderTransformer(vocab_size, n_embed, context_size, n_layer, n_head, dropout)
+    model = m.to(device)
+
+    if args.inference:
+        exit()
+    ## END MODEL ##
+    ## START TRAINING ##
+    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)
+
+    for step in range(max_iters):
+        if step % eval_interval == 0:
+            losses = estimate_loss()
+            print(f'step {step:4d}: train loss {losses["train"]:.4f}, val loss: {losses["val"]:.4f}')
+
+        xb, yb = get_batch('train')
+
+        logits, loss = model(xb, yb)
+        optimizer.zero_grad(set_to_none=True)
+        loss.backward()
+        optimizer.step()
+
+    print()
+    print('Loss:')
+    print(loss.item())
+
+    ## END TRAINING ##
+    ## START VALIDATION ##
+
+    ## END VALIDATION ##
+
+    # save model weights
+    torch.save(model.state_dict(), f'./models/{base_name}' + ''.join([f'{key}={v}' for key, v in params.items()]))
+    with open('train.log', 'a') as f:
+        f.write(f'{max_iters},{learning_rate}\n')