chessdata

.gitattributes

@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text
+*.wandb filter=lfs diff=lfs merge=lfs -text
+*.log filter=lfs diff=lfs merge=lfs -text
+*.jsonl
+*.wandb
+*.log
+*.jsonl*
+*.wandb*
+*.log*
+*.pdf  

chess/chessgpt_data_chess_puzzle_dataset_info.json

@@ -0,0 +1,24 @@
+{
+  "shard_lengths": [
+    199851,
+    199924,
+    199453,
+    199726,
+    199272,
+    199512,
+    199747,
+    199343,
+    199682,
+    198787,
+    199206,
+    199248,
+    199706,
+    199924,
+    199539,
+    198900
+  ],
+  "total_size": 1525978293,
+  "filename_pattern": "chess_puzzle-data.jsonl-{shard_id:05d}-of-{num_shards:05d}",
+  "num_examples": 3191820,
+  "dataset_name": "chess_puzzle"
+}

chess/process_jsonlines.py

@@ -0,0 +1,24 @@
+import jsonlines
+import re
+
+def extract_fen(puzzle_data):
+    text = puzzle_data["text"]
+    fen = re.search('(?<=is )[0-9A-Za-z]+/[0-9A-Za-z]+/[0-9A-Za-z]+/[0-9A-Za-z]+/[0-9A-Za-z]+/[0-9A-Za-z]+/[0-9A-Za-z]+/.+?[,.]', text).group(0)[:-1]
+
+    return fen
+
+
+for i in range(16): 
+    read_file = 'chess_puzzle-data.jsonl-000' + str(i).zfill(2) + '-of-00016'
+    with jsonlines.open(read_file) as reader:
+        write_file = 'puzzle' + str(i).zfill(2) + '.jsonl'
+        with jsonlines.open(write_file, mode='w') as writer: 
+            # count = 0
+            for puzzle in reader:
+                # count += 1
+                writer.write(extract_fen(puzzle))
+                # for key, value in puzzle.items():
+                #     writer.write("key: " + key + " value: " + str(value))
+                # writer.write("-----")
+                # if count == 10:
+                    # break

chess/puzzle00.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e65c604357197a5ddfdadc131f21ccd11d8fc3d0961a0b2bb4f3e3f5ab7eafc0
+size 11582431

chess/puzzle01.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83b7730299910a88c0b61b217b8623ff10ade2165c3a599b5e498410a7fb6b7e
+size 11591576

chess/puzzle02.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc50ac1a84612dbff90d3079fe2e126420fcb5d475edf16be621449b99df225d
+size 11561724

chess/puzzle03.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e137c284018cbd7da06d4f6af76b0f4e2fd8c9b2ea6b78f668a3071d6e56f67
+size 11581850

chess/puzzle04.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0c9e27136067121f97d670af6837da130b8040c588b6c89c9ca9315f7c54752
+size 11551762

chess/puzzle05.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22d7e3e0308506625eae9234b40d2224014d71d4f7e8be48f244a393d6449398
+size 11568178

chess/puzzle06.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35123330882cc045444050a044cdee04ec2067a2351491318e1c604e9053c7da
+size 11577697

chess/puzzle07.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f60c2689fc013ac9d7e0434e66a203d167cb3e21355eabfb55b49a8136dab935
+size 11555884

chess/puzzle08.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e759021b88813aec161ee676bb723105ff5ef1a1673f4fbf17485d86a7bb9e69
+size 11580910

chess/puzzle09.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e58cc12c5e42a98ca2ea1c3b5de4f0e248020a96c8b74e729e22530aea3c710
+size 11520781

chess/puzzle10.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:561492edd423940a845204030489fa0890c40fb248c94c91de2ab0352d8e9a68
+size 11542376

chess/puzzle11.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99d81aec2219794d7c0ace6d9067bd68a51b4bb6f332bfe5c80890a44dab7e39
+size 11548297

chess/puzzle12.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4bcfe91fc66133dc83ebac6460fd9ad62cd414be1eaad9599b0e1037cbdbb5b
+size 11576974

chess/puzzle13.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51d17eab86eb8d2c1d3b6e6a22ea95b91c0f1bc77120f3fb78e7117f62cc2488
+size 11588079

chess/puzzle14.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ebc6aa1241c858b1c7145ca863e6bdba6f41db6db4520a833972af483069046
+size 11568015

chess/puzzle15.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c47c8013506927eae1c019afaeb479035cee935a5ba7c63b1a44d33b0222df3
+size 11529247