Chess proven 1M

README.md

@@ -5,11 +5,17 @@ tags:
 license: mit
 ---
 
-# Chess GPT - Proven Architecture
+# Chess GPT - Prof's Architecture
 
-Params: 975,808
-Vocab: 1804 (TOP_K=2000)
-Dataset: 1M samples x 10 epochs
-UNK rate: 24.2%
+Params: 998,656
+Vocab: 1604 (TOP_K=2000)
+Dataset: 1M samples x 5 epochs
+
+Config:
+- n_embd: 128
+- n_layer: 4
+- n_head: 4
+- LR: 5e-4
+- UNK rate: 25.7%
 
 Target: 60-70% legal rate

config.json

@@ -15,10 +15,10 @@
   "n_ctx": 256,
   "n_embd": 128,
   "n_head": 4,
-  "n_inner": 320,
-  "n_layer": 5,
+  "n_inner": 512,
+  "n_layer": 4,
   "pad_token_id": 0,
   "tie_weights": true,
   "transformers_version": "4.57.6",
-  "vocab_size": 1804
+  "vocab_size": 1604
 }

model.py

@@ -26,7 +26,7 @@ def apply_rope(q, k):
 
 class ChessConfig(PretrainedConfig):
     model_type = "chess_transformer"
-    def __init__(self, vocab_size=1804, n_embd=128, n_layer=6, n_head=8, n_ctx=256, n_inner=None, dropout=0.1, layer_norm_epsilon=1e-05, tie_weights=True, pad_token_id=0, bos_token_id=1, eos_token_id=2, **kwargs):
+    def __init__(self, vocab_size=1604, n_embd=128, n_layer=4, n_head=4, n_ctx=256, n_inner=None, dropout=0.1, layer_norm_epsilon=1e-05, tie_weights=True, pad_token_id=0, bos_token_id=1, eos_token_id=2, **kwargs):
         super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
         self.vocab_size = vocab_size
         self.n_embd = n_embd

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6e3cbb9b7f597f471f675808f06654046cde64cf9d14196d6226640eb5dc8c6
-size 3908576
+oid sha256:55dfdcc3502c61d227c19892983221a8a00ae4c084f1ce855076fda72325c02b
+size 3998960

vocab.json

@@ -1602,205 +1602,5 @@
   "WNc3d1(x)": 1600,
   "BNe4g5(x)": 1601,
   "BBe4g6": 1602,
-  "WNh3f2": 1603,
-  "BRa2b2": 1604,
-  "BQh4f6": 1605,
-  "WQe2c4(x)": 1606,
-  "BRf8f4": 1607,
-  "BQg5f6": 1608,
-  "WKe4e3": 1609,
-  "BRg8e8": 1610,
-  "WQd3e4(x)": 1611,
-  "BBf5d3": 1612,
-  "BRe8e3": 1613,
-  "WNg4e5": 1614,
-  "WKb3b4": 1615,
-  "BBh3g4": 1616,
-  "WQe1g3": 1617,
-  "BNd4b3(x)": 1618,
-  "BBe5g7": 1619,
-  "BKd5e5": 1620,
-  "BBc6f3(x)": 1621,
-  "BKh5g4": 1622,
-  "BNc4b2(x)": 1623,
-  "BBb4d2(x)": 1624,
-  "BRe8a8": 1625,
-  "WQg3g4": 1626,
-  "WKc4d5": 1627,
-  "WBb2d4": 1628,
-  "BBc6d5(x)": 1629,
-  "WRa1a6(x)": 1630,
-  "WBc1d2(x)": 1631,
-  "WBb3e6(x)": 1632,
-  "BQe7e6(x)": 1633,
-  "WQf3h5": 1634,
-  "BNc4e5": 1635,
-  "WBc4f7(x)": 1636,
-  "BRd2b2(x)": 1637,
-  "BRh8b8": 1638,
-  "WKd4c3": 1639,
-  "WNb5d6(+)": 1640,
-  "BRf8a8(x)": 1641,
-  "WKb3c2": 1642,
-  "WBf3c6(x)": 1643,
-  "WBd5b7(x)": 1644,
-  "WKd4d5": 1645,
-  "WQc2d3(x)": 1646,
-  "BBg7h6(x)": 1647,
-  "WBb5d7(x)": 1648,
-  "BQe6g6": 1649,
-  "BBg4h3": 1650,
-  "WKg4h3": 1651,
-  "WKc3b2": 1652,
-  "WQd2f4(x)": 1653,
-  "WKg5f6": 1654,
-  "WBd3a6(x)": 1655,
-  "BRd8e8(x)": 1656,
-  "BNe8d6": 1657,
-  "WQf3d5(x)": 1658,
-  "WRa7b7": 1659,
-  "WNb5c7(x)": 1660,
-  "WRf1e1(x)": 1661,
-  "WBd2c1": 1662,
-  "WKd4e4": 1663,
-  "WQb7a7(x)": 1664,
-  "WKb2c2": 1665,
-  "WRe2e3": 1666,
-  "BBc6d5": 1667,
-  "BPe7d6(x)": 1668,
-  "WQd3c3": 1669,
-  "WQb3d1": 1670,
-  "BNc6d8(x)": 1671,
-  "WQg3h4": 1672,
-  "WBf1a6(x)": 1673,
-  "BRc2a2(x)": 1674,
-  "WNd3c5": 1675,
-  "WBd2f4(x)": 1676,
-  "WBc2e4(x)": 1677,
-  "BRa2a1(+)": 1678,
-  "BKb5c4": 1679,
-  "WKe5f6": 1680,
-  "BPd2d1(Q)": 1681,
-  "WBb3d5": 1682,
-  "WNe4f6(x)": 1683,
-  "BQd6c6": 1684,
-  "WKb3a4": 1685,
-  "WRd7b7(x)": 1686,
-  "BKe5e6": 1687,
-  "WRc1c6": 1688,
-  "BQd6c7": 1689,
-  "BBg6e4": 1690,
-  "WNe3c4": 1691,
-  "WQf3e4": 1692,
-  "WKc4d3": 1693,
-  "WPd7d8(Q)": 1694,
-  "BRd8d2(+)": 1695,
-  "WBe3c1": 1696,
-  "WRf2f1": 1697,
-  "BQd7f7": 1698,
-  "WKf5g6": 1699,
-  "WNf4e2": 1700,
-  "WBd5c6(x)": 1701,
-  "BKc7d8": 1702,
-  "BNd6c4": 1703,
-  "WNc5e4": 1704,
-  "WBc3b4": 1705,
-  "BQe7b4": 1706,
-  "WQe3e2": 1707,
-  "BBd6b8": 1708,
-  "WQe2d3(x)": 1709,
-  "WNf5e7(+)": 1710,
-  "BBd4b6": 1711,
-  "WKd5c6": 1712,
-  "WRh1h5(x)": 1713,
-  "BQe5f6": 1714,
-  "WQb3c3": 1715,
-  "BKf4e3": 1716,
-  "WNe2c1": 1717,
-  "WKf5e6": 1718,
-  "BQf6b2(x)": 1719,
-  "WQb3a4": 1720,
-  "BBf8b4(x)": 1721,
-  "BKc6c7": 1722,
-  "BQd7d8": 1723,
-  "WQd1b1": 1724,
-  "WRe1e7(+)": 1725,
-  "WKg5g6": 1726,
-  "WRh1d1(x)": 1727,
-  "WNg5h7(x)": 1728,
-  "BBe6g4(x)": 1729,
-  "BRa2a3": 1730,
-  "WNf5e3": 1731,
-  "BRa8a7(x)": 1732,
-  "WPg4g5(+)": 1733,
-  "BRd7c7": 1734,
-  "WPe6f7(x+)": 1735,
-  "BKg5g6": 1736,
-  "BPf5f4(+)": 1737,
-  "BQb4b2(x)": 1738,
-  "BPd7e6(x)": 1739,
-  "BKf4f3": 1740,
-  "BNf4e6": 1741,
-  "BNc4b6": 1742,
-  "BQc7b8": 1743,
-  "BKc4b3": 1744,
-  "WQc2d1": 1745,
-  "BRe8h8": 1746,
-  "BNf4g6": 1747,
-  "BNe6c5": 1748,
-  "BBd4e5": 1749,
-  "WQd2c3(x)": 1750,
-  "WQe4f3": 1751,
-  "WNd6e4": 1752,
-  "WPf7f8(Q)": 1753,
-  "WNe1d3": 1754,
-  "BNe5f3(x)": 1755,
-  "WRd2c2": 1756,
-  "WBe5g7(x)": 1757,
-  "WRa1a4(x)": 1758,
-  "WKb3c3": 1759,
-  "BRa8a5(x)": 1760,
-  "BQe7e5": 1761,
-  "BBh5e2(x)": 1762,
-  "BNf6g4(+)": 1763,
-  "WQd3b3": 1764,
-  "WBe4c6(x)": 1765,
-  "WRe2f2": 1766,
-  "BKg5h6": 1767,
-  "BKc5b6": 1768,
-  "BBd4b2(x)": 1769,
-  "WNc4d6(x)": 1770,
-  "WKf5f6": 1771,
-  "BBf8a3(x)": 1772,
-  "BNc2d4": 1773,
-  "BBa6c4(x)": 1774,
-  "WRa1a7": 1775,
-  "WQd4d2": 1776,
-  "BRf8f2": 1777,
-  "BNc7e6": 1778,
-  "WPe2f3(x)": 1779,
-  "WBg2e4": 1780,
-  "BRc8c1(x+)": 1781,
-  "BRa8a4(x)": 1782,
-  "WNe5c4(x)": 1783,
-  "BQd5e6(+)": 1784,
-  "WNd2b1": 1785,
-  "WQd2d3(x)": 1786,
-  "BRh8d8(x)": 1787,
-  "WRc7a7(x)": 1788,
-  "WQe2e5(x)": 1789,
-  "WRf1f5": 1790,
-  "BBf5h7": 1791,
-  "WRc1c4": 1792,
-  "BKb7c8": 1793,
-  "BNd8e6": 1794,
-  "WRc1c8(x+)": 1795,
-  "BNg5e4": 1796,
-  "BQf6d6": 1797,
-  "BRf7g7": 1798,
-  "BNc4e3(x)": 1799,
-  "WKc2c1": 1800,
-  "WNf4e6": 1801,
-  "BKe4f3": 1802,
-  "BKg7f6(x)": 1803
+  "WNh3f2": 1603
 }