Download pro_data.py from Convergent-ch/Arkon-Engine-0.5-Max: direct link, hf CLI and curl.
- Browser
- Download file 6.38 kB
-
https://huggingface.co/Convergent-ch/Arkon-Engine-0.5-Max/resolve/main/pro_data.py
- Command line
-
hf download hf://Convergent-ch/Arkon-Engine-0.5-Max/pro_data.py
-
curl -L -o pro_data.py https://huggingface.co/Convergent-ch/Arkon-Engine-0.5-Max/resolve/main/pro_data.py
6.38 kB
| """Generate game positions and verified mating positions for Arkon Pro.""" | |
| from concurrent.futures import ProcessPoolExecutor | |
| import json | |
| import math | |
| import random | |
| import subprocess | |
| import time | |
| from arkon import DATA, ROOT, TEACHER, chess | |
| OUT = ROOT / "pro" | |
| PRO_DATA = OUT / "positions.jsonl" | |
| TACTICS = OUT / "mate-test.jsonl" | |
| def key(board): | |
| return tuple(board.fen().split()[:4]) | |
| def endgame(rng): | |
| board = chess.Board.empty() | |
| pieces = rng.choice(("KQk", "KRk", "KRRk", "KQrk", "KRpk", "KQNk")) | |
| for square, symbol in zip(rng.sample(range(64), len(pieces)), pieces): | |
| board.set_piece_at(square, chess.Piece.from_symbol(symbol)) | |
| if rng.random() < .5: | |
| board = board.mirror() | |
| return board | |
| def mate_moves(board, distance=1): | |
| for move in list(board.legal_moves): | |
| board.push(move) | |
| try: | |
| if board.is_checkmate(): | |
| yield_move = True | |
| elif distance == 2 and not board.is_game_over(): | |
| yield_move = True | |
| for reply in list(board.legal_moves): | |
| board.push(reply) | |
| try: | |
| if next(mate_moves(board), None) is None: | |
| yield_move = False | |
| break | |
| finally: | |
| board.pop() | |
| else: | |
| yield_move = False | |
| finally: | |
| board.pop() | |
| if yield_move: | |
| yield move | |
| def shard(job): | |
| worker, goal, nodes, deadline, *destination = job | |
| directory = destination[0] if destination else OUT | |
| rng = random.Random(13919 + worker) | |
| engine = chess.engine.SimpleEngine.popen_uci( | |
| str(TEACHER), timeout=20, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0) | |
| ) | |
| engine.configure({"Threads": 1, "Hash": 24}) | |
| count = game = 0 | |
| path = directory / f"shard-{worker}.jsonl" | |
| try: | |
| with path.open("w", encoding="utf-8") as output: | |
| while count < goal and time.monotonic() < deadline: | |
| game += 1 | |
| board = chess.Board() | |
| if game % 5 == 0: | |
| for _ in range(50): | |
| candidate = endgame(rng) | |
| if candidate.is_valid() and not candidate.is_game_over(): | |
| board = candidate | |
| break | |
| if board.ply() == 0 and board.castling_rights: | |
| for _ in range(rng.randrange(4, 13)): | |
| legal = list(board.legal_moves) | |
| if not legal: | |
| break | |
| board.push(rng.choice(legal)) | |
| for ply in range(100): | |
| if board.is_game_over(claim_draw=True) or time.monotonic() >= deadline: | |
| break | |
| info = engine.analyse(board, chess.engine.Limit(nodes=nodes), multipv=3) | |
| moves = [item["pv"][0] for item in info] | |
| scores = [item["score"].pov(board.turn).score(mate_score=3500) for item in info] | |
| if ply % 2 == game % 2 or scores[0] > 3400: | |
| labels = [[m.uci(), s] for m, s in zip(moves, scores)] | |
| output.write(json.dumps([f"{directory.name}:{worker}:{game}", board.fen(), labels]) + "\n") | |
| count += 1 | |
| if count % 1000 == 0: | |
| output.flush() | |
| print(f"Worker {worker}: {count}/{goal}", flush=True) | |
| if count >= goal: | |
| break | |
| weights = [math.exp(max(-12, (s - scores[0]) / 85)) for s in scores] | |
| board.push(rng.choices(moves, weights)[0]) | |
| finally: | |
| engine.quit() | |
| return path, count | |
| def tactical_rows(seen, *, seed, count, test=False): | |
| rng = random.Random(seed) | |
| rows = [] | |
| for _ in range(count * 150): | |
| board = endgame(rng) | |
| if not board.is_valid() or board.is_game_over() or key(board) in seen: | |
| continue | |
| immediate = list(mate_moves(board)) | |
| distance = 1 | |
| matches = immediate | |
| if not matches and len(rows) % 3 == 0: | |
| matches = list(mate_moves(board, 2)) | |
| distance = 2 | |
| if not matches: | |
| continue | |
| seen.add(key(board)) | |
| if test: | |
| rows.append([board.fen(), distance, [m.uci() for m in matches]]) | |
| else: | |
| rng.shuffle(matches) | |
| rows.append([f"mate:{len(rows)}", board.fen(), [[m.uci(), 3500 - distance] for m in matches[:3]]]) | |
| if len(rows) >= count: | |
| break | |
| return rows | |
| def assemble(paths): | |
| seen = set() | |
| for line in DATA.read_text(encoding="utf-8").splitlines(): | |
| seen.add(key(chess.Board(json.loads(line)[1]))) | |
| count = 0 | |
| with PRO_DATA.open("w", encoding="utf-8") as output: | |
| for path in paths: | |
| with path.open(encoding="utf-8") as source: | |
| for line in source: | |
| _, fen, _ = json.loads(line) | |
| identity = key(chess.Board(fen)) | |
| if identity in seen: | |
| continue | |
| seen.add(identity) | |
| output.write(line) | |
| count += 1 | |
| tests = tactical_rows(seen, seed=70113, count=90, test=True) | |
| rows = tactical_rows(seen, seed=72113, count=2600) | |
| for row in rows: | |
| output.write(json.dumps(row) + "\n") | |
| TACTICS.write_text("".join(json.dumps(row) + "\n" for row in tests), encoding="utf-8") | |
| return count, len(rows), len(tests) | |
| def prepare(): | |
| OUT.mkdir(exist_ok=True) | |
| with ProcessPoolExecutor(max_workers=6) as pool: | |
| start = time.monotonic() | |
| pilot = list(pool.map(shard, [(i + 6, 200, 12000, start + 180) for i in range(6)])) | |
| rate = sum(count for _, count in pilot) / (time.monotonic() - start) | |
| nodes = max(4000, min(40000, round(12000 * rate / 125))) | |
| print(f"Pilot: {rate:.1f} positions/s. Main search: {nodes} nodes per position.", flush=True) | |
| deadline = time.monotonic() + 17 * 60 | |
| jobs = [(i, 17500, nodes, deadline) for i in range(6)] | |
| result = list(pool.map(shard, jobs)) | |
| return assemble([p for p, _ in pilot + result]), nodes | |