""" train_local.py - Treino PPO do Ankh no PC (CPU em varios processos + GPU em lote). Instalar: pip install gymnasium stable-baselines3 sb3-contrib psutil (e o PyTorch com CUDA: https://pytorch.org/get-started/locally/) Rodar: python train_local.py --dir ~/ankh_runs/run1 Retomar: rode o mesmo comando (usa o ultimo checkpoint de --dir). Fase 2 (soma zero): python train_local.py --dir ~/ankh_runs/fase2 --zero-sum \ --init-from ~/ankh_runs/run1/checkpoints/round_0040.zip Parar: Ctrl+C (salva um checkpoint antes de sair). Deixe os arquivos ankh_*.py na mesma pasta deste script. Os processos de CPU usam 'spawn', por isso o bloco `if __name__ == "__main__"` no fim e obrigatorio. Saidas em --dir: checkpoints/, snapshots/ (oponentes do self-play), tb/ e metrics.csv. """ import argparse import csv import gc import glob import os import random import shutil import time import numpy as np import torch from sb3_contrib import MaskablePPO from stable_baselines3.common.callbacks import BaseCallback from ankh_eval import evaluate, evaluate_matrix from ankh_report import Agg, log_round, make_stats_callback, render_markdown from ankh_game import GODS, AnkhGame from ankh_map import SCENARIOS from ankh_vec import BatchedSelfPlayVecEnv, TorchOpponent def default_workers(): """Nucleos fisicos - 1 (deixa um para o processo principal, que cuida da GPU).""" n = 0 try: import psutil n = psutil.cpu_count(logical=False) or 0 except ImportError: pass if not n: n = max((os.cpu_count() or 2) // 2, 1) return max(n - 1, 1) class DiskCheckpoint(BaseCallback): """Salva o modelo a cada `every` passos e apaga os mais antigos.""" def __init__(self, every, directory, keep=3): super().__init__() self.every, self.dir, self.keep, self.last = every, directory, keep, 0 def _on_training_start(self): self.last = self.num_timesteps def _on_step(self): if self.num_timesteps - self.last >= self.every: self.last = self.num_timesteps self.model.save(f"{self.dir}/step_{self.num_timesteps:010d}.zip") for old in sorted(glob.glob(f"{self.dir}/step_*.zip"))[:-self.keep]: os.remove(old) return True def newest(directory): files = glob.glob(f"{directory}/*.zip") return max(files, key=os.path.getmtime) if files else None def single(opp): """Adapta um oponente em lote (TorchOpponent) para decisoes individuais.""" return lambda obs, mask: int(opp(obs[None], mask[None])[0]) def parse(argv=None): p = argparse.ArgumentParser() p.add_argument("--dir", default="ankh_run", help="pasta de saida (checkpoints, snapshots, tb, metrics)") p.add_argument("--workers", type=int, default=0, help="processos de CPU (0 = nucleos fisicos - 1)") p.add_argument("--envs", type=int, default=0, help="partidas em paralelo (0 = 16 por worker)") p.add_argument("--round-steps", type=int, default=400_000) p.add_argument("--total-rounds", type=int, default=40) p.add_argument("--save-every", type=int, default=100_000) p.add_argument("--keep-step-ckpts", type=int, default=3) p.add_argument("--keep-rounds", type=int, default=10) p.add_argument("--snapshot-every", type=int, default=1) p.add_argument("--eval-per-god", type=int, default=8, help="partidas de avaliacao por deus E por cenario") p.add_argument("--scenarios", default="all", help="cenarios usados (nomes separados por virgula) ou all") p.add_argument("--players", default="2,3,4,5", help="numeros de jogadores treinados, ex.: 2 ou 2,3") p.add_argument("--matrix-every", type=int, default=5, help="a cada N rounds, matriz de confrontos entre deuses (0 = nunca)") p.add_argument("--matrix-games", type=int, default=3, help="partidas por par de deuses na matriz") p.add_argument("--god-pool", default="0,1,2,3,4", help="deuses sorteados (0 Amun, 1 Anubis, 2 Isis, 3 Osiris, 4 Ra), ex.: 4 ou 0,4") p.add_argument("--n-steps", type=int, default=128) p.add_argument("--lr", type=float, default=3e-4) p.add_argument("--merged-reward", choices=["normal", "zero"], default="normal", help="zero: os assentos de um deus fundido recebem recompensa 0 depois da fusao") p.add_argument("--zero-sum", action="store_true", help="recompensas de soma zero (pesos simetricos, Egito ateu = 0)") p.add_argument("--init-from", default="", help="checkpoint .zip de uma fase anterior: usa os pesos como ponto de partida " "e o semeia como primeiro oponente do self-play") return p.parse_args(argv) def main(argv=None): a = parse(argv) root = os.path.abspath(os.path.expanduser(a.dir)) ckpt, snap, tb, metrics = (f"{root}/checkpoints", f"{root}/snapshots", f"{root}/tb", f"{root}/metrics.csv") for d in (ckpt, snap, tb): os.makedirs(d, exist_ok=True) device = "cuda" if torch.cuda.is_available() else "cpu" if device == "cpu": print("AVISO: GPU nao encontrada; treinando na CPU (bem mais lento).") workers = a.workers or default_workers() n_envs = a.envs or 16 * workers batch = max((a.n_steps * n_envs) // 4, 64) print("saida em:", root, "| relatorio:", f"{root}/relatorio.md") print(f"dispositivo: {device} | workers: {workers} | partidas: {n_envs} | lote de coleta: {a.n_steps * n_envs}") god_pool = [int(x) for x in a.god_pool.split(",")] print("deuses sorteados:", [GODS[x] for x in god_pool]) wanted = [int(x) for x in a.players.split(",")] names = list(SCENARIOS) if a.scenarios == "all" else a.scenarios.split(",") names = [n for n in names if SCENARIOS[n]["n_players"] in wanted] probe = AnkhGame(names) scenarios = list(probe.scenarios) print("cenarios usados:", {n: SCENARIOS[n]["n_players"] for n in scenarios}) for n, errs in probe.skipped.items(): print(f"AVISO: cenario {n} ignorado (erro nos dados): {errs[0]}") venv = BatchedSelfPlayVecEnv(n_envs=n_envs, n_workers=workers, seed=int(time.time()) % 100000, zero_sum=a.zero_sum, god_pool=god_pool, scenarios=scenarios, merged_reward=a.merged_reward) print("recompensas:", "SOMA ZERO" if a.zero_sum else "padrao (nao simetricas)") last = newest(ckpt) if last: model = MaskablePPO.load(last, env=venv, device=device, tensorboard_log=tb) print("RETOMANDO de", os.path.basename(last), "| passos ja feitos:", model.num_timesteps) else: model = MaskablePPO("MlpPolicy", venv, n_steps=a.n_steps, batch_size=batch, n_epochs=4, gamma=0.995, gae_lambda=0.95, ent_coef=0.01, learning_rate=a.lr, verbose=0, device=device, tensorboard_log=tb, policy_kwargs=dict(net_arch=[512, 512])) print("treino NOVO") if a.init_from: model.set_parameters(os.path.expanduser(a.init_from), device=device) print("pesos iniciais carregados de", a.init_from) if not glob.glob(f"{snap}/snap_*.zip"): shutil.copy(os.path.expanduser(a.init_from), f"{snap}/snap_0000.zip") if not os.path.exists(metrics): with open(metrics, "w", newline="") as f: csv.writer(f).writerow(["timesteps", "win_vs_prev", "atheist_vs_prev", "prev_snapshot", "train_opponent", "steps_per_s", "minutes"] + [f"w_{g}" for g in GODS]) goal, t0, opp = a.total_rounds * a.round_steps, time.time(), None try: while model.num_timesteps < goal: todo = a.round_steps - model.num_timesteps % a.round_steps del opp gc.collect() if device == "cuda": torch.cuda.empty_cache() # Seleção do oponente de treino pool = sorted(glob.glob(f"{snap}/snap_*.zip")) if not pool: opp, opp_name = None, "aleatorio" elif len(pool) == 1: path = pool[0] opp, opp_name = TorchOpponent(path, device=device), os.path.basename(path) else: if random.random() < 0.5: path = pool[-1] # 50% último modelo else: candidates = pool[:-1] if len(candidates) <= 4: path = random.choice(candidates) else: if random.random() < 0.8: path = random.choice(candidates[-4:]) else: path = random.choice(candidates[:-4]) opp, opp_name = TorchOpponent(path, device=device), os.path.basename(path) venv.set_opponent(opp) t_round = time.time() agg_train = Agg() model.learn(todo, reset_num_timesteps=False, tb_log_name="ppo", progress_bar=False, callback=[DiskCheckpoint(a.save_every, ckpt, a.keep_step_ckpts), make_stats_callback(agg_train)]) sps = todo / (time.time() - t_round) r = model.num_timesteps // a.round_steps model.save(f"{ckpt}/round_{r:04d}.zip") for old in sorted(glob.glob(f"{ckpt}/round_*.zip"))[:-a.keep_rounds]: os.remove(old) # Avaliação única contra o snapshot anterior agg_p = Agg() prev_files = sorted(glob.glob(f"{snap}/snap_*.zip")) # ANTES de salvar o novo snapshot if prev_files: prev_name = os.path.basename(prev_files[-1]) prev_fn = single(TorchOpponent(prev_files[-1], device=device)) pw, pa, by_god = evaluate(model, a.eval_per_god, prev_fn, god_pool=god_pool, agg=agg_p, scenarios=scenarios) else: prev_name, pw, pa, prev_fn, by_god = "", 0.0, 0.0, None, {} if r % a.snapshot_every == 0: model.save(f"{snap}/snap_{r:04d}.zip") minutes = (time.time() - t0) / 60 matrix = None if a.matrix_every and r % a.matrix_every == 0 and prev_fn: matrix = evaluate_matrix(model, prev_fn, god_pool, a.matrix_games, scenarios=scenarios) log_round(root, {"round": r, "steps": model.num_timesteps, "minutes": minutes, "opponent": opp_name, "prev_snapshot": prev_name, "rewards": "soma zero" if a.zero_sum else "padrao", "gods": [GODS[x] for x in god_pool], "scenarios": {n: SCENARIOS[n]["n_players"] for n in scenarios}, "train": agg_train.to_dict(), "eval_prev": agg_p.to_dict() if prev_name else None, "matrix": matrix, "matrix_vs": prev_name or "nenhum"}) render_markdown(root) with open(metrics, "a", newline="") as f: csv.writer(f).writerow([model.num_timesteps, pw, pa, prev_name, opp_name, round(sps), round(minutes, 1)] + [round(by_god.get(g, float("nan")), 2) for g in range(len(GODS))]) vs_prev = f"vs {prev_name}: vitorias {pw:.0%}, Egito ateu {pa:.0%}" if prev_name else "vs anterior: (sem snapshot ainda)" per_god = " ".join(f"{GODS[g][:3]} {w:.0%}" for g, w in by_god.items()) if by_god else "" god_str = f" ({per_god})" if per_god else "" print(f"round {r:3d} | passos {model.num_timesteps:>10,} | {sps:,.0f} passos/s | {vs_prev}{god_str} | treinou contra: {opp_name} | {minutes:.0f} min") print("treino concluido") except KeyboardInterrupt: path = f"{ckpt}/step_{model.num_timesteps:010d}.zip" model.save(path) print("\ninterrompido; checkpoint salvo em", path) finally: venv.close() if __name__ == "__main__": main()