diff --git a/Alpha_MCTS.py b/Alpha_MCTS.py new file mode 100644 index 0000000000000000000000000000000000000000..0c6fc5936188ae58c2959083f2ca6f8ab49a0471 --- /dev/null +++ b/Alpha_MCTS.py @@ -0,0 +1,120 @@ +import math +import torch +import numpy as np + +class Node: + + def __init__(self, game, args, state, parent = None, action = None, prob = 0, visits = 0): + self.game = game + self.args = args + self.state = state + self.parent = parent + self.action = action + self.prob = prob + + self.children = [] + self.visits = visits + self.value = 0 + + def leaf_or_not(self): + return len(self.children) > 0 + + def search(self): + best_child = None + best_ucb = -np.inf + for child in self.children: + ucb = self.get_ucb(child) + if best_ucb < ucb: + best_ucb = ucb + best_child = child + return best_child + + def get_ucb(self, child): + if child.visits == 0: + q_value = 0 + else: + q_value = 1 - (((child.value / child.visits) + 1) / 2) + + return q_value + self.args['EXPLORATION_CONSTANT'] * (math.sqrt(self.visits) / (child.visits + 1)) * child.prob + + def expand(self, policy): + + for move, prob in enumerate(policy): + if prob > 0: + child = self.state.copy() + child = self.game.make_move(child, move, 1) + if self.args["ADVERSARIAL"]: + child = self.game.change_perspective(child, player = -1) + + child = Node(self.game, self.args, child, self, move, prob) + self.children.append(child) + + def backpropagate(self,state_value): + self.value += state_value + self.visits += 1 + if self.args["ADVERSARIAL"]: + state_value = self.game.get_opponent_value(state_value) + if self.parent is not None: + self.parent.backpropagate(state_value) + + +class Alpha_MCTS: + + def __init__(self, game, args, model): + self.game = game + self.args = args + self.model = model + + @torch.no_grad() + def search(self, state): + root = Node(self.game, self.args, state, visits = 1) + + if self.args["ROOT_RANDOMNESS"]: + policy, _ = self.model( + torch.tensor(self.game.get_encoded_state(state), device = self.model.device + ).unsqueeze(0)) + + policy = torch.softmax(policy, axis = 1).squeeze(0).cpu().numpy() + + policy = (1 - self.args["DIRICHLET_EPSILON"]) * policy + self.args["DIRICHLET_EPSILON"] * np.random.dirichlet([self.args["DIRICHLET_ALPHA"]] * self.game.possible_state) + + valid_state = self.game.get_valid_moves(state) + policy *= valid_state + policy /= np.sum(policy) + root.expand(policy) + + for _ in range(self.args["NO_OF_SEARCHES"]): + node = root + no_moves = 0 + while node.leaf_or_not(): + node = node.search() + no_moves += 1 + + is_terminal, value = self.game.know_terminal_value(node.state, node.action) + + if self.args["ADVERSARIAL"]: + value = self.game.get_opponent_value(value) + + if not is_terminal: + + policy, value = self.model( + torch.tensor(self.game.get_encoded_state(node.state), device = self.model.device).unsqueeze(0) + ) + + valid_state = self.game.get_valid_moves(node.state) + policy = torch.softmax(policy, axis = 1).squeeze(0).cpu().numpy().astype(np.float64) + + policy *= valid_state + policy /= np.sum(policy) + + value = value.item() + + node.expand(policy) + + node.backpropagate(value) + + move_probability = np.zeros(self.game.possible_state) + for children in root.children: + move_probability[children.action] = children.visits + move_probability /= np.sum(move_probability) + return move_probability diff --git a/Alpha_MCTS_Parallel.py b/Alpha_MCTS_Parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..b184c57ba3b63be675691e5b4276a1c4830d58c1 --- /dev/null +++ b/Alpha_MCTS_Parallel.py @@ -0,0 +1,128 @@ +import math +import torch +import numpy as np + +class Node: + + def __init__(self, game, args, state, parent = None, action = None, prob = 0, visits = 0): + self.game = game + self.args = args + self.state = state + self.parent = parent + self.action = action + self.prob = prob + + self.children = [] + self.visits = visits + self.value = 0 + + def leaf_or_not(self): + return len(self.children) > 0 + + def search(self): + best_child = None + best_ucb = -np.inf + for child in self.children: + ucb = self.get_ucb(child) + if best_ucb < ucb: + best_ucb = ucb + best_child = child + return best_child + + def get_ucb(self, child): + if child.visits == 0: + q_value = 0 + else: + q_value = 1 - ((child.value / child.visits) + 1) / 2 + + return q_value + self.args['EXPLORATION_CONSTANT'] * (math.sqrt(self.visits) / (child.visits + 1)) * child.prob + + def expand(self, policy): + + for move, prob in enumerate(policy): + if prob > 0: + child = self.state.copy() + child = self.game.make_move(child, move, 1) + if self.args["ADVERSARIAL"]: + child = self.game.change_perspective(child, player = -1) + + child = Node(self.game, self.args, child, self, move, prob) + self.children.append(child) + + def backpropagate(self,state_value): + self.value += state_value + self.visits += 1 + if self.args["ADVERSARIAL"]: + state_value = self.game.get_opponent_value(state_value) + if self.parent is not None: + self.parent.backpropagate(state_value) + + +class Alpha_MCTS: + + def __init__(self, game, args, model): + self.game = game + self.args = args + self.model = model + + @torch.no_grad() + def search(self, states, spGames): + + policy, _ = self.model( + torch.tensor(self.game.get_encoded_state(states), device = self.model.device + )) + + policy = torch.softmax(policy, axis = 1).cpu().numpy() + + if self.args["ROOT_RANDOMNESS"]: + policy = (1 - self.args["DIRICHLET_EPSILON"]) * policy + self.args["DIRICHLET_EPSILON"] * np.random.dirichlet([self.args["DIRICHLET_ALPHA"]] * self.game.possible_state, size = policy.shape[0]) + + for i, spg in enumerate(spGames): + spg_policy = policy[i] + valid_state = self.game.get_valid_moves(states[i]) + spg_policy *= valid_state + spg_policy /= np.sum(spg_policy) + + spg.root = Node(self.game, self.args, states[i], visits = 1) + spg.root.expand(spg_policy) + + for _ in range(self.args["NO_OF_SEARCHES"]): + + for spg in spGames: + spg.node = None + node = spg.root + + while node.leaf_or_not(): + node = node.search() + + is_terminal, value = self.game.know_terminal_value(node.state, node.action) + + if self.args["ADVERSARIAL"]: + value = self.game.get_opponent_value(value) + + if is_terminal: + node.backpropagate(value) + else: + spg.node = node + + expandabel_spgs = [mapping_index for mapping_index in range(len(spGames)) if spGames[mapping_index].node is not None] + + if len(expandabel_spgs) > 0: + states = np.stack([spGames[mapping_index].node.state for mapping_index in expandabel_spgs]) + policy, value = self.model( + torch.tensor(self.game.get_encoded_state(states), device = self.model.device) + ) + + policy = torch.softmax(policy, axis = 1).cpu().numpy().astype(np.float64) + value = value.cpu().numpy() + + for i, mapping_index in enumerate(expandabel_spgs): + node = spGames[mapping_index].node + spg_policy, spg_value = policy[i], value[i] + valid_state = self.game.get_valid_moves(node.state) + + spg_policy *= valid_state + spg_policy /= np.sum(spg_policy) + + node.expand(spg_policy) + node.backpropagate(spg_value) diff --git a/Alpha_Zero.py b/Alpha_Zero.py new file mode 100644 index 0000000000000000000000000000000000000000..23d68fa48211c0fc3be7a229582bdce8e61f5917 --- /dev/null +++ b/Alpha_Zero.py @@ -0,0 +1,143 @@ +import os +import random +import copy +import numpy as np + +import torch +import torch.nn.functional as F + +from tqdm import trange +from Alpha_MCTS import Alpha_MCTS +from Arena import Arena + +class Colors: + RESET = "\033[0m" + RED = "\033[91m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + BLUE = "\033[94m" + MAGENTA = "\033[95m" + CYAN = "\033[96m" + WHITE = "\033[97m" + + +class Alpha_Zero: + + def __init__(self, game, args, model, optimizer): + self.game = game + self.args = args + self.model = model + self.optimizer = optimizer + self.mcts = Alpha_MCTS(game, args, model) + + def self_play(self): + + single_game_memory = [] + player = 1 + state = self.game.initialise_state() + + while True: + neutral_state = self.game.change_perspective(state, player) if self.args["ADVERSARIAL"] else state + prob = self.mcts.search(neutral_state) + + single_game_memory.append((neutral_state, prob, player)) + + temp_prob = prob ** (1 / self.args["TEMPERATURE"]) + + temp_prob[temp_prob == 0] = - np.inf + + temp_prob = torch.softmax(torch.tensor(temp_prob), axis = 0).cpu().numpy() + + move = np.random.choice(self.game.possible_state, p = temp_prob) + + state = self.game.make_move(state, move, player) + is_terminal, value = self.game.know_terminal_value(state, move) + + if is_terminal: + return_memory = [] + for return_state, return_action_prob, return_player in single_game_memory: + if self.args["ADVERSARIAL"]: + return_value = value if return_player == player else self.game.get_opponent_value(value) + else: + return_value = value + + return_memory.append(( + self.game.get_encoded_state(return_state), + return_action_prob, + return_value + )) + return return_memory + + if self.args["ADVERSARIAL"]: + player = self.game.get_opponent(player) + + def train(self, memory): + + random.shuffle(memory) + + for batch_start in range(0, len(memory), self.args["BATCH_SIZE"]): + batch_end = batch_start + self.args["BATCH_SIZE"] + + training_memory = memory[batch_start : batch_end] + + state, action_prob, value = zip(*training_memory) + + state, action_prob, value = np.array(state), np.array(action_prob), np.array(value).reshape(-1, 1) + + state = torch.tensor(state, device = self.model.device, dtype=torch.float32) + policy_targets = torch.tensor(action_prob, device = self.model.device, dtype=torch.float32) + value_targets = torch.tensor(value, device = self.model.device, dtype=torch.float32) + + out_policy, out_value = self.model(state) + + policy_loss = F.cross_entropy(out_policy, policy_targets) + value_loss = F.mse_loss(out_value, value_targets) + loss = policy_loss + value_loss + + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + def learn(self): + try: + model_path = os.path.join(self.args["MODEL_PATH"], 'model.pt') + optimizer_path = os.path.join(self.args["MODEL_PATH"], 'optimizer.pt') + + self.model.load_state_dict(torch.load(model_path)) + self.optimizer.load_state_dict(torch.load(optimizer_path)) + except: + print(Colors.RED + "UNABLE TO LOAD MODEL") + print(Colors.GREEN + "SETTING UP NEW MODEL..." + Colors.RESET) + + else: + print(Colors.GREEN + "MODEL FOUND\nLOADING MODEL..." + Colors.RESET) + finally: + + initial_model = copy.copy(self.model) + + for iteration in range(self.args["NO_ITERATIONS"]): + memory = [] + + print(Colors.BLUE + "\nIteration no: " , iteration + 1, Colors.RESET) + + print(Colors.YELLOW + "Self Play" + Colors.RESET) + self.model.eval() + + for _ in trange(self.args["SELF_PLAY_ITERATIONS"]): + memory += self.self_play() + + print(Colors.YELLOW + "Training..." + Colors.RESET) + self.model.train() + for _ in trange(self.args["EPOCHS"]): + self.train(memory) + + print(Colors.YELLOW + "Testing..." + Colors.RESET) + self.model.eval() + wins, draws, defeats = Arena(self.game, self.args, self.model, initial_model) + print(Colors.GREEN + "Testing Completed" + Colors.WHITE + "\nTrained Model Stats:") + print(Colors.GREEN, "Wins: ", wins, Colors.RESET, "|", Colors.RED, "Loss: ", defeats, Colors.RESET, "|", Colors.WHITE," Draw: ", draws, Colors.RESET) + + print(Colors.YELLOW + "Saving Model...") + torch.save(self.model.state_dict(), os.path.join(self.args["MODEL_PATH"], "model_non_parallel.pt")) + torch.save(self.optimizer.state_dict(), os.path.join(self.args["MODEL_PATH"], "optimizer_non_parallel.pt")) + print("Saved!" + Colors.RESET) diff --git a/Games/2048/2048.py b/Games/2048/2048.py new file mode 100644 index 0000000000000000000000000000000000000000..bb3bbda933407c090fadaac65ca0d453a5debca9 --- /dev/null +++ b/Games/2048/2048.py @@ -0,0 +1,41 @@ +"""NOTE : THIS IS THE FORMAT YOU SHOULD FOLLOW TO MAKE GAMES""" + +class GAME: + def __init__(self): + self.row = 4 + self.col = 4 + self.possible_state = self.row * self.col # no of possible state game can have (no of possible moves) + + def initialise_state(self): + """NOTE: this fuction creates starting position for your game""" + + def get_valid_moves(self, state): + """ NOTE: inputs state and output all possible valid moves""" + + def know_terminal_value(self, state, action): + """NOTE: input state and action and the output will be tuple True + if is terminal state and False if not and value 0 is + player lost and 1 if the player won the game""" + + def make_move(self, state, action, player): + """NOTE: input a action and a state and a player, the + action is taken on given state and output is the final state""" + + def get_encoded_state(self, state): + """input a state the output should be the encoded state that will be + given to the neural network""" + + def get_opponent(self, player): + """NOTE: taking the input of current player this should change the + player to the opponent""" + return -1 * player + + def get_opponent_value(self, value): + """this should switch the reward obtained from the know_terminal_state() + to the reward of the opponent""" + return -1 * value + + def change_perspective(self, state, player = -1): + """this function taking input state and player flip the board and return + the game with the perspective of the opponent""" + return player * state diff --git a/Games/2048/2048NN.py b/Games/2048/2048NN.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Games/Chess/Chess.py b/Games/Chess/Chess.py new file mode 100644 index 0000000000000000000000000000000000000000..276cc20ec9cac65072c481356480cc1560c999f8 --- /dev/null +++ b/Games/Chess/Chess.py @@ -0,0 +1,41 @@ +"""NOTE : THIS IS THE FORMAT YOU SHOULD FOLLOW TO MAKE GAMES""" + +class GAME: + def __init__(self): + self.row = 8 + self.col = 8 + self.possible_state = self.row * self.col + + def initialise_state(self): + """NOTE: this fuction creates starting position for your game""" + + def get_valid_moves(self, state): + """ NOTE: inputs state and output all possible valid moves""" + + def know_terminal_value(self, state, action): + """NOTE: input state and action and the output will be tuple True + if is terminal state and False if not and value 0 is + player lost and 1 if the player won the game""" + + def make_move(self, state, action, player): + """NOTE: input a action and a state and a player, the + action is taken on given state and output is the final state""" + + def get_encoded_state(self, state): + """input a state the output should be the encoded state that will be + given to the neural network""" + + def get_opponent(self, player): + """NOTE: taking the input of current player this should change the + player to the opponent""" + return -1 * player + + def get_opponent_value(self, value): + """this should switch the reward obtained from the know_terminal_state() + to the reward of the opponent""" + return -1 * value + + def change_perspective(self, state, player = -1): + """this function taking input state and player flip the board and return + the game with the perspective of the opponent""" + return player * state diff --git a/Games/Chess/ChessNN.py b/Games/Chess/ChessNN.py new file mode 100644 index 0000000000000000000000000000000000000000..79b761219ed777c56fe016b9734bc3c265008053 --- /dev/null +++ b/Games/Chess/ChessNN.py @@ -0,0 +1,63 @@ +import torch.nn as nn +import torch.nn.functional as F + +class ResNet(nn.Module): + def __init__(self, game, num_resBlocks, num_hidden, device): + super().__init__() + self.device = device + self.startBlock = nn.Sequential( + nn.Conv2d(3, num_hidden, kernel_size=3, padding=1), + nn.BatchNorm2d(num_hidden), + nn.ReLU(), + nn.Conv2d(3, num_hidden, kernel_size=3, padding=1), + nn.BatchNorm2d(num_hidden), + nn.ReLU() + ) + + self.backBone = nn.ModuleList( + [ResBlock(num_hidden) for i in range(num_resBlocks)] + ) + + self.policyHead = nn.Sequential( + nn.Conv2d(num_hidden, 128, kernel_size=3, padding=1), + nn.BatchNorm2d(128), + nn.ReLU(), + nn.Flatten(), + nn.Linear(128 * game.row * game.col, game.possible_state) + ) + + self.valueHead = nn.Sequential( + nn.Conv2d(num_hidden, 3, kernel_size=3, padding=1), + nn.BatchNorm2d(3), + nn.ReLU(), + nn.Flatten(), + nn.Linear(3 * game.row * game.col, 1), + nn.Tanh() + ) + + self.to(device) + + def forward(self, x): + x = self.startBlock(x) + for resBlock in self.backBone: + x = resBlock(x) + policy = self.policyHead(x) + value = self.valueHead(x) + return policy, value + + +class ResBlock(nn.Module): + def __init__(self, num_hidden): + super().__init__() + self.conv1 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1) + self.bn1 = nn.BatchNorm2d(num_hidden) + self.conv2 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1) + self.bn2 = nn.BatchNorm2d(num_hidden) + + def forward(self, x): + residual = x + x = F.relu(self.bn1(self.conv1(x))) + x = self.bn2(self.conv2(x)) + x += residual + x = F.relu(x) + return x diff --git a/Games/Chess/Stokfish.py b/Games/Chess/Stokfish.py new file mode 100644 index 0000000000000000000000000000000000000000..33ba8a3f7e53aaf120c29cdb8c931370d614fce3 --- /dev/null +++ b/Games/Chess/Stokfish.py @@ -0,0 +1 @@ +import Stokfish \ No newline at end of file diff --git a/Games/ConnectFour/ConnectFour.py b/Games/ConnectFour/ConnectFour.py new file mode 100644 index 0000000000000000000000000000000000000000..01cc3b2d90cbe13ead81faa217bd791a9254f65f --- /dev/null +++ b/Games/ConnectFour/ConnectFour.py @@ -0,0 +1,76 @@ +import numpy as np + +class ConnectFour: + def __init__(self): + self.row = 6 + self.col = 7 + self.possible_state = self.col + self.in_a_row = 4 + + def __repr__(self): + return "ConnectFour" + + def initialise_state(self): + return np.zeros((self.row, self.col)) + + def make_move(self, state, action, player): + row = np.max(np.where(state[:, action] == 0)) + state[row, action] = player + return state + + def get_valid_moves(self, state): + return (state[0] == 0).astype(np.uint8) + + def check_win(self, state, action): + if action == None: + return False + + row = np.min(np.where(state[:, action] != 0)) + column = action + player = state[row][column] + + def count(offset_row, offset_column): + for i in range(1, self.in_a_row): + r = row + offset_row * i + c = action + offset_column * i + if ( + r < 0 + or r >= self.row + or c < 0 + or c >= self.col + or state[r][c] != player + ): + return i - 1 + return self.in_a_row - 1 + + return ( + count(1, 0) >= self.in_a_row - 1 # vertical + or (count(0, 1) + count(0, -1)) >= self.in_a_row - 1 # horizontal + or (count(1, 1) + count(-1, -1)) >= self.in_a_row - 1 # top left diagonal + or (count(1, -1) + count(-1, 1)) >= self.in_a_row - 1 # top right diagonal + ) + + def know_terminal_value(self, state, action): + if self.check_win(state, action): + return True, 1 + if np.sum(self.get_valid_moves(state)) == 0: + return True, 0 + return False, 0 + + def get_opponent(self, player): + return -player + + def get_opponent_value(self, value): + return -value + + def change_perspective(self, state, player): + return state * player + + def get_encoded_state(self, state): + encoded_state = np.stack( + (state == -1, state == 0, state == 1) + ).astype(np.float32) + if len(state.shape) == 3: + encoded_state = np.swapaxes(encoded_state, 0, 1) + + return encoded_state \ No newline at end of file diff --git a/Games/ConnectFour/ConnectFourNN.py b/Games/ConnectFour/ConnectFourNN.py new file mode 100644 index 0000000000000000000000000000000000000000..d32a8a401549f83be66302f848ef4f62ac7119cf --- /dev/null +++ b/Games/ConnectFour/ConnectFourNN.py @@ -0,0 +1,61 @@ +import torch.nn as nn +import torch.nn.functional as F + + +class ResNet(nn.Module): + def __init__(self, game, num_resBlocks, num_hidden, device): + super().__init__() + self.device = device + self.startBlock = nn.Sequential( + nn.Conv2d(3, num_hidden, kernel_size=3, padding=1), + nn.BatchNorm2d(num_hidden), + nn.ReLU() + ) + + self.backBone = nn.ModuleList( + [ResBlock(num_hidden) for i in range(num_resBlocks)] + ) + + self.policyHead = nn.Sequential( + nn.Conv2d(num_hidden, 32, kernel_size=3, padding=1), + nn.BatchNorm2d(32), + nn.ReLU(), + nn.Flatten(), + nn.Linear(32 * game.row * game.col, game.possible_state) + ) + + self.valueHead = nn.Sequential( + nn.Conv2d(num_hidden, 3, kernel_size=3, padding=1), + nn.BatchNorm2d(3), + nn.ReLU(), + nn.Flatten(), + nn.Linear(3 * game.row * game.col, 1), + nn.Tanh() + ) + + self.to(device) + + def forward(self, x): + x = self.startBlock(x) + for resBlock in self.backBone: + x = resBlock(x) + policy = self.policyHead(x) + value = self.valueHead(x) + return policy, value + + +class ResBlock(nn.Module): + def __init__(self, num_hidden): + super().__init__() + self.conv1 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1) + self.bn1 = nn.BatchNorm2d(num_hidden) + self.conv2 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1) + self.bn2 = nn.BatchNorm2d(num_hidden) + + def forward(self, x): + residual = x + x = F.relu(self.bn1(self.conv1(x))) + x = self.bn2(self.conv2(x)) + x += residual + x = F.relu(x) + return x \ No newline at end of file diff --git a/Games/ConnectFour/__pycache__/ConnectFour.cpython-310.pyc b/Games/ConnectFour/__pycache__/ConnectFour.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ceed04de4e68a1124904d3551205774942e4593 Binary files /dev/null and b/Games/ConnectFour/__pycache__/ConnectFour.cpython-310.pyc differ diff --git a/Games/ConnectFour/__pycache__/ConnectFour.cpython-311.pyc b/Games/ConnectFour/__pycache__/ConnectFour.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24cad37a7d2ad2720064e1417026dc8b4cb36188 Binary files /dev/null and b/Games/ConnectFour/__pycache__/ConnectFour.cpython-311.pyc differ diff --git a/Games/ConnectFour/__pycache__/ConnectFour.cpython-312.pyc b/Games/ConnectFour/__pycache__/ConnectFour.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b75ff59498a8159a7a7c12327ac4048b3ec74095 Binary files /dev/null and b/Games/ConnectFour/__pycache__/ConnectFour.cpython-312.pyc differ diff --git a/Games/ConnectFour/__pycache__/ConnectFour.cpython-313.pyc b/Games/ConnectFour/__pycache__/ConnectFour.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a09a829fc531fd625008cca0cbf64cc8b7b812c4 Binary files /dev/null and b/Games/ConnectFour/__pycache__/ConnectFour.cpython-313.pyc differ diff --git a/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-310.pyc b/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85f35ecfb5782de50f67e8c323c1b76c18fc5d7e Binary files /dev/null and b/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-310.pyc differ diff --git a/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-311.pyc b/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ef5e3c382e2ae12b9e2936b362f0e513eab9260 Binary files /dev/null and b/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-311.pyc differ diff --git a/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-312.pyc b/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25dc4b08b5a20d6509c6bb1b4b147b76bbd0e976 Binary files /dev/null and b/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-312.pyc differ diff --git a/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-313.pyc b/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9759e86d8bfa0c0bf8b51f4db3936eae350eb5d Binary files /dev/null and b/Games/ConnectFour/__pycache__/ConnectFourNN.cpython-313.pyc differ diff --git a/Games/ConnectFour/models_n_optimizers/model.pt b/Games/ConnectFour/models_n_optimizers/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..552d5208ddde07207cd70d5c767cce913ca77dbb --- /dev/null +++ b/Games/ConnectFour/models_n_optimizers/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ecbcdf83dc322c859ab3188f11e2f7b25ee504515a8bcf2b84798be7b1b3eb7 +size 10920462 diff --git a/Games/ConnectFour/models_n_optimizers/optimizer.pt b/Games/ConnectFour/models_n_optimizers/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..6df71d8afedaa332153e935042e7dad55afc19c3 --- /dev/null +++ b/Games/ConnectFour/models_n_optimizers/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7165fd808da4ff2c3535392d8cf651ee905b0d0a15aadbef7e2e8300a7387fd +size 21792762 diff --git a/Games/TicTacToe/TicTacToe.py b/Games/TicTacToe/TicTacToe.py new file mode 100644 index 0000000000000000000000000000000000000000..755ca1c6ec23c725c6424fcf906483e25b099b64 --- /dev/null +++ b/Games/TicTacToe/TicTacToe.py @@ -0,0 +1,59 @@ +import numpy as np + +class TicTacToe: + + def __init__(self): + self.col = 3 + self.row = 3 + self.possible_state = self.col * self.row + + def initialise_state(self): + return np.zeros((self.row, self.col)) + + def get_valid_moves(self, state): + return (state.reshape(-1) == 0).astype(np.uint8) + + def know_terminal_value(self, state, action): + + if action is None: + return False, 0 + + row=action//3 + col=action%3 + player = state[row,col] + + if (sum(state[row,:])==player*3 + or sum(state[:,col])==player*3 + or np.trace(state)==player*3 + or np.trace(np.fliplr(state))==player*3): + return True, 1 + + if 0 not in state.reshape(-1): + return True, 0 + else: + return False,0 + + def make_move(self, state, action, player): + row=action//3 + col=action%3 + + state[row,col]=player + + return state + + def get_encoded_state(self, state): + encoded_state = np.stack( + (state == -1, state == 0, state == 1) + ).astype(np.float32) + if len(state.shape) == 3: + encoded_state = np.swapaxes(encoded_state, 0, 1) + + return encoded_state + def get_opponent(self, player): + return -1 * player + + def get_opponent_value(self, value): + return -1 * value + + def change_perspective(self, state, player = -1): + return (player * state).astype(np.float32) diff --git a/Games/TicTacToe/TicTacToeNN.py b/Games/TicTacToe/TicTacToeNN.py new file mode 100644 index 0000000000000000000000000000000000000000..8c1f9a717f96e64f7f23ab525ab3be6cc3de7c59 --- /dev/null +++ b/Games/TicTacToe/TicTacToeNN.py @@ -0,0 +1,61 @@ +import torch.nn as nn +import torch.nn.functional as F + +class ResNet(nn.Module): + + def __init__(self, game, num_resBlocks, num_hidden, device): + super().__init__() + self.device = device + self.startBlock = nn.Sequential( + nn.Conv2d(3, num_hidden, kernel_size=3, padding=1), + nn.BatchNorm2d(num_hidden), + nn.ReLU() + ) + + self.backBone = nn.ModuleList( + [ResBlock(num_hidden) for i in range(num_resBlocks)] + ) + + self.policyHead = nn.Sequential( + nn.Conv2d(num_hidden, 32, kernel_size=3, padding=1), + nn.BatchNorm2d(32), + nn.ReLU(), + nn.Flatten(), + nn.Linear(32 * game.row * game.col, game.possible_state) + ) + + self.valueHead = nn.Sequential( + nn.Conv2d(num_hidden, 3, kernel_size=3, padding=1), + nn.BatchNorm2d(3), + nn.ReLU(), + nn.Flatten(), + nn.Linear(3 * game.row * game.col, 1), + nn.Tanh() + ) + + self.to(device) + + def forward(self, x): + x = self.startBlock(x) + for resBlock in self.backBone: + x = resBlock(x) + policy = self.policyHead(x) + value = self.valueHead(x) + return policy, value + + +class ResBlock(nn.Module): + def __init__(self, num_hidden): + super().__init__() + self.conv1 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1) + self.bn1 = nn.BatchNorm2d(num_hidden) + self.conv2 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1) + self.bn2 = nn.BatchNorm2d(num_hidden) + + def forward(self, x): + residual = x + x = F.relu(self.bn1(self.conv1(x))) + x = self.bn2(self.conv2(x)) + x += residual + x = F.relu(x) + return x diff --git a/Games/TicTacToe/__pycache__/TicTacToe.cpython-310.pyc b/Games/TicTacToe/__pycache__/TicTacToe.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1da5d22ee110058d3210eba9d1559f6a4765bf18 Binary files /dev/null and b/Games/TicTacToe/__pycache__/TicTacToe.cpython-310.pyc differ diff --git a/Games/TicTacToe/__pycache__/TicTacToe.cpython-311.pyc b/Games/TicTacToe/__pycache__/TicTacToe.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb56032c440a2904e4e1d18fa9f2c9ab19c3eda0 Binary files /dev/null and b/Games/TicTacToe/__pycache__/TicTacToe.cpython-311.pyc differ diff --git a/Games/TicTacToe/__pycache__/TicTacToe.cpython-313.pyc b/Games/TicTacToe/__pycache__/TicTacToe.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d8976c5b885f7810d608e28ff77534d7ba40f2d1 Binary files /dev/null and b/Games/TicTacToe/__pycache__/TicTacToe.cpython-313.pyc differ diff --git a/Games/TicTacToe/__pycache__/TicTacToeNN.cpython-310.pyc b/Games/TicTacToe/__pycache__/TicTacToeNN.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2845d1e7d38dbc158e1f66392aa0936b14671dbe Binary files /dev/null and b/Games/TicTacToe/__pycache__/TicTacToeNN.cpython-310.pyc differ diff --git a/Games/TicTacToe/__pycache__/TicTacToeNN.cpython-311.pyc b/Games/TicTacToe/__pycache__/TicTacToeNN.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d02c362739551b860dd517157bb00988b7d7968 Binary files /dev/null and b/Games/TicTacToe/__pycache__/TicTacToeNN.cpython-311.pyc differ diff --git a/Games/TicTacToe/__pycache__/TicTacToeNN.cpython-313.pyc b/Games/TicTacToe/__pycache__/TicTacToeNN.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..884e1dd92df21010a41412aab7487ec49d1c9da6 Binary files /dev/null and b/Games/TicTacToe/__pycache__/TicTacToeNN.cpython-313.pyc differ diff --git a/Games/TicTacToe/models_n_optimizers/model.pt b/Games/TicTacToe/models_n_optimizers/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..432f13d9e6d812a0b9fa6647f7ecf6e6dd4c4058 --- /dev/null +++ b/Games/TicTacToe/models_n_optimizers/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5877c49bfb164d48c9ccf7d165d9e647c14ed6956cb047e6f40b6e7237e9af2d +size 10900420 diff --git a/Games/TicTacToe/models_n_optimizers/model_non_parallel.pt b/Games/TicTacToe/models_n_optimizers/model_non_parallel.pt new file mode 100644 index 0000000000000000000000000000000000000000..df9ed63ad7fa5983841e490c0e948e807553da06 --- /dev/null +++ b/Games/TicTacToe/models_n_optimizers/model_non_parallel.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4cefe7c002f47fe17ee7b6304bdd6f32b44191bc00090604923300803062740 +size 10903056 diff --git a/Games/TicTacToe/models_n_optimizers/optimizer.pt b/Games/TicTacToe/models_n_optimizers/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e7380bf2156baf1cca4e94908195e8004600ba6 --- /dev/null +++ b/Games/TicTacToe/models_n_optimizers/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38db7c19396de4e717b28891427dcf3ff4efc1451c3a0cc7baff70b3cc5824c5 +size 21738002 diff --git a/Games/TicTacToe/models_n_optimizers/optimizer_non_parallel.pt b/Games/TicTacToe/models_n_optimizers/optimizer_non_parallel.pt new file mode 100644 index 0000000000000000000000000000000000000000..367f2c635e24d7b89ec6d6ac45276ccf7c2762bb --- /dev/null +++ b/Games/TicTacToe/models_n_optimizers/optimizer_non_parallel.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f346133359c24c36ae633ab24f2e4c47773d5e40506c5f2fb64a06f8c54c7fa +size 21741505 diff --git a/Games/game.py b/Games/game.py new file mode 100644 index 0000000000000000000000000000000000000000..0aa2c6577c63ab1c2eb49a5110a5c68b2ffc4dc2 --- /dev/null +++ b/Games/game.py @@ -0,0 +1,41 @@ +"""NOTE : THIS IS THE FORMAT YOU SHOULD FOLLOW TO MAKE GAMES""" + +class GAME: + def __init__(self): + self.row = None + self.col = None + self.possible_state = self.row * self.col # no of possible state game can have (no of possible moves) + + def initialise_state(self): + """NOTE: this fuction creates starting position for your game""" + + def get_valid_moves(self, state): + """ NOTE: inputs state and output all possible valid moves""" + + def know_terminal_value(self, state, action): + """NOTE: input state and action and the output will be tuple True + if is terminal state and False if not and value 0 is + player lost and 1 if the player won the game""" + + def make_move(self, state, action, player): + """NOTE: input a action and a state and a player, the + action is taken on given state and output is the final state""" + + def get_encoded_state(self, state): + """input a state the output should be the encoded state that will be + given to the neural network""" + + def get_opponent(self, player): + """NOTE: taking the input of current player this should change the + player to the opponent""" + return -1 * player + + def get_opponent_value(self, value): + """this should switch the reward obtained from the know_terminal_state() + to the reward of the opponent""" + return -1 * value + + def change_perspective(self, state, player = -1): + """this function taking input state and player flip the board and return + the game with the perspective of the opponent""" + return player * state diff --git a/MCTS.py b/MCTS.py new file mode 100644 index 0000000000000000000000000000000000000000..01d17190338ae60bdf5e4041198cb09a4492b401 --- /dev/null +++ b/MCTS.py @@ -0,0 +1,104 @@ +import math +import numpy as np + +class Node: + + def __init__(self, game, args, state, parent = None, action = None): + self.game = game + self.args = args + self.state = state + self.parent = parent + self.action = action + + self.children = list() + self.expandable_moves = self.game.get_moves(self.state) + self.visits = 0 + self.total_value = 0 + + def leaf_or_not(self): + return (len(self.children) > 0 and len(self.expandable_moves) == 0) + + def search(self): + best_child = None + best_ucb = -np.inf + for child in self.children: + ucb = self.get_ucb(child) + if best_ucb < ucb: + best_ucb = ucb + best_child = child + + return best_child + + def get_ucb(self, child): + q_value = 1 - ((child.total_value / child.visits) + 1) / 2 + return q_value + self.args["EXPLORATION_CONSTANT"] * math.sqrt(math.log(self.visits) / child.visits) + + def expand(self): + rand_move = np.random.choice(self.expandable_moves) + self.expandable_moves.remove(rand_move) + + child = self.game.make_move(self.state.copy(), rand_move, 1) + child = self.game.change_perspective(child) + child = Node(self.game, self.args, child, self, rand_move) + + self.children.append(child) + + return child + + def simulate(self): + is_terminal, value = self.game.know_terminal_value(self.state, self.action) + value = self.game.get_opponent_value(value) + + if is_terminal: + return value + + state = self.state.copy() + player = 1 + while True: + possible_moves = self.game.get_moves(state) + rand_move = np.random.choice(possible_moves) + state = self.game.make_move(state, rand_move, player) + is_terminal, value = self.game.know_terminal_value(state, rand_move) + if is_terminal: + if player == -1: + value = self.game.get_opponent_value(value) + return value + player = self.game.get_opponent(player) + + def backpropagate(self,value): + self.total_value += value + self.visits += 1 + + value = self.game.get_opponent_value(value) + if self.parent is not None: + self.parent.backpropagate(value) + + +class MCTS: + def __init__(self, game, args): + self.game = game + self.args = args + + def search(self, node): + root = Node(self.game, self.args, node) + + for _ in range(self.args["NO_OF_SEARCHES"]): + node = root + while node.leaf_or_not(): + node = node.search() + + is_terminal, value = self.game.know_terminal_value(node.state, node.action) + value = self.game.get_opponent_value(value) + + if not is_terminal: + node = node.expand() + value = node.simulate() + + node.backpropagate(value) + + move_probability = np.zeros(self.game.possible_state) + for children in root.children: + move_probability[children.action] = children.visits + move_probability /= np.sum(move_probability) + + return move_probability diff --git a/Play.py b/Play.py new file mode 100644 index 0000000000000000000000000000000000000000..fb6c8cf1d7dcbbc66c7b382a0e70723d757cdbdd --- /dev/null +++ b/Play.py @@ -0,0 +1,86 @@ + +import os +import numpy as np + +import torch + +from Games.ConnectFour.ConnectFour import ConnectFour +from Games.ConnectFour.ConnectFourNN import ResNet +from Alpha_MCTS import Alpha_MCTS + +class Colors: + RESET = "\033[0m" + RED = "\033[91m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + BLUE = "\033[94m" + MAGENTA = "\033[95m" + CYAN = "\033[96m" + WHITE = "\033[97m" + +GAME = "ConnectFour" +args = { + "MODEL_PATH" : os.path.join(os.getcwd(), "Games", GAME, "models_n_optimizers"), + + "ADVERSARIAL" : True, + "ROOT_RANDOMNESS": False, + + "TEMPERATURE" : 1, + + "NO_OF_SEARCHES" : 1200, + "EXPLORATION_CONSTANT" : 1, +} + + +game = ConnectFour() +device = torch.device("cuda" if torch.cuda.is_available else "cpu") + +model = ResNet(game, 9, 128, device) +model.eval() + +path = os.path.join(args["MODEL_PATH"], "model.pt") + +try: + model.load_state_dict(torch.load(path)) + print(Colors.GREEN, "Model Found\n Model Successfully Loaded", Colors.RESET) + +except: + print(Colors.RED, "Model Not Found!!!", Colors.RESET) + +finally: + mcts = Alpha_MCTS(game, args, model) + + state = game.initialise_state() + player = -1 + + while True: + print(state) + + if player == 1: + valid_moves = game.get_valid_moves(state) + print("valid_moves", [i for i in range(game.possible_state) if valid_moves[i] == 1]) + action = int(input(f"{player}:")) + + if valid_moves[action] == 0: + print("action not valid") + continue + + else: + neutral_state = game.change_perspective(state, player) + mcts_probs = mcts.search(neutral_state) + print(Colors.GREEN, "MCTS Move Probabilities:", Colors.RESET,mcts_probs ) + action = np.argmax(mcts_probs) + + state = game.make_move(state, action, player) + + is_terminal, value = game.know_terminal_value(state, action) + + if is_terminal: + print(state) + if value == 1: + print(player, "won") + else: + print("draw") + break + + player = game.get_opponent(player) diff --git a/Train.py b/Train.py new file mode 100644 index 0000000000000000000000000000000000000000..e961431b38e0dc9a2f66288246118cf159b006d6 --- /dev/null +++ b/Train.py @@ -0,0 +1,50 @@ +import os +import torch + +from Games.TicTacToe.TicTacToe import TicTacToe +from Games.TicTacToe.TicTacToeNN import ResNet +from Alpha_Zero_Parallel import Alpha_Zero + +GAME = "TicTacToe" + +args = { + "MODEL_PATH" : os.path.join(os.getcwd(), "Games", GAME, "models_n_optimizers"), + "SAVE_GAME_PATH" : os.path.join(os.getcwd(), "Games", GAME, "games"), + + "EXPLORATION_CONSTANT" : 2, + + "TEMPERATURE" : 2, + + "DIRICHLET_EPSILON" : 0.25, + "DIRICHLET_ALPHA" : 0.3, + "ROOT_RANDOMNESS": True, + + "ADVERSARIAL" : True, + + "NO_OF_SEARCHES" : 800, + "NO_ITERATIONS" : 1, + "SELF_PLAY_ITERATIONS" : 3000, + "PARALLEL_PROCESS" : 500, + "EPOCHS" : 1, + "BATCH_SIZE" : 128, + "MODEL_CHECK_GAMES" : 200, + "WIN_RATIO_FOR_SAVING": 0.5, +} + + +game = TicTacToe() +torch.backends.cudnn.enabled = False + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +print(device, "in use") + +model = ResNet(game, 9, 128, device) + +optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay = 0.0001) + +state = game.initialise_state() + +alpha_zero = Alpha_Zero(game, args, model, optimizer) + +alpha_zero.learn() diff --git a/__pycache__/Alpha_MCTS.cpython-310.pyc b/__pycache__/Alpha_MCTS.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d3098d9b7d7dbc58911b3245dc13038181493172 Binary files /dev/null and b/__pycache__/Alpha_MCTS.cpython-310.pyc differ diff --git a/__pycache__/Alpha_MCTS.cpython-311.pyc b/__pycache__/Alpha_MCTS.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c4e07da90865c0adb6e2eaef8c430b9d4b48880 Binary files /dev/null and b/__pycache__/Alpha_MCTS.cpython-311.pyc differ diff --git a/__pycache__/Alpha_MCTS.cpython-313.pyc b/__pycache__/Alpha_MCTS.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5c10ef9bb206d3f940cbc46e1ed56f7c9cf49c6 Binary files /dev/null and b/__pycache__/Alpha_MCTS.cpython-313.pyc differ diff --git a/__pycache__/Alpha_MCTS_Parallel.cpython-310.pyc b/__pycache__/Alpha_MCTS_Parallel.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd4559dbe02942cf96f0d587f04629ae3a58b89d Binary files /dev/null and b/__pycache__/Alpha_MCTS_Parallel.cpython-310.pyc differ diff --git a/__pycache__/Alpha_MCTS_Parallel.cpython-311.pyc b/__pycache__/Alpha_MCTS_Parallel.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f4455b5857ad15fd26614b987ad370744c68711e Binary files /dev/null and b/__pycache__/Alpha_MCTS_Parallel.cpython-311.pyc differ diff --git a/__pycache__/Alpha_MCTS_Parallel.cpython-313.pyc b/__pycache__/Alpha_MCTS_Parallel.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2abda6c57bd68bc8be29295dabbcee1f07d99388 Binary files /dev/null and b/__pycache__/Alpha_MCTS_Parallel.cpython-313.pyc differ diff --git a/__pycache__/Alpha_Zero.cpython-310.pyc b/__pycache__/Alpha_Zero.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..186ff9d4ba5b83c0bd82ab935ae9f7e10d770a89 Binary files /dev/null and b/__pycache__/Alpha_Zero.cpython-310.pyc differ diff --git a/__pycache__/Alpha_Zero.cpython-311.pyc b/__pycache__/Alpha_Zero.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..af569ff819f580fae5c53dac5c75705a0ee2403b Binary files /dev/null and b/__pycache__/Alpha_Zero.cpython-311.pyc differ diff --git a/__pycache__/Alpha_Zero.cpython-313.pyc b/__pycache__/Alpha_Zero.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2cbbb868001b0682c4ccc4adc1d912da201f01f Binary files /dev/null and b/__pycache__/Alpha_Zero.cpython-313.pyc differ diff --git a/__pycache__/Alpha_Zero_Parallel.cpython-310.pyc b/__pycache__/Alpha_Zero_Parallel.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c6877030323f48cb0ee8528530ee14db50a61120 Binary files /dev/null and b/__pycache__/Alpha_Zero_Parallel.cpython-310.pyc differ diff --git a/__pycache__/Alpha_Zero_Parallel.cpython-311.pyc b/__pycache__/Alpha_Zero_Parallel.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a6f4744a0255907bf30794cbc1338e1aba2728f9 Binary files /dev/null and b/__pycache__/Alpha_Zero_Parallel.cpython-311.pyc differ diff --git a/__pycache__/Alpha_Zero_Parallel.cpython-313.pyc b/__pycache__/Alpha_Zero_Parallel.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59d67d02bc0a04572d90b0c18257367c3fa118bf Binary files /dev/null and b/__pycache__/Alpha_Zero_Parallel.cpython-313.pyc differ diff --git a/__pycache__/Arena.cpython-313.pyc b/__pycache__/Arena.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77dfac7089177171151b61e5255526798f8382ef Binary files /dev/null and b/__pycache__/Arena.cpython-313.pyc differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b3888ba335aab58b0f9fdeb8e6a236183c2dffd5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,64 @@ +altair==6.0.0 +attrs==25.4.0 +blinker==1.9.0 +cachetools==7.0.5 +certifi==2026.2.25 +charset-normalizer==3.4.5 +click==8.3.1 +cuda-bindings==12.9.4 +cuda-pathfinder==1.4.2 +filelock==3.25.2 +fsspec==2026.2.0 +gitdb==4.0.12 +GitPython==3.1.46 +idna==3.11 +Jinja2==3.1.6 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +MarkupSafe==3.0.3 +mpmath==1.3.0 +narwhals==2.18.0 +networkx==3.6.1 +numpy==2.4.3 +nvidia-cublas-cu12==12.8.4.1 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cudnn-cu12==9.10.2.21 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cufile-cu12==1.13.1.3 +nvidia-curand-cu12==10.3.9.90 +nvidia-cusolver-cu12==11.7.3.90 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cusparselt-cu12==0.7.1 +nvidia-nccl-cu12==2.27.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvtx-cu12==12.8.90 +packaging==26.0 +pandas==2.3.3 +pillow==12.1.1 +protobuf==6.33.5 +pyarrow==23.0.1 +pydeck==0.9.1 +python-dateutil==2.9.0.post0 +pytz==2026.1.post1 +referencing==0.37.0 +requests==2.32.5 +rpds-py==0.30.0 +setuptools==82.0.1 +six==1.17.0 +smmap==5.0.3 +streamlit==1.55.0 +sympy==1.14.0 +tenacity==9.1.4 +toml==0.10.2 +torch==2.10.0 +torchvision==0.25.0 +tornado==6.5.5 +tqdm==4.67.3 +triton==3.6.0 +typing_extensions==4.15.0 +tzdata==2025.3 +urllib3==2.6.3 +watchdog==6.0.0 diff --git a/save_games.py b/save_games.py new file mode 100644 index 0000000000000000000000000000000000000000..cbba92c9ff6b92e81f089273e95fc223a1ff7886 --- /dev/null +++ b/save_games.py @@ -0,0 +1,83 @@ +import os +import shelve + +import torch + +from tqdm import trange +from Alpha_Zero_Parallel import Alpha_Zero +from Games.ConnectFour.ConnectFour import ConnectFour +from Games.ConnectFour.ConnectFourNN import ResNet + + +class Colors: + RESET = "\033[0m" + RED = "\033[91m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + BLUE = "\033[94m" + MAGENTA = "\033[95m" + CYAN = "\033[96m" + WHITE = "\033[97m" + +def save_games(args, game, model, optimizer): + try: + model_path = os.path.join(args["MODEL_PATH"], 'model.pt') + optimizer_path = os.path.join(args["MODEL_PATH"], 'optimizer.pt') + + model.load_state_dict(torch.load(model_path)) + optimizer.load_state_dict(torch.load(optimizer_path)) + except: + print(Colors.RED + "UNABLE TO LOAD MODEL") + print(Colors.GREEN + "SETTING UP NEW MODEL..." + Colors.RESET) + + else: + print(Colors.GREEN + "MODEL FOUND\nLOADING MODEL..." + Colors.RESET) + finally: + for iteration in range(args["NO_ITERATIONS"]): + memory = [] + + print(Colors.BLUE + "\nIteration no: " , iteration + 1, Colors.RESET) + print(Colors.YELLOW + "Self Play" + Colors.RESET) + model.eval() + alpha_zero = Alpha_Zero(game, args, model, optimizer) + for _ in trange(args["SELF_PLAY_ITERATIONS"] // args["PARALLEL_PROCESS"]): + memory = alpha_zero.self_play() + + with shelve.open( os.path.join(args["SAVE_GAME_PATH"],"games_5.pkl"), writeback=True) as db: + if "data" in db: + existing_data = db["data"] + existing_data.extend(memory) + else: + db["data"] = memory + +GAME = "ConnectFour" +args = { + "MODEL_PATH" : os.path.join(os.getcwd(), "Games", GAME, "models_n_optimizers"), + "SAVE_GAME_PATH" : os.path.join(os.getcwd(), "Games", GAME, "games"), + + "EXPLORATION_CONSTANT" : 2.25, + + "TEMPERATURE" : 1.75, + + "DIRICHLET_EPSILON" : 0.25, + "DIRICHLET_ALPHA" : 0.3, + "ROOT_RANDOMNESS": True, + + "ADVERSARIAL" : True, + + "NO_OF_SEARCHES" : 12000, + "NO_ITERATIONS" : 100, + "SELF_PLAY_ITERATIONS" : 100, + "PARALLEL_PROCESS" : 50, +} + +game = ConnectFour() +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print(device, "in use") + +model = ResNet(game, 9, 128, device) +model.eval() + +optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay = 0.0001) + +save_games(args, game, model, optimizer) diff --git a/train_from_saved_games.py b/train_from_saved_games.py new file mode 100644 index 0000000000000000000000000000000000000000..e597e57fa61c1ad093da50019bce4ed873147c15 --- /dev/null +++ b/train_from_saved_games.py @@ -0,0 +1,102 @@ +import os +import random +import shelve +import numpy as np + +import torch +import torch.nn.functional as F + +from Games.ConnectFour.ConnectFour import ConnectFour +from Games.ConnectFour.ConnectFourNN import ResNet +from tqdm import trange + +class Colors: + RESET = "\033[0m" + RED = "\033[91m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + BLUE = "\033[94m" + MAGENTA = "\033[95m" + CYAN = "\033[96m" + WHITE = "\033[97m" + + + + +def train(args, model, optimizer, memory): + + random.shuffle(memory) + + for batch_start in range(0, len(memory), args["BATCH_SIZE"]): + batch_end = batch_start + args["BATCH_SIZE"] + + training_memory = memory[batch_start : batch_end] + + state, action_prob, value = zip(*training_memory) + + state, action_prob, value = np.array(state), np.array(action_prob), np.array(value).reshape(-1, 1) + + state = torch.tensor(state, device = model.device, dtype=torch.float32) + policy_targets = torch.tensor(action_prob, device = model.device, dtype=torch.float32) + value_targets = torch.tensor(value, device = model.device, dtype=torch.float32) + + out_policy, out_value = model(state) + + policy_loss = F.cross_entropy(out_policy, policy_targets) + value_loss = F.mse_loss(out_value, value_targets) + loss = policy_loss + value_loss + + optimizer.zero_grad() + loss.backward() + optimizer.step() + +def train_from_saved_data(args, model, optimizer, memory): + try: + model_path = os.path.join(args["MODEL_PATH"], 'model_.pt') + optimizer_path = os.path.join(args["MODEL_PATH"], 'optimizer_.pt') + + model.load_state_dict(torch.load(model_path)) + optimizer.load_state_dict(torch.load(optimizer_path)) + except: + print(Colors.RED + "UNABLE TO LOAD MODEL") + print(Colors.GREEN + "SETTING UP NEW MODEL..." + Colors.RESET) + + else: + print(Colors.GREEN + "MODEL FOUND\nLOADING MODEL..." + Colors.RESET) + finally: + print(Colors.YELLOW + "Training..." + Colors.RESET) + model.train() + for _ in trange(args["EPOCHS"]): + train(args, model, optimizer, memory) + + print(Colors.YELLOW + "Saving Model...") + torch.save(model.state_dict(), os.path.join(args["MODEL_PATH"], "model_.pt")) + torch.save(optimizer.state_dict(), os.path.join(args["MODEL_PATH"], "optimizer_.pt")) + print("Saved!" + Colors.RESET) + + +GAME = "ConnectFour" + +args = { + "MODEL_PATH" : os.path.join(os.getcwd(), "Games", GAME, "models_n_optimizers"), + "EPOCHS" : 4, + "BATCH_SIZE" : 124, +} + + +game = ConnectFour() +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print(device, "in use") + +SAVE_GAME_PATH = os.path.join(os.getcwd(), "Games", GAME, "games", "games_6.pkl") + +with shelve.open(SAVE_GAME_PATH) as db: + if "data" in db: + memory = db["data"] + +model = ResNet(game, 9, 128, device) +model.train() + +optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay = 0.0001) + +train_from_saved_data(args, model, optimizer, memory)