Upload 52 files

Alpha_MCTS.py

@@ -0,0 +1,120 @@
+import math
+import torch
+import numpy as np
+
+class Node:
+
+    def __init__(self, game, args, state, parent = None, action = None, prob = 0, visits = 0):
+        self.game = game
+        self.args = args
+        self.state = state
+        self.parent = parent
+        self.action = action
+        self.prob = prob
+
+        self.children = []
+        self.visits = visits
+        self.value = 0
+
+    def leaf_or_not(self):
+        return len(self.children) > 0
+
+    def search(self):
+        best_child = None
+        best_ucb = -np.inf
+        for child in self.children:
+            ucb = self.get_ucb(child)
+            if best_ucb < ucb:
+                best_ucb = ucb
+                best_child = child
+        return best_child
+
+    def get_ucb(self, child):
+        if child.visits == 0:
+            q_value = 0
+        else:
+            q_value = 1 - (((child.value / child.visits) + 1) / 2)
+
+        return q_value + self.args['EXPLORATION_CONSTANT'] * (math.sqrt(self.visits) / (child.visits + 1)) * child.prob
+
+    def expand(self, policy):
+
+        for move, prob in enumerate(policy):
+            if prob > 0:
+                child = self.state.copy()
+                child = self.game.make_move(child, move, 1)
+                if self.args["ADVERSARIAL"]:
+                    child = self.game.change_perspective(child, player = -1)
+
+                child = Node(self.game, self.args, child, self, move, prob)
+                self.children.append(child)
+
+    def backpropagate(self,state_value):
+        self.value += state_value
+        self.visits += 1
+        if self.args["ADVERSARIAL"]:
+            state_value = self.game.get_opponent_value(state_value)
+        if self.parent is not None:
+            self.parent.backpropagate(state_value)
+
+
+class Alpha_MCTS:
+
+    def __init__(self, game, args, model):
+        self.game = game
+        self.args = args
+        self.model = model
+
+    @torch.no_grad()
+    def search(self, state):
+        root = Node(self.game, self.args, state, visits = 1)
+
+        if self.args["ROOT_RANDOMNESS"]:
+            policy, _ = self.model(
+                        torch.tensor(self.game.get_encoded_state(state), device = self.model.device
+                    ).unsqueeze(0))
+
+            policy = torch.softmax(policy, axis = 1).squeeze(0).cpu().numpy()
+
+            policy = (1 - self.args["DIRICHLET_EPSILON"]) * policy + self.args["DIRICHLET_EPSILON"] * np.random.dirichlet([self.args["DIRICHLET_ALPHA"]] * self.game.possible_state)
+
+            valid_state = self.game.get_valid_moves(state)
+            policy *= valid_state
+            policy /= np.sum(policy)
+            root.expand(policy)
+
+        for _ in range(self.args["NO_OF_SEARCHES"]):
+            node = root
+            no_moves = 0
+            while node.leaf_or_not():
+                node = node.search()
+                no_moves += 1
+
+            is_terminal, value = self.game.know_terminal_value(node.state, node.action)
+
+            if self.args["ADVERSARIAL"]:
+                value = self.game.get_opponent_value(value)
+
+            if not is_terminal:
+
+                policy, value = self.model(
+                    torch.tensor(self.game.get_encoded_state(node.state), device = self.model.device).unsqueeze(0)
+                )
+
+                valid_state = self.game.get_valid_moves(node.state)
+                policy = torch.softmax(policy, axis = 1).squeeze(0).cpu().numpy().astype(np.float64)
+
+                policy *= valid_state
+                policy /= np.sum(policy)
+
+                value = value.item()
+
+                node.expand(policy)
+
+            node.backpropagate(value)
+
+        move_probability = np.zeros(self.game.possible_state)
+        for children in root.children:
+            move_probability[children.action] = children.visits
+        move_probability /= np.sum(move_probability)
+        return move_probability

Alpha_MCTS_Parallel.py

@@ -0,0 +1,128 @@
+import math
+import torch
+import numpy as np
+
+class Node:
+
+    def __init__(self, game, args, state, parent = None, action = None, prob = 0, visits = 0):
+        self.game = game
+        self.args = args
+        self.state = state
+        self.parent = parent
+        self.action = action
+        self.prob = prob
+
+        self.children = []
+        self.visits = visits
+        self.value = 0
+
+    def leaf_or_not(self):
+        return len(self.children) > 0
+
+    def search(self):
+        best_child = None
+        best_ucb = -np.inf
+        for child in self.children:
+            ucb = self.get_ucb(child)
+            if best_ucb < ucb:
+                best_ucb = ucb
+                best_child = child
+        return best_child
+
+    def get_ucb(self, child):
+        if child.visits == 0:
+            q_value = 0
+        else:
+            q_value = 1 - ((child.value / child.visits) + 1) / 2
+
+        return q_value + self.args['EXPLORATION_CONSTANT'] * (math.sqrt(self.visits) / (child.visits + 1)) * child.prob
+
+    def expand(self, policy):
+
+        for move, prob in enumerate(policy):
+            if prob > 0:
+                child = self.state.copy()
+                child = self.game.make_move(child, move, 1)
+                if self.args["ADVERSARIAL"]:
+                    child = self.game.change_perspective(child, player = -1)
+
+                child = Node(self.game, self.args, child, self, move, prob)
+                self.children.append(child)
+
+    def backpropagate(self,state_value):
+        self.value += state_value
+        self.visits += 1
+        if self.args["ADVERSARIAL"]:
+            state_value = self.game.get_opponent_value(state_value)
+        if self.parent is not None:
+            self.parent.backpropagate(state_value)
+
+
+class Alpha_MCTS:
+
+    def __init__(self, game, args, model):
+        self.game = game
+        self.args = args
+        self.model = model
+
+    @torch.no_grad()
+    def search(self, states, spGames):
+
+        policy, _ = self.model(
+                        torch.tensor(self.game.get_encoded_state(states), device = self.model.device
+                    ))
+
+        policy = torch.softmax(policy, axis = 1).cpu().numpy()
+
+        if self.args["ROOT_RANDOMNESS"]:
+            policy = (1 - self.args["DIRICHLET_EPSILON"]) * policy + self.args["DIRICHLET_EPSILON"] * np.random.dirichlet([self.args["DIRICHLET_ALPHA"]] * self.game.possible_state, size = policy.shape[0])
+
+        for i, spg in enumerate(spGames):
+            spg_policy = policy[i]
+            valid_state = self.game.get_valid_moves(states[i])
+            spg_policy *= valid_state
+            spg_policy /= np.sum(spg_policy)
+
+            spg.root = Node(self.game, self.args, states[i], visits = 1)
+            spg.root.expand(spg_policy)
+
+        for _ in range(self.args["NO_OF_SEARCHES"]):
+
+            for spg in spGames:
+                spg.node = None
+                node = spg.root
+
+                while node.leaf_or_not():
+                    node = node.search()
+
+                is_terminal, value = self.game.know_terminal_value(node.state, node.action)
+
+                if self.args["ADVERSARIAL"]:
+                    value = self.game.get_opponent_value(value)
+
+                if is_terminal:
+                    node.backpropagate(value)
+                else:
+                    spg.node = node
+
+            expandabel_spgs = [mapping_index for mapping_index in range(len(spGames)) if spGames[mapping_index].node is not None]
+
+            if len(expandabel_spgs) > 0:
+                states = np.stack([spGames[mapping_index].node.state for mapping_index in expandabel_spgs])
+                policy, value = self.model(
+                    torch.tensor(self.game.get_encoded_state(states), device = self.model.device)
+                )
+
+                policy = torch.softmax(policy, axis = 1).cpu().numpy().astype(np.float64)
+                value = value.cpu().numpy()
+
+            for i, mapping_index in enumerate(expandabel_spgs):
+                node = spGames[mapping_index].node
+                spg_policy, spg_value = policy[i], value[i]
+                valid_state = self.game.get_valid_moves(node.state)
+
+                spg_policy *= valid_state
+                spg_policy /= np.sum(spg_policy)
+
+                node.expand(spg_policy)
+                node.backpropagate(spg_value)

Alpha_Zero.py

@@ -0,0 +1,143 @@
+import os
+import random
+import copy
+import numpy as np
+
+import torch
+import torch.nn.functional as F
+
+from tqdm import trange
+from Alpha_MCTS import Alpha_MCTS
+from Arena import Arena
+
+class Colors:
+    RESET = "\033[0m"
+    RED = "\033[91m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    BLUE = "\033[94m"
+    MAGENTA = "\033[95m"
+    CYAN = "\033[96m"
+    WHITE = "\033[97m"
+
+
+class Alpha_Zero:
+
+    def __init__(self, game, args, model, optimizer):
+        self.game = game
+        self.args = args
+        self.model = model
+        self.optimizer = optimizer
+        self.mcts = Alpha_MCTS(game, args, model)
+
+    def self_play(self):
+
+        single_game_memory = []
+        player = 1
+        state = self.game.initialise_state()
+
+        while True:
+            neutral_state = self.game.change_perspective(state, player) if self.args["ADVERSARIAL"] else state
+            prob = self.mcts.search(neutral_state)
+
+            single_game_memory.append((neutral_state, prob, player))
+
+            temp_prob = prob ** (1 / self.args["TEMPERATURE"])
+
+            temp_prob[temp_prob == 0] = - np.inf
+
+            temp_prob = torch.softmax(torch.tensor(temp_prob), axis = 0).cpu().numpy()
+
+            move = np.random.choice(self.game.possible_state, p = temp_prob)
+
+            state = self.game.make_move(state, move, player)
+            is_terminal, value = self.game.know_terminal_value(state, move)
+
+            if is_terminal:
+                return_memory = []
+                for return_state, return_action_prob, return_player in single_game_memory:
+                    if self.args["ADVERSARIAL"]:
+                        return_value = value if return_player == player else self.game.get_opponent_value(value)
+                    else:
+                        return_value = value
+
+                    return_memory.append((
+                        self.game.get_encoded_state(return_state),
+                        return_action_prob,
+                        return_value
+                    ))
+                return return_memory
+
+            if self.args["ADVERSARIAL"]:
+                player = self.game.get_opponent(player)
+
+    def train(self, memory):
+
+        random.shuffle(memory)
+
+        for batch_start in range(0, len(memory), self.args["BATCH_SIZE"]):
+            batch_end = batch_start + self.args["BATCH_SIZE"]
+
+            training_memory = memory[batch_start : batch_end]
+
+            state, action_prob, value = zip(*training_memory)
+
+            state, action_prob, value =  np.array(state), np.array(action_prob), np.array(value).reshape(-1, 1)
+
+            state = torch.tensor(state, device = self.model.device, dtype=torch.float32)
+            policy_targets = torch.tensor(action_prob, device = self.model.device, dtype=torch.float32)
+            value_targets = torch.tensor(value, device = self.model.device, dtype=torch.float32)
+
+            out_policy, out_value = self.model(state)
+
+            policy_loss = F.cross_entropy(out_policy, policy_targets)
+            value_loss = F.mse_loss(out_value, value_targets)
+            loss = policy_loss + value_loss
+
+            self.optimizer.zero_grad()
+            loss.backward()
+            self.optimizer.step()
+
+    def learn(self):
+        try:
+            model_path = os.path.join(self.args["MODEL_PATH"], 'model.pt')
+            optimizer_path = os.path.join(self.args["MODEL_PATH"], 'optimizer.pt')
+
+            self.model.load_state_dict(torch.load(model_path))
+            self.optimizer.load_state_dict(torch.load(optimizer_path))
+        except:
+            print(Colors.RED + "UNABLE TO LOAD MODEL")
+            print(Colors.GREEN + "SETTING UP NEW MODEL..." + Colors.RESET)
+
+        else:
+            print(Colors.GREEN + "MODEL FOUND\nLOADING MODEL..." + Colors.RESET)
+        finally:
+
+            initial_model = copy.copy(self.model)
+
+            for iteration in range(self.args["NO_ITERATIONS"]):
+                memory = []
+
+                print(Colors.BLUE + "\nIteration no: " , iteration + 1, Colors.RESET)
+
+                print(Colors.YELLOW + "Self Play" + Colors.RESET)
+                self.model.eval()
+
+                for _ in trange(self.args["SELF_PLAY_ITERATIONS"]):
+                    memory += self.self_play()
+
+                print(Colors.YELLOW + "Training..." + Colors.RESET)
+                self.model.train()
+                for _ in trange(self.args["EPOCHS"]):
+                    self.train(memory)
+
+                print(Colors.YELLOW + "Testing..." + Colors.RESET)
+                self.model.eval()
+                wins, draws, defeats = Arena(self.game, self.args, self.model, initial_model)
+                print(Colors.GREEN + "Testing Completed" + Colors.WHITE + "\nTrained Model Stats:")
+                print(Colors.GREEN, "Wins: ", wins, Colors.RESET, "|", Colors.RED, "Loss: ", defeats, Colors.RESET, "|", Colors.WHITE," Draw: ", draws, Colors.RESET)
+
+            print(Colors.YELLOW + "Saving Model...")
+            torch.save(self.model.state_dict(), os.path.join(self.args["MODEL_PATH"], "model_non_parallel.pt"))
+            torch.save(self.optimizer.state_dict(), os.path.join(self.args["MODEL_PATH"], "optimizer_non_parallel.pt"))
+            print("Saved!" + Colors.RESET)

Games/2048/2048.py

@@ -0,0 +1,41 @@
+"""NOTE : THIS IS THE FORMAT YOU SHOULD FOLLOW TO MAKE GAMES"""
+
+class GAME:
+    def __init__(self):
+        self.row = 4
+        self.col = 4
+        self.possible_state = self.row * self.col # no of possible state game can have (no of possible moves)
+
+    def initialise_state(self):
+        """NOTE: this fuction creates starting position for your game"""
+
+    def get_valid_moves(self, state):
+        """ NOTE: inputs state and output all possible valid moves"""
+
+    def know_terminal_value(self, state, action):
+        """NOTE: input state and action and the output will be tuple True 
+        if is terminal state and False if not and value 0 is
+        player lost and 1 if the player won the game"""
+
+    def make_move(self, state, action, player):
+        """NOTE: input a action and a state and a player, the 
+        action is taken on given state and output is the final state"""
+
+    def get_encoded_state(self, state):
+        """input a state the output should be the encoded state that will be
+        given to the neural network"""
+
+    def get_opponent(self, player):
+        """NOTE: taking the input of current player this should change the 
+        player to the opponent"""
+        return -1 * player
+
+    def get_opponent_value(self, value):
+        """this should switch the reward obtained from the know_terminal_state() 
+        to the reward of the opponent"""
+        return -1 * value
+
+    def change_perspective(self, state, player = -1):
+        """this function taking input state and player flip the board and return 
+        the game with the perspective of the opponent"""
+        return player * state

Games/Chess/Chess.py

@@ -0,0 +1,41 @@
+"""NOTE : THIS IS THE FORMAT YOU SHOULD FOLLOW TO MAKE GAMES"""
+
+class GAME:
+    def __init__(self):
+        self.row = 8
+        self.col = 8
+        self.possible_state = self.row * self.col
+
+    def initialise_state(self):
+        """NOTE: this fuction creates starting position for your game"""
+
+    def get_valid_moves(self, state):
+        """ NOTE: inputs state and output all possible valid moves"""
+
+    def know_terminal_value(self, state, action):
+        """NOTE: input state and action and the output will be tuple True 
+        if is terminal state and False if not and value 0 is
+        player lost and 1 if the player won the game"""
+
+    def make_move(self, state, action, player):
+        """NOTE: input a action and a state and a player, the 
+        action is taken on given state and output is the final state"""
+
+    def get_encoded_state(self, state):
+        """input a state the output should be the encoded state that will be
+        given to the neural network"""
+
+    def get_opponent(self, player):
+        """NOTE: taking the input of current player this should change the 
+        player to the opponent"""
+        return -1 * player
+
+    def get_opponent_value(self, value):
+        """this should switch the reward obtained from the know_terminal_state() 
+        to the reward of the opponent"""
+        return -1 * value
+
+    def change_perspective(self, state, player = -1):
+        """this function taking input state and player flip the board and return 
+        the game with the perspective of the opponent"""
+        return player * state

Games/Chess/ChessNN.py

@@ -0,0 +1,63 @@
+import torch.nn as nn
+import torch.nn.functional as F
+
+class ResNet(nn.Module):
+    def __init__(self, game, num_resBlocks, num_hidden, device):
+        super().__init__()
+        self.device = device
+        self.startBlock = nn.Sequential(
+            nn.Conv2d(3, num_hidden, kernel_size=3, padding=1),
+            nn.BatchNorm2d(num_hidden),
+            nn.ReLU(),
+            nn.Conv2d(3, num_hidden, kernel_size=3, padding=1),
+            nn.BatchNorm2d(num_hidden),
+            nn.ReLU()
+        )
+
+        self.backBone = nn.ModuleList(
+            [ResBlock(num_hidden) for i in range(num_resBlocks)]
+        )
+
+        self.policyHead = nn.Sequential(
+            nn.Conv2d(num_hidden, 128, kernel_size=3, padding=1),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.Flatten(),
+            nn.Linear(128 * game.row * game.col, game.possible_state)
+        )
+
+        self.valueHead = nn.Sequential(
+            nn.Conv2d(num_hidden, 3, kernel_size=3, padding=1),
+            nn.BatchNorm2d(3),
+            nn.ReLU(),
+            nn.Flatten(),
+            nn.Linear(3 * game.row * game.col, 1),
+            nn.Tanh()
+        )
+
+        self.to(device)
+
+    def forward(self, x):
+        x = self.startBlock(x)
+        for resBlock in self.backBone:
+            x = resBlock(x)
+        policy = self.policyHead(x)
+        value = self.valueHead(x)
+        return policy, value
+
+
+class ResBlock(nn.Module):
+    def __init__(self, num_hidden):
+        super().__init__()
+        self.conv1 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1)
+        self.bn1 = nn.BatchNorm2d(num_hidden)
+        self.conv2 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1)
+        self.bn2 = nn.BatchNorm2d(num_hidden)
+
+    def forward(self, x):
+        residual = x
+        x = F.relu(self.bn1(self.conv1(x)))
+        x = self.bn2(self.conv2(x))
+        x += residual
+        x = F.relu(x)
+        return x

Games/Chess/Stokfish.py

@@ -0,0 +1 @@
+import Stokfish

Games/ConnectFour/ConnectFour.py

@@ -0,0 +1,76 @@
+import numpy as np
+
+class ConnectFour:
+    def __init__(self):
+        self.row = 6
+        self.col = 7
+        self.possible_state = self.col
+        self.in_a_row = 4
+        
+    def __repr__(self):
+        return "ConnectFour"
+        
+    def initialise_state(self):
+        return np.zeros((self.row, self.col))
+    
+    def make_move(self, state, action, player):
+        row = np.max(np.where(state[:, action] == 0))
+        state[row, action] = player
+        return state
+    
+    def get_valid_moves(self, state):
+        return (state[0] == 0).astype(np.uint8)
+    
+    def check_win(self, state, action):
+        if action == None:
+            return False
+        
+        row = np.min(np.where(state[:, action] != 0))
+        column = action
+        player = state[row][column]
+
+        def count(offset_row, offset_column):
+            for i in range(1, self.in_a_row):
+                r = row + offset_row * i
+                c = action + offset_column * i
+                if (
+                    r < 0 
+                    or r >= self.row
+                    or c < 0 
+                    or c >= self.col
+                    or state[r][c] != player
+                ):
+                    return i - 1
+            return self.in_a_row - 1
+
+        return (
+            count(1, 0) >= self.in_a_row - 1 # vertical
+            or (count(0, 1) + count(0, -1)) >= self.in_a_row - 1 # horizontal
+            or (count(1, 1) + count(-1, -1)) >= self.in_a_row - 1 # top left diagonal
+            or (count(1, -1) + count(-1, 1)) >= self.in_a_row - 1 # top right diagonal
+        )
+    
+    def know_terminal_value(self, state, action):
+        if self.check_win(state, action):
+            return True, 1
+        if np.sum(self.get_valid_moves(state)) == 0:
+            return True, 0
+        return False, 0
+    
+    def get_opponent(self, player):
+        return -player
+    
+    def get_opponent_value(self, value):
+        return -value
+    
+    def change_perspective(self, state, player):
+        return state * player
+    
+    def get_encoded_state(self, state):
+        encoded_state = np.stack(
+            (state == -1, state == 0, state == 1)
+        ).astype(np.float32)
+        if len(state.shape) == 3:
+            encoded_state = np.swapaxes(encoded_state, 0, 1)
+        
+        return encoded_state

Games/ConnectFour/ConnectFourNN.py

@@ -0,0 +1,61 @@
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class ResNet(nn.Module):
+    def __init__(self, game, num_resBlocks, num_hidden, device):
+        super().__init__()
+        self.device = device
+        self.startBlock = nn.Sequential(
+            nn.Conv2d(3, num_hidden, kernel_size=3, padding=1),
+            nn.BatchNorm2d(num_hidden),
+            nn.ReLU()
+        )
+        
+        self.backBone = nn.ModuleList(
+            [ResBlock(num_hidden) for i in range(num_resBlocks)]
+        )
+        
+        self.policyHead = nn.Sequential(
+            nn.Conv2d(num_hidden, 32, kernel_size=3, padding=1),
+            nn.BatchNorm2d(32),
+            nn.ReLU(),
+            nn.Flatten(),
+            nn.Linear(32 * game.row * game.col, game.possible_state)
+        )
+        
+        self.valueHead = nn.Sequential(
+            nn.Conv2d(num_hidden, 3, kernel_size=3, padding=1),
+            nn.BatchNorm2d(3),
+            nn.ReLU(),
+            nn.Flatten(),
+            nn.Linear(3 * game.row * game.col, 1),
+            nn.Tanh()
+        )
+        
+        self.to(device)
+        
+    def forward(self, x):
+        x = self.startBlock(x)
+        for resBlock in self.backBone:
+            x = resBlock(x)
+        policy = self.policyHead(x)
+        value = self.valueHead(x)
+        return policy, value
+        
+        
+class ResBlock(nn.Module):
+    def __init__(self, num_hidden):
+        super().__init__()
+        self.conv1 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1)
+        self.bn1 = nn.BatchNorm2d(num_hidden)
+        self.conv2 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1)
+        self.bn2 = nn.BatchNorm2d(num_hidden)
+        
+    def forward(self, x):
+        residual = x
+        x = F.relu(self.bn1(self.conv1(x)))
+        x = self.bn2(self.conv2(x))
+        x += residual
+        x = F.relu(x)
+        return x

Games/ConnectFour/models_n_optimizers/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ecbcdf83dc322c859ab3188f11e2f7b25ee504515a8bcf2b84798be7b1b3eb7
+size 10920462

Games/ConnectFour/models_n_optimizers/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7165fd808da4ff2c3535392d8cf651ee905b0d0a15aadbef7e2e8300a7387fd
+size 21792762

Games/TicTacToe/TicTacToe.py

@@ -0,0 +1,59 @@
+import numpy as np
+
+class TicTacToe:
+
+    def __init__(self):
+        self.col = 3
+        self.row = 3
+        self.possible_state = self.col * self.row
+
+    def initialise_state(self):
+        return np.zeros((self.row, self.col))
+
+    def get_valid_moves(self, state):
+        return (state.reshape(-1) == 0).astype(np.uint8)
+
+    def know_terminal_value(self, state, action):
+
+        if action is None:
+            return False, 0
+
+        row=action//3
+        col=action%3
+        player = state[row,col]
+
+        if (sum(state[row,:])==player*3
+            or sum(state[:,col])==player*3
+            or np.trace(state)==player*3
+            or np.trace(np.fliplr(state))==player*3):
+            return True, 1
+
+        if 0 not in state.reshape(-1):
+            return True, 0
+        else:
+            return False,0
+
+    def make_move(self, state, action, player):
+        row=action//3
+        col=action%3
+
+        state[row,col]=player
+
+        return state
+
+    def get_encoded_state(self, state):
+        encoded_state = np.stack(
+            (state == -1, state == 0, state == 1)
+        ).astype(np.float32)
+        if len(state.shape) == 3:
+            encoded_state = np.swapaxes(encoded_state, 0, 1)
+
+        return encoded_state
+    def get_opponent(self, player):
+        return -1 * player
+
+    def get_opponent_value(self, value):
+        return -1 * value
+
+    def change_perspective(self, state, player = -1):
+        return (player * state).astype(np.float32)

Games/TicTacToe/TicTacToeNN.py

@@ -0,0 +1,61 @@
+import torch.nn as nn
+import torch.nn.functional as F
+
+class ResNet(nn.Module):
+
+    def __init__(self, game, num_resBlocks, num_hidden, device):
+        super().__init__()
+        self.device = device
+        self.startBlock = nn.Sequential(
+            nn.Conv2d(3, num_hidden, kernel_size=3, padding=1),
+            nn.BatchNorm2d(num_hidden),
+            nn.ReLU()
+        )
+
+        self.backBone = nn.ModuleList(
+            [ResBlock(num_hidden) for i in range(num_resBlocks)]
+        )
+
+        self.policyHead = nn.Sequential(
+            nn.Conv2d(num_hidden, 32, kernel_size=3, padding=1),
+            nn.BatchNorm2d(32),
+            nn.ReLU(),
+            nn.Flatten(),
+            nn.Linear(32 * game.row * game.col, game.possible_state)
+        )
+
+        self.valueHead = nn.Sequential(
+            nn.Conv2d(num_hidden, 3, kernel_size=3, padding=1),
+            nn.BatchNorm2d(3),
+            nn.ReLU(),
+            nn.Flatten(),
+            nn.Linear(3 * game.row * game.col, 1),
+            nn.Tanh()
+        )
+
+        self.to(device)
+
+    def forward(self, x):
+        x = self.startBlock(x)
+        for resBlock in self.backBone:
+            x = resBlock(x)
+        policy = self.policyHead(x)
+        value = self.valueHead(x)
+        return policy, value
+
+
+class ResBlock(nn.Module):
+    def __init__(self, num_hidden):
+        super().__init__()
+        self.conv1 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1)
+        self.bn1 = nn.BatchNorm2d(num_hidden)
+        self.conv2 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1)
+        self.bn2 = nn.BatchNorm2d(num_hidden)
+
+    def forward(self, x):
+        residual = x
+        x = F.relu(self.bn1(self.conv1(x)))
+        x = self.bn2(self.conv2(x))
+        x += residual
+        x = F.relu(x)
+        return x

Games/TicTacToe/models_n_optimizers/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5877c49bfb164d48c9ccf7d165d9e647c14ed6956cb047e6f40b6e7237e9af2d
+size 10900420

Games/TicTacToe/models_n_optimizers/model_non_parallel.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4cefe7c002f47fe17ee7b6304bdd6f32b44191bc00090604923300803062740
+size 10903056

Games/TicTacToe/models_n_optimizers/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38db7c19396de4e717b28891427dcf3ff4efc1451c3a0cc7baff70b3cc5824c5
+size 21738002

Games/TicTacToe/models_n_optimizers/optimizer_non_parallel.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f346133359c24c36ae633ab24f2e4c47773d5e40506c5f2fb64a06f8c54c7fa
+size 21741505

Games/game.py

@@ -0,0 +1,41 @@
+"""NOTE : THIS IS THE FORMAT YOU SHOULD FOLLOW TO MAKE GAMES"""
+
+class GAME:
+    def __init__(self):
+        self.row = None
+        self.col = None
+        self.possible_state = self.row * self.col # no of possible state game can have (no of possible moves)
+
+    def initialise_state(self):
+        """NOTE: this fuction creates starting position for your game"""
+
+    def get_valid_moves(self, state):
+        """ NOTE: inputs state and output all possible valid moves"""
+
+    def know_terminal_value(self, state, action):
+        """NOTE: input state and action and the output will be tuple True 
+        if is terminal state and False if not and value 0 is
+        player lost and 1 if the player won the game"""
+
+    def make_move(self, state, action, player):
+        """NOTE: input a action and a state and a player, the 
+        action is taken on given state and output is the final state"""
+
+    def get_encoded_state(self, state):
+        """input a state the output should be the encoded state that will be
+        given to the neural network"""
+
+    def get_opponent(self, player):
+        """NOTE: taking the input of current player this should change the 
+        player to the opponent"""
+        return -1 * player
+
+    def get_opponent_value(self, value):
+        """this should switch the reward obtained from the know_terminal_state() 
+        to the reward of the opponent"""
+        return -1 * value
+
+    def change_perspective(self, state, player = -1):
+        """this function taking input state and player flip the board and return 
+        the game with the perspective of the opponent"""
+        return player * state

MCTS.py

@@ -0,0 +1,104 @@
+import math
+import numpy as np
+
+class Node:
+
+    def __init__(self, game, args, state, parent = None, action = None):
+        self.game = game
+        self.args = args
+        self.state = state
+        self.parent = parent
+        self.action = action
+
+        self.children = list()
+        self.expandable_moves = self.game.get_moves(self.state)
+        self.visits = 0
+        self.total_value = 0
+
+    def leaf_or_not(self):
+        return (len(self.children) > 0 and len(self.expandable_moves) == 0)
+
+    def search(self):
+        best_child = None
+        best_ucb = -np.inf
+        for child in self.children:
+            ucb = self.get_ucb(child)
+            if best_ucb < ucb:
+                best_ucb = ucb
+                best_child = child
+
+        return best_child
+
+    def get_ucb(self, child):
+        q_value = 1 - ((child.total_value / child.visits) + 1) / 2
+        return q_value + self.args["EXPLORATION_CONSTANT"] * math.sqrt(math.log(self.visits) / child.visits)
+
+    def expand(self):
+        rand_move = np.random.choice(self.expandable_moves)
+        self.expandable_moves.remove(rand_move)
+
+        child = self.game.make_move(self.state.copy(), rand_move, 1)
+        child = self.game.change_perspective(child)
+        child = Node(self.game, self.args, child, self, rand_move)
+
+        self.children.append(child)
+
+        return child
+
+    def simulate(self):
+        is_terminal, value = self.game.know_terminal_value(self.state, self.action)
+        value = self.game.get_opponent_value(value)
+
+        if is_terminal:
+            return value
+
+        state = self.state.copy()
+        player = 1
+        while True:
+            possible_moves = self.game.get_moves(state)
+            rand_move = np.random.choice(possible_moves)
+            state = self.game.make_move(state, rand_move, player)
+            is_terminal, value = self.game.know_terminal_value(state, rand_move)
+            if is_terminal:
+                if player == -1:
+                    value = self.game.get_opponent_value(value)
+                return value
+            player = self.game.get_opponent(player)
+
+    def backpropagate(self,value):
+        self.total_value += value
+        self.visits += 1
+
+        value = self.game.get_opponent_value(value)
+        if self.parent is not None:
+            self.parent.backpropagate(value)
+
+
+class MCTS:
+    def __init__(self, game, args):
+        self.game = game
+        self.args = args
+
+    def search(self, node):
+        root = Node(self.game, self.args, node)
+
+        for _ in range(self.args["NO_OF_SEARCHES"]):
+            node = root
+            while node.leaf_or_not():
+                node = node.search()
+
+            is_terminal, value = self.game.know_terminal_value(node.state, node.action)
+            value = self.game.get_opponent_value(value)
+
+            if not is_terminal:
+                node = node.expand()
+                value = node.simulate()
+
+            node.backpropagate(value)
+
+        move_probability = np.zeros(self.game.possible_state)
+        for children in root.children:
+            move_probability[children.action] = children.visits
+        move_probability /= np.sum(move_probability)
+
+        return move_probability

Play.py

@@ -0,0 +1,86 @@
+
+import os
+import numpy as np
+
+import torch
+
+from Games.ConnectFour.ConnectFour import ConnectFour
+from Games.ConnectFour.ConnectFourNN import ResNet
+from Alpha_MCTS import Alpha_MCTS
+
+class Colors:
+    RESET = "\033[0m"
+    RED = "\033[91m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    BLUE = "\033[94m"
+    MAGENTA = "\033[95m"
+    CYAN = "\033[96m"
+    WHITE = "\033[97m"
+
+GAME = "ConnectFour"
+args = {
+    "MODEL_PATH" : os.path.join(os.getcwd(), "Games", GAME, "models_n_optimizers"),
+
+    "ADVERSARIAL" : True,
+    "ROOT_RANDOMNESS": False,
+
+    "TEMPERATURE" : 1,
+
+    "NO_OF_SEARCHES" : 1200,
+    "EXPLORATION_CONSTANT" : 1,    
+}
+
+
+game = ConnectFour()
+device = torch.device("cuda" if torch.cuda.is_available else "cpu")
+
+model = ResNet(game, 9, 128, device)
+model.eval()
+
+path = os.path.join(args["MODEL_PATH"], "model.pt")
+
+try:
+    model.load_state_dict(torch.load(path))
+    print(Colors.GREEN, "Model Found\n Model Successfully Loaded", Colors.RESET)
+
+except:
+    print(Colors.RED, "Model Not Found!!!", Colors.RESET)
+
+finally:
+    mcts = Alpha_MCTS(game, args, model)
+
+    state = game.initialise_state()
+    player = -1
+
+    while True:
+        print(state)
+
+        if player == 1:
+            valid_moves = game.get_valid_moves(state)
+            print("valid_moves", [i for i in range(game.possible_state) if valid_moves[i] == 1])
+            action = int(input(f"{player}:"))
+
+            if valid_moves[action] == 0:
+                print("action not valid")
+                continue
+
+        else:
+            neutral_state = game.change_perspective(state, player)
+            mcts_probs = mcts.search(neutral_state)
+            print(Colors.GREEN, "MCTS Move Probabilities:", Colors.RESET,mcts_probs )
+            action = np.argmax(mcts_probs)
+
+        state = game.make_move(state, action, player)
+
+        is_terminal, value = game.know_terminal_value(state, action)
+
+        if is_terminal:
+            print(state)
+            if value == 1:
+                print(player, "won")
+            else:
+                print("draw")
+            break
+
+        player = game.get_opponent(player)

Train.py

@@ -0,0 +1,50 @@
+import os
+import torch
+
+from Games.TicTacToe.TicTacToe import TicTacToe
+from Games.TicTacToe.TicTacToeNN import ResNet
+from Alpha_Zero_Parallel import Alpha_Zero
+
+GAME = "TicTacToe"
+
+args = {
+    "MODEL_PATH" : os.path.join(os.getcwd(), "Games", GAME, "models_n_optimizers"),
+    "SAVE_GAME_PATH" :  os.path.join(os.getcwd(), "Games", GAME, "games"),
+
+    "EXPLORATION_CONSTANT" : 2,
+
+    "TEMPERATURE" : 2,
+
+    "DIRICHLET_EPSILON" : 0.25,
+    "DIRICHLET_ALPHA" : 0.3,
+    "ROOT_RANDOMNESS": True,
+
+    "ADVERSARIAL" : True,
+
+    "NO_OF_SEARCHES" : 800,
+    "NO_ITERATIONS" : 1,
+    "SELF_PLAY_ITERATIONS" : 3000,
+    "PARALLEL_PROCESS" : 500,
+    "EPOCHS" : 1,
+    "BATCH_SIZE" : 128,
+    "MODEL_CHECK_GAMES" : 200,
+    "WIN_RATIO_FOR_SAVING": 0.5,
+}
+
+
+game = TicTacToe()
+torch.backends.cudnn.enabled = False
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+print(device, "in use")
+
+model = ResNet(game, 9, 128, device)
+
+optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay = 0.0001)
+
+state = game.initialise_state()
+
+alpha_zero = Alpha_Zero(game, args, model, optimizer)
+
+alpha_zero.learn()

requirements.txt

@@ -0,0 +1,64 @@
+altair==6.0.0
+attrs==25.4.0
+blinker==1.9.0
+cachetools==7.0.5
+certifi==2026.2.25
+charset-normalizer==3.4.5
+click==8.3.1
+cuda-bindings==12.9.4
+cuda-pathfinder==1.4.2
+filelock==3.25.2
+fsspec==2026.2.0
+gitdb==4.0.12
+GitPython==3.1.46
+idna==3.11
+Jinja2==3.1.6
+jsonschema==4.26.0
+jsonschema-specifications==2025.9.1
+MarkupSafe==3.0.3
+mpmath==1.3.0
+narwhals==2.18.0
+networkx==3.6.1
+numpy==2.4.3
+nvidia-cublas-cu12==12.8.4.1
+nvidia-cuda-cupti-cu12==12.8.90
+nvidia-cuda-nvrtc-cu12==12.8.93
+nvidia-cuda-runtime-cu12==12.8.90
+nvidia-cudnn-cu12==9.10.2.21
+nvidia-cufft-cu12==11.3.3.83
+nvidia-cufile-cu12==1.13.1.3
+nvidia-curand-cu12==10.3.9.90
+nvidia-cusolver-cu12==11.7.3.90
+nvidia-cusparse-cu12==12.5.8.93
+nvidia-cusparselt-cu12==0.7.1
+nvidia-nccl-cu12==2.27.5
+nvidia-nvjitlink-cu12==12.8.93
+nvidia-nvshmem-cu12==3.4.5
+nvidia-nvtx-cu12==12.8.90
+packaging==26.0
+pandas==2.3.3
+pillow==12.1.1
+protobuf==6.33.5
+pyarrow==23.0.1
+pydeck==0.9.1
+python-dateutil==2.9.0.post0
+pytz==2026.1.post1
+referencing==0.37.0
+requests==2.32.5
+rpds-py==0.30.0
+setuptools==82.0.1
+six==1.17.0
+smmap==5.0.3
+streamlit==1.55.0
+sympy==1.14.0
+tenacity==9.1.4
+toml==0.10.2
+torch==2.10.0
+torchvision==0.25.0
+tornado==6.5.5
+tqdm==4.67.3
+triton==3.6.0
+typing_extensions==4.15.0
+tzdata==2025.3
+urllib3==2.6.3
+watchdog==6.0.0