commit

app.py

@@ -6,7 +6,7 @@ import pandas as pd
 from smolagents import CodeAgent, tool, InferenceClientModel, WebSearchTool, load_tool, PromptTemplates, Tool, FinalAnswerTool
 from dotenv import load_dotenv
 from my_prompt_config import PromptConfig
-from my_tools import ReverseStringTool, ImageLoadTool
+from my_reverse_string import ReverseStringTool, ImageLoadTool
 from PIL import Image
 
 # (Keep Constants as is)

chess_pieces_detection/__init__.py

@@ -1 +0,0 @@
-from train_chess_pieces_recognition import ChessPiecesRecognition

chess_pieces_detection/train.sh

@@ -1,2 +0,0 @@
-. ../venv/bin/activate
-python3 train_chess_pieces_recognition.py

chess_board_tool.py → my_chess_board_tool.py

@@ -5,13 +5,14 @@ import cv2
 import numpy as np
 import math
 import numpy
-from .chess_pieces_detection import ChessPiecesRecognition
-
+from my_train_chess_pieces_recognition import ChessPiecesRecognition
+import traceback
 
+# Based on; https://github.com/kratos606/chessboard-recogniser/tree/main
 class ChessBoard(Tool):
     name = "_my_chess_board"
     description = """
-        Analyze an image representing a chess board and extract board state in FEN notation
+        Analyze an image representing a chess board and return board position as a list of chess pieces
     """
 
     inputs = {
@@ -23,6 +24,8 @@ class ChessBoard(Tool):
 
     output_type = "string"
 
+    is_initialized = False
+
     # Steps to do
     # - break board image into array of images representing pieces
     #      Image -> Image[]
@@ -31,18 +34,22 @@ class ChessBoard(Tool):
     # - construct FEN from string of chess pieces
     #      str[] -> []
 
+    def __init__(self, _chess_board_model_name, _chess_board_model_dir):
+        print(f"***KS*** ChessBoard initializing ...")
+        self.recognition = ChessPiecesRecognition(_chess_board_model_name, _chess_board_model_dir)
+        self.is_initialized = True
 
-    def gradientx(self, img):
+    def __gradientx(self, img):
         # Compute gradient in x-direction using larger Sobel kernel
         grad_x = cv2.Sobel(img, cv2.CV_32F, 1, 0, ksize=31)
         return grad_x
 
-    def gradienty(self, img):
+    def __gradienty(self, img):
         # Compute gradient in y-direction using larger Sobel kernel
         grad_y = cv2.Sobel(img, cv2.CV_32F, 0, 1, ksize=31)
         return grad_y
 
-    def checkMatch(self, lineset):
+    def __checkMatch(self, lineset):
         linediff = np.diff(lineset)
         x = 0
         cnt = 0
@@ -54,7 +61,7 @@ class ChessBoard(Tool):
                 x = line
         return cnt == 5
 
-    def pruneLines(self, lineset, image_dim, margin=20):
+    def __pruneLines(self, lineset, image_dim, margin=20):
         # Remove lines near the margins
         lineset = [x for x in lineset if x > margin and x < image_dim - margin]
         if not lineset:
@@ -75,7 +82,7 @@ class ChessBoard(Tool):
                 start_pos = i
         return lineset
 
-    def skeletonize_1d(self, arr):
+    def __skeletonize_1d(self, arr):
         _arr = arr.copy()
         for i in range(len(_arr) - 1):
             if _arr[i] <= _arr[i + 1]:
@@ -85,7 +92,7 @@ class ChessBoard(Tool):
                 _arr[i] = 0
         return _arr
 
-    def getChessLines(self, hdx, hdy, hdx_thresh, hdy_thresh, image_shape):
+    def __getChessLines(self, hdx, hdy, hdx_thresh, hdy_thresh, image_shape):
         # Generate Gaussian window
         window_size = 21
         sigma = 8.0
@@ -102,24 +109,24 @@ class ChessBoard(Tool):
         blur_y = np.convolve(hdy_thresh_binary, gausswin, mode='same')
 
         # Skeletonize signals
-        skel_x = self.skeletonize_1d(blur_x)
-        skel_y = self.skeletonize_1d(blur_y)
+        skel_x = self.__skeletonize_1d(blur_x)
+        skel_y = self.__skeletonize_1d(blur_y)
 
         # Find line positions
         lines_x = np.where(skel_x > 0)[0].tolist()
         lines_y = np.where(skel_y > 0)[0].tolist()
 
         # Prune lines
-        lines_x = self.pruneLines(lines_x, image_shape[1])
-        lines_y = self.pruneLines(lines_y, image_shape[0])
+        lines_x = self.__pruneLines(lines_x, image_shape[1])
+        lines_y = self.__pruneLines(lines_y, image_shape[0])
 
         # Check if lines match expected pattern
         is_match = (len(lines_x) == 7) and (len(lines_y) == 7) and \
-                   self.checkMatch(lines_x) and self.checkMatch(lines_y)
+                   self.__checkMatch(lines_x) and self.__checkMatch(lines_y)
 
         return lines_x, lines_y, is_match
 
-    def getChessTiles(self, img, lines_x, lines_y):
+    def __getChessTiles(self, img, lines_x, lines_y):
         stepx = int(round(np.mean(np.diff(lines_x))))
         stepy = int(round(np.mean(np.diff(lines_y))))
 
@@ -159,7 +166,7 @@ class ChessBoard(Tool):
         return squares
 
     # Image(PIL) --> Image(CV2)[]
-    def extract_pieces_from_image_board(self, image):
+    def __extract_pieces_from_image_board(self, image):
         # Load the image
         if image is None:
             print(f"Image not provided")
@@ -172,8 +179,8 @@ class ChessBoard(Tool):
         norm_image = equ.astype(np.float32) / 255.0
 
         # Compute the gradients
-        grad_x = self.gradientx(norm_image)
-        grad_y = self.gradienty(norm_image)
+        grad_x = self.__gradientx(norm_image)
+        grad_y = self.__gradienty(norm_image)
 
         # Clip the gradients
         Dx_pos = np.clip(grad_x, 0, None)
@@ -195,8 +202,8 @@ class ChessBoard(Tool):
             threshold_x = np.max(hough_Dx) * (a / 5.0)
             threshold_y = np.max(hough_Dy) * (a / 5.0)
 
-            lines_x, lines_y, is_match = self.getChessLines(hough_Dx, hough_Dy, threshold_x, threshold_y,
-                                                          norm_image.shape)
+            lines_x, lines_y, is_match = self.__getChessLines(hough_Dx, hough_Dy, threshold_x, threshold_y,
+                                                              norm_image.shape)
 
             if is_match:
                 break
@@ -205,7 +212,7 @@ class ChessBoard(Tool):
 
         squares_resized = []
         if is_match:
-            squares = self.getChessTiles(gray, lines_x, lines_y)
+            squares = self.__getChessTiles(gray, lines_x, lines_y)
             for square in squares:
                 resized = cv2.resize(square, (32, 32), interpolation=cv2.INTER_AREA)
                 squares_resized.append(resized)
@@ -229,24 +236,32 @@ class ChessBoard(Tool):
 
         return squares_resized
 
-    def detect_chess_pieces(self, images):
-        recognition = ChessPiecesRecognition()
-        return recognition.classify_pieces(images)
+    def __detect_chess_pieces(self, images):
+        return self.recognition.classify_pieces(images)
 
-    def convert_pieces_list_to_fen(self, pieces):
-        print()
+    #def __convert_pieces_list_to_fen(self, pieces):
+    #    return "dupa"
 
     def forward(self, img: Image) -> str:
-        print(f"***KS*** Analyzing chess board image")
-        cv2_image = cv2.cvtColor(numpy.array(img), cv2.COLOR_RGB2BGR)
-
-        # Image(PIL) -> Image(CV2)(32x32) []
-        squares_resized = self.extract_pieces_from_image_board(cv2_image)
-
-        # Image(CV2)(32x32) [] -> str[]
-        pieces_list = self.detect_chess_pieces(squares_resized)
-
-        # str[] -> str(FEN)
-        fen = self.convert_pieces_list_to_fen(pieces_list)
-
-        return f"FEN is: \"{fen}\n  "
+        pieces_list = ""
+        try:
+            print(f"***KS*** Analyzing chess board image for image: {img}")
+            cv2_image = cv2.cvtColor(numpy.array(img), cv2.COLOR_RGB2BGR)
+            print(f"***KS*** Got CV2 image shape: {cv2_image.shape}")
+
+            # Image(PIL) -> Image(CV2)(32x32) []
+            squares_resized = self.__extract_pieces_from_image_board(cv2_image)
+            print(f"***KS*** Squares resized: {len(squares_resized)}")
+
+            # Image(CV2)(32x32) [] -> str[]
+            pieces_list = self.__detect_chess_pieces(squares_resized)
+            print(f"***KS*** Pieces list: {pieces_list}")
+
+            # str[] -> str(FEN)
+            #fen = self.__convert_pieces_list_to_fen(pieces_list)
+            #print(f"***KS*** FEN acquired: {fen}")
+        except Exception as ex:
+            print(traceback.format_exc())
+            print(f"***KS*** Exception invoking ChessBoard: {ex}")
+
+        return pieces_list

my_fen_tool.py

@@ -0,0 +1,69 @@
+from smolagents import Tool
+import numpy as np
+import textwrap
+
+
+#AUTHORIZED_TYPES = [
+#    "string",
+#    "boolean",
+#    "integer",
+#    "number",
+#    "image",
+#    "audio",
+#    "array",
+#    "object",
+#    "any",
+#    "null",
+#]
+
+# https://chessvision.ai/docs/tools/fen2image/
+# https://en.wikipedia.org/wiki/Forsyth%E2%80%93Edwards_Notation
+
+
+class FENTool(Tool):
+    name = "_my_fen"
+    description = """
+        Convert a bord provided a s alist of chest pieces into FEN notation
+    """
+
+    inputs = {
+        "_inp": {
+            "type": "string",
+            "description": "string with chest pieces",
+        }
+    }
+
+    output_type = "string"
+
+    def __shortenFEN(self, fen):
+        """Reduce FEN to shortest form (ex. '111p11Q' becomes '3p2Q')"""
+        return fen.replace('11111111','8').replace('1111111','7') \
+            .replace('111111','6').replace('11111','5') \
+            .replace('1111','4').replace('111','3').replace('11','2')
+
+    def forward(self, _inp: str) -> str:
+        #arr = np.array(list(_inp)).reshape(8, 8)
+        #print(f"Arr: {arr.shape}\n{arr}")
+
+        #arr = np.flip(arr, axis=0)
+        #print(f"Arr flipped: {arr.shape}\n{arr}")
+
+        #fen = [str(r) for r in arr]
+        #print(f"rows as str: {fen}")
+
+        arr = textwrap.wrap(_inp, 8)
+        print(f"Arr: \n{arr}")
+
+        arr = arr[::-1]
+        print(f"Arr: \n{arr}")
+
+        fen_long = "/".join(arr)
+        print(f"FEN long: \n{fen_long}")
+
+        fen_short = self.__shortenFEN(fen_long)
+        print(f"FEN short: \n{fen_short}")
+
+        fen_enhanced = f"{fen_short} b - - 0 1"
+
+        _out = fen_enhanced
+        return _out

my_tools.py → my_image_load.py

@@ -17,28 +17,6 @@ from io import BytesIO
 #]
 
 
-class ReverseStringTool(Tool):
-    name = "_my_reverse_string"
-    description = """
-        Decode a string which is provided in a reversed form.
-    """
-
-    inputs = {
-        "_inp": {
-            "type": "string",
-            "description": "encoded input string",
-        }
-    }
-
-    output_type = "string"
-
-    def forward(self, _inp: str) -> str:
-        _out = ""
-        for a in _inp:
-            _out = a + _out
-        return _out
-
-
 class ImageLoadTool(Tool):
     name = "_my_image_load"
     description = """

my_reverse_string.py

@@ -0,0 +1,37 @@
+from smolagents import Tool
+
+
+#AUTHORIZED_TYPES = [
+#    "string",
+#    "boolean",
+#    "integer",
+#    "number",
+#    "image",
+#    "audio",
+#    "array",
+#    "object",
+#    "any",
+#    "null",
+#]
+
+
+class ReverseStringTool(Tool):
+    name = "_my_reverse_string"
+    description = """
+        Decode a string which is provided in a reversed form.
+    """
+
+    inputs = {
+        "_inp": {
+            "type": "string",
+            "description": "encoded input string",
+        }
+    }
+
+    output_type = "string"
+
+    def forward(self, _inp: str) -> str:
+        _out = ""
+        for a in _inp:
+            _out = a + _out
+        return _out

chess_pieces_detection/train_chess_pieces_recognition.py → my_train_chess_pieces_recognition.py

@@ -27,7 +27,7 @@ from PIL import Image
 # - black Pawn          p
 # - empty
 
-TRAIN_DIR = "/mnt/c/Users/krzsa/IdeaProjects/Agents-Course-Assignment/chess_pieces_detection/train-data"
+TRAIN_DIR = "/mnt/c/Users/krzsa/IdeaProjects/Agents-Course-Assignment/train-data"
 TRAIN_DIR_BLACK = f"{TRAIN_DIR}/black"
 TRAIN_DIR_WHITE = f"{TRAIN_DIR}/white"
 TRAIN_DIR_EMPTY = f"{TRAIN_DIR}/empty"
@@ -82,9 +82,10 @@ TEST_DATA = TRAIN_DATA
 # https://docs.pytorch.org/docs/stable/generated/torch.nn.Module.html#torch.nn.Module
 class CNNModel(nn.Module):
 
-    def __init__(self, _name):
+    def __init__(self, _model_name, _model_dir):
         super(CNNModel, self).__init__()
-        self.name = _name
+        self.name = _model_name
+        self.model_dir = _model_dir
         print("***KS*** Model: Creating layers")
         # First Convolutional Layer: 32 features, 5x5 kernel
         # https://docs.pytorch.org/docs/stable/generated/torch.nn.Conv2d.html#torch.nn.Conv2d
@@ -108,7 +109,7 @@ class CNNModel(nn.Module):
 
     def _initialize_weights(self):
         # Load the pre-trained model
-        model_name = f"saved_models/{self.name}.pth"
+        model_name =  os.path.join(self.model_dir, self.name)
         print(f"***KS*** Checking pre-trained model: '{model_name}'")
         if os.path.exists(model_name):
             print(f"***KS*** Model '{model_name}' exists, loading weights ...")
@@ -135,7 +136,7 @@ class CNNModel(nn.Module):
         print(f"***KS*** Saving model ...")
         # Save the model checkpoint
         os.makedirs('saved_models', exist_ok=True)
-        model_save_path = f"saved_models/{self.name}.pth"
+        model_save_path = f"../saved_models/{self.name}.pth"
         torch.save(self.state_dict(), model_save_path)
         print(f'*** KS *** Model saved in file: {model_save_path}')
 
@@ -249,6 +250,8 @@ class ChessDataset(Dataset):
 
 
 class ChessImagesDataset(Dataset):
+    CHESS_PIECES = '1KQRBNPkqrbnp'
+
     def __init__(self, images):
         self.num_images = len(images)
         self.images = images
@@ -269,9 +272,9 @@ class ChessImagesDataset(Dataset):
 
 
 class ChessPiecesRecognition:
-    def __init__(self):
-        print(f"***KS*** Chess pieces recognition initialized")
-        self.model = CNNModel("test-1")
+    def __init__(self, _model_name, _model_dir):
+        print(f"***KS*** Chess pieces recognition initializing ...")
+        self.model = CNNModel(_model_name, _model_dir)
         self.__load_train_data__()
 
     def __load_train_data__(self):
@@ -378,10 +381,9 @@ class ChessPiecesRecognition:
             for inputs, labels in loader:
                 # Move inputs and labels to device
                 inputs = inputs.to(self.model.get_device())
-                labels = labels.to(self.model.get_device())
 
                 outputs = self.model(inputs)
-                print(f"***KS*** Got model outputs: \nshape: {outputs.shape}\n{outputs}")
+                print(f"***KS*** Got model outputs: \nshape: {outputs.shape}")
 
                 labels_detected = np.argmax(outputs.cpu(), axis=1)
                 print(f"***KS*** Got labels idx detected: \nshape: {labels_detected.shape}\n{labels_detected}")
@@ -394,7 +396,4 @@ class ChessPiecesRecognition:
 
 #t = ChessPiecesRecognition()
 #t.train()
-#t.eval()
-
-if __name__ == "__main__":
-    print("This is a module and should not be executed directly")
+#t.eval()

my_train_chess_pieces_recognition.sh

@@ -0,0 +1,2 @@
+. ./venv/bin/activate
+python3 my_train_chess_pieces_recognition.py

simple.py

@@ -7,15 +7,14 @@ from smolagents import CodeAgent, tool, InferenceClientModel, WebSearchTool, loa
 from smolagents import PromptTemplates, PlanningPromptTemplate, FinalAnswerPromptTemplate, ManagedAgentPromptTemplate
 
 from dotenv import load_dotenv
-from my_tools import ReverseStringTool, ImageLoadTool
-from chess_board_tool import ChessBoard
+from my_reverse_string import ReverseStringTool
+from my_image_load import ImageLoadTool
+from my_chess_board_tool import ChessBoard
 from PIL import Image
 
 
 task_id = "cca530fc-4052-43b2-b130-b30968d8aa44"
 
-# https://github.com/kratos606/chessboard-recogniser/tree/main
-
 MODEL_REASONING = "Qwen/Qwen2.5-Coder-32B-Instruct"
 #MODEL_REASONING = "Qwen/Qwen2.5-72B-Instruct"  not good
 #"meta-llama/Meta-Llama-3-70B-Instruct"
@@ -30,6 +29,7 @@ PROMPT_TEMPLATES = PromptTemplates(
             
             Describe your initial plan as a set of bullet points. 
             Each bullet point should describe in one sentence an action which is to be taken in this step.
+            Do NOT provide hypothetical examples for the final answer.
             
             Use the tools provided. If you are going to use a tool, describe in detail how you are going
             to use that particular tool and explain parameters used to invoke the tool.
@@ -73,10 +73,18 @@ PROMPT_TEMPLATES = PromptTemplates(
 #question = f"Load an image for task id {task_id} and display it using matplotlib "
 question = f"Load an image for task id {task_id} and analyze the chess board "
 
+chess_board_model_name = "my_chess_pieces_recognition.pth"
+chess_board_model_dir = "/mnt/c/Users/krzsa/IdeaProjects/Agents-Course-Assignment/saved_models"
+
 reasoning_agent = CodeAgent(
     name="CourseAssistant",
     description="General AI Assistant",
-    tools=[ImageLoadTool(), FinalAnswerTool(), ReverseStringTool(), ChessBoard()],
+    tools=[
+        ImageLoadTool(),
+        FinalAnswerTool(),
+        ReverseStringTool(),
+        ChessBoard(chess_board_model_name, chess_board_model_dir)
+    ],
     model=InferenceClientModel(model_id=MODEL_REASONING),
     planning_interval=3, # This is where you activate planning!,
     prompt_templates=PROMPT_TEMPLATES,

test.sh

@@ -0,0 +1,2 @@
+. ./venv/bin/activate
+pytest --capture=no

test_tools.py

@@ -1,20 +1,22 @@
-from my_tools import ReverseStringTool, ImageLoadTool
-from chess_board_tool import ChessBoard
+from my_reverse_string import ReverseStringTool
+from my_image_load import ImageLoadTool
+from my_chess_board_tool import ChessBoard
+from my_fen_tool import FENTool
 import pytest
 import matplotlib.pyplot as plt
 import matplotlib as mp
 
 #pytest --capture=no
 
+@pytest.mark.skip(reason="disabled")
 @pytest.mark.parametrize("_inp,_exp",[("abc", "cba"),("ihg fed cba", "abc def ghi")])
 def test_tool_reverse_string(_inp,_exp):
     assert ReverseStringTool().forward(_inp) == _exp
 
-
+@pytest.mark.skip(reason="disabled")
 @pytest.mark.parametrize("_task_id,_exp",[("cca530fc-4052-43b2-b130-b30968d8aa44", "")])
 def test_tool_image_load(_task_id,_exp):
-    #assert ReverseStringTool().forward(_inp) == _exp
-    print(f"Loading image for task id: {_task_id}")
+    print(f"\nLoading image for task id: {_task_id}")
     t = ImageLoadTool()
     result = t.forward(_task_id)
     print(f"Got result: {result}")
@@ -22,13 +24,28 @@ def test_tool_image_load(_task_id,_exp):
     #plt.imshow(result)
     #plt.show()
 
-
-@pytest.mark.parametrize("_task_id,_exp",[("cca530fc-4052-43b2-b130-b30968d8aa44", "")])
+@pytest.mark.skip(reason="disabled")
+@pytest.mark.parametrize("_task_id,_exp",[("cca530fc-4052-43b2-b130-b30968d8aa44", "1K1111111PP11111P11RBBqP1111n111Q1111111p11b11111pp111pp1k11r111")])
 def test_tool_chess_board(_task_id,_exp):
-    #assert ReverseStringTool().forward(_inp) == _exp
-    print(f"Loading image for task id: {_task_id}")
+    print(f"\nLoading image for task id: {_task_id}")
     t = ImageLoadTool()
     image = t.forward(_task_id)
     print(f"Got result: {image}")
-    board_tool = ChessBoard()
-    fen = board_tool.forward(image)
+    chess_board_model_name = "my_chess_pieces_recognition.pth"
+    chess_board_model_dir = "/mnt/c/Users/krzsa/IdeaProjects/Agents-Course-Assignment/saved_models"
+    board_tool = ChessBoard(chess_board_model_name, chess_board_model_dir)
+    pieces = board_tool.forward(image)
+    assert pieces == _exp
+
+
+@pytest.mark.parametrize("_pieces_list,_exp",
+    [
+        ("1K1111111PP11111P11RBBqP1111n111Q1111111p11b11111pp111pp1k11r111",
+         "1k2r3/1pp3pp/p2b4/Q7/4n3/P2RBBqP/1PP5/1K6 b - - 0 1")
+    ])
+def test_tool_fen(_pieces_list,_exp):
+    print(f"\nConverting pieces list to FEN: {_pieces_list}")
+    t = FENTool()
+    fen = t.forward(_pieces_list)
+    print(f"Got result: {fen}")
+    assert fen == _exp