make app

app_gradio/README.md

@@ -0,0 +1,3 @@
+## Image to Fen
+
+[check out the GitHub repo](https://github.com/DerekLiu35/ChessCV).

app_gradio/app.py

@@ -0,0 +1,169 @@
+"""Provide an image of a chessboard and get the FEN (https://en.wikipedia.org/wiki/Forsyth–Edwards_Notation ) representation of the board."""
+import argparse
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Callable
+
+import gradio as gr
+from PIL import ImageStat
+from PIL.Image import Image
+import requests
+
+from image_to_fen.fen import ImageToFen
+import image_to_fen.util as util
+
+os.environ["CUDA_VISIBLE_DEVICES"] = ""  # do not use GPU
+
+logging.basicConfig(level=logging.INFO)
+
+APP_DIR = Path(__file__).resolve().parent
+FAVICON = APP_DIR / "265f.png"
+README = APP_DIR / "README.md"
+
+DEFAULT_PORT = 11700
+
+
+def main(args):
+    predictor = PredictorBackend(url=args.model_url)
+    frontend = make_frontend(
+        predictor.run
+    )
+    frontend.launch(
+        server_name="0.0.0.0",  # noqa: S104
+        server_port=args.port,  # set a port to bind to, failing if unavailable
+        share=True,
+        favicon_path=FAVICON,
+    )
+
+
+def make_frontend(
+    fn: Callable[[Image], str],
+    app_name: str = "image-to-fen"
+):
+    """Creates a gradio.Interface frontend for an image to text function."""
+    examples_dir = Path("image_to_fen") / "tests" / "support" / "boards"
+    example_fnames = [elem for elem in os.listdir(examples_dir) if elem.endswith(".png")]
+    example_paths = [examples_dir / fname for fname in example_fnames]
+    examples = [[str(path)] for path in example_paths]
+
+    allow_flagging = "never"
+
+    readme = _load_readme(with_logging=allow_flagging == "manual")
+
+    # build a basic browser interface to a Python function
+    frontend = gr.Interface(
+        fn=fn,
+        outputs=gr.components.Textbox(),
+        inputs=gr.components.Image(type="pil", label="Chess Board"),
+        title="♟️ Image to Fen",
+        thumbnail="FAVICON",
+        description=__doc__,
+        article=readme,
+        examples=examples,
+        cache_examples=False,
+        allow_flagging=allow_flagging
+    )
+
+    return frontend
+
+
+class PredictorBackend:
+    """Interface to a backend that serves predictions.
+
+    To communicate with a backend accessible via a URL, provide the url kwarg.
+
+    Otherwise, runs a predictor locally.
+    """
+
+    def __init__(self, url=None):
+        if url is not None:
+            self.url = url
+            self._predict = self._predict_from_endpoint
+        else:
+            model = ImageToFen()
+            self._predict = model.predict
+
+    def run(self, image):
+        pred, metrics = self._predict_with_metrics(image)
+        self._log_inference(pred, metrics)
+        return pred
+
+    def _predict_with_metrics(self, image):
+        pred = self._predict(image)
+
+        stats = ImageStat.Stat(image)
+        metrics = {
+            "image_mean_intensity": stats.mean,
+            "image_median": stats.median,
+            "image_extrema": stats.extrema,
+            "image_area": image.size[0] * image.size[1],
+            "pred_length": len(pred),
+        }
+        return pred, metrics
+
+    def _predict_from_endpoint(self, image):
+        """Send an image to an endpoint that accepts JSON and return the predicted text.
+
+        The endpoint should expect a base64 representation of the image, encoded as a string,
+        under the key "image". It should return the predicted text under the key "pred".
+
+        Parameters
+        ----------
+        image
+            A PIL image of a chess board.
+
+        Returns
+        -------
+        pred
+            A string containing the predictor's guess of the FEN representation of the chess board.
+        """
+        encoded_image = util.encode_b64_image(image)
+
+        headers = {"Content-type": "application/json"}
+        payload = json.dumps({"image": "data:image/png;base64," + encoded_image})
+
+        response = requests.post(self.url, data=payload, headers=headers)
+        pred = response.json()["pred"]
+
+        return pred
+
+    def _log_inference(self, pred, metrics):
+        for key, value in metrics.items():
+            logging.info(f"METRIC {key} {value}")
+        logging.info(f"PRED >begin\n{pred}\nPRED >end")
+
+
+def _load_readme(with_logging=False):
+    with open(README) as f:
+        lines = f.readlines()
+        # if not with_logging:
+        #     lines = lines[: lines.index("<!-- logging content below -->\n")]
+
+        readme = "".join(lines)
+    return readme
+
+
+def _make_parser():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--model_url",
+        default=None,
+        type=str,
+        help="Identifies a URL to which to send image data. Data is base64-encoded, converted to a utf-8 string, and then set via a POST request as JSON with the key 'image'. Default is None, which instead sends the data to a model running locally.",
+    )
+    parser.add_argument(
+        "--port",
+        default=DEFAULT_PORT,
+        type=int,
+        help=f"Port on which to expose this server. Default is {DEFAULT_PORT}.",
+    )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = _make_parser()
+    args = parser.parse_args()
+    main(args)

image_to_fen/__init__.py

@@ -0,0 +1 @@
+"""Modules for creating and running image to fen."""

image_to_fen/artifacts/image-to-fen/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9623198845b7c54ad9f3ada08d6cd87a036f5a3fd49a63d03f7e35e3d59bc3d
+size 166133826

image_to_fen/fen.py

@@ -0,0 +1,109 @@
+import argparse
+from pathlib import Path
+from typing import Sequence, Union
+
+from PIL import Image
+import torch
+import torchvision
+import numpy as np
+import re
+
+import image_to_fen.util as util
+
+
+STAGED_MODEL_DIRNAME = Path(__file__).resolve().parent / "artifacts" / "image-to-fen"
+MODEL_FILE = "model.pt"
+
+class ImageToFen:
+    """Takes image of chess board and returns FEN string."""
+
+    def __init__(self, model_path=None):
+        if model_path is None:
+            model_path = STAGED_MODEL_DIRNAME / MODEL_FILE
+        self.model = torch.jit.load(model_path)
+
+    @torch.no_grad()
+    def predict(self, image: Union[str, Path, Image.Image]) -> str:
+        """Predict FEN string for image of chess board."""
+        image = image
+        if not isinstance(image, Image.Image):
+            image = util.read_image_pil(image, grayscale=True)
+        image = image.resize((200, 200))
+        image = torchvision.transforms.PILToTensor()(image)/255
+        pred = self.model([image])[1][0]
+        nms_pred = apply_nms(pred, iou_thresh=0.2)
+        pred_str = boxes_labels_to_fen(nms_pred['boxes'], nms_pred['labels'])
+        return pred_str
+    
+def apply_nms(orig_prediction, iou_thresh=0.3):
+
+    # torchvision returns the indices of the bboxes to keep
+    keep = torchvision.ops.nms(orig_prediction['boxes'], orig_prediction['scores'], iou_thresh)
+
+    final_prediction = orig_prediction
+    final_prediction['boxes'] = final_prediction['boxes'][keep]
+    final_prediction['scores'] = final_prediction['scores'][keep]
+    final_prediction['labels'] = final_prediction['labels'][keep]
+
+    return final_prediction
+
+def boxes_labels_to_fen(boxes, labels, square_size=25):
+  boxes = torch.round(boxes / 25) * 25
+  eye = np.eye(13)
+  one_hot = onehot_from_fen("8-8-8-8-8-8-8-8")
+  for i, box in enumerate(boxes):
+    x = box[0]
+    y = box[1]
+    ind = int((x / square_size) + (y / square_size) * 8)
+    if (ind >= 64):
+      continue
+    one_hot[ind] = eye[12 - labels[i]].reshape((1, 13)).astype(int)
+  return fen_from_onehot(one_hot)
+
+def onehot_from_fen(fen):
+    piece_symbols = 'prbnkqPRBNKQ'
+    eye = np.eye(13)
+    output = np.empty((0, 13))
+    fen = re.sub('[-]', '', fen)
+
+    for char in fen:
+        if(char in '12345678'):
+            output = np.append(
+              output, np.tile(eye[12], (int(char), 1)), axis=0)
+        else:
+            idx = piece_symbols.index(char)
+            output = np.append(output, eye[idx].reshape((1, 13)), axis=0)
+
+    return output
+
+def fen_from_onehot(one_hot):
+    piece_symbols = 'prbnkqPRBNKQ'
+    output = ''
+    for j in range(8):
+        for i in range(8):
+            idx = np.where(one_hot[j*8 + i]==1)[0][0]
+            if(idx == 12):
+                output += ' '
+            else:
+                output += piece_symbols[idx]
+        if(j != 7):
+            output += '-'
+
+    for i in range(8, 0, -1):
+        output = output.replace(' ' * i, str(i))
+
+    return output
+
+def main():
+    """Run prediction on image."""
+    parser = argparse.ArgumentParser(description="Predict FEN string for image of chess board.")
+    parser.add_argument("image", type=Path, help="Path to image file.")
+    parser.add_argument("--model-path", type=Path, help="Path to model file.")
+    args = parser.parse_args()
+    image_to_fen = ImageToFen(args.model_path)
+    pred = image_to_fen.predict(args.image)
+    print(f"Prediction: {pred}")
+    
+    # image_to_fen/tests/support/boards/phpSrRLQ1.png
+if __name__ == "__main__":
+    main()

image_to_fen/util.py

@@ -0,0 +1,88 @@
+"""Utility functions for image_to_fen module."""
+import base64
+import contextlib
+import hashlib
+from io import BytesIO
+import os
+from pathlib import Path
+from typing import Union
+from urllib.request import urlretrieve
+
+import numpy as np
+from PIL import Image
+import smart_open
+from tqdm import tqdm
+
+
+def to_categorical(y, num_classes):
+    """1-hot encode a tensor."""
+    return np.eye(num_classes, dtype="uint8")[y]
+
+
+def read_image_pil(image_uri: Union[Path, str], grayscale=False) -> Image:
+    with smart_open.open(image_uri, "rb") as image_file:
+        return read_image_pil_file(image_file, grayscale)
+
+
+def read_image_pil_file(image_file, grayscale=False) -> Image:
+    with Image.open(image_file) as image:
+        if grayscale:
+            image = image.convert(mode="L")
+        else:
+            image = image.convert(mode=image.mode)
+        return image
+
+
+@contextlib.contextmanager
+def temporary_working_directory(working_dir: Union[str, Path]):
+    """Temporarily switches to a directory, then returns to the original directory on exit."""
+    curdir = os.getcwd()
+    os.chdir(working_dir)
+    try:
+        yield
+    finally:
+        os.chdir(curdir)
+
+def encode_b64_image(image, format="png"):
+    """Encode a PIL image as a base64 string."""
+    _buffer = BytesIO()  # bytes that live in memory
+    image.save(_buffer, format=format)  # but which we write to like a file
+    encoded_image = base64.b64encode(_buffer.getvalue()).decode("utf8")
+    return encoded_image
+
+
+def compute_sha256(filename: Union[Path, str]):
+    """Return SHA256 checksum of a file."""
+    with open(filename, "rb") as f:
+        return hashlib.sha256(f.read()).hexdigest()
+
+
+class TqdmUpTo(tqdm):
+    """From https://github.com/tqdm/tqdm/blob/master/examples/tqdm_wget.py"""
+
+    def update_to(self, blocks=1, bsize=1, tsize=None):
+        """
+        Parameters
+        ----------
+        blocks: int, optional
+            Number of blocks transferred so far [default: 1].
+        bsize: int, optional
+            Size of each block (in tqdm units) [default: 1].
+        tsize: int, optional
+            Total size (in tqdm units). If [default: None] remains unchanged.
+        """
+        if tsize is not None:
+            self.total = tsize
+        self.update(blocks * bsize - self.n)  # will also set self.n = b * bsize
+
+
+def download_url(url, filename):
+    """Download a file from url to filename, with a progress bar."""
+    with TqdmUpTo(unit="B", unit_scale=True, unit_divisor=1024, miniters=1) as t:
+        urlretrieve(url, filename, reporthook=t.update_to, data=None)  # noqa: S310
+
+# the function takes the original prediction and the iou threshold.
+
+# function to convert a torchtensor back to PIL image
+def torch_to_pil(img):
+    return torchvision.transforms.ToPILImage()(img).convert('RGB')

requirements.txt

@@ -0,0 +1,369 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile requirements/prod.in
+#
+aiofiles==23.2.1
+    # via gradio
+aiohttp==3.8.5
+    # via gradio
+aiosignal==1.3.1
+    # via aiohttp
+altair==5.0.1
+    # via gradio
+annotated-types==0.5.0
+    # via pydantic
+anyio==3.7.1
+    # via
+    #   httpcore
+    #   starlette
+async-timeout==4.0.3
+    # via aiohttp
+attrs==23.1.0
+    # via
+    #   aiohttp
+    #   jsonschema
+    #   referencing
+boto3==1.28.34
+    # via
+    #   boto3-extensions
+    #   smart-open
+boto3-extensions==0.20.0
+    # via gantry
+botocore==1.31.34
+    # via
+    #   boto3
+    #   boto3-extensions
+    #   s3transfer
+cachetools==4.2.4
+    # via gantry
+certifi==2023.7.22
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+charset-normalizer==3.2.0
+    # via
+    #   aiohttp
+    #   requests
+click==8.1.7
+    # via
+    #   gantry
+    #   uvicorn
+click-spinner==0.1.10
+    # via gantry
+cmake==3.27.2
+    # via triton
+colorama==0.4.6
+    # via
+    #   gantry
+    #   halo
+    #   log-symbols
+contourpy==1.1.0
+    # via matplotlib
+cycler==0.11.0
+    # via matplotlib
+dateparser==1.1.8
+    # via gantry
+exceptiongroup==1.1.3
+    # via anyio
+fastapi==0.101.1
+    # via gradio
+ffmpy==0.3.1
+    # via gradio
+filelock==3.12.2
+    # via
+    #   huggingface-hub
+    #   torch
+    #   triton
+fonttools==4.42.1
+    # via matplotlib
+frozenlist==1.4.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec==2023.6.0
+    # via
+    #   gradio-client
+    #   huggingface-hub
+gantry==0.4.9
+    # via -r requirements/prod.in
+gradio==3.40.1
+    # via -r requirements/prod.in
+gradio-client==0.5.0
+    # via gradio
+h11==0.14.0
+    # via
+    #   httpcore
+    #   uvicorn
+h5py==3.9.0
+    # via -r requirements/prod.in
+halo==0.0.31
+    # via gantry
+httpcore==0.17.3
+    # via httpx
+httpx==0.24.1
+    # via
+    #   gradio
+    #   gradio-client
+huggingface-hub==0.16.4
+    # via
+    #   gradio
+    #   gradio-client
+idna==3.4
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
+importlib-metadata==6.8.0
+    # via -r requirements/prod.in
+importlib-resources==6.0.1
+    # via gradio
+isodate==0.6.1
+    # via gantry
+jinja2==3.1.2
+    # via
+    #   -r requirements/prod.in
+    #   altair
+    #   gradio
+    #   torch
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+jsonschema==4.19.0
+    # via altair
+jsonschema-specifications==2023.7.1
+    # via jsonschema
+kiwisolver==1.4.5
+    # via matplotlib
+linkify-it-py==2.0.2
+    # via markdown-it-py
+lit==16.0.6
+    # via triton
+log-symbols==0.0.14
+    # via halo
+markdown-it-py[linkify]==2.2.0
+    # via
+    #   gradio
+    #   mdit-py-plugins
+markupsafe==2.1.3
+    # via
+    #   gradio
+    #   jinja2
+marshmallow==3.20.1
+    # via
+    #   gantry
+    #   marshmallow-oneofschema
+marshmallow-oneofschema==3.0.1
+    # via gantry
+matplotlib==3.7.2
+    # via gradio
+mdit-py-plugins==0.3.3
+    # via gradio
+mdurl==0.1.2
+    # via markdown-it-py
+monotonic==1.6
+    # via gantry
+mpmath==1.3.0
+    # via sympy
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+networkx==3.1
+    # via torch
+numpy==1.25.2
+    # via
+    #   -r requirements/prod.in
+    #   altair
+    #   contourpy
+    #   gantry
+    #   gradio
+    #   h5py
+    #   matplotlib
+    #   pandas
+    #   torchvision
+nvidia-cublas-cu11==11.10.3.66
+    # via
+    #   nvidia-cudnn-cu11
+    #   nvidia-cusolver-cu11
+    #   torch
+nvidia-cuda-cupti-cu11==11.7.101
+    # via torch
+nvidia-cuda-nvrtc-cu11==11.7.99
+    # via torch
+nvidia-cuda-runtime-cu11==11.7.99
+    # via torch
+nvidia-cudnn-cu11==8.5.0.96
+    # via torch
+nvidia-cufft-cu11==10.9.0.58
+    # via torch
+nvidia-curand-cu11==10.2.10.91
+    # via torch
+nvidia-cusolver-cu11==11.4.0.1
+    # via torch
+nvidia-cusparse-cu11==11.7.4.91
+    # via torch
+nvidia-nccl-cu11==2.14.3
+    # via torch
+nvidia-nvtx-cu11==11.7.91
+    # via torch
+orjson==3.9.5
+    # via gradio
+packaging==23.1
+    # via
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   marshmallow
+    #   matplotlib
+pandas==2.0.3
+    # via
+    #   altair
+    #   gantry
+    #   gradio
+pillow==9.4.0
+    # via
+    #   -r requirements/prod.in
+    #   gradio
+    #   matplotlib
+    #   torchvision
+pydantic==2.3.0
+    # via
+    #   fastapi
+    #   gradio
+pydantic-core==2.6.3
+    # via pydantic
+pydub==0.25.1
+    # via gradio
+pyngrok==6.0.0
+    # via -r requirements/prod.in
+pyparsing==3.0.9
+    # via matplotlib
+python-dateutil==2.8.2
+    # via
+    #   botocore
+    #   dateparser
+    #   gantry
+    #   matplotlib
+    #   pandas
+python-multipart==0.0.6
+    # via gradio
+pytz==2023.3
+    # via
+    #   dateparser
+    #   pandas
+pyyaml==6.0.1
+    # via
+    #   gantry
+    #   gradio
+    #   huggingface-hub
+    #   pyngrok
+referencing==0.30.2
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+regex==2023.8.8
+    # via dateparser
+requests==2.31.0
+    # via
+    #   -r requirements/prod.in
+    #   gantry
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   torchvision
+rpds-py==0.9.2
+    # via
+    #   jsonschema
+    #   referencing
+s3transfer==0.6.2
+    # via boto3
+semantic-version==2.10.0
+    # via gradio
+six==1.16.0
+    # via
+    #   halo
+    #   isodate
+    #   python-dateutil
+smart-open[s3]==6.3.0
+    # via -r requirements/prod.in
+sniffio==1.3.0
+    # via
+    #   anyio
+    #   httpcore
+    #   httpx
+spinners==0.0.24
+    # via halo
+starlette==0.27.0
+    # via fastapi
+sympy==1.12
+    # via torch
+tabulate==0.9.0
+    # via gantry
+termcolor==2.3.0
+    # via halo
+toolz==0.12.0
+    # via altair
+torch==2.0.1
+    # via
+    #   -r requirements/prod.in
+    #   torchvision
+    #   triton
+torchvision==0.15.2
+    # via -r requirements/prod.in
+tqdm==4.66.1
+    # via
+    #   -r requirements/prod.in
+    #   gantry
+    #   huggingface-hub
+triton==2.0.0
+    # via torch
+typeguard==2.13.3
+    # via gantry
+typing-extensions==4.7.1
+    # via
+    #   altair
+    #   fastapi
+    #   gantry
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   pydantic
+    #   pydantic-core
+    #   torch
+    #   uvicorn
+tzdata==2023.3
+    # via pandas
+tzlocal==5.0.1
+    # via dateparser
+uc-micro-py==1.0.2
+    # via linkify-it-py
+urllib3==1.26.16
+    # via
+    #   botocore
+    #   requests
+uvicorn==0.23.2
+    # via gradio
+websockets==11.0.3
+    # via
+    #   gradio
+    #   gradio-client
+wheel==0.41.2
+    # via
+    #   nvidia-cublas-cu11
+    #   nvidia-cuda-cupti-cu11
+    #   nvidia-cuda-runtime-cu11
+    #   nvidia-curand-cu11
+    #   nvidia-cusparse-cu11
+    #   nvidia-nvtx-cu11
+yarl==1.9.2
+    # via aiohttp
+zipp==3.16.2
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools