webcam.py

import cv2
import numpy as np
import tensorflow as tf
from PIL import Image

TFLITE_PATH: str = "./models/model_mobilenet_v2.tflite"

IMAGE_SIZE: tuple[int, int] = (160, 160)
CLASS_NAMES: list[str] = [
    "A", "B", "C", "D", "E",
    "F", "G", "H", "I", "J",
    "K", "L", "M", "N", "O",
    "P", "Q", "R", "S", "T",
    "U", "V", "W", "X", "Y",
    "Z", "del", "space",
]

TARGET_FRAME_COUNT: int = 3
TARGET_CONSECUTIVE_PREDICTIONS: int = 4
TARGET_PREDICTION_SCORE: float = 0.92


def load_model():
    interpreter = tf.lite.Interpreter(model_path=TFLITE_PATH)
    # print(interpreter.get_signature_list())
    classify_lite = interpreter.get_signature_runner("serving_default")
    return classify_lite


def get_image_array(image_data):
    img_array = tf.keras.utils.img_to_array(image_data)
    img_array = tf.expand_dims(img_array, 0)  # Create a batch

    return img_array


def predict(classify_lite, image_array):
    score_lite = classify_lite(input_2=image_array)["outputs"]

    predicted_char = CLASS_NAMES[np.argmax(score_lite)]
    prediction_score = np.max(score_lite)

    return predicted_char, prediction_score

def max_predicted(predictions: dict[str, int]) -> tuple[str, int]:
    return max(predictions.items(), key=lambda k: k[1])


if __name__ == "__main__":
    classify_lite = load_model()

    x1, y1 = 100, 100
    x2, y2 = (x1 + IMAGE_SIZE[0]), (y1 + IMAGE_SIZE[1])

    video_capture = cv2.VideoCapture(0)

    frame_count: int = 0
    previous_predictions: dict[str, int] = {letter: 0 for letter in CLASS_NAMES}
    text: str = ""

    while True:
        ret, img = video_capture.read()
        img = cv2.flip(img, 1)

        predicted_char: str = ""
        prediction_score: float = 0.0

        if ret:
            frame_count += 1

            if frame_count == TARGET_FRAME_COUNT:
                frame_count = 0

                img_cropped = img[y1:y2, x1:x2]
                image_data = Image.fromarray(img_cropped)
                image_array = get_image_array(image_data)

                predicted_char, prediction_score = predict(classify_lite, image_array)

                if (
                    prediction_score >= TARGET_PREDICTION_SCORE
                ):
                    previous_predictions[predicted_char] += 1

                letter, count = max_predicted(previous_predictions)
                if (
                    count >= TARGET_CONSECUTIVE_PREDICTIONS
                ):
                    previous_predictions = {letter: 0 for letter in CLASS_NAMES}

                    if letter == "space":
                        text += " "
                    elif letter == "del":
                        text = text[:-1]
                    else:
                        text += letter

            cv2.putText(
                img,
                predicted_char.upper(),
                (100, 400),
                cv2.FONT_HERSHEY_SIMPLEX,
                4,
                (255, 255, 255),
                4,
            )
            cv2.putText(
                img,
                f"(score = {prediction_score:.2f})",
                (100, 450),
                cv2.FONT_HERSHEY_SIMPLEX,
                1,
                (255, 255, 255),
            )
            cv2.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0), 2)
            cv2.imshow("Video Capture", img)

            blank_img = np.zeros((200, 1200, 3), np.uint8)
            cv2.putText(
                blank_img,
                text.upper(),
                (30, 30),
                cv2.FONT_HERSHEY_SIMPLEX,
                1,
                (255, 255, 255),
                2,
            )
            cv2.imshow("Text", blank_img)

            keyboard_key = cv2.waitKey(1)
            if keyboard_key == 27:  # when `esc` is pressed
                break

    cv2.destroyAllWindows()
    cv2.VideoCapture(0).release()