push code

.gitattributes

@@ -1,35 +1,3 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.pdiparams filter=lfs diff=lfs merge=lfs -text
+*.pdmodel filter=lfs diff=lfs merge=lfs -text
+*.otf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,113 @@
+# virtual environment
+.venv
+common/cpp
+common/cpp_gapi
+sample_videos/*
+models/vietocr/*
+models/soundbar/*
+results.json
+output/*
+models/soundbar_detection_yolov7/*
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+# C extensions
+*.so
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+# source/yolov7/*
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+# Translations
+*.mo
+*.pot
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+# Flask stuff:
+instance/
+.webassets-cache
+# Scrapy stuff:
+.scrapy
+# Sphinx documentation
+docs/_build/
+# PyBuilder
+target/
+# Jupyter Notebook
+.ipynb_checkpoints
+# IPython
+profile_default/
+ipython_config.py
+# pyenv
+.python-version
+# celery beat schedule file
+celerybeat-schedule
+# SageMath parsed files
+*.sage.py
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+# Spyder project settings
+.spyderproject
+.spyproject
+# Rope project settings
+.ropeproject
+# mkdocs documentation
+/site
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+# Pyre type checker
+.pyre/
+# custom
+old/
+notebook/
+static/images/face
+static/images/celeba
+celeba_pics.gz
+celeba_vecs.gz
+face_pics.gz
+face_vecs.gz

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    #-   id: check-added-large-files
+    -   id: fix-byte-order-marker
+    -   id: check-case-conflict
+    -   id: check-json
+    -   id: check-yaml
+        args: ['--unsafe']
+    -   id: detect-aws-credentials
+        args: [--allow-missing-credentials]
+    -   id: detect-private-key
+    -   id: end-of-file-fixer
+    -   id: mixed-line-ending
+    -   id: trailing-whitespace
+-   repo: https://github.com/asottile/add-trailing-comma
+    rev: v3.1.0
+    hooks:
+    -   id: add-trailing-comma
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+    -   id: isort
+        name: isort (python)
+        args: [--settings-path=pyproject.toml]
+    -   id: isort
+        name: isort (cython)
+        types: [cython]
+    -   id: isort
+        name: isort (pyi)
+        types: [pyi]
+-   repo: https://github.com/psf/black
+    rev: 23.11.0
+    hooks:
+    -   id: black
+        args: [--config=pyproject.toml]
+-   repo: https://github.com/pycqa/flake8.git
+    rev: 6.1.0
+    hooks:
+    -   id: flake8
+        args: [--ignore, "E203", --max-line-length, "79"]
+-   repo: https://github.com/kynan/nbstripout
+    rev: 0.6.1
+    hooks:
+    -   id: nbstripout
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.0
+    hooks:
+    -   id: pyupgrade
+        args: [--py36-plus]

.pre-commit-setting.toml

@@ -0,0 +1,22 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+[tool.black]
+line-length = 79
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.idea
+  | \.pytest_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
+
+[flake8]
+ignore = E203
+max-line-length = 79

README.md

@@ -1,14 +1,59 @@
----
-title: Kleverocr
-emoji: 🏆
-colorFrom: purple
-colorTo: green
-sdk: gradio
-sdk_version: 4.1.2
-app_file: app.py
-pinned: false
-models:
-- kleverocr/
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+<h1 align="center">Project title</h1>
+
+[Demo on Streamlit](https://khanhphantt-demo.streamlitapp.com/)
+
+Project description
+
+<p align="center"><img src="data/images/out.jpeg" width="700" height="400"></p>
+
+
+### :file_folder: Dataset
+This dataset consists of __XXX samples__ (X classes):
+*	__class_1: YYY samples__
+*	__class_2: YYY samples__
+
+## Installation
+1. Clone the repo
+```
+$ git clone https://github.com/khanhphantt/project
+```
+
+2. Change your directory to the cloned repo
+```
+$ cd Face-Mask-Detection
+```
+
+3. Create a Python virtual environment named '.venv' and activate it
+```
+$ python -m venv .venv
+$ .venv/bin/activate
+```
+
+4. Install the libraries required
+```
+$ pip install -r requirements.txt
+```
+
+## Working
+
+1. Train data:
+```
+$ python3
+```
+
+2. Detect face masks in an image:
+```
+$ python3
+```
+
+3. Detect face masks in real-time video (webcam):
+```
+$ python3
+```
+
+## Streamlit app
+
+Face Mask Detector webapp using Tensorflow & Streamlit:
+```
+$ streamlit run app.py
+```

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-modernist

app.py

@@ -15,11 +15,12 @@ from src.visualization import visualize_result
 print("[INFO] Loaded recognition model")
 
 
-def do_ocr(inp):
-    print(type(inp))
+def do_ocr(image):
+    print(type(image))
     # img = cv2.imread(inp, cv2.IMREAD_GRAYSCALE)
     # image = cv2.imread(inp, cv2.IMREAD_GRAYSCALE)
-    image = cv2.cvtColor(inp, cv2.COLOR_BGR2GRAY)
+    visualized_image = image
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
     """
     # Recognize all text with boxes and scores
     cls = False: to improve the performance
@@ -36,7 +37,7 @@ def do_ocr(inp):
     print(f"OCR time: {time_ocr - time_start}")
     print(f"post time: {time.time() - time_ocr}")
     time_total = time.time() - time_start
-    img_box, img_text = visualize_result(result, inp)
+    img_box, img_text = visualize_result(result, visualized_image)
     return img_text, img_box, time_total
 
 

ocr.py

@@ -0,0 +1,54 @@
+import time
+from os import listdir
+from os.path import (
+    isfile,
+    join,
+)
+
+import cv2
+
+from src.postprocessing import postprocess_result
+from src.settings import (
+    OCR_JA,
+    OCR_ML,
+)
+from src.visualization import visualize_result
+
+path = "data/"
+
+
+def paddleOCR(path):
+    """
+    perform ocr
+    args:
+        path(array): path to input folder
+    return(str):
+        text in markdown format
+    """
+    imgs = [f for f in listdir(path) if isfile(join(path, f))]
+    for img_file in imgs:
+        if not img_file.endswith("JA_0.png"):
+            continue
+
+        img_path = join(path, img_file)
+        out_path = join(path, "output", img_file)
+        print(f"Process {img_path}")
+        image = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
+        """
+        # Recognize all text with boxes and scores
+        cls = False: to improve the performance
+            recognize text only from -90 to 90 degree
+        bboxes = OCR.ocr(image_crop, cls=False, det=True, rec=False)
+        result = recognize_text(image_crop, bboxes)
+        """
+        time_start = time.time()
+        result = OCR_JA.ocr(image, cls=True, det=True, rec=True)
+        time_ocr = time.time()
+        result = postprocess_result(image, result, OCR_ML)
+        print(f"OCR time: {time_ocr - time_start}")
+        print(f"post time: {time.time() - time_ocr}")
+        visualize_result(result, img_path, out_path)
+
+
+if __name__ == "__main__":
+    paddleOCR(path)

pyproject.toml

@@ -0,0 +1,21 @@
+[tool.black]
+line-length = 79
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.idea
+  | \.pytest_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
+
+[tool.isort]
+profile = "black"
+force_grid_wrap=2
+multi_line_output=3

requirements.txt

@@ -0,0 +1,2 @@
+paddlepaddle
+paddleocr

src/postprocessing.py

@@ -0,0 +1,47 @@
+from src.settings import RECOGNITION_THRESHOLD
+from src.utilities import crop_image
+
+
+def postprocess_result(image, result, OCR):
+    """
+    Post-processing steps to improve the results
+    args:
+        image(Image|array): RGB image
+        result(list): boxes with shape(N, 4, 2), text and score
+    return(Image|array):
+        updated result
+    """
+    new_result = []
+    for line in result[0]:
+        if line[1][1] < RECOGNITION_THRESHOLD:
+            """
+            boxes = line[0], txts = line[1][0], scores = line[1][1]
+            """
+            line[1] = recognize_text_by_multilanguage(image, line, OCR)
+        new_result.append(line)
+
+    return [new_result]
+
+
+def recognize_text_by_multilanguage(image, line, OCR):
+    """
+    Do recognition again on the text having low recognition score.
+    args:
+        image(Image|array): RGB image
+        result(list): boxes with shape(N, 4, 2), text and score
+    return(Image|array):
+        updated result
+    """
+    box = line[0]
+    txt = line[1][0]
+    score = line[1][1]
+    cropped_image = crop_image(image, box)
+    result = OCR.ocr(cropped_image, cls=True, det=False, rec=True)
+    if result[0][0][1] > score:
+        print(f"[{score}]{txt} -----> [{result[0][0][1]}]{result[0][0][0]}")
+        txt = result[0][0][0]
+        score = result[0][0][1]
+    else:
+        print(f"[{score}]{txt} --X--> [{result[0][0][1]}]{result[0][0][0]}")
+
+    return (txt, score)

src/settings.py

@@ -0,0 +1,60 @@
+import logging
+import sys
+
+from paddleocr import PaddleOCR
+
+# check GPU
+# gpu = True if torch.cuda.is_available() else False
+
+RECOGNITION_THRESHOLD = 0.90
+FONTPATH = "fonts/NotoSerifJP-SemiBold.otf"
+""" Models list
+# Detection:
+detection_ch_PP-OCRv3: Original lightweight model,
+    supporting Chinese, English, multilingual text detection
+detection_ml_PP-OCRv3: Original lightweight detection model,
+    supporting English, multilingual text detection
+
+# Recognition:
+recognition_ch_PP-OCRv3: [New] Original lightweight model,
+    supporting Chinese, English, multilingual text recognition.
+    No dict.
+recognition_japan_PP-OCRv3_rec: Lightweight model for Japanese recognition
+    Need dict: ppocr/utils/dict/japan_dict.txt
+
+"""
+
+
+OCR_JA = PaddleOCR(
+    det_model_dir="models/detection/ch_PP-OCRv4_server_det/",
+    rec_model_dir="models/recognition/japan_PP-OCRv4_rec/",
+    rec_char_dict_path="models/char_dict/japan_dict.txt",
+    cls_model_dir="models/cls/ch_ppocr_mobile_v2.0_cls",
+    use_angle_cls=True,
+)
+
+OCR_ML = PaddleOCR(
+    det_model_dir="models/detection/ch_PP-OCRv4_server_det/",
+    rec_model_dir="models/recognition/ch_PP-OCRv4_rec/",
+    # rec_char_dict_path="models/char_dict/japan_dict.txt",
+    cls_model_dir="models/cls/ch_ppocr_mobile_v2.0_cls",
+    use_angle_cls=True,
+)
+
+# ocr = PaddleOCR(use_angle_cls=True, lang='japan')
+
+"""Define logging"""
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+# logger_value = logging.getLogger("Value")
+# logger_value.disabled()
+# logger_date = logging.getLogger("Date")
+# logger_date.disable()
+"""
+PaddleOCR logger
+ppstrcuture//table/predict_table.py
+dt_boxes num: line 81
+rec_res num: line 93
+tools/infer/predict_system.py
+dt_boxes num: line 71
+rec_res num: line 90
+"""

src/utilities.py

@@ -0,0 +1,14 @@
+def crop_image(image, bbox):
+    """
+    crop the text area (got from text detection) to recognize
+    args:
+        image(Image|array): RGB image
+        bbox(list): boxes with shape(N, 4, 2)
+    return(Image|array):
+        cropped image
+    """
+    top = int(min(bbox[0][1], bbox[1][1]))
+    bot = int(max(bbox[2][1], bbox[3][1]))
+    left = int(min(bbox[0][0], bbox[3][0]))
+    right = int(max(bbox[1][0], bbox[2][0]))
+    return image[top:bot, left:right]

src/visualization.py

@@ -0,0 +1,271 @@
+import math
+import string
+
+import cv2
+import numpy as np
+from PIL import (
+    Image,
+    ImageDraw,
+    ImageFont,
+)
+
+from src.settings import FONTPATH
+
+
+def count_characters(str: str):
+    """
+    Count the number of Japanese characters,
+    a single English character and a single number
+    equal to half the length of Japanese characters.
+    args:
+        s(string): the input of string
+    return(int):
+        the number of Japanese characters
+    """
+
+    count_zh = count_pu = 0
+    s_len = len(str)
+    en_dg_count = 0
+    for c in str:
+        if c in string.ascii_letters or c.isdigit() or c.isspace():
+            en_dg_count += 1
+        elif c.isalpha():
+            count_zh += 1
+        else:
+            count_pu += 1
+    return s_len - math.ceil(en_dg_count / 2)
+
+
+def create_blank_img(img_h, img_w):
+    """
+    create new blank img
+    args:
+        img_h(int): the height of blank img
+        img_w(int): the width of blank img
+    return(Image|array):
+        blank image
+    """
+    blank_img = np.ones(shape=[img_h, img_w], dtype=np.int8) * 255
+    blank_img[:, img_w - 1 :] = 0
+    blank_img = Image.fromarray(blank_img).convert("RGB")
+    draw_txt = ImageDraw.Draw(blank_img)
+    return blank_img, draw_txt
+
+
+def text_visual(
+    texts,
+    scores,
+    img_h=400,
+    img_w=600,
+    threshold=0.0,
+    font_path=FONTPATH,
+):
+    """
+    create new img with recognized text
+    args:
+        texts(list): the text will be draw
+        scores(list|None): corresponding score of each txt
+        img_h(int): the height of blank img
+        img_w(int): the width of blank img
+        font_path: the path of font which is used to draw text
+    return(Image|array): image with recognized text
+    """
+    if scores is not None:
+        assert len(texts) == len(
+            scores,
+        ), "The number of txts and corresponding scores must match"
+
+    blank_img, draw_txt = create_blank_img()
+
+    font_size = 20
+    txt_color = (0, 0, 0)
+    font = ImageFont.truetype(font_path, font_size, encoding="utf-8")
+
+    gap = font_size + 5
+    txt_img_list = []
+    count, index = 1, 0
+    for idx, txt in enumerate(texts):
+        index += 1
+        if scores[idx] < threshold or math.isnan(scores[idx]):
+            index -= 1
+            continue
+        first_line = True
+        while count_characters(txt) >= img_w // font_size - 4:
+            tmp = txt
+            txt = tmp[: img_w // font_size - 4]
+            if first_line:
+                new_txt = str(index) + ": " + txt
+                first_line = False
+            else:
+                new_txt = "    " + txt
+            draw_txt.text((0, gap * count), new_txt, txt_color, font=font)
+            txt = tmp[img_w // font_size - 4 :]
+            if count >= img_h // gap - 1:
+                txt_img_list.append(np.array(blank_img))
+                blank_img, draw_txt = create_blank_img()
+                count = 0
+            count += 1
+        if first_line:
+            new_txt = str(index) + ": " + txt + "   " + "%.3f" % (scores[idx])
+        else:
+            new_txt = "  " + txt + "  " + "%.3f" % (scores[idx])
+        draw_txt.text((0, gap * count), new_txt, txt_color, font=font)
+        # whether add new blank img or not
+        if count >= img_h // gap - 1 and idx + 1 < len(texts):
+            txt_img_list.append(np.array(blank_img))
+            blank_img, draw_txt = create_blank_img()
+            count = 0
+        count += 1
+    txt_img_list.append(np.array(blank_img))
+    if len(txt_img_list) == 1:
+        blank_img = np.array(txt_img_list[0])
+    else:
+        blank_img = np.concatenate(txt_img_list, axis=1)
+    return np.array(blank_img)
+
+
+def resize_img(img, input_size=600):
+    """
+    resize img and limit the longest side of the image to input_size
+    args:
+        img(np.array): original image
+        input_size(int): new size of the longest side of the image
+    return(Image|array):
+        a new-size image
+    """
+    img = np.array(img)
+    im_shape = img.shape
+    im_size_max = np.max(im_shape[0:2])
+    im_scale = float(input_size) / float(im_size_max)
+    img = cv2.resize(img, None, None, fx=im_scale, fy=im_scale)
+    return img
+
+
+def draw_ocr(
+    image,
+    boxes,
+    txts=None,
+    scores=None,
+    drop_score=0.0,
+    font_path=FONTPATH,
+):
+    """
+    Visualize the results of OCR detection and recognition
+    args:
+        image(Image|array): RGB image
+        boxes(list): boxes with shape(N, 4, 2)
+        txts(list): the texts
+        scores(list): txxs corresponding scores
+        drop_score(float): only scores > drop_threshold will be visualized
+        font_path: the path of font which is used to draw text
+    return(Image|array):
+        the visualized img
+    """
+    if scores is None:
+        scores = [1] * len(boxes)
+    box_num = len(boxes)
+    for i in range(box_num):
+        if scores is not None and (
+            scores[i] < drop_score or math.isnan(scores[i])
+        ):
+            continue
+        box = np.reshape(np.array(boxes[i]), [-1, 1, 2]).astype(np.int64)
+        image = cv2.polylines(np.array(image), [box], True, (255, 0, 0), 2)
+    if txts is not None:
+        img = np.array(image)
+        txt_img = text_visual(
+            txts,
+            scores,
+            img_h=img.shape[0],
+            img_w=600,
+            threshold=drop_score,
+            font_path=font_path,
+        )
+        img = np.concatenate([np.array(img), np.array(txt_img)], axis=1)
+        return img
+    return image
+
+
+def draw_ocr_2(img, results):
+    """
+    Visualize the results of OCR detection and recognition
+    args:
+        image(Image|array): RGB image
+        results(list): boxes with shape(N, 4, 2), texts and scores
+    return(Image|array):
+        the visualized img
+    """
+    img = np.asarray(img)
+    # img_box = np.ones((img.shape[0], img.shape[1], 3), np.uint8) * 255
+    img_text = np.ones((img.shape[0], img.shape[1], 3), np.uint8) * 255
+
+    for line in results:
+        text = line[1][0]
+        # score = line[1][1]
+
+        top = int(min(line[0][0][1], line[0][1][1]))
+        # bottom = int(max(line[0][2][1], line[0][3][1]))
+        left = int(min(line[0][0][0], line[0][3][0]))
+        right = int(max(line[0][1][0], line[0][2][0]))
+
+        text_size = int((right - left) / len(text))
+        color = (
+            np.random.randint(0, 255),
+            np.random.randint(0, 255),
+            np.random.randint(0, 255),
+        )
+
+        box = np.reshape(np.array(line[0]), [-1, 1, 2]).astype(np.int64)
+        img = cv2.polylines(np.array(img), [box], True, color, 2)
+
+        img_text = place_text(img_text, text, (left, top), text_size, color)
+
+    return img, img_text
+
+
+def place_text(img, text, top_left_point, text_size, text_color):
+    """
+    Put text into image
+    args:
+        img(Image|array): RGB image
+        text(list): text to be put
+        top_left_point(array): top-left point to start the text
+        text_color(tuple): text color in ()
+    return(Image|array):
+        the visualized img
+    """
+    font = ImageFont.truetype(FONTPATH, text_size, encoding="utf-8")
+    img_pil = Image.fromarray(img)
+    draw = ImageDraw.Draw(img_pil)
+    draw.text(top_left_point, text, fill=text_color, font=font)
+    return np.array(img_pil)
+
+
+def visualize_result(result, image):
+    """
+    make visualization in image foramt
+    args:
+        result(array): RGB image
+        img_path(str): path to input image
+        out_path(str): path to output image
+    return(Image|array):
+        the visualized img
+    """
+    result = result[0]
+    # image = Image.open(img_path).convert("RGB")
+    image = Image.fromarray(image)
+    """
+    boxes = [line[0] for line in result]
+    txts = [line[1][0] for line in result]
+    scores = [line[1][1] for line in result]
+    """
+    img_boxes, img_text = draw_ocr_2(image, result)
+    if isinstance(image, str):
+        img_combination = np.concatenate(
+            [np.array(img_boxes), np.array(img_text)],
+            axis=1,
+        )
+        img_combination = Image.fromarray(img_combination)
+        img_text.save(image + "_out.jpg")
+
+    return img_boxes, img_text