source/PaddleOCR/ppocr/data/imaug/random_crop_data.py

# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This code is refer from:
https://github.com/WenmuZhou/DBNet.pytorch/blob/master/data_loader/modules/random_crop_data.py
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
import cv2
import random

from paddle import get_device

from shapely.geometry import Polygon, box as shapely_box
from shapely import intersection


def is_poly_in_rect(poly, x, y, w, h):
    poly = np.array(poly)
    if poly[:, 0].min() < x or poly[:, 0].max() > x + w:
        return False
    if poly[:, 1].min() < y or poly[:, 1].max() > y + h:
        return False
    return True


def is_poly_outside_rect(poly, x, y, w, h):
    poly = np.array(poly)
    if poly[:, 0].max() < x or poly[:, 0].min() > x + w:
        return True
    if poly[:, 1].max() < y or poly[:, 1].min() > y + h:
        return True
    return False


def split_regions(axis):
    regions = []
    min_axis = 0
    for i in range(1, axis.shape[0]):
        if axis[i] != axis[i - 1] + 1:
            region = axis[min_axis:i]
            min_axis = i
            regions.append(region)
    return regions


def random_select(axis, max_size):
    xx = np.random.choice(axis, size=2)
    xmin = np.min(xx)
    xmax = np.max(xx)
    xmin = np.clip(xmin, 0, max_size - 1)
    xmax = np.clip(xmax, 0, max_size - 1)
    return xmin, xmax


def region_wise_random_select(regions, max_size):
    selected_index = list(np.random.choice(len(regions), 2))
    selected_values = []
    for index in selected_index:
        axis = regions[index]
        xx = int(np.random.choice(axis, size=1))
        selected_values.append(xx)
    xmin = min(selected_values)
    xmax = max(selected_values)
    return xmin, xmax


def get_min_rotated_rect_side(poly):
    """
    Compute the minimum side length of the minimum bounding rotated rectangle of a polygon.
    """
    poly = np.array(poly).astype(np.float32)
    if len(poly) < 3:
        return 0
    rect = cv2.minAreaRect(poly)
    width, height = rect[1]
    return min(width, height)


def get_min_quad_side(quad):
    """
    Compute the minimum side length of a quadrilateral.
    """
    if len(quad) != 4:
        return 0
    quad = np.array(quad)
    sides = []
    for i in range(4):
        side = np.linalg.norm(quad[i] - quad[(i + 1) % 4])
        sides.append(side)
    return min(sides) if sides else 0


def clip_poly_to_rect(poly, x, y, w, h):
    """
    Clip a polygon to a rectangular region and return the clipped quadrilateral.

    Args:
        poly: Original polygon vertices [[x1, y1], [x2, y2], ...]
        x, y, w, h: Position and size of the clipping rectangle

    Returns:
        Clipped quadrilateral vertices, or None if the result is invalid.
    """
    try:
        # Create polygon and clipping rectangle
        poly_shape = Polygon(poly)
        crop_rect = shapely_box(x, y, x + w, y + h)

        # Compute intersection
        clipped = intersection(poly_shape, crop_rect)

        # No intersection or empty
        if clipped.is_empty:
            return None

        # Get intersection coordinates
        if clipped.geom_type == "Polygon":
            coords = list(clipped.exterior.coords[:-1])  # Remove duplicate last point
        elif clipped.geom_type == "MultiPolygon":
            # Multiple polygons, select the largest by area
            largest = max(clipped.geoms, key=lambda p: p.area)
            coords = list(largest.exterior.coords[:-1])
        elif clipped.geom_type == "GeometryCollection":
            # Extract polygons from geometry collection
            polygons = [g for g in clipped.geoms if g.geom_type == "Polygon"]
            if not polygons:
                return None
            largest = max(polygons, key=lambda p: p.area)
            coords = list(largest.exterior.coords[:-1])
        else:
            return None

        # Less than 3 points, invalid
        if len(coords) <= 3:
            return None

        # Convert to numpy array
        coords = np.array(coords)

        # Exactly 4 points, return directly
        if len(coords) == 4:
            return coords

        # More than 4 points, use Douglas-Peucker to simplify to quadrilateral
        # Output points are a subset of original coords (no out-of-bounds), IoU~0.99
        if len(coords) > 4:
            poly_cv = coords.reshape(-1, 1, 2).astype(np.float32)
            peri = cv2.arcLength(poly_cv, True)
            if peri < 1e-6:
                return None
            lo, hi = 0.0, 0.5
            best = None
            for _ in range(50):
                mid = (lo + hi) / 2
                approx = cv2.approxPolyDP(poly_cv, mid * peri, True)
                if len(approx) <= 4:
                    best = approx
                    hi = mid
                else:
                    lo = mid
            if best is not None and len(best) >= 3:
                return best.reshape(-1, 2)
            return None

        return coords
    except Exception as e:
        return None


class RandomCrop(object):
    def __init__(
        self,
        size=(640, 640),
        max_tries=10,
        min_crop_side_ratio=0.1,
        keep_ratio=True,
        **kwargs,
    ):
        self.size = size
        self.max_tries = max_tries
        self.min_crop_side_ratio = min_crop_side_ratio
        self.keep_ratio = keep_ratio

    def __call__(self, data):
        img = data["image"]
        text_polys = data["polys"]
        ignore_tags = data["ignore_tags"]
        texts = data["texts"]

        # Separate care and ignore text boxes
        care_indices = [i for i, tag in enumerate(ignore_tags) if not tag]
        all_care_polys = [text_polys[i] for i in care_indices]

        h, w, _ = img.shape

        # If no valid text boxes, still need to resize and pad the image
        if len(all_care_polys) == 0:
            # Use entire image as crop region, skip crop loop
            crop_x, crop_y, crop_w, crop_h = 0, 0, w, h
            valid_care_data = []
        else:
            # Pre-compute char heights (min side of min bounding rotated rect) for all care boxes
            char_heights = np.array(
                [get_min_rotated_rect_side(poly) for poly in all_care_polys]
            )

            # Try to find a suitable crop region
            valid_care_data = []
            for attempt in range(self.max_tries):
                # Randomly determine crop region width and height
                crop_w_min = min(int(w * self.min_crop_side_ratio), self.size[0])
                crop_w_max = int(self.size[0] * 3)
                crop_w = (
                    w
                    if crop_w_min >= crop_w_max
                    else min(random.randint(crop_w_min, crop_w_max), w)
                )

                crop_h_min = min(int(h * self.min_crop_side_ratio), self.size[1])
                crop_h_max = int(self.size[1] * 3)
                crop_h = (
                    h
                    if crop_h_min >= crop_h_max
                    else min(random.randint(crop_h_min, crop_h_max), h)
                )

                # Randomly determine crop region start position
                crop_x = 0 if crop_w >= w else random.randint(0, w - crop_w)
                crop_y = 0 if crop_h >= h else random.randint(0, h - crop_h)

                # Check each care text box, clip and validate simultaneously (computed once)
                valid_care_data = []
                for care_idx, (poly, char_height) in enumerate(
                    zip(all_care_polys, char_heights)
                ):
                    # Quick check: completely outside, skip
                    if is_poly_outside_rect(poly, crop_x, crop_y, crop_w, crop_h):
                        continue

                    # Completely inside, no clipping needed
                    if is_poly_in_rect(poly, crop_x, crop_y, crop_w, crop_h):
                        valid_care_data.append(
                            (care_idx, None)
                        )  # None means no clipping needed
                        continue

                    # Truncated box, clip and validate (executed once)
                    clipped_poly = clip_poly_to_rect(
                        poly, crop_x, crop_y, crop_w, crop_h
                    )
                    if clipped_poly is None:
                        continue

                    # Validate clipped polygon - area check
                    clipped_area = cv2.contourArea(clipped_poly.astype(np.float32))
                    if clipped_area < 80:
                        continue

                    # Validate - char height check
                    clipped_char_height = get_min_rotated_rect_side(clipped_poly)
                    if clipped_char_height < char_height * 0.35:
                        continue

                    # Validate - min side length check (quadrilaterals only)
                    if len(clipped_poly) == 4:
                        min_side = get_min_quad_side(clipped_poly)
                        if min_side < char_height * 0.35:
                            continue

                    # All validations passed, save the clipped polygon
                    valid_care_data.append((care_idx, clipped_poly))

                # At least one valid text box, use this crop region
                if len(valid_care_data) >= 1:
                    break
            else:
                # All attempts failed, use original region
                crop_x, crop_y, crop_w, crop_h = 0, 0, w, h
                valid_care_data = [(i, None) for i in range(len(all_care_polys))]

        # Crop and scale image
        # Only shrink when crop region is larger than target size, otherwise just pad
        need_resize = crop_w > self.size[0] or crop_h > self.size[1]

        if need_resize:
            # Crop region larger than target, need to shrink
            scale_w = self.size[0] / crop_w
            scale_h = self.size[1] / crop_h
            scale = min(scale_w, scale_h)
            h_resized = int(crop_h * scale)
            w_resized = int(crop_w * scale)
        else:
            # Crop region smaller than or equal to target, no upscaling
            scale = 1.0
            h_resized = crop_h
            w_resized = crop_w

        if self.keep_ratio:
            # Random padding - compute padding size
            pad_h = self.size[1] - h_resized
            pad_w = self.size[0] - w_resized

            # Randomly distribute padding to each side
            pad_top = random.randint(0, pad_h) if pad_h > 0 else 0
            pad_left = random.randint(0, pad_w) if pad_w > 0 else 0

            # Resize cropped image (only when shrinking is needed)
            cropped_img = img[crop_y : crop_y + crop_h, crop_x : crop_x + crop_w]
            if need_resize:
                resized_img = cv2.resize(cropped_img, (w_resized, h_resized))
            else:
                resized_img = cropped_img

            # Create padded image
            padimg = np.zeros((self.size[1], self.size[0], img.shape[2]), img.dtype)
            padimg[pad_top : pad_top + h_resized, pad_left : pad_left + w_resized] = (
                resized_img
            )
            img = padimg
        else:
            img = cv2.resize(
                img[crop_y : crop_y + crop_h, crop_x : crop_x + crop_w],
                tuple(self.size),
            )
            pad_left = 0
            pad_top = 0

        # Build fast lookup set of valid care indices
        valid_care_indices_set = {care_idx for care_idx, _ in valid_care_data}

        # Build mapping from care_idx to clipped polygon
        care_idx_to_clipped = {
            care_idx: clipped for care_idx, clipped in valid_care_data
        }

        # Build output text box list
        text_polys_crop = []
        ignore_tags_crop = []
        texts_crop = []

        for all_idx, (poly, text, tag) in enumerate(
            zip(text_polys, texts, ignore_tags)
        ):
            if tag:
                # Ignore text box, simple processing
                if not is_poly_outside_rect(poly, crop_x, crop_y, crop_w, crop_h):
                    adjusted_poly = (poly - (crop_x, crop_y)) * scale + (
                        pad_left,
                        pad_top,
                    )
                    adjusted_poly[:, 0] = np.clip(adjusted_poly[:, 0], 0, self.size[0])
                    adjusted_poly[:, 1] = np.clip(adjusted_poly[:, 1], 0, self.size[1])
                    text_polys_crop.append(adjusted_poly.tolist())
                    ignore_tags_crop.append(tag)
                    texts_crop.append(text)
            else:
                # Care text box, find corresponding care_idx
                try:
                    care_idx = care_indices.index(all_idx)
                except ValueError:
                    continue

                # Check if this is a valid text box
                if care_idx not in valid_care_indices_set:
                    continue

                # Get clipped polygon (if any)
                clipped_poly = care_idx_to_clipped[care_idx]

                if clipped_poly is None:
                    # Completely inside, use original polygon
                    adjusted_poly = (poly - (crop_x, crop_y)) * scale + (
                        pad_left,
                        pad_top,
                    )
                else:
                    # Use clipped polygon
                    adjusted_poly = (clipped_poly - (crop_x, crop_y)) * scale + (
                        pad_left,
                        pad_top,
                    )

                text_polys_crop.append(adjusted_poly.tolist())
                ignore_tags_crop.append(tag)
                texts_crop.append(text)
        data["image"] = img
        # Pad polygons to uniform point count to avoid inhomogeneous array error
        if text_polys_crop:
            max_points = max(len(p) for p in text_polys_crop)
            for i, poly in enumerate(text_polys_crop):
                if len(poly) < max_points:
                    text_polys_crop[i] = poly + [poly[-1]] * (max_points - len(poly))
        data["polys"] = np.array(text_polys_crop, dtype=np.float32)
        data["ignore_tags"] = ignore_tags_crop
        data["texts"] = texts_crop
        return data


def crop_area(im, text_polys, min_crop_side_ratio, max_tries):
    h, w, _ = im.shape
    h_array = np.zeros(h, dtype=np.int32)
    w_array = np.zeros(w, dtype=np.int32)
    for points in text_polys:
        points = np.round(points, decimals=0).astype(np.int32)
        minx = np.min(points[:, 0])
        maxx = np.max(points[:, 0])
        w_array[minx:maxx] = 1
        miny = np.min(points[:, 1])
        maxy = np.max(points[:, 1])
        h_array[miny:maxy] = 1
    # ensure the cropped area not across a text
    h_axis = np.where(h_array == 0)[0]
    w_axis = np.where(w_array == 0)[0]

    if len(h_axis) == 0 or len(w_axis) == 0:
        return 0, 0, w, h

    h_regions = split_regions(h_axis)
    w_regions = split_regions(w_axis)

    for i in range(max_tries):
        if len(w_regions) > 1:
            xmin, xmax = region_wise_random_select(w_regions, w)
        else:
            xmin, xmax = random_select(w_axis, w)
        if len(h_regions) > 1:
            ymin, ymax = region_wise_random_select(h_regions, h)
        else:
            ymin, ymax = random_select(h_axis, h)

        if (
            xmax - xmin < min_crop_side_ratio * w
            or ymax - ymin < min_crop_side_ratio * h
        ):
            # area too small
            continue
        num_poly_in_rect = 0
        for poly in text_polys:
            if not is_poly_outside_rect(poly, xmin, ymin, xmax - xmin, ymax - ymin):
                num_poly_in_rect += 1
                break

        if num_poly_in_rect > 0:
            return xmin, ymin, xmax - xmin, ymax - ymin

    return 0, 0, w, h


class EastRandomCropData(object):
    def __init__(
        self,
        size=(640, 640),
        max_tries=10,
        min_crop_side_ratio=0.1,
        keep_ratio=True,
        **kwargs,
    ):
        self.size = size
        self.max_tries = max_tries
        self.min_crop_side_ratio = min_crop_side_ratio
        self.keep_ratio = keep_ratio

    def __call__(self, data):
        img = data["image"]
        text_polys = data["polys"]
        ignore_tags = data["ignore_tags"]
        texts = data["texts"]
        all_care_polys = [text_polys[i] for i, tag in enumerate(ignore_tags) if not tag]
        # 计算crop区域
        crop_x, crop_y, crop_w, crop_h = crop_area(
            img, all_care_polys, self.min_crop_side_ratio, self.max_tries
        )
        # crop 图片 保持比例填充
        scale_w = self.size[0] / crop_w
        scale_h = self.size[1] / crop_h
        scale = min(scale_w, scale_h)
        h = int(crop_h * scale)
        w = int(crop_w * scale)
        if self.keep_ratio:
            padimg = np.zeros((self.size[1], self.size[0], img.shape[2]), img.dtype)
            padimg[:h, :w] = cv2.resize(
                img[crop_y : crop_y + crop_h, crop_x : crop_x + crop_w], (w, h)
            )
            img = padimg
        else:
            img = cv2.resize(
                img[crop_y : crop_y + crop_h, crop_x : crop_x + crop_w],
                tuple(self.size),
            )
        # crop 文本框
        text_polys_crop = []
        ignore_tags_crop = []
        texts_crop = []
        for poly, text, tag in zip(text_polys, texts, ignore_tags):
            poly = ((poly - (crop_x, crop_y)) * scale).tolist()
            if not is_poly_outside_rect(poly, 0, 0, w, h):
                text_polys_crop.append(poly)
                ignore_tags_crop.append(tag)
                texts_crop.append(text)
        data["image"] = img
        data["polys"] = np.array(text_polys_crop)
        if "iluvatar_gpu" in get_device():
            data["polys"] = np.array(text_polys_crop).astype(np.float32)
        data["ignore_tags"] = ignore_tags_crop
        data["texts"] = texts_crop
        return data


class RandomCropImgMask(object):
    def __init__(self, size, main_key, crop_keys, p=3 / 8, **kwargs):
        self.size = size
        self.main_key = main_key
        self.crop_keys = crop_keys
        self.p = p

    def __call__(self, data):
        image = data["image"]

        h, w = image.shape[0:2]
        th, tw = self.size
        if w == tw and h == th:
            return data

        mask = data[self.main_key]
        if np.max(mask) > 0 and random.random() > self.p:
            # make sure to crop the text region
            tl = np.min(np.where(mask > 0), axis=1) - (th, tw)
            tl[tl < 0] = 0
            br = np.max(np.where(mask > 0), axis=1) - (th, tw)
            br[br < 0] = 0

            br[0] = min(br[0], h - th)
            br[1] = min(br[1], w - tw)

            i = random.randint(tl[0], br[0]) if tl[0] < br[0] else 0
            j = random.randint(tl[1], br[1]) if tl[1] < br[1] else 0
        else:
            i = random.randint(0, h - th) if h - th > 0 else 0
            j = random.randint(0, w - tw) if w - tw > 0 else 0

        # return i, j, th, tw
        for k in data:
            if k in self.crop_keys:
                if len(data[k].shape) == 3:
                    if np.argmin(data[k].shape) == 0:
                        img = data[k][:, i : i + th, j : j + tw]
                        if img.shape[1] != img.shape[2]:
                            a = 1
                    elif np.argmin(data[k].shape) == 2:
                        img = data[k][i : i + th, j : j + tw, :]
                        if img.shape[1] != img.shape[0]:
                            a = 1
                    else:
                        img = data[k]
                else:
                    img = data[k][i : i + th, j : j + tw]
                    if img.shape[0] != img.shape[1]:
                        a = 1
                data[k] = img
        return data