Add Egyption ID information Extraction

.gitignore

@@ -0,0 +1,4 @@
+# Ignore all .log files
+*.pt
+*.pth
+models

app.py

@@ -0,0 +1,50 @@
+import gradio as gr
+from code.detection.recognize_id.detect_and_recognize_id import Recognize_ID
+from code.detection.detection import detection
+from code.recognization.recognization import TextRecognition
+import os
+
+# Define a dummy prediction function
+def predict_image(image):
+
+    # Recognize ID
+    rec_id = Recognize_ID()
+    id = rec_id.give_me_id_number(image)
+
+    # Detection
+    det = detection()
+    detection_list = det.full_pipeline(image)
+
+    result = ''
+    # Loop on all detected images and recognize them
+    recognizer = TextRecognition()
+    for line in detection_list[2:6]:
+        for word in line:
+            recognized_word = recognizer.recognize_image(word)
+            result = result + recognized_word + ' '
+        result += '\n'    
+    
+    # Add Id number 
+    result = result + id
+
+    return result
+
+# List of paths to your sample images
+current_dir = os.path.dirname(os.path.abspath(__file__))
+sample_images = [
+    os.path.join(current_dir , "samples/id_1.png" )
+]
+
+# Create the Gradio interface
+interface = gr.Interface(
+    fn=predict_image,  # Function to run    
+    inputs="image",    # Input type
+    outputs="text",    # Output type
+    title="Recognization",  
+    description="Upload an image",
+    examples=sample_images
+)
+
+# Launch the app
+interface.launch()
+

code/detection/detection.py

@@ -0,0 +1,77 @@
+from ultralytics import YOLO
+from glob import glob
+import matplotlib.pyplot as plt
+import cv2
+import os
+from PIL import Image
+from ultralytics.engine.results import Results
+import numpy as np
+
+
+class detection:
+
+    def __init__(self,model_path='detection.pt'):
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        model_path = os.path.join(current_dir , model_path )
+        self.model = YOLO(model_path)
+
+    def get_distance(self,res):
+        boxes = res[0].boxes.xywh.numpy()  # Convert to numpy array
+        # Sort primarily by Y (vertical), then X (horizontal) using lexsort
+        sorted_indices = np.lexsort((boxes[:, 0], boxes[:, 1]))
+        sorted_boxes = boxes[sorted_indices]
+        return sorted_boxes[:, 1], sorted_indices  # Return sorted Y values and indices
+
+    def handle_the_boxes(self,res, img, y_threshold=30):
+        distance_sorted, sorted_indices = self.get_distance(res)
+        PB = res[0].boxes.xyxy.numpy()[sorted_indices]  # Get boxes in sorted order
+        same_object = []
+        current_line = [PB[0]]
+
+        # Group boxes into lines using Y threshold
+        for i in range(1, len(PB)):
+            prev_y = current_line[-1][1]  # Use ymin from XYXY format
+            current_y = PB[i][1]
+            if abs(current_y - prev_y) > y_threshold:
+                # Sort line left-to-right before adding
+                current_line = sorted(current_line, key=lambda x: x[0] , reverse=True)
+                same_object.append(current_line)
+                current_line = [PB[i]]
+            else:
+                current_line.append(PB[i])
+
+        # Add the last line and sort it
+        if current_line:
+            current_line = sorted(current_line, key=lambda x: x[0])
+            same_object.append(current_line)
+
+        # Extract word images in final order
+        return [
+            [self.words_pixels(img, box) for box in line]
+            for line in same_object
+        ]
+
+    # Keep words_pixels as original
+    def words_pixels(self,img, xyxy):
+        xmin, ymin, xmax, ymax = xyxy.tolist()
+        return img[int(ymin):int(ymax)+1, int(xmin):int(xmax)+1]
+
+    def full_pipeline(self,image,show=False):
+
+        if isinstance(image, str):  # If the input is a file path
+            img  = cv2.imread(image)
+        elif isinstance(image, np.ndarray):  # If the input is a NumPy array
+            image = image
+            img = image
+    
+        res = self.model(image)
+
+        if show:
+            res[0].show()
+
+
+        return  self.handle_the_boxes(res , img)
+
+
+   
+    

code/detection/recognize_id/detect_and_recognize_id.py

@@ -0,0 +1,36 @@
+from ultralytics import YOLO
+from ultralytics.engine.results import Results
+import cv2
+import os
+import numpy as np
+
+class Recognize_ID:
+
+    def __init__(self,model_path='recognization_id.pt'):
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        model_path = os.path.join(current_dir , model_path )
+        self.model = YOLO(model=model_path )
+   
+    def give_me_id_number(self,image:str):
+        """
+        image_dir : input image directory
+        model : yolo model
+        """
+        if isinstance(image, str):  # If the input is a file path
+            current_dir = os.path.dirname(os.path.abspath(__file__))
+            image_path = os.path.join(current_dir , image )
+            img = cv2.imread(image_path)
+        elif isinstance(image, np.ndarray):  # If the input is a NumPy array
+            img = image
+
+        print(type(img))
+        res = self.model(img)
+        boxes = res[0].boxes.xywh[::,0].tolist()
+        classes = res[0].boxes.cls.tolist()
+        boxes_labels =[(int(key) , int(value)) for key , value in zip(boxes, classes)]
+        boxes_labels.sort()
+        national_id = "".join([str(i[1]) for  i in boxes_labels])
+
+        return national_id
+    

code/recognization/.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

code/recognization/augmentation.py

@@ -0,0 +1,134 @@
+"""
+Paper: "UTRNet: High-Resolution Urdu Text Recognition In Printed Documents" presented at ICDAR 2023
+Authors: Abdur Rahman, Arjun Ghosh, Chetan Arora
+GitHub Repository: https://github.com/abdur75648/UTRNet-High-Resolution-Urdu-Text-Recognition
+Project Website: https://abdur75648.github.io/UTRNet/
+Copyright (c) 2023-present: This work is licensed under the Creative Commons Attribution-NonCommercial
+4.0 International License (http://creativecommons.org/licenses/by-nc/4.0/)
+"""
+
+from functools import partial
+import random as rnd
+import imgaug.augmenters as iaa
+import numpy as np
+from PIL import ImageFilter, Image
+from timm.data import auto_augment
+
+_OP_CACHE = {}
+
+def _get_op(key, factory):
+    try:
+        op = _OP_CACHE[key]
+    except KeyError:
+        op = factory()
+        _OP_CACHE[key] = op
+    return op
+
+
+def _get_param(level, img, max_dim_factor, min_level=1):
+    max_level = max(min_level, max_dim_factor * max(img.size))
+    return round(min(level, max_level))
+
+def gaussian_blur(img, radius, **__):
+    radius = _get_param(radius, img, 0.02)
+    key = 'gaussian_blur_' + str(radius)
+    op = _get_op(key, lambda: ImageFilter.GaussianBlur(radius))
+    return img.filter(op)
+
+def motion_blur(img, k, **__):
+    k = _get_param(k, img, 0.08, 3) | 1  # bin to odd values
+    key = 'motion_blur_' + str(k)
+    op = _get_op(key, lambda: iaa.MotionBlur(k))
+    return Image.fromarray(op(image=np.asarray(img)))
+
+def gaussian_noise(img, scale, **_):
+    scale = _get_param(scale, img, 0.25) | 1  # bin to odd values
+    key = 'gaussian_noise_' + str(scale)
+    op = _get_op(key, lambda: iaa.AdditiveGaussianNoise(scale=scale))
+    return Image.fromarray(op(image=np.asarray(img)))
+
+def poisson_noise(img, lam, **_):
+    lam = _get_param(lam, img, 0.2) | 1  # bin to odd values
+    key = 'poisson_noise_' + str(lam)
+    op = _get_op(key, lambda: iaa.AdditivePoissonNoise(lam))
+    return Image.fromarray(op(image=np.asarray(img)))
+
+def salt_and_pepper_noise(image, prob=0.05):
+    if prob <= 0:
+        return image
+    arr = np.asarray(image)
+    original_dtype = arr.dtype
+    intensity_levels = 2 ** (arr[0, 0].nbytes * 8)
+    min_intensity = 0
+    max_intensity = intensity_levels - 1
+    random_image_arr = np.random.choice([min_intensity, 1, np.nan], p=[prob / 2, 1 - prob, prob / 2], size=arr.shape)
+    salt_and_peppered_arr = arr.astype(np.float) * random_image_arr
+    salt_and_peppered_arr = np.nan_to_num(salt_and_peppered_arr, nan=max_intensity).astype(original_dtype)
+    return Image.fromarray(salt_and_peppered_arr)
+
+def random_border_crop(image):
+    img_width,img_height = image.size
+    crop_left = int(img_width * rnd.uniform(0.0, 0.025))
+    crop_top = int(img_height * rnd.uniform(0.0, 0.075))            
+    crop_right = int(img_width * rnd.uniform(0.975, 1.0))
+    crop_bottom = int(img_height * rnd.uniform(0.925, 1.0))
+    final_image = image.crop((crop_left, crop_top, crop_right, crop_bottom))
+    return final_image
+
+def random_resize(image):
+    size = image.size
+    new_size = [rnd.randint(int(0.5*size[0]), int(1.5*size[0])), rnd.randint(int(0.5*size[1]), int(1.5*size[1]))]
+    reduce_factor = rnd.randint(1,4)
+    new_size = tuple([int(x/reduce_factor) for x in new_size])
+    final_image = image.resize(new_size)
+    return final_image
+
+def _level_to_arg(level, _hparams, max):
+    level = max * level / auto_augment._LEVEL_DENOM
+    return level,
+
+_RAND_TRANSFORMS = [
+    'AutoContrast',
+    'Equalize',
+    'Invert',
+    # 'Rotate',
+    'Posterize',
+    'Solarize',
+    'SolarizeAdd',
+    'Color',
+    'Contrast',
+    'Brightness',
+    'Sharpness',
+    'ShearX',
+]
+#_RAND_TRANSFORMS.remove('SharpnessIncreasing')  # remove, interferes with *blur ops
+_RAND_TRANSFORMS.extend([
+    'GaussianBlur',
+    'GaussianNoise',
+    'PoissonNoise'
+])
+auto_augment.LEVEL_TO_ARG.update({
+    'GaussianBlur': partial(_level_to_arg, max=4),
+    'MotionBlur': partial(_level_to_arg, max=20),
+    'GaussianNoise': partial(_level_to_arg, max=0.1 * 255),
+    'PoissonNoise': partial(_level_to_arg, max=40)
+})
+auto_augment.NAME_TO_OP.update({
+    'GaussianBlur': gaussian_blur,
+    'MotionBlur': motion_blur,
+    'GaussianNoise': gaussian_noise,
+    'PoissonNoise': poisson_noise
+})
+
+def rand_augment_transform(magnitude=5, num_layers=3):
+    # These are tuned for magnitude=5, which means that effective magnitudes are half of these values.
+    hparams = {
+        'img_mean':128,
+        # 'rotate_deg': 5,
+        'shear_x_pct': 0.9,
+        'shear_y_pct': 0.0,
+    }
+    ra_ops = auto_augment.rand_augment_ops(magnitude, hparams, transforms=_RAND_TRANSFORMS)
+    # Supply weights to disable replacement in random selection (i.e. avoid applying the same op twice)
+    choice_weights = [1. / len(ra_ops) for _ in range(len(ra_ops))]
+    return auto_augment.RandAugment(ra_ops, num_layers, choice_weights)

code/recognization/config.py

@@ -0,0 +1,30 @@
+class Config:
+    FeatureExtraction = 'HRNet'  # or any other feature extraction method
+    SequenceModeling = 'DBiLSTM'  # or any other sequential model
+    Prediction = 'CTC'  # or 'Attn'
+    input_channel = 1  # e.g., RGB image has 3 channels
+    output_channel = 32  # Adjust based on your architecture
+    hidden_size = 256  # Adjust based on your architecture
+    num_class = 182  # Number of output classes
+    device = 'cpu'  # or 'cuda' for GPU
+    batch_max_length = 8  # Maximum sequence length for prediction
+    # Adam optimizer
+    adam = False
+    lr = 0.1
+    batch_size = 4
+    beta1 = 0.9
+    workers = 4
+    num_epochs = 5
+    rho = 0.95
+    eps = 1e-8
+
+    imgH = 32
+    imgW = 400
+    train_data = 'result/train/'  # path to train data
+    valid_data = 'result/validate/'  # path to validation data
+    saved_model = 'model/'
+
+    character =''
+    rgb = False
+    grad_clip = 5
+    

code/recognization/custom_test.py

@@ -0,0 +1,235 @@
+"""
+Paper: "UTRNet: High-Resolution Urdu Text Recognition In Printed Documents" presented at ICDAR 2023
+Authors: Abdur Rahman, Arjun Ghosh, Chetan Arora
+GitHub Repository: https://github.com/abdur75648/UTRNet-High-Resolution-Urdu-Text-Recognition
+Project Website: https://abdur75648.github.io/UTRNet/
+Copyright (c) 2023-present: This work is licensed under the Creative Commons Attribution-NonCommercial
+4.0 International License (http://creativecommons.org/licenses/by-nc/4.0/)
+"""
+
+import os,shutil
+import time
+import argparse
+import random
+import numpy as np
+import matplotlib.pyplot as plt
+from datetime import datetime
+import pytz
+
+import torch
+import torch.utils.data
+import torch.nn.functional as F
+from tqdm import tqdm
+from nltk.metrics.distance import edit_distance
+
+from utils import CTCLabelConverter, AttnLabelConverter, Averager, Logger
+from dataset import hierarchical_dataset, AlignCollate
+from model import Model
+
+def validation(model, criterion, evaluation_loader, converter, opt, device):
+    """ validation or evaluation """
+    eval_arr = []
+    sum_len_gt = 0
+
+    n_correct = 0
+    
+    norm_ED = 0
+    length_of_data = 0
+    infer_time = 0
+    valid_loss_avg = Averager()
+
+    for i, (image_tensors, labels) in enumerate(tqdm(evaluation_loader)):
+        batch_size = image_tensors.size(0)
+        length_of_data = length_of_data + batch_size
+        image = image_tensors.to(device)
+        # For max length prediction
+        length_for_pred = torch.IntTensor([opt.batch_max_length] * batch_size).to(device)
+        text_for_pred = torch.LongTensor(batch_size, opt.batch_max_length + 1).fill_(0).to(device)
+
+        text_for_loss, length_for_loss = converter.encode(labels, batch_max_length=opt.batch_max_length)
+
+        start_time = time.time()
+        if 'CTC' in opt.Prediction:
+            preds = model(image)
+            forward_time = time.time() - start_time
+            preds_size = torch.IntTensor([preds.size(1)] * batch_size)
+            cost = criterion(preds.log_softmax(2).permute(1, 0, 2), text_for_loss, preds_size, length_for_loss)
+            _, preds_index = preds.max(2)
+            preds_str = converter.decode(preds_index.data, preds_size.data)
+        else:
+            preds = model(image, text=text_for_pred, is_train=False)
+            forward_time = time.time() - start_time
+
+            preds = preds[:, :text_for_loss.shape[1] - 1, :].to(device)
+            target = text_for_loss[:, 1:].to(device)  # without [GO] Symbol
+            cost = criterion(preds.contiguous().view(-1, preds.shape[-1]), target.contiguous().view(-1))
+            _, preds_index = preds.max(2)
+            preds_str = converter.decode(preds_index, length_for_pred)
+            labels = converter.decode(text_for_loss[:, 1:], length_for_loss)
+
+        infer_time += forward_time
+        valid_loss_avg.add(cost)
+
+        # calculate accuracy & confidence score
+        preds_prob = F.softmax(preds, dim=2)
+        preds_max_prob, _ = preds_prob.max(dim=2)
+        confidence_score_list = []
+        for gt, pred, pred_max_prob in zip(labels, preds_str, preds_max_prob):
+            if 'Attn' in opt.Prediction:
+                gt = gt[:gt.find('[s]')]
+                pred_EOS = pred.find('[s]')
+                pred = pred[:pred_EOS]  # prune after "end of sentence" token ([s])
+                pred_max_prob = pred_max_prob[:pred_EOS]
+
+            if pred == gt:
+                n_correct += 1
+
+            # ICDAR2019 Normalized Edit Distance
+            if len(gt) == 0 or len(pred) == 0:
+                ED = 0
+            elif len(gt) > len(pred):
+                ED = 1 - edit_distance(pred, gt) / len(gt)
+            else:
+                ED = 1 - edit_distance(pred, gt) / len(pred)
+
+            eval_arr.append([gt,pred,ED])
+
+            sum_len_gt += len(gt)
+            norm_ED += (ED*len(gt))
+
+            # calculate confidence score (= multiply of pred_max_prob)
+            try:
+                confidence_score = pred_max_prob.cumprod(dim=0)[-1]
+            except:
+                confidence_score = 0  # for empty pred case, when prune after "end of sentence" token ([s])
+            confidence_score_list.append(confidence_score)
+            # print(pred, gt, pred==gt, confidence_score)
+
+    accuracy = n_correct / float(length_of_data) * 100
+    norm_ED = norm_ED / float(sum_len_gt)
+
+    return valid_loss_avg.val(), accuracy, norm_ED, eval_arr
+
+
+def test(opt, device):
+    opt.device = device
+    os.makedirs("test_outputs", exist_ok=True)
+    datetime_now = str(datetime.now(pytz.timezone('Asia/Kolkata')).strftime("%Y-%m-%d_%H-%M-%S"))
+    logger = Logger(f'test_outputs/{datetime_now}.txt')
+    """ model configuration """
+    if 'CTC' in opt.Prediction:
+        converter = CTCLabelConverter(opt.character)
+    else:
+        converter = AttnLabelConverter(opt.character)
+    opt.num_class = len(converter.character)
+
+    if opt.rgb:
+        opt.input_channel = 3
+    model = Model(opt)
+    logger.log('model input parameters', opt.imgH, opt.imgW, opt.input_channel, opt.output_channel,
+          opt.hidden_size, opt.num_class, opt.batch_max_length, opt.FeatureExtraction,
+          opt.SequenceModeling, opt.Prediction)
+    model = model.to(device)
+
+    # load model
+    model.load_state_dict(torch.load(opt.saved_model, map_location=device))
+    logger.log('Loaded pretrained model from %s' % opt.saved_model)
+    # logger.log(model)
+
+    """ setup loss """
+    if 'CTC' in opt.Prediction:
+        criterion = torch.nn.CTCLoss(zero_infinity=True).to(device)
+    else:
+        criterion = torch.nn.CrossEntropyLoss(ignore_index=0).to(device)  # ignore [GO] token = ignore index 0
+
+    """ evaluation """
+    model.eval()
+    with torch.no_grad():
+        AlignCollate_evaluation = AlignCollate(imgH=opt.imgH, imgW=opt.imgW)#, keep_ratio_with_pad=opt.PAD)
+        eval_data, eval_data_log = hierarchical_dataset(root=opt.eval_data, opt=opt, rand_aug=False)
+        logger.log(eval_data_log)
+        evaluation_loader = torch.utils.data.DataLoader(
+            eval_data, batch_size=opt.batch_size,
+            shuffle=False,
+            num_workers=int(opt.workers),
+            collate_fn=AlignCollate_evaluation, pin_memory=True)
+        _, accuracy, norm_ED, eval_arr = validation( model, criterion, evaluation_loader, converter, opt,device)
+    logger.log("="*20)
+    logger.log(f'Accuracy : {accuracy:0.4f}\n')
+    logger.log(f'Norm_ED : {norm_ED:0.4f}\n')
+    logger.log("="*20)
+    
+    if opt.visualize:
+        logger.log("Threshold - ", opt.threshold)
+        logger.log("ED","\t","gt","\t","pred")
+        arr = []
+        for gt,pred,ED in eval_arr:
+            ED = ED*100.0
+            arr.append(ED)
+            if ED<=(opt.threshold):
+                logger.log(ED,"\t",gt,"\t",pred)
+        plt.hist(arr, edgecolor="red")
+        plt.savefig('test_outputs/'+str(datetime_now)+".png")
+        plt.close()
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--visualize', action='store_true', help='for visualization of bad samples')
+    parser.add_argument('--threshold', type=float, help='Save samples below this threshold in txt file', default=50.0)
+    parser.add_argument('--eval_data', required=True, help='path to evaluation dataset')
+    parser.add_argument('--workers', type=int, help='number of data loading workers', default=4)
+    parser.add_argument('--batch_size', type=int, default=32, help='input batch size')
+    parser.add_argument('--saved_model', required=True, help="path to saved_model to evaluation")
+    """ Data processing """
+    parser.add_argument('--batch_max_length', type=int, default=100, help='maximum-label-length')
+    parser.add_argument('--imgH', type=int, default=32, help='the height of the input image')
+    parser.add_argument('--imgW', type=int, default=400, help='the width of the input image')
+    parser.add_argument('--rgb', action='store_true', help='use rgb input')
+    """ Model Architecture """
+    parser.add_argument('--FeatureExtraction', type=str, default="HRNet", #required=True,
+                        help='FeatureExtraction stage VGG|RCNN|ResNet|UNet|HRNet|Densenet|InceptionUnet|ResUnet|AttnUNet|UNet|VGG')
+    parser.add_argument('--SequenceModeling', type=str, default="DBiLSTM", #required=True,
+                        help='SequenceModeling stage LSTM|GRU|MDLSTM|BiLSTM|DBiLSTM')
+    parser.add_argument('--Prediction', type=str, default="CTC", #required=True,
+                        help='Prediction stage CTC|Attn')
+    parser.add_argument('--input_channel', type=int, default=1, help='the number of input channel of Feature extractor')
+    parser.add_argument('--output_channel', type=int, default=512, help='the number of output channel of Feature extractor')
+    parser.add_argument('--hidden_size', type=int, default=256, help='the size of the LSTM hidden state')
+    """ GPU Selection """
+    parser.add_argument('--device_id', type=str, default=None, help='cuda device ID')
+
+    opt = parser.parse_args()
+    if opt.FeatureExtraction == "HRNet":
+        opt.output_channel = 32
+    
+    # Fix random seeds for both numpy and pytorch
+    seed = 1111
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    np.random.seed(seed)
+    random.seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+    """ vocab / character number configuration """
+    file = open("UrduGlyphs.txt","r",encoding="utf-8")
+    content = file.readlines()
+    content = ''.join([str(elem).strip('\n') for elem in content])
+    opt.character = content+" "
+    
+    cuda_str = 'cuda'
+    if opt.device_id is not None:
+        cuda_str = f'cuda:{opt.device_id}'
+    device = torch.device(cuda_str if torch.cuda.is_available() else 'cpu')
+    print("Device : ", device)
+    
+    # opt.eval_data = "/DATA/parseq/val/"
+    # test(opt, device)
+    
+    # opt.eval_data = "/DATA/parseq/IIITH/lmdb_new/"
+    # test(opt, device)
+    
+    # opt.eval_data = "/DATA/public_datasets/UPTI/valid/"
+    # test(opt, device)
+    
+    test(opt, device)