Upload 18 files

.gitattributes

@@ -1,5 +1,35 @@
-models/charrec.pt filter=lfs diff=lfs merge=lfs -text
-models/large_LP_YOLOm_best.pt filter=lfs diff=lfs merge=lfs -text
-models/yolov8n_lp_det.pt filter=lfs diff=lfs merge=lfs -text
-models/yolov8n_lpchar_det.pt filter=lfs diff=lfs merge=lfs -text
-models/yolov8n.pt filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,5 +1,7 @@
 import gradio as gr
 import model1 as m1
+import model2 as m2
+
 cars = []
 lps = []
 lp_texts = []
@@ -12,6 +14,20 @@ def model1(image):
     counter = 0
     return cars[0], lps[0], lp_texts[0]
 
+# this is the main function that passes the images to the model 1
+def model2(image):    
+    global lps, lp_texts, counter
+    (lps, lp_texts) = m2.run([image])
+    counter = 0
+    return lps[0], lp_texts[0]
+
+# this is the main function that passes the images to the model 1
+def model3(image):    
+    global cars, lps, lp_texts, counter
+    (cars, lps, lp_texts) = m3.run([image])
+    counter = 0
+    return cars[0], lps[0], lp_texts[0]
+
 # function to go to next detected car licence plate
 def next_img():
     global counter
@@ -26,6 +42,21 @@ def prev_img():
     index =  int(counter % len(cars))
     return cars[index], lps[index], lp_texts[index]
 
+
+# function to go to next detected licence plate
+def next_img_lp():
+    global counter
+    counter += 1
+    index =  int(counter % len(lps))
+    return lps[index], lp_texts[index]
+
+# function to go to prev detected  licence plate
+def prev_img_lp():
+    global counter
+    counter -= 1
+    index =  int(counter % len(lps))
+    return lps[index], lp_texts[index]
+
 # this code is responcible for the front end part of the page
 with gr.Blocks() as demo:
     gr.Markdown("## ANPR Project")
@@ -47,5 +78,24 @@ with gr.Blocks() as demo:
         submit.click(model1, inputs=[img], outputs=[car, lp, lp_text])
         next.click(next_img, outputs=[car, lp, lp_text])
         prev.click(prev_img, outputs=[car, lp, lp_text])
+    
+    with gr.Tab("Model 2"):
+        gr.Markdown("Using 2 different ML models")
+        gr.Markdown("YOLOv8m for car dection + easy ocr for text detection")
+        gr.Markdown("YOLOv8m for car dection is trained on a large dataset of 25K training images")
+        img2 = gr.Image(label="Input")
+        submit2 = gr.Button(value="submit")
+        
+        with gr.Row():       
+            lp2 = gr.Image(label="Licence Plate")
+            lp_text2 = gr.Text(label="Plate Number")    
+            
+        with gr.Row():
+            next2 = gr.Button(value="next")
+            prev2 = gr.Button(value="prev")
+        
+        submit2.click(model2, inputs=[img2], outputs=[lp2, lp_text2])
+        next2.click(next_img_lp, outputs=[lp2, lp_text2])
+        prev2.click(prev_img_lp, outputs=[lp2, lp_text2])
 
 demo.launch(share=False)

model1.py

@@ -1,10 +1,12 @@
 from ultralytics import YOLO
-import easyocr
 import numpy as np
+from transformers import TrOCRProcessor, VisionEncoderDecoderModel
 
 car_detection = YOLO("models/yolov8n.pt")
 lp_detection = YOLO("models/yolov8n_lp_det.pt")
-reader = easyocr.Reader(['en'])
+
+processor = TrOCRProcessor.from_pretrained('models/processor')
+model = VisionEncoderDecoderModel.from_pretrained('models/model')
 
 # char_dect = YOLO("models/yolov8n_lpchar_det.pt")
 # char_rec = torch.load("models/charrec.pt", map_location="cpu")
@@ -54,14 +56,16 @@ def detect_lp_text(inputs):
     # iterating through each licence plate
     for input in inputs:
         # finding the number/text in licence plate
-        result = reader.readtext(input)
+        pixel_values = processor(input, return_tensors="pt").pixel_values
+        generated_ids = model.generate(pixel_values)
+        generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
 
         # if no text is found in the licence plate, then adding a default text not found
-        if len(result) == 0:
+        if len(generated_text) == 0:
             plate_number.append("not found")
         else:
             # adding the licence plate number to a list
-            plate_number.append(result[0][1])
+            plate_number.append(generated_text)
             
     return plate_number
 

model2.py

@@ -0,0 +1,81 @@
+from model1 import np, YOLO, processor, model
+
+lp_detection = YOLO("models/yolov8n_lp_det.pt")
+
+# processor = TrOCRProcessor.from_pretrained('models/processor')
+# model = VisionEncoderDecoderModel.from_pretrained('models/model')
+
+# set special tokens used for creating the decoder_input_ids from the labels
+model.config.decoder_start_token_id = processor.tokenizer.cls_token_id
+model.config.pad_token_id = processor.tokenizer.pad_token_id
+# make sure vocab size is set correctly
+model.config.vocab_size = model.config.decoder.vocab_size
+
+# set beam search parameters
+model.config.eos_token_id = processor.tokenizer.sep_token_id
+model.config.max_length = 20
+model.config.early_stopping = True
+model.config.no_repeat_ngram_size = 3
+model.config.length_penalty = 2.0
+model.config.num_beams = 4
+
+
+# function to detect licence plates in the given car images
+def detect_lp(inputs):
+    lps = []
+    # running the license plate detection model with 50% confidence threshold
+    lp_results = lp_detection.predict(source=inputs, conf=0.5, verbose=False)
+    # iterating through each output (num of outputs will be same as num of inputs)
+    for lp_result in lp_results:
+        # finding the bounding boxes of the license plate detected
+        lp_boxes = lp_result.boxes.xyxy.tolist()    
+        # iterating through each license plate detected
+        for lp_box in lp_boxes:
+            # cropping license plate  image from the car image
+            lp = lp_result.orig_img[int(lp_box[1]):int(lp_box[3]), int(lp_box[0]):int(lp_box[2])]
+            lps.append(lp)
+            # breaking as we only want to detect one licence plate per car
+            break
+        
+        # if no licence plate is detected then we are adding a black image 
+        if len(lp_boxes) == 0:
+            lps.append(np.zeros((100,100,3), np.uint8))
+            
+    return lps
+
+# function to detect licence plate number in the given licence plate images
+def detect_lp_text(inputs):
+    plate_number = []
+    # iterating through each licence plate
+    for input in inputs:
+        # finding the number/text in licence plate
+        pixel_values = processor(input, return_tensors="pt").pixel_values
+        generated_ids = model.generate(pixel_values)
+        generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+
+
+        # if no text is found in the licence plate, then adding a default text not found
+        if len(generated_text) == 0:
+            plate_number.append("not found")
+        else:
+            # adding the licence plate number to a list
+            plate_number.append(generated_text)
+            
+    return plate_number
+
+def run(inputs):
+        
+    # for future, to handle multiple inputs
+    # currently using just one input
+    inputs = inputs[0]
+    
+    # detecting licence plates from the input images
+    # returns licence plate images, if it cant find a license plate a black image is returned
+    lps = detect_lp(inputs)
+    
+    # detecting licence plate number from licence plate images
+    # returns text from the licence plate images, if none is detected "not found" text is returned
+    lp_text = detect_lp_text(lps)
+    
+    return lps, lp_text
+

model3.py

@@ -0,0 +1,143 @@
+from model1 import reader, np, YOLO, car_detection, lp_detection
+import torch
+from PIL import Image
+import cv2
+from torchvision import transforms
+
+char_dect = YOLO("models/yolov8n_lpchar_det.pt")
+char_rec = torch.load("models/charrec.pt", map_location="cpu")
+
+# function to detect cars in the given image
+def detect_cars(inputs):
+    cars = []
+    # running the cars detection model with 50% confidence threshold
+    car_results = car_detection.predict(source=inputs, classes=[2], conf=0.5, verbose=False)
+    # iterating through each output (num of outputs will be same as num of inputs)
+    for car_result in car_results:       
+        # finding the bounding boxes of the cars detected
+        boxes = car_result.boxes.xyxy.tolist()    
+        # iterating through each car detected
+        for box in boxes:
+            # cropping car image from the input image
+            car = car_result.orig_img[int(box[1]):int(box[3]), int(box[0]):int(box[2])]
+            cars.append(car)            
+    return cars
+
+# function to detect licence plates in the given car images
+def detect_lp(inputs):
+    lps = []
+    # running the license plate detection model with 50% confidence threshold
+    lp_results = lp_detection.predict(source=inputs, conf=0.5, verbose=False)
+    # iterating through each output (num of outputs will be same as num of inputs)
+    for lp_result in lp_results:
+        # finding the bounding boxes of the license plate detected
+        lp_boxes = lp_result.boxes.xyxy.tolist()    
+        # iterating through each license plate detected
+        for lp_box in lp_boxes:
+            # cropping license plate  image from the car image
+            lp = lp_result.orig_img[int(lp_box[1]):int(lp_box[3]), int(lp_box[0]):int(lp_box[2])]
+            lps.append(lp)
+            # breaking as we only want to detect one licence plate per car
+            break
+        
+        # if no licence plate is detected then we are adding a black image 
+        if len(lp_boxes) == 0:
+            lps.append(np.zeros((100,100,3), np.uint8))
+            
+    return lps
+
+# function to detect licence plates character in the given LP images
+def chars_lp_det(inputs):
+    vis_lp = []
+    chars = []
+    # running the license plate detection model with 50% confidence threshold
+    chars_results = char_dect.predict(source=inputs, conf=0.5, verbose=False)
+    # iterating through each output (num of outputs will be same as num of inputs)
+    for chars_result in chars_results:
+        # finding the bounding boxes of the license plate detected
+        chars_boxes = chars_result.boxes.xyxy.tolist()    
+        # iterating through each license plate detected
+        vis = chars_result.orig_img.copy()
+        c_list  =[]
+        for chars_box in chars_boxes:
+            # cropping license plate  image from the car image
+            cv2.rectangle(vis, (int(chars_box[0]),int(chars_box[1])), (int(chars_box[2]), int(chars_box[3])), (0,255,0), 1)
+            chrs = chars_result.orig_img[int(chars_box[1]):int(chars_box[3]), int(chars_box[0]):int(chars_box[2])]            
+            c_list.append(chrs)
+
+        chars.append(c_list)
+        vis_lp.append(vis)
+        # if no licence plate is detected then we are adding a black image 
+        if len(vis_lp) == 0:
+            vis_lp.append(np.zeros((100,100,3), np.uint8))
+            
+    return vis_lp, chars
+
+# function to detect licence plate number in the given licence plate images
+def detect_lp_text(inputs):
+    plate_number = []
+    # iterating through each licence plate
+    for input in inputs:
+        # finding the number/text in licence plate
+        result = reader.readtext(input)
+
+        # if no text is found in the licence plate, then adding a default text not found
+        if len(result) == 0:
+            plate_number.append("not found")
+        else:
+            # adding the licence plate number to a list
+            plate_number.append(result[0][1])
+            
+    return plate_number
+
+def rec_lp_char(inputs):
+    m = ['0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z']
+    transform = transforms.Compose([
+        transforms.Resize((224, 224)),
+        transforms.ToTensor(),
+    ])
+    lptexts = []
+    for input in inputs:
+        imgs = [transform(Image.fromarray(input[i])) for i in range(len(input))]
+        if len(imgs) <= 1:
+            lptexts.append("not found") 
+            continue
+        imgs = torch.stack(imgs)
+        output = char_rec(imgs)
+        preds = torch.argmax(output, dim=1).tolist()
+        lptext = ""
+        for pred in preds:
+            lptext += m[int(pred)]
+        lptexts.append(lptext)
+    return lptexts
+        
+
+def run(inputs):
+        
+    # for future, to handle multiple inputs
+    # currently using just one input
+    inputs = inputs[0]
+    
+    # detecting cars, this function returns all detected car images
+    cars = detect_cars(inputs)
+    
+    # if no car is detected black images are returned
+    if len(cars) == 0:
+        return [np.zeros((100,100,3), np.uint8)], [np.zeros((100,100,3), np.uint8)], "not found"
+    
+    # detecting licence plates from the car images
+    # returns licence plate images, if it cant find a license plate a black image is returned
+    lps = detect_lp(cars)
+    
+    vis_lp, chars_lp = chars_lp_det(lps)
+    
+    
+    lptexts = rec_lp_char(chars_lp)
+    
+    # detecting licence plate number from licence plate images
+    # returns text from the licence plate images, if none is detected "not found" text is returned
+    # lp_text = detect_lp_text(lps)
+        
+    
+    return cars, vis_lp, lptexts
+

models/model/config.json

@@ -0,0 +1,176 @@
+{
+  "_name_or_path": "microsoft/trocr-small-printed",
+  "architectures": [
+    "VisionEncoderDecoderModel"
+  ],
+  "decoder": {
+    "_name_or_path": "",
+    "activation_dropout": 0.0,
+    "activation_function": "relu",
+    "add_cross_attention": true,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 0,
+    "chunk_size_feed_forward": 0,
+    "classifier_dropout": 0.0,
+    "cross_attention_hidden_size": 384,
+    "d_model": 256,
+    "decoder_attention_heads": 8,
+    "decoder_ffn_dim": 1024,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 6,
+    "decoder_start_token_id": 2,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout": 0.1,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "init_std": 0.02,
+    "is_decoder": true,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layernorm_embedding": true,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 512,
+    "min_length": 0,
+    "model_type": "trocr",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 1,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "scale_embedding": true,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": false,
+    "use_learned_position_embeddings": true,
+    "vocab_size": 64044
+  },
+  "decoder_start_token_id": 0,
+  "early_stopping": true,
+  "encoder": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_probs_dropout_prob": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "encoder_stride": 16,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 384,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 384,
+    "initializer_range": 0.02,
+    "intermediate_size": 1536,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-12,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "deit",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 6,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 16,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qkv_bias": true,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  },
+  "eos_token_id": 2,
+  "is_encoder_decoder": true,
+  "length_penalty": 2.0,
+  "model_type": "vision-encoder-decoder",
+  "no_repeat_ngram_size": 3,
+  "num_beams": 4,
+  "pad_token_id": 1,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.0",
+  "vocab_size": 64044
+}

models/model/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 0,
+  "decoder_start_token_id": 0,
+  "early_stopping": true,
+  "eos_token_id": 2,
+  "length_penalty": 2.0,
+  "no_repeat_ngram_size": 3,
+  "num_beams": 4,
+  "pad_token_id": 1,
+  "transformers_version": "4.37.0",
+  "use_cache": false
+}

models/model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:819da9002f0817676c7db10c0ea64c16b17e3c71690be9b3cb1bff95280bee76
+size 246430696

models/processor/preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "crop_size": {
+    "height": 224,
+    "width": 224
+  },
+  "do_center_crop": false,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "DeiTImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "TrOCRProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 384,
+    "width": 384
+  }
+}

models/processor/sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f5e2fefcf793761a76a6bfb8ad35489f9c203b25557673284b6d032f41043f4
+size 1356293

models/processor/special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

models/processor/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "processor_class": "TrOCRProcessor",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}