# -*- coding: utf-8 -*- """MartheDeployment_Doors_fasterRCNN.ipynb Automatically generated by Colab. Original file is located at https://colab.research.google.com/drive/1kgEtpfNt0jxSwPRhOzODIC6P_prg-c4L ## Libraries """ # from google.colab.patches import cv2_imshow import cv2 import numpy as np import pandas as pd import statistics from statistics import mode from PIL import Image # pip install PyPDF2 # pip install PyMuPDF # pip install pip install PyMuPDF==1.19.0 import io # !pip install pypdfium2 import pypdfium2 as pdfium import fitz # PyMuPDF import os #drive.mount("/content/drive", force_remount=True) import torch from torchvision.models.detection.faster_rcnn import FastRCNNPredictor from PIL import Image, ImageDraw import torchvision.transforms.functional as F import matplotlib.pyplot as plt import google_sheet_Legend """# updated for (fullpath, pdf_name) """ def convert2pillow(path): pdf = pdfium.PdfDocument(path) page = pdf.get_page(0) pil_image = page.render().to_pil() return pil_image import torch import torchvision from torchvision.models.detection.faster_rcnn import FastRCNNPredictor # Function to get the model def get_model(num_classes): # Load a pre-trained Faster R-CNN model with a ResNet-50-FPN backbone model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True) # Get the number of input features for the classifier in_features = model.roi_heads.box_predictor.cls_score.in_features # Replace the pre-trained head with a new one for our number of classes model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes) return model '''def ev_model(img, model, device, threshold): image_tensor = F.to_tensor(img).unsqueeze(0) image_tensor = image_tensor.to(device) model.eval() with torch.no_grad(): predictions = model(image_tensor) single_boxes = [] double_boxes = [] for element in range(len(predictions[0]['boxes'])): score = predictions[0]['scores'][element].item() if score > threshold: if predictions[0]['labels'][element].item() == 1: single_boxes.append(predictions[0]['boxes'][element].tolist()) else: double_boxes.append(predictions[0]['boxes'][element].tolist()) return single_boxes, double_boxes''' def ev_model(img, model, device, threshold): image_tensor = F.to_tensor(img).unsqueeze(0) image_tensor = image_tensor.to(device) model.eval() with torch.no_grad(): predictions = model(image_tensor) doors_info = [] for element in range(len(predictions[0]['boxes'])): score = predictions[0]['scores'][element].item() if score > threshold: box = predictions[0]['boxes'][element].tolist() label = predictions[0]['labels'][element].item() doors_info.append((box,label)) return doors_info def distance(point1, point2): x1, y1 = point1 x2, y2 = point2 return np.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2) def calculate_midpoint(p1, p2): x1, y1 = p1 x2, y2 = p2 # Calculate the midpoint xm = int((x1 + x2) / 2) ym = int((y1 + y2) / 2) return (xm, ym) def get_door_info(doors_info): width_pixels = [] lines = [] sanda = [] line_midpoint = [] for door_inf in doors_info: xmin, ymin, xmax, ymax = door_inf[0] #horz_bottom if door_inf[1] == 2: #for drawing point_st = (int(xmin), int(ymax) + 5) point_end = (int(xmax),int(ymax) + 5) lines.append((point_st,point_end)) sanda_st = (int(xmin), int(ymax)) sand_end = (int(xmax),int(ymax)) sanda.append((sanda_st, sand_end)) line_midpoint.append(calculate_midpoint(sanda_st,sand_end)) #for calculation width = distance((xmin,ymax), (xmax,ymax)) width_pixels.append(width) #horz_upper if door_inf[1] == 3: #for drawing point_st = (int(xmin),int(ymin) -5) point_end = (int(xmax),int(ymin) - 5) lines.append((point_st,point_end)) sanda_st = (int(xmin),int(ymin)) sand_end = (int(xmax),int(ymin)) sanda.append((sanda_st, sand_end)) line_midpoint.append(calculate_midpoint(sanda_st,sand_end)) #for calculation width = distance((xmin,ymin), (xmax,ymin)) width_pixels.append(width) #vert_right if door_inf[1] == 4: #for drawing point_st = (int(xmax) + 5,int(ymin)) point_end = (int(xmax) + 5,int(ymax)) lines.append((point_st,point_end)) sanda_st = (int(xmax), int(ymin)) sand_end = (int(xmax), int(ymax)) sanda.append((sanda_st, sand_end)) line_midpoint.append(calculate_midpoint(sanda_st,sand_end)) #for calculation width = distance((xmax,ymin), (xmax,ymax)) width_pixels.append(width) #vert_left if door_inf[1] == 5: #for drawing point_st = (int(xmin) -5,int(ymin)) point_end = (int(xmin) -5,int(ymax)) lines.append((point_st,point_end)) sanda_st = (int(xmin),int(ymin)) sand_end = (int(xmin),int(ymax)) sanda.append((sanda_st, sand_end)) line_midpoint.append(calculate_midpoint(sanda_st,sand_end)) #for calculation width = distance((xmin,ymin), (xmin,ymax)) width_pixels.append(width) return width_pixels, lines, sanda, line_midpoint def pxl2meter(width_pixels, ratio): real_width = [] for width in width_pixels: real_width.append(round(width*ratio, 2)) return real_width def width_as_char(real_width): char_width = [] for width in real_width: char_width.append(f"{width}m") return char_width def add_annotations_to_pdf(image, pdf_name, lines, sanda, char_width, line_midpoint): image_width, image_height = image.size # Create a new PDF document pdf_document = fitz.open() # Add a new page to the document with the same dimensions as the image page = pdf_document.new_page(width=image_width, height=image_height) # Insert the image into the PDF page image_stream = io.BytesIO() image.save(image_stream, format="PNG") page.insert_image(page.rect, stream=image_stream.getvalue()) #Annotation for drawin lines as in the markups for i in range(len(line_midpoint)): x, y = line_midpoint[i] text = char_width[i] rect = fitz.Rect(x, y, x + 200, y + 50) # Adjust the width and height as needed annot = page.add_freetext_annot(rect, text, fontsize=10, fontname="helv", text_color=(1,0,0)) annot.update() #Annotation for drawin lines as in the markups for i in range(len(lines)): annot = page.add_line_annot(lines[i][0], lines[i][1]) annot = page.add_line_annot(sanda[i][0], lines[i][0]) annot = page.add_line_annot(sanda[i][1], lines[i][1]) annot.set_border(width=2, dashes=None) # Optional border styling annot.set_colors(stroke=(1, 0, 0)) # Set the line color to red annot.update() output_pdf_path = pdf_name+"_annotated.pdf" # Save the PDF with annotations return pdf_document '''def calculate_width(bbox): #if looking right or left, width < height bbox_width = bbox[2] - bbox[0] bbox_height = bbox[3] - bbox[1] if bbox_width > bbox_height: door_width = bbox_width else: door_width = bbox_height return door_width ''' '''def width_annotations(bbox, ratio): lines = [] width = [] for box in bbox: door_width = calculate_width(box) door_width = round(door_width*ratio, 2) x1,y1,x2,y2 = box b_width = x2 - x1 b_height = y2 - y1 if b_width > b_height: lines.append(((x1, y1), (x2, y1))) x = (x1+x2)/2 y = (y1+y1)/2 width.append(((x,y),door_width)) else: lines.append(((x1, y1), (x1, y2))) x = (x1+x1)/2 y = (y1+y2)/2 width.append(((x,y), door_width)) return lines, width''' '''def create_width_annotations(width_info): annotations = [] for i in range(len(width_info)): annotations.append(((width_info[i][0][0]),(width_info[i][0][1]),f"{width_info[i][1]} m")) return annotations''' '''def calculate_midpoint(top_left, bottom_right): x1, y1 = top_left x2, y2 = bottom_right # Calculate the midpoint xm = int((x1 + x2) / 2) ym = int((y1 + y2) / 2) return (xm, ym)''' '''def mid_points_bbox(bbox): midpoint = [] for i in range(len(bbox)): x1 = int(bbox[i][0]) y1 = int(bbox[i][1]) x2 = int(bbox[i][2]) y2 = int(bbox[i][3]) top_left_corner = (x1, y1) bottom_right_corner = (x2, y2) midpoint.append(calculate_midpoint(top_left_corner, bottom_right_corner)) return midpoint''' '''def create_annotations(door_kind, midpoints): door = door_kind annotations = [] for i in range(len(midpoints)): annotations.append((midpoints[i][0],midpoints[i][1], door)) return annotations def add_annotations_to_pdf(image, pdf_name, annotation_s, annotation_d,width_ann_single, width_ann_double,line_single,line_double): image_width, image_height = image.size # Create a new PDF document pdf_document = fitz.open() # Add a new page to the document with the same dimensions as the image page = pdf_document.new_page(width=image_width, height=image_height) # Insert the image into the PDF page image_stream = io.BytesIO() image.save(image_stream, format="PNG") page.insert_image(page.rect, stream=image_stream.getvalue()) # Add annotations for annotation in annotation_s: x, y, text = annotation # Create an annotation (sticky note) annot = page.add_text_annot(fitz.Point(x, y), text) annot.set_border(width=0.2, dashes=(1, 2)) # Optional border styling annot.set_colors(stroke=(1, 0, 0), fill=None) # Set the stroke color to red annot.update() for annotation in annotation_d: x, y, text = annotation # Create an annotation (sticky note) annot = page.add_text_annot(fitz.Point(x, y), text) annot.set_border(width=0.2, dashes=(1, 2)) # Optional border styling annot.set_colors(stroke=(0, 1, 0), fill=None) # Set the stroke color to red annot.update() #Annotations for width measurement (marra single we marra double) for annotation in width_ann_single: x, y, text = annotation rect = fitz.Rect(x, y, x + 200, y + 50) # Adjust the width and height as needed annot = page.add_freetext_annot(rect, text, fontsize=10, fontname="helv", text_color=(1,0,0)) annot.update() for annotation in width_ann_double: x, y, text = annotation rect = fitz.Rect(x, y, x + 200, y + 50) # Adjust the width and height as needed annot = page.add_freetext_annot(rect, text, fontsize=10, fontname="helv", text_color=(1,0,0)) annot.update() #Annotation kind of the line drawings (marra single we marra double) for line in line_single: start_point, end_point = line annot = page.add_line_annot(start_point, end_point) annot.set_border(width=2, dashes=None) # Optional border styling annot.set_colors(stroke=(1, 0, 0)) # Set the line color to red annot.update() for line in line_double: start_point, end_point = line annot = page.add_line_annot(start_point, end_point) annot.set_border(width=2, dashes=None) # Optional border styling annot.set_colors(stroke=(1, 0, 0)) # Set the line color to red annot.update() output_pdf_path = pdf_name+"_annotated.pdf" # Save the PDF with annotations return pdf_document # pdf_document.save(output_pdf_path) # pdf_document.close() ''' def main_run(pdf_fullpath, weights_path, pdf_name,pdfpath,ratio): ####pdf_fullpath here is the data and not the path img_pillow = convert2pillow(pdf_fullpath) #new_image = img_pillow.resize((2384, 1684)) # Specify the number of classes (including the background) num_classes = 10 # Ensure this matches the saved model's number of classes # Load the model with the specified number of classes model = get_model(num_classes) # Load the saved model's state dictionary with map_location to handle CPU device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') try: model.load_state_dict(torch.load(weights_path, map_location=device), strict=False) except RuntimeError as e: print(f"Error loading model state_dict: {e}") return # Set the model to evaluation mode model.eval() # Move the model to the appropriate device model.to(device) # START INFERENCE #sbox, dbox = ev_model(img_pillow, model, device, 0.6) #Dataframe for Doors count #doors_count = {'Type': ['Single Doors', 'Double Doors'], 'Quantity': [len(sbox), len(dbox)]} #df_doors = pd.DataFrame(doors_count) #single_midpoint = mid_points_bbox(sbox) #double_midpoint = mid_points_bbox(dbox) #Kind Annotations #single_annotations = create_annotations("single door", single_midpoint) #double_annotations = create_annotations("double door", double_midpoint) #Lines Annotations #line_single, width_signle = width_annotations(sbox, 1) #line_double, width_double = width_annotations(dbox, 1) #Width Annotations #width_single_ann = create_width_annotations(width_signle) #width_double_ann = create_width_annotations(width_double) # add_annotations_to_pdf(new_image, pdf_name, single_annotations, double_annotations) #NEW doors_info = ev_model(img_pillow, model, device, 0.9) width_pixels, lines, sanda, line_midpoint = get_door_info(doors_info) real_width = pxl2meter(width_pixels, ratio) char_width = width_as_char(real_width) pdf_document = add_annotations_to_pdf(img_pillow, plan, lines, sanda, char_width, line_midpoint) #pdf_document=add_annotations_to_pdf(img_pillow, pdf_name, single_annotations, double_annotations,width_single_ann,width_double_ann,line_single,line_double) page=pdf_document[0] pix = page.get_pixmap() # render page to an image pl=Image.frombytes('RGB', [pix.width,pix.height],pix.samples) img=np.array(pl) annotatedimg = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) df_doors = df_doors.fillna(' ') gc,spreadsheet_service,spreadsheetId, spreadsheet_url , namepathArr=google_sheet_Legend.legendGoogleSheets(df_doors , pdf_name,pdfpath) list1=pd.DataFrame(columns=['content', 'id', 'subject']) for page in pdf_document: for annot in page.annots(): list1.loc[len(list1)] =annot.info # modify this return return annotatedimg, pdf_document , spreadsheet_url, list1 , df_doors # model_path = '/content/drive/MyDrive/separated_classes.pth' # #pdf_name = data # for i in range(len(fullpath)): # main_run(fullpath[i], model_path, pdf_name[i])