dxf__omar3_2.py

# -*- coding: utf-8 -*-wj
"""Version to be deployed of 3.2 Calculating area/perimeter

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1XPeCoTBgWSNBYZ3aMKBteP4YG3w4bORs
"""

# pip install ezdxf[draw]

# pip install --upgrade ezdxf

# pip install pymupdf #==1.22.5

# pip install PyPDF2

# pip install ezdxf scipy

"""## Imports"""

import numpy as np
import cv2
from matplotlib import pyplot as plt
import math
from PIL import Image  , ImageDraw, ImageFont , ImageColor
import fitz
import ezdxf as ez
import sys
from ezdxf import units
from ezdxf.math import OCS, Matrix44, Vec3
import ezdxf
import matplotlib.pyplot as plt
from matplotlib.patches import Polygon
from shapely.geometry import Polygon as ShapelyPolygon
from ezdxf.math import Vec2
import random
import pandas as pd
import google_sheet_Legend
import tsadropboxretrieval
from ezdxf import bbox

"""## Notes"""

#new approach to get width and height of dxf plan
'''
This portion is used to convert vertices read from dxf to pixels in order to accurately locate shapes in the image and pdf
  ratio :
  MeasuredMetric* PixelValue/ DxfMetric = MeasuredPixel
  PixelValue: get from pixel conversion code , second number in the bracker represents the perimeter
  DxfMetric: measured perimeter from foxit

  divide pixelvalue by dxfmetric, will give u a ratio , this is ur dxfratio


'''


"""PDF to image"""

def pdftoimg(datadoc):
  doc = fitz.open('pdf',datadoc)
  page=doc[0]
  pix = page.get_pixmap()  # render page to an image
  pl=Image.frombytes('RGB', [pix.width,pix.height],pix.samples)
  img=np.array(pl)
  img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
  return img


# Standard ISO paper sizes in inches
ISO_SIZES_INCHES = {
    "A0": (33.11, 46.81),
    "A1": (23.39, 33.11),
    "A2": (16.54, 23.39),
    "A3": (11.69, 16.54),
    "A4": (8.27, 11.69),
    "A5": (5.83, 8.27),
    "A6": (4.13, 5.83),
    "A7": (2.91, 4.13),
    "A8": (2.05, 2.91),
    "A9": (1.46, 2.05),
    "A10": (1.02, 1.46)
}

def get_paper_size_in_inches(width, height):
    """Find the closest matching paper size in inches."""
    for size, (w, h) in ISO_SIZES_INCHES.items():
        if (abs(w - width) < 0.1 and abs(h - height) < 0.1) or (abs(w - height) < 0.1 and abs(h - width) < 0.1):
            return size
    return "Unknown Size"

def analyze_pdf(datadoc):
    # Open the PDF file
    pdf_document = fitz.open('pdf',datadoc)

    # Iterate through pages and print their sizes
    for page_number in range(len(pdf_document)):
        page = pdf_document[page_number]
        rect = page.rect
        width_points, height_points = rect.width, rect.height

        # Convert points to inches
        width_inches, height_inches = width_points / 72, height_points / 72

        paper_size = get_paper_size_in_inches(width_inches, height_inches)

        print(f"Page {page_number + 1}: {width_inches:.2f} x {height_inches:.2f} inches ({paper_size})")

    pdf_document.close()
    return width_inches , height_inches , paper_size


def get_dxfSize(dxfpath):

  doc = ezdxf.readfile(dxfpath)
  msp = doc.modelspace()
  # Create a cache for bounding box calculations
  # Get the overall bounding box for all entities in the modelspace
  cache = bbox.Cache()
  overall_bbox = bbox.extents(msp, cache=cache)
  print("Overall Bounding Box:", overall_bbox)
  print(overall_bbox.extmin[0]+overall_bbox.extmax[0], overall_bbox.extmin[1]+overall_bbox.extmax[1])

  return overall_bbox.extmin[0]+overall_bbox.extmax[0], overall_bbox.extmin[1]+overall_bbox.extmax[1]



def switch_case(argument):
    switcher = {
      "A0": 1.27,
      "A1": 2.54,
      "A2": 5.08,
      "A3": 10.16,
      "A4": 20.32,
      "A5": 40.64,
      "A6": 81.28,
      "A7": 162.56,
      "A8": 325.12,
      "A9": 650.24,
      "A10": 1300.48
    }
    # Get the value from the dictionary; if not found, return a default value
    print("Final Ratio=",switcher.get(argument, 1))
    return switcher.get(argument, 1)




def RetriveRatio(datadoc,dxfpath):

  width,height,paper_size = analyze_pdf (datadoc)

  if(width > height ):
    bigger=width
  else:
    bigger=height

  width_dxf,height_dxf = get_dxfSize(dxfpath)

  if(width_dxf > height_dxf ):
    bigger_dxf=width_dxf
  else:
    bigger_dxf=height_dxf

  if(0.2 < bigger_dxf/bigger < 1.2):
    print("bigger_dxf/bigger",bigger/bigger_dxf)
    argument = paper_size
    FinalRatio=switch_case(argument)
  else:
    FinalRatio=1
  return FinalRatio


"""Flips image
DXF origin is at the bottom left while img origin is top left
"""

def flip(img):
  height, width = img.shape[:2]

  # Define the rotation angle (clockwise)
  angle = 180

  # Calculate the rotation matrix
  rotation_matrix = cv2.getRotationMatrix2D((width/2, height/2), angle, 1)

  # Rotate the image
  rotated_image = cv2.warpAffine(img, rotation_matrix, (width, height))
  flipped_horizontal = cv2.flip(rotated_image, 1)
  return flipped_horizontal

"""### Hatched areas"""

def get_hatched_areas(filename,FinalRatio):
      doc = ezdxf.readfile(filename)
      doc.header['$MEASUREMENT'] = 1
      msp = doc.modelspace()
      trial=0
      hatched_areas = []

      for entity in msp:

        if entity.dxftype() == 'HATCH':
            flag=0
            trial=0

            print(entity.dxftype())

            for path in entity.paths:
              if str(path.type)=='BoundaryPathType.POLYLINE':
                print('First type of Hatch')
                vertices = [(vertex[0]* (FinalRatio), vertex[1]* (FinalRatio))for vertex in path.vertices]
                if(len(vertices)>3):
                 poly = ShapelyPolygon(vertices)

                 minx, miny, maxx, maxy = poly.bounds

                 # Calculate the width and height of the bounding box
                 width = maxx - minx
                 height = maxy - miny



                 if (poly.area > 1.5 and (height > 0.7 and width > 0.7)):
                   area1 = round(poly.area,3)
                   perimeter = round (poly.length,3)
                   if trial==0:
                     hatched_areas.append([vertices,area1,perimeter])
                     trial=1
                   else:
                        for i in range(len(hatched_areas)):
                            if(area1 == hatched_areas[i][1]):
                              flag=1
              elif str(path.type) == 'BoundaryPathType.EDGE':
                print('Second type of Hatch')
              
                vert=[]
                flag=0
                flag2=0
                for edge in path.edges:
                  x,y=edge.start
                  x1,y1=edge.end
                  if(flag==0):
                    vert=[(x* (FinalRatio),y* (FinalRatio)),(x1* (FinalRatio),y1* (FinalRatio))]
                  else:
                      vert.append([x1* (FinalRatio),y1* (FinalRatio)])
                  flag=1
                poly = ShapelyPolygon(vert)
                minx, miny, maxx, maxy = poly.bounds

                 # Calculate the width and height of the bounding box
                width = maxx - minx
                height = maxy - miny

                if (poly.area > 1.5 and (height > 0.7 and width > 0.7)):
                   area1= round(poly.area,3)
                   perimeter = round (poly.length,3)
                   for i in range(len(hatched_areas)):
                      if(area1 == hatched_areas[i][1]):
                        flag2=1
                   if(flag2==0):
                       hatched_areas.append([vert,area1,perimeter])
              else:
                print(path.type)

        elif entity.dxftype() == 'SOLID':
            vertices = [entity.dxf.vtx0 * (FinalRatio), entity.dxf.vtx1* (FinalRatio), entity.dxf.vtx2* (FinalRatio), entity.dxf.vtx3* (FinalRatio)]
            poly = ShapelyPolygon(vertices)
            minx, miny, maxx, maxy = poly.bounds

            # Calculate the width and height of the bounding box
            width = maxx - minx
            height = maxy - miny

            if (poly.area > 1.5 and (height > 0.7 and width > 0.7)):
             hatched_areas.append([vertices,poly.area,poly.length])

        elif entity.dxftype() == 'LWPOLYLINE':

           vertices=[]
           lwpolyline = entity
           points = lwpolyline.get_points()
           flag=0

           for i in range(len(points)):
             vertices.append([points[i][0]* (FinalRatio),points[i][1]* (FinalRatio)])
           if(len(vertices)>3):

             if(vertices[0][0] == vertices[len(vertices)-1][0]  or vertices[0][1] == vertices[len(vertices)-1][1]):

               poly=ShapelyPolygon(vertices)
               minx, miny, maxx, maxy = poly.bounds

               # Calculate the width and height of the bounding box
               width = maxx - minx
               height = maxy - miny

               if (poly.area > 1.5 and (height > 0.7 and width > 0.7)):
                area1 = round(poly.area,3)
                perimeter = round (poly.length,3)
                for i in range(len(hatched_areas)):
                  if(area1 == hatched_areas[i][1]):
                     flag=1
                if(flag==0):
                   hatched_areas.append([vertices,area1,perimeter])


        elif entity.dxftype() == 'POLYLINE':

          flag=0
          vertices = [(v.dxf.location.x * (FinalRatio), v.dxf.location.y * (FinalRatio)) for v in entity.vertices]
          print('Vertices:', vertices)

          if(len(vertices)>3):

             if(vertices[0][0] == vertices[len(vertices)-1][0]  or vertices[0][1] == vertices[len(vertices)-1][1]):

               poly=ShapelyPolygon(vertices)
               minx, miny, maxx, maxy = poly.bounds

               # Calculate the width and height of the bounding box
               width = maxx - minx
               height = maxy - miny

               if (poly.area > 1.5 and (height > 0.7 and width > 0.7)):
                area1 = round(poly.area,3)
                perimeter = round (poly.length,3)
                for i in range(len(hatched_areas)):
                  if(area1 == hatched_areas[i][1]):
                     flag=1
                if(flag==0):
                   hatched_areas.append([vertices,area1,perimeter])

        elif entity.dxftype() == 'SPLINE':
          spline_entity = entity
          vertices = []
          control_points = spline_entity.control_points
          if(len(control_points)>3):
            for i in range(len(control_points)):
              vertices.append([control_points[i][0]* (FinalRatio),control_points[i][1]* (FinalRatio)])
            poly=ShapelyPolygon(vertices)

            minx, miny, maxx, maxy = poly.bounds

            # Calculate the width and height of the bounding box
            width = maxx - minx
            height = maxy - miny


            if (poly.area > 1.5 and (height > 0.7 and width > 0.7)):
                area1 = round(poly.area,3)
                perimeter = round (poly.length,3)
                hatched_areas.append([vertices,area1,perimeter])

      sorted_data = sorted(hatched_areas, key=lambda x: x[1])
      return sorted_data

"""### Rotate polygon"""

from math import sin, cos, radians

def rotate_point(point, angle,pdfrotation,width,height, center_point=(0, 0)):
    """Rotates a point around center_point(origin by default)
    Angle is in degrees.
    Rotation is counter-clockwise
    """
    angle_rad = radians(angle % 360)
    # Shift the point so that center_point becomes the origin
    new_point = (point[0] - center_point[0], point[1] - center_point[1])
    new_point = (new_point[0] * cos(angle_rad) - new_point[1] * sin(angle_rad),
                 new_point[0] * sin(angle_rad) + new_point[1] * cos(angle_rad))
    # Reverse the shifting we have done
    if pdfrotation!=0:

      new_point = (new_point[0]+width  + center_point[0], new_point[1]  + center_point[1]) #pdfsize[2] is the same as +width
    else:

      new_point = (new_point[0] + center_point[0], new_point[1]+ height  + center_point[1]) # pdfsize[3] is the same as +height
    # new_point = (new_point[0] + center_point[0], new_point[1] + center_point[1])
    return new_point


def rotate_polygon(polygon, angle, pdfrotation,width,height,center_point=(0, 0)):
    """Rotates the given polygon which consists of corners represented as (x,y)
    around center_point (origin by default)
    Rotation is counter-clockwise
    Angle is in degrees
    """
    rotated_polygon = []
    for corner in polygon:
        rotated_corner = rotate_point(corner, angle,pdfrotation,width,height, center_point)
        rotated_polygon.append(rotated_corner)
    return rotated_polygon

#create a dataframe containing color , count(how many times is this object found in the plan), area of 1 of these shapes, total area
#perimeter, totat perimeter, length, total length
#import pandas as pd
#SimilarAreaDictionary= pd.DataFrame(columns=['Guess','Color','Occurences','Area','Total Area','Perimeter','Total Perimeter','Length','Total Length','R','G','B'])
#loop 3la hatched areas and count the occurences of each shape w create a table bl hagat di

def generate_color_array(length):
    colorRanges = []
    while len(colorRanges) < length:
        # Generate random RGB values
        r = random.randint(0, 255)
        g = random.randint(0, 255)
        b = random.randint(0, 255)
        # Ensure no duplicate colors
        if (r, g, b) not in colorRanges:
            colorRanges.append((r, g, b))
    return colorRanges

def Create_DF(dxfpath,datadoc):
    
  FinalRatio= RetriveRatio(datadoc,dxfpath)
    
  hatched_areas = get_hatched_areas(dxfpath,FinalRatio)
  # SimilarAreaDictionary= pd.DataFrame(columns=['Area', 'Total Area', 'Perimeter', 'Total Perimeter', 'Occurences', 'Color'])
  SimilarAreaDictionary= pd.DataFrame(columns=['Guess','Color','Occurences','Area','Total Area','Perimeter','Total Perimeter','Length','Total Length','Texts','Comments'])
    
  colorRanges2=generate_color_array(300)
  colorRanges = [[255, 0, 0], [0, 0, 255], [0, 255, 255], [0, 64, 0], [255, 204, 0], [255, 128, 64], [255, 0, 128], [255, 128, 192], [128, 128, 255], [128, 64, 0],[0, 255, 0],[0, 200, 0],[255, 128, 255], [128, 0, 255], [0, 128, 192], [128, 0, 128],[128, 0, 0], [0, 128, 255], [149, 1, 70], [255, 182, 128], [222, 48, 71], [240, 0, 112], [255, 0, 255], [192, 46, 65], [0, 0, 128],[0, 128, 64],[255, 255, 0], [128, 0, 80], [255, 255, 128], [90, 255, 140],[255, 200, 20],[91, 16, 51], [90, 105, 138], [114, 10, 138], [36, 82, 78], [225, 105, 190], [108, 150, 170], [11, 35, 75], [42, 176, 170], [255, 176, 170], [209, 151, 15],[81, 27, 85], [226, 106, 122], [67, 119, 149], [159, 179, 140], [159, 179, 30],[255, 85, 198], [255, 27, 85], [188, 158, 8],[140, 188, 120], [59, 61, 52], [65, 81, 21], [212, 255, 174], [15, 164, 90],[41, 217, 245], [213, 23, 182], [11, 85, 169], [78, 153, 239], [0, 66, 141],[64, 98, 232], [140, 112, 255], [57, 33, 154], [194, 117, 252], [116, 92, 135], [74, 43, 98], [188, 13, 123], [129, 58, 91], [255, 128, 100], [171, 122, 145],  [255, 98, 98], [222, 48, 77]]
  colorUsed=[]
  TotalArea=0
  TotalPerimeter=0
  for i in range(len(hatched_areas)):
      area = hatched_areas[i][1]  # area
      perimeter = hatched_areas[i][2]  # perimeter
      if(i < len(colorRanges)):
        color = colorRanges[i]
        colorUsed.append(color)
      else:
        color = colorRanges2[i]
        colorUsed.append(color)
      TotalArea = area
      TotalPerimeter = perimeter
      tol=2
      condition1 = (SimilarAreaDictionary['Area'] >= area - tol) & (SimilarAreaDictionary['Area'] <= area +tol)
      condition2 = (SimilarAreaDictionary['Perimeter'] >= perimeter -tol) & (SimilarAreaDictionary['Perimeter'] <= perimeter +tol)
      combined_condition = condition1 & condition2

      if any(combined_condition):
          index = np.where(combined_condition)[0][0]
          SimilarAreaDictionary.at[index, 'Occurences'] += 1
          SimilarAreaDictionary.at[index, 'Total Area'] = SimilarAreaDictionary.at[index, 'Total Area'] + area
          SimilarAreaDictionary.at[index, 'Total Perimeter'] = SimilarAreaDictionary.at[index, 'Total Perimeter'] + perimeter
      else:
          TotalArea=area
          TotalPerimeter=perimeter
          new_data = {'Area': area, 'Total Area': TotalArea ,'Perimeter': perimeter, 'Total Perimeter': TotalPerimeter, 'Occurences': 1, 'Color':color,'Comments':''} #add color here and read color to insert in
          SimilarAreaDictionary = pd.concat([SimilarAreaDictionary, pd.DataFrame([new_data])], ignore_index=True)

  # print(SimilarAreaDictionary)
  return SimilarAreaDictionary
"""### Draw on Image and PDF"""

def mainFunctionDrawImgPdf(datadoc,dxfpath, dxfratio,pdfpath,pdfname):
  FinalRatio= RetriveRatio(datadoc,dxfpath)
  hatched_areas = get_hatched_areas(dxfpath,FinalRatio)
  img=pdftoimg(datadoc)
  flipped_horizontal=flip(img)
  allcnts = []
  imgg = flipped_horizontal
  # imgtransparent1=imgg.copy()
  doc = fitz.open('pdf',datadoc)
  page2 = doc[0]
  rotationOld=page2.rotation
  derotationMatrix=page2.derotation_matrix
  pix=page2.get_pixmap()
  width=abs(page2.mediabox[2])+abs(page2.mediabox[0])
  height=abs(page2.mediabox[3])+abs(page2.mediabox[1])
  print('mediabox', width , height)
  if page2.rotation!=0:

    rotationangle = page2.rotation
    page2.set_rotation(0)
    ratio =  pix.width/ img.shape[0]
  else:
    ratio =  pix.width/ img.shape[1]
    rotationangle = 270

  allshapes=[]
  # Iterate through each polygon in metric units
  NewColors = []
  SimilarAreaDictionary=Create_DF(dxfpath,datadoc)
  i=0


  for polygon in hatched_areas:
      cntPoints = []
      cntPoints1 = []
      shapee = []
      # Convert each vertex from metric to pixel coordinates
      for vertex in polygon[0]:
          x = (vertex[0]) *dxfratio
          y = (vertex[1]) *dxfratio
          if rotationangle==0:
            if y<0:
              y=y*-1
          cntPoints.append([int(x), int(y)])
          cntPoints1.append([x, y])

      for poi in np.array(cntPoints1):
          x1, y1 = poi
          p1 = fitz.Point(x1,y1)
          # p1 = fitz.Point(x1,y1)
          p1=p1*derotationMatrix
          shapee.append([p1[0],p1[1]])

      shapee=np.flip(shapee,1)
      shapee=rotate_polygon(shapee,rotationangle,rotationOld,width,height)
      tol=2
      condition1 = (SimilarAreaDictionary['Area'] >= polygon[1] - tol) & (SimilarAreaDictionary['Area'] <= polygon[1] +tol)
      condition2 = (SimilarAreaDictionary['Perimeter'] >= polygon[2] -tol) & (SimilarAreaDictionary['Perimeter'] <= polygon[2] +tol)
      combined_condition = condition1 & condition2

      if any(combined_condition):

          index = np.where(combined_condition)[0][0]
          # print(SimilarAreaDictionary.at[index, 'Color'])
          NewColors=SimilarAreaDictionary.at[index, 'Color']
      else:
        NewColors=SimilarAreaDictionary.at[i, 'Color']
  
      # cv2.drawContours(imgg, [np.array(cntPoints)], -1, (NewColors), thickness=2)
      cv2.drawContours(imgg, [np.array(cntPoints)], -1, ([NewColors[2],NewColors[1],NewColors[0]]), thickness=-1)
      annot11 = page2.add_polygon_annot( points=shapee)  # 'Polygon'
      annot11.set_border(width=0.2)
      annot11.set_colors(stroke=(int(NewColors[0])/255,int(NewColors[1])/255,int(NewColors[2])/255), fill= (int(NewColors[0])/255,int(NewColors[1])/255,int(NewColors[2])/255) )
      annot11.set_info(content='Area='+str(polygon[1])+' m^2',subject='ADR Team')
      annot11.set_opacity(0.9)
      # annot.set_line_ends(fitz.PDF_ANNOT_LE_DIAMOND, fitz.PDF_ANNOT_LE_CIRCLE)
      annot11.update()



      annot12 = page2.add_polygon_annot( points=shapee)  # 'Polygon'
      annot12.set_border(width=0.2)
      annot12.set_colors(stroke=(int(NewColors[0])/255,int(NewColors[1])/255,int(NewColors[2])/255))
      annot12.set_info(content='Perimeter='+str(polygon[2])+' m',subject='ADR Team')
      annot12.set_opacity(0.8)
      # annot.set_line_ends(fitz.PDF_ANNOT_LE_DIAMOND, fitz.PDF_ANNOT_LE_CIRCLE)
      annot12.update()
      i += 1
  alpha = 0.8  # Transparency factor.
  
  page2.set_rotation(rotationOld)
  Correct_img=flip(imgg)

  image_new1 = cv2.addWeighted(Correct_img, alpha,  img, 1 - alpha, 0)
  SimilarAreaDictionary = SimilarAreaDictionary.fillna(' ')
  gc,spreadsheet_service,spreadsheetId, spreadsheet_url , namepathArr=google_sheet_Legend.legendGoogleSheets(SimilarAreaDictionary , pdfname,pdfpath)
  # dbxTeam=tsadropboxretrieval.ADR_Access_DropboxTeam('user')
  # md, res =dbxTeam.files_download(path= pdfpath+pdfname)
  # data = res.content
  # doc=fitz.open("pdf", data)
  # list1=pd.DataFrame(columns=['content', 'creationDate', 'id', 'modDate', 'name', 'subject', 'title'])
  list1=pd.DataFrame(columns=['content',  'id',  'subject','color'])

  for page in doc:    
    # Iterate through annotations on the page
    for annot in page.annots():
        # Get the color of the annotation
        annot_color = annot.colors
        if annot_color is not None:
            # annot_color is a dictionary with 'stroke' and 'fill' keys
            stroke_color = annot_color.get('stroke')  # Border color
            fill_color = annot_color.get('fill')      # Fill color
            if fill_color:
              v='fill'
              print('fill')
            if stroke_color:
              v='stroke'
            x,y,z=int(annot_color.get(v)[0]*255),int(annot_color.get(v)[1]*255),int(annot_color.get(v)[2]*255)
            list1.loc[len(list1)] =[annot.info['content'],annot.info['id'],annot.info['subject'],[x,y,z]]
  return doc,image_new1, SimilarAreaDictionary ,spreadsheetId, spreadsheet_url , namepathArr , list1,hatched_areas


def deletemarkupsDXF(list1, dbPath, path):
    '''list1 : original markup pdf
       list2 : deleted markup pdf
       deletedrows : deleted markups - difference between both dfs
    '''

    myDict1 = eval(list1)
    list1 = pd.DataFrame(myDict1)

    dbxTeam = tsadropboxretrieval.ADR_Access_DropboxTeam('user')
    md, res = dbxTeam.files_download(path=dbPath + path)
    data = res.content
    doc = fitz.open("pdf", data)
    
    # Prepare a DataFrame for the annotations in the new PDF
    list2 = pd.DataFrame(columns=['content', 'id', 'subject', 'color'])

    for page in doc:
        # Iterate through annotations on the page
        for annot in page.annots():
            # Get the color of the annotation
            annot_color = annot.colors
            if annot_color is not None:
                # Check for fill or stroke color
                stroke_color = annot_color.get('stroke')
                fill_color = annot_color.get('fill')

                v = 'stroke' if stroke_color else 'fill'
                color = annot_color.get(v)
                if color:
                    # Convert color to tuple and multiply by 255 to get RGB values
                    color_tuple = (int(color[0] * 255), int(color[1] * 255), int(color[2] * 255))
                    # Append annotation data to list2
                    list2.loc[len(list2)] = [annot.info['content'], annot.info['id'], annot.info['subject'], color_tuple]

    # Ensure that colors are stored as tuples (which are hashable)
    list1['color'] = list1['color'].apply(lambda x: tuple(x) if isinstance(x, list) else x)
    
    # Find the deleted rows by checking the difference between original and current annotations
    deletedrows = pd.concat([list1, list2]).drop_duplicates(keep=False)

    print(deletedrows, len(deletedrows))
    flag = 0
    if len(deletedrows) != 0:
        flag = 1
        deletedrows = deletedrows[['content', 'id', 'subject', 'color']]
        # Drop rows where 'content' starts with 'Scale'
        deletedrows = deletedrows.drop(deletedrows.index[deletedrows['content'].str.startswith('Scale')])
    else:
        flag = 0
    
    return deletedrows