Update pilecaps_adr.py

pilecaps_adr.py

@@ -6,39 +6,23 @@ Automatically generated by Colaboratory.
 Original file is located at
     https://colab.research.google.com/drive/16RHtRae7VU_fqHMAjOUL4ET5slEFo3pf
 """
-# pip install pdf-annotate
-# pip install pdf-annotate
-#pip install pdf2image
-#!pip install -q gradio
-#pip install pygsheets
-# !apt-get install poppler-utils
 import numpy as np 
 import cv2 
-#from google.colab.patches import cv2_imshow
 from matplotlib import pyplot as plt 
-#from pdf2image import convert_from_path
-import math
 from math import sin, cos, radians
 import pandas as pd
-import random
-# import imutils
-# from imutils import contours 
-import colorsys
-from PIL import Image  , ImageDraw, ImageFont , ImageColor
+from PIL import Image
 import numpy as np
-#import gradio as gr
-
-# from __future__ import print_function
 from googleapiclient.discovery import build 
 from google.oauth2 import service_account
 import pygsheets
 import re
-import pandas
 import fitz 
-import json
 import db
 import ast
-
+import Dropbox_TSA_API
+import tsadropboxretrieval
+#Prepare preprocessing
 def detectCircles(imgOriginal ):
   im=imgOriginal.copy()
   imgGry1 = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
@@ -46,9 +30,6 @@ def detectCircles(imgOriginal ):
   er1=cv2.erode(imgGry1,kernel, iterations=2)
  
   er1=cv2.dilate(er1,kernel, iterations=1)
-  # gray_blurred = cv2.blur(er1, (3,3 ))
-  # Apply Hough transform on the blurred image.
-   # min distance between circles,  Upper threshold for the internal Canny edge detector.
   detected_circles = cv2.HoughCircles( er1, cv2.HOUGH_GRADIENT, 1, 50, param1= 550,
                     param2 =21, minRadius = 20, maxRadius = 50) #18 param2
     
@@ -83,7 +64,7 @@ def detectSmallCircles(img ):
         cv2.circle(im, (x, y), r+3, (255, 255, 255), -1)
   # cv2_imshow(im)
   return im
-# c=detectCircles(img)
+
 
 def DashedPreprocessing(imgOriginal,imgnoSmall):
   h,w=imgOriginal.shape[0:2]
@@ -182,20 +163,12 @@ def ConnectBeamLines(smalldashesOut):
   green1=cv2.bitwise_not(smalldashesOut)
   green2=smalldashesOut.copy()
   green2=cv2.cvtColor(green2,cv2.COLOR_GRAY2BGR)
-  # imgLines= cv2.HoughLinesP(green1,0.05,np.pi/250,10,minLineLength=25,maxLineGap = 20)  #was w-h , gap=150  #50
-  # for i in range(len(imgLines)):
-  #     for x1,y1,x2,y2 in imgLines[i]:
-  #         cv2.line(green2,(x1,y1),(x2,y2),(0,0,0),2)
-
   imgLines= cv2.HoughLinesP(green1,0.3,np.pi/180,20,minLineLength=25,maxLineGap = 25)  #try 180
-
-
   for i in range(len(imgLines)):
       for x1,y1,x2,y2 in imgLines[i]:
           cv2.line(green2,(x1,y1),(x2,y2),(0,0,0),2)
-  # cv2_imshow(green2)
   return green2
-
+#create img with solid lines
 def allpreSteps(imgOriginal):
   noCircles=detectCircles(imgOriginal)
   imgnoSmall=detectSmallCircles(noCircles )
@@ -209,10 +182,9 @@ def allpreSteps(imgOriginal):
 
 def ChangeBrightness(img,k):
   imgdarker = 255 * (img/255)**k # k>1 darker , k <1 lighter
-  # cv2_imshow(imgdarker)
   imgdarker = imgdarker.astype('uint8')
   return imgdarker
-
+#OLD method (White shapes)
 def preprocessold(img,number):
   blurG = ChangeBrightness(img,3)
   imgGry = cv2.cvtColor(blurG, cv2.COLOR_BGR2GRAY)
@@ -223,8 +195,7 @@ def preprocessold(img,number):
   ret3, thresh = cv2.threshold(er1, 254, 255, cv2.THRESH_BINARY_INV  )
   thresh=cv2.medianBlur(thresh,5)
   return thresh
-
-
+#anding of old method output with solid lines img
 def preprocess(imgOriginal,number,green2,flag,layeredflag,BlackmaskDetected1=0):
   #first preprocessing ( old method - black img with white shapes)
   img1=preprocessold(imgOriginal,number)
@@ -258,13 +229,12 @@ def preprocess(imgOriginal,number,green2,flag,layeredflag,BlackmaskDetected1=0):
 """# ROI (levels)
 ## Detect regions with specific color and mask them
 """
-
 def hexRGB(color):
   color=color.lstrip('#')
-
   color= tuple(int(color[i:i+2], 16) for i in (0, 2, 4)) #hex to rgb
   color=np.array(color) #rgb to bgr 
   return color
+
 def DetectColor(img,color=0):
 
   imgCopy=img.copy()
@@ -292,45 +262,6 @@ def DetectColor(img,color=0):
   cv2.imwrite('det.png',detectedColors)
   return mask, detectedColors, color
 
-    
-def detectAllColors(img,finalColorArray):
-  for i in range(len(finalColorArray)):
-    detectedColors= DetectColor(img,finalColorArray[i])[1]
-    if i == 0:
-      allcolorsImg=cv2.bitwise_or(detectedColors,detectedColors)
-    else:
-      allcolorsImg=cv2.bitwise_or(allcolorsImg,detectedColors)
-  allcolorsImg= cv2.medianBlur(allcolorsImg,7)
-  
-  return allcolorsImg
-
-def colorOrder(img,finalColorArray):
-  newimg=img.copy()
-  arraycolor=[]
-  allcolorsImg= detectAllColors(img,finalColorArray)
-  allcolorsImgG=  cv2.cvtColor(allcolorsImg, cv2.COLOR_BGR2GRAY)
-
-  ColoredContour, Coloredhierarchy = cv2.findContours(allcolorsImgG, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
-  Coloredhierarchy=Coloredhierarchy[0]
-  for cnt in ColoredContour :
-    Blackmask = np.zeros(img.shape[:2], dtype="uint8")
-    cv2.drawContours(Blackmask,[cnt],0,(255,255,255),20) 
-    coloredand=cv2.bitwise_and(allcolorsImg,allcolorsImg,mask=Blackmask)
-    
-    for colors in finalColorArray:
-      getColor=DetectColor(coloredand,colors)[1]
-      
-      pil_image=Image.fromarray(getColor)
-      extrema = pil_image.convert("L").getextrema()
-      if extrema != (0, 0): # if image is not black --> has a colored mask within
-        arraycolor.append(colors)
-        break
-
-  res = []
-  [res.append(x) for x in arraycolor if x not in res]
-  
-  return res
-
 def getinnerColor(BlackmaskDetected,img,detectedColors,finalColorArray,ratioarea,ratioperim,flag,eachcolor):
 
   countBlackMasks=0
@@ -441,8 +372,6 @@ def allLevelsofColor(BlackmaskDetected,img,levelonly, invertedmask,color,finalCo
   return firstLevel1, BlackmaskDetected
     
 def getColoredContour(mask,img,finalColorArray,ratioarea,ratioperim,flag,eachcolor):
-    print('uuuuuuuuummmmmmmmmmmmm')
-
     ColoredContour, Coloredhierarchy = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
     Coloredhierarchy=Coloredhierarchy[0]
 
@@ -498,8 +427,7 @@ def findContoursFullImage(green2,img,number,finalColorArray,ratioarea,ratioperim
 
       return contourss,color ,mask
 
-
-
+#Straighten tilted shapes
 def StraightenImage(contour,imgArea):
     rect = cv2.minAreaRect(contour)
     
@@ -526,7 +454,7 @@ def StraightenImage(contour,imgArea):
     # warped = cv2.warpPerspective(imgArea, M, (width, height))
     ##############  
     return angleR,width,height
-
+#get all areas and perimeter present
 def getAreasPerimeter(green2,img,number,ratioarea,ratioperim,flag,finalColorArray,color=[0,0,0]):
   appended=[]
   if number==0:
@@ -556,7 +484,7 @@ def getAreasPerimeter(green2,img,number,ratioarea,ratioperim,flag,finalColorArra
             # print('numss')
             # if flag=='area':
               # addedMargin=area1+perimeter1*2
-              areaa=area1* ratioarea# true value of area of any shape/ area px value of same shape
+              areaa=round(area1* ratioarea,3)# true value of area of any shape/ area px value of same shape
               appended.append([areaa,width,height])
 
         #else:
@@ -564,8 +492,7 @@ def getAreasPerimeter(green2,img,number,ratioarea,ratioperim,flag,finalColorArra
           #appended.append([perimeter,width,height])
 
   return appended
-
-
+#fill dictionary with areas and perimeters and occurences
 def FillDictionary(green2,SimilarAreaDictionary,img,number,ratioarea,ratioperim,flag,finalColorArray,rgbcolor=[0,0,0],color=[0,0,0]):
   #fills dictionary with key areas and number of occurences
   areas_Perimeters=sorted(getAreasPerimeter(green2,img,number,ratioarea,ratioperim,flag,finalColorArray,color) )
@@ -644,13 +571,10 @@ def FillDictionary(green2,SimilarAreaDictionary,img,number,ratioarea,ratioperim,
 
 
   return SimilarAreaDictionary, colorsUsed , areas_Perimeters
-
-
-
-
-def drawAllContours(img,number,finalColorArray,ratioarea,ratioperim,flag , path,pdfpath):
+#detect and draw and measure
+def drawAllContours(dataDoc,img,number,finalColorArray,ratioarea,ratioperim,flag , path,pdfpath):
   green2=allpreSteps(img)
-  doc = fitz.open('dropbox_plans/1.0/'+path)
+  doc = fitz.open("pdf",dataDoc)
   page = doc[0]
   pix=page.get_pixmap()
   ratio =  pix.width / img.shape[1]   
@@ -861,16 +785,11 @@ def drawAllContours(img,number,finalColorArray,ratioarea,ratioperim,flag , path,
 
   alpha = 0.4  # Transparency factor.
   image_new1 = cv2.addWeighted(imgArea1, alpha,  imgtransparent1, 1 - alpha, 0)
-  # SimilarAreaDictionary.drop(['Rounded', 'Width','Height','R','G','B'], axis=1, inplace=True)
-
-  print(SimilarAreaDictionary)
  
-  # annotationsSave
-
-  pdflink= db.dropbox_upload_file(doc=doc,pdfname=path,pdfpath=pdfpath+'Measured Plan/')
-  print('pdfpath,link',pdfpath,pdflink)
-  dbx=db.dropbox_connect()
-  md, res =dbx.files_download(path= pdfpath+path)
+  dbPath='/TSA Team Folder/ADR Test'+pdfpath+'Measured Plan/' 
+  pdflink= tsadropboxretrieval.uploadanyFile(doc=doc,path=dbPath,pdfname=path) #doc=doc,pdfname=path,pdfpath=pdfpath+'Measured Plan/'
+  dbxTeam=tsadropboxretrieval.ADR_Access_DropboxTeam('user')
+  md, res =dbxTeam.files_download(path= dbPath+path)
   data = res.content
   doc=fitz.open("pdf", data)
   # list1=pd.DataFrame(columns=['content', 'creationDate', 'id', 'modDate', 'name', 'subject', 'title'])
@@ -885,7 +804,7 @@ def drawAllContours(img,number,finalColorArray,ratioarea,ratioperim,flag , path,
 
 ######################################################
 
-def deletemarkups(list1, pdfpath , path):
+def deletemarkups(list1, dbPath , path):
   '''list1 : original markup pdf
      list2 : deleted markup pdf
      deletedrows : deleted markups - difference betw both dfs
@@ -898,9 +817,9 @@ def deletemarkups(list1, pdfpath , path):
   areastodelete = []
   perimstodelete=[]
 
-  dbx=db.dropbox_connect()
-
-  md, res =dbx.files_download(path= pdfpath+path)
+  dbxTeam=tsadropboxretrieval.ADR_Access_DropboxTeam('user')
+  print('pathhhhh',dbPath+path)
+  md, res =dbxTeam.files_download(path= dbPath+path)
   data = res.content
   doc=fitz.open("pdf", data)
   list2=pd.DataFrame(columns=['content',  'id',  'subject'])
@@ -921,8 +840,6 @@ def deletemarkups(list1, pdfpath , path):
     flag=0
   return deletedrows
    
-
-
 def deletefromlegend(deletedrows,SimilarAreaDictionarycopy,section, areaPermArr=[]):
   items=[]
   idx=0
@@ -931,17 +848,20 @@ def deletefromlegend(deletedrows,SimilarAreaDictionarycopy,section, areaPermArr=
   myDict=eval(SimilarAreaDictionarycopy)
   SimilarAreaDictionarycopy=pd.DataFrame(myDict)
   strings=deletedrows['content']
-
   areastodelete = []
   perimstodelete=[]
-
+  lengthstodelete=[]
+  print('areaPermArr',areaPermArr)
   for item in strings:
     items.append(str(item).split('\n \n'))
   # print('itemsssssss',float(re.findall("\d+\.\d+", str(items[i][0]).split()[0])[0])) #take area and perim mn hna l sec 3.2 and +/- value  margin 
+  
   for i in range(len(items)):
-    areastodelete.append(float(re.findall("\d+\.\d+", str(items[i][0]).split()[0])[0]))
-    perimstodelete.append(float(re.findall("\d+\.\d+", str(items[i][1]).split()[0])[0]) )
-  print(areastodelete,perimstodelete)
+    print('ITEMSS',str(items[i]).split())
+    areastodelete.append(float(re.findall("\d+\.\d+", str(items[i][0]).split()[1])[0]))
+    perimstodelete.append(float(re.findall("\d+\.\d+", str(items[i][1]).split()[1])[0]) )
+    lengthstodelete.append(float(re.findall("\d+\.\d+", str(items[i][2]).split()[1])[0]) )
+  
   for i in range(len(areastodelete)):#item in areastodelete:
     areamin=round(areastodelete[i],1)- 0.3
     areamax=round(areastodelete[i],1)+ 0.3
@@ -949,16 +869,17 @@ def deletefromlegend(deletedrows,SimilarAreaDictionarycopy,section, areaPermArr=
     perimmax=round(perimstodelete[i],1)+ 0.3
     if section.startswith('1.0'):
       for p in range(len(areaPermArr)):
+        
         if areastodelete[i] in areaPermArr[p]:
+          print('AAA',areaPermArr[p])
           area= areaPermArr[p][0]
           width= areaPermArr[p][1]
           height= areaPermArr[p][2]
-    if section.startswith('1.0'):
+    # if section.startswith('1.0'):
       widthMin= width -10
       widthMax= width +10
       heightMin = height-10
       heightMax=height+10
-      print(width, widthMin ,height, heightMin)
       found=SimilarAreaDictionarycopy.loc[SimilarAreaDictionarycopy.index[((SimilarAreaDictionarycopy['Rounded'] >=areamin) & (SimilarAreaDictionarycopy['Rounded']<=areamax) & (SimilarAreaDictionarycopy['Perimeter'] >=perimmin) & (SimilarAreaDictionarycopy['Perimeter']<=perimmax)   ) & ( ((SimilarAreaDictionarycopy['Width']>=widthMin) & (SimilarAreaDictionarycopy['Width']<=widthMax) & (SimilarAreaDictionarycopy['Height']>=heightMin) & (SimilarAreaDictionarycopy['Height']<=heightMax) ) | ((SimilarAreaDictionarycopy['Width']>=heightMin) & (SimilarAreaDictionarycopy['Width']<=heightMax) & (SimilarAreaDictionarycopy['Height']>=widthMin) & (SimilarAreaDictionarycopy['Height']<=widthMax) )) ]]
     elif section.startswith('3.2'):
       areamin=round(areastodelete[i],1)- 0.1
@@ -982,52 +903,11 @@ def deletefromlegend(deletedrows,SimilarAreaDictionarycopy,section, areaPermArr=
           idx=SimilarAreaDictionarycopy.index[((SimilarAreaDictionarycopy['Area'] >=areamin) & (SimilarAreaDictionarycopy['Area']<=areamax)  & (SimilarAreaDictionarycopy['Perimeter'] >=perimmin) & (SimilarAreaDictionarycopy['Perimeter']<=perimmax)    )]
         SimilarAreaDictionarycopy.loc[idx,'Total Area'] = SimilarAreaDictionarycopy.loc[idx,'Total Area'] - areastodelete[i]
         SimilarAreaDictionarycopy.loc[idx,'Total Perimeter'] = SimilarAreaDictionarycopy.loc[idx,'Total Perimeter'] - perimstodelete[i]
-        print('ppppppppppppppppppp',SimilarAreaDictionarycopy.loc[idx,'Occurences'])
+        SimilarAreaDictionarycopy.loc[idx,'Total Length'] = SimilarAreaDictionarycopy.loc[idx,'Total Length'] - lengthstodelete[i]
         SimilarAreaDictionarycopy.loc[idx,'Occurences'] = int(SimilarAreaDictionarycopy.loc[idx,'Occurences']) - 1
-  print(SimilarAreaDictionarycopy)
   return SimilarAreaDictionarycopy
 #######################################################
-
-def getTitle(path):
-  planName= path.split("/")[-1].split('.')
-  LegendName='Legend of ' + str(planName[0]) + ' plan'
-  return LegendName
-
-def retrieveMCCol(gc):
-  ws=gc.open_by_key('1A8VtqLFhe2NXPxIjfAilbxF9xV2eSzZ-yZ9GP8_5jSo')
-  worksheet = ws.worksheet(0)
-  mcT_Names=worksheet.get_col(1)
-  newMcTNames=[]
-  for i in mcT_Names:
-    if i != '':
-      newMcTNames.append(i)
-  return newMcTNames
-
-
-def getdropdownValues(gc,spreadsheet_service,spreadsheetid):
-  dropdownValues=[]
-  ws=gc.open_by_key('1A8VtqLFhe2NXPxIjfAilbxF9xV2eSzZ-yZ9GP8_5jSo')  ## spreadsheet containing mc-t names 
- 
-  worksheet = ws.worksheet(0)
-  response = spreadsheet_service.spreadsheets().get(
-        spreadsheetId=spreadsheetid, fields='*',
-        ranges='A2:A60',includeGridData=True).execute()
-  r=list(response['sheets'][0]['data'][0]['rowData'][0]['values'][0])
-  print(r)
-  if 'dataValidation' in r:
-    print('yes')
-    colvals= response['sheets'][0]['data'][0]['rowData'][0]['values'][0]['dataValidation']
-    colvalsList=list(colvals.items())
-    print(colvalsList[0][1])
-    lengthVals=len(colvalsList[0][1]['values'])
-    for i in range(lengthVals):
-      dictVal=(colvalsList[0][1]['values'][i].values())
-      # val=[*dictVal]
-     
-      dropdownValues.append(*dictVal)
-    print(dropdownValues)
-  worksheet.update_col(index=1, values=dropdownValues)
-  return dropdownValues
+##Legend
 def authorizeLegend():
   SCOPES = [
   'https://www.googleapis.com/auth/spreadsheets',
@@ -1039,7 +919,7 @@ def authorizeLegend():
   drive_service = build('drive', 'v3', credentials=credentials)
   gc = pygsheets.authorize(custom_credentials=credentials, client_secret='credentials.json')
   return spreadsheet_service,drive_service,gc
-
+#create/update legend
 def legendGoogleSheets(SimilarAreaDictionary,path ,pdfpath, spreadsheetId=0):
 
   spreadsheet_service,drive_service,gc=authorizeLegend()
@@ -1086,7 +966,7 @@ def legendGoogleSheets(SimilarAreaDictionary,path ,pdfpath, spreadsheetId=0):
   worksheet = ws.worksheet(0)
   worksheet.title='Legend and data created'
   worksheet.clear()
-
+  print('PDFPATHHH',pdfpath)
   ws.create_developer_metadata('path',pdfpath)
   splittedpdfpath=re.split(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,/<>?]', pdfpath)
   namepathArr=[legendTitle , spreadsheetId,ws.get_developer_metadata('path')[0].value]
@@ -1336,7 +1216,7 @@ def mapnametoLegend(McTName):
   spreadsheet_key =str(key)  # Please set the Spreadsheet ID.
   
   ws = gc.open_by_key(spreadsheet_key)
-  guessednamesfinal=getguessnames(gc,ws)
+  # guessednamesfinal=getguessnames(gc,ws)
   sheetnames=[]
   unit=''
   # ws.add_worksheet("Summary")  # Please set the new sheet name.
@@ -1351,6 +1231,8 @@ def mapnametoLegend(McTName):
   worksheetw = ws.worksheet(0) #legend
   worksheetS = ws.worksheet_by_title('XML Export Summary')
   summaryId= ws[1].id
+  print('summaryyyID',summaryId)
+  print('summaryyyID2',worksheetS.id)
   worksheetS.clear()
   countnames=0
   row0=['MC_T Name','Qty','Unit']
@@ -1363,7 +1245,7 @@ def mapnametoLegend(McTName):
     print(McTName[i][0])
     
     # firstpart= re.split(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./<>?]', McTName[i][0])
-
+    print('kkk' ,McTName[i][2])
     if McTName[i][2].startswith('Area'):
       if section.startswith('1.0'):
         rowvalue=5# column 5 
@@ -1396,9 +1278,7 @@ def mapnametoLegend(McTName):
     print('mcct',McTName[i][1])
     if isinstance(McTName[i][1], list):
       guessednames=worksheetw.get_col(1, returnas='matrix', include_tailing_empty=False)
-      print('gg',guessednames)
-      # guessednames=guessednames[3:]
-      print('gg',guessednames)
+
       for m in McTName[i][1]:
         if m:
           if m.startswith('text1'):
@@ -1513,9 +1393,10 @@ def mapnametoLegend(McTName):
   ]
   }
   spreadsheet_service.spreadsheets().batchUpdate(spreadsheetId=spreadsheet_key, body=body2).execute()
-  return summaryId,guessednamesfinal   
+  return summaryId #,guessednamesfinal   
 
   # print(x,xarea)
+
 def getguessnames(gc,ws):
   guessednamesfinal=[]
   worksheetw = ws.worksheet(0)