Update tsadropboxretrieval.py

tsadropboxretrieval.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 """TSADropboxRetrieval.ipynb
-
 Automatically generated by Colaboratory.
-
 Original file is located at
     https://colab.research.google.com/drive/1d-UI3Y-z7Dj-vqu69CxluOUnN4rvsUuE
 """
@@ -10,7 +8,7 @@ Original file is located at
 # !pip install dropbox -q
 
 # pip install pymupdf #==1.22.5
-
+import os
 import base64
 import requests
 import json
@@ -23,8 +21,16 @@ import io
 import re
 import pyarrow
 from io import BytesIO
+from functools import lru_cache
+from io import BytesIO
+import os
+
+
+
+@lru_cache(maxsize=1)
+def load_parquet_df():
+    return GetParquetDF()
 
-"""### NEW CODE - OCTOBER 26 - Marthe"""
 
 files_list=[]
 
@@ -101,6 +107,7 @@ def dropbox_upload_file(df, flag=0):
         print('Error uploading file to Dropbox: ' + str(e))
     return dbxTeam
 
+
 def check_if_file_exists(dbxTeam,path):
     try:
         md = dbxTeam.files_get_metadata(path)
@@ -109,39 +116,93 @@ def check_if_file_exists(dbxTeam,path):
     except Exception as error_response:
         exists_bool = False
         return exists_bool
+def uploadmarkupPDFTable(doc,pdfname,path):
+   dbxTeam= ADR_Access_DropboxTeam('admin')
+   try:
+      path=path+pdfname
+      exists_bool=check_if_file_exists(dbxTeam,path)
+      if exists_bool:
+        print("if gowa el else <3")
+        dbxTeam.files_delete(path)
+      print("abl el meta <3")
+      meta=dbxTeam.files_upload(doc.read() ,path)
+      try:
+        shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings(path)
+      except:
+        shared_link_metadata=dbxTeam.sharing_create_shared_link(path)
+      # print(shared_link_metadata.url)
+      return shared_link_metadata.url
+   except Exception as e:
+        print('Error uploading file to Dropbox: ' + str(e))
+    
+
 
-def uploadanyFile(doc,pdfname,path,flag=0):
+def upload_string_file(content_str, filename, path):
     try:
-        dbxTeam= ADR_Access_DropboxTeam('admin')
-        print('ppp')
-        if flag: #tree = doc
-          pdfname=str(pdfname).split('.pdf')[0]+'.xml'
-          path=path+pdfname
-          print(path)
-          f = BytesIO()
-          doc.write(f, encoding='utf-8', xml_declaration=True)
-          exists_bool=check_if_file_exists(dbxTeam,path)
-          if exists_bool:
-            dbxTeam.files_delete(path) 
-          meta=dbxTeam.files_upload(f.getvalue() ,path)
+        dbxTeam = ADR_Access_DropboxTeam('admin')
+        full_path = path + filename
+        # convert string to bytes
+        f = BytesIO(content_str.encode("utf-8"))
+        # delete if file already exists
+        if check_if_file_exists(dbxTeam, full_path):
+            dbxTeam.files_delete(full_path)
+        # upload
+        meta = dbxTeam.files_upload(f.getvalue(), full_path)
+        # create shared link
+        try:
+            shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings(full_path)
+        except:
+            shared_link_metadata = dbxTeam.sharing_create_shared_link(full_path)
+        return shared_link_metadata.url
+    except Exception as e:
+        print("Error uploading file to Dropbox:", str(e))
+        return "Error uploading file to Dropbox."
+
+
+
+
+def uploadanyFile(doc, pdfname, path, flag=0):
+    try:
+        dbxTeam = ADR_Access_DropboxTeam('admin')
+
+        # Determine initial file path
+        if flag:  # XML upload
+            pdfname = str(pdfname).split('.pdf')[0] + '.xml'
+            file_path = path + pdfname
+            f = BytesIO()
+            doc.write(f, encoding='utf-8', xml_declaration=True)
+            data_to_upload = f.getvalue()
         else:
-           path=path+pdfname
-           exists_bool=check_if_file_exists(dbxTeam,path)
-           if exists_bool:
-              dbxTeam.files_delete(path)
-           meta=dbxTeam.files_upload(doc.write() ,path)
+            file_path = path + pdfname
+            data_to_upload = doc.write()
+
+        # Extract base name and extension for duplicate handling
+        base_name, ext = os.path.splitext(pdfname)
+        counter = 1
+
+        # If file exists, increment suffix until unique
+        while check_if_file_exists(dbxTeam, file_path):
+            new_name = f"{base_name}({counter}){ext}"
+            file_path = path + new_name
+            counter += 1
+
+        # Upload file
+        meta = dbxTeam.files_upload(data_to_upload, file_path)
+
+        # Try to create or retrieve shared link
         try:
-          print('hereintry')
-          shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings(path)
+            shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings(file_path)
         except:
-          print('hereinexcept')
-          shared_link_metadata=dbxTeam.sharing_create_shared_link(path)
+            shared_link_metadata = dbxTeam.sharing_create_shared_link(file_path)
+
         return shared_link_metadata.url
+
     except Exception as e:
         print('Error uploading file to Dropbox: ' + str(e))
     
     return 'Error uploading file to Dropbox.'
 
+
 # Call when the dropbox is updated with new items - if not , call parquet saved version of the df of saved items
 # Call when the dropbox is updated with new items - if not , call parquet saved version of the df of saved items
 def DropboxItemstoDF(folder_path):
@@ -181,11 +242,14 @@ def GetParquetDF():
     return df
 
 
-def getPathtoPDF_File(nameofPDF):
-  parquetDf=GetParquetDF()
+def getPathtoPDF_File(nameofPDF,progress_callback=None):
+  parquetDf = load_parquet_df()
   nameofPDF=nameofPDF.replace('"', '')
   try:
-    path=parquetDf.loc[parquetDf['name'] == nameofPDF, 'path_display'].iloc[0]
+    # path=parquetDf.loc[parquetDf['name'] == nameofPDF, 'path_display'].iloc[0]
+    path = parquetDf.at[parquetDf.index[parquetDf['name'] == nameofPDF][0], 'path_display']
+    if progress_callback:
+        progress_callback(60)
     link=getSharedLink(path)
     print(path,link)
   except:
@@ -200,29 +264,63 @@ def getPDFData(path):
   data = res.content
   return data
 
-def retrieveProjects(projname):
-  print('retrieve')
-
-  parquetDf=GetParquetDF()
-  documentsToMeasure = []
-  RelevantDocuments = []
-  projnameWithDetails = f'{projname} 01 Project Details'
-  # Split the project name into words and convert to lowercase
-  matches = set(re.split(r'[`\-= ~!@#$%^&*()_+\[\]{};\'\\:"|<,/<>?]', projnameWithDetails.lower()))
-
-  # Convert the 'path_display' column to lowercase for case-insensitive matching
-  parquetDf['path_display_lower'] = parquetDf['path_display'].str.lower()
-
-  # Create a mask to filter relevant documents
-  mask = parquetDf['path_display_lower'].apply(lambda x: all(match in x for match in matches))
-
-  # Filter RelevantDocuments and documentsToMeasure using the mask
-  RelevantDocuments = parquetDf[mask][['name', 'path_display']].values.tolist()
-  documentsToMeasure = [doc for doc in RelevantDocuments if doc[0].endswith('.pdf')] # Filter documentsToMeasure for PDF files later if needed
-
-  # Remove the temporary 'path_display_lower' column
-  parquetDf.drop(columns=['path_display_lower'], inplace=True)
-  print('done')
-  return documentsToMeasure,RelevantDocuments
-
-
+def retrieveProjects(projname, progress_callback=None):
+    # if progress_callback:
+    progress_callback(20)
+    projnameNospaces = projname.strip().replace('"', '').replace("'", '').replace(" ", "")
+    print(projname,projnameNospaces)
+    projname = '/' + projnameNospaces[:4]  # Extract main project name
+    projname = projname.replace('/"', '')  # Remove unwanted characters
+    print('projname', projname)
+    
+    parquetDf = load_parquet_df()
+    documentsToMeasure = []
+    RelevantDocuments = []
+
+    # Send progress update (20%)
+    if progress_callback:
+        progress_callback(40)
+
+    # Store the original path before converting it to lowercase
+    parquetDf['original_path_display'] = parquetDf['path_display']
+
+    # Create a lowercase column for case-insensitive matching
+    parquetDf['path_display_lower'] = parquetDf['path_display'].str.lower()
+    if progress_callback:
+        progress_callback(50)
+    ##### Updated code #######
+    # Filter using the lowercase column but retrieve the original paths
+    def path_matches(x):
+        #remove the file name from the path
+        folder_path = os.path.dirname(x)
+        #Check if project number (like /2564) appears in folder path
+        has_projnum = projname in folder_path
+        #Check if "01 project details" appears in folder path
+        has_details = '01 project details' in folder_path
+        return has_projnum and has_details
+
+    #Apply the mask 
+    mask = parquetDf['path_display_lower'].apply(path_matches)
+    #### End of updated code #####
+    if progress_callback:
+        progress_callback(60)   
+    # Retrieve the original (case-sensitive) paths before lowering them
+    RelevantDocuments = parquetDf[mask][['name', 'original_path_display']].values.tolist()
+    documentsToMeasure = [doc for doc in RelevantDocuments if doc[0].endswith('.pdf')]  # Keep only PDFs
+    print('documentsToMeasure',documentsToMeasure)
+    # Send progress update (80%)
+    if progress_callback:
+        progress_callback(70)
+
+    # Extract path from the original (case-sensitive) column
+    if RelevantDocuments:
+        extracted_path = RelevantDocuments[0][1].split("01 Project Details")[0] + "01 Project Details"
+    else:
+        extracted_path = None  # Handle case when no match is found
+
+    # Remove temporary columns
+    parquetDf.drop(columns=['original_path_display', 'path_display_lower'], inplace=True)
+    if progress_callback:
+        progress_callback(80)
+    return documentsToMeasure, RelevantDocuments, extracted_path
+