Update tsadropboxretrieval.py

tsadropboxretrieval.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 """TSADropboxRetrieval.ipynb
-
 Automatically generated by Colaboratory.
-
 Original file is located at
     https://colab.research.google.com/drive/1d-UI3Y-z7Dj-vqu69CxluOUnN4rvsUuE
 """
@@ -23,8 +21,14 @@ import io
 import re
 import pyarrow
 from io import BytesIO
+from functools import lru_cache
+
+
+
+@lru_cache(maxsize=1)
+def load_parquet_df():
+    return GetParquetDF()
 
-"""### NEW CODE - OCTOBER 26 - Marthe"""
 
 files_list=[]
 
@@ -101,6 +105,7 @@ def dropbox_upload_file(df, flag=0):
         print('Error uploading file to Dropbox: ' + str(e))
     return dbxTeam
 
+
 def check_if_file_exists(dbxTeam,path):
     try:
         md = dbxTeam.files_get_metadata(path)
@@ -109,7 +114,25 @@ def check_if_file_exists(dbxTeam,path):
     except Exception as error_response:
         exists_bool = False
         return exists_bool
-
+def uploadmarkupPDFTable(doc,pdfname,path):
+   dbxTeam= ADR_Access_DropboxTeam('admin')
+   try:
+      path=path+pdfname
+      exists_bool=check_if_file_exists(dbxTeam,path)
+      if exists_bool:
+        print("if gowa el else <3")
+        dbxTeam.files_delete(path)
+      print("abl el meta <3")
+      meta=dbxTeam.files_upload(doc.read() ,path)
+      try:
+        shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings(path)
+      except:
+        shared_link_metadata=dbxTeam.sharing_create_shared_link(path)
+      # print(shared_link_metadata.url)
+      return shared_link_metadata.url
+   except Exception as e:
+        print('Error uploading file to Dropbox: ' + str(e))
+    
 def uploadanyFile(doc,pdfname,path,flag=0):
     try:
         dbxTeam= ADR_Access_DropboxTeam('admin')
@@ -131,11 +154,10 @@ def uploadanyFile(doc,pdfname,path,flag=0):
               dbxTeam.files_delete(path)
            meta=dbxTeam.files_upload(doc.write() ,path)
         try:
-          print('hereintry')
           shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings(path)
         except:
-          print('hereinexcept')
           shared_link_metadata=dbxTeam.sharing_create_shared_link(path)
+        # print(shared_link_metadata.url)
         return shared_link_metadata.url
     except Exception as e:
         print('Error uploading file to Dropbox: ' + str(e))
@@ -181,30 +203,14 @@ def GetParquetDF():
     return df
 
 
-def getPathtoPDF_File(nameofPDF):
-  parquetDf=GetParquetDF()
-  nameofPDF=nameofPDF.replace('"', '')
-  try:
-    path=parquetDf.loc[parquetDf['name'] == nameofPDF, 'path_display'].iloc[0]
-    link=getSharedLink(path)
-    print(path,link)
-  except:
-    return 'Project does not exist'
-  return path,link
-# parquetDf
-
-# getPathtoPDF_File('A5157-EBLA-V5-XX-SH-L-0004-D2-01.pdf')
-def getPDFData(path):
-  dbxTeam= ADR_Access_DropboxTeam('admin')
-  md, res =dbxTeam.files_download(path)
-  data = res.content
-  return data
-
-def getPathtoPDF_File(nameofPDF):
-  parquetDf=GetParquetDF()
+def getPathtoPDF_File(nameofPDF,progress_callback=None):
+  parquetDf = load_parquet_df()
   nameofPDF=nameofPDF.replace('"', '')
   try:
-    path=parquetDf.loc[parquetDf['name'] == nameofPDF, 'path_display'].iloc[0]
+    # path=parquetDf.loc[parquetDf['name'] == nameofPDF, 'path_display'].iloc[0]
+    path = parquetDf.at[parquetDf.index[parquetDf['name'] == nameofPDF][0], 'path_display']
+    if progress_callback:
+        progress_callback(60)
     link=getSharedLink(path)
     print(path,link)
   except:
@@ -219,26 +225,49 @@ def getPDFData(path):
   data = res.content
   return data
 
-def retrieveProjects(projname):
-  print('retrieve',projname)
-  
-  projname='/'+projname.split(' ')[0]
-  projname=projname.replace('/"', '')
-  print(projname)
-  parquetDf=GetParquetDF()
-  documentsToMeasure = []
-  RelevantDocuments = []
-  parquetDf['path_display_lower'] = parquetDf['path_display'].str.lower()
-
-  # Filter based on the presence of '/2221' and '01 Project Details'
-  mask = parquetDf['path_display_lower'].apply(lambda x: projname in x and '01 project details' in x)
-  print(mask)
-  # Filter RelevantDocuments and documentsToMeasure using the mask
-  RelevantDocuments = parquetDf[mask][['name', 'path_display_lower']].values.tolist()
-  documentsToMeasure = [doc for doc in RelevantDocuments if doc[0].endswith('.pdf')] # Filter documentsToMeasure for PDF files later if needed
-  print(documentsToMeasure)
-  # Remove the temporary 'path_display_lower' column
-  parquetDf.drop(columns=['path_display_lower'], inplace=True)
-  print(len(documentsToMeasure))
-  return documentsToMeasure,RelevantDocuments
-
+def retrieveProjects(projname, progress_callback=None):
+    # if progress_callback:
+    progress_callback(20)
+    
+    projname = '/' + projname.split(' ')[0]  # Extract main project name
+    projname = projname.replace('/"', '')  # Remove unwanted characters
+    print('projname', projname)
+    
+    parquetDf = load_parquet_df()
+    documentsToMeasure = []
+    RelevantDocuments = []
+
+    # Send progress update (20%)
+    if progress_callback:
+        progress_callback(40)
+
+    # Store the original path before converting it to lowercase
+    parquetDf['original_path_display'] = parquetDf['path_display']
+
+    # Create a lowercase column for case-insensitive matching
+    parquetDf['path_display_lower'] = parquetDf['path_display'].str.lower()
+    if progress_callback:
+        progress_callback(50)
+    # Filter using the lowercase column but retrieve the original paths
+    mask = parquetDf['path_display_lower'].apply(lambda x: '/'+projname in x and '01 project details' in x)
+    if progress_callback:
+        progress_callback(60)
+    # Retrieve the original (case-sensitive) paths before lowering them
+    RelevantDocuments = parquetDf[mask][['name', 'original_path_display']].values.tolist()
+    documentsToMeasure = [doc for doc in RelevantDocuments if doc[0].endswith('.pdf')]  # Keep only PDFs
+    
+    # Send progress update (80%)
+    if progress_callback:
+        progress_callback(70)
+
+    # Extract path from the original (case-sensitive) column
+    if RelevantDocuments:
+        extracted_path = RelevantDocuments[0][1].split("01 Project Details")[0] + "01 Project Details"
+    else:
+        extracted_path = None  # Handle case when no match is found
+
+    # Remove temporary columns
+    parquetDf.drop(columns=['original_path_display', 'path_display_lower'], inplace=True)
+    if progress_callback:
+        progress_callback(80)
+    return documentsToMeasure, RelevantDocuments, extracted_path