Update tsadropboxretrieval.py

tsadropboxretrieval.py

@@ -126,10 +126,10 @@ def uploadanyFile(doc,pdfname,path,flag=0):
 
 # Call when the dropbox is updated with new items - if not , call parquet saved version of the df of saved items
 # Call when the dropbox is updated with new items - if not , call parquet saved version of the df of saved items
-def DropboxItemstoDF():
+def DropboxItemstoDF(folder_path):
   files_list=[]
   dbxTeam=ADR_Access_DropboxTeam('user') # or pass dbx in parameters
-  folder_path = "/TSA Team Folder"
+  # folder_path = "/TSA Team Folder"
   res = dbxTeam.files_list_folder(path=folder_path, recursive=True )
   # df1=handle_entries(res.entries , files_list)
   if res.has_more:
@@ -138,7 +138,7 @@ def DropboxItemstoDF():
         df2=handle_entries(res.entries , files_list)
 
   # dbxTeam=dropbox_upload_file(df2)
-  print(df2)
+  # print(df2)
   return df2 , files_list
 
 # df2=DropboxItemstoDF()
@@ -149,19 +149,41 @@ def DropboxItemstoDF():
 #   fthr=pd.read_feather('df2.feather')
 #   return fthr
 
+# def GetParquetDF():
+#   # # read the parquet file in current directory, back into a pandas data frame
+#   dbxTeam=ADR_Access_DropboxTeam('user') # or pass dbx in parameters
+#   try:
+#     shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings( path='/TSA Team Folder/ADR Test/DropboxDirectory/df.parquet.gzip')
+#   except:
+#     shared_link_metadata=dbxTeam.sharing_create_shared_link( path='/TSA Team Folder/ADR Test/DropboxDirectory/df.parquet.gzip')
+#   metadata, res = dbxTeam.sharing_get_shared_link_file(url=shared_link_metadata.url)
+#   data=res.content  # or res.content, or iter_content, or iter_lines, etc. as needed
+
+#   pq_file = io.BytesIO(data)
+#   df = pd.read_parquet(pq_file)
+#   return df
+
+
 def GetParquetDF():
-  # # read the parquet file in current directory, back into a pandas data frame
-  dbxTeam=ADR_Access_DropboxTeam('user') # or pass dbx in parameters
-  try:
-    shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings( path='/TSA Team Folder/ADR Test/DropboxDirectory/df.parquet.gzip')
-  except:
-    shared_link_metadata=dbxTeam.sharing_create_shared_link( path='/TSA Team Folder/ADR Test/DropboxDirectory/df.parquet.gzip')
-  metadata, res = dbxTeam.sharing_get_shared_link_file(url=shared_link_metadata.url)
-  data=res.content  # or res.content, or iter_content, or iter_lines, etc. as needed
+    # Initialize Dropbox client
+    dbxTeam = ADR_Access_DropboxTeam('user')  # or pass dbx in parameters
+    # Define the path to the Parquet file on Dropbox
+    path = '/TSA Team Folder/ADR Test/DropboxDirectory/df.parquet.gzip'
+    try:
+        # Try to create a shared link with settings
+        shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings(path=path)
+    except dropbox.exceptions.ApiError:
+        # If settings are not supported, create a shared link without settings
+        shared_link_metadata = dbxTeam.sharing_create_shared_link(path=path)
+    # Get the file content from the shared link
+    _, res = dbxTeam.sharing_get_shared_link_file(url=shared_link_metadata.url)
+    data = res.content
+    # Read the Parquet file content into a pandas DataFrame
+    with io.BytesIO(data) as pq_file:
+        df = pd.read_parquet(pq_file)
+
+    return df
 
-  pq_file = io.BytesIO(data)
-  df = pd.read_parquet(pq_file)
-  return df
 
 def getPathtoPDF_File(nameofPDF):
   parquetDf=GetParquetDF()
@@ -181,20 +203,57 @@ def getPDFData(path):
   data = res.content
   return data
 
+# def retrieveProjects(projname):
+#     print('retrieve')
+    
+#     parquetDf=GetParquetDF()
+#     documentsToMeasure = []
+#     RelevantDocuments = []
+#     projnameWithDetails = f'{projname} 01 Project Details'
+    
+#     # Split the project name into words and convert to lowercase
+#     matches = set(re.findall(r'\b\w+\b', projnameWithDetails.lower()))
+    
+#     # Convert the 'path_display' column to lowercase for case-insensitive matching
+#     parquetDf['path_display_lower'] = parquetDf['path_display'].str.lower()
+    
+#     # Check if all words in matches are present in the path
+#     mask = parquetDf['path_display_lower'].apply(lambda x: all(match in x for match in matches))
+    
+#     # Filter relevant documents
+#     RelevantDocuments = parquetDf.loc[mask, ['name', 'path_display']].values.tolist()
+    
+#     # Filter documents to measure
+#     documentsToMeasure = parquetDf.loc[(mask) & (parquetDf['name'].str.endswith('.pdf')), ['name', 'path_display']].values.tolist()
+    
+#     # Remove the temporary 'path_display_lower' column
+#     parquetDf.drop(columns=['path_display_lower'], inplace=True)
+#     return documentsToMeasure,RelevantDocuments
+
 def retrieveProjects(projname):
   print('retrieve')
+
   parquetDf=GetParquetDF()
-  print('p',parquetDf)
-  documentsToMeasure=[]
-  RelevantDocuments=[]
-  projnameWithDetails=''
-  projnameWithDetails=projname+' 01 Project Details'
-  matches=re.split(r'[`\-= ~!@#$%^&*()_+\[\]{};\'\\:"|<,/<>?]' , projnameWithDetails.lower())
-  for s in parquetDf['path_display']:
-    if all(x in s.lower() for x in matches):
-        name=parquetDf.loc[parquetDf['path_display'] == s, 'name'].iloc[0]
-        path=parquetDf.loc[parquetDf['name'] == name, 'path_display'].iloc[0]
-        RelevantDocuments.append([name,path])
-        if name.endswith('.pdf'):
-          documentsToMeasure.append([name,path])
-  return documentsToMeasure,RelevantDocuments
+  documentsToMeasure = []
+  RelevantDocuments = []
+  projnameWithDetails = f'{projname} 01 Project Details'
+  # Split the project name into words and convert to lowercase
+  matches = set(re.split(r'[`\-= ~!@#$%^&*()_+\[\]{};\'\\:"|<,/<>?]', projnameWithDetails.lower()))
+
+  # Convert the 'path_display' column to lowercase for case-insensitive matching
+  parquetDf['path_display_lower'] = parquetDf['path_display'].str.lower()
+
+  # Create a mask to filter relevant documents
+  mask = parquetDf['path_display_lower'].apply(lambda x: all(match in x for match in matches))
+
+  # Filter RelevantDocuments and documentsToMeasure using the mask
+  RelevantDocuments = parquetDf[mask][['name', 'path_display']].values.tolist()
+  documentsToMeasure = RelevantDocuments  # Filter documentsToMeasure for PDF files later if needed
+
+  # Remove the temporary 'path_display_lower' column
+  parquetDf.drop(columns=['path_display_lower'], inplace=True)
+  return documentsToMeasure,RelevantDocuments
+
+
+
+