Upload tsadropboxretrieval.py

tsadropboxretrieval.py

@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+"""TSADropboxRetrieval.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1d-UI3Y-z7Dj-vqu69CxluOUnN4rvsUuE
+"""
+
+# !pip install dropbox -q
+
+# pip install pymupdf #==1.22.5
+
+import base64
+import requests
+import json
+import pathlib
+import pandas as pd
+import dropbox
+from dropbox.exceptions import AuthError
+import fitz
+import io
+import re
+import pyarrow
+
+
+"""### NEW CODE - OCTOBER 26 - Marthe"""
+
+files_list=[]
+
+app_key='9bljerefjumct38'
+app_secret='nl6k66clw1j1k12'
+access_code='sl.Bou05Rb15xPy851-I1UV8oOabHPY21AEPl5nrYl-Q0ninFSy0kTuRWPSve_JPbd3Z03E7eBY4r9R454rdzDM0AxLkyqrQEDzyAGUwP7kZ7s2CR6EwvdLD2a7Xh8nFEs38voLTH2IHzrQ2QEx7rji4OJ8aSQStKtJkI7_dh8tYHj5'
+refresh_token='qK2VqvbxWMMAAAAAAAAAAXFQvrHM4xUwWUcZ6l5vGOygn1iAA6zlDjmAQNBbZprL'
+basic_auth=base64.b64encode(f'{app_key}:{app_secret}'.encode())
+
+def ADR_Access_DropboxTeam(flag):
+  if flag=='user':
+    dbxTeam = dropbox.DropboxTeam(app_key=app_key,
+              app_secret=app_secret,
+              oauth2_refresh_token = refresh_token).as_user('dbmid:AACjJg2GKc3tI42iOnD01dd6s0XDyyx6Thw')
+  elif flag=='admin':
+    dbxTeam = dropbox.DropboxTeam(app_key=app_key,
+              app_secret=app_secret,
+              oauth2_refresh_token = refresh_token).as_admin('dbmid:AACjJg2GKc3tI42iOnD01dd6s0XDyyx6Thw')
+  root_namespace_id = dbxTeam.users_get_current_account().root_info.root_namespace_id
+  dbxTeam = dbxTeam.with_path_root(dropbox.common.PathRoot.root(root_namespace_id))
+  return dbxTeam
+
+def getSharedLink(path):
+  dbxTeam=ADR_Access_DropboxTeam('user')
+  try:
+    shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings(path)
+  except:
+    shared_link_metadata=dbxTeam.sharing_create_shared_link(path)
+  return shared_link_metadata.url
+
+def handle_entries(entries):
+
+  for file in entries:
+      if isinstance(file, dropbox.files.FileMetadata):
+      #     if str(file.name).endswith(".pdf"):
+          metadata = {
+                    'name': file.name,
+                    'path_display': file.path_display,
+                    'client_modified': file.client_modified,
+                    'server_modified': file.server_modified
+                }
+          files_list.append(metadata)
+  df = pd.DataFrame.from_records(files_list)
+  return df
+
+def dropbox_connect():
+    """Create a connection to myyyyyyy Dropbox."""
+    print('connecy')
+    try:
+        # print('ayhaga')
+        dbxMe = dropbox.Dropbox(
+            app_key='67w6ibpa9d2b60x',
+            app_secret='d3ecz8g1604fu04',
+            oauth2_refresh_token = 'R_LACBBNhysAAAAAAAAAAXt9mMy9OYIV_v4pF45lG6Z8DHNV66rq1q7acWjj_H5g',
+
+        )
+        # dbx=dropbox.Dropbox(access_token)
+
+    except AuthError as e:
+        print('Error connecting to Dropbox with access token: ' + str(e))
+    return dbxMe
+
+
+def dropbox_upload_file(df, flag=0):
+    try:
+        dbxTeam= ADR_Access_DropboxTeam('admin')
+        path='/TSA Team Folder/ADR Test/DropboxDirectory/df.parquet.gzip'
+        doc=df.to_parquet()
+        meta=dbxTeam.files_upload(doc,path,mode=dropbox.files.WriteMode("overwrite"))
+
+    except Exception as e:
+        print('Error uploading file to Dropbox: ' + str(e))
+    return dbxTeam
+
+
+def uploadanyFile(doc,pdfname,path):
+    try:
+        dbxTeam= ADR_Access_DropboxTeam('admin')
+        path=path+pdfname
+        meta=dbxTeam.files_upload( doc.write() ,path,mode=dropbox.files.WriteMode("overwrite"))
+        try:
+          shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings(path)
+        except:
+          shared_link_metadata=dbxTeam.sharing_create_shared_link(path)
+        return shared_link_metadata.url
+    except Exception as e:
+        print('Error uploading file to Dropbox: ' + str(e))
+    
+    return 'Error uploading file to Dropbox.'
+
+# Call when the dropbox is updated with new items - if not , call parquet saved version of the df of saved items
+def DropboxItemstoDF():
+  dbxTeam=ADR_Access_DropboxTeam('user') # or pass dbx in parameters
+  folder_path = "/TSA Team Folder"
+  res = dbxTeam.files_list_folder(path=folder_path, recursive=True)
+  df1=handle_entries(res.entries)
+  while res.has_more:
+      res = dbxTeam.files_list_folder_continue(cursor=res.cursor)
+      df2=handle_entries(res.entries)
+
+  dbxTeam=dropbox_upload_file(df2)
+  return df2
+
+# df2=DropboxItemstoDF()
+
+# Feather format for storing data
+# def ToFeather(df2):
+#   df2.to_feather('df2.feather')
+#   fthr=pd.read_feather('df2.feather')
+#   return fthr
+
+def GetParquetDF():
+  # # read the parquet file in current directory, back into a pandas data frame
+  dbxTeam=ADR_Access_DropboxTeam('user') # or pass dbx in parameters
+  try:
+    shared_link_metadata = dbxTeam.sharing_create_shared_link_with_settings( path='/TSA Team Folder/ADR Test/DropboxDirectory/df.parquet.gzip')
+  except:
+    shared_link_metadata=dbxTeam.sharing_create_shared_link( path='/TSA Team Folder/ADR Test/DropboxDirectory/df.parquet.gzip')
+  metadata, res = dbxTeam.sharing_get_shared_link_file(url=shared_link_metadata.url)
+  data=res.content  # or res.content, or iter_content, or iter_lines, etc. as needed
+
+  pq_file = io.BytesIO(data)
+  df = pd.read_parquet(pq_file)
+  return df
+def getPathtoPDF_File(nameofPDF):
+  parquetDf=GetParquetDF()
+  nameofPDF=nameofPDF.replace('"', '')
+  try:
+    path=parquetDf.loc[parquetDf['name'] == nameofPDF, 'path_display'].iloc[0]
+    link=getSharedLink(path)
+  except:
+    return 'Project does not exist'
+  return path,link
+# parquetDf
+
+# getPathtoPDF_File('A5157-EBLA-V5-XX-SH-L-0004-D2-01.pdf')
+def getPDFData(path):
+  dbxTeam= ADR_Access_DropboxTeam('admin')
+  md, res =dbxTeam.files_download(path)
+  data = res.content
+  return data
+
+def retrieveProjects(projname):
+  parquetDf=GetParquetDF()
+  documnetsToMeasure=[]
+  RelevantDocuments=[]
+  projnameWithDetails=''
+  projnameWithDetails=projname+' 01 Project Details'
+  matches=re.split(r'[`\-= ~!@#$%^&*()_+\[\]{};\'\\:"|<,/<>?]' , projnameWithDetails.lower())
+  for s in parquetDf['path_display']:
+    if all(x in s.lower() for x in matches):
+        name=parquetDf.loc[parquetDf['path_display'] == s, 'name'].iloc[0]
+        path=parquetDf.loc[parquetDf['name'] == name, 'path_display'].iloc[0]
+        RelevantDocuments.append([name,path])
+        if name.endswith('.pdf'):
+          documnetsToMeasure.append([name,path])
+  return documnetsToMeasure,RelevantDocuments