import pathlib import pandas as pd import dropbox from dropbox.exceptions import AuthError from datetime import datetime import time # from main import keep_alive import re import os import tsadropboxretrieval # active_path = "/TSA Team Folder/ADR Test/" # file1 = open("newprojects.txt", "w") # file1.close() # ###################################################### # def jan_update(): #if month==1 call this function to change active path # year = datetime.now().date().year # df = dropbox_list_files('') # fol = [] # for l0 in df.name.values: # matched = re.findall('.*' + str(year) + '.pdf', l0) # #print(l0) # if matched: # fol.append(matched) # if len(fol) > 0: # filtered = df[df.name == fol[0][0]] # path = filtered.iloc[0]['path_display'] # active_path = '/' + path.split('/')[1] # return active_path # else: # return # ####################################################### # def feb_update(reponame): # if month=2, clean 2022 folder # year = datetime.now().date().year # li = api.list_repo_files( # repo_id=reponame, # repo_type='space', # ) # found = [] # for i in li: #match regex to find prev year files # matched = re.findall(str(year - 1)[2:4] + '.*.pdf$', i) # if matched: # found.append(matched) # for f in found: #delete files # print(f[0]) # api.delete_file( # path_in_repo=f[0], # repo_id=reponame, # repo_type="space", # ) # ####################################################### # #sorting/cleaning files(Jan & Feb only) # def yearly_update(): # month = datetime.now().date().month # day=datetime.now().date().day # if month == 1 and day == 10: #move to new years folder # global active_path # active_path = jan_update() # elif month == 2 and day == 1: # print('feb') # feb_update('alaaadr/SearchDocuments') # feb_update('alaaadr/ImageSearch') # feb_update('alaaadr/Measurements') ############################################ def download_NewlyAddedFiles(active_path): today = datetime.now().date() df= tsadropboxretrieval.DropboxItemstoDF(active_path)[0] #print(df.head()) d = df[df.server_modified.dt.date == today].reset_index(drop=True) dfParquet=tsadropboxretrieval.GetParquetDF() dictionaryDF= pd.DataFrame(columns=['name','path_display','client_modified','server_modified']) # dictionaryDF={} for i in range(0, len(d)): #print('am here') down_name = d.iloc[i]['name'] path = d.iloc[i]['path_display'] clientDate = d.iloc[i]['client_modified'] serverDate = d.iloc[i]['server_modified'] dfParquet.loc[-1]=[down_name,path,clientDate,serverDate] dfParquet.index = dfParquet.index + 1 # shifting index dfParquet = dfParquet.sort_index() # sorting by index print(dfParquet) tsadropboxretrieval.dropbox_upload_file(dfParquet) # parquetDF=tsadropboxretrieval.GetParquetDF() # Append-adds at last file1 = open("newprojects.txt", "a") # append mode file1.write(str(dictionaryDF)) file1.close() return dictionaryDF #download todays files # dropbox_download_file(path, 'dropbox/' + down_name) #push to hf # push_to_hf() ################################################################################################# # def dropbox_connect(): # """Create a connection to Dropbox.""" # try: # dbx = dropbox.Dropbox(app_key='cujlb5a2hoesvpy', # app_secret='eb8khe4fuqda1kh', # oauth2_refresh_token=refresh_token) # except AuthError as e: # print('Error connecting to Dropbox with access token: ' + str(e)) # return dbx ####################################################### def dropbox_list_files(path): """Return a Pandas dataframe of files in a given Dropbox folder path in the Apps directory. """ dbx = tsadropboxretrieval.ADR_Access_DropboxTeam('admin') print(dbx) try: files = dbx.files_list_folder(path, recursive=True) files_list = [] for file in files: if isinstance(file, dropbox.files.FileMetadata): metadata = { 'name': file.name, 'path_display': file.path_display, 'client_modified': file.client_modified, 'server_modified': file.server_modified } files_list.append(metadata) df = pd.DataFrame.from_records(files_list) return df.sort_values(by='server_modified', ascending=False) except Exception as e: print('Error getting list of files from Dropbox: ' + str(e)) ############################################### # def dropbox_download_file(dropbox_file_path, local_file_path): # """Download a file from Dropbox to the local machine.""" # print('downloading....') # try: # dbx = dropbox_connect() # with open(local_file_path, 'wb') as f: # metadata, result = dbx.files_download(path=dropbox_file_path) # f.write(result.content) # except Exception as e: # print('Error downloading file from Dropbox: ' + str(e))