from huggingface_hub import HfFileSystem from datetime import datetime import pytz import os from config import HUGGING_FACE_TOKEN import csv logsDir = os.getenv("HF_HOME", "/data") # # Create a new file # with open(os.path.join(data_dir, "my_data.txt"), "a") as f: # f.write("Hello World! From pesistent storage line 2") # # Read the data from the file # with open(os.path.join(data_dir, "my_data.txt"), "r") as f: # data = f.read() # # Print the data # print(data) # Set the time zone to Pacific Time Zone TIME_ZONE = 'US/Pacific' TIMEZONE_OBJ = pytz.timezone(TIME_ZONE) def append_dict_to_csv(file_path, row_data): fieldnames = row_data.keys() with open(file_path, 'a') as csv_file: csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames) # Check if the file is empty, and if so, write the header if csv_file.tell() == 0: csv_writer.writeheader() csv_writer.writerow(row_data) def saveLog(message, level='info') -> None: global logsDir if not os.path.isdir(logsDir): print("Log directory/Data Directory not available.") return current_time = datetime.now(TIMEZONE_OBJ) message = str(message) log_file_path = os.path.join(logsDir, f"{current_time.strftime('%Y-%m')}-log.csv") data_dict = {"time":str(current_time), "level": level, "message": message} append_dict_to_csv(log_file_path, data_dict) def getAllLogFilesPaths(): global logsDir # Save processed data to temporary file if not os.path.isdir(logsDir): print("Log directory/Data Directory not available.") return [] logFiles = [file for file in os.listdir(logsDir) if 'log' in file.lower()] print(logFiles,"avaiable logs") downloadableFilesPaths = [os.path.join(os.path.abspath(logsDir), logFilePath) for logFilePath in logFiles] return downloadableFilesPaths