anumaurya114exp commited on
Commit
68a4e98
·
1 Parent(s): 0f256eb

using huggingface file system instead of commit schedular

Browse files
Files changed (1) hide show
  1. persistStorage.py +5 -8
persistStorage.py CHANGED
@@ -1,4 +1,5 @@
1
- from huggingface_hub import CommitScheduler
 
2
  from datetime import datetime
3
  import pytz
4
  import os
@@ -12,17 +13,13 @@ TIMEZONE_OBJ = pytz.timezone(TIME_ZONE)
12
  LOGS_DIR = "logs"
13
  os.makedirs(LOGS_DIR, exist_ok=True)
14
 
15
- scheduler = CommitScheduler(
16
- repo_id="QueryHelper",
17
- folder_path=LOGS_DIR,
18
- path_in_repo="data",
19
- token=HUGGING_FACE_TOKEN
20
- )
21
 
22
  def append_dict_to_csv(file_path, row_data):
23
  fieldnames = row_data.keys()
24
 
25
- with open(file_path, 'a+', newline='\n') as csv_file:
26
  csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
27
 
28
  # Check if the file is empty, and if so, write the header
 
1
+ from huggingface_hub import HfFileSystem
2
+
3
  from datetime import datetime
4
  import pytz
5
  import os
 
13
  LOGS_DIR = "logs"
14
  os.makedirs(LOGS_DIR, exist_ok=True)
15
 
16
+ fs = HfFileSystem(token=HUGGING_FACE_TOKEN)
17
+
 
 
 
 
18
 
19
  def append_dict_to_csv(file_path, row_data):
20
  fieldnames = row_data.keys()
21
 
22
+ with fs.open(file_path, 'a+', newline='\n') as csv_file:
23
  csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
24
 
25
  # Check if the file is empty, and if so, write the header