Spaces:
Running
Running
优化日志推送逻辑,简化代码并修复日期判断
Browse files- logging_helper.py +19 -35
logging_helper.py
CHANGED
|
@@ -75,35 +75,20 @@ class LoggingHelper:
|
|
| 75 |
|
| 76 |
def push_yesterday(self) -> bool:
|
| 77 |
try:
|
|
|
|
| 78 |
year = self.today.year.__str__()
|
| 79 |
month = self.today.month.__str__()
|
| 80 |
day = self.today.day.__str__()
|
| 81 |
remotedir = "/".join([year, month, day])
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
localpath = os.sep.join([self.local_dir, filename])
|
| 92 |
-
self.buffer[filename].to_json(localpath)
|
| 93 |
-
files_to_push.append(filename)
|
| 94 |
-
print(f"[push_yesterday] {filename} finished")
|
| 95 |
-
print("[push_yesterday] Done")
|
| 96 |
-
print("[push_yesterday] Pushing log files to remote")
|
| 97 |
-
if len(files_to_push):
|
| 98 |
-
localdir = os.sep.join([self.local_dir, remotedir])
|
| 99 |
-
res = hf.upload_folder(
|
| 100 |
-
repo_id=self.repo_id,
|
| 101 |
-
folder_path=localdir,
|
| 102 |
-
path_in_repo=remotedir,
|
| 103 |
-
repo_type=self.repo_type,
|
| 104 |
-
commit_message=f"Updated at {beijing()}",
|
| 105 |
-
)
|
| 106 |
-
print(f"[push_yesterday] Log files pushed to {res}")
|
| 107 |
print("[push_yesterday] Done")
|
| 108 |
return True
|
| 109 |
except Exception as e:
|
|
@@ -112,22 +97,23 @@ class LoggingHelper:
|
|
| 112 |
|
| 113 |
def push(self):
|
| 114 |
try:
|
| 115 |
-
now = beijing().date()
|
| 116 |
-
if now != self.today: # new day comes
|
| 117 |
-
if not self.push_yesterday():
|
| 118 |
-
print("[push] Failed to upload yesterday's log files")
|
| 119 |
-
self.today = now
|
| 120 |
files_to_push = [
|
| 121 |
filename for filename in self.need_push if self.need_push[filename]
|
| 122 |
]
|
| 123 |
-
if len(files_to_push) == 0:
|
| 124 |
-
return True
|
| 125 |
print("[push] Writing datasets to json files")
|
| 126 |
for filename in files_to_push:
|
| 127 |
localpath = os.path.join(self.local_dir, filename)
|
| 128 |
self.buffer[filename].to_json(localpath)
|
| 129 |
print(f"[push] {filename} finished")
|
| 130 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
print("[push] Pushing log files to remote")
|
| 132 |
remotedir = self.remotedir()
|
| 133 |
localdir = "/".join([self.local_dir, remotedir])
|
|
@@ -138,8 +124,6 @@ class LoggingHelper:
|
|
| 138 |
repo_type=self.repo_type,
|
| 139 |
commit_message=f"Updated at {beijing()}",
|
| 140 |
)
|
| 141 |
-
for filename in files_to_push:
|
| 142 |
-
self.need_push[filename] = False
|
| 143 |
print(f"[push] Log files pushed to {res}")
|
| 144 |
print("[push] Done")
|
| 145 |
return True
|
|
|
|
| 75 |
|
| 76 |
def push_yesterday(self) -> bool:
|
| 77 |
try:
|
| 78 |
+
print("[push_yesterday] Pushing yesterday's log files to remote")
|
| 79 |
year = self.today.year.__str__()
|
| 80 |
month = self.today.month.__str__()
|
| 81 |
day = self.today.day.__str__()
|
| 82 |
remotedir = "/".join([year, month, day])
|
| 83 |
+
localdir = os.sep.join([self.local_dir, remotedir])
|
| 84 |
+
res = hf.upload_folder(
|
| 85 |
+
repo_id=self.repo_id,
|
| 86 |
+
folder_path=localdir,
|
| 87 |
+
path_in_repo=remotedir,
|
| 88 |
+
repo_type=self.repo_type,
|
| 89 |
+
commit_message=f"Updated at {beijing()}",
|
| 90 |
+
)
|
| 91 |
+
print(f"[push_yesterday] Log files pushed to {res}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
print("[push_yesterday] Done")
|
| 93 |
return True
|
| 94 |
except Exception as e:
|
|
|
|
| 97 |
|
| 98 |
def push(self):
|
| 99 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
files_to_push = [
|
| 101 |
filename for filename in self.need_push if self.need_push[filename]
|
| 102 |
]
|
|
|
|
|
|
|
| 103 |
print("[push] Writing datasets to json files")
|
| 104 |
for filename in files_to_push:
|
| 105 |
localpath = os.path.join(self.local_dir, filename)
|
| 106 |
self.buffer[filename].to_json(localpath)
|
| 107 |
print(f"[push] {filename} finished")
|
| 108 |
+
self.need_push[filename] = False
|
| 109 |
+
now = beijing().date()
|
| 110 |
+
if len(files_to_push) == 0:
|
| 111 |
+
print("[push] Done")
|
| 112 |
+
return True
|
| 113 |
+
if now != self.today: # new day comes
|
| 114 |
+
if not self.push_yesterday():
|
| 115 |
+
print("[push] Failed to upload yesterday's log files")
|
| 116 |
+
self.today = now
|
| 117 |
print("[push] Pushing log files to remote")
|
| 118 |
remotedir = self.remotedir()
|
| 119 |
localdir = "/".join([self.local_dir, remotedir])
|
|
|
|
| 124 |
repo_type=self.repo_type,
|
| 125 |
commit_message=f"Updated at {beijing()}",
|
| 126 |
)
|
|
|
|
|
|
|
| 127 |
print(f"[push] Log files pushed to {res}")
|
| 128 |
print("[push] Done")
|
| 129 |
return True
|