Beracles commited on
Commit
9ee80d0
·
1 Parent(s): 40c3225

优化日志推送逻辑,简化代码并修复日期判断

Browse files
Files changed (1) hide show
  1. logging_helper.py +19 -35
logging_helper.py CHANGED
@@ -75,35 +75,20 @@ class LoggingHelper:
75
 
76
  def push_yesterday(self) -> bool:
77
  try:
 
78
  year = self.today.year.__str__()
79
  month = self.today.month.__str__()
80
  day = self.today.day.__str__()
81
  remotedir = "/".join([year, month, day])
82
- files_to_push = []
83
- for filename in self.buffer.keys():
84
- if not filename.startswith(remotedir):
85
- continue
86
- files_to_push.append(filename)
87
- if len(files_to_push) == 0:
88
- return True
89
- print("[push_yesterday] Writing datasets to json files")
90
- for filename in files_to_push:
91
- localpath = os.sep.join([self.local_dir, filename])
92
- self.buffer[filename].to_json(localpath)
93
- files_to_push.append(filename)
94
- print(f"[push_yesterday] {filename} finished")
95
- print("[push_yesterday] Done")
96
- print("[push_yesterday] Pushing log files to remote")
97
- if len(files_to_push):
98
- localdir = os.sep.join([self.local_dir, remotedir])
99
- res = hf.upload_folder(
100
- repo_id=self.repo_id,
101
- folder_path=localdir,
102
- path_in_repo=remotedir,
103
- repo_type=self.repo_type,
104
- commit_message=f"Updated at {beijing()}",
105
- )
106
- print(f"[push_yesterday] Log files pushed to {res}")
107
  print("[push_yesterday] Done")
108
  return True
109
  except Exception as e:
@@ -112,22 +97,23 @@ class LoggingHelper:
112
 
113
  def push(self):
114
  try:
115
- now = beijing().date()
116
- if now != self.today: # new day comes
117
- if not self.push_yesterday():
118
- print("[push] Failed to upload yesterday's log files")
119
- self.today = now
120
  files_to_push = [
121
  filename for filename in self.need_push if self.need_push[filename]
122
  ]
123
- if len(files_to_push) == 0:
124
- return True
125
  print("[push] Writing datasets to json files")
126
  for filename in files_to_push:
127
  localpath = os.path.join(self.local_dir, filename)
128
  self.buffer[filename].to_json(localpath)
129
  print(f"[push] {filename} finished")
130
- print("[push] Done")
 
 
 
 
 
 
 
 
131
  print("[push] Pushing log files to remote")
132
  remotedir = self.remotedir()
133
  localdir = "/".join([self.local_dir, remotedir])
@@ -138,8 +124,6 @@ class LoggingHelper:
138
  repo_type=self.repo_type,
139
  commit_message=f"Updated at {beijing()}",
140
  )
141
- for filename in files_to_push:
142
- self.need_push[filename] = False
143
  print(f"[push] Log files pushed to {res}")
144
  print("[push] Done")
145
  return True
 
75
 
76
  def push_yesterday(self) -> bool:
77
  try:
78
+ print("[push_yesterday] Pushing yesterday's log files to remote")
79
  year = self.today.year.__str__()
80
  month = self.today.month.__str__()
81
  day = self.today.day.__str__()
82
  remotedir = "/".join([year, month, day])
83
+ localdir = os.sep.join([self.local_dir, remotedir])
84
+ res = hf.upload_folder(
85
+ repo_id=self.repo_id,
86
+ folder_path=localdir,
87
+ path_in_repo=remotedir,
88
+ repo_type=self.repo_type,
89
+ commit_message=f"Updated at {beijing()}",
90
+ )
91
+ print(f"[push_yesterday] Log files pushed to {res}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
92
  print("[push_yesterday] Done")
93
  return True
94
  except Exception as e:
 
97
 
98
  def push(self):
99
  try:
 
 
 
 
 
100
  files_to_push = [
101
  filename for filename in self.need_push if self.need_push[filename]
102
  ]
 
 
103
  print("[push] Writing datasets to json files")
104
  for filename in files_to_push:
105
  localpath = os.path.join(self.local_dir, filename)
106
  self.buffer[filename].to_json(localpath)
107
  print(f"[push] {filename} finished")
108
+ self.need_push[filename] = False
109
+ now = beijing().date()
110
+ if len(files_to_push) == 0:
111
+ print("[push] Done")
112
+ return True
113
+ if now != self.today: # new day comes
114
+ if not self.push_yesterday():
115
+ print("[push] Failed to upload yesterday's log files")
116
+ self.today = now
117
  print("[push] Pushing log files to remote")
118
  remotedir = self.remotedir()
119
  localdir = "/".join([self.local_dir, remotedir])
 
124
  repo_type=self.repo_type,
125
  commit_message=f"Updated at {beijing()}",
126
  )
 
 
127
  print(f"[push] Log files pushed to {res}")
128
  print("[push] Done")
129
  return True