Beracles commited on
Commit
33c7184
·
1 Parent(s): 61cccd8

优化日志加载时间戳的处理逻辑,修正时间范围过滤条件

Browse files
Files changed (1) hide show
  1. logging_helper.py +6 -5
logging_helper.py CHANGED
@@ -10,7 +10,6 @@ from huggingface_hub import HfApi
10
  import pandas as pd
11
  from datetime import datetime, date, timedelta
12
  from zoneinfo import ZoneInfo
13
- import glob
14
 
15
  hf = HfApi()
16
  hf.token = os.environ.get("hf_token")
@@ -209,15 +208,15 @@ class LoggingHelper:
209
 
210
  start_timestamp = self.cutoff_timestamp()
211
  end_timestamp = (
212
- beijing()
213
- .replace(hour=23, minute=59, second=59, microsecond=999999)
214
  .isoformat(timespec="microseconds")
215
  )
216
  from_timestamp = from_timestamp or start_timestamp
217
  to_timestamp = to_timestamp or end_timestamp
218
  total_files_loaded = 0
219
  for remotepath, timestamp in self.timestamps.items():
220
- if timestamp < from_timestamp or timestamp >= to_timestamp:
221
  continue
222
  localpath = "/".join([self.local_dir, remotepath])
223
  print(f"[load_logs] Loading file {localpath}")
@@ -349,7 +348,9 @@ class LoggingHelper:
349
  ) + timedelta(days=1)
350
  to_timestamp = to_datetime.isoformat(timespec="microseconds")
351
 
352
- print(f"[refresh] Starting to load logs from {from_date} to {to_date}")
 
 
353
  # 如果查询范围超出缓存范围,则加载相应的日志文件
354
  self.load_logs(from_timestamp=from_timestamp, to_timestamp=to_timestamp)
355
  if self.dataframe_refresh_needed:
 
10
  import pandas as pd
11
  from datetime import datetime, date, timedelta
12
  from zoneinfo import ZoneInfo
 
13
 
14
  hf = HfApi()
15
  hf.token = os.environ.get("hf_token")
 
208
 
209
  start_timestamp = self.cutoff_timestamp()
210
  end_timestamp = (
211
+ (beijing() + timedelta(days=1))
212
+ .replace(hour=0, minute=0, second=0, microsecond=0)
213
  .isoformat(timespec="microseconds")
214
  )
215
  from_timestamp = from_timestamp or start_timestamp
216
  to_timestamp = to_timestamp or end_timestamp
217
  total_files_loaded = 0
218
  for remotepath, timestamp in self.timestamps.items():
219
+ if timestamp < from_timestamp or timestamp > to_timestamp:
220
  continue
221
  localpath = "/".join([self.local_dir, remotepath])
222
  print(f"[load_logs] Loading file {localpath}")
 
348
  ) + timedelta(days=1)
349
  to_timestamp = to_datetime.isoformat(timespec="microseconds")
350
 
351
+ print(
352
+ f"[refresh] Starting to load logs from {from_timestamp} to {to_timestamp}"
353
+ )
354
  # 如果查询范围超出缓存范围,则加载相应的日志文件
355
  self.load_logs(from_timestamp=from_timestamp, to_timestamp=to_timestamp)
356
  if self.dataframe_refresh_needed: