PhilipL commited on
Commit
0a2bede
·
unverified ·
1 Parent(s): ca78357

fix function sync_from_hf

Browse files
Files changed (1) hide show
  1. logger_service.py +18 -7
logger_service.py CHANGED
@@ -1,19 +1,34 @@
1
  import os
2
  import pandas as pd
3
- from huggingface_hub import upload_file, login
4
 
5
  # --- 配置區 ---
6
  DATASET_REPO_ID = "My-Project-Team/Digital-Bodyguard-Dataset"
7
  CSV_FILENAME = "baseline_logs.csv"
8
  HF_TOKEN = os.getenv("HF_TOKEN")
9
 
10
- # 🌟 新增:批次上傳門檻 (每累積幾筆資料才備份到雲端一次)
11
  BATCH_SIZE = 30
12
- unsynced_count = 0 # 計算目前累積了幾筆未上傳的操作
13
 
14
  if HF_TOKEN:
15
  login(token=HF_TOKEN)
16
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  def upload_to_hf():
18
  """將本地的 CSV 檔案同步回雲端"""
19
  try:
@@ -37,19 +52,15 @@ def push_new_log(new_log_dict):
37
  """
38
  global unsynced_count
39
 
40
- # 每收到一筆日誌,計數器 +1
41
  unsynced_count += 1
42
 
43
- # 檢查是否達到批次上傳的門檻
44
  if unsynced_count >= BATCH_SIZE:
45
  print(f"📦 已累積 {BATCH_SIZE} 筆操作,正在打包同步至 Hugging Face...")
46
  success = upload_to_hf()
47
 
48
- # 如果上傳成功,就把計數器歸零,重新開始算;如果失敗就保留計數,下次再試
49
  if success:
50
  print("✅ 批次同步成功!")
51
  unsynced_count = 0
52
  return success
53
 
54
- # 還沒滿 30 筆,就不做任何雲端連線,讓網頁瞬間跑完!
55
  return True
 
1
  import os
2
  import pandas as pd
3
+ from huggingface_hub import hf_hub_download, upload_file, login
4
 
5
  # --- 配置區 ---
6
  DATASET_REPO_ID = "My-Project-Team/Digital-Bodyguard-Dataset"
7
  CSV_FILENAME = "baseline_logs.csv"
8
  HF_TOKEN = os.getenv("HF_TOKEN")
9
 
10
+ # 🌟 批次上傳門檻 (每累積幾筆資料才備份到雲端一次)
11
  BATCH_SIZE = 30
12
+ unsynced_count = 0
13
 
14
  if HF_TOKEN:
15
  login(token=HF_TOKEN)
16
 
17
+ # 🌟 把這個啟動時需要用到的「下載功能」加回來!
18
+ def sync_from_hf():
19
+ """從雲端下載最新的 CSV 並回傳 DataFrame (系統啟動時只會呼叫一次)"""
20
+ try:
21
+ path = hf_hub_download(
22
+ repo_id=DATASET_REPO_ID,
23
+ filename=CSV_FILENAME,
24
+ repo_type="dataset",
25
+ token=HF_TOKEN
26
+ )
27
+ return pd.read_csv(path)
28
+ except Exception as e:
29
+ print(f"⚠️ 無法下載雲端資料,可能為首次運行: {e}")
30
+ return pd.DataFrame(columns=["timestamp", "ip_address", "cookie_id", "account", "role", "action", "status"])
31
+
32
  def upload_to_hf():
33
  """將本地的 CSV 檔案同步回雲端"""
34
  try:
 
52
  """
53
  global unsynced_count
54
 
 
55
  unsynced_count += 1
56
 
 
57
  if unsynced_count >= BATCH_SIZE:
58
  print(f"📦 已累積 {BATCH_SIZE} 筆操作,正在打包同步至 Hugging Face...")
59
  success = upload_to_hf()
60
 
 
61
  if success:
62
  print("✅ 批次同步成功!")
63
  unsynced_count = 0
64
  return success
65
 
 
66
  return True