mengtaoguo commited on
Commit
b08675f
·
verified ·
1 Parent(s): fd4a31c

Update quality_audit_job.py

Browse files
Files changed (1) hide show
  1. quality_audit_job.py +15 -16
quality_audit_job.py CHANGED
@@ -2,20 +2,18 @@ import os
2
  import re
3
  import time
4
  import sys
5
- import logging
6
  from supabase import create_client
7
 
8
  # 强制不缓存输出
9
- logging.basicConfig(level=logging.INFO, format='%(message)s', stream=sys.stdout)
10
- logger = logging.getLogger(__name__)
11
 
12
- # --- 核心修正点:匹配你截图里的变量名 ---
13
  url = os.environ.get("SUPABASE_URL")
14
- # 尝试读取截图中的 SERVICE_ROLE_KEY
15
  key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY")
16
 
17
  if not url or not key:
18
- print(f"❌ 环境变量缺失! URL: {'OK' if url else 'MISSING'}, KEY: {'OK' if key else 'MISSING'}", flush=True)
19
  sys.exit(1)
20
 
21
  supabase = create_client(url, key)
@@ -39,29 +37,30 @@ def analyze_text(text):
39
  return {"is_filled": not is_empty, "char_count": char_count, "has_garbage_chars": has_garbage, "quality_score": score}
40
 
41
  def run_audit():
42
- # 你的数据湖大小
43
  total_count = 7038
44
- logger.info(f"🚀 审计引擎启动 | 目标量: {total_count}", flush=True)
45
 
46
  while True:
47
  try:
48
- # --- 修正点:不使用 RPC,改用普通 COUNT ---
49
- # 每个植物审计3个字段,所以 count / 3
50
  count_res = supabase.table("data_quality_audit").select("id", count="exact").execute()
51
- processed_plants = (count_res.count or 0) // 3
52
 
53
- # 打印进度条
54
- print(f"📊 进度: {get_progress_bar(processed_plants, total_count)} ({processed_plants}/{total_count})", flush=True)
 
55
 
56
- # 水位线逻辑
57
  last_res = supabase.table("data_quality_audit").select("updated_at").order("updated_at", desc=True).limit(1).execute()
58
  last_time = last_res.data[0]['updated_at'] if last_res.data else "1970-01-01T00:00:00Z"
59
 
60
- # 抓取主表变动
61
  res = supabase.table("bs4_plants").select("plant_id, edible_uses, medicinal_uses, cultivation_details, updated_at")\
62
  .gt("updated_at", last_time).order("updated_at").limit(20).execute()
63
 
64
  if not res.data:
 
65
  time.sleep(30)
66
  continue
67
 
@@ -76,7 +75,7 @@ def run_audit():
76
  }).execute()
77
 
78
  except Exception as e:
79
- print(f"❌ 运行异常: {e}", flush=True)
80
  time.sleep(10)
81
 
82
  if __name__ == "__main__":
 
2
  import re
3
  import time
4
  import sys
 
5
  from supabase import create_client
6
 
7
  # 强制不缓存输出
8
+ def log(msg):
9
+ print(msg, flush=True)
10
 
11
+ # 1. 匹配你 Space 里的真实变量名
12
  url = os.environ.get("SUPABASE_URL")
 
13
  key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY")
14
 
15
  if not url or not key:
16
+ log(f"❌ 环境变量缺失! URL: {'OK' if url else 'MISSING'}, KEY: {'OK' if key else 'MISSING'}")
17
  sys.exit(1)
18
 
19
  supabase = create_client(url, key)
 
37
  return {"is_filled": not is_empty, "char_count": char_count, "has_garbage_chars": has_garbage, "quality_score": score}
38
 
39
  def run_audit():
40
+ # 你的数据湖总数
41
  total_count = 7038
42
+ log(f"🚀 审计引擎启动 | 目标量: {total_count}")
43
 
44
  while True:
45
  try:
46
+ # 2. 统计已审计数量 (每个植物有 3 个字段,所以除以 3)
 
47
  count_res = supabase.table("data_quality_audit").select("id", count="exact").execute()
48
+ processed_count = (count_res.count or 0) // 3
49
 
50
+ # 显示进度条
51
+ bar = get_progress_bar(processed_count, total_count)
52
+ log(f"📊 进度: {bar} ({processed_count}/{total_count})")
53
 
54
+ # 3. 获取水位线 (基于审计表的最后更新时间)
55
  last_res = supabase.table("data_quality_audit").select("updated_at").order("updated_at", desc=True).limit(1).execute()
56
  last_time = last_res.data[0]['updated_at'] if last_res.data else "1970-01-01T00:00:00Z"
57
 
58
+ # 4. 抓取主表变动
59
  res = supabase.table("bs4_plants").select("plant_id, edible_uses, medicinal_uses, cultivation_details, updated_at")\
60
  .gt("updated_at", last_time).order("updated_at").limit(20).execute()
61
 
62
  if not res.data:
63
+ # 如果没数据,休眠 30 秒
64
  time.sleep(30)
65
  continue
66
 
 
75
  }).execute()
76
 
77
  except Exception as e:
78
+ log(f"❌ 运行异常: {e}")
79
  time.sleep(10)
80
 
81
  if __name__ == "__main__":