Spaces:
Runtime error
Runtime error
Update quality_audit_job.py
Browse files- quality_audit_job.py +15 -16
quality_audit_job.py
CHANGED
|
@@ -2,20 +2,18 @@ import os
|
|
| 2 |
import re
|
| 3 |
import time
|
| 4 |
import sys
|
| 5 |
-
import logging
|
| 6 |
from supabase import create_client
|
| 7 |
|
| 8 |
# 强制不缓存输出
|
| 9 |
-
|
| 10 |
-
|
| 11 |
|
| 12 |
-
#
|
| 13 |
url = os.environ.get("SUPABASE_URL")
|
| 14 |
-
# 尝试读取截图中的 SERVICE_ROLE_KEY
|
| 15 |
key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY")
|
| 16 |
|
| 17 |
if not url or not key:
|
| 18 |
-
|
| 19 |
sys.exit(1)
|
| 20 |
|
| 21 |
supabase = create_client(url, key)
|
|
@@ -39,29 +37,30 @@ def analyze_text(text):
|
|
| 39 |
return {"is_filled": not is_empty, "char_count": char_count, "has_garbage_chars": has_garbage, "quality_score": score}
|
| 40 |
|
| 41 |
def run_audit():
|
| 42 |
-
# 你的数据湖
|
| 43 |
total_count = 7038
|
| 44 |
-
|
| 45 |
|
| 46 |
while True:
|
| 47 |
try:
|
| 48 |
-
#
|
| 49 |
-
# 每个植物审计3个字段,所以 count / 3
|
| 50 |
count_res = supabase.table("data_quality_audit").select("id", count="exact").execute()
|
| 51 |
-
|
| 52 |
|
| 53 |
-
#
|
| 54 |
-
|
|
|
|
| 55 |
|
| 56 |
-
# 水位线
|
| 57 |
last_res = supabase.table("data_quality_audit").select("updated_at").order("updated_at", desc=True).limit(1).execute()
|
| 58 |
last_time = last_res.data[0]['updated_at'] if last_res.data else "1970-01-01T00:00:00Z"
|
| 59 |
|
| 60 |
-
# 抓取主表变动
|
| 61 |
res = supabase.table("bs4_plants").select("plant_id, edible_uses, medicinal_uses, cultivation_details, updated_at")\
|
| 62 |
.gt("updated_at", last_time).order("updated_at").limit(20).execute()
|
| 63 |
|
| 64 |
if not res.data:
|
|
|
|
| 65 |
time.sleep(30)
|
| 66 |
continue
|
| 67 |
|
|
@@ -76,7 +75,7 @@ def run_audit():
|
|
| 76 |
}).execute()
|
| 77 |
|
| 78 |
except Exception as e:
|
| 79 |
-
|
| 80 |
time.sleep(10)
|
| 81 |
|
| 82 |
if __name__ == "__main__":
|
|
|
|
| 2 |
import re
|
| 3 |
import time
|
| 4 |
import sys
|
|
|
|
| 5 |
from supabase import create_client
|
| 6 |
|
| 7 |
# 强制不缓存输出
|
| 8 |
+
def log(msg):
|
| 9 |
+
print(msg, flush=True)
|
| 10 |
|
| 11 |
+
# 1. 匹配你 Space 里的真实变量名
|
| 12 |
url = os.environ.get("SUPABASE_URL")
|
|
|
|
| 13 |
key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY")
|
| 14 |
|
| 15 |
if not url or not key:
|
| 16 |
+
log(f"❌ 环境变量缺失! URL: {'OK' if url else 'MISSING'}, KEY: {'OK' if key else 'MISSING'}")
|
| 17 |
sys.exit(1)
|
| 18 |
|
| 19 |
supabase = create_client(url, key)
|
|
|
|
| 37 |
return {"is_filled": not is_empty, "char_count": char_count, "has_garbage_chars": has_garbage, "quality_score": score}
|
| 38 |
|
| 39 |
def run_audit():
|
| 40 |
+
# 你的数据湖总数
|
| 41 |
total_count = 7038
|
| 42 |
+
log(f"🚀 审计引擎启动 | 目标量: {total_count}")
|
| 43 |
|
| 44 |
while True:
|
| 45 |
try:
|
| 46 |
+
# 2. 统计已审计数量 (每个植物有 3 个字段,所以除以 3)
|
|
|
|
| 47 |
count_res = supabase.table("data_quality_audit").select("id", count="exact").execute()
|
| 48 |
+
processed_count = (count_res.count or 0) // 3
|
| 49 |
|
| 50 |
+
# 显示进度条
|
| 51 |
+
bar = get_progress_bar(processed_count, total_count)
|
| 52 |
+
log(f"📊 进度: {bar} ({processed_count}/{total_count})")
|
| 53 |
|
| 54 |
+
# 3. 获取水位线 (基于审计表的最后更新时间)
|
| 55 |
last_res = supabase.table("data_quality_audit").select("updated_at").order("updated_at", desc=True).limit(1).execute()
|
| 56 |
last_time = last_res.data[0]['updated_at'] if last_res.data else "1970-01-01T00:00:00Z"
|
| 57 |
|
| 58 |
+
# 4. 抓取主表新变动
|
| 59 |
res = supabase.table("bs4_plants").select("plant_id, edible_uses, medicinal_uses, cultivation_details, updated_at")\
|
| 60 |
.gt("updated_at", last_time).order("updated_at").limit(20).execute()
|
| 61 |
|
| 62 |
if not res.data:
|
| 63 |
+
# 如果没数据,休眠 30 秒
|
| 64 |
time.sleep(30)
|
| 65 |
continue
|
| 66 |
|
|
|
|
| 75 |
}).execute()
|
| 76 |
|
| 77 |
except Exception as e:
|
| 78 |
+
log(f"❌ 运行异常: {e}")
|
| 79 |
time.sleep(10)
|
| 80 |
|
| 81 |
if __name__ == "__main__":
|