Update bs4_quality_scanner.py

bs4_quality_scanner.py

@@ -8,8 +8,7 @@ url = os.environ.get("SUPABASE_URL")
 key = os.environ.get("SUPABASE_SERVICE_ROLE_KEY")
 supabase = create_client(url, key)
 
-# --- 定义每个字段的“正常范围”和“异常规则” ---
-# min_len: 低于此长度疑似抓取不全; max_len: 高于此长度疑似抓错位置或吞了后面内容
+# --- 扩展后的字段规则字典 (同步 JSON 补齐字段) ---
 FIELD_RULES = {
     'latin_name': {'min': 5, 'max': 80},
     'common_name': {'min': 2, 'max': 100},
@@ -24,11 +23,17 @@ FIELD_RULES = {
     'medicinal_uses': {'min': 10, 'max': 5000},
     'other_uses': {'min': 10, 'max': 5000},
     'cultivation_details': {'min': 20, 'max': 8000},
-    'propagation': {'min': 20, 'max': 4000}
+    'propagation': {'min': 20, 'max': 4000},
+    # --- 新增补齐字段 ---
+    'weed_potential': {'min': 2, 'max': 50},
+    'found_in': {'min': 5, 'max': 5000},
+    'conservation_status': {'min': 5, 'max': 500},
+    'special_uses': {'min': 5, 'max': 1000},
+    'author': {'min': 1, 'max': 50}
 }
 
 def analyze_value_pro(field_name, val):
-    """增强版检测：判定空值、乱码及离群异常"""
+    """深度检测逻辑"""
     raw_s = str(val or "")
     s = raw_s.strip()
     
@@ -39,31 +44,30 @@ def analyze_value_pro(field_name, val):
     # 1. 乱码检测
     has_garbage = bool(re.search(r'[ÂÃÅÐÑÒÓÔÕÖ×Ø]', s))
     
-    # 2. 离群值/异常检测 (Outlier)
+    # 2. 离群值检测
     is_outlier = False
     length = len(s)
     
     if field_name in FIELD_RULES:
         rule = FIELD_RULES[field_name]
-        # 长度异常判定
         if length < rule['min'] or length > rule['max']:
             is_outlier = True
-        # 结构异常判定：如果包含超过 5 个连续空格，或者包含 HTML 标签残余
-        if "     " in raw_s or "<div>" in raw_s.lower() or "<span>" in raw_s.lower():
+        # 结构异常判定：HTML残留或异常空白
+        if "     " in raw_s or "<div>" in raw_s.lower():
             is_outlier = True
 
     return False, (has_garbage or is_outlier), length
 
 def run_bs4_audit():
     target_columns = list(FIELD_RULES.keys())
-    # 增加 outlier 统计项
     stats = {col: {'filled': 0, 'empty': 0, 'outlier': 0, 'len_sum': 0, 'score_sum': 0} for col in target_columns}
     
-    log(f"\n[{time.strftime('%H:%M:%S')}] 🛡️ 启动离群值扫描 (深度体检模式)...")
+    log(f"\n[{time.strftime('%H:%M:%S')}] 🛡️ 启动全字段深度扫描 (含新增字段)...")
     
     offset = 0
     limit = 200
     while True:
+        # 动态获取字段
         res = supabase.table("bs4_plants").select(",".join(target_columns)).range(offset, offset + limit - 1).execute()
         if not res.data: break
         
@@ -79,21 +83,18 @@ def run_bs4_audit():
                     if is_bad:
                         stats[col]['outlier'] += 1
                 
-                # 评分逻辑优化：离群异常和乱码按 50 分计（警告级），空值 0 分
-                if is_emp: score = 0
-                elif is_bad: score = 50 
-                else: score = 100
+                # 质量评分
+                score = 0 if is_emp else (50 if is_bad else 100)
                 stats[col]['score_sum'] += score
         
         offset += len(res.data)
 
-    # --- 控制台可视化报告 ---
-    log("\n" + "="*95)
+    # --- 输出可视化表格 ---
+    log("\n" + "="*100)
     log(f"{'字段名称':<25} | {'填充':<6} | {'异常/离群':<10} | {'空缺':<6} | {'均长':<6} | {'健康分'}")
-    log("-" * 95)
+    log("-" * 100)
     
     report_batch = []
-    # 按健康分从低到高排序
     sorted_cols = sorted(stats.items(), key=lambda x: (x[1]['score_sum']/offset if offset>0 else 0))
     
     for col, data in sorted_cols:
@@ -102,26 +103,26 @@ def run_bs4_audit():
         
         log(f"{col:<25} | {data['filled']:<6} | {data['outlier']:<10} | {data['empty']:<6} | {avg_len:<6} | {total_score}")
         
-        # 为了兼容你之前的数据库表，我们将 outlier 存入原本的 dirty_count 字段，或者你可以扩充字段
         report_batch.append({
             "field_name": col,
             "total_records": offset,
             "filled_count": data['filled'],
             "empty_count": data['empty'],
-            "dirty_count": data['outlier'], # 现在这个字段代表广义的“坏数据”
+            "dirty_count": data['outlier'],
             "avg_char_count": avg_len,
             "quality_score": total_score,
             "updated_at": "now()"
         })
 
+    # 回写到统计表
     supabase.table("bs4_field_quality_report").upsert(report_batch).execute()
-    log("="*95)
-    log(f"✅ 深度巡检完成。异常记录已标记，下次巡检 10 分钟后...")
+    log("="*100)
+    log(f"✅ 巡检完成。共处理 {offset} 条记录。")
 
 if __name__ == "__main__":
-    while True:
-        try:
-            run_bs4_audit()
-        except Exception as e:
-            log(f"❌ 运行报错: {e}")
-        time.sleep(600)
+    # 保持之前商定的逻辑：启动时运行一遍。
+    # 如果想手动触发，重启 Space 即可。
+    try:
+        run_bs4_audit()
+    except Exception as e:
+        log(f"❌ 运行报错: {e}")