Update app.py

app.py

@@ -14,7 +14,7 @@ import numpy as np
 # ==================== 配置區 ====================
 HF_TOKEN = os.environ.get("HF_TOKEN", "您的_HuggingFace_Token")
 DATASET_REPO_ID = "Paul720810/Text-to-SQL-Softline"
-SIMILARITY_THRESHOLD = 0.70  # 降低閾值，因為很多數據有問題
+SIMILARITY_THRESHOLD = 0.75
 
 # 多個備用LLM模型
 LLM_MODELS = [
@@ -25,7 +25,7 @@ LLM_MODELS = [
 
 print("=" * 60)
 print("🤖 智能 Text-to-SQL 系統啟動中...")
-print("⚠️  檢測到大量無效數據，啟用增強修復模式")
+print("📊 模式: 讀取全部4276條數據（包含空白SQL）")
 print("=" * 60)
 
 # ==================== 增強工具函數 ====================
@@ -38,8 +38,8 @@ def validate_sql(sql_query: str) -> Dict:
         return {"valid": False, "issues": ["SQL語句為空"], "is_safe": False, "empty": True}
     
     sql_clean = sql_query.strip()
-    if len(sql_clean) < 10:  # 非常短的SQL可能無效
-        return {"valid": False, "issues": ["SQL過短"], "is_safe": False, "empty": False}
+    if len(sql_clean) < 5:
+        return {"valid": False, "issues": ["SQL過短"], "is_safe": False, "empty": True}
     
     security_issues = []
     sql_upper = sql_clean.upper()
@@ -73,30 +73,32 @@ def analyze_question_type(question: str) -> Dict:
         "keywords": [],
         "has_count": False,
         "has_date": False,
-        "has_group": False
+        "has_group": False,
+        "has_comparison": False
     }
     
     # 檢測關鍵詞
     keywords_sets = {
-        "sales": ["銷售", "業績", "金額", "收入", "sale", "revenue"],
-        "customer": ["客戶", "買家", "用戶", "customer", "client"],
-        "product": ["產品", "商品", "項目", "product", "item"],
-        "time": ["時間", "日期", "月份", "年", "月", "最近", "date", "month", "year"],
-        "report": ["報告", "完成", "份", "report", "complete"],
-        "count": ["多少", "幾個", "數量", "count", "how many"]
+        "sales": ["銷售", "業績", "金額", "收入", "sale", "revenue", "金額"],
+        "customer": ["客戶", "買家", "用戶", "customer", "client", "買家"],
+        "product": ["產品", "商品", "項目", "product", "item", "產品"],
+        "time": ["時間", "日期", "月份", "年", "月", "最近", "date", "month", "year", "時間"],
+        "report": ["報告", "完成", "份", "report", "complete", "報告"],
+        "count": ["多少", "幾個", "數量", "count", "how many", "多少"],
+        "comparison": ["比較", "vs", " versus", "對比", "相比", "比較"]
     }
     
     for category, keywords in keywords_sets.items():
         for keyword in keywords:
             if keyword in question_lower:
-                analysis["keywords"].append(category)
                 if category not in analysis["keywords"]:
                     analysis["keywords"].append(category)
     
     # 特殊檢測
     analysis["has_count"] = any(kw in question_lower for kw in keywords_sets["count"])
     analysis["has_date"] = any(kw in question_lower for kw in keywords_sets["time"])
-    analysis["has_group"] = "每" in question_lower or "各" in question_lower or "group" in question_lower
+    analysis["has_group"] = any(word in question_lower for word in ["每", "各", "group", "每個"])
+    analysis["has_comparison"] = any(kw in question_lower for kw in keywords_sets["comparison"])
     
     # 確定主要類型
     if analysis["keywords"]:
@@ -104,71 +106,78 @@ def analyze_question_type(question: str) -> Dict:
     
     return analysis
 
-def generate_intelligent_sql(question: str, analysis: Dict) -> str:
+def generate_sql_from_question(question: str, analysis: Dict) -> str:
     """根據問題分析生成智能SQL"""
+    question_lower = question.lower()
     question_type = analysis["type"]
-    has_count = analysis["has_count"]
-    has_date = analysis["has_date"]
-    has_group = analysis["has_group"]
-    
-    # 根據問題類型生成相應的SQL
-    if question_type == "sales":
-        if has_count and has_group and has_date:
-            return "SELECT strftime('%Y-%m', sale_date) as month, COUNT(*) as sales_count, SUM(amount) as total_sales FROM sales GROUP BY month ORDER BY month;"
-        elif has_count:
-            return "SELECT product_name, COUNT(*) as sale_count FROM sales GROUP BY product_name ORDER BY sale_count DESC LIMIT 10;"
-        else:
-            return "SELECT product_name, SUM(amount) as total_sales FROM sales GROUP BY product_name ORDER BY total_sales DESC LIMIT 10;"
-    
-    elif question_type == "customer":
-        if has_count and has_group:
-            return "SELECT customer_name, COUNT(*) as order_count, SUM(amount) as total_spent FROM orders GROUP BY customer_name ORDER BY total_spent DESC;"
-        else:
-            return "SELECT customer_name, email, join_date FROM customers ORDER BY join_date DESC LIMIT 10;"
-    
-    elif question_type == "product":
-        if has_count:
-            return "SELECT category, COUNT(*) as product_count FROM products GROUP BY category ORDER BY product_count DESC;"
-        else:
-            return "SELECT product_name, price, stock_quantity FROM products WHERE stock_quantity > 0 ORDER BY price DESC LIMIT 10;"
-    
-    elif question_type == "report" or question_type == "time":
-        if has_count and has_group and has_date:
-            return "SELECT strftime('%Y-%m', report_date) as month, COUNT(*) as report_count FROM reports GROUP BY month ORDER BY month;"
-        elif has_date:
-            return "SELECT report_id, report_name, report_date FROM reports ORDER BY report_date DESC LIMIT 10;"
-        else:
-            return "SELECT report_type, COUNT(*) as count FROM reports GROUP BY report_type ORDER BY count DESC;"
-    
-    # 默認SQL
-    if has_count and has_group:
-        return "SELECT category, COUNT(*) as item_count FROM items GROUP BY category ORDER BY item_count DESC;"
-    elif has_count:
-        return "SELECT COUNT(*) as total_count FROM records;"
+    
+    # 針對常見問題模式的SQL生成
+    if "每月" in question_lower and ("完成" in question_lower or "報告" in question_lower):
+        year_match = re.search(r'(\d{4})年', question_lower)
+        year = year_match.group(1) if year_match else "2023"
+        return f"SELECT strftime('%Y-%m', completion_date) as month, COUNT(*) as report_count FROM reports WHERE strftime('%Y', completion_date) = '{year}' GROUP BY month ORDER BY month;"
+    
+    elif "銷售" in question_lower and ("最高" in question_lower or "最好" in question_lower):
+        return "SELECT product_name, SUM(sales_amount) as total_sales FROM sales GROUP BY product_name ORDER BY total_sales DESC LIMIT 10;"
+    
+    elif "客戶" in question_lower and ("訂單" in question_lower or "購買" in question_lower):
+        return "SELECT customer_name, COUNT(*) as order_count, SUM(order_amount) as total_spent FROM orders GROUP BY customer_name ORDER BY total_spent DESC;"
+    
+    elif "比較" in question_lower and ("年" in question_lower or "年份" in question_lower):
+        return "SELECT strftime('%Y', order_date) as year, COUNT(*) as order_count, SUM(order_amount) as yearly_revenue FROM orders GROUP BY year ORDER BY year;"
+    
+    elif "庫存" in question_lower and ("不足" in question_lower or "缺少" in question_lower):
+        return "SELECT product_name, stock_quantity FROM products WHERE stock_quantity < 10 ORDER BY stock_quantity ASC;"
+    
+    # 根據分析結果生成通用SQL
+    if analysis["has_count"] and analysis["has_group"] and analysis["has_date"]:
+        return "SELECT strftime('%Y-%m', date_column) as period, COUNT(*) as item_count FROM appropriate_table GROUP BY period ORDER BY period;"
+    
+    elif analysis["has_count"] and analysis["has_group"]:
+        return "SELECT category_column, COUNT(*) as count FROM appropriate_table GROUP BY category_column ORDER BY count DESC;"
+    
+    elif analysis["has_count"]:
+        return "SELECT COUNT(*) as total_count FROM appropriate_table;"
+    
+    elif analysis["has_group"]:
+        return "SELECT group_column, AVG(value_column) as average_value FROM appropriate_table GROUP BY group_column;"
+    
     else:
-        return "SELECT * FROM data_table LIMIT 10;"
+        return "SELECT * FROM appropriate_table LIMIT 10;"
+
+def repair_empty_sql(original_sql: str, user_question: str, similar_question: str) -> str:
+    """修復空白SQL"""
+    if not original_sql or original_sql.strip() == "":
+        # 分析問題並生成合適的SQL
+        analysis = analyze_question_type(user_question)
+        repaired_sql = generate_sql_from_question(user_question, analysis)
+        
+        # 添加註釋說明這是修復的SQL
+        return f"-- 根據類似問題 '{similar_question}' 修復生成的SQL\n{repaired_sql}"
+    
+    return original_sql
 
-# ==================== 智能數據加載模塊 ====================
-class SmartDataLoader:
+# ==================== 完整數據加載模塊 ====================
+class CompleteDataLoader:
     def __init__(self, hf_token: str):
         self.hf_token = hf_token
         self.questions = []
         self.sql_answers = []
-        self.valid_indices = []  # 記錄有效數據的索引
+        self.sql_quality = []  # 記錄每個SQL的質量評分
         self.schema_data = {}
         
-    def load_and_clean_dataset(self) -> bool:
-        """加載並清理數據集"""
+    def load_complete_dataset(self) -> bool:
+        """加載完整數據集（包括空白SQL）"""
         try:
-            print(f"[{get_current_time()}] 加載數據集 '{DATASET_REPO_ID}'...")
+            print(f"[{get_current_time()}] 正在加載完整數據集 '{DATASET_REPO_ID}'...")
             raw_dataset = load_dataset(DATASET_REPO_ID, token=self.hf_token)['train']
             
-            print("解析 messages 格式並過濾無效數據...")
-            valid_count = 0
+            print("解析全部 messages 格式...")
+            total_count = 0
             empty_count = 0
-            invalid_count = 0
+            valid_count = 0
             
-            for i, item in enumerate(raw_dataset):
+            for item in raw_dataset:
                 try:
                     if 'messages' in item and len(item['messages']) >= 2:
                         user_content = item['messages'][0]['content']
@@ -187,50 +196,31 @@ class SmartDataLoader:
                         else:
                             sql_query = assistant_content
                         
-                        # 驗證SQL - 只保留真正有效的數據
+                        # 保存所有數據
+                        self.questions.append(question)
+                        self.sql_answers.append(sql_query)
+                        
+                        # 評估SQL質量
                         validation = validate_sql(sql_query)
+                        quality_score = 1.0 if validation["valid"] else 0.3
+                        self.sql_quality.append(quality_score)
                         
+                        total_count += 1
+                        if validation["empty"]:
+                            empty_count += 1
                         if validation["valid"]:
-                            self.questions.append(question)
-                            self.sql_answers.append(sql_query)
-                            self.valid_indices.append(i)
                             valid_count += 1
-                        elif validation["empty"]:
-                            empty_count += 1
-                        else:
-                            invalid_count += 1
                             
                 except Exception as e:
                     continue
             
-            print(f"數據清理完成: {valid_count} 有效, {empty_count} 空, {invalid_count} 無效")
-            
-            # 如果有效數據太少，添加一些備用問題
-            if valid_count < 100:
-                print("有效數據過少，添加備用問題...")
-                self.add_backup_examples()
-                
+            print(f"數據加載完成: 總數 {total_count}, 有效 {valid_count}, 空白 {empty_count}")
             return True
             
         except Exception as e:
             print(f"數據集加載失敗: {e}")
-            self.add_backup_examples()
             return False
     
-    def add_backup_examples(self):
-        """添加備用範例"""
-        backup_data = [
-            {"question": "查詢銷售額最高的產品", "sql": "SELECT product_name, SUM(sales_amount) as total_sales FROM sales GROUP BY product_name ORDER BY total_sales DESC LIMIT 10;"},
-            {"question": "顯示最近30天的訂單", "sql": "SELECT * FROM orders WHERE order_date >= date('now', '-30 days') ORDER BY order_date DESC;"},
-            {"question": "統計每個客戶的訂單數量", "sql": "SELECT customer_name, COUNT(*) as order_count FROM orders GROUP BY customer_name ORDER BY order_count DESC;"},
-            {"question": "2023年每月銷售額", "sql": "SELECT strftime('%Y-%m', sale_date) as month, SUM(amount) as monthly_sales FROM sales WHERE strftime('%Y', sale_date) = '2023' GROUP BY month ORDER BY month;"},
-            {"question": "庫存不足的商品", "sql": "SELECT product_name, stock_quantity FROM products WHERE stock_quantity < 10 ORDER BY stock_quantity ASC;"}
-        ]
-        
-        for data in backup_data:
-            self.questions.append(data["question"])
-            self.sql_answers.append(data["sql"])
-    
     def load_schema(self) -> bool:
         """加載數據庫Schema"""
         try:
@@ -250,60 +240,73 @@ class SmartDataLoader:
             return False
 
 # ==================== 主系統 ====================
-class EnhancedTextToSQLSystem:
+class CompleteTextToSQLSystem:
     def __init__(self, hf_token: str):
         self.hf_token = hf_token
-        self.data_loader = SmartDataLoader(hf_token)
+        self.data_loader = CompleteDataLoader(hf_token)
         self.retrieval_system = RetrievalSystem()
         
         self.initialize_system()
     
     def initialize_system(self):
         """初始化系統組件"""
-        print("初始化系統組件...")
+        print("正在初始化完整數據系統...")
         
-        self.data_loader.load_and_clean_dataset()
+        self.data_loader.load_complete_dataset()
         self.data_loader.load_schema()
         
-        # 只為有效數據計算向量
+        # 為所有問題計算向量（包括空白SQL的）
         if self.data_loader.questions:
             self.retrieval_system.compute_embeddings(self.data_loader.questions)
         
-        print(f"系統初始化完成，可用有效問題: {len(self.data_loader.questions)}")
+        print(f"系統初始化完成，載入問題總數: {len(self.data_loader.questions)}")
     
     def generate_sql(self, user_question: str) -> Tuple[str, str]:
-        """生成SQL查詢"""
+        """生成SQL查詢 - 處理所有數據"""
         log_messages = [f"⏰ {get_current_time()} 開始處理"]
         
         if not user_question or user_question.strip() == "":
             return "請輸入您的問題。", "錯誤: 問題為空"
         
-        # 分析問題
-        question_analysis = analyze_question_type(user_question)
-        log_messages.append(f"🔍 問題分析: {question_analysis['type']}類型")
-        
-        # 1. 嘗試檢索相似問題（只在有有效數據時）
+        # 1. 檢索最相似的問題（從所有4276條中）
         if self.data_loader.questions:
             hits = self.retrieval_system.retrieve_similar(user_question)
             
             if hits:
                 best_hit = hits[0]
                 similarity_score = best_hit['score']
-                similar_question = self.data_loader.questions[best_hit['corpus_id']]
-                original_sql = self.data_loader.sql_answers[best_hit['corpus_id']]
+                corpus_id = best_hit['corpus_id']
+                similar_question = self.data_loader.questions[corpus_id]
+                original_sql = self.data_loader.sql_answers[corpus_id]
+                sql_quality = self.data_loader.sql_quality[corpus_id]
                 
-                log_messages.append(f"📋 檢索到: '{similar_question}'")
-                log_messages.append(f"📊 相似度: {similarity_score:.3f}")
+                log_messages.append(f"🔍 檢索到: '{similar_question}'")
+                log_messages.append(f"📊 相似度: {similarity_score:.3f}, 質量分數: {sql_quality:.1f}")
                 
                 if similarity_score > SIMILARITY_THRESHOLD:
-                    log_messages.append(f"✅ 相似度 > {SIMILARITY_THRESHOLD}，使用預先SQL")
-                    return original_sql, "\n".join(log_messages)
+                    # 檢查並修復SQL（如果是空白的）
+                    validation = validate_sql(original_sql)
+                    
+                    if validation["empty"] or not validation["valid"]:
+                        log_messages.append(f"⚠️ 原始SQL需要修復: {', '.join(validation['issues'])}")
+                        log_messages.append("🛠️ 正在智能修復SQL...")
+                        
+                        repaired_sql = repair_empty_sql(original_sql, user_question, similar_question)
+                        log_messages.append("✅ 修復完成")
+                        
+                        return repaired_sql, "\n".join(log_messages)
+                    else:
+                        log_messages.append(f"✅ 相似度 > {SIMILARITY_THRESHOLD}，使用預先SQL")
+                        return original_sql, "\n".join(log_messages)
                 else:
-                    log_messages.append(f"ℹ️ 相似度不足，嘗試其他方法")
+                    log_messages.append(f"ℹ️ 相似度 {similarity_score:.3f} 低於閾值 {SIMILARITY_THRESHOLD}")
         
-        # 2. 智能生成SQL
+        # 2. 如果檢索失敗或相似度不足，智能生成SQL
         log_messages.append("🤖 智能生成SQL...")
-        intelligent_sql = generate_intelligent_sql(user_question, question_analysis)
+        analysis = analyze_question_type(user_question)
+        intelligent_sql = generate_sql_from_question(user_question, analysis)
+        
+        log_messages.append(f"📋 問題分析: {analysis['type']}類型")
         log_messages.append("✅ 智能生成完成")
         
         return intelligent_sql, "\n".join(log_messages)
@@ -335,21 +338,24 @@ class RetrievalSystem:
     
     def compute_embeddings(self, questions: List[str]) -> None:
         if questions:
-            self.question_embeddings = self.embedder.encode(questions, convert_to_tensor=True)
+            print(f"正在為 {len(questions)} 個問題計算向量...")
+            self.question_embeddings = self.embedder.encode(questions, convert_to_tensor=True, show_progress_bar=False)
+            print("向量計算完成")
     
-    def retrieve_similar(self, user_question: str, top_k: int = 3) -> List[Dict]:
+    def retrieve_similar(self, user_question: str, top_k: int = 5) -> List[Dict]:
         if self.question_embeddings is None or len(self.question_embeddings) == 0:
             return []
         try:
             question_embedding = self.embedder.encode(user_question, convert_to_tensor=True)
             hits = util.semantic_search(question_embedding, self.question_embeddings, top_k=top_k)
             return hits[0] if hits and hits[0] else []
-        except:
+        except Exception as e:
+            print(f"檢索錯誤: {e}")
             return []
 
 # ==================== 初始化系統 ====================
-print("正在初始化增強版Text-to-SQL系統...")
-text_to_sql_system = EnhancedTextToSQLSystem(HF_TOKEN)
+print("正在初始化完整數據Text-to-SQL系統...")
+text_to_sql_system = CompleteTextToSQLSystem(HF_TOKEN)
 
 # ==================== Gradio界面 ====================
 def process_query(user_question: str) -> Tuple[str, str]:
@@ -358,17 +364,16 @@ def process_query(user_question: str) -> Tuple[str, str]:
 
 with gr.Blocks(title="智能Text-to-SQL系統", theme=gr.themes.Soft()) as demo:
     gr.Markdown("# 🚀 ��能 Text-to-SQL 系統")
-    gr.Markdown("💡 針對大量無效數據優化的增強版本")
+    gr.Markdown("📊 完整模式: 讀取全部4276條數據")
     
     with gr.Row():
         question_input = gr.Textbox(
             label="📝 輸入問題", 
-            placeholder="例如：查詢2023年每月報告數量", 
-            lines=2
+            placeholder="例如：2023年每月完成多少份報告", 
+            lines=2,
+            scale=4
         )
-    
-    with gr.Row():
-        submit_btn = gr.Button("🚀 生成SQL", variant="primary")
+        submit_btn = gr.Button("🚀 生成SQL", variant="primary", scale=1)
     
     with gr.Row():
         sql_output = gr.Code(