Update app.py

app.py

@@ -46,27 +46,27 @@ def parse_sql_from_response(response_text: str) -> Optional[str]:
     """從模型輸出提取 SQL，增強版"""
     if not response_text:
         return None
-    
+
     # 清理回應文本
     response_text = response_text.strip()
-    
+
     # 1. 先找 ```sql ... ```
     match = re.search(r"```sql\s*\n(.*?)\n```", response_text, re.DOTALL | re.IGNORECASE)
     if match:
         return match.group(1).strip()
-    
+
     # 2. 找任何 ``` 包圍的內容
     match = re.search(r"```\s*\n?(.*?)\n?```", response_text, re.DOTALL)
     if match:
         sql_candidate = match.group(1).strip()
         if sql_candidate.upper().startswith('SELECT'):
             return sql_candidate
-    
+
     # 3. 找 SQL 語句（更寬鬆的匹配）
     match = re.search(r"(SELECT\s+.*?;)", response_text, re.DOTALL | re.IGNORECASE)
     if match:
         return match.group(1).strip()
-    
+
     # 4. 找沒有分號的 SQL
     match = re.search(r"(SELECT\s+.*?)(?=\n\n|\n```|$|\n[^,\s])", response_text, re.DOTALL | re.IGNORECASE)
     if match:
@@ -74,7 +74,7 @@ def parse_sql_from_response(response_text: str) -> Optional[str]:
         if not sql.endswith(';'):
             sql += ';'
         return sql
-    
+
     # 5. 如果包含 SELECT，嘗試提取整行
     if 'SELECT' in response_text.upper():
         lines = response_text.split('\n')
@@ -84,7 +84,7 @@ def parse_sql_from_response(response_text: str) -> Optional[str]:
                 if not line.endswith(';'):
                     line += ';'
                 return line
-    
+
     return None
 
 # ==================== Text-to-SQL 核心類 ====================
@@ -113,7 +113,7 @@ class TextToSQLSystem:
         self._log("✅ 系統初始化完成")
         # 載入數據庫結構
         self.schema = self._load_schema()
-            
+
         # 暫時添加：打印 schema 信息
         if self.schema:
             print("=" * 50)
@@ -125,7 +125,7 @@ class TextToSQLSystem:
                 for col in columns[:5]:  # 只顯示前5個
                     print(f"  - {col['name']} ({col['type']})")
             print("=" * 50)
-    
+
     # in class TextToSQLSystem:
 
     def _load_gguf_model(self):
@@ -137,7 +137,7 @@ class TextToSQLSystem:
                 filename=GGUF_FILENAME,
                 repo_type="dataset"
             )
-            
+
             # 使用一組更基礎、更穩定的參數來載入模型
             self.llm = Llama(
                 model_path=model_path,
@@ -147,16 +147,16 @@ class TextToSQLSystem:
                 verbose=False,   # 設為 False 避免 llama.cpp 本身的日誌干擾
                 n_gpu_layers=0   # 確認在 CPU 上運行
             )
-            
+
             # 簡單測試模型是否能回應
             self.llm("你好", max_tokens=3)
             self._log("✅ GGUF 模型載入成功")
-            
+
         except Exception as e:
             self._log(f"❌ GGUF 載入失敗: {e}", "ERROR")
             self._log("系統將無法生成 SQL。請檢查模型檔案或 llama-cpp-python 安裝。", "CRITICAL")
             self.llm = None
-        
+
     def _try_gguf_loading(self):
         """嘗試載入 GGUF"""
         try:
@@ -165,7 +165,7 @@ class TextToSQLSystem:
                 filename=GGUF_FILENAME,
                 repo_type="dataset"
             )
-            
+
             self.llm = Llama(
                 model_path=model_path,
                 n_ctx=512,
@@ -173,24 +173,24 @@ class TextToSQLSystem:
                 verbose=False,
                 n_gpu_layers=0
             )
-            
+
             # 測試生成
             test_result = self.llm("SELECT", max_tokens=5)
             self._log("✅ GGUF 模型載入成功")
             return True
-            
+
         except Exception as e:
             self._log(f"GGUF 載入失敗: {e}", "WARNING")
             return False
-    
+
     def _load_transformers_model(self):
         """使用 Transformers 載入你的微調模型"""
         try:
             from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
             import torch
-            
+
             self._log(f"載入 Transformers 模型: {FINETUNED_MODEL_PATH}")
-            
+
             # 載入你的微調模型
             self.transformers_tokenizer = AutoTokenizer.from_pretrained(FINETUNED_MODEL_PATH)
             self.transformers_model = AutoModelForCausalLM.from_pretrained(
@@ -199,7 +199,7 @@ class TextToSQLSystem:
                 device_map="cpu",           # 強制使用 CPU
                 trust_remote_code=True      # Qwen 模型可能需要
             )
-            
+
             # 創建生成管道
             self.generation_pipeline = pipeline(
                 "text-generation",
@@ -212,20 +212,20 @@ class TextToSQLSystem:
                 top_p=0.9,
                 pad_token_id=self.transformers_tokenizer.eos_token_id
             )
-            
+
             self.llm = "transformers"  # 標記使用 transformers
             self._log("✅ Transformers 模型��入成功")
-            
+
         except Exception as e:
             self._log(f"❌ Transformers 載入也失敗: {e}", "ERROR")
             self.llm = None
-    
+
     def huggingface_api_call(self, prompt: str) -> str:
         """調用 GGUF 模型，並加入詳細的原始輸出日誌"""
         if self.llm is None:
             self._log("模型未載入，返回 fallback SQL。", "ERROR")
             return self._generate_fallback_sql(prompt)
-        
+
         try:
             output = self.llm(
                 prompt,
@@ -236,9 +236,9 @@ class TextToSQLSystem:
                 # --- 將 stop 參數加回來 ---
                 stop=["```", ";", "\n\n", "</s>"],
             )
-            
+
             self._log(f"🧠 模型原始輸出 (Raw Output): {output}", "DEBUG")
-            
+
             if output and "choices" in output and len(output["choices"]) > 0:
                 generated_text = output["choices"][0]["text"]
                 self._log(f"📝 提取出的生成文本: {generated_text.strip()}", "DEBUG")
@@ -246,13 +246,13 @@ class TextToSQLSystem:
             else:
                 self._log("❌ 模型的原始輸出格式不正確或為空。", "ERROR")
                 return ""
-                
+
         except Exception as e:
             self._log(f"❌ 模型生成過程中發生嚴重錯誤: {e}", "CRITICAL")
             import traceback
             self._log(traceback.format_exc(), "DEBUG")
             return ""
-        
+
     def _load_gguf_model_fallback(self, model_path):
         """備用載入方式"""
         try:
@@ -286,7 +286,7 @@ class TextToSQLSystem:
             )
             with open(schema_path, "r", encoding="utf-8") as f:
                 schema_data = json.load(f)
-                
+
                 # 添加調試信息
                 self._log(f"📊 Schema 載入成功，包含 {len(schema_data)} 個表格:")
                 for table_name, columns in schema_data.items():
@@ -294,14 +294,14 @@ class TextToSQLSystem:
                     # 顯示前3個欄位作為範例
                     sample_cols = [col['name'] for col in columns[:3]]
                     self._log(f"    範例欄位: {', '.join(sample_cols)}")
-                
+
                 self._log("✅ 數據庫結構載入完成")
                 return schema_data
-                
+
         except Exception as e:
             self._log(f"❌ 載入 schema 失敗: {e}", "ERROR")
             return {}
-    
+
     # 也可以添加一個方法來檢查生成的 SQL 是否使用了正確的表格和欄位
     def _analyze_sql_correctness(self, sql: str) -> Dict:
         """分析 SQL 的正確性"""
@@ -312,15 +312,15 @@ class TextToSQLSystem:
             'invalid_columns': [],
             'suggestions': []
         }
-        
+
         if not self.schema:
             return analysis
-        
+
         # 提取 SQL 中的表格名稱
         table_pattern = r'FROM\s+(\w+)|JOIN\s+(\w+)'
         table_matches = re.findall(table_pattern, sql, re.IGNORECASE)
         used_tables = [match[0] or match[1] for match in table_matches]
-        
+
         # 檢查表格是否存在
         valid_tables = list(self.schema.keys())
         for table in used_tables:
@@ -332,26 +332,26 @@ class TextToSQLSystem:
                 for valid_table in valid_tables:
                     if table.lower() in valid_table.lower() or valid_table.lower() in table.lower():
                         analysis['suggestions'].append(f"{table} -> {valid_table}")
-        
+
         # 提取欄位名稱（簡單版本）
         column_pattern = r'SELECT\s+(.*?)\s+FROM|WHERE\s+(\w+)\s*[=<>]|GROUP BY\s+(\w+)|ORDER BY\s+(\w+)'
         column_matches = re.findall(column_pattern, sql, re.IGNORECASE)
-        
+
         return analysis
 
     def _encode_texts(self, texts):
         """編碼文本為嵌入向量"""
         if isinstance(texts, str):
             texts = [texts]
-        
-        inputs = self.embed_tokenizer(texts, padding=True, truncation=True, 
+
+        inputs = self.embed_tokenizer(texts, padding=True, truncation=True,
                                     return_tensors="pt", max_length=512)
         if DEVICE == "cuda":
             inputs = {k: v.cuda() for k, v in inputs.items()}
-        
+
         with torch.no_grad():
             outputs = self.embed_model(**inputs)
-        
+
         # 使用平均池化
         embeddings = outputs.last_hidden_state.mean(dim=1)
         return embeddings.cpu()
@@ -360,28 +360,53 @@ class TextToSQLSystem:
         """載入數據集並建立 FAISS 索引"""
         try:
             dataset = load_dataset(DATASET_REPO_ID, data_files="training_data.jsonl", split="train")
+
+            # 先過濾不完整樣本，避免 messages 長度不足導致索引或檢索報錯
+            try:
+                original_count = len(dataset)
+            except Exception:
+                original_count = None
+
+            dataset = dataset.filter(
+                lambda ex: isinstance(ex.get("messages"), list)
+                and len(ex["messages"]) >= 2
+                and all(
+                    isinstance(m.get("content"), str) and m.get("content") and m["content"].strip()
+                    for m in ex["messages"][:2]
+                )
+            )
+
+            if original_count is not None:
+                self._log(
+                    f"資料集清理: 原始 {original_count} 筆, 過濾後 {len(dataset)} 筆, 移除 {original_count - len(dataset)} 筆"
+                )
+
+            if len(dataset) == 0:
+                self._log("清理後資料集為空，無法建立索引。", "ERROR")
+                return None, None
+
             corpus = [item['messages'][0]['content'] for item in dataset]
             self._log(f"正在編碼 {len(corpus)} 個問題...")
-            
+
             # 批量編碼
             embeddings_list = []
             batch_size = 32
-            
+
             for i in range(0, len(corpus), batch_size):
                 batch_texts = corpus[i:i+batch_size]
                 batch_embeddings = self._encode_texts(batch_texts)
                 embeddings_list.append(batch_embeddings)
                 self._log(f"已編碼 {min(i+batch_size, len(corpus))}/{len(corpus)}")
-            
+
             all_embeddings = torch.cat(embeddings_list, dim=0).numpy()
-            
+
             # 建立 FAISS 索引
             index = faiss.IndexFlatIP(all_embeddings.shape[1])
             index.add(all_embeddings.astype('float32'))
-            
+
             self._log("✅ 向量索引建立完成")
             return dataset, index
-            
+
         except Exception as e:
             self._log(f"❌ 載入數據失敗: {e}", "ERROR")
             return None, None
@@ -390,7 +415,7 @@ class TextToSQLSystem:
         """根據實際 Schema 識別相關表格"""
         question_lower = question.lower()
         relevant_tables = []
-        
+
         # 根據實際表格的關鍵詞映射
         keyword_to_table = {
             'TSR53SampleDescription': ['客戶', '買方', '申請', '發票對象', 'customer', 'invoice', 'sample'],
@@ -400,18 +425,18 @@ class TextToSQLSystem:
             'JobEventsLog': ['事件', '操作', '用戶', 'event', 'log', 'user'],
             'calendar_days': ['工作日', '假期', 'workday', 'holiday', 'calendar']
         }
-        
+
         for table, keywords in keyword_to_table.items():
             if any(keyword in question_lower for keyword in keywords):
                 relevant_tables.append(table)
-        
+
         # 預設重要表格
         if not relevant_tables:
             if any(word in question_lower for word in ['客戶', '買家', '申請', '工作單', '數量']):
                 return ['TSR53SampleDescription', 'JobsInProgress']
             else:
                 return ['JobTimeline', 'TSR53SampleDescription']
-        
+
         return relevant_tables[:3]  # 最多返回3個相關表格
 
     # 請將這整個函數複製到您的 TextToSQLSystem class 內部
@@ -422,7 +447,7 @@ class TextToSQLSystem:
         """
         if not self.schema:
             return "No schema available.\n"
-        
+
         actual_table_names_map = {name.lower(): name for name in self.schema.keys()}
         real_table_names = []
         for table in table_names:
@@ -453,7 +478,7 @@ class TextToSQLSystem:
                     else:
                         cols_str.append(f"{col_name} ({col_type})")
                 formatted += f"Columns: {', '.join(cols_str)}\n\n"
-        
+
         return formatted.strip()
 
 
@@ -470,14 +495,14 @@ class TextToSQLSystem:
         返回一個元組 (SQL字符串或None, 狀態消息)。
         """
         q_lower = question.lower()
-        
+
         # ==============================================================================
         #  第一層：高價值意圖識別與模板覆寫 (Intent Recognition & Templating)
         # ==============================================================================
-        
+
         # --- 預先檢測所有可能的意圖和實體 ---
         job_no_match = re.search(r"(?:工單|jobno)\s*'\"?([A-Z]{2,3}\d+)'\"?", question, re.IGNORECASE)
-        
+
         entity_match_data = None
         ENTITY_TO_COLUMN_MAP = {
             '申請廠商': 'sd.ApplicantName', '申請方': 'sd.ApplicantName', 'applicant': 'sd.ApplicantName',
@@ -491,7 +516,7 @@ class TextToSQLSystem:
                 if match:
                     entity_match_data = {"type": keyword, "name": match.group(1).strip(), "column": column}
                     break
-        
+
         lab_group_match_data = None
         LAB_GROUP_MAP = {'A':'TA','B':'TB','C':'TC','D':'TD','E':'TE','Y':'TY','TA':'TA','TB':'TB','TC':'TC','TD':'TD','TE':'TE','TY':'TY','WC':'WC','EO':'EO','GCI':'GCI','GCO':'GCO','MI':'MI'}
         lab_group_match = re.findall(r"([A-Z]+)\s*組", question, re.IGNORECASE)
@@ -528,7 +553,7 @@ class TextToSQLSystem:
             self._log(f"🔄 檢測到查詢【{entity_type} '{entity_name}' 在 {year} 年的總業績】意圖，啟用模板。", "INFO")
             template_sql = f"WITH JobTotalAmount AS (SELECT JobNo, SUM(LocalAmount) AS TotalAmount FROM (SELECT DISTINCT JobNo, InvoiceCreditNoteNo, LocalAmount FROM TSR53Invoice) GROUP BY JobNo) SELECT SUM(jta.TotalAmount) AS total_revenue FROM TSR53SampleDescription AS sd JOIN JobTotalAmount AS jta ON sd.JobNo = jta.JobNo WHERE {column_name} LIKE '%{entity_name}%' AND strftime('%Y', sd.FirstReportAuthorizedDate) = '{year}';"
             return self._finalize_sql(template_sql, f"模板覆寫: 查詢 {entity_type}='{entity_name}' ({year}年) 的總業績")
-            
+
         if not entity_match_data and any(kw in q_lower for kw in ['業績', '營收', '金額', 'sales', 'revenue']):
             year_match, month_match = re.search(r'(\d{4})\s*年?', question), re.search(r'(\d{1,2})\s*月', question)
             time_condition, time_log = "", "總"
@@ -571,17 +596,17 @@ class TextToSQLSystem:
         #  第二層：常規修正流程 (Fallback Corrections)
         # ==============================================================================
         self._log("未觸發任何模板，嘗試解析並修正 AI 輸出...", "INFO")
-        
+
         parsed_sql = parse_sql_from_response(raw_response)
         if not parsed_sql:
             self._log(f"❌ 未能從模型回應中解析出任何 SQL。原始回應: {raw_response}", "ERROR")
             return None, f"無法解析SQL。原始回應:\n{raw_response}"
 
         self._log(f"📊 解析出的原始 SQL: {parsed_sql}", "DEBUG")
-        
+
         fixed_sql = " " + parsed_sql.strip() + " "
         fixes_applied_fallback = []
-        
+
         dialect_corrections = {
             r'YEAR\s*\(([^)]+)\)': r"strftime('%Y', \1)",
             r"(strftime\('%Y',\s*[^)]+\))\s*=\s*(\d{4})": r"\1 = '\2'",
@@ -632,47 +657,52 @@ class TextToSQLSystem:
         """使用 FAISS 快速檢索相似問題"""
         if self.faiss_index is None or self.dataset is None:
             return []
-        
+
         try:
             # 編碼問題
             q_embedding = self._encode_texts([question]).numpy().astype('float32')
-            
+
             # FAISS 搜索
             distances, indices = self.faiss_index.search(q_embedding, min(top_k + 2, len(self.dataset)))
-            
+
             results = []
             seen_questions = set()
-            
+
             for i, idx in enumerate(indices[0]):
                 if len(results) >= top_k:
                     break
-                
+
                 # 修復：將 numpy.int64 轉換為 Python int
                 idx = int(idx)  # ← 添加這行轉換
-                
+
                 if idx >= len(self.dataset):  # 確保索引有效
                     continue
-                    
+
                 item = self.dataset[idx]
-                q_content = item['messages'][0]['content']
-                a_content = item['messages'][1]['content']
-                
+                # 防呆：若樣本不完整則跳過
+                if not isinstance(item.get('messages'), list) or len(item['messages']) < 2:
+                    continue
+                q_content = (item['messages'][0].get('content') or '').strip()
+                a_content = (item['messages'][1].get('content') or '').strip()
+                if not q_content or not a_content:
+                    continue
+
                 # 提取純淨問題
                 clean_q = re.sub(r"以下是一個SQL查詢任務：\s*指令:\s*", "", q_content).strip()
                 if clean_q in seen_questions:
                     continue
-                    
+
                 seen_questions.add(clean_q)
                 sql = parse_sql_from_response(a_content) or "無法解析範例SQL"
-                
+
                 results.append({
                     "similarity": float(distances[0][i]),
                     "question": clean_q,
                     "sql": sql
                 })
-            
+
             return results
-            
+
         except Exception as e:
             self._log(f"❌ 檢索失敗: {e}", "ERROR")
             return []
@@ -684,7 +714,7 @@ class TextToSQLSystem:
         建立一個高度結構化、以任務為導向的提示詞，使用清晰的標題分隔符。
         """
         relevant_tables = self._identify_relevant_tables(user_q)
-        
+
         # 使用我們新的、更簡單的 schema 格式化函數
         schema_str = self._format_relevant_schema(relevant_tables)
 
@@ -721,7 +751,7 @@ SQL:
     def _generate_fallback_sql(self, prompt: str) -> str:
         """當模型不可用時的備用 SQL 生成"""
         prompt_lower = prompt.lower()
-        
+
         # 簡單的關鍵詞匹配生成基本 SQL
         if "統計" in prompt or "數量" in prompt or "多少" in prompt:
             if "月" in prompt:
@@ -730,13 +760,13 @@ SQL:
                 return "SELECT applicant, COUNT(*) as count FROM tsr53sampledescription GROUP BY applicant ORDER BY count DESC;"
             else:
                 return "SELECT COUNT(*) as total_count FROM jobtimeline WHERE completed_time IS NOT NULL;"
-        
+
         elif "金額" in prompt or "總額" in prompt:
             return "SELECT SUM(amount) as total_amount FROM tsr53invoice;"
-        
+
         elif "評級" in prompt or "pass" in prompt_lower or "fail" in prompt_lower:
             return "SELECT rating, COUNT(*) as count FROM tsr53sampledescription GROUP BY rating;"
-        
+
         else:
             return "SELECT * FROM jobtimeline LIMIT 10;"
 
@@ -745,22 +775,22 @@ SQL:
         try:
             if not os.path.exists(model_path):
                 return False
-            
+
             # 檢查檔案大小（至少應該有幾MB）
             file_size = os.path.getsize(model_path)
             if file_size < 10 * 1024 * 1024:  # 小於 10MB 可能有問題
                 return False
-            
+
             # 檢查 GGUF 檔案頭部
             with open(model_path, 'rb') as f:
                 header = f.read(8)
                 if not header.startswith(b'GGUF'):
                     return False
-            
+
             return True
         except Exception:
             return False
-    
+
 # in class TextToSQLSystem:
 
     def process_question(self, question: str) -> Tuple[str, str]:
@@ -769,7 +799,7 @@ SQL:
         if question in self.query_cache:
             self._log("⚡ 使用緩存結果")
             return self.query_cache[question]
-        
+
         self.log_history = []
         self._log(f"⏰ 處理問題: {question}")
 
@@ -788,12 +818,12 @@ SQL:
 
         # 4. **新的核心步驟**: 呼叫決策引擎來生成最終 SQL
         final_sql, status_message = self._validate_and_fix_sql(question, response)
-        
+
         if final_sql:
             result = (final_sql, status_message)
         else:
             result = (status_message, "生成失敗")
-        
+
         # 緩存結果
         self.query_cache[question] = result
         return result
@@ -804,10 +834,10 @@ text_to_sql_system = TextToSQLSystem()
 def process_query(q: str):
     if not q.strip():
         return "", "等待輸入", "請輸入問題"
-    
+
     sql, status = text_to_sql_system.process_question(q)
     logs = "\n".join(text_to_sql_system.log_history[-10:])  # 只顯示最後10條日誌
-    
+
     return sql, status, logs
 
 # 範例問題
@@ -822,19 +852,19 @@ examples = [
 with gr.Blocks(theme=gr.themes.Soft(), title="Text-to-SQL 智能助手") as demo:
     gr.Markdown("# ⚡ Text-to-SQL 智能助手")
     gr.Markdown("輸入自然語言問題，自動生成SQL查詢語句")
-    
+
     with gr.Row():
         with gr.Column(scale=2):
             inp = gr.Textbox(lines=3, label="💬 您的問題", placeholder="例如：2024年每月完成多少份報告？")
             btn = gr.Button("🚀 生成 SQL", variant="primary")
             status = gr.Textbox(label="狀態", interactive=False)
-        
+
         with gr.Column(scale=3):
             sql_out = gr.Code(label="🤖 生成的 SQL", language="sql", lines=8)
-    
+
     with gr.Accordion("📋 處理日誌", open=False):
         logs = gr.Textbox(lines=8, label="日誌", interactive=False)
-    
+
     # 範例區
     gr.Examples(
         examples=examples,