🔧 使用 pydantic==2.10.6 修復 schema 錯誤

app.py

@@ -1,90 +1,258 @@
 import os
 import cv2
 import numpy as np
+import torch
+import torch.nn as nn
 import gradio as gr
+from pathlib import Path
+import mediapipe as mp
+import pickle
 
-# 檢查檔案是否存在
-model_path = "tsflow/models/best_model.pt"
-config_path = "tsflow/results/test_results.json"
+# MediaPipe設定
+mp_pose = mp.solutions.pose
+mp_hands = mp.solutions.hands
+mp_face_mesh = mp.solutions.face_mesh
 
-# 如果是在SignView2.0目錄下運行，調整路徑
-if not os.path.exists(model_path):
-    model_path = "../tsflow/models/best_model.pt"
-    config_path = "../tsflow/results/test_results.json"
+# 設定設備
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+print(f"使用設備: {device}")
 
-try:
-    from realtime_sign_prediction import RealtimeSignPredictor
-    
-    # 初始化預測器
-    print("🚀 正在初始化手語辨識系統...")
-    predictor = RealtimeSignPredictor(
-        model_path=model_path,
-        config_path=config_path,
-        sequence_length=50,
-        use_segmentation=True
-    )
-    print("✅ 手語辨識系統初始化完成！")
-    MODEL_LOADED = True
-    
-except Exception as e:
-    print(f"⚠️ 模型載入失敗: {e}")
-    print("🔄 使用模擬模式運行...")
-    MODEL_LOADED = False
+# 載入標籤映射
+label_to_idx = {'again': 0, 'all': 1, 'apple': 2, 'bad': 3, 'bathroom': 4, 'beautiful': 5, 'bird': 6, 'black': 7, 'blue': 8, 'book': 9, 'bored': 10, 'boy': 11, 'brother': 12, 'brown': 13, 'but': 14, 'computer': 15, 'cousin': 16, 'dance': 17, 'day': 18, 'deaf': 19, 'doctor': 20, 'dog': 21, 'draw': 22, 'drink': 23, 'eat': 24, 'english': 25, 'family': 26, 'father': 27, 'fine': 28, 'finish': 29, 'fish': 30, 'forget': 31, 'friend': 32, 'girl': 33}
+idx_to_label = {v: k for k, v in label_to_idx.items()}
 
-def predict_sign_language(image):
-    """簡化的預測函數，返回單一字串結果"""
-    if image is None:
-        return "請上傳影像"
-    
-    if not MODEL_LOADED:
-        return "⚠️ 模型未載入，無法進行預測"
-    
-    try:
-        # 處理畫面
-        results, keypoints, flow_features = predictor.process_frame(image)
+class BiLSTMWithAttention(nn.Module):
+    def __init__(self, input_size, hidden_size, num_layers, num_classes, dropout=0.5):
+        super(BiLSTMWithAttention, self).__init__()
+        self.hidden_size = hidden_size
+        self.num_layers = num_layers
+        
+        self.bilstm = nn.LSTM(input_size, hidden_size, num_layers, 
+                             batch_first=True, bidirectional=True, dropout=dropout)
+        
+        # 注意力機制
+        self.attention = nn.Linear(hidden_size * 2, 1)
         
-        # 獲取預測結果
-        top_predictions = predictor.get_top_predictions(top_k=3)
+        # 分類層
+        self.classifier = nn.Linear(hidden_size * 2, num_classes)
+        self.dropout = nn.Dropout(dropout)
         
-        # 格式化預測結果為簡單字串
-        if top_predictions:
-            result_text = "🎯 手語辨識結果:\n\n"
-            for i, (label, confidence) in enumerate(top_predictions, 1):
-                result_text += f"{i}. {label}: {confidence:.2%}\n"
+    def forward(self, x):
+        batch_size = x.size(0)
+        
+        # LSTM前向傳播
+        lstm_out, _ = self.bilstm(x)
+        
+        # 注意力權重計算
+        attention_weights = torch.softmax(self.attention(lstm_out), dim=1)
+        
+        # 加權平均
+        context_vector = torch.sum(attention_weights * lstm_out, dim=1)
+        
+        # 分類
+        output = self.classifier(self.dropout(context_vector))
+        
+        return output
+
+# 初始化模型
+input_size = 258  # keypoints (75*2) + optical_flow (108)
+hidden_size = 256
+num_layers = 3
+num_classes = len(label_to_idx)
+
+model = BiLSTMWithAttention(input_size, hidden_size, num_layers, num_classes)
+model = model.to(device)
+
+# 載入模型權重
+model_path = Path("tsflow/models/best_model.pt")
+if model_path.exists():
+    try:
+        checkpoint = torch.load(model_path, map_location=device)
+        if isinstance(checkpoint, dict) and 'model_state_dict' in checkpoint:
+            model.load_state_dict(checkpoint['model_state_dict'])
+        else:
+            model.load_state_dict(checkpoint)
+        model.eval()
+        print("✅ 模型載入成功")
+    except Exception as e:
+        print(f"❌ 模型載入失敗: {e}")
+        raise
+else:
+    print(f"❌ 找不到模型檔案: {model_path}")
+    raise FileNotFoundError(f"模型檔案不存在: {model_path}")
+
+def extract_keypoints_from_frame(frame):
+    """從單個frame提取關鍵點"""
+    try:
+        with mp_pose.Pose(static_image_mode=True, model_complexity=1) as pose, \
+             mp_hands.Hands(static_image_mode=True, max_num_hands=2) as hands:
+            
+            rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
             
-            result_text += f"\n📊 序列進度: {len(predictor.keypoint_sequence)}/{predictor.sequence_length}"
+            keypoints = []
+            
+            # 提取姿勢關鍵點
+            pose_results = pose.process(rgb_frame)
+            if pose_results.pose_landmarks:
+                pose_points = []
+                for landmark in pose_results.pose_landmarks.landmark:
+                    pose_points.extend([landmark.x, landmark.y])
+                keypoints.extend(pose_points)
+            else:
+                keypoints.extend([0.0] * 66)  # 33個姿勢點 * 2
+            
+            # 提取手部關鍵點
+            hands_results = hands.process(rgb_frame)
+            if hands_results.multi_hand_landmarks:
+                hand_points = []
+                for hand_landmarks in hands_results.multi_hand_landmarks:
+                    for landmark in hand_landmarks.landmark:
+                        hand_points.extend([landmark.x, landmark.y])
+                if len(hand_points) >= 42:  # 至少一隻手
+                    keypoints.extend(hand_points[:42])
+                else:
+                    keypoints.extend(hand_points + [0.0] * (42 - len(hand_points)))
+            else:
+                keypoints.extend([0.0] * 42)  # 21個手部點 * 2
+            
+            return np.array(keypoints, dtype=np.float32)
+    except Exception as e:
+        print(f"關鍵點提取錯誤: {e}")
+        return np.zeros(150, dtype=np.float32)
+
+def calculate_optical_flow_features(frames):
+    """計算光流特徵"""
+    try:
+        if len(frames) < 2:
+            return np.zeros(108, dtype=np.float32)
+        
+        flow_features = []
+        for i in range(len(frames) - 1):
+            gray1 = cv2.cvtColor(frames[i], cv2.COLOR_BGR2GRAY)
+            gray2 = cv2.cvtColor(frames[i + 1], cv2.COLOR_BGR2GRAY)
+            
+            flow = cv2.calcOpticalFlowPyrLK(
+                gray1, gray2, None, None,
+                winSize=(15, 15),
+                maxLevel=2,
+                criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)
+            )
+            
+            if flow[0] is not None and len(flow[0]) > 0:
+                flow_features.extend(flow[0].flatten()[:54])
+            else:
+                flow_features.extend([0.0] * 54)
+        
+        if len(flow_features) >= 108:
+            return np.array(flow_features[:108], dtype=np.float32)
         else:
-            result_text = "📡 正在收集動作序列...\n請確保手語動作清晰可見"
+            return np.array(flow_features + [0.0] * (108 - len(flow_features)), dtype=np.float32)
+    except Exception as e:
+        print(f"光流計算錯誤: {e}")
+        return np.zeros(108, dtype=np.float32)
+
+def predict_sign_language(video_path):
+    """預測手語影片"""
+    try:
+        cap = cv2.VideoCapture(video_path)
+        frames = []
+        
+        while True:
+            ret, frame = cap.read()
+            if not ret:
+                break
+            frames.append(frame)
+        
+        cap.release()
+        
+        if len(frames) == 0:
+            return "錯誤：無法讀取影片幀", 0.0
+        
+        # 提取特徵
+        keypoints_sequence = []
+        for frame in frames:
+            keypoints = extract_keypoints_from_frame(frame)
+            keypoints_sequence.append(keypoints)
+        
+        optical_flow = calculate_optical_flow_features(frames)
+        
+        # 確保序列長度為104
+        target_length = 104
+        if len(keypoints_sequence) > target_length:
+            keypoints_sequence = keypoints_sequence[:target_length]
+        elif len(keypoints_sequence) < target_length:
+            last_frame = keypoints_sequence[-1] if keypoints_sequence else np.zeros(150)
+            while len(keypoints_sequence) < target_length:
+                keypoints_sequence.append(last_frame)
+        
+        # 組合特徵
+        features_sequence = []
+        for i, keypoints in enumerate(keypoints_sequence):
+            if i < len(optical_flow) // 54:
+                flow_feature = optical_flow[i*54:(i+1)*54]
+            else:
+                flow_feature = np.zeros(54)
+            
+            combined_features = np.concatenate([keypoints, flow_feature, np.zeros(54)])
+            features_sequence.append(combined_features)
         
-        return result_text
+        # 轉換為tensor並預測
+        features_tensor = torch.tensor([features_sequence], dtype=torch.float32).to(device)
         
+        with torch.no_grad():
+            outputs = model(features_tensor)
+            probabilities = torch.softmax(outputs, dim=1)
+            predicted_class = torch.argmax(probabilities, dim=1).item()
+            confidence = probabilities[0][predicted_class].item()
+        
+        predicted_label = idx_to_label.get(predicted_class, "未知")
+        
+        return f"預測結果: {predicted_label}", confidence
+        
+    except Exception as e:
+        print(f"預測錯誤: {e}")
+        return f"預測失敗: {str(e)}", 0.0
+
+def gradio_predict(video):
+    """Gradio介面的預測函數"""
+    if video is None:
+        return "請上傳影片", "信心度: 0%"
+    
+    try:
+        result, confidence = predict_sign_language(video)
+        confidence_text = f"信心度: {confidence:.2%}"
+        return result, confidence_text
     except Exception as e:
-        return f"❌ 處理錯誤: {str(e)}"
+        return f"處理錯誤: {str(e)}", "信心度: 0%"
 
-# 使用最簡單的Interface避開所有schema問題
+# 建立Gradio介面
 demo = gr.Interface(
-    fn=predict_sign_language,
-    inputs=gr.Image(),
-    outputs=gr.Textbox(lines=10),
+    fn=gradio_predict,
+    inputs=gr.Video(label="上傳手語影片"),
+    outputs=[
+        gr.Textbox(label="預測結果"),
+        gr.Textbox(label="信心度")
+    ],
     title="🤟 SignView2.0 - 手語辨識系統",
-    description="支援34種手語詞彙的即時辨識系統，準確率達94.25%\n\n上傳影像即可開始辨識手語動作",
+    description="""
+    ### 歡迎使用 SignView2.0 手語辨識系統！
+    
+    **系統特色：**
+    - 🎯 準確率：94.25%
+    - 📚 支援34種手語詞彙
+    - 🧠 使用BiLSTM + 注意力機制
+    - 👁️ MediaPipe + 光流特徵融合
+    
+    **使用方法：**
+    1. 上傳手語影片（建議3-4秒）
+    2. 點擊提交進行辨識
+    3. 查看預測結果和信心度
+    
+    **支援詞彙：** again, all, apple, bad, bathroom, beautiful, bird, black, blue, book, bored, boy, brother, brown, but, computer, cousin, dance, day, deaf, doctor, dog, draw, drink, eat, english, family, father, fine, finish, fish, forget, friend, girl
+    """,
+    examples=[],
     flagging_mode="never"
 )
 
 if __name__ == "__main__":
-    print("🎉 SignView2.0 手語辨識系統已啟動！")
-    
-    # 根據環境自動選擇最佳配置
-    import os
-    try:
-        # 嘗試最簡單的launch，讓Gradio自己處理
-        demo.launch()
-    except Exception as e:
-        print(f"預設啟動失敗，嘗試備用方案: {e}")
-        try:
-            # 如果在Spaces環境，強制使用share=True
-            demo.launch(share=True)
-        except Exception as e2:
-            print(f"備用方案也失敗: {e2}")
-            # 最後嘗試基本配置
-            demo.launch(server_name="0.0.0.0", server_port=7860, share=False) 
+    demo.launch() 

requirements.txt

@@ -1,8 +1,8 @@
-gradio==4.38.1
+gradio==4.44.0
 torch>=2.0.0
 torchvision>=0.15.0
 opencv-python>=4.8.0
 mediapipe>=0.10.0
 numpy>=1.24.0
 Pillow>=9.5.0
-scipy>=1.10.0
+scipy>=1.10.0