init

.gitattributes

@@ -33,3 +33,14 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.mov filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+*.m4a filter=lfs diff=lfs merge=lfs -text
+*.m4v filter=lfs diff=lfs merge=lfs -text
+*.m4b filter=lfs diff=lfs merge=lfs -text
+*.m4p filter=lfs diff=lfs merge=lfs -text
+*.m4r filter=lfs diff=lfs merge=lfs -text
+*.m4b filter=lfs diff=lfs merge=lfs -text
+*.db filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,11 @@
+predefined/
+predefined/*
+test/
+test/*
+data/
+data/*
+instructions/
+instructions/*
+tmp/
+tmp/*
+hand_landmarker.task

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

Dockerfile

@@ -0,0 +1,14 @@
+# FROM tensorflow/tensorflow:latest-gpu
+FROM mediapipegpu
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 libespeak1 -y
+
+RUN pip install -r requirements.txt 
+
+COPY . .
+
+CMD ["bash", "run.sh"]

README.md

@@ -1,10 +1,5 @@
----
-title: Finger
-emoji: 📈
-colorFrom: purple
-colorTo: green
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# gradio_finger_recognition
+## video
+Just for video finger recognition
+## video-audio
+Gradio implementation for video with audio

backend.log

@@ -0,0 +1,6 @@
+INFO:     Will watch for changes in these directories: ['/app']
+INFO:     Uvicorn running on https://0.0.0.0:8000 (Press CTRL+C to quit)
+INFO:     Started reloader process [7] using StatReload
+INFO:     Started server process [103]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.

backend.py

@@ -0,0 +1,192 @@
+from fastapi import FastAPI, HTTPException, Body
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+from sqlmodel import Session, select, SQLModel
+from passlib.context import CryptContext
+from models import engine, User, RecordedSession, Feedback, Exercise
+from db_utils import save_session_frames, load_session_frames, create_temp_video_audio, mux_audio_video, cleanup_temp_files
+from typing import List, Dict
+import numpy as np
+import io
+import base64
+import secrets
+from contextlib import asynccontextmanager
+from datetime import datetime
+
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+async def lifespan(app: FastAPI):
+    # ✅ 启动时执行
+    SQLModel.metadata.create_all(engine)
+    yield
+    # ❌ 关闭时（可选）
+    # print("App is shutting down...")
+
+app = FastAPI(lifespan=lifespan)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# 注册用户
+class RegisterUser(BaseModel):
+    username: str
+
+@app.post("/register")
+def register(data: RegisterUser):
+    print(data)
+    with Session(engine) as session:
+        if session.exec(select(User).where(User.username == data.username)).first():
+            raise HTTPException(status_code=400, detail="Username already exists")
+        user = User(username=data.username)
+        session.add(user)
+        session.commit()
+        return {
+            "status": "success",
+            "message": "User registered"
+            }
+
+# 登录用户
+class LoginRequest(BaseModel):
+    username: str
+    password: str
+
+@app.post("/login")
+def login(login_data: LoginRequest):
+    with Session(engine) as session:
+        user = session.exec(select(User).where(User.username == login_data.username)).first()
+        if not user:
+            hashed = pwd_context.hash(login_data.password)
+            user = User(username=login_data.username, password_hash=hashed)
+            session.add(user)
+            session.commit()
+
+        session_id = secrets.token_urlsafe(16)
+        token = f"mock-token-{secrets.token_urlsafe(8)}"
+
+        return {
+            "status": "success",
+            "token": token,
+            "session_id": session_id,
+            "username": user.username
+        }
+    
+# @app.post("/logout")
+# def logout():
+#     # 如果你有实际的 session 存储，比如数据库或 Redis，可以在这里清除它
+#     # 目前是 mock 登录，所以这里只返回个消息
+#     return {"status": "success", "message": "Logged out"}
+
+# 获取用户录制列表
+@app.get("/user_sessions/{username}")
+def get_user_sessions(username: str):
+    with Session(engine) as session:
+        user = session.exec(select(User).where(User.username == username)).first()
+        if not user:
+            raise HTTPException(status_code=404, detail="User not found")
+
+        sessions_list = session.exec(
+            select(RecordedSession)
+            .where(RecordedSession.user_id == user.id)
+            .order_by(RecordedSession.timestamp.desc())
+        ).all()
+
+        return [{
+            "session_id": s.id,
+            "session_name": s.session_name,
+            "timestamp": s.timestamp.strftime("%Y-%m-%d %H:%M:%S")
+        } for s in sessions_list]
+
+# 上传 session 帧
+class UploadSession(BaseModel):
+    username: str
+    session_name: str
+    fps: int
+    frames: List[str]
+    keypoints: Dict[str, List[str]]
+    audio_chunks: List[str]
+
+@app.post("/upload_session")
+def upload_session(data: UploadSession):
+    def decode_b64_array(b64):
+        return np.load(io.BytesIO(base64.b64decode(b64)), allow_pickle=True)
+
+    min_cnt = min(len(data.frames), len(data.keypoints["Left"]))
+    data.frames = data.frames[:min_cnt]
+    data.keypoints["Left"] = data.keypoints["Left"][:min_cnt]
+    data.keypoints["Right"] = data.keypoints["Right"][:min_cnt]
+    left_list = data.keypoints.get("Left", [])
+    right_list = data.keypoints.get("Right", [])
+    frames = [decode_b64_array(f) for f in data.frames]
+    if len(left_list) != len(right_list):
+        raise HTTPException(status_code=400, detail="Keypoints Left and Right lengths mismatch")
+    keypoints = []
+    for l_b64, r_b64 in zip(left_list, right_list):
+        left_kps = decode_b64_array(l_b64) if l_b64 else None
+        right_kps = decode_b64_array(r_b64) if r_b64 else None
+        keypoints.append({"Left": left_kps, "Right": right_kps})
+    audio = [decode_b64_array(a) for a in data.audio_chunks]
+
+    session_id = save_session_frames(
+        username=data.username,
+        session_name=data.session_name,
+        frames=frames,
+        keypoints=keypoints,
+        audio_chunks=audio,
+        fps=data.fps
+    )
+    return {"message": "Saved", "session_id": session_id}
+
+# 加载帧（播放用）
+@app.get("/play_session/{session_id}")
+def generate_playback_response(session_id: str):
+    video_path, audio_path = create_temp_video_audio(session_id)
+    muxed_path = mux_audio_video(video_path, audio_path)
+    # cleanup_temp_files(video_path, audio_path, muxed_path)
+    return FileResponse(muxed_path, media_type="video/mp4")
+
+
+@app.post("/submit_feedback")
+def submit_feedback(username: str = Body(...), content: str = Body(...)):
+    with Session(engine) as session:
+        fb = Feedback(username=username, content=content)
+        session.add(fb)
+        session.commit()
+    return {"status": "success"}
+
+@app.get("/get_feedback/{username}")
+def get_feedback(username: str):
+    with Session(engine) as session:
+        feedbacks = session.exec(
+            select(Feedback).where(Feedback.username == username).order_by(Feedback.timestamp.desc())
+        ).all()
+        return [f"- {f.timestamp.strftime('%Y-%m-%d %H:%M:%S')}: {f.content}" for f in feedbacks]
+    
+class ExerciseIn(BaseModel):
+    username: str
+    duration: str
+    sta_time: str
+
+@app.post("/submit_exercise")
+def submit_exercise(data: ExerciseIn):
+    with Session(engine) as session:
+        fb = Exercise(**data.dict())
+        session.add(fb)
+        session.commit()
+    return {"status": "success"}
+    
+@app.get("/get_exercise/{username}")
+def get_exercise(username: str):
+    with Session(engine) as session:
+        exercises = session.exec(
+            select(Exercise).where(Exercise.username == username).order_by(Exercise.sta_time.desc())
+        ).all()
+        return [f"- {f.sta_time}: {f.duration}" for f in exercises]
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run("backend:app", host="0.0.0.0", port=8000, reload=True, ssl_keyfile="key.pem", ssl_certfile="cert.pem")

cert.pem

@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIIFpzCCA4+gAwIBAgIULUvEMyRzfP652DVlibqIbe/E/ScwDQYJKoZIhvcNAQEL
+BQAwYzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlBBMRUwEwYDVQQHDAxQaGlsYWRl
+bHBoaWExDjAMBgNVBAoMBVVQZW5uMQ8wDQYDVQQLDAZWaXNpb24xDzANBgNVBAMM
+BkZpbmdlcjAeFw0yNTAzMTkxOTAwMjhaFw0yNjAzMTkxOTAwMjhaMGMxCzAJBgNV
+BAYTAlVTMQswCQYDVQQIDAJQQTEVMBMGA1UEBwwMUGhpbGFkZWxwaGlhMQ4wDAYD
+VQQKDAVVUGVubjEPMA0GA1UECwwGVmlzaW9uMQ8wDQYDVQQDDAZGaW5nZXIwggIi
+MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDSjMNis1BSEtHA59bYWW2wkAZs
+lL2+hbCDXwioDLvUDOjSen422tkr/8zyzyqhLKZAO7TaCbIvlBsQdRZ1w3Y5iNNc
+P1wgjmwfKXkefTJjheU6+IuggB0H8a7oS5berSAizKAaLrXIeRDaVGd0uNWatwal
+Kj5yoTE8ZeDnmVicnf5yhwP0WiUVFqRSoQYqWZIVpcx1xzqxBwR6HuO9XclZLtz9
+fGicfb0RgDXFpmpIZs1DV6ZRYqln5PqQ5OweiqZ5llhzA6uJh4N6fiZKQqN3pjJZ
+UOieujEFzimNVAGUxOkcYzzuUUS5KothIHnoRvs9kJU5gTJCX4Sz8VfF2u13IgOW
+SUxHtS7pQPxdoZUaRF0px7P7e+5oXTbktizynS1pIKlmPm49x8pDg04PPujbY770
+JEJIKaAUyMjMPIbtbJv3txx3FM9yrszwgzgEia4gvwS+ZJOmZIDXKRZTLF70pcx4
+5CpaUUUew7V0Pm2/3xCjN1TY9saMGCSpFth0VQNEV06uHPGmdtbPP1a6BU05KU+4
+Tq0KoMEhD/34lqyeiJfaDVHXQiJyXNtMPSxf+YmOZIq5RzMJuedBITLn/c/VajSn
+mLVI3ySQBiiOUT0MuIdLRzE+sOT5KuLXRv2wwn6HJYgcmbfbntRQ3ZPqdKl4hO64
+8IdbpfO1YAYVH/LG7QIDAQABo1MwUTAdBgNVHQ4EFgQUdNKLpfRiZK+0nrDfVSdF
+RRozQtYwHwYDVR0jBBgwFoAUdNKLpfRiZK+0nrDfVSdFRRozQtYwDwYDVR0TAQH/
+BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAEomnTV+8LgrgJRLf4m6qUgqi7Qvo
+FF58+3bZlBshlN/Kb6h8jyTYcgO0J9Glg9aOZ00luRahN+Ahg7y3jFwJnb05k72m
+JRtUdcACYJEGPLgLTvOmie3WiSsIgZHiXzE04cYcLGg7YT/E2WtyAS3+avnLtwyA
+48C4ikDOlhzZLYD0KkNtTs0U2qgLXSGcAkKpRFxaKFayxPTxbhw0zgqhYpcc7G8f
+TXvIF4uTxr+RHo719b2fqkwnzU7BWJAbJhjQoCFo5eMF4azYmHbFPJAQfMpCxgVw
+d2A8bfTjLyvZW7opLqzdYYbTgckHoio/oMm7vaAHYbUcR3GDeFCnfZmXeFL3rtqs
+UqviWwLODRDFuNhaW/8MBIxgHGNFL1IreDVxBzkqxCU/fMvrjJpncIESt4OTDSxb
+sTy4luBlbZ6oLFIKplulM8ANBk+UX3bvyK9QueFDLvtkCS3YeSEcVW1yphS6aJ/q
+upxwLhAFPUgohSlashf1AWolTj4RDHqk0j3os9+K64hE5R5/lHbhABQs1UKfIVJs
+Wun2HyxM9CUjdirBOKqpBl9/hcYcZIIlg+fNp8JAMVk7rgRWxGg7u2pmTnK0jAOO
+3FbQX8igNmnDBySKdoNbl4Y9S74hdD9SN2s7T0ewu9kgOKNuv+Q2JNeOoRgScd8N
+67W65RLVDoYuroM=
+-----END CERTIFICATE-----

db_utils.py

@@ -0,0 +1,157 @@
+import numpy as np
+import io
+from sqlmodel import Session, select
+from models import engine, RecordedSession, VideoFrame, User
+import base64
+import  tempfile
+import os
+import threading
+import cv2
+import subprocess
+from scipy.io.wavfile import write as write_wav
+import pdb
+
+SAMPLING_RATE = 48000
+
+def encode_ndarray(arr: np.ndarray) -> str:
+    buf = io.BytesIO()
+    np.save(buf, arr)
+    return base64.b64encode(buf.getvalue()).decode("utf-8")
+
+def ndarray_to_bytes(arr: np.ndarray) -> bytes:
+    buf = io.BytesIO()
+    np.save(buf, arr)
+    return buf.getvalue()
+
+def bytes_to_ndarray(blob: bytes) -> np.ndarray:
+    return np.load(io.BytesIO(blob), allow_pickle=True)
+
+def save_session_frames(username: str,
+    session_name: str,
+    frames: list[np.ndarray],
+    keypoints: list[np.ndarray],
+    audio_chunks: list[np.ndarray],
+    fps: int
+) -> int:
+    samples_per_frame = int(SAMPLING_RATE // fps)
+    with Session(engine) as session:
+        # 🔍 根据用户名查找 user_id
+        user = session.exec(select(User).where(User.username == username)).first()
+        if not user:
+            raise ValueError(f"Username '{username}' not found!")
+
+        # 💾 新建一个 session 记录
+        rec = RecordedSession(user_id=user.id, session_name=session_name)
+        session.add(rec)
+        session.commit()
+        session.refresh(rec)
+
+        rec_id = rec.id
+
+        # 💾 存储帧数据
+        for i in range(len(frames)):
+            start_idx = int(i * samples_per_frame)
+            end_idx = int((i+1) * samples_per_frame)
+            f = VideoFrame(
+                session_id=rec.id,
+                frame_index=i,
+                image_array=ndarray_to_bytes(frames[i]),
+                keypoints=ndarray_to_bytes(keypoints[i]),
+                audio_chunk=ndarray_to_bytes(audio_chunks[start_idx: end_idx]),
+                fps=fps
+            )
+            session.add(f)
+        session.commit()
+    return rec_id
+
+def load_session_frames(session_id: int):
+    fps = 0
+    with Session(engine) as session:
+        result = session.exec(
+            select(VideoFrame)
+            .where(VideoFrame.session_id == session_id)
+            .order_by(VideoFrame.frame_index)
+        ).all()
+
+        frames = []
+        keypoints_all = []
+        audio_chunks = []
+
+        for f in result:
+            frames.append(bytes_to_ndarray(f.image_array))
+            keypoints_all.append(bytes_to_ndarray(f.keypoints))
+            audio_chunks.append(bytes_to_ndarray(f.audio_chunk))
+            fps = f.fps
+
+        # 🔁 拼接所有音频 chunk 为一个完整的音频数组
+        full_audio = np.concatenate(audio_chunks, axis=0) if audio_chunks else None
+
+    return frames, keypoints_all, full_audio, fps
+
+def draw_keypoints_on_frame(frame: np.ndarray, keypoints: dict) -> np.ndarray:
+    for hand in ["Left", "Right"]:
+        if keypoints.get(hand) is not None:
+            for x, y, z in keypoints[hand]:
+                cx, cy = int(x * frame.shape[1]), int(y * frame.shape[0])
+                cv2.circle(frame, (cx, cy), 3, (0, 255, 0), -1)
+    return frame
+
+def create_temp_video_audio(session_id: int):
+    frames, keypoints, audio_chunks, fps = load_session_frames(session_id)
+
+    # ✅ 视频文件路径
+    video_temp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
+    video_path = video_temp.name
+
+    height, width, _ = frames[0].shape
+    out = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (width, height))
+    for f, kpts in zip(frames, keypoints):
+        # frame_with_kpts = draw_keypoints_on_frame(f.copy(), kpts)
+        # out.write(frame_with_kpts)
+        out.write(f)
+    out.release()
+
+    # ✅ 音频文件路径
+    audio_temp = tempfile.NamedTemporaryFile(delete=False, suffix=".wav")
+    audio_path = audio_temp.name
+
+    if audio_chunks.ndim == 1:
+        audio_chunks = audio_chunks[:, np.newaxis]
+    scaled_audio = (audio_chunks * 32767).astype(np.int16)
+    write_wav(audio_path, SAMPLING_RATE, scaled_audio)
+
+    return video_path, audio_path
+
+def mux_audio_video(video_path: str, audio_path: str) -> str:
+    output = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4").name
+    print("📄 合并后文件路径:", output)
+    print("video path: ", video_path)
+    print("audio path: ", audio_path)
+    cmd = [
+        "ffmpeg", "-y",
+        "-i", video_path,
+        "-i", audio_path,
+        "-c:v", "libx264",  # 视频编码器
+        "-c:a", "aac",      # 音频编码器
+        "-b:a", "192k",     # 音频比特率
+        "-ac", "2",         # 设置为立体声 (2个声道)
+        "-shortest",        # 保证音频和视频时间一致
+        output
+    ]
+    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if result.returncode != 0:
+        print("❌ ffmpeg 合并失败：", result.stderr.decode())
+        raise RuntimeError("ffmpeg failed to mux audio and video")
+    return output
+
+
+def cleanup_temp_files(*files):
+    def _cleanup():
+        import time
+        time.sleep(60)  # 延迟 60 秒后删除
+        for f in files:
+            try:
+                os.remove(f)
+            except Exception as e:
+                print(f"❌ 删除临时文件失败 {f}: {e}")
+    threading.Thread(target=_cleanup).start()

error.log

@@ -0,0 +1,14 @@
+[2025-03-20 15:08:44 -0400] [54511] [INFO] Starting gunicorn 23.0.0
+[2025-03-20 15:08:44 -0400] [54511] [ERROR] connection to ('0.0.0.0', 80) failed: [Errno 13] Permission denied
+[2025-03-20 15:08:45 -0400] [54511] [ERROR] connection to ('0.0.0.0', 80) failed: [Errno 13] Permission denied
+[2025-03-20 15:08:46 -0400] [54511] [ERROR] connection to ('0.0.0.0', 80) failed: [Errno 13] Permission denied
+[2025-03-20 15:08:47 -0400] [54511] [ERROR] connection to ('0.0.0.0', 80) failed: [Errno 13] Permission denied
+[2025-03-20 15:08:48 -0400] [54511] [ERROR] connection to ('0.0.0.0', 80) failed: [Errno 13] Permission denied
+[2025-03-20 15:08:49 -0400] [54511] [ERROR] Can't connect to ('0.0.0.0', 80)
+[2025-03-20 15:10:31 -0400] [57021] [INFO] Starting gunicorn 23.0.0
+[2025-03-20 15:10:31 -0400] [57021] [ERROR] connection to ('0.0.0.0', 80) failed: [Errno 13] Permission denied
+[2025-03-20 15:10:32 -0400] [57021] [ERROR] connection to ('0.0.0.0', 80) failed: [Errno 13] Permission denied
+[2025-03-20 15:10:33 -0400] [57021] [ERROR] connection to ('0.0.0.0', 80) failed: [Errno 13] Permission denied
+[2025-03-20 15:10:34 -0400] [57021] [ERROR] connection to ('0.0.0.0', 80) failed: [Errno 13] Permission denied
+[2025-03-20 15:10:35 -0400] [57021] [ERROR] connection to ('0.0.0.0', 80) failed: [Errno 13] Permission denied
+[2025-03-20 15:10:36 -0400] [57021] [ERROR] Can't connect to ('0.0.0.0', 80)

key.pem

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDSjMNis1BSEtHA
+59bYWW2wkAZslL2+hbCDXwioDLvUDOjSen422tkr/8zyzyqhLKZAO7TaCbIvlBsQ
+dRZ1w3Y5iNNcP1wgjmwfKXkefTJjheU6+IuggB0H8a7oS5berSAizKAaLrXIeRDa
+VGd0uNWatwalKj5yoTE8ZeDnmVicnf5yhwP0WiUVFqRSoQYqWZIVpcx1xzqxBwR6
+HuO9XclZLtz9fGicfb0RgDXFpmpIZs1DV6ZRYqln5PqQ5OweiqZ5llhzA6uJh4N6
+fiZKQqN3pjJZUOieujEFzimNVAGUxOkcYzzuUUS5KothIHnoRvs9kJU5gTJCX4Sz
+8VfF2u13IgOWSUxHtS7pQPxdoZUaRF0px7P7e+5oXTbktizynS1pIKlmPm49x8pD
+g04PPujbY770JEJIKaAUyMjMPIbtbJv3txx3FM9yrszwgzgEia4gvwS+ZJOmZIDX
+KRZTLF70pcx45CpaUUUew7V0Pm2/3xCjN1TY9saMGCSpFth0VQNEV06uHPGmdtbP
+P1a6BU05KU+4Tq0KoMEhD/34lqyeiJfaDVHXQiJyXNtMPSxf+YmOZIq5RzMJuedB
+ITLn/c/VajSnmLVI3ySQBiiOUT0MuIdLRzE+sOT5KuLXRv2wwn6HJYgcmbfbntRQ
+3ZPqdKl4hO648IdbpfO1YAYVH/LG7QIDAQABAoICAB25SYwRjHFJvtrg3+/DED0w
+1/dUOEVBUl35eJtg0NNqzyOHr3HHC4munxxYKsh+KSpAQ5PUcpdM7VUxcm5FndcZ
+fd00BKmD8bekfzjSq95o9KeSlwOdknugnvSkVzQwdKz0/lUz6u3WSY7JA6HyxhUF
+aaa/g/li0DSanVmUyHGzzEwWy3QflcSYYbJYkSpBYuIzed3Wtm5vcy5aVgRRzS8N
+8pzGh24wQhTKaMzyZWa7PJcZNJ2gtBG6vbTe3IajREU9+FakWf8cZm9Qh89MAQ/P
+IqSBS5W861bKeAs/pMl/0vjy+ZMbXfxWCT84+nzUKHvUgXyQxKuiGHXeyGliMsux
+LB0hNX46jvlv3kC5my4fJIoIpLpOGjoU+K5IVYb3XTFjdMmGFBtrzge1HA47hLol
+HYDld+JOKpZI2MnCEST9sF30+NBGibw1HLpbL6kd2fNMK84vlNHoyE+sIb2kYZou
+8RHjn7vTOIbmdjvdT4R6FoaS6LXa/XKOgyDdLIHCvJizoXnhY4QT66GyRrtY7E+X
+OOT7T674r2Z+u1eC7M1CtTeoHAJixyclnFI/XP/eSzSZ2GJffFRiUdPPuqLGrdLg
+rTzCfiXn9diYzTBs0P0AfiL+D9eh+fC9Jw1u6lWuwCMgYfx8bAs9peOEMKSTOYbK
+ePJ3Cx4XmPJHx4DLRsgBAoIBAQDtd5kCR87lCyD+uhNbnCZL8QR15Dy79hefXW/l
+hWr3C88syjdoaYCNuB9LQ+i678zmmFOFlK28Ge0v7CX+lzMQN3vqcvaqf2Vb1QeS
+I9N8NoJ1ftzqm/r8EqBYKnYVqedNBOBMOcaTUHRUJrDnQL9/+gN0+OkIR4TNJOQm
+Fci0km8o0Q7dkygrigmwfnprGWfjH5gLngjVfhLOfQ+i5nCCxxT4x5vcHJyCCKSU
+TgrxyhVEB7K0mWW4godovNu6O3VBoY1MIraqThoudIKevUOMDSQBMWG8duAupCk8
+ZXgkshojH5tpN2Wj2QQr2rrG4gpRZZ2NHmUOEK2oD6Uv4hNtAoIBAQDi+2EStFsm
+U0N58F8VuBpPyPkD7Yv7DscwqFXoWE2g7MlqdZF7cCqwyf4O/AB0LLKTEzP2NBWO
+e0uxrftGYN4kdlALVx9B2/wXELd4fm9SqtSutUv3ZN5GnK/GltAw9YHEDE36ESHF
+lTO0lv/k+O7W+yhMEDY4j0FmgvDfH+Q0WmFnwxYl39YT9/Jw1sPDKbtDE1gJ0b0N
+YdjZhn4VohfmiRhid5GmndrQbwxGfvLEXsU+syo2/nj9IGCxFq8lka0fKesL92Nx
+DcjHRf8+6VK023RM5M+DKYMmwFLl92NrR1aG+HlDvshHUZjIoSXVAwfSz0Ba9GMY
+FFdGzLIL99GBAoIBAG6AxgoComuBR0RiEJoDyupx6LJ3mC+bcBiv4V88O69kpm7g
+VvJWjgTk1mMu4cED0CTKY6t4qXQr2G0Bhhi4AYIdX6OVBeYHTIJ0WoaN918I+qJ0
+e5cNKLlebZE5iSPBoan8h+fQxvBMcyWpr46dWb/S9wLaxY4dwdW4whZa8r/cmK+0
+wSco0HuaS7H+2Ta3ZtmRHS7ixpeaiGPgXINgmqCwxbiTIptGESqNnRCKVJt5f3Xd
+4zIZY/V9gEekAtfhzUnSRK9WRAxyNcrCWvpFdoZYoxWPBj0uUFpD+BBr44GoA3Ou
+xKIsrjaVyVQi/+GG1GhWUf+WUk4+QqE/To4+tO0CggEADDSAL8VK8XCgvDnUoxJX
+N9sSqMdpM4LD6zXiCLBW3ERfQD46KG0Lnp3970hVremJYKczsBV040h19YPpcwta
+ZpOGElYI9D2j/ImFlBEYY1WUQiC1iQP/f8SFHySU5U0OQUB4IO4y5rDzKs7Dy8gm
+76Bptk1Y3Qm29pAr65OHbdk+S5oN4tN6a3B1tOOXezMPQrgTj9ObWtDcHDZDKV6h
+8l+E1CahylfKoFKYUmIZI74E1S5FItfkIZhQGrWhjV+b84UJgoc27alUFoMJCpT5
+QYhbZJcZIXBmAPtuebcnvWkEmhVaT+4+Trdwg7lGk4GqNge26i0h9vWBC+mN2V7m
+AQKCAQBMdq4Q9Jvr6Mfg9spP/J478/ZvmbYVUuaIpxSgYvVPx2ZRdhjpIrA1OvdS
+nDS7r9Qvhh9lpq7yRxtFjUvH7OFwFDVxUH3X1l5hmeRIX/LTIYw/Gt3fcrpyoaei
+bmXNuKH9FcGdVUcgHbji7pb4ZPgJFmpDXDde1cwNzrvkq5kkPCGiABJYlS2c/FTY
+o72PhSj8nze3VlVwb9Yx5CPS0GqJM202uzUvJN5SdPr4/PbeZcU1fWWoUGvdzOTE
+hhaGD0ByCgBEDJ86uiu28ppVCbHdduWjbgp4OsQM9wcLXY9L0ligi0+C9dyW/fmk
+eDuRtd0FawsrIDxvndnhBysNpuzv
+-----END PRIVATE KEY-----

keypoints_process.py

@@ -0,0 +1,123 @@
+import gradio as gr
+import cv2
+import numpy as np
+from gradio_webrtc import WebRTC
+from twilio.rest import Client
+import os
+import spaces
+from threading import Lock
+from collections import defaultdict
+import time
+from bisect import bisect_left
+from scipy.spatial.distance import cdist
+import mediapipe as mp
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+from mediapipe.framework.formats import landmark_pb2
+from mediapipe import solutions
+import pdb
+import json
+from moviepy.editor import VideoFileClip
+import librosa
+
+# 启用 GPU 选项
+base_options = mp.tasks.BaseOptions(model_asset_path='gesture_recognizer.task',
+    delegate=mp.tasks.BaseOptions.Delegate.GPU)
+
+mp_drawing = mp.solutions.drawing_utils
+mp_hands = mp.solutions.hands
+
+base_options=mp.tasks.BaseOptions(model_asset_path='hand_landmarker.task',
+    delegate=mp.tasks.BaseOptions.Delegate.GPU)
+options = vision.HandLandmarkerOptions(base_options=base_options,
+    running_mode=mp.tasks.vision.RunningMode.VIDEO,
+    num_hands=2)
+detector = vision.HandLandmarker.create_from_options(options)
+
+options_image = vision.HandLandmarkerOptions(base_options=base_options,
+    running_mode=mp.tasks.vision.RunningMode.IMAGE,
+    num_hands=2)
+detector_image = vision.HandLandmarker.create_from_options(options_image)
+
+video_size = (500, 500)
+
+previous_timestamp = None
+
+class ReferenceVideo:
+    def __init__(self):
+        self.keypoints = {"Left": [], "Right": []}
+        # self.timestamps = []
+        # self.duration = 0
+        self.frames = []       # 存储原始视频帧
+    
+    def load_video(self, video_path):
+        global previous_timestamp
+        self.keypoints = {"Left": [], "Right": []}
+        self.frames = []
+
+        video = VideoFileClip(video_path)
+        fps = video.fps  # 需要存储 fps
+        video_size = (video.size[0], video.size[1])
+        audio = video.audio.to_soundarray()
+        original_sr = video.audio.fps
+        audio = librosa.resample(audio.T, orig_sr=original_sr, target_sr=48000).T  # 采样率转换
+
+        if audio.ndim == 2 and audio.shape[1] == 2:
+            audio = 0.5 * (audio[:, 0] + audio[:, 1])  # 立体声转单声道
+        audio = audio.astype(np.float32)
+
+        cap = cv2.VideoCapture(video_path)
+        while cap.isOpened():
+            ret, frame = cap.read()
+            if not ret:
+                break
+            
+            # 1. 显式拷贝并创建独立的图像数据
+            rgb_data = frame.astype(np.uint8).copy()
+            rgb = mp.Image(image_format=mp.ImageFormat.SRGB,data=np.array(cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)))
+            
+            # 2. 生成严格递增的时间戳（单位：毫秒）
+            timestamp_ms = int(cap.get(cv2.CAP_PROP_POS_MSEC))
+            while previous_timestamp is not None and timestamp_ms <= previous_timestamp:
+                timestamp_ms = previous_timestamp + 1
+            previous_timestamp = timestamp_ms
+            
+            # 3. 调用检测器
+            results = detector.detect_for_video(rgb, timestamp_ms)
+            
+            # 4. 处理检测结果
+            frame_landmarks = {"Left": None, "Right": None}
+            if results.hand_landmarks and results.handedness:
+                for idx, hand_landmarks in enumerate(results.hand_landmarks):
+                    label = results.handedness[idx][0].category_name
+                    landmarks = [(lm.x, lm.y, lm.z) for lm in hand_landmarks]
+                    frame_landmarks[label] = landmarks
+            
+            self.keypoints["Left"].append(frame_landmarks["Left"])
+            self.keypoints["Right"].append(frame_landmarks["Right"])
+            # self.timestamps.append(timestamp_ms / 1000)  # 统一使用计算后的时间戳
+            self.frames.append(cv2.resize(frame, video_size))
+            output_path = os.path.splitext(video_path)[0] + "_keypoints.json"
+            with open(output_path, "w") as f:
+                json.dump(self.keypoints, f)
+            
+            # 5. 显式释放资源
+            del results  # 关键：释放检测结果占用的GPU资源
+            del rgb     # 释放Image实例
+        
+        # self.duration = self.timestamps[-1] if self.timestamps else 0
+        cap.release()
+        video.close()
+
+        # np.save(f"{os.path.splitext(video_path)[0]}_frames.npy", np.array(self.frames, dtype=np.uint8))
+        # np.save(f"{os.path.splitext(video_path)[0]}_audio.npy", audio)
+        metadata = {"fps": fps, "video_size": video_size}
+        with open(f"{os.path.splitext(video_path)[0]}_meta.json", "w") as f:
+            json.dump(metadata, f)
+
+ref_video = ReferenceVideo()
+
+video_paths = ['predefined/Move12_preview.mp4', 'predefined/Move12_main.mp4']
+
+for video_path in video_paths:
+    ref_video.load_video(video_path)

load_balancer.py

@@ -0,0 +1,132 @@
+from flask import Flask, redirect, request, jsonify
+import requests
+from threading import Lock
+import logging
+from datetime import datetime
+import json
+import os
+import time
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler('load_balancer.log'),
+        logging.StreamHandler()
+    ]
+)
+
+app = Flask(__name__)
+
+# Configuration
+SERVERS = [f"https://158.130.50.41:{port}" for port in range(7850, 7860)]  # Changed to HTTPS
+HEALTH_CHECK_INTERVAL = 30  # seconds
+MAX_FAILURES = 3
+
+class LoadBalancer:
+    def __init__(self):
+        self.servers = SERVERS
+        self.current_index = 0
+        self.lock = Lock()
+        self.server_status = {server: {'healthy': True, 'failures': 0} for server in self.servers}
+        self.last_health_check = {server: datetime.now() for server in self.servers}
+
+    def get_next_server(self):
+        with self.lock:
+            # Find the next healthy server
+            start_index = self.current_index
+            while True:
+                server = self.servers[self.current_index]
+                if self.server_status[server]['healthy']:
+                    self.current_index = (self.current_index + 1) % len(self.servers)
+                    return server
+                
+                self.current_index = (self.current_index + 1) % len(self.servers)
+                if self.current_index == start_index:
+                    # All servers are unhealthy
+                    return None
+
+    def mark_server_failure(self, server):
+        with self.lock:
+            self.server_status[server]['failures'] += 1
+            if self.server_status[server]['failures'] >= MAX_FAILURES:
+                self.server_status[server]['healthy'] = False
+                logging.warning(f"Server {server} marked as unhealthy")
+
+    def mark_server_success(self, server):
+        with self.lock:
+            self.server_status[server]['failures'] = 0
+            self.server_status[server]['healthy'] = True
+
+    def health_check(self, server):
+        try:
+            # Disable SSL verification for self-signed certificates
+            response = requests.get(f"{server}/health", timeout=5, verify=False)
+            if response.status_code == 200:
+                self.mark_server_success(server)
+                return True
+            else:
+                self.mark_server_failure(server)
+                return False
+        except Exception as e:
+            logging.error(f"Health check failed for {server}: {str(e)}")
+            self.mark_server_failure(server)
+            return False
+
+    def get_status(self):
+        return {
+            'servers': [
+                {
+                    'url': server,
+                    'healthy': status['healthy'],
+                    'failures': status['failures']
+                }
+                for server, status in self.server_status.items()
+            ]
+        }
+
+# Initialize load balancer
+load_balancer = LoadBalancer()
+
+@app.route('/')
+def proxy():
+    server = load_balancer.get_next_server()
+    if not server:
+        return jsonify({'error': 'No healthy servers available'}), 503
+    
+    # Get the original request path and query parameters
+    path = request.path
+    query_string = request.query_string.decode('utf-8')
+    
+    # Construct the target URL
+    target_url = f"{server}{path}"
+    if query_string:
+        target_url += f"?{query_string}"
+    
+    logging.info(f"Redirecting to {target_url}")
+    return redirect(target_url)
+
+@app.route('/health')
+def health():
+    return jsonify({'status': 'healthy'}), 200
+
+@app.route('/status')
+def status():
+    return jsonify(load_balancer.get_status()), 200
+
+def run_health_checks():
+    """Background task to check server health"""
+    while True:
+        for server in SERVERS:
+            load_balancer.health_check(server)
+        time.sleep(HEALTH_CHECK_INTERVAL)
+
+if __name__ == '__main__':
+    # Start health check thread
+    import threading
+    health_check_thread = threading.Thread(target=run_health_checks, daemon=True)
+    health_check_thread.start()
+    
+    # Run the Flask app on HTTP port 80
+    app.run(host='0.0.0.0', port=7860) 

manage_nginx.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Function to check if nginx is installed
+check_nginx() {
+    if ! command -v nginx &> /dev/null; then
+        echo "Nginx is not installed. Installing..."
+        sudo apt-get update
+        sudo apt-get install -y nginx
+    fi
+}
+
+# Function to start nginx
+start_nginx() {
+    echo "Starting Nginx..."
+    sudo cp nginx.conf /etc/nginx/nginx.conf
+    sudo nginx -t  # Test configuration
+    if [ $? -eq 0 ]; then
+        sudo systemctl start nginx
+        echo "Nginx started successfully"
+    else
+        echo "Nginx configuration test failed"
+        exit 1
+    fi
+}
+
+# Function to stop nginx
+stop_nginx() {
+    echo "Stopping Nginx..."
+    sudo systemctl stop nginx
+    echo "Nginx stopped"
+}
+
+# Function to restart nginx
+restart_nginx() {
+    echo "Restarting Nginx..."
+    sudo cp nginx.conf /etc/nginx/nginx.conf
+    sudo nginx -t  # Test configuration
+    if [ $? -eq 0 ]; then
+        sudo systemctl restart nginx
+        echo "Nginx restarted successfully"
+    else
+        echo "Nginx configuration test failed"
+        exit 1
+    fi
+}
+
+# Function to show status
+show_status() {
+    echo "Checking Nginx status..."
+    sudo systemctl status nginx
+}
+
+# Main script
+case "$1" in
+    start)
+        check_nginx
+        start_nginx
+        ;;
+    stop)
+        stop_nginx
+        ;;
+    restart)
+        check_nginx
+        restart_nginx
+        ;;
+    status)
+        show_status
+        ;;
+    *)
+        echo "Usage: $0 {start|stop|restart|status}"
+        exit 1
+        ;;
+esac
+
+exit 0 

mediapipe

@@ -0,0 +1 @@
+Subproject commit 357dfdba090c5298263b067193d002a4c7a53859

models.py

@@ -0,0 +1,38 @@
+from sqlmodel import SQLModel, Field, create_engine
+from typing import Optional
+from datetime import datetime
+
+class User(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    username: str = Field(unique=True, index=True)
+    password_hash: str
+    display_name: Optional[str]
+
+class RecordedSession(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    user_id: int
+    session_name: str
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+class VideoFrame(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    session_id: int = Field(foreign_key="recordedsession.id")
+    frame_index: int
+    image_array: bytes  # NumPy-encoded frame
+    keypoints: bytes    # NumPy-encoded keypoints
+    audio_chunk: bytes  # NumPy-encoded audio chunk
+    fps: int
+
+class Feedback(SQLModel, table=True):
+    id: int = Field(default=None, primary_key=True)
+    username: str
+    content: str
+    timestamp: datetime = Field(default_factory=datetime.now)
+
+class Exercise(SQLModel, table=True):
+    id: int = Field(default=None, primary_key=True)
+    username: str
+    duration: str
+    sta_time: str
+
+engine = create_engine("sqlite:///data/recordings.db")

nginx.conf

@@ -0,0 +1,104 @@
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+    access_log /var/log/nginx/access.log main;
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+
+    # SSL configuration
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_prefer_server_ciphers on;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+    ssl_session_timeout 1d;
+    ssl_session_cache shared:SSL:50m;
+    ssl_session_tickets off;
+    ssl_stapling on;
+    ssl_stapling_verify on;
+    resolver 8.8.8.8 8.8.4.4 valid=300s;
+    resolver_timeout 5s;
+
+    # Define upstream servers for load balancing
+    upstream app_servers {
+        least_conn;  # Use least connections algorithm for load balancing
+        
+        # Add all 10 servers
+        server localhost:7850;
+        server localhost:7851;
+        server localhost:7852;
+        server localhost:7853;
+        server localhost:7854;
+        server localhost:7855;
+        server localhost:7856;
+        server localhost:7857;
+        server localhost:7858;
+        server localhost:7859;
+    }
+
+    # Redirect HTTP to HTTPS
+    server {
+        listen 80;
+        server_name 158.130.50.41;  # Change this to your domain name
+        return 301 https://$server_name$request_uri;
+    }
+
+    # HTTPS server configuration
+    server {
+        listen 443 ssl http2;
+        server_name 158.130.50.41;  # Change this to your domain name
+
+        # SSL certificate paths
+        ssl_certificate /etc/nginx/ssl/cert.pem;
+        ssl_certificate_key /etc/nginx/ssl/key.pem;
+
+        # WebSocket support
+        location / {
+            proxy_pass http://app_servers;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-Forwarded-Ssl on;
+            
+            # Timeout settings for WebSocket
+            proxy_read_timeout 300s;
+            proxy_send_timeout 300s;
+            proxy_connect_timeout 75s;
+            
+            # Buffer settings
+            proxy_buffering off;
+            proxy_buffer_size 128k;
+            proxy_buffers 4 256k;
+            proxy_busy_buffers_size 256k;
+        }
+
+        # Health check endpoint
+        location /health {
+            access_log off;
+            return 200 'healthy\n';
+        }
+
+        # Security headers
+        add_header Strict-Transport-Security "max-age=63072000" always;
+        add_header X-Frame-Options DENY;
+        add_header X-Content-Type-Options nosniff;
+        add_header X-XSS-Protection "1; mode=block";
+    }
+} 

preprocess_videos.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Directory containing the videos
+VIDEO_DIR="predefined"
+
+# Target dimensions
+TARGET_WIDTH=720
+TARGET_HEIGHT=640
+
+# Process each video in the directory
+for video in "$VIDEO_DIR"/*.mp4; do
+    if [ -f "$video" ]; then
+        echo "Processing: $video"
+        
+        # Get video dimensions
+        width=$(ffprobe -v error -select_streams v:0 -show_entries stream=width -of default=noprint_wrappers=1:nokey=1 "$video")
+        height=$(ffprobe -v error -select_streams v:0 -show_entries stream=height -of default=noprint_wrappers=1:nokey=1 "$video")
+        
+        # Calculate crop dimensions for 720:640 aspect ratio
+        if [ "$width" -gt "$height" ]; then
+            # If video is wider than tall
+            crop_width=$(( height * TARGET_WIDTH / TARGET_HEIGHT ))
+            crop_height=$height
+            x_offset=$(( (width - crop_width) / 2 ))
+            y_offset=0
+        else
+            # If video is taller than wide
+            crop_width=$width
+            crop_height=$(( width * TARGET_HEIGHT / TARGET_WIDTH ))
+            x_offset=0
+            y_offset=$(( (height - crop_height) / 2 ))
+        fi
+        
+        # Process video with ffmpeg
+        ffmpeg -i "$video" \
+            -af "aresample=48000" \
+            -vf "crop=$crop_width:$crop_height:$x_offset:$y_offset,scale=$TARGET_WIDTH:$TARGET_HEIGHT" \
+            -c:v libx264 \
+            -c:a aac \
+            -y \
+            "${video%.*}_processed.mp4"
+            
+        echo "Completed: $video"
+
+        # move the original video to a new name
+        mv "$video" "${video%.*}_old.mp4"
+        mv "${video%.*}_processed.mp4" "$video"
+    fi
+done
+
+echo "All videos processed!"

process_videos.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Directory containing the videos
+VIDEO_DIR="predefined"
+
+# Target dimensions
+TARGET_WIDTH=720
+TARGET_HEIGHT=640
+
+# Process each video in the directory
+for video in "$VIDEO_DIR"/*.mov; do
+    if [ -f "$video" ]; then
+        echo "Processing: $video"
+        
+        # Get video dimensions
+        width=$(ffprobe -v error -select_streams v:0 -show_entries stream=width -of default=noprint_wrappers=1:nokey=1 "$video")
+        height=$(ffprobe -v error -select_streams v:0 -show_entries stream=height -of default=noprint_wrappers=1:nokey=1 "$video")
+        
+        # Calculate crop dimensions for 720:640 aspect ratio
+        if [ "$width" -gt "$height" ]; then
+            # If video is wider than tall
+            crop_width=$(( height * TARGET_WIDTH / TARGET_HEIGHT ))
+            crop_height=$height
+            x_offset=$(( (width - crop_width) / 2 ))
+            y_offset=0
+        else
+            # If video is taller than wide
+            crop_width=$width
+            crop_height=$(( width * TARGET_HEIGHT / TARGET_WIDTH ))
+            x_offset=0
+            y_offset=$(( (height - crop_height) / 2 ))
+        fi
+        
+        # Process video with ffmpeg
+        ffmpeg -i "$video" \
+            -af "aresample=48000" \
+            -vf "crop=$crop_width:$crop_height:$x_offset:$y_offset,scale=$TARGET_WIDTH:$TARGET_HEIGHT" \
+            -c:v libx264 \
+            -c:a aac \
+            -y \
+            "${video%.*}_processed.mov"
+            
+        echo "Completed: $video"
+    fi
+done
+
+echo "All videos processed!" 

requirements.txt

@@ -0,0 +1,20 @@
+flask==2.0.1
+gunicorn==20.1.0
+gradio==5.22.0
+gradio_client==1.8.0
+gradio_webrtc==0.0.31
+opencv-python==4.10.0.84
+numpy==1.24.4
+twilio==9.3.7
+scipy==1.15.2
+mediapipe==0.10.18
+moviepy==1.0.3
+librosa==0.10.2.post1
+pydub==0.25.1
+pyttsx3==2.98
+absl-py==2.1.0
+fastapi==0.115.4
+pydantic==2.9.2
+sqlmodel==0.0.24
+passlib==1.7.4
+gTTS==2.5.4

run.sh

@@ -0,0 +1,2 @@
+python backend.py > backend.log 2>&1 & 
+python app.py

run_load_balancer.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Run the load balancer with gunicorn
+echo "Starting load balancer..."
+gunicorn --bind 0.0.0.0:80 \
+         --workers 4 \
+         --timeout 120 \
+         --access-logfile access.log \
+         --error-logfile error.log \
+         load_balancer:app 

run_multiple.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Kill any existing Python processes running on these ports
+for port in {7850..7869}; do
+    lsof -ti:$port | xargs kill -9 2>/dev/null
+done
+
+# Start 10 instances of the app
+for port in {7850..7869}; do
+    echo "Starting app on port $port"
+    python app.py --port $port &
+    # Store the process ID
+    echo $! > "app_$port.pid"
+    # Wait a bit between launches to prevent port conflicts
+    sleep 2
+done
+
+echo "All apps started. To stop them, run: ./stop_apps.sh" 

run_multiple2.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Kill any existing Python processes running on these ports
+for port in {7860..7869}; do
+    lsof -ti:$port | xargs kill -9 2>/dev/null
+done
+
+# Start 10 instances of the app
+for port in {7860..7869}; do
+    echo "Starting app on port $port"
+    python app.py --port $port &
+    # Store the process ID
+    echo $! > "app_$port.pid"
+    # Wait a bit between launches to prevent port conflicts
+    sleep 2
+done
+
+echo "All apps started. To stop them, run: ./stop_apps.sh" 

setup_firewall.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Function to check if ufw is installed
+check_ufw() {
+    if ! command -v ufw &> /dev/null; then
+        echo "UFW is not installed. Installing..."
+        sudo apt-get update
+        sudo apt-get install -y ufw
+    fi
+}
+
+# Function to enable ufw if not already enabled
+enable_ufw() {
+    if ! sudo ufw status | grep -q "Status: active"; then
+        echo "Enabling UFW..."
+        sudo ufw enable
+    fi
+}
+
+# Function to allow SSH (important to do this first to prevent lockout)
+allow_ssh() {
+    echo "Allowing SSH connections..."
+    sudo ufw allow ssh
+}
+
+# Function to allow required ports
+allow_ports() {
+    echo "Setting up firewall rules..."
+    
+    # Allow HTTP (80)
+    sudo ufw allow 80/tcp
+    echo "Port 80 (HTTP) allowed"
+    
+    # Allow HTTPS (443)
+    sudo ufw allow 443/tcp
+    echo "Port 443 (HTTPS) allowed"
+    
+    # Allow ports 7850-7869
+    for port in {7850..7869}; do
+        sudo ufw allow $port/tcp
+        echo "Port $port allowed"
+    done
+    
+    # Allow WebSocket connections
+    sudo ufw allow 7860/tcp
+    echo "Port 7860 (WebSocket) allowed"
+}
+
+# Function to show firewall status
+show_status() {
+    echo "Current firewall status:"
+    sudo ufw status verbose
+}
+
+# Main script
+echo "Setting up firewall rules..."
+
+# Check and install UFW if needed
+check_ufw
+
+# Enable UFW
+enable_ufw
+
+# Allow SSH first (important!)
+allow_ssh
+
+# Allow required ports
+allow_ports
+
+# Show final status
+show_status
+
+echo "Firewall setup completed. The following ports are now open:"
+echo "- Port 80 (HTTP)"
+echo "- Port 443 (HTTPS)"
+echo "- Ports 7850-7869 (Application servers)"
+echo "- Port 7860 (WebSocket)"
+echo "- Port 22 (SSH)" 

setup_ssl.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Create SSL directory if it doesn't exist
+sudo mkdir -p /etc/nginx/ssl
+
+# Function to generate self-signed certificate
+generate_self_signed() {
+    echo "Generating self-signed SSL certificate..."
+    sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
+        -keyout /etc/nginx/ssl/key.pem \
+        -out /etc/nginx/ssl/cert.pem \
+        -subj "/C=US/ST=State/L=City/O=Organization/CN=localhost"
+    echo "Self-signed certificate generated successfully"
+}
+
+# Function to copy existing certificates
+copy_certificates() {
+    if [ -f "cert.pem" ] && [ -f "key.pem" ]; then
+        echo "Copying existing SSL certificates..."
+        sudo cp cert.pem /etc/nginx/ssl/
+        sudo cp key.pem /etc/nginx/ssl/
+        sudo chmod 600 /etc/nginx/ssl/key.pem
+        echo "Certificates copied successfully"
+    else
+        echo "Certificate files not found. Generating self-signed certificate..."
+        generate_self_signed
+    fi
+}
+
+# Function to check SSL configuration
+check_ssl_config() {
+    echo "Checking SSL configuration..."
+    sudo nginx -t
+    if [ $? -eq 0 ]; then
+        echo "SSL configuration is valid"
+    else
+        echo "SSL configuration check failed"
+        exit 1
+    fi
+}
+
+# Main script
+echo "Setting up SSL for Nginx..."
+
+# Copy or generate certificates
+copy_certificates
+
+# Check SSL configuration
+check_ssl_config
+
+# Restart Nginx to apply changes
+echo "Restarting Nginx to apply SSL configuration..."
+sudo systemctl restart nginx
+
+echo "SSL setup completed. Nginx is now configured for HTTPS on port 443"
+echo "You can access your application at https://localhost" 

stop_apps.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Kill all running instances
+for port in {7850..7869}; do
+    if [ -f "app_$port.pid" ]; then
+        pid=$(cat "app_$port.pid")
+        kill $pid 2>/dev/null
+        rm "app_$port.pid"
+        echo "Stopped app on port $port"
+    fi
+done
+
+# Clean up any remaining processes on these ports
+for port in {7850..7869}; do
+    lsof -ti:$port | xargs kill -9 2>/dev/null
+done
+
+echo "All apps stopped" 

video.py

@@ -0,0 +1,815 @@
+import gradio as gr # 0.0.4
+import cv2
+import numpy as np
+from gradio_webrtc import WebRTC
+from twilio.rest import Client
+import os
+import spaces
+import time
+from bisect import bisect_left
+from scipy.spatial.distance import cdist
+import mediapipe as mp
+from mediapipe.tasks.python import vision
+from mediapipe.framework.formats import landmark_pb2
+from mediapipe import solutions
+import pdb
+import json
+
+# 启用 GPU 选项
+base_options = mp.tasks.BaseOptions(model_asset_path='gesture_recognizer.task',
+    delegate=mp.tasks.BaseOptions.Delegate.GPU)
+
+mp_drawing = mp.solutions.drawing_utils
+mp_hands = mp.solutions.hands
+
+base_options=mp.tasks.BaseOptions(model_asset_path='hand_landmarker.task',
+    delegate=mp.tasks.BaseOptions.Delegate.GPU)
+options = vision.HandLandmarkerOptions(base_options=base_options,
+    running_mode=mp.tasks.vision.RunningMode.VIDEO,
+    num_hands=2)
+detector = vision.HandLandmarker.create_from_options(options)
+
+options_image = vision.HandLandmarkerOptions(base_options=base_options,
+    running_mode=mp.tasks.vision.RunningMode.IMAGE,
+    num_hands=2)
+detector_image = vision.HandLandmarker.create_from_options(options_image)
+
+# # 配置 Hand Landmarker
+# hands = HandLandmarker.HandLandmarkerOptions(
+#     base_options=base_options,
+#     num_hands=2,  # 检测最多 2 只手
+#     min_hand_detection_confidence=0.5,
+#     min_hand_presence_confidence=0.5,
+#     min_tracking_confidence=0.5
+# )
+
+# # 初始化 MediaPipe Hands 模型
+# hands = mp_hands.Hands(
+#     static_image_mode=False,
+#     max_num_hands=2,
+#     min_detection_confidence=0.3,
+#     # model_complexity=1  # 启用分类,
+# )
+
+account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
+auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
+
+if account_sid and auth_token:
+    client = Client(account_sid, auth_token)
+
+    token = client.tokens.create()
+
+    rtc_configuration = {
+        "iceServers": token.ice_servers,
+        "iceTransportPolicy": "relay",
+        "video": {
+            "width": 500,
+            "height": 500
+        }
+    }
+else:
+    rtc_configuration = None
+
+
+# 全局变量
+previous_timestamp = None
+start_time = None
+frame_cnt = 0
+video_size = (500, 500)
+SAMPLING_RATE = 48000
+HAND_CONNECTIONS = [
+    (0, 1), (1, 2), (2, 3), (3, 4),  # 拇指
+    (0, 5), (5, 6), (6, 7), (7, 8),  # 食指
+    (0, 9), (9, 10), (10, 11), (11, 12),  # 中指
+    (0, 13), (13, 14), (14, 15), (15, 16),  # 无名指
+    (0, 17), (17, 18), (18, 19), (19, 20)  # 小指
+]
+PREDEFINED_VIDEOS = {
+    "Example Video 1": 'predefined/Trim1.mov',
+    "Example Video 2": 'predefined/Trim2.mov',
+    "Example Video 3": 'predefined/Trim3.mov',
+    "Example Video 4": 'predefined/Trim4.mov',
+    "Example Video 5": 'predefined/Trim5.mov',
+    "Example Video 6": 'predefined/Trim6.mov',
+}
+
+
+class ReferenceVideo:
+    def __init__(self):
+        self.keypoints = {"Left": [], "Right": []}
+        # self.timestamps = []
+        # self.duration = 0
+        self.frames = []       # 存储原始视频帧
+    
+    def load_video(self, video_path):
+        self.keypoints = {"Left": [], "Right": []}
+        self.frames = []
+        cap = cv2.VideoCapture(video_path)
+        global previous_timestamp
+        while cap.isOpened():
+            ret, frame = cap.read()
+            if not ret:
+                break
+            
+            # 1. 显式拷贝并创建独立的图像数据
+            rgb_data = frame.astype(np.uint8).copy()
+            rgb = mp.Image(image_format=mp.ImageFormat.SRGB,data=np.array(cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)))
+            
+            # 2. 生成严格递增的时间戳（单位：毫秒）
+            timestamp_ms = int(cap.get(cv2.CAP_PROP_POS_MSEC))
+            while previous_timestamp is not None and timestamp_ms <= previous_timestamp:
+                timestamp_ms = previous_timestamp + 1
+            previous_timestamp = timestamp_ms
+            
+            # 3. 调用检测器
+            results = detector.detect_for_video(rgb, timestamp_ms)
+            
+            # 4. 处理检测结果
+            frame_landmarks = {"Left": None, "Right": None}
+            if results.hand_landmarks and results.handedness:
+                for idx, hand_landmarks in enumerate(results.hand_landmarks):
+                    label = results.handedness[idx][0].category_name
+                    landmarks = [(lm.x, lm.y) for lm in hand_landmarks]
+                    frame_landmarks[label] = landmarks
+            
+            self.keypoints["Left"].append(frame_landmarks["Left"])
+            self.keypoints["Right"].append(frame_landmarks["Right"])
+            # self.timestamps.append(timestamp_ms / 1000)  # 统一使用计算后的时间戳
+            self.frames.append(cv2.resize(frame, video_size))
+            
+            # 5. 显式释放资源
+            del results  # 关键：释放检测结果占用的GPU资源
+            del rgb     # 释放Image实例
+        
+        # self.duration = self.timestamps[-1] if self.timestamps else 0
+        cap.release()
+        if len(self.frames) != 0:
+            return "Loaded predefined successfully"
+        return "Error: Video not found"
+    
+    def load_keypoints(self, json_path, video_path):
+        self.keypoints = {"Left": [], "Right": []}
+        self.frames = []
+        cap = cv2.VideoCapture(video_path)
+        while cap.isOpened():
+            ret, frame = cap.read()
+            if not ret:
+                break
+            self.frames.append(cv2.resize(frame, video_size))
+        cap.release()
+        with open(json_path, "r") as f:
+            self.keypoints = json.load(f)
+
+# 初始化参考视频
+ref_video = ReferenceVideo()
+
+def process_selected_video(video_name):
+    video_path = PREDEFINED_VIDEOS.get(video_name, None)
+    if video_path and os.path.exists(video_path):
+        keypoints_path = os.path.splitext(video_path)[0] + "_keypoints.json"  # 假设 keypoints 存在
+        ref_video.load_keypoints(keypoints_path, video_path)  # 加载关键点
+        return "Loaded predefined successfully", video_path  # 返回状态 & 视频路径
+    return "Error: Video not found", None
+
+def compute_overall_similarity(sim_matrix):
+    """计算整体相似度（加权综合）"""
+    if sim_matrix is None:
+        return 0.0
+    
+    # 对角线权重更高（关键点自身相似度）
+    diag_weight = 0.6
+    diag_mean = np.mean(np.diag(sim_matrix))
+    
+    # 非对角线部分
+    off_diag_weight = 0.4
+    mask = np.ones(sim_matrix.shape, dtype=bool)
+    np.fill_diagonal(mask, 0)
+    off_diag_mean = np.mean(sim_matrix[mask])
+    
+    return diag_weight * diag_mean + off_diag_weight * off_diag_mean
+
+# def normalize_hand(hand):
+#     """使用手掌宽度归一化手部关键点，避免影响手势形态"""
+#     if hand is None:
+#         return None
+#     hand = np.array(hand)
+    
+#     # 计算手的中心点
+#     center = np.mean(hand, axis=0)
+
+#     # 计算手掌宽度（食指掌关节 - 小指掌关节）
+#     palm_width = np.linalg.norm(hand[5] - hand[17])  # keypoints 5 和 17 分别是食指掌关节和小指掌关节
+
+#     # 避免除零错误
+#     palm_width = max(palm_width, 1e-6)
+
+#     # 归一化关键点
+#     normalized_hand = (hand - center) / palm_width
+#     return normalized_hand
+
+# def compute_distance_matrix(hand):
+#     """计算关键点之间的距离矩阵"""
+#     if hand is None:
+#         return None
+#     num_points = 21
+#     matrix = np.zeros((num_points, num_points))
+#     for i in range(num_points):
+#         for j in range(num_points):
+#             matrix[i, j] = np.linalg.norm(np.array(hand[i]) - np.array(hand[j]))
+#     return matrix
+
+# def kl_divergence(p, q):
+#     """计算 KL 散度"""
+#     p = np.asarray(p) + 1e-10
+#     q = np.asarray(q) + 1e-10
+#     return np.sum(p * np.log(p / q))
+
+# def compute_similarity_matrix(matrix1, matrix2):
+#     """计算相似度矩阵，使用 KL 散度"""
+#     if matrix1 is None or matrix2 is None:
+#         return None
+#     similarity_matrix = np.zeros((21, 21))
+#     for i in range(21):
+#         similarity_matrix[i] = 1 - kl_divergence(matrix1[i], matrix2[i])  # 相似度归一化
+#     return similarity_matrix
+
+def normalize_hand(hand):
+    # """使用手掌对角线归一化手部关键点，避免影响手势形态"""
+    # if hand is None:
+    #     return None
+    # hand = np.array(hand)
+    
+    # # 计算手的中心点
+    # center = hand[0]  # 以手腕 (关键点 0) 作为中心
+    
+    # # 计算手掌对角线（手腕到小指掌关节的距离）
+    # palm_size = np.linalg.norm(hand[0] - hand[17])  # keypoints 0 (手腕) 和 17 (小指掌关节)
+    
+    # # 避免除零错误
+    # palm_size = max(palm_size, 1e-6)
+
+    # # 归一化关键点
+    # normalized_hand = (hand - center) / palm_size
+    # return normalized_hand
+    """改进的归一化：使用多关键点计算缩放因子"""
+    if hand is None:
+        return None
+    hand = np.array(hand)
+    
+    # 使用手腕到中指根部 + 手掌宽度的综合缩放因子
+    palm_length = np.linalg.norm(hand[0] - hand[9])   # 手腕到中指根部
+    palm_width = np.linalg.norm(hand[5] - hand[17])   # 食指到小指宽度
+    scale_factor = 0.5 * (palm_length + palm_width)  # 综合指标
+    
+    # 防零处理
+    scale_factor = max(scale_factor, 1e-6)
+    
+    # 以手腕为中心归一化
+    normalized_hand = (hand - hand[0]) / scale_factor
+    return normalized_hand
+
+def compute_distance_matrix(hand):
+    """计算关键点之间的欧几里得距离矩阵"""
+    if hand is None:
+        return None
+    num_points = 21
+    matrix = np.zeros((num_points, num_points))
+    for i in range(num_points):
+        for j in range(num_points):
+            matrix[i, j] = np.linalg.norm(hand[i] - hand[j])
+    return matrix
+
+# def compute_angle_features(hand):
+#     """计算关键点相对手掌中心的角度特征"""
+#     if hand is None:
+#         return None
+#     hand = np.array(hand)
+#     angles = []
+#     for i in range(1, 21):  # 计算关键点 1-20 相对手腕 (0) 的角度
+#         vec = hand[i] - hand[0]
+#         angle = np.arctan2(vec[1], vec[0])  # 计算 2D 角度
+#         angles.append(angle)
+#     return np.array(angles)
+
+# def cosine_similarity(p, q):
+#     """计算余弦相似度"""
+#     if p is None or q is None:
+#         return None
+#     p = np.asarray(p) + 1e-10
+#     q = np.asarray(q) + 1e-10
+#     return 1 - cosine(p, q)
+
+# def compute_similarity_matrix(matrix1, matrix2):
+#     """计算相似度矩阵，使用余弦相似度"""
+#     if matrix1 is None or matrix2 is None:
+#         return None
+#     similarity_matrix = np.zeros((21, 21))
+#     for i in range(21):
+#         similarity_matrix[i] = cosine_similarity(matrix1[i], matrix2[i])  # 计算相似度
+#     return similarity_matrix
+
+def kl_divergence(p, q):
+    """计算 KL 散度，并增加动态阈值控制"""
+    p = np.asarray(p) + 1e-10
+    q = np.asarray(q) + 1e-10
+    divergence = np.sum(p * np.log(p / q))
+    return divergence
+
+def compute_similarity_matrix(matrix1, matrix2):
+    # """计算相似度矩阵，结合欧氏距离和 KL 散度"""
+    # if matrix1 is None or matrix2 is None:
+    #     return None
+    # num_points = 21
+    # similarity_matrix = np.zeros((num_points, num_points))
+    
+    # for i in range(num_points):
+    #     for j in range(num_points):
+    #         euclidean_diff = abs(matrix1[i, j] - matrix2[i, j])  # 计算欧氏距离差异
+    #         euclidean_similarity = np.exp(-euclidean_diff)  # 转换为相似性（越小相似度越高）
+            
+    #         kl_sim = 1 / (1 + kl_divergence(matrix1[i], matrix2[i]))  # KL 散度转换为相似性
+            
+    #         similarity_matrix[i, j] = 0.5 * euclidean_similarity + 0.5 * kl_sim  # 加权组合相似性
+            
+    # return similarity_matrix
+    """改进的相似度计算：引入高斯核平滑和对称性处理"""
+    if matrix1 is None or matrix2 is None:
+        return None
+    
+    num_points = 21
+    similarity_matrix = np.zeros((num_points, num_points))
+    
+    # 对距离矩阵进行高斯平滑（减少噪声影响）
+    sigma = 0.5  # 高斯核宽度，控制平滑程度
+    gauss_kernel = np.exp(-(matrix1 - matrix2)**2 / (2 * sigma**2))
+    
+    for i in range(num_points):
+        for j in range(num_points):
+            # 欧氏距离相似度（高斯核转换）
+            euclidean_sim = gauss_kernel[i, j]
+            
+            # JS散度替代KL散度（对称且更稳定）
+            p = matrix1[i] + 1e-10
+            q = matrix2[i] + 1e-10
+            m = 0.5 * (p + q)
+            js_div = 0.5 * (kl_divergence(p, m) + kl_divergence(q, m))
+            js_sim = 1 / (1 + js_div)  # 转换为相似度
+            
+            # 动态权重调整（更依赖欧氏距离）
+            similarity_matrix[i, j] = 0.7 * euclidean_sim + 0.3 * js_sim
+    
+    # 增加矩阵整体均值作为稳定性补偿
+    matrix_sim = np.mean(similarity_matrix)
+    return similarity_matrix * 0.9 + matrix_sim * 0.1
+
+
+# def compute_distance_matrix(hand):
+#     """计算关键点归一化距离矩阵（21x21）"""
+#     if hand is None or len(hand) != 21:
+#         return None
+    
+#     # 转换为numpy数组
+#     points = np.array([(x, y) for (x, y) in hand], dtype=np.float32)
+    
+#     # 计算欧氏距离矩阵
+#     dist_matrix = np.linalg.norm(points[:, None] - points, axis=2)
+    
+#     # 归一化到[0,1]
+#     max_dist = np.max(dist_matrix)
+#     if max_dist > 0:
+#         dist_matrix /= max_dist
+#     return dist_matrix
+
+# def row_to_probability(row, temperature=0.1):
+#     """将距离行转换为概率分布（Softmax归一化）"""
+#     scaled = -row / temperature  # 距离越小，概率越高
+#     exp_values = np.exp(scaled - np.max(scaled))  # 数值稳定性优化
+#     return exp_values / np.sum(exp_values)
+
+# def js_divergence(p_row, q_row):
+#     """计算对称化散度（Jensen-Shannon）"""
+#     p = p_row + 1e-10  # 防止除零
+#     q = q_row + 1e-10
+#     m = 0.5 * (p + q)
+#     return 0.5 * (np.sum(p * np.log(p / m)) + np.sum(q * np.log(q / m)))
+
+# def compute_similarity_matrix(matrix1, matrix2, temperature=0.1):
+#     """生成21x21相似度矩阵"""
+#     if matrix1 is None or matrix2 is None:
+#         return None
+    
+#     similarity_matrix = np.zeros((21, 21))
+    
+#     for i in range(21):
+#         # 转换行概率分布
+#         p_row = row_to_probability(matrix1[i], temperature)
+#         q_row = row_to_probability(matrix2[i], temperature)
+        
+#         # 计算对称散度并映射到相似度
+#         js = js_divergence(p_row, q_row)
+#         similarity_matrix[i] = 1 - np.sqrt(js)  # 映射到[0,1]
+#     return np.clip(similarity_matrix, 0.0, 1.0)
+
+# 存储平滑后的相似度
+smooth_sim_left = np.ones(21) * 0.5  # 初始值设为 0.8
+smooth_sim_right = np.ones(21) * 0.5
+alpha = 0.2
+
+# def draw_hand(img, left_landmarks, left_similarity_matrix, right_landmarks, right_similarity_matrix):
+#     """绘制手部关键点，颜色由相似度平滑过渡（红 -> 绿）"""
+#     global smooth_sim_left, smooth_sim_right, alpha  # 平滑参数需在外部初始化
+
+#     if left_similarity_matrix is not None:
+#         for i in range(21):
+#             x, y = int(left_landmarks[i][0] * img.shape[1]), int(left_landmarks[i][1] * img.shape[0])
+#             # 计算当前关键点的平均相似度（可根据需要调整权重）
+#             sim_score = np.mean(left_similarity_matrix[i])
+            
+#             # 指数平滑处理（建议alpha=0.2~0.3）
+#             smooth_sim_left[i] = alpha * sim_score + (1 - alpha) * smooth_sim_left[i]
+#             sim_score = smooth_sim_left[i]
+
+#             # 非线性颜色映射增强对比度（sim_score范围[0,1]）
+#             g = int(255 * (sim_score ** 2))  # 高相似度时绿色更突出
+#             r = int(255 * ((1 - sim_score) ** 2))  # 低相似度时红色更突出
+#             color = (0, g, r)  # OpenCV使用BGR格式
+
+#             cv2.circle(img, (x, y), 6, color, -1)
+
+#         # 绘制连接线（保持原逻辑）
+#         for i, j in HAND_CONNECTIONS:
+#             p1 = (int(left_landmarks[i][0] * img.shape[1]), int(left_landmarks[i][1] * img.shape[0]))
+#             p2 = (int(left_landmarks[j][0] * img.shape[1]), int(left_landmarks[j][1] * img.shape[0]))
+#             cv2.line(img, p1, p2, (0, 255, 255), 2)
+
+#     # 右侧手部同理
+#     if right_similarity_matrix is not None:
+#         for i in range(21):
+#             x, y = int(right_landmarks[i][0] * img.shape[1]), int(right_landmarks[i][1] * img.shape[0])
+#             sim_score = np.mean(right_similarity_matrix[i])
+#             smooth_sim_right[i] = alpha * sim_score + (1 - alpha) * smooth_sim_right[i]
+#             sim_score = smooth_sim_right[i]
+
+#             g = int(255 * (sim_score ** 2))
+#             r = int(255 * ((1 - sim_score) ** 2))
+#             color = (0, g, r)
+#             cv2.circle(img, (x, y), 6, color, -1)
+
+#         for i, j in HAND_CONNECTIONS:
+#             p1 = (int(right_landmarks[i][0] * img.shape[1]), int(right_landmarks[i][1] * img.shape[0]))
+#             p2 = (int(right_landmarks[j][0] * img.shape[1]), int(right_landmarks[j][1] * img.shape[0]))
+#             cv2.line(img, p1, p2, (0, 255, 255), 2)
+
+#     return img
+
+def draw_hand(img, left_landmarks, left_similarity_matrix, right_landmarks, right_similarity_matrix):
+    """绘制手部关键点，颜色深度由相似度决定"""
+
+    # if left_similarity_matrix is not None:
+    #     for i in range(21):
+    #         x, y = int(left_landmarks[i][0] * img.shape[1]), int(left_landmarks[i][1] * img.shape[0])
+    #         sim_score = np.mean(left_similarity_matrix[i])
+    #         color = (0, int(sim_score * 255), int((1 - sim_score) * 255))  # 绿色->红色
+    #         cv2.circle(img, (x, y), 6, color, -1)
+
+    #     for i, j in HAND_CONNECTIONS:
+    #         p1 = (int(left_landmarks[i][0] * img.shape[1]), int(left_landmarks[i][1] * img.shape[0]))
+    #         p2 = (int(left_landmarks[j][0] * img.shape[1]), int(left_landmarks[j][1] * img.shape[0]))
+    #         cv2.line(img, p1, p2, (0, 255, 255), 2)
+
+    # if right_similarity_matrix is not None:
+    #     for i in range(21):
+    #         x, y = int(right_landmarks[i][0] * img.shape[1]), int(right_landmarks[i][1] * img.shape[0])
+    #         sim_score = np.mean(right_similarity_matrix[i])
+    #         color = (0, int(sim_score * 255), int((1 - sim_score) * 255))  # 绿色->红色
+    #         cv2.circle(img, (x, y), 6, color, -1)
+
+    #     for i, j in HAND_CONNECTIONS:
+    #         p1 = (int(right_landmarks[i][0] * img.shape[1]), int(right_landmarks[i][1] * img.shape[0]))
+    #         p2 = (int(right_landmarks[j][0] * img.shape[1]), int(right_landmarks[j][1] * img.shape[0]))
+    #         cv2.line(img, p1, p2, (0, 255, 255), 2)
+
+    global smooth_sim_left, smooth_sim_right, alpha  # 需要在多帧之间保持平滑值
+
+    if left_similarity_matrix is not None:
+        for i in range(21):
+            x, y = int(left_landmarks[i][0] * img.shape[1]), int(left_landmarks[i][1] * img.shape[0])
+            sim_score = np.mean(left_similarity_matrix[i])
+
+            # 平滑处理，减少抖动
+            smooth_sim_left[i] = alpha * sim_score + (1 - alpha) * smooth_sim_left[i]
+            sim_score = smooth_sim_left[i]
+
+            # 颜色映射优化
+            if sim_score > 0.7:
+                color = (0, 255, 0)  # 纯绿色（高相似度）
+            else:
+                color = (0, 0, 255)  # 纯红色（低相似度）
+            # else:
+            #     g = int(255 * (sim_score ** 2))  # 平滑过渡的绿色
+            #     r = int(255 * ((1 - sim_score) ** 2))  # 平滑过渡的红色
+            #     color = (0, g, r)
+
+            cv2.circle(img, (x, y), 6, color, -1)
+
+        for i, j in HAND_CONNECTIONS:
+            p1 = (int(left_landmarks[i][0] * img.shape[1]), int(left_landmarks[i][1] * img.shape[0]))
+            p2 = (int(left_landmarks[j][0] * img.shape[1]), int(left_landmarks[j][1] * img.shape[0]))
+            cv2.line(img, p1, p2, (0, 255, 255), 2)
+
+    if right_similarity_matrix is not None:
+        for i in range(21):
+            x, y = int(right_landmarks[i][0] * img.shape[1]), int(right_landmarks[i][1] * img.shape[0])
+            sim_score = np.mean(right_similarity_matrix[i])
+
+            # 平滑处理，减少抖动
+            smooth_sim_right[i] = alpha * sim_score + (1 - alpha) * smooth_sim_right[i]
+            sim_score = smooth_sim_right[i]
+
+            # 颜色映射优化
+            if sim_score > 0.7:
+                color = (0, 255, 0)  # 纯绿色（高相似度）
+            else:
+                color = (0, 0, 255)  # 纯红色（低相似度）
+            # else:
+            #     g = int(255 * (sim_score ** 2))  # 平滑过渡的绿色
+            #     r = int(255 * ((1 - sim_score) ** 2))  # 平滑过渡的红色
+            #     color = (0, g, r)
+
+            cv2.circle(img, (x, y), 6, color, -1)
+
+        for i, j in HAND_CONNECTIONS:
+            p1 = (int(right_landmarks[i][0] * img.shape[1]), int(right_landmarks[i][1] * img.shape[0]))
+            p2 = (int(right_landmarks[j][0] * img.shape[1]), int(right_landmarks[j][1] * img.shape[0]))
+            cv2.line(img, p1, p2, (0, 255, 255), 2)
+
+    return img
+
+def draw_hand_gt(img, left_landmarks, right_landmarks):
+    """绘制手部关键点，全部标记为蓝色"""
+
+    if left_landmarks is not None:
+        for i in range(21):
+            x, y = int(left_landmarks[i][0] * img.shape[1]), int(left_landmarks[i][1] * img.shape[0])
+            color = (255, 0, 0)  # 纯蓝色
+            cv2.circle(img, (x, y), 6, color, -1)
+        
+        for i, j in HAND_CONNECTIONS:
+            p1 = (int(left_landmarks[i][0] * img.shape[1]), int(left_landmarks[i][1] * img.shape[0]))
+            p2 = (int(left_landmarks[j][0] * img.shape[1]), int(left_landmarks[j][1] * img.shape[0]))
+            cv2.line(img, p1, p2, (255, 0, 0), 2)  # 连接线也是蓝色
+
+    if right_landmarks is not None:
+        for i in range(21):
+            x, y = int(right_landmarks[i][0] * img.shape[1]), int(right_landmarks[i][1] * img.shape[0])
+            color = (255, 0, 0)  # 纯蓝色
+            cv2.circle(img, (x, y), 6, color, -1)
+        
+        for i, j in HAND_CONNECTIONS:
+            p1 = (int(right_landmarks[i][0] * img.shape[1]), int(right_landmarks[i][1] * img.shape[0]))
+            p2 = (int(right_landmarks[j][0] * img.shape[1]), int(right_landmarks[j][1] * img.shape[0]))
+            cv2.line(img, p1, p2, (255, 0, 0), 2)  # 连接线也是蓝色
+
+    return img
+
+def draw_hand_landmarks(img, real_left_matrix, ref_left_matrix, real_right_matrix, ref_right_matrix, real_hand_data):
+    """在实时视频帧上绘制手势关键点，并连线"""
+    img_h, img_w, _ = img.shape
+
+    def draw_hand(hand_data, real_matrix, ref_matrix):
+        if hand_data is not None and real_matrix is not None and ref_matrix is not None:
+            # 计算并绘制关键点
+            for i, (x, y) in enumerate(hand_data):
+                sim = np.exp(-kl_divergence(real_matrix[i], ref_matrix[i]))
+                red_intensity = int(255 * (1 - sim))  # 低相似度 => 更红
+                green_intensity = int(255 * sim)  # 高相似度 => 更绿
+                color = (0, green_intensity, red_intensity)
+                cv2.circle(img, (int(x * img_w), int(y * img_h)), 5, color, -1)
+
+            # 绘制关键点连线
+            for connection in HAND_CONNECTIONS:
+                start_idx, end_idx = connection
+                if start_idx < len(hand_data) and end_idx < len(hand_data):
+                    x1, y1 = hand_data[start_idx]
+                    x2, y2 = hand_data[end_idx]
+                    start_pos = (int(x1 * img_w), int(y1 * img_h))
+                    end_pos = (int(x2 * img_w), int(y2 * img_h))
+                    cv2.line(img, start_pos, end_pos, (255, 255, 255), 2)  # 线条颜色为白色，宽度2px
+
+    # 处理左手
+    draw_hand(real_hand_data["Left"], real_left_matrix, ref_left_matrix)
+    
+    # 处理右手
+    draw_hand(real_hand_data["Right"], real_right_matrix, ref_right_matrix)
+    
+    return img
+
+def draw_landmarks_on_image(rgb_image, detection_result):
+
+    hand_landmarks_list = detection_result.hand_landmarks
+    handedness_list = detection_result.handedness
+    annotated_image = np.copy(rgb_image)
+    
+    for idx in range(len(hand_landmarks_list)):
+        hand_landmarks = hand_landmarks_list[idx]
+        handedness = handedness_list[idx]
+
+        # Draw the hand landmarks.
+        hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
+        hand_landmarks_proto.landmark.extend([
+            landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in hand_landmarks
+            ])
+        solutions.drawing_utils.draw_landmarks(
+            annotated_image,
+            hand_landmarks_proto,
+            solutions.hands.HAND_CONNECTIONS,
+            solutions.drawing_styles.get_default_hand_landmarks_style(),
+            solutions.drawing_styles.get_default_hand_connections_style())
+    return annotated_image
+
+def detection(img):
+    global start_time, frame_cnt, video_size
+    img = cv2.resize(img, video_size)
+    # 降低分辨率处理
+    # small_img = cv2.resize(img, (640, 480))
+    img = cv2.flip(img, 1)
+    rgb = mp.Image(image_format=mp.ImageFormat.SRGB,data=np.array(cv2.cvtColor(img,cv2.COLOR_BGR2RGB)))
+    results = detector_image.detect(rgb)
+
+    # annotated_image = draw_landmarks_on_image(rgb.numpy_view(), results)
+    # return annotated_image
+
+    if not ref_video.frames:
+        cv2.putText(img, "Reference video not loaded", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 
+                    1, (0, 0, 255), 2)
+        return img, (SAMPLING_RATE, np.zeros(int(SAMPLING_RATE * 0.5), dtype=np.float32))
+
+    if not results.hand_landmarks:
+        cv2.putText(img, "No hand detected", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 
+                    1, (0, 0, 255), 2)
+
+        combined_img = cv2.hconcat([img, ref_video.frames[frame_cnt]])
+        frame_cnt += 1
+        frame_cnt = frame_cnt % len(ref_video.frames)
+        return combined_img, (SAMPLING_RATE, np.zeros(int(SAMPLING_RATE * 0.5), dtype=np.float32))
+
+    if start_time is None:
+        start_time = time.time()
+
+    # # 计算当前时间戳
+    # current_time = time.time() - start_time
+    # aligned_time = max(0, current_time - time_offset)
+
+    # 找到最近的参考帧
+    # ref_index = find_nearest_frame(aligned_time % ref_video.duration)
+    ref_index = frame_cnt
+    ref_left = ref_video.keypoints["Left"][ref_index]
+    ref_right = ref_video.keypoints["Right"][ref_index]
+
+    # 解析当前帧
+    real_hand_data = {"Left": None, "Right": None}
+    for idx, hand_landmarks in enumerate(results.hand_landmarks):
+        label = results.handedness[idx][0].category_name
+        real_hand_data[label] = [(lm.x, lm.y) for lm in hand_landmarks]
+    
+    # real_left_norm = normalize_hand(real_hand_data["Left"])
+    # real_right_norm = normalize_hand(real_hand_data["Right"])
+    # ref_left_norm = normalize_hand(ref_left)
+    # ref_right_norm = normalize_hand(ref_right)
+    # real_left_matrix = compute_distance_matrix(real_left_norm)
+    # real_right_matrix = compute_distance_matrix(real_right_norm)
+    # ref_left_matrix = compute_distance_matrix(ref_left_norm)
+    # ref_right_matrix = compute_distance_matrix(ref_right_norm)
+    # sim_left_matrix = compute_similarity_matrix(real_left_matrix, ref_left_matrix)
+    # sim_right_matrix = compute_similarity_matrix(real_right_matrix, ref_right_matrix)
+
+    # 归一化手部关键点
+    real_left_norm = normalize_hand(real_hand_data["Left"])
+    real_right_norm = normalize_hand(real_hand_data["Right"])
+    ref_left_norm = normalize_hand(ref_left)
+    ref_right_norm = normalize_hand(ref_right)
+
+    # 计算距离矩阵
+    real_left_matrix = compute_distance_matrix(real_left_norm)
+    real_right_matrix = compute_distance_matrix(real_right_norm)
+    ref_left_matrix = compute_distance_matrix(ref_left_norm)
+    ref_right_matrix = compute_distance_matrix(ref_right_norm)
+
+    # # 计算角度特征（用于辅助相似度判断）
+    # real_left_angles = compute_angle_features(real_left_norm)
+    # real_right_angles = compute_angle_features(real_right_norm)
+    # ref_left_angles = compute_angle_features(ref_left_norm)
+    # ref_right_angles = compute_angle_features(ref_right_norm)
+
+    # 计算距离矩阵的相似度
+    sim_left_matrix = compute_similarity_matrix(real_left_matrix, ref_left_matrix)
+    sim_right_matrix = compute_similarity_matrix(real_right_matrix, ref_right_matrix)
+
+    # # 计算角度特征的相似度（可选）
+    # angle_similarity_left = cosine_similarity(real_left_angles, ref_left_angles)
+    # angle_similarity_right = cosine_similarity(real_right_angles, ref_right_angles)
+
+    # # 归一化最终相似度矩阵
+    # final_sim_left = sim_left_matrix * angle_similarity_left
+    # final_sim_right = sim_right_matrix * angle_similarity_right
+
+    # sim_left_matrix = compute_similarity_matrix(real_hand_data["Left"], ref_left)
+    # sim_right_matrix = compute_similarity_matrix(real_hand_data["Right"], ref_right)
+
+    # 在实时视频帧 `img` 上绘制关键点
+    # img = draw_hand_landmarks(img, real_left_matrix, ref_left_matrix, real_right_matrix, ref_right_matrix, real_hand_data)
+    img_left = draw_hand(img, real_hand_data["Left"], sim_left_matrix, real_hand_data["Right"], sim_right_matrix)
+    img_right = draw_hand_gt(ref_video.frames[frame_cnt], ref_left, ref_right)
+
+    # left_sim = compute_overall_similarity(sim_left_matrix)
+    # right_sim = compute_overall_similarity(sim_right_matrix)
+    # overall_sim = max(left_sim, right_sim) 
+    # feedback_manager.check_and_generate(overall_sim)
+    # try:
+    #     audio_data = feedback_manager.audio_queue.get_nowait()
+    # except feedback_manager.audio_queue.Empty:
+    #     audio_data = (SAMPLING_RATE, np.zeros(int(SAMPLING_RATE * 0.1), dtype=np.float32))
+    # if not feedback_manager.audio_queue.empty():
+    #     audio_data = feedback_manager.audio_queue.get()
+    # else:
+    #     audio_data = (SAMPLING_RATE, np.zeros(int(SAMPLING_RATE * 0.1), dtype=np.float32))
+
+    # 拼接 `img` 和比对视频的对应帧
+    combined_img = cv2.hconcat([img_left, img_right])
+
+    frame_cnt += 1
+    frame_cnt = frame_cnt % len(ref_video.frames)
+
+    return combined_img
+
+# Gradio 界面
+css = """
+.container { max-width: 1200px; margin: auto; }
+.video-column { background: #f5f5f5; padding: 20px; border-radius: 10px; width: 3000px; height: 50%}
+.alert-panel { min-height: 150px; }
+.loading { text-align: center; padding: 50px; }
+.custom-blue-button {
+    background-color: #87CEEB;  /* 天蓝色背景 */
+    color: white;  /* 白色字体 */
+    border: none;  /* 去掉默认边框 */
+    border-radius: 5px;  /* 圆角 */
+    font-weight: bold;  /* 加粗字体 */
+    width: 100%;  /* 按钮占满列宽 */
+    text-align: center;  /* 文字居中 */
+    cursor: pointer;  /* 鼠标悬停时显示手形 */
+    transition: background-color 0.3s ease;  /* 添加过渡效果 */
+.webcam-container {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    height: 50%;  /* 限制 webcam 的高度 */
+    max-width: 100%;  /* 限制 webcam 宽度 */
+}
+"""
+
+with gr.Blocks(css=css) as demo:
+    gr.HTML("<h1 style='text-align:center;color:#2c3e50;'>Intelligent Gesture Comparison System</h1>")
+    
+    with gr.Accordion("Reference Video Configuration", open=True):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("## Reference video configuration")
+
+                # 创建两个并排的部分：上传新视频 和 选择已有视频
+                with gr.Row():  # 用 gr.Row() 实现平行布局
+                    with gr.Column():
+                        # 上传新视频部分
+                        upload_video = gr.Video(label="Upload standard action video", sources=["upload"], elem_classes="video-upload")
+                        upload_btn = gr.Button("Analysis video", variant="primary", elem_classes="custom-blue-button")
+                    
+                    with gr.Column():
+                        # 选择已有视频部分
+                        selected_video_display = gr.Video(label="Selected Video", interactive=False)
+                        video_dropdown = gr.Dropdown(choices=list(PREDEFINED_VIDEOS.keys()), label="Choose Predefined Video")
+                        select_btn = gr.Button("Load Selected Video", variant="primary", elem_classes="custom-blue-button")
+
+                upload_status = gr.Textbox(label="Processing status", interactive=False)
+        
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("## Real-time detection screen")
+            video_output = WebRTC(
+                label="Stream",
+                rtc_configuration=rtc_configuration,
+            )
+    # with gr.Row():
+    #     sim_display = gr.Number(label="Real-time similarity", value=0.0)
+    #     feedback_audio = gr.Audio(
+    #         label="Audio", 
+    #         autoplay=True,
+    #         # interactive=False,
+    #         # # every=0.5  # 每0.5秒检查更新
+    #         # type="numpy", 
+    #     )
+
+    select_btn.click(process_selected_video, inputs=video_dropdown, outputs=[upload_status, selected_video_display])
+    upload_btn.click(ref_video.load_video, inputs=upload_video, outputs=upload_status)
+    video_output.stream(fn=detection, inputs=[video_output], outputs=[video_output], time_limit=90, concurrency_limit=2)
+
+if __name__ == "__main__":
+    demo.launch(share=True)

video_audio.py

@@ -0,0 +1,122 @@
+import asyncio
+import os
+import cv2
+import numpy as np
+import gradio as gr # 0.0.28
+from mediapipe import solutions
+from gradio_webrtc import WebRTC, AsyncAudioVideoStreamHandler, AudioEmitType, VideoEmitType
+from twilio.rest import Client
+import io
+from pydub import AudioSegment
+import pdb
+
+# 🔹 初始化 MediaPipe Hands
+hands = solutions.hands.Hands(static_image_mode=False, max_num_hands=2, min_detection_confidence=0.3)
+
+# 🔹 Twilio ICE 配置（WebRTC 服务器）
+account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
+auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
+
+if account_sid and auth_token:
+    client = Client(account_sid, auth_token)
+    token = client.tokens.create()
+    rtc_configuration = {"iceServers": token.ice_servers, "iceTransportPolicy": "relay"}
+else:
+    rtc_configuration = None
+
+# 🎤 采样率
+SAMPLING_RATE = 16000
+
+# ================================
+# 📌 实现 WebRTC 处理音视频数据
+# ================================
+class AVHandler(AsyncAudioVideoStreamHandler):
+    def __init__(self):
+        super().__init__(expected_layout="mono", output_sample_rate=SAMPLING_RATE, output_frame_size=480)
+        self.audio_queue = asyncio.Queue()
+        self.video_queue = asyncio.Queue()
+        self.very_good_audio = self.generate_very_good_audio()
+    
+    def copy(self):
+        return AVHandler()
+    
+    async def video_receive(self, frame: np.ndarray):
+        """处理 WebRTC 采集到的视频帧"""
+        # 进行手势检测
+        img = cv2.resize(frame, (640, 480))
+        results = hands.process(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+
+        if results.multi_hand_landmarks:
+            for landmarks in results.multi_hand_landmarks:
+                solutions.drawing_utils.draw_landmarks(img, landmarks, solutions.hands.HAND_CONNECTIONS)
+
+        self.video_queue.put_nowait(cv2.resize(img, (500, 500)))
+
+    async def video_emit(self) -> VideoEmitType:
+        """发送处理后的视频流"""
+        return await self.video_queue.get()
+
+    async def receive(self, frame: tuple[int, np.ndarray]) -> None:
+        """接收音频流"""
+        _, array = frame
+        # if array.ndim == 1:  # 确保数据为二维
+        #     array = np.expand_dims(array, axis=-1)  # 变为 (samples, 1)
+        self.audio_queue.put_nowait(array)
+
+    async def emit(self) -> AudioEmitType:
+        """发送音频流"""
+        # array = await self.audio_queue.get()
+        array = self.very_good_audio.reshape(1, -1)
+        # if array.ndim == 1:  # 确保数据为二维
+        #     array = np.expand_dims(array, axis=-1)
+        return (SAMPLING_RATE, array)
+
+    def generate_very_good_audio(self):
+        """使用 pydub 生成‘very good’的音频数据"""
+        from gtts import gTTS
+
+        # 生成语音文件
+        tts = gTTS("very good", lang="en")
+        audio_io = io.BytesIO()
+        tts.write_to_fp(audio_io)
+        audio_io.seek(0)
+
+        # 加载为 pydub AudioSegment
+        audio = AudioSegment.from_file(audio_io, format="mp3")
+        audio = audio.set_frame_rate(SAMPLING_RATE).set_channels(1)
+
+        # 转换为 NumPy 数组
+        samples = np.array(audio.get_array_of_samples()).astype(np.float32)
+        samples /= np.iinfo(audio.array_type).max  # 归一化到 [-1, 1]
+        
+        return samples
+
+# ================================
+# 📌 Gradio 界面
+# ================================
+css = """
+.my-group {max-width: 600px !important; max-height: 600px !important;}
+.my-column {display: flex !important; justify-content: center !important; align-items: center !important;}
+"""
+
+with gr.Blocks(css=css) as demo:
+    gr.HTML("<h1 style='text-align: center'>实时音视频手势检测 🎥🎤</h1>")
+    
+    with gr.Column(elem_classes=["my-column"]):
+        with gr.Group(elem_classes=["my-group"]):
+            webrtc = WebRTC(
+                label="Stream",
+                modality="audio-video",
+                mode="send-receive",
+                rtc_configuration=rtc_configuration,
+                icon="🔵",
+                pulse_color="rgb(35, 157, 225)"
+            )
+    
+        # 绑定音视频处理
+        webrtc.stream(
+            AVHandler(), inputs=[webrtc], outputs=[webrtc], time_limit=90, concurrency_limit=2
+        )
+
+if __name__ == "__main__":
+    demo.launch(share=True)