feat: implement HQ AI Bridge mode with high-fidelity motion restoration and HF Hub integration

.gitignore

@@ -0,0 +1,3 @@
+weights_hq/
+*.ckpt
+*.pt

backend/debug_large.py

@@ -0,0 +1,45 @@
+import os
+import sys
+import numpy as np
+import torch
+
+# Setup paths
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+if CURRENT_DIR not in sys.path:
+    sys.path.append(CURRENT_DIR)
+
+from sign_bridge_inference import SignBridgeInference
+
+# POINT TO THE LARGE CHECKPOINT
+LARGE_MODEL_PATH = "/Users/harshit/Documents/WEBSITE_EXPLO/sign_idd_model_20260121_171210/best.ckpt"
+MODEL_ROOT = os.path.dirname(LARGE_MODEL_PATH)
+
+print(f"Loading LARGE model from: {LARGE_MODEL_PATH}")
+# SignBridgeInference expects a weights directory with 'best.ckpt'
+engine = SignBridgeInference(MODEL_ROOT)
+
+text = "Today weather rain"
+print(f"Translating: {text}")
+skeletons = engine.translate(text, sampling_steps=50)
+
+skel_array = np.array(skeletons)
+skel_std = np.std(skel_array, axis=0).mean()
+skel_mean = np.mean(skel_array)
+skel_min = np.min(skel_array)
+skel_max = np.max(skel_array)
+
+print("-" * 30)
+print(f"Frames: {len(skeletons)}")
+print(f"Mean Coordinate Value: {skel_mean:.6f}")
+print(f"Min Coord: {skel_min:.6f}, Max Coord: {skel_max:.6f}")
+print(f"Average Variance (STD) across frames: {skel_std:.6f}")
+
+if skel_std < 1e-4:
+    print("CRITICAL: The LARGE model is also still?!")
+else:
+    print("SUCCESS: Motion detected in LARGE model!")
+
+from video_renderer import render_skeleton_to_video
+output_path = os.path.join(CURRENT_DIR, "debug_large_model.mp4")
+render_skeleton_to_video(skeletons, output_path)
+print(f"Video rendered to: {output_path}")

backend/main.py

@@ -8,8 +8,10 @@ if BACKEND_DIR not in sys.path:
 
 try:
     from .model_loader import SignModel
+    from .model_loader_hq import sign_model_hq
 except (ImportError, ValueError):
     from model_loader import SignModel
+    from model_loader_hq import sign_model_hq
 
 from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
@@ -66,3 +68,14 @@ async def translate_text(request: TranslationRequest):
         }
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
+@app.post("/translate_hq", response_model=TranslationResponse)
+async def translate_text_hq(request: TranslationRequest):
+    try:
+        result = sign_model_hq.inference(request.text)
+        return {
+            "skeletons": result.get("skeletons", []),
+            "video_url": result.get("video_url"),
+            "text_processed": request.text
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

backend/model_loader_hq.py

@@ -0,0 +1,120 @@
+import os
+import sys
+import threading
+from typing import Dict, Any, List
+
+# Add backend directory to path
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+if CURRENT_DIR not in sys.path:
+    sys.path.append(CURRENT_DIR)
+
+try:
+    from .sign_bridge_inference_hq import SignBridgeInferenceHQ
+except (ImportError, ValueError):
+    from sign_bridge_inference_hq import SignBridgeInferenceHQ
+
+# HQ Weights are stored in a dedicated directory
+MODEL_ROOT_HQ = os.path.join(os.path.dirname(CURRENT_DIR), "weights_hq")
+CONFIG_PATH_HQ = os.path.join(os.path.dirname(CURRENT_DIR), "model_configs", "Sign-IDD-HQ.yaml")
+
+class SignModelHQ:
+    """
+    Singleton wrapper for the High Fidelity SignBridgeInference engine.
+    Uses the uncompressed 1.1GB weights and optimized motion sampling.
+    """
+    _instance = None
+    _lock = threading.Lock()
+
+    def __new__(cls):
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = super(SignModelHQ, cls).__new__(cls)
+                cls._instance._initialized = False
+            return cls._instance
+
+    def __init__(self):
+        if self._initialized:
+            return
+            
+        print("Initializing SignBridge HQ Model (High Fidelity Path)...")
+        self.engine = None
+        self.is_loaded = False
+        self._load_error = None
+        self._initialized = True
+        
+        # Load in background
+        threading.Thread(target=self._load_model_async, daemon=True).start()
+
+    def _load_model_async(self):
+        try:
+            # 1. Ensure weight directory exists
+            os.makedirs(MODEL_ROOT_HQ, exist_ok=True)
+            weight_path = os.path.join(MODEL_ROOT_HQ, "best.ckpt")
+
+            # 2. Check if weights need to be downloaded (Runtime bypass for 1GB repo limit)
+            if not os.path.exists(weight_path):
+                print(f"HQ Weights not found at {weight_path}. Attempting download from Hub...")
+                from huggingface_hub import hf_hub_download
+                
+                repo_id = os.environ.get("HF_REPO_ID_HQ", "Harshit2907/SignBridge-Weights")
+                token = os.environ.get("HF_TOKEN") # Optional: needed if repo is private
+                
+                print(f"Downloading HQ Weights from {repo_id}...")
+                downloaded_file = hf_hub_download(
+                    repo_id=repo_id,
+                    filename="best.ckpt",
+                    local_dir=MODEL_ROOT_HQ,
+                    token=token
+                )
+                print(f"✅ Download complete: {downloaded_file}")
+
+            # 3. Initialize the HQ-specific inference engine
+            self.engine = SignBridgeInferenceHQ(MODEL_ROOT_HQ)
+            self.is_loaded = True
+            print("✅ SignBridge HQ Model loaded and ready for high-fidelity inference.")
+        except Exception as e:
+            self._load_error = str(e)
+            print(f"❌ Failed to load SignBridge HQ Model: {e}")
+            import traceback
+            traceback.print_exc()
+
+    def inference(self, text: str) -> Dict[str, Any]:
+        if not self.is_loaded:
+            if self._load_error:
+                raise RuntimeError(f"HQ Model failed to load: {self._load_error}")
+            raise RuntimeError("HQ Model is still loading. Please try again in 30 seconds.")
+
+        print(f"HQ Inference Request: '{text}'")
+        
+        try:
+            # HIGH FIDELITY PARAMS:
+            # We use 90 steps (matching original training) and potentially different guidance or length heuristics
+            skeletons = self.engine.translate(text, sampling_steps=90)
+            
+            import uuid
+            from video_renderer import render_skeleton_to_video
+            
+            filename = f"hq_gen_{uuid.uuid4().hex[:8]}.mp4"
+            output_dir = os.path.join(CURRENT_DIR, "output")
+            os.makedirs(output_dir, exist_ok=True)
+            output_path = os.path.join(output_dir, filename)
+            
+            # Use standard renderer (stable)
+            render_skeleton_to_video(skeletons, output_path)
+            
+            # URL resolution (assumes same static mount)
+            video_url = f"https://harshit2907-sign-idd-inference.hf.space/static/{filename}"
+            if os.environ.get("LOCAL_DEV"):
+                 video_url = f"http://127.0.0.1:8001/static/{filename}"
+
+            return {
+                "skeletons": None,
+                "video_url": video_url,
+                "glosses": self.engine.text_to_glosses(text)
+            }
+        except Exception as e:
+            print(f"HQ Inference error: {e}")
+            raise e
+
+# Global singleton instance for HQ
+sign_model_hq = SignModelHQ()

backend/sign_bridge_inference_hq.py

@@ -0,0 +1,82 @@
+import os
+import sys
+import torch
+import numpy as np
+from typing import List, Dict
+
+# Resolve imports
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+if CURRENT_DIR not in sys.path:
+    sys.path.append(CURRENT_DIR)
+
+try:
+    from .sign_bridge_inference import SignBridgeInference
+except (ImportError, ValueError):
+    from sign_bridge_inference import SignBridgeInference
+
+class SignBridgeInferenceHQ(SignBridgeInference):
+    """
+    High Fidelity version of the SignBridge Inference Engine.
+    Uses original uncompressed weights and a 'HQ Sampler' tuned for motion.
+    """
+    
+    def translate(self, text: str, sampling_steps: int = 90) -> List[List[List[float]]]:
+        """
+        Translates text to HQ skeletons using the high-fidelity sampler.
+        """
+        glosses = self.text_to_glosses(text)
+        if not glosses:
+             return []
+
+        # Map glosses to indices
+        tokens = [self.bos_token] + glosses + [self.eos_token]
+        indices = [self.vocab.stoi[t] for t in tokens]
+        
+        dev = self.device
+        src_tensor = torch.tensor([indices], dtype=torch.long, device=dev)
+        src_mask = (src_tensor != self.vocab.stoi[self.pad_token]).unsqueeze(1).unsqueeze(2)
+        src_lengths = torch.tensor([len(indices)], dtype=torch.long, device=dev)
+
+        # 1. Encode source
+        with torch.no_grad():
+            encoder_output = self.model.encode(src_tensor, src_lengths, src_mask)
+
+        # 2. HQ Dynamic Frame Estimation
+        # We increase the frames per word to allow for more fluid motion
+        # Validation videos usually have approx 100-200 frames for a sentence
+        n_frames = max(80, len(glosses) * 20 + 30) 
+        
+        trg_mask = torch.ones((1, 1, n_frames), device=dev, dtype=torch.bool)
+        
+        # 3. HQ Sampling with deterministic/stochastic blend
+        # Note: We can manually call ddim_sample or use our own loop for better variance control
+        # Currently, we'll use the model's ddim_sample but with HQ steps
+        print(f"HQ Sampler: Generating {n_frames} frames over {sampling_steps} steps...")
+        
+        with torch.no_grad():
+            # Create a mock input_3d just for shape
+            mock_input_3d = torch.zeros((1, n_frames, 150), device=dev)
+            
+            # The LARGE model typically performs better at 80-100 steps
+            raw_skels = self.model.ACD.ddim_sample(
+                encoder_output, 
+                mock_input_3d, 
+                src_mask, 
+                trg_mask, 
+                sampling_steps=sampling_steps
+            )
+            
+            # Use the final prediction (x0)
+            raw_skel = raw_skels[-1][0] # (T, 150)
+            
+            # HQ MOTION CALIBRATION:
+            # If the model is slightly shy, we can apply a very subtle Dynamic Range expansion
+            # skel_std = raw_skel.std()
+            # if skel_std < 0.15:
+            #     raw_skel = (raw_skel - raw_skel.mean()) * 1.2 + raw_skel.mean()
+
+            return raw_skel.reshape(n_frames, 50, 3).tolist()
+
+    def text_to_glosses(self, text: str) -> List[str]:
+        # Reuse base class preprocessing
+        return super().text_to_glosses(text)

model_configs/Sign-IDD-HQ.yaml

@@ -0,0 +1,82 @@
+data:
+  src: gloss
+  trg: skels
+  files: files
+  train: ./Data/P2014T_Ben/train
+  dev: ./Data/P2014T_Ben/dev
+  test: ./Data/P2014T_Ben/test
+  max_sent_length: 300
+  skip_frames: 1
+  src_vocab: ./Configs/src_vocab.txt
+
+training:
+  # ---- keep everything as-is (your PINN config) ----
+  overwrite: false
+  save_model: true
+  random_seed: 27
+  optimizer: adam
+  learning_rate: 0.001
+  learning_rate_min: 0.0002
+  weight_decay: 0.0
+  clip_grad_norm: 5.0
+  batch_size: 64
+  scheduling: plateau
+  patience: 7
+  decrease_factor: 0.7
+  early_stopping_metric: dtw
+  epochs: 20000
+  validation_freq: 2000
+  logging_freq: 250
+  eval_metric: dtw
+
+  # ---- checkpoint saving behavior SAME AS the "Base" YAML ----
+  # (Base uses: model_dir, overwrite, continue, keep_last_ckpts)
+  model_dir: /home/user2/THESIS/Sign-IDD-main/Models/PINN_scratch_run1
+  overwrite: false
+  continue: true
+  keep_last_ckpts: 1
+
+  # ---- rest of your training config (unchanged) ----
+  shuffle: true
+  use_cuda: true
+  max_output_length: 300
+  loss: L1
+  bone_loss: MSE
+  hand_joint_weight: 1.5
+  body_joint_weight: 1.0
+  body_bonelen_weight: 0.05
+  hand_bonelen_weight: 0.05
+  lambda_bone: 0.1
+
+  # ---------------- PINN (NEW) ----------------
+  use_pinn: true
+  pinn_weight: 0.1
+
+  pinn_lambda_bone: 1.0
+  pinn_lambda_vel: 0.1
+  pinn_lambda_acc: 0.05
+  pinn_lambda_fk: 0.5
+
+  pinn_dt: 1.0
+  pinn_rest_from: first_valid
+  pinn_detach_rest: true
+  pinn_use_huber: true
+  pinn_huber_delta: 1.0
+
+model:
+  encoder:
+    type: transformer
+    embeddings:
+      embedding_dim: 512
+      dropout: 0.1
+    hidden_size: 512
+    ff_size: 2048
+    num_layers: 6
+    num_heads: 8
+    dropout: 0.1
+
+  trg_size: 150
+
+  diffusion:
+    timesteps: 100
+    sampling_timesteps: 90

requirements.txt

@@ -7,3 +7,4 @@ pyyaml>=6.0.1
 opencv-python-headless>=4.8.1.78
 python-dotenv
 requests
+huggingface-hub