fix: model paths (.pth), landmark normalization, WS URL, GPU fallback; add ModelSelector; mobile layout improvements

backend/app/config.py

@@ -24,11 +24,12 @@ BASE_DIR = Path(__file__).resolve().parent.parent.parent  # repo root
 WEIGHTS_DIR = os.getenv("WEIGHTS_DIR", str(BASE_DIR))
 
 # Individual model paths (relative to repo root)
-PIPELINE_A_MODEL   = os.path.join(WEIGHTS_DIR, "Mediapipe_XGBoost",       "model.pkl")
-PIPELINE_B_AE      = os.path.join(WEIGHTS_DIR, "CNN_Autoencoder_LightGBM", "autoencoder_model.pkl")
-PIPELINE_B_LGBM    = os.path.join(WEIGHTS_DIR, "CNN_Autoencoder_LightGBM", "lgbm_model.pkl")
-PIPELINE_C_CNN     = os.path.join(WEIGHTS_DIR, "CNN_PreTrained",            "cnn_model.pkl")
-PIPELINE_C_SVM     = os.path.join(WEIGHTS_DIR, "CNN_PreTrained",            "svm_model.pkl")
+# Note: the actual files on disk use .pth extension (identical content to .pkl)
+PIPELINE_A_MODEL   = os.path.join(WEIGHTS_DIR, "Mediapipe_XGBoost",       "model.pth")
+PIPELINE_B_AE      = os.path.join(WEIGHTS_DIR, "CNN_Autoencoder_LightGBM", "autoencoder_model.pth")
+PIPELINE_B_LGBM    = os.path.join(WEIGHTS_DIR, "CNN_Autoencoder_LightGBM", "lgbm_model.pth")
+PIPELINE_C_CNN     = os.path.join(WEIGHTS_DIR, "CNN_PreTrained",            "cnn_model.pth")
+PIPELINE_C_SVM     = os.path.join(WEIGHTS_DIR, "CNN_PreTrained",            "svm_model.pth")
 
 # ---------------------------------------------------------------------------
 # Inference thresholds

backend/app/main.py

@@ -118,8 +118,10 @@ async def global_exception_handler(request: Request, exc: Exception):
 def _run_ensemble(
     landmarks: list[float],
     image_b64: str | None = None,
+    model_mode: str | None = None,
 ) -> PredictionResponse:
     store = get_model_store()
+    effective_mode = _resolve_pipeline_mode(model_mode)
     result = ensemble.run(
         landmarks,
         image_input=image_b64,
@@ -128,7 +130,7 @@ def _run_ensemble(
         lgbm_model=store.lgbm_model,
         cnn_model=store.cnn_model,
         svm_model=store.svm_model,
-        pipeline_mode=config.PIPELINE_MODE,
+        pipeline_mode=effective_mode,
         confidence_threshold=config.CONFIDENCE_THRESHOLD,
         secondary_threshold=config.SECONDARY_THRESHOLD,
     )
@@ -157,6 +159,30 @@ def _available_pipelines() -> list[str]:
     return pipelines
 
 
+def _resolve_pipeline_mode(requested_mode: str | None) -> str:
+    """
+    Resolve a per-request pipeline mode safely.
+    Falls back to configured default when requested mode is unavailable.
+    """
+    default_mode = config.PIPELINE_MODE
+    if requested_mode is None:
+        return default_mode
+
+    available = set(_available_pipelines())
+    if requested_mode == "ensemble":
+        return "ensemble"
+    if requested_mode in available:
+        return requested_mode
+
+    logger.warning(
+        "Requested mode '%s' is unavailable. Falling back to '%s'. Available: %s",
+        requested_mode,
+        default_mode,
+        sorted(available),
+    )
+    return default_mode
+
+
 # ---------------------------------------------------------------------------
 # REST endpoints
 # ---------------------------------------------------------------------------
@@ -190,7 +216,11 @@ async def health():
 @app.post("/api/predict", response_model=PredictionResponse)
 async def predict_landmarks(body: LandmarkMessage):
     """REST fallback: send 63 landmark floats, receive prediction."""
-    return _run_ensemble(body.landmarks)
+    return _run_ensemble(
+        body.landmarks,
+        image_b64=body.image_b64,
+        model_mode=body.model_mode,
+    )
 
 
 @app.post("/api/predict/image", response_model=PredictionResponse)
@@ -232,7 +262,11 @@ async def ws_landmarks(ws: WebSocket):
                 msg  = LandmarkMessage(**data)
                 session_id = msg.session_id
 
-                response = _run_ensemble(msg.landmarks)
+                response = _run_ensemble(
+                    msg.landmarks,
+                    image_b64=msg.image_b64,
+                    model_mode=msg.model_mode,
+                )
                 await ws.send_text(response.model_dump_json())
 
             except ValueError as ve:

backend/app/schemas.py

@@ -3,7 +3,7 @@ Pydantic request / response schemas for SanketSetu backend.
 """
 from __future__ import annotations
 
-from typing import List, Optional
+from typing import List, Optional, Literal
 
 from pydantic import BaseModel, Field, field_validator
 
@@ -20,6 +20,14 @@ class LandmarkMessage(BaseModel):
     """
     landmarks: List[float] = Field(..., min_length=63, max_length=63)
     session_id: str = Field(default="default")
+    model_mode: Optional[Literal["A", "B", "C", "ensemble"]] = Field(
+        default=None,
+        description="Optional per-request model mode override",
+    )
+    image_b64: Optional[str] = Field(
+        default=None,
+        description="Optional base-64 hand image used when model_mode='C' or ensemble needs C fallback",
+    )
 
     @field_validator("landmarks")
     @classmethod
@@ -45,6 +53,7 @@ class EnsembleMessage(BaseModel):
     landmarks: List[float] = Field(..., min_length=63, max_length=63)
     image_b64: Optional[str] = Field(default=None)
     session_id: str = Field(default="default")
+    model_mode: Optional[Literal["A", "B", "C", "ensemble"]] = Field(default=None)
 
 
 # ---------------------------------------------------------------------------

fly.toml

@@ -1,42 +0,0 @@
-# fly.toml — Fly.io deployment for SanketSetu backend
-# Deploy: flyctl deploy --dockerfile Dockerfile (from repo root)
-
-app = "sanketsetu-backend"
-primary_region = "maa"   # Mumbai — closest to India
-
-[build]
-  dockerfile = "Dockerfile"
-
-[http_service]
-  internal_port       = 8000
-  force_https         = true
-  auto_stop_machines  = "stop"
-  auto_start_machines = true
-  min_machines_running = 0
-  processes           = ["app"]
-
-  [http_service.concurrency]
-    type       = "requests"
-    soft_limit = 10
-    hard_limit = 25
-
-[vm]
-  size   = "shared-cpu-2x"   # 2 shared vCPUs, 512 MB RAM
-  memory = "1gb"
-
-[[vm.mounts]]
-  # Optional: mount model weights externally to keep image small
-  # source      = "sanketsetu_models"
-  # destination = "/models"
-
-[env]
-  KERAS_BACKEND       = "tensorflow"
-  TF_CPP_MIN_LOG_LEVEL = "3"
-  CUDA_VISIBLE_DEVICES = ""
-  TF_ENABLE_ONEDNN_OPTS = "0"
-  OMP_NUM_THREADS     = "4"
-  PIPELINE_MODE       = "ensemble"
-  CORS_ORIGINS        = "https://sanketsetu.vercel.app,http://localhost:5173"
-
-# Secrets to set via flyctl:
-#   flyctl secrets set CONFIDENCE_THRESHOLD=0.70

frontend/src/App.tsx

@@ -10,19 +10,64 @@ import { WebcamFeed }      from './components/WebcamFeed'
 import { PredictionHUD }   from './components/PredictionHUD'
 import { OnboardingGuide } from './components/OnboardingGuide'
 import { Calibration }     from './components/Calibration'
+import { ModelSelector }   from './components/ModelSelector'
+import type { ModelMode } from './types'
 
-type AppStage = 'onboarding' | 'calibration' | 'running'
+type AppStage = 'onboarding' | 'model-select' | 'calibration' | 'running'
 
 function App() {
   // ── Stage management ─────────────────────────────────────────
   const showOnboarding = !localStorage.getItem('sanketsetu-onboarded')
-  const [stage, setStage] = useState<AppStage>(showOnboarding ? 'onboarding' : 'calibration')
+  const [stage, setStage] = useState<AppStage>(showOnboarding ? 'onboarding' : 'model-select')
+  const savedModel = localStorage.getItem('sanketsetu-model-mode') as ModelMode | null
+  const [selectedModel, setSelectedModel] = useState<ModelMode>(savedModel ?? 'ensemble')
+  const [availableModes, setAvailableModes] = useState<Set<ModelMode>>(new Set(['ensemble']))
 
   const handleOnboardingDone = () => {
     localStorage.setItem('sanketsetu-onboarded', '1')
+    setStage('model-select')
+  }
+
+  const handleModelContinue = () => {
+    localStorage.setItem('sanketsetu-model-mode', selectedModel)
     setStage('calibration')
   }
 
+  useEffect(() => {
+    let active = true
+    const healthUrl = `${resolveBackendHttpBase()}/health`
+
+    const loadAvailability = async () => {
+      try {
+        const res = await fetch(healthUrl)
+        if (!res.ok) return
+        const data = (await res.json()) as { pipelines_available?: string[] }
+        if (!active) return
+
+        const next = new Set<ModelMode>(['ensemble'])
+        for (const mode of data.pipelines_available ?? []) {
+          if (mode === 'A' || mode === 'B' || mode === 'C') {
+            next.add(mode)
+          }
+        }
+        setAvailableModes(next)
+      } catch {
+        // Keep local defaults when backend health is unavailable.
+      }
+    }
+
+    loadAvailability()
+    return () => {
+      active = false
+    }
+  }, [])
+
+  useEffect(() => {
+    if (selectedModel !== 'ensemble' && !availableModes.has(selectedModel)) {
+      setSelectedModel('ensemble')
+    }
+  }, [availableModes, selectedModel])
+
   // ── Webcam ───────────────────────────────────────────────────
   const { videoRef, isReady, error, facingMode, switchCamera } = useWebcam()
 
@@ -47,13 +92,21 @@ function App() {
 
   // ── WebSocket ────────────────────────────────────────────────
   const { lastPrediction, isConnected, latency, lowBandwidth, sendLandmarks } = useWebSocket()
+  const imageCanvasRef = useRef<HTMLCanvasElement | null>(null)
 
   // Send landmarks on every new frame
   useEffect(() => {
     if (stage === 'running' && landmarks) {
-      sendLandmarks(landmarks)
+      let imageB64: string | undefined
+      if (selectedModel === 'C' && videoRef.current) {
+        imageB64 = captureVideoFrame(videoRef.current, imageCanvasRef)
+      }
+      sendLandmarks(landmarks, {
+        modelMode: selectedModel,
+        imageB64,
+      })
     }
-  }, [landmarks, stage, sendLandmarks])
+  }, [landmarks, selectedModel, sendLandmarks, stage, videoRef])
 
   // Was the last prediction recently (within 1.5s)?
   const lastPredTs = useRef(0)
@@ -75,6 +128,18 @@ function App() {
         )}
       </AnimatePresence>
 
+      {/* ── Model selector overlay ─────────────────────────────── */}
+      <AnimatePresence>
+        {stage === 'model-select' && (
+          <ModelSelector
+            selectedMode={selectedModel}
+            availableModes={availableModes}
+            onSelectMode={setSelectedModel}
+            onContinue={handleModelContinue}
+          />
+        )}
+      </AnimatePresence>
+
       {/* ── Calibration overlay ────────────────────────────────── */}
       <AnimatePresence>
         {stage === 'calibration' && (
@@ -118,23 +183,24 @@ function App() {
       </AnimatePresence>
 
       {/* ── Header ─────────────────────────────────────────────── */}
-      <header className="flex items-center justify-between px-6 py-4">
-        <div className="flex items-center gap-3">
-          <Hand size={22} style={{ color: '#00f5d4' }} />
-          <h1 className="text-xl font-bold tracking-wide" style={{ color: '#e2e8f0' }}>
+      <header className="flex items-center justify-between px-3 py-3 sm:px-6 sm:py-4">
+        <div className="flex items-center gap-2 sm:gap-3">
+          <Hand size={20} style={{ color: '#00f5d4' }} />
+          <h1 className="text-base sm:text-xl font-bold tracking-wide" style={{ color: '#e2e8f0' }}>
             Sanket<span className="glow-text">Setu</span>
-            <span className="ml-2 text-base font-normal text-slate-500">| સંકેત-સેતુ</span>
+            <span className="hidden sm:inline ml-2 text-sm font-normal text-slate-500">| સંકેત-સેતુ</span>
           </h1>
         </div>
-        <div className="flex items-center gap-3 text-slate-500 text-xs">
-          {mpLoading && <span>Loading AI…</span>}
-          {mpError   && <span className="text-rose-400">{mpError}</span>}
-          <Settings size={18} className="cursor-pointer hover:text-slate-300 transition-colors" />
+        <div className="flex items-center gap-2 text-slate-500 text-xs">
+          {mpLoading && <span className="hidden sm:inline">Loading AI…</span>}
+          {mpLoading && <span className="sm:hidden">AI…</span>}
+          {mpError   && <span className="text-rose-400 text-xs max-w-[120px] truncate">{mpError}</span>}
+          <Settings size={16} className="cursor-pointer hover:text-slate-300 transition-colors" />
         </div>
       </header>
 
       {/* ── Main content ───────────────────────────────────────── */}
-      <main className="flex-1 flex flex-col lg:flex-row items-start justify-center gap-6 px-4 pb-8 lg:px-8">
+      <main className="flex-1 flex flex-col lg:flex-row items-stretch lg:items-start justify-center gap-3 sm:gap-6 px-2 sm:px-4 pb-4 sm:pb-8 lg:px-8">
 
         {/* Webcam panel */}
         <motion.div
@@ -166,6 +232,7 @@ function App() {
             isConnected={isConnected}
             latency={latency}
             lowBandwidth={lowBandwidth}
+            selectedModel={selectedModel}
           />
         </motion.div>
 
@@ -174,4 +241,40 @@ function App() {
   )
 }
 
+function captureVideoFrame(
+  video: HTMLVideoElement,
+  canvasRef: { current: HTMLCanvasElement | null },
+): string | undefined {
+  if (!video.videoWidth || !video.videoHeight) return undefined
+
+  if (!canvasRef.current) {
+    canvasRef.current = document.createElement('canvas')
+  }
+  const canvas = canvasRef.current
+  canvas.width = 128
+  canvas.height = 128
+
+  const ctx = canvas.getContext('2d')
+  if (!ctx) return undefined
+
+  // Center-crop to square before resizing to model input size.
+  const side = Math.min(video.videoWidth, video.videoHeight)
+  const sx = (video.videoWidth - side) / 2
+  const sy = (video.videoHeight - side) / 2
+  ctx.drawImage(video, sx, sy, side, side, 0, 0, 128, 128)
+
+  return canvas.toDataURL('image/jpeg', 0.85).replace(/^data:image\/jpeg;base64,/, '')
+}
+
+function resolveBackendHttpBase(): string {
+  const envWs = import.meta.env.VITE_WS_URL as string | undefined
+  if (envWs) {
+    return envWs
+      .replace(/^wss:\/\//i, 'https://')
+      .replace(/^ws:\/\//i, 'http://')
+  }
+  if (import.meta.env.DEV) return 'http://localhost:8000'
+  return window.location.origin
+}
+
 export default App

frontend/src/components/ModelSelector.tsx

@@ -0,0 +1,135 @@
+import { motion } from 'framer-motion'
+import { Brain, Gauge, Layers, Aperture } from 'lucide-react'
+import type { ComponentType } from 'react'
+import type { ModelMode } from '../types'
+
+interface Props {
+  selectedMode: ModelMode
+  availableModes: Set<ModelMode>
+  onSelectMode: (mode: ModelMode) => void
+  onContinue: () => void
+}
+
+type ModeMeta = {
+  key: ModelMode
+  title: string
+  subtitle: string
+  details: string
+  icon: ComponentType<{ size?: number }>
+}
+
+const MODE_OPTIONS: ModeMeta[] = [
+  {
+    key: 'ensemble',
+    title: 'Ensemble (Recommended)',
+    subtitle: 'Balanced accuracy and reliability',
+    details: 'Starts with A, falls back to B and C when confidence is low.',
+    icon: Layers,
+  },
+  {
+    key: 'A',
+    title: 'Pipeline A',
+    subtitle: 'Fastest response',
+    details: 'XGBoost using hand landmarks only.',
+    icon: Gauge,
+  },
+  {
+    key: 'B',
+    title: 'Pipeline B',
+    subtitle: 'Stronger landmark reasoning',
+    details: 'Autoencoder embeddings with LightGBM.',
+    icon: Brain,
+  },
+  {
+    key: 'C',
+    title: 'Pipeline C',
+    subtitle: 'Image-based fallback model',
+    details: 'CNN features with SVM using webcam snapshots.',
+    icon: Aperture,
+  },
+]
+
+export function ModelSelector({
+  selectedMode,
+  availableModes,
+  onSelectMode,
+  onContinue,
+}: Props) {
+  const canContinue = selectedMode === 'ensemble' || availableModes.has(selectedMode)
+
+  return (
+    <div className="fixed inset-0 z-40 flex items-center justify-center px-4">
+      <motion.div
+        initial={{ opacity: 0, y: 20 }}
+        animate={{ opacity: 1, y: 0 }}
+        className="w-full max-w-4xl rounded-2xl p-4 sm:p-6"
+        style={{
+          background: 'rgba(5,8,22,0.92)',
+          backdropFilter: 'blur(16px)',
+          border: '1px solid rgba(255,255,255,0.12)',
+          boxShadow: '0 12px 36px rgba(0,0,0,0.45)',
+        }}
+      >
+        <h2 className="text-2xl sm:text-3xl font-bold glow-text text-center">Choose Recognition Model</h2>
+        <p className="text-slate-400 text-center mt-2 text-sm sm:text-base">
+          Select how predictions should be generated for this session.
+        </p>
+
+        <div className="mt-5 grid grid-cols-1 md:grid-cols-2 gap-3">
+          {MODE_OPTIONS.map((option) => {
+            const Icon = option.icon
+            const selected = selectedMode === option.key
+            const available = option.key === 'ensemble' || availableModes.has(option.key)
+
+            return (
+              <button
+                key={option.key}
+                type="button"
+                onClick={() => onSelectMode(option.key)}
+                className="text-left rounded-xl p-4 transition-all"
+                style={{
+                  background: selected ? 'rgba(0,245,212,0.12)' : 'rgba(255,255,255,0.04)',
+                  border: selected
+                    ? '1px solid rgba(0,245,212,0.55)'
+                    : '1px solid rgba(255,255,255,0.10)',
+                  opacity: available ? 1 : 0.5,
+                }}
+              >
+                <div className="flex items-center justify-between gap-2">
+                  <div className="flex items-center gap-2">
+                    <Icon size={16} />
+                    <span className="font-semibold text-slate-100">{option.title}</span>
+                  </div>
+                  {!available && <span className="text-xs text-rose-300">Unavailable</span>}
+                </div>
+                <p className="text-sm text-slate-300 mt-2">{option.subtitle}</p>
+                <p className="text-xs text-slate-500 mt-1">{option.details}</p>
+              </button>
+            )
+          })}
+        </div>
+
+        <div className="mt-6 flex flex-col sm:flex-row items-center justify-between gap-3">
+          <p className="text-xs text-slate-500">
+            Tip: Ensemble is best for most users. Use A for low-latency demos.
+          </p>
+          <button
+            type="button"
+            onClick={onContinue}
+            disabled={!canContinue}
+            className="px-5 py-2.5 rounded-lg font-semibold disabled:cursor-not-allowed"
+            style={{
+              background: canContinue ? 'rgba(0,245,212,0.22)' : 'rgba(148,163,184,0.2)',
+              color: canContinue ? '#99f6e4' : '#94a3b8',
+              border: canContinue
+                ? '1px solid rgba(0,245,212,0.45)'
+                : '1px solid rgba(148,163,184,0.25)',
+            }}
+          >
+            Continue
+          </button>
+        </div>
+      </motion.div>
+    </div>
+  )
+}

frontend/src/components/PredictionHUD.tsx

@@ -1,13 +1,14 @@
 import { useEffect, useRef, useState } from 'react';
 import { motion, AnimatePresence } from 'framer-motion';
 import { Cpu, Zap, Clock } from 'lucide-react';
-import type { PredictionResponse } from '../types';
+import type { ModelMode, PredictionResponse } from '../types';
 
 interface Props {
   prediction: PredictionResponse | null;
   isConnected: boolean;
   latency: number;
   lowBandwidth?: boolean;
+  selectedModel: ModelMode;
 }
 
 const PIPELINE_COLORS: Record<string, string> = {
@@ -21,6 +22,7 @@ const PIPELINE_LABELS: Record<string, string> = {
   A: 'XGBoost',
   B: 'AE + LGBM',
   C: 'CNN + SVM',
+  ensemble: 'A -> B -> C fallback',
 };
 
 function confidenceColor(c: number) {
@@ -47,7 +49,7 @@ function ConfidenceBar({ value }: { value: number }) {
  * Floating HUD panel that shows the current sign prediction, confidence,
  * active pipeline, latency and a rolling history of the last 10 signs.
  */
-export function PredictionHUD({ prediction, isConnected, latency, lowBandwidth = false }: Props) {
+export function PredictionHUD({ prediction, isConnected, latency, lowBandwidth = false, selectedModel }: Props) {
   const [history, setHistory] = useState<PredictionResponse[]>([]);
   const prevSignRef = useRef<string | null>(null);
 
@@ -62,9 +64,10 @@ export function PredictionHUD({ prediction, isConnected, latency, lowBandwidth =
   const pipelineKey = prediction?.pipeline ?? 'A';
   const pipelineColor = PIPELINE_COLORS[pipelineKey] ?? 'text-slate-400';
   const pipelineLabel = PIPELINE_LABELS[pipelineKey] ?? pipelineKey;
+  const selectedLabel = PIPELINE_LABELS[selectedModel] ?? selectedModel;
 
   return (
-    <div className="glass glow-border flex flex-col gap-4 p-5 min-w-[260px] max-w-xs w-full">
+    <div className="glass glow-border flex flex-col gap-3 sm:gap-4 p-3 sm:p-5 w-full lg:min-w-[260px] lg:max-w-xs">
       {/* Connection status */}
       <div className="flex items-center justify-between text-xs text-slate-400">
         <span className="flex items-center gap-1.5">
@@ -90,90 +93,99 @@ export function PredictionHUD({ prediction, isConnected, latency, lowBandwidth =
         </span>
       </div>
 
-      {/* Main sign display */}
-      <div className="flex flex-col items-center gap-1 py-2">
-        <AnimatePresence mode="popLayout">
-          {prediction ? (
-            <motion.div
-              key={prediction.sign}
-              initial={{ opacity: 0, scale: 0.6, y: 10 }}
-              animate={{ opacity: 1, scale: 1, y: 0 }}
-              exit={{ opacity: 0, scale: 0.4, y: -10 }}
-              transition={{ type: 'spring', stiffness: 300, damping: 22 }}
-              className="glow-text text-7xl font-bold select-none"
-              style={{ color: '#00f5d4' }}
-            >
-              {prediction.sign}
-            </motion.div>
-          ) : (
-            <motion.div
-              key="placeholder"
-              initial={{ opacity: 0 }}
-              animate={{ opacity: 0.3 }}
-              exit={{ opacity: 0 }}
-              className="text-5xl font-bold text-slate-500 select-none"
-            >
-              ?
-            </motion.div>
-          )}
-        </AnimatePresence>
+      {/* Main content: sign + history side by side on mobile, stacked on lg */}
+      <div className="flex lg:flex-col gap-3">
+        {/* Main sign display */}
+        <div className="flex flex-col items-center justify-center gap-1 py-1 sm:py-2 min-w-[80px] sm:min-w-0">
+          <AnimatePresence mode="popLayout">
+            {prediction ? (
+              <motion.div
+                key={prediction.sign}
+                initial={{ opacity: 0, scale: 0.6, y: 10 }}
+                animate={{ opacity: 1, scale: 1, y: 0 }}
+                exit={{ opacity: 0, scale: 0.4, y: -10 }}
+                transition={{ type: 'spring', stiffness: 300, damping: 22 }}
+                className="glow-text font-bold select-none leading-none"
+                style={{ color: '#00f5d4', fontSize: 'clamp(2.5rem, 10vw, 4.5rem)' }}
+              >
+                {prediction.sign}
+              </motion.div>
+            ) : (
+              <motion.div
+                key="placeholder"
+                initial={{ opacity: 0 }}
+                animate={{ opacity: 0.3 }}
+                exit={{ opacity: 0 }}
+                className="font-bold text-slate-500 select-none leading-none"
+                style={{ fontSize: 'clamp(2rem, 8vw, 3.5rem)' }}
+              >
+                ?
+              </motion.div>
+            )}
+          </AnimatePresence>
+          <span className="text-[10px] text-slate-500 mt-0.5">Current sign</span>
+        </div>
 
-        {/* Confidence bar */}
-        {prediction && (
-          <div className="w-full px-2 mt-1">
-            <div className="flex justify-between text-xs text-slate-400 mb-1">
-              <span>Confidence</span>
-              <span style={{ color: confidenceColor(prediction.confidence) }}>
-                {Math.round(prediction.confidence * 100)}%
-              </span>
+        {/* Right column on mobile: confidence + pipeline + history */}
+        <div className="flex flex-1 flex-col gap-2 justify-center">
+          {/* Confidence bar */}
+          {prediction && (
+            <div className="w-full">
+              <div className="flex justify-between text-xs text-slate-400 mb-1">
+                <span>Confidence</span>
+                <span style={{ color: confidenceColor(prediction.confidence) }}>
+                  {Math.round(prediction.confidence * 100)}%
+                </span>
+              </div>
+              <ConfidenceBar value={prediction.confidence} />
             </div>
-            <ConfidenceBar value={prediction.confidence} />
-          </div>
-        )}
-      </div>
+          )}
 
-      {/* Pipeline badge */}
-      {prediction && (
-        <div className="flex items-center gap-1.5 text-xs">
-          <Cpu size={12} className={pipelineColor} />
-          <span className={pipelineColor}>Pipeline {pipelineKey}</span>
-          <span className="text-slate-500">·</span>
-          <span className="text-slate-400">{pipelineLabel}</span>
-        </div>
-      )}
+          {/* Pipeline badge */}
+          {prediction && (
+            <div className="flex items-center gap-1.5 text-xs">
+              <Cpu size={12} className={pipelineColor} />
+              <span className={pipelineColor}>Pipeline {pipelineKey}</span>
+              <span className="text-slate-500">·</span>
+              <span className="text-slate-400">{pipelineLabel}</span>
+            </div>
+          )}
 
-      {/* Divider */}
-      <div className="border-t" style={{ borderColor: 'rgba(255,255,255,0.08)' }} />
+          <div className="text-xs text-slate-500">
+            Selected mode: <span className="text-slate-300">{selectedModel} ({selectedLabel})</span>
+          </div>
 
-      {/* History */}
-      <div className="flex flex-col gap-1">
-        <p className="text-xs text-slate-500 mb-1 flex items-center gap-1">
-          <Zap size={11} /> Recent signs
-        </p>
-        <div className="flex flex-wrap gap-1.5">
-          <AnimatePresence>
-            {history.map((h, i) => (
-              <motion.span
-                key={`${h.sign}-${i}`}
-                initial={{ opacity: 0, scale: 0.5 }}
-                animate={{ opacity: 1 - i * 0.08, scale: 1 }}
-                exit={{ opacity: 0, scale: 0.3 }}
-                transition={{ duration: 0.2 }}
-                className="px-2 py-0.5 rounded-full text-sm font-semibold"
-                style={{
-                  background: 'rgba(0,245,212,0.08)',
-                  border: '1px solid rgba(0,245,212,0.2)',
-                  color: '#00f5d4',
-                  fontSize: i === 0 ? '1.1rem' : '0.85rem',
-                }}
-              >
-                {h.sign}
-              </motion.span>
-            ))}
-          </AnimatePresence>
-          {history.length === 0 && (
-            <span className="text-xs text-slate-600 italic">None yet</span>
-          )}
+          {/* History */}
+          <div>
+            <p className="text-xs text-slate-500 mb-1 flex items-center gap-1">
+              <Zap size={11} /> Recent signs
+            </p>
+            <div className="flex flex-wrap gap-1">
+              <AnimatePresence>
+                {history.map((h, i) => (
+                  <motion.span
+                    key={`${h.sign}-${i}`}
+                    initial={{ opacity: 0, scale: 0.5 }}
+                    animate={{ opacity: 1 - i * 0.08, scale: 1 }}
+                    exit={{ opacity: 0, scale: 0.3 }}
+                    transition={{ duration: 0.2 }}
+                    className="px-1.5 py-0.5 rounded-full font-semibold"
+                    style={{
+                      background: 'rgba(0,245,212,0.08)',
+                      border: '1px solid rgba(0,245,212,0.2)',
+                      color: '#00f5d4',
+                      fontSize: i === 0 ? '1rem' : '0.75rem',
+                    }}
+                  >
+                    {h.sign}
+                  </motion.span>
+                ))}
+              </AnimatePresence>
+              {history.length === 0 && (
+                <span className="text-xs text-slate-600 italic">None yet</span>
+              )}
+            </div>
+          </div>
         </div>
       </div>
     </div>

frontend/src/components/WebcamFeed.tsx

@@ -49,7 +49,7 @@ export function WebcamFeed({
       ref={containerRef}
       className="relative rounded-2xl overflow-hidden w-full max-w-2xl"
       style={{
-        aspectRatio: '16/9',
+        aspectRatio: window.innerWidth < 640 ? '4/3' : '16/9',
         border: '1px solid rgba(0,245,212,0.2)',
         boxShadow: '0 0 30px rgba(0,245,212,0.08)',
         background: '#0a0a1a',

frontend/src/hooks/useMediaPipe.ts

@@ -47,14 +47,30 @@ export function useMediaPipe(): MediaPipeState {
     (async () => {
       try {
         const vision = await FilesetResolver.forVisionTasks(WASM_URL);
-        const hl = await HandLandmarker.createFromOptions(vision, {
-          baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },
-          runningMode: 'VIDEO',
-          numHands: 1,
-          minHandDetectionConfidence: 0.5,
-          minHandPresenceConfidence: 0.5,
-          minTrackingConfidence: 0.5,
-        });
+
+        // Try GPU first for best performance; fall back to CPU if unavailable.
+        let hl: HandLandmarker | null = null;
+        try {
+          hl = await HandLandmarker.createFromOptions(vision, {
+            baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },
+            runningMode: 'VIDEO',
+            numHands: 1,
+            minHandDetectionConfidence: 0.4,
+            minHandPresenceConfidence: 0.4,
+            minTrackingConfidence: 0.4,
+          });
+        } catch {
+          console.warn('GPU delegate unavailable, falling back to CPU.');
+          hl = await HandLandmarker.createFromOptions(vision, {
+            baseOptions: { modelAssetPath: MODEL_URL, delegate: 'CPU' },
+            runningMode: 'VIDEO',
+            numHands: 1,
+            minHandDetectionConfidence: 0.4,
+            minHandPresenceConfidence: 0.4,
+            minTrackingConfidence: 0.4,
+          });
+        }
+
         if (!cancelled) {
           landmarkerRef.current = hl;
           setIsLoading(false);

frontend/src/hooks/useWebSocket.ts

@@ -1,11 +1,14 @@
 import { useEffect, useRef, useState, useCallback } from 'react';
-import type { PredictionResponse } from '../types';
+import type { ModelMode, PredictionResponse } from '../types';
 
-// Derive WebSocket base URL from the current page origin so the hook works
-// on any deployment (HF Space, Vercel + backend, localhost) without extra config.
+// Derive WebSocket base URL.
+// Priority: VITE_WS_URL env var → dev fallback (port 8000) → same host (production).
 function _defaultWsUrl(): string {
   if (import.meta.env.VITE_WS_URL) return import.meta.env.VITE_WS_URL as string;
   const proto = window.location.protocol === 'https:' ? 'wss' : 'ws';
+  // In Vite dev mode the frontend is served on 5173 but FastAPI runs on 8000.
+  if (import.meta.env.DEV) return `${proto}://localhost:8000`;
+  // In production the backend is co-located (HF Spaces Docker).
   return `${proto}://${window.location.host}`;
 }
 const WS_URL = _defaultWsUrl();
@@ -20,7 +23,14 @@ export interface WebSocketState {
   isConnected: boolean;
   latency: number;
   lowBandwidth: boolean;
-  sendLandmarks: (landmarks: number[], sessionId?: string) => void;
+  sendLandmarks: (
+    landmarks: number[],
+    options?: {
+      sessionId?: string;
+      modelMode?: ModelMode;
+      imageB64?: string;
+    },
+  ) => void;
 }
 
 /**
@@ -92,7 +102,14 @@ export function useWebSocket(): WebSocketState {
   }, [connect]);
 
   /** Throttled send — adapts to 5fps in low-bandwidth mode (latency > 500ms) */
-  const sendLandmarks = useCallback((landmarks: number[], sessionId = 'browser') => {
+  const sendLandmarks = useCallback((
+    landmarks: number[],
+    options?: {
+      sessionId?: string;
+      modelMode?: ModelMode;
+      imageB64?: string;
+    },
+  ) => {
     const ws = wsRef.current;
     if (!ws || ws.readyState !== WebSocket.OPEN) return;
 
@@ -103,7 +120,12 @@ export function useWebSocket(): WebSocketState {
     lastSendTime.current = now;
 
     inflightTs.current = now;
-    ws.send(JSON.stringify({ landmarks, session_id: sessionId }));
+    ws.send(JSON.stringify({
+      landmarks,
+      session_id: options?.sessionId ?? 'browser',
+      model_mode: options?.modelMode,
+      image_b64: options?.imageB64,
+    }));
   }, [lowBandwidth]);
 
   return { lastPrediction, isConnected, latency, lowBandwidth, sendLandmarks };

frontend/src/lib/landmarkUtils.ts

@@ -19,18 +19,20 @@ export interface RawLandmark {
 
 /**
  * Convert MediaPipe NormalizedLandmark[] (21 points) to a flat 63-element
- * array, then subtract the wrist position to centre the hand.
+ * array of raw [0,1]-normalised coordinates as expected by the trained models.
+ *
+ * The XGBoost, Autoencoder+LGBM models were trained directly on the raw
+ * MediaPipe landmark coordinates (x, y, z per landmark, no wrist-centering).
+ * Sending wrist-subtracted coords produces incorrect predictions.
  */
 export function normaliseLandmarks(raw: RawLandmark[]): number[] {
   if (raw.length !== 21) {
     throw new Error(`Expected 21 landmarks, got ${raw.length}`);
   }
 
-  const wrist = raw[0];
-
   const flat: number[] = [];
   for (const lm of raw) {
-    flat.push(lm.x - wrist.x, lm.y - wrist.y, lm.z - wrist.z);
+    flat.push(lm.x, lm.y, lm.z);
   }
   return flat; // length 63
 }

frontend/src/types.ts

@@ -2,10 +2,12 @@
  * Shared TypeScript types for SanketSetu frontend.
  */
 
+export type ModelMode = 'A' | 'B' | 'C' | 'ensemble'
+
 export interface PredictionResponse {
   sign: string;
   confidence: number;
-  pipeline: string;
+  pipeline: ModelMode | `${'A' | 'B' | 'C'}+${string}`;
   label_index: number;
   probabilities?: number[];
   latency_ms?: number;