Deepak Roshan commited on
Commit
f124d78
Β·
0 Parent(s):

fresh clean upload

Browse files
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ #images
3
+ images/
4
+
5
+ # Byte-compiled / optimized / DLL files
6
+ __pycache__/
7
+ *.py[cod]
8
+ *$py.class
9
+
10
+ # C extensions
11
+ *.so
12
+
13
+ # Distribution / packaging
14
+ .Python
15
+ build/
16
+ develop-eggs/
17
+ dist/
18
+ downloads/
19
+ eggs/
20
+ .eggs/
21
+ lib/
22
+ lib64/
23
+ parts/
24
+ sdist/
25
+ var/
26
+ wheels/
27
+ share/python-wheels/
28
+ *.egg-info/
29
+ .installed.cfg
30
+ *.egg
31
+ MANIFEST
32
+
33
+ # PyInstaller
34
+ # Usually these files are written by a python script from a template
35
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
36
+ *.manifest
37
+ *.spec
38
+
39
+ # Installer logs
40
+ pip-log.txt
41
+ pip-delete-this-directory.txt
42
+
43
+ # Unit test / coverage reports
44
+ htmlcov/
45
+ .tox/
46
+ .nox/
47
+ .coverage
48
+ .coverage.*
49
+ .cache
50
+ nosetests.xml
51
+ coverage.xml
52
+ *.cover
53
+ *.py,cover
54
+ .hypothesis/
55
+ .pytest_cache/
56
+ cover/
57
+
58
+ # Translations
59
+ *.mo
60
+ *.pot
61
+
62
+ # Django stuff:
63
+ *.log
64
+ local_settings.py
65
+ db.sqlite3
66
+ db.sqlite3-journal
67
+
68
+ # Flask stuff:
69
+ instance/
70
+ .webassets-cache
71
+
72
+ # Scrapy stuff:
73
+ .scrapy
74
+
75
+ # Sphinx documentation
76
+ docs/_build/
77
+
78
+ # PyBuilder
79
+ .pybuilder/
80
+ target/
81
+
82
+ # Jupyter Notebook
83
+ .ipynb_checkpoints
84
+
85
+ # IPython
86
+ profile_default/
87
+ ipython_config.py
88
+
89
+ # pyenv
90
+ # For a library or package, you might want to ignore these files since the code is
91
+ # intended to run in multiple environments; otherwise, check them in:
92
+ # .python-version
93
+
94
+ # pipenv
95
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
96
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
97
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
98
+ # install all needed dependencies.
99
+ #Pipfile.lock
100
+
101
+ # poetry
102
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
103
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
104
+ # commonly ignored for libraries.
105
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
106
+ #poetry.lock
107
+
108
+ # pdm
109
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
110
+ #pdm.lock
111
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
112
+ # in version control.
113
+ # https://pdm.fming.dev/#use-with-ide
114
+ .pdm.toml
115
+
116
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
117
+ __pypackages__/
118
+
119
+ # Celery stuff
120
+ celerybeat-schedule
121
+ celerybeat.pid
122
+
123
+ # SageMath parsed files
124
+ *.sage.py
125
+
126
+ # Environments
127
+ .env
128
+ .venv
129
+ env/
130
+ venv/
131
+ ENV/
132
+ env.bak/
133
+ venv.bak/
134
+ mp_env/
135
+
136
+ # Spyder project settings
137
+ .spyderproject
138
+ .spyproject
139
+
140
+ # Rope project settings
141
+ .ropeproject
142
+
143
+ # mkdocs documentation
144
+ /site
145
+
146
+ # mypy
147
+ .mypy_cache/
148
+ .dmypy.json
149
+ dmypy.json
150
+
151
+ # Pyre type checker
152
+ .pyre/
153
+
154
+ # pytype static type analyzer
155
+ .pytype/
156
+
157
+ # Cython debug symbols
158
+ cython_debug/
159
+
160
+ # PyCharm
161
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
162
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
163
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
164
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
165
+ #.idea/
166
+ voice_*.mp3
167
+ tmp_audio/
168
+ *.mp3
Dockerfile ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ============================================================
2
+ # Dockerfile β€” Hugging Face Spaces (Docker SDK)
3
+ # Free tier: 16GB RAM, 2 vCPU
4
+ # HF exposes only port 7860 β€” nginx routes both Flask apps
5
+ #
6
+ # / β†’ Flask 5000 (gesture + emotion)
7
+ # /avatar/ β†’ Flask 5001 (speech-to-sign)
8
+ # ============================================================
9
+
10
+ FROM python:3.10-slim
11
+
12
+ # HF Spaces runs as non-root user 1000
13
+ RUN useradd -m -u 1000 user
14
+ USER root
15
+
16
+ # ── System deps ───────────────────────────────────────────────
17
+ RUN apt-get update && apt-get install -y --no-install-recommends \
18
+ libgl1 \
19
+ libglib2.0-0 \
20
+ libsm6 \
21
+ libxext6 \
22
+ libxrender-dev \
23
+ libgomp1 \
24
+ nginx \
25
+ supervisor \
26
+ curl \
27
+ git-lfs \
28
+ && rm -rf /var/lib/apt/lists/*
29
+
30
+ WORKDIR /app
31
+
32
+ # ── Python deps ───────────────────────────────────────────────
33
+ COPY requirements.txt .
34
+ RUN pip install --no-cache-dir --upgrade pip && \
35
+ pip install --no-cache-dir -r requirements.txt
36
+
37
+ # ── App files ─────────────────────────────────────────────────
38
+ COPY isl_detection.py .
39
+ COPY isl_ui_dashboard.py .
40
+ COPY launcher.py .
41
+ COPY speech_to_sign_avatar.py .
42
+ COPY translation.py .
43
+ COPY unified_launcher.py .
44
+ COPY model.h5 .
45
+ COPY templates/ templates/
46
+
47
+ # ── Configs ───────────────────────────────────────────────────
48
+ COPY nginx.conf /etc/nginx/nginx.conf
49
+ COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
50
+
51
+ # ── Nginx log dir permissions ─────────────────────────────────
52
+ RUN mkdir -p /var/log/nginx /var/log/supervisor /var/run \
53
+ && chmod -R 777 /var/log/nginx /var/log/supervisor /var/run \
54
+ && chown -R user:user /app
55
+
56
+ # ── Environment ───────────────────────────────────────────────
57
+ ENV RUN_MODE=CLOUD
58
+ ENV PYTHONUNBUFFERED=1
59
+ ENV TF_CPP_MIN_LOG_LEVEL=3
60
+ ENV CUDA_VISIBLE_DEVICES=-1
61
+
62
+ # HF Spaces MUST use port 7860
63
+ EXPOSE 7860
64
+
65
+ # ── Start supervisor as root (nginx needs it) ─────────────────
66
+ USER root
67
+ CMD ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
README.md ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: ISL Indian Sign Language Detection
3
+ emoji: "🀟"
4
+ colorFrom: yellow
5
+ colorTo: green
6
+ sdk: docker
7
+ pinned: false
8
+ license: mit
9
+ app_port: 7860
10
+ ---
11
+
12
+ # 🀟 ISL Indian Sign Language Detection System
13
+
14
+ Real-time Indian Sign Language detection with emotion recognition and speech-to-sign avatar.
15
+
16
+ ## Features
17
+ - Gesture Recognition β€” Real-time ISL A-Z letter detection via MediaPipe + TensorFlow
18
+ - Emotion Detection β€” Happy, Sad, Angry, Surprise, Neutral via facial landmarks
19
+ - Speech-to-Sign Avatar β€” 3D animated hand avatar for fingerspelling
20
+
21
+ ## Usage
22
+ - Main dashboard: `/`
23
+ - Speech-to-Sign avatar: `/avatar/`
24
+
25
+ ## Built with
26
+ - Flask + Socket.IO
27
+ - TensorFlow CNN model
28
+ - MediaPipe Hand Tracking
29
+ - Three.js 3D Avatar
isl_detection.py ADDED
@@ -0,0 +1,918 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # isl_detection.py β€” CLOUD PRODUCTION VERSION
2
+ #
3
+ # ════════════════════════════════════════════════════════════════════════════
4
+ # CLOUD FIXES APPLIED (3 changes only β€” everything else identical):
5
+ #
6
+ # FIX 1 β€” pygame TTS: init only in LOCAL mode (crashes on headless server)
7
+ # FIX 2 β€” cv2.CAP_DSHOW: Windows-only flag removed for Linux cloud server
8
+ # FIX 3 β€” SharedMemory cleanup on startup (prevents crash on restart)
9
+ #
10
+ # LOCAL CAMERA FIXES (built-in laptop webcam static/noise):
11
+ # FIX 4 β€” MJPG codec + 30-frame flush to clear DirectShow buffer on Windows
12
+ # FIX 5 β€” CAP_PROP_BUFFERSIZE set before flush (not after)
13
+ #
14
+ # ALL original features preserved:
15
+ # βœ… Emotion detection v3 (happy/sad/angry/surprise/neutral)
16
+ # βœ… Letter hold timer, same-letter cooldown, confidence buffer
17
+ # βœ… SharedMemory zero-copy frame transfer
18
+ # βœ… CLOUD mode browser frame ingestion (_cloud_frame)
19
+ # βœ… gTTS speak button (disabled silently in cloud β€” browser TTS handles it)
20
+ # βœ… Word suggestions, auto-TTS, reset, backspace, accept_suggestion
21
+ # βœ… Oracle Free Tier safe: TARGET_FPS=15, EMOTION_PROCESS_INTERVAL=5
22
+ # ════════════════════════════════════════════════════════════════════════════
23
+
24
+ import cv2
25
+ import mediapipe as mp
26
+ import numpy as np
27
+ import string
28
+ import time
29
+ import copy
30
+ import itertools
31
+ import os
32
+ import base64
33
+ import platform
34
+ import threading
35
+ from collections import deque, defaultdict, Counter
36
+ from multiprocessing import Manager, Value, shared_memory
37
+ import ctypes
38
+
39
+ os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
40
+ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
41
+
42
+ from tensorflow.keras.models import load_model
43
+ from gtts import gTTS
44
+ import pygame
45
+
46
+ try:
47
+ import translation
48
+ TRANSLATION_AVAILABLE = True
49
+ except ImportError:
50
+ TRANSLATION_AVAILABLE = False
51
+
52
+ # ─────────────────────────────────────────────────────────────
53
+ # CONFIG
54
+ # ─────────────────────────────────────────────────────────────
55
+ RUN_MODE = os.getenv("RUN_MODE", "LOCAL")
56
+ ISL_MODEL_PATH = "model.h5"
57
+ CAM_INDEX = 0
58
+
59
+ # ── Letter timing ──────────────────────────────────────────────────────────
60
+ LETTER_HOLD_SEC = 0.9
61
+ SAME_LETTER_COOLDOWN_SEC = 1.2
62
+
63
+ # ── Emotion thresholds ────────────────────────────────────────────────────
64
+ SMOOTHING_FRAMES = 2
65
+ MIN_CONF_TO_SHOW = 0.06
66
+ EMOTIONS = ["happy", "sad", "angry", "surprise", "neutral"]
67
+
68
+ CONF_THRESHOLD = 0.32
69
+ SPACE_THRESHOLD = 18
70
+ ENABLE_AUTO_TTS = True
71
+ TTS_LANGUAGE = "en"
72
+
73
+ SHOW_LETTER_OVERLAY = True
74
+ OVERLAY_COLOR_LETTER = (0, 255, 255)
75
+ OVERLAY_COLOR_CONF = (255, 255, 0)
76
+ OVERLAY_POSITION = (20, 50)
77
+ OVERLAY_FONT_SCALE = 1.5
78
+ OVERLAY_THICKNESS = 3
79
+
80
+ COMMON_WORDS = [
81
+ "HELLO", "HELP", "PLEASE", "THANK", "YOU", "YES", "NO", "GOOD", "BAD",
82
+ "MORNING", "AFTERNOON", "EVENING", "NIGHT", "TODAY", "TOMORROW", "WATER",
83
+ "FOOD", "HOME", "SCHOOL", "WORK", "HAPPY", "SAD", "SORRY", "WELCOME"
84
+ ]
85
+
86
+ TARGET_FPS = 15
87
+ CAMERA_RESOLUTION = (640, 480)
88
+ EMOTION_PROCESS_INTERVAL = 5
89
+ FRAME_W, FRAME_H = 640, 480
90
+
91
+
92
+ # ─────────────────────────────────────────────────────────────
93
+ # SHARED STATE
94
+ # ─────────────────────────────────────────────────────────────
95
+ class SharedState:
96
+ def __init__(self):
97
+ manager = Manager()
98
+ self.ui_queue = manager.Queue(200)
99
+ self.command_queue = manager.Queue(20)
100
+ self.processing_fps = Value(ctypes.c_double, 0.0)
101
+
102
+
103
+ # ─────────────────────────────────────────────────────────────
104
+ # TTS ENGINE
105
+ # FIX 1: pygame.mixer.init() only runs in LOCAL mode.
106
+ # On cloud (headless Linux) there is no audio device β€”
107
+ # init() would crash the entire process at import time.
108
+ # In CLOUD mode TTS is silently disabled here; the browser
109
+ # handles speech via the Web Speech API on the client side.
110
+ # ─────────────────────────────────────────────────────────────
111
+ TTS_ENABLED = False
112
+ if RUN_MODE == "LOCAL":
113
+ try:
114
+ pygame.mixer.init()
115
+ TTS_ENABLED = True
116
+ print("βœ… TTS engine initialized")
117
+ except Exception as e:
118
+ print(f"⚠️ TTS init warning: {e}")
119
+ else:
120
+ print("ℹ️ TTS disabled (CLOUD/headless mode) β€” browser handles speech")
121
+
122
+ speaking_word = ""
123
+ pause_audio = False
124
+
125
+
126
+ def speak_text(text, lang=TTS_LANGUAGE):
127
+ global speaking_word
128
+
129
+ # FIX 1 continued: guard every call β€” silently skip on cloud
130
+ if not TTS_ENABLED or not text or not text.strip():
131
+ return
132
+
133
+ def _speak():
134
+ global pause_audio, speaking_word
135
+ try:
136
+ fname = f"voice_{int(time.time()*1000)}.mp3"
137
+ gTTS(text=text, lang=lang).save(fname)
138
+ speaking_word = text
139
+ pygame.mixer.music.load(fname)
140
+ pygame.mixer.music.play()
141
+ while pygame.mixer.music.get_busy():
142
+ if pause_audio:
143
+ pygame.mixer.music.pause()
144
+ while pause_audio:
145
+ time.sleep(0.1)
146
+ pygame.mixer.music.unpause()
147
+ pygame.time.Clock().tick(10)
148
+ speaking_word = ""
149
+ pygame.mixer.music.stop()
150
+ pygame.mixer.music.unload()
151
+ time.sleep(0.15)
152
+ for _ in range(3):
153
+ try:
154
+ if os.path.exists(fname):
155
+ os.remove(fname)
156
+ break
157
+ except PermissionError:
158
+ time.sleep(0.2)
159
+ except Exception as e:
160
+ print(f"TTS error: {e}")
161
+ speaking_word = ""
162
+
163
+ threading.Thread(target=_speak, daemon=True).start()
164
+
165
+
166
+ # ─────────────────────────────────────────────────────────────
167
+ # LANDMARK UTILITIES
168
+ # ─────────────────────────────────────────────────────────────
169
+ def calc_landmark_list(image, landmarks):
170
+ w, h = image.shape[1], image.shape[0]
171
+ return [
172
+ [min(int(lm.x * w), w - 1), min(int(lm.y * h), h - 1)]
173
+ for lm in landmarks.landmark
174
+ ]
175
+
176
+
177
+ def pre_process_landmark(landmark_list):
178
+ temp = copy.deepcopy(landmark_list)
179
+ bx, by = temp[0]
180
+ for i in range(len(temp)):
181
+ temp[i][0] -= bx
182
+ temp[i][1] -= by
183
+ flat = list(itertools.chain.from_iterable(temp))
184
+ mv = max(map(abs, flat)) if flat else 1
185
+ return [n / mv for n in flat]
186
+
187
+
188
+ def apply_clahe(frame):
189
+ """Enhance low-light / low-contrast frames for mobile cameras."""
190
+ try:
191
+ lab = cv2.cvtColor(frame, cv2.COLOR_BGR2LAB)
192
+ l, a, b = cv2.split(lab)
193
+ clahe = cv2.createCLAHE(clipLimit=2.5, tileGridSize=(8, 8))
194
+ return cv2.cvtColor(cv2.merge((clahe.apply(l), a, b)), cv2.COLOR_LAB2BGR)
195
+ except Exception:
196
+ return frame
197
+
198
+
199
+ # ─────────────────────────────────────────────────────────────
200
+ # EMOTION DETECTION β€” v3
201
+ # Redesigned SAD / ANGRY / SURPRISE with external facial cues.
202
+ # HAPPY unchanged.
203
+ # ─────────────────────────────────────────────────────────────
204
+
205
+ def _dist(a, b):
206
+ """Euclidean distance between two 2-D landmark points."""
207
+ return float(np.linalg.norm(
208
+ np.array(a, dtype=np.float32) - np.array(b, dtype=np.float32)
209
+ ))
210
+
211
+
212
+ def detect_emotion_from_landmarks(face_landmarks, img_shape):
213
+ """
214
+ Derive emotion scores from 468 MediaPipe Face Mesh landmarks.
215
+ Returns {emotion: float} summing to 1.0.
216
+
217
+ Key landmark reference:
218
+ Mouth corners : 61 (L), 291 (R)
219
+ Lip inner : 13 (top), 14 (bottom)
220
+ Lip outer : 0 (top), 17 (bottom)
221
+ L eye lids : 159 (upper), 145 (lower)
222
+ R eye lids : 386 (upper), 374 (lower)
223
+ L eye corners : 33 (outer), 133 (inner)
224
+ R eye corners : 362 (outer), 263 (inner)
225
+ L brow : 70 (inner), 105 (outer), 52 (arch)
226
+ R brow : 300 (inner), 334 (outer), 282 (arch)
227
+ Nose tip : 4
228
+ Nostrils : 129 (L ala), 358 (R ala)
229
+ Cheeks : 50 (L), 280 (R)
230
+ """
231
+ try:
232
+ pts = np.array(face_landmarks, dtype=np.float32)
233
+ face_w = max(float(np.max(pts[:, 0]) - np.min(pts[:, 0])), 1.0)
234
+ face_h = max(float(np.max(pts[:, 1]) - np.min(pts[:, 1])), 1.0)
235
+
236
+ def p(i):
237
+ return pts[i]
238
+
239
+ def nw(a, b):
240
+ return _dist(p(a), p(b)) / face_w
241
+
242
+ def nh(a, b):
243
+ return _dist(p(a), p(b)) / face_h
244
+
245
+ def clamp01(v):
246
+ return max(0.0, min(1.0, float(v)))
247
+
248
+ # ── Mouth ────────────────────────────────────────
249
+ lc, rc = p(61), p(291)
250
+ tl, bl = p(13), p(14)
251
+ ut, lb = p(0), p(17)
252
+
253
+ mid_y = (tl[1] + bl[1]) / 2.0
254
+ mouth_curve = ((lc[1] - mid_y) + (rc[1] - mid_y)) / (2.0 * face_h)
255
+ mouth_open = _dist(ut, lb) / face_h
256
+ inner_gap = _dist(tl, bl) / face_h
257
+ mouth_w = _dist(lc, rc) / face_w
258
+ o_ratio = (mouth_open / (mouth_w + 1e-6)) if mouth_w > 0.05 else 0.0
259
+
260
+ # ── Eyes ─────────────────────────────────────────
261
+ leu, led = p(159), p(145)
262
+ reu, red = p(386), p(374)
263
+ lel, ler = p(33), p(133)
264
+ rel, rer = p(362), p(263)
265
+
266
+ l_eye_h = _dist(leu, led)
267
+ r_eye_h = _dist(reu, red)
268
+ l_eye_w = _dist(lel, ler)
269
+ r_eye_w = _dist(rel, rer)
270
+
271
+ l_ear = l_eye_h / (l_eye_w + 1e-6)
272
+ r_ear = r_eye_h / (r_eye_w + 1e-6)
273
+ avg_ear = (l_ear + r_ear) / 2.0
274
+ eye_open = ((l_eye_h + r_eye_h) / 2.0) / face_h
275
+
276
+ # ── Eyebrows ──────────────────────────────────────
277
+ lib, rib = p(70), p(300)
278
+ lob, rob = p(105), p(334)
279
+ lab, rab = p(52), p(282)
280
+
281
+ l_brow_in_h = (leu[1] - lib[1]) / face_h
282
+ r_brow_in_h = (reu[1] - rib[1]) / face_h
283
+ avg_brow_in_h = (l_brow_in_h + r_brow_in_h) / 2.0
284
+
285
+ l_brow_out_h = (leu[1] - lob[1]) / face_h
286
+ r_brow_out_h = (reu[1] - rob[1]) / face_h
287
+ avg_brow_out_h = (l_brow_out_h + r_brow_out_h) / 2.0
288
+
289
+ brow_inner_gap = nw(70, 300)
290
+ brow_oblique = avg_brow_in_h - avg_brow_out_h
291
+
292
+ l_arch_h = (leu[1] - lab[1]) / face_h
293
+ r_arch_h = (reu[1] - rab[1]) / face_h
294
+ avg_arch_h = (l_arch_h + r_arch_h) / 2.0
295
+
296
+ # ── Nose ─────────────────────────────────────────
297
+ nostril_w = nw(129, 358)
298
+ nose_flare = max(0.0, nostril_w - 0.26)
299
+
300
+ # ── Cheeks ───────────────────────────────────────
301
+ lch, rch, nb = p(50), p(280), p(4)
302
+ cheek_raise = ((nb[1] - lch[1]) + (nb[1] - rch[1])) / (2.0 * face_h)
303
+
304
+ # ── HAPPY ────────────────────────────────────────
305
+ happy = 0.0
306
+ if mouth_curve < -0.003:
307
+ happy += clamp01(abs(mouth_curve) / 0.018) * 0.55
308
+ if mouth_w > 0.36:
309
+ happy += clamp01((mouth_w - 0.36) / 0.07) * 0.28
310
+ if cheek_raise > 0.007:
311
+ happy += clamp01(cheek_raise / 0.018) * 0.26
312
+ if avg_ear < 0.27 and eye_open > 0.04:
313
+ happy += 0.12
314
+ if mouth_curve > 0.004: happy *= 0.15
315
+ if avg_brow_in_h < -0.05: happy *= 0.40
316
+ if mouth_open > 0.08: happy *= 0.55
317
+ happy = clamp01(happy * 1.30)
318
+
319
+ # ── SAD β€” v3 ──────────────────────────────────────
320
+ sad = 0.0
321
+ if brow_oblique > 0.02:
322
+ sad += clamp01((brow_oblique - 0.02) / 0.055) * 0.45
323
+ if 0.08 < avg_ear < 0.23:
324
+ peak_dist = abs(avg_ear - 0.16)
325
+ sad += clamp01(1.0 - peak_dist / 0.08) * 0.30
326
+ if 0.02 < eye_open < 0.065:
327
+ sad += clamp01((0.065 - eye_open) / 0.025) * 0.18
328
+ if mouth_curve > 0.002:
329
+ sad += clamp01(mouth_curve / 0.020) * 0.40
330
+ if mouth_w < 0.36:
331
+ sad += clamp01((0.36 - mouth_w) / 0.055) * 0.18
332
+ if avg_brow_in_h > 0.04:
333
+ sad += clamp01((avg_brow_in_h - 0.04) / 0.040) * 0.20
334
+ if mouth_curve < -0.004: sad *= 0.08
335
+ if cheek_raise > 0.014: sad *= 0.20
336
+ if avg_ear > 0.30: sad *= 0.25
337
+ if avg_brow_in_h < -0.06 and brow_inner_gap < 0.14: sad *= 0.15
338
+ sad = clamp01(sad * 1.50)
339
+
340
+ # ── ANGRY β€” v3 ────────────────────────────────────
341
+ angry = 0.0
342
+ if nose_flare > 0.005:
343
+ angry += clamp01(nose_flare / 0.055) * 0.35
344
+ if avg_brow_in_h < -0.04:
345
+ angry += clamp01(abs(avg_brow_in_h + 0.04) / 0.045) * 0.40
346
+ if brow_inner_gap < 0.155:
347
+ angry += clamp01((0.155 - brow_inner_gap) / 0.045) * 0.35
348
+ if avg_brow_out_h < -0.03:
349
+ angry += clamp01(abs(avg_brow_out_h + 0.03) / 0.040) * 0.20
350
+ if avg_ear < 0.20 and eye_open < 0.055:
351
+ angry += clamp01((0.055 - eye_open) / 0.025) * 0.25
352
+ if inner_gap < 0.030:
353
+ angry += clamp01((0.030 - inner_gap) / 0.025) * 0.22
354
+ if brow_oblique < 0.015:
355
+ angry += 0.08
356
+ if mouth_curve < -0.005: angry *= 0.15
357
+ if cheek_raise > 0.014: angry *= 0.20
358
+ if avg_brow_in_h > -0.01: angry *= 0.20
359
+ if mouth_open > 0.07: angry *= 0.40
360
+ if brow_oblique > 0.04: angry *= 0.35
361
+ angry = clamp01(angry * 1.45)
362
+
363
+ # ── SURPRISE β€” v3 ─────────────────────────────────
364
+ surprise = 0.0
365
+ if 0.40 < o_ratio < 1.60:
366
+ peak_dist = abs(o_ratio - 0.90)
367
+ surprise += clamp01(1.0 - peak_dist / 0.55) * 0.50
368
+ if 0.03 < mouth_open < 0.11:
369
+ surprise += clamp01((mouth_open - 0.03) / 0.06) * 0.25
370
+ if mouth_w > 0.40:
371
+ surprise *= max(0.10, 1.0 - (mouth_w - 0.40) / 0.10)
372
+ if avg_ear > 0.27:
373
+ surprise += clamp01((avg_ear - 0.27) / 0.10) * 0.40
374
+ if eye_open > 0.07:
375
+ surprise += clamp01((eye_open - 0.07) / 0.055) * 0.28
376
+ if avg_arch_h > 0.05:
377
+ surprise += clamp01((avg_arch_h - 0.05) / 0.055) * 0.35
378
+ if avg_brow_in_h > 0.04 and avg_brow_out_h > 0.02:
379
+ surprise += 0.15
380
+ if abs(brow_oblique) < 0.025:
381
+ surprise += 0.08
382
+ if mouth_curve < -0.005: surprise *= 0.30
383
+ if avg_brow_in_h < -0.03: surprise *= 0.20
384
+ if avg_ear < 0.18: surprise *= 0.25
385
+ if brow_inner_gap < 0.13: surprise *= 0.25
386
+ if mouth_w > 0.44: surprise *= 0.25
387
+ if mouth_open > 0.12 and o_ratio < 0.40:
388
+ surprise *= 0.30
389
+ surprise = clamp01(surprise * 1.35)
390
+
391
+ # ── NEUTRAL ───────────────────────────────────────
392
+ neutral = max(0.0, 1.0 - (happy + sad + angry + surprise) * 1.05)
393
+
394
+ total = happy + sad + angry + surprise + neutral
395
+ if total > 1e-6:
396
+ happy /= total
397
+ sad /= total
398
+ angry /= total
399
+ surprise /= total
400
+ neutral /= total
401
+
402
+ return {k: round(float(v), 4)
403
+ for k, v in zip(EMOTIONS, [happy, sad, angry, surprise, neutral])}
404
+
405
+ except Exception as ex:
406
+ print(f"⚠️ Emotion detection error: {ex}")
407
+ return {e: (1.0 if e == "neutral" else 0.0) for e in EMOTIONS}
408
+
409
+
410
+ # ─────────────────────────────────────────────────────────────
411
+ # DISPLAY HELPERS
412
+ # ─────────────────────────────────────────────────────────────
413
+ def get_word_suggestions(partial, word_freq):
414
+ if not partial:
415
+ return []
416
+ p_up = partial.upper()
417
+ seen = {}
418
+ out = []
419
+ for word, freq in word_freq.most_common(20):
420
+ if word.startswith(p_up) and word != p_up:
421
+ out.append((word, freq)); seen[word] = True
422
+ for word in COMMON_WORDS:
423
+ if word.startswith(p_up) and word not in seen:
424
+ out.append((word, 0))
425
+ out.sort(key=lambda x: (-x[1], x[0]))
426
+ return [m[0] for m in out[:3]]
427
+
428
+
429
+ def draw_hand_landmarks(frame, hand_landmarks):
430
+ mp.solutions.drawing_utils.draw_landmarks(
431
+ frame, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS,
432
+ mp.solutions.drawing_utils.DrawingSpec(
433
+ color=(0, 255, 0), thickness=2, circle_radius=3),
434
+ mp.solutions.drawing_utils.DrawingSpec(
435
+ color=(0, 255, 255), thickness=2),
436
+ )
437
+
438
+
439
+ def draw_letter_overlay(frame, letter, confidence):
440
+ if not SHOW_LETTER_OVERLAY or not letter or confidence <= 0:
441
+ return frame
442
+ x, y = OVERLAY_POSITION
443
+ overlay = frame.copy()
444
+ cv2.rectangle(overlay, (x - 10, y - 40), (x + 220, y + 20), (0, 0, 0), -1)
445
+ frame = cv2.addWeighted(overlay, 0.5, frame, 0.5, 0)
446
+ cv2.putText(frame, letter, (x, y),
447
+ cv2.FONT_HERSHEY_SIMPLEX, OVERLAY_FONT_SCALE,
448
+ OVERLAY_COLOR_LETTER, OVERLAY_THICKNESS, cv2.LINE_AA)
449
+ tw = cv2.getTextSize(letter, cv2.FONT_HERSHEY_SIMPLEX,
450
+ OVERLAY_FONT_SCALE, OVERLAY_THICKNESS)[0][0]
451
+ cv2.putText(frame, f"({confidence:.2f})", (x + tw + 10, y),
452
+ cv2.FONT_HERSHEY_SIMPLEX, OVERLAY_FONT_SCALE * 0.7,
453
+ OVERLAY_COLOR_CONF, OVERLAY_THICKNESS - 1, cv2.LINE_AA)
454
+ return frame
455
+
456
+
457
+ # ─────────────────────────────────────────────────────────────
458
+ # CORE PROCESSING ENGINE
459
+ # ──────────────────────────���──────────────────────────────────
460
+ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, frame_seq=None):
461
+ print("=" * 70)
462
+ print("πŸš€ ISL Detection System Starting… (emotion engine v3)")
463
+ print(f" RUN_MODE = {RUN_MODE}")
464
+ print(f" CONF_THRESHOLD = {CONF_THRESHOLD}")
465
+ print(f" LETTER_HOLD_SEC = {LETTER_HOLD_SEC}s")
466
+ print(f" SharedMemory = {'enabled' if shm_name else 'disabled (base64 fallback)'}")
467
+ print("=" * 70)
468
+
469
+ # ── SharedMemory init ──────────────────────────────────────────────────
470
+ shm = None
471
+ shm_array = None
472
+
473
+ if shm_name is not None:
474
+ try:
475
+ shm = shared_memory.SharedMemory(name=shm_name)
476
+ shm_array = np.ndarray((480, 640, 3), dtype=np.uint8, buffer=shm.buf)
477
+ print("βœ… SharedMemory connected (Zero-copy mode)")
478
+ except Exception as e:
479
+ print(f"❌ SharedMemory init failed: {e}")
480
+ return
481
+
482
+ # ── Load ISL model ─────────────────────────────────────────────────────
483
+ try:
484
+ model = load_model(ISL_MODEL_PATH)
485
+ print("βœ… ISL model loaded")
486
+ print(f" Input shape : {model.input_shape}")
487
+ print(f" Output shape : {model.output_shape}")
488
+ except Exception as e:
489
+ print(f"❌ Failed to load model: {e}")
490
+ if shm:
491
+ shm.close()
492
+ return
493
+
494
+ try:
495
+ model.predict(np.zeros((1, 42), np.float32), verbose=0)
496
+ print("βœ… Model warmed up")
497
+ except Exception as e:
498
+ print(f"⚠️ Warmup: {e}")
499
+
500
+ alphabet = list(string.ascii_uppercase) + [str(i) for i in range(1, 10)]
501
+ print(f" Alphabet : {len(alphabet)} classes (model outputs {model.output_shape[1]})")
502
+
503
+ # ── Detection state ────────────────────────────────────────────────────
504
+ current_word = []
505
+ sentence = []
506
+ no_hand_frames = 0
507
+
508
+ candidate_letter = ""
509
+ candidate_since = 0.0
510
+ last_accepted_letter = ""
511
+ last_accepted_time = 0.0
512
+
513
+ letter_confidence_buffer = deque(maxlen=4)
514
+ word_frequency = Counter()
515
+ current_detected_letter = ""
516
+ current_confidence = 0.0
517
+
518
+ emotion_history = deque(maxlen=SMOOTHING_FRAMES)
519
+ current_emotion = "neutral"
520
+ emotion_scores = {e: (1.0 if e == "neutral" else 0.0) for e in EMOTIONS}
521
+ emotion_timeline = deque(maxlen=100)
522
+ emotion_frame_counter = 0
523
+
524
+ frame_count = 0
525
+ fps_frame_times = deque(maxlen=30)
526
+
527
+ stats = {
528
+ "letters_detected": 0,
529
+ "words_formed": 0,
530
+ "sentences_formed": 0,
531
+ "emotion_changes": 0,
532
+ "session_start": time.time(),
533
+ }
534
+
535
+ cmd_queue = shared_state.command_queue
536
+ ui_queue = shared_state.ui_queue
537
+
538
+ # ── Camera init ────────────────────────────────────────────────────────
539
+ print("πŸŽ₯ Initializing camera…")
540
+ if RUN_MODE == "LOCAL":
541
+ # FIX 2: cv2.CAP_DSHOW is Windows-only β€” use platform check
542
+ if platform.system() == "Windows":
543
+ cap = cv2.VideoCapture(CAM_INDEX, cv2.CAP_DSHOW)
544
+ else:
545
+ cap = cv2.VideoCapture(CAM_INDEX) # Linux/Mac β€” no CAP_DSHOW
546
+
547
+ if not cap.isOpened():
548
+ cap = cv2.VideoCapture(CAM_INDEX) # fallback without backend flag
549
+ if not cap.isOpened():
550
+ print("❌ Camera not found β€” is another app using it?")
551
+ if shm:
552
+ shm.close()
553
+ return
554
+
555
+ # FIX 4: Set MJPG codec BEFORE resolution β€” forces hardware MJPEG path
556
+ # on built-in laptop webcams, avoiding the raw uncompressed buffer that
557
+ # causes the horizontal static/noise stripes seen in the MJPEG stream.
558
+ # Must be set before width/height or it has no effect on some drivers.
559
+ if platform.system() == "Windows":
560
+ cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'))
561
+
562
+ cap.set(cv2.CAP_PROP_FRAME_WIDTH, CAMERA_RESOLUTION[0])
563
+ cap.set(cv2.CAP_PROP_FRAME_HEIGHT, CAMERA_RESOLUTION[1])
564
+ cap.set(cv2.CAP_PROP_FPS, TARGET_FPS)
565
+
566
+ # FIX 5: Buffer size must be 1 to prevent stale frame accumulation.
567
+ # Set AFTER resolution so the driver doesn't reset it.
568
+ cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
569
+
570
+ # FIX 4 continued: Flush 30 frames (was 10) to drain the DirectShow
571
+ # ring buffer. Built-in webcams pre-fill ~20 frames before stabilizing.
572
+ print("οΏ½οΏ½οΏ½οΏ½ Flushing camera buffer (30 frames)…")
573
+ for _ in range(30):
574
+ cap.read()
575
+
576
+ # Verify the camera is actually producing valid frames
577
+ ret, test_frame = cap.read()
578
+ if not ret or test_frame is None:
579
+ print("❌ Camera flush failed β€” no valid frame received")
580
+ cap.release()
581
+ if shm:
582
+ shm.close()
583
+ return
584
+
585
+ actual_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
586
+ actual_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
587
+ print(f"βœ… Camera: {actual_w}x{actual_h}")
588
+ print("βœ… Camera ready!")
589
+ else:
590
+ cap = None
591
+ print("☁️ CLOUD mode β€” waiting for browser frames via socket")
592
+
593
+ print("=" * 70)
594
+
595
+ mp_hands = mp.solutions.hands
596
+ mp_face = mp.solutions.face_mesh
597
+
598
+ with mp_hands.Hands(
599
+ static_image_mode=False,
600
+ model_complexity=0,
601
+ max_num_hands=1,
602
+ min_detection_confidence=0.6,
603
+ min_tracking_confidence=0.6,
604
+ ) as hands, \
605
+ mp_face.FaceMesh(
606
+ static_image_mode=False,
607
+ max_num_faces=1,
608
+ refine_landmarks=True,
609
+ min_detection_confidence=0.6,
610
+ min_tracking_confidence=0.6,
611
+ ) as face_mesh:
612
+
613
+ print("βœ… MediaPipe initialized β€” entering main loop")
614
+
615
+ while True:
616
+ t0 = time.time()
617
+ now = t0
618
+
619
+ # ── Commands ───────────────────────────────────────────────────
620
+ try:
621
+ while not cmd_queue.empty():
622
+ cmd = cmd_queue.get_nowait()
623
+ action = cmd.get("action", "")
624
+
625
+ if action == "reset":
626
+ current_word = []
627
+ sentence = []
628
+ no_hand_frames = 0
629
+ candidate_letter = ""
630
+ candidate_since = 0.0
631
+ last_accepted_letter = ""
632
+ last_accepted_time = 0.0
633
+ letter_confidence_buffer.clear()
634
+ current_detected_letter = ""
635
+ current_confidence = 0.0
636
+ print("πŸ”„ Reset")
637
+
638
+ elif action == "backspace":
639
+ if current_word:
640
+ current_word.pop()
641
+ elif sentence:
642
+ sentence.pop()
643
+
644
+ elif action == "accept_suggestion":
645
+ w = cmd.get("word", "")
646
+ if w:
647
+ current_word = list(w)
648
+
649
+ elif action == "speak":
650
+ txt = (cmd.get("text", "")
651
+ or (" ".join(sentence) if sentence
652
+ else "".join(current_word)))
653
+ if txt:
654
+ speak_text(txt) # silently skipped in CLOUD mode
655
+
656
+ elif action == "stop":
657
+ if cap:
658
+ cap.release()
659
+ if shm:
660
+ shm.close()
661
+ return
662
+ except Exception:
663
+ pass
664
+
665
+ # ── Capture ────────────────────────────────────────────────────
666
+ if RUN_MODE == "LOCAL":
667
+ ret, raw = cap.read()
668
+ if not ret or raw is None:
669
+ print("⚠️ Camera read failed β€” retrying")
670
+ time.sleep(0.02)
671
+ continue
672
+ else:
673
+ fd = getattr(shared_state, "_cloud_frame", None)
674
+ if fd is None:
675
+ time.sleep(0.03)
676
+ continue
677
+ shared_state._cloud_frame = None
678
+ try:
679
+ raw = cv2.imdecode(
680
+ np.frombuffer(base64.b64decode(fd), np.uint8),
681
+ cv2.IMREAD_COLOR)
682
+ if raw is None:
683
+ continue
684
+ except Exception:
685
+ continue
686
+
687
+ frame = cv2.resize(raw, (FRAME_W, FRAME_H))
688
+ frame = cv2.flip(frame, 1)
689
+ frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
690
+ frame_rgb.flags.writeable = False
691
+
692
+ # ── Hand detection ─────────────────────────────────────────────
693
+ hand_results = hands.process(frame_rgb)
694
+ frame_rgb.flags.writeable = True
695
+
696
+ detected_letter = ""
697
+ hand_detected = False
698
+
699
+ if hand_results and hand_results.multi_hand_landmarks:
700
+ hand_detected = True
701
+ no_hand_frames = 0
702
+
703
+ for hlm in hand_results.multi_hand_landmarks:
704
+ draw_hand_landmarks(frame, hlm)
705
+ try:
706
+ lm_list = calc_landmark_list(frame, hlm)
707
+ pre_list = pre_process_landmark(lm_list)
708
+
709
+ if model is not None and len(pre_list) == 42:
710
+ features = np.array(pre_list, dtype=np.float32).reshape(1, 42)
711
+ pred = model.predict(features, verbose=0)
712
+ prob = float(np.max(pred))
713
+ raw_letter = alphabet[int(np.argmax(pred))]
714
+
715
+ current_detected_letter = raw_letter
716
+ current_confidence = prob
717
+
718
+ print(f"DEBUG β†’ {raw_letter} {prob:.4f}")
719
+
720
+ if prob >= CONF_THRESHOLD:
721
+ if raw_letter == candidate_letter:
722
+ letter_confidence_buffer.append(prob)
723
+ else:
724
+ candidate_letter = raw_letter
725
+ candidate_since = now
726
+ letter_confidence_buffer.clear()
727
+ letter_confidence_buffer.append(prob)
728
+
729
+ avg_conf = float(np.mean(letter_confidence_buffer))
730
+ hold_time = now - candidate_since
731
+
732
+ if hold_time >= LETTER_HOLD_SEC and avg_conf >= CONF_THRESHOLD:
733
+ same_letter_ok = (
734
+ raw_letter != last_accepted_letter or
735
+ (now - last_accepted_time) >= SAME_LETTER_COOLDOWN_SEC
736
+ )
737
+ if same_letter_ok:
738
+ detected_letter = raw_letter
739
+ last_accepted_letter = raw_letter
740
+ last_accepted_time = now
741
+ current_word.append(raw_letter)
742
+ stats["letters_detected"] += 1
743
+ print(f"✍️ Letter: {raw_letter} "
744
+ f"(held {hold_time:.2f}s, conf {avg_conf:.3f})")
745
+ candidate_letter = ""
746
+ candidate_since = 0.0
747
+ letter_confidence_buffer.clear()
748
+ else:
749
+ candidate_letter = ""
750
+ candidate_since = 0.0
751
+ letter_confidence_buffer.clear()
752
+
753
+ except Exception as e:
754
+ print(f"⚠️ Prediction error: {e}")
755
+
756
+ else:
757
+ no_hand_frames += 1
758
+ letter_confidence_buffer.clear()
759
+ candidate_letter = ""
760
+ candidate_since = 0.0
761
+ current_detected_letter = ""
762
+ current_confidence = 0.0
763
+
764
+ if no_hand_frames >= SPACE_THRESHOLD and current_word:
765
+ word = "".join(current_word)
766
+ sentence.append(word)
767
+ word_frequency[word] += 1
768
+ stats["words_formed"] += 1
769
+ print(f"πŸ“ Word committed: {word}")
770
+ if ENABLE_AUTO_TTS:
771
+ speak_text(word) # silently skipped in CLOUD mode
772
+ current_word = []
773
+ last_accepted_letter = ""
774
+ last_accepted_time = 0.0
775
+ no_hand_frames = 0
776
+
777
+ frame = draw_letter_overlay(frame, current_detected_letter, current_confidence)
778
+
779
+ # ── Emotion (every N frames) ───────────────────────────────────
780
+ emotion_frame_counter += 1
781
+ if emotion_frame_counter % EMOTION_PROCESS_INTERVAL == 0:
782
+ ns = {e: (1.0 if e == "neutral" else 0.0) for e in EMOTIONS}
783
+ try:
784
+ cr = cv2.cvtColor(apply_clahe(frame), cv2.COLOR_BGR2RGB)
785
+ cr.flags.writeable = False
786
+ mr = face_mesh.process(cr)
787
+ cr.flags.writeable = True
788
+ if mr and mr.multi_face_landmarks:
789
+ ns = detect_emotion_from_landmarks(
790
+ calc_landmark_list(frame, mr.multi_face_landmarks[0]),
791
+ frame.shape)
792
+ except Exception:
793
+ pass
794
+
795
+ emotion_history.append(ns)
796
+ avg_e = defaultdict(float)
797
+ for d in emotion_history:
798
+ for k, v in d.items():
799
+ avg_e[k] += v
800
+ for k in avg_e:
801
+ avg_e[k] /= len(emotion_history)
802
+
803
+ dominant = (max(avg_e, key=avg_e.get)
804
+ if avg_e and max(avg_e.values()) >= MIN_CONF_TO_SHOW
805
+ else "neutral")
806
+
807
+ if current_emotion != dominant:
808
+ stats["emotion_changes"] += 1
809
+ scores_str = " | ".join(
810
+ [f"{k.upper()}:{avg_e[k]:.3f}" for k in EMOTIONS]
811
+ )
812
+ print(f"😊 Emotion: {current_emotion.upper()} β†’ {dominant.upper()}")
813
+ print(f" Scores : {scores_str}")
814
+
815
+ current_emotion = dominant
816
+ emotion_scores = dict(avg_e)
817
+ emotion_timeline.append({
818
+ "time": time.time(),
819
+ "emotion": dominant,
820
+ "scores": dict(avg_e),
821
+ })
822
+
823
+ # ── FPS ────────────────────────────────────────────────────────
824
+ frame_count += 1
825
+ fps_frame_times.append(time.time())
826
+ if len(fps_frame_times) > 1:
827
+ td = fps_frame_times[-1] - fps_frame_times[0]
828
+ if td > 0:
829
+ shared_state.processing_fps.value = (len(fps_frame_times) - 1) / td
830
+
831
+ # ── Frame output ───────────────────────────────────────────────
832
+ if shm_array is not None:
833
+ shm_array[:] = frame
834
+ if frame_lock_val is not None:
835
+ frame_lock_val.value = 1
836
+ if frame_seq is not None:
837
+ frame_seq.value += 1
838
+ frame_base64 = None
839
+ else:
840
+ _, buffer_img = cv2.imencode(
841
+ ".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 65])
842
+ frame_base64 = base64.b64encode(buffer_img.tobytes()).decode("utf-8")
843
+
844
+ # ── State packet ───────────────────────────────────────────────
845
+ state_packet = {
846
+ "frame": frame_base64 if frame_base64 else "",
847
+ "current_word": "".join(current_word),
848
+ "sentence": " ".join(sentence),
849
+ "suggestions": get_word_suggestions(
850
+ "".join(current_word), word_frequency),
851
+ "emotion": current_emotion,
852
+ "emotion_scores": emotion_scores,
853
+ "emotion_timeline": list(emotion_timeline)[-20:],
854
+ "stats": stats.copy(),
855
+ "detected_letter": detected_letter,
856
+ "hand_detected": hand_detected,
857
+ "fps": shared_state.processing_fps.value,
858
+ "timestamp": time.time(),
859
+ "speaking": speaking_word,
860
+ "overlay_letter": current_detected_letter,
861
+ "overlay_confidence": current_confidence,
862
+ "hold_progress": min(1.0,
863
+ (now - candidate_since) / LETTER_HOLD_SEC)
864
+ if candidate_letter else 0.0,
865
+ }
866
+
867
+ try:
868
+ ui_queue.put_nowait(state_packet)
869
+ except Exception:
870
+ try:
871
+ ui_queue.get_nowait()
872
+ ui_queue.put_nowait(state_packet)
873
+ except Exception:
874
+ pass
875
+
876
+ if frame_count == 1:
877
+ print("πŸ“€ First frame sent β€” dashboard can read now")
878
+ if shm_array is not None:
879
+ print("πŸ“€ SharedMemory active β€” zero-copy mode")
880
+
881
+ # ── Frame-rate limiter ─────────────────────────────────────────
882
+ wait = (1.0 / TARGET_FPS) - (time.time() - t0)
883
+ if wait > 0:
884
+ time.sleep(wait)
885
+
886
+ if cap:
887
+ cap.release()
888
+ if shm is not None:
889
+ shm.close()
890
+
891
+ print("βœ… System stopped")
892
+
893
+
894
+ # ─────────────────────────────────────────────────────────────
895
+ # STANDALONE ENTRY POINT
896
+ # ─────────────────────────────────────────────────────────────
897
+ if __name__ == "__main__":
898
+ FRAME_BYTES = FRAME_W * FRAME_H * 3
899
+
900
+ # FIX 3: Clean up leftover shared memory from a previous crash
901
+ # Without this, restarting after a crash raises FileExistsError
902
+ try:
903
+ _old_shm = shared_memory.SharedMemory(name="isl_frame_shm")
904
+ _old_shm.close()
905
+ _old_shm.unlink()
906
+ print("🧹 Cleaned up leftover shared memory from previous run")
907
+ except FileNotFoundError:
908
+ pass # normal β€” no leftover
909
+
910
+ shm = shared_memory.SharedMemory(create=True, size=FRAME_BYTES, name="isl_frame_shm")
911
+ seq = Value(ctypes.c_uint64, 0)
912
+ lock = Value(ctypes.c_bool, False)
913
+ ss = SharedState()
914
+ try:
915
+ core_processing_engine(ss, shm_name=shm.name, frame_lock_val=lock, frame_seq=seq)
916
+ finally:
917
+ shm.close()
918
+ shm.unlink()
isl_ui_dashboard.py ADDED
@@ -0,0 +1,941 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # isl_ui_dashboard.py β€” CLOUD PRODUCTION VERSION
2
+ #
3
+ # ════════════════════════════════════════════════════════════════════════════
4
+ # CLOUD FIXES APPLIED (4 changes only β€” everything else identical):
5
+ #
6
+ # FIX 1 β€” Browser camera: HTML now captures + emits browser_frame via socket
7
+ # FIX 2 β€” Nav links: localhost:5000/5001 β†’ dynamic window.location URLs
8
+ # FIX 3 β€” _cloud_frame scope: shared_state stored as module-level variable
9
+ # FIX 4 β€” /avatar route added
10
+ #
11
+ # LOCAL CAMERA FIX:
12
+ # FIX 5 β€” generate_frames() now waits for frame_seq > 0 before streaming.
13
+ # Without this, the MJPEG client receives raw uninitialized
14
+ # SharedMemory bytes (all zeros or garbage) before the detection
15
+ # engine writes the first valid frame β€” causing static/noise on
16
+ # the built-in laptop webcam display.
17
+ # Also removed unreachable `return` after infinite fallback loop.
18
+ #
19
+ # TRANSPORT FIX:
20
+ # FIX 6 β€” transports updated to ['websocket', 'polling'] with upgrade:true
21
+ # in the embedded HTML Socket.IO client. Polling-only breaks camera
22
+ # streaming on HuggingFace Spaces (HTTPS) and other cloud hosts.
23
+ # WebSocket is tried first; polling is the automatic fallback.
24
+ #
25
+ # ALL original features preserved:
26
+ # βœ… MJPEG /video_feed from SharedMemory (zero-copy)
27
+ # βœ… Socket.IO state_update events (emotion, word, suggestions, stats)
28
+ # βœ… Speak, Backspace, Reset commands
29
+ # βœ… Emotion bars, Chart.js timeline, word suggestions
30
+ # βœ… async_mode='threading', transports=['websocket','polling']
31
+ # ════════════════════════════════════════════════════════════════════════════
32
+
33
+ import time
34
+ import threading
35
+ import numpy as np
36
+ import cv2
37
+ import multiprocessing.shared_memory
38
+
39
+ from queue import Queue, Empty
40
+ from flask import Flask, render_template_string, Response, jsonify, request
41
+ from flask_socketio import SocketIO
42
+
43
+ # ── App & SocketIO ────────────────────────────────────────────────────────────
44
+ app = Flask(__name__)
45
+ app.config["SECRET_KEY"] = "isl_secret_2024"
46
+
47
+ socketio = SocketIO(
48
+ app,
49
+ async_mode="threading",
50
+ cors_allowed_origins="*",
51
+ ping_timeout=60,
52
+ ping_interval=25,
53
+ logger=False,
54
+ engineio_logger=False,
55
+ )
56
+
57
+ # ── Module-level state ────────────────────────────────────────────────────────
58
+ _local_state_q: Queue = Queue(100)
59
+ FRAME_W, FRAME_H = 640, 480
60
+ _shm_name = None
61
+ _frame_seq = None
62
+
63
+ # FIX 3: shared_state stored at module level so on_browser_frame() can
64
+ # set _cloud_frame without being inside dashboard_process() scope.
65
+ _shared_state = None
66
+
67
+ # ── HTML ──────────────────────────────────────────────────────────────────────
68
+ HTML = r"""
69
+ <!DOCTYPE html>
70
+ <html lang="en">
71
+ <head>
72
+ <meta charset="UTF-8"/>
73
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover"/>
74
+ <title>ISL Gesture Recognition β€” AI Sign Language</title>
75
+ <link rel="preconnect" href="https://fonts.googleapis.com"/>
76
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin/>
77
+ <link href="https://fonts.googleapis.com/css2?family=Playfair+Display:wght@400;600;700&family=DM+Sans:wght@300;400;500;600&family=DM+Mono:wght@400;500&display=swap" rel="stylesheet"/>
78
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.7.2/socket.io.min.js"></script>
79
+ <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
80
+ <style>
81
+ :root {
82
+ --bg: #0d1117;
83
+ --surf: #131922;
84
+ --card: #1a2233;
85
+ --bdr: rgba(212,168,83,.13);
86
+ --bdrm: rgba(212,168,83,.30);
87
+ --bdrh: rgba(212,168,83,.58);
88
+ --gold: #d4a853;
89
+ --goldb: #f0c76a;
90
+ --goldd: #a07c3a;
91
+ --sage: #7eb89a;
92
+ --saged: #4e7d66;
93
+ --txt: #f0ede6;
94
+ --txtm: #a8a096;
95
+ --txtl: #5a5650;
96
+ --red: #c0614a;
97
+ --r8: 8px;
98
+ --r14: 14px;
99
+ --fD: 'Playfair Display', Georgia, serif;
100
+ --fB: 'DM Sans', system-ui, sans-serif;
101
+ --fM: 'DM Mono', monospace;
102
+ }
103
+ *, *::before, *::after { margin:0; padding:0; box-sizing:border-box; }
104
+ html, body { height:100%; overflow:hidden; }
105
+ body {
106
+ font-family: var(--fB);
107
+ background: var(--bg);
108
+ color: var(--txt);
109
+ -webkit-font-smoothing: antialiased;
110
+ display: flex;
111
+ flex-direction: column;
112
+ }
113
+
114
+ #app { display:flex; flex:1; overflow:hidden; height:100%; }
115
+
116
+ /* ── Camera pane ── */
117
+ #cam-pane {
118
+ flex: 1;
119
+ display: flex;
120
+ flex-direction: column;
121
+ min-width: 0;
122
+ background: var(--bg);
123
+ position: relative;
124
+ overflow: hidden;
125
+ }
126
+ .topbar {
127
+ display: flex;
128
+ align-items: center;
129
+ justify-content: space-between;
130
+ padding: 10px 16px 0;
131
+ flex-shrink: 0;
132
+ position: relative;
133
+ z-index: 5;
134
+ }
135
+ .brand { display:flex; align-items:baseline; gap:8px; }
136
+ .brand-t { font-family:var(--fD); font-size:1.1rem; font-weight:700; color:var(--gold); }
137
+ .brand-s { font-size:.65rem; color:var(--txtl); text-transform:uppercase; letter-spacing:.07em; }
138
+ .live-badge {
139
+ font-size:.65rem; font-family:var(--fM); color:var(--sage);
140
+ background:rgba(126,184,154,.10); border:1px solid rgba(126,184,154,.25);
141
+ padding:3px 9px; border-radius:20px; transition:all .3s;
142
+ }
143
+ .live-badge.rec { color:var(--gold); background:rgba(212,168,83,.1); border-color:var(--bdrm); animation:pd .9s infinite; }
144
+ .live-badge.err { color:var(--red); background:rgba(192,97,74,.1); border-color:rgba(192,97,74,.3); }
145
+ @keyframes pd { 0%,100%{opacity:1} 50%{opacity:.55} }
146
+
147
+ #camera-feed {
148
+ flex: 1; width:100%; display:block; object-fit:cover; background:#000;
149
+ }
150
+ .cam-overlay {
151
+ position:absolute; bottom:0; left:0; right:0;
152
+ padding:12px 18px 14px;
153
+ background: linear-gradient(to top, rgba(13,17,23,1) 55%, transparent);
154
+ display:flex; align-items:flex-end; justify-content:space-between;
155
+ gap:10px; pointer-events:none; z-index:5;
156
+ }
157
+ .co-letter {
158
+ font-family:var(--fD); font-size:clamp(2rem,5vw,3.2rem);
159
+ font-weight:700; color:var(--goldb); line-height:1;
160
+ }
161
+ .co-word {
162
+ font-size:.85rem; font-weight:600; color:var(--gold);
163
+ font-family:var(--fM); letter-spacing:.05em;
164
+ }
165
+ .co-desc { font-size:.72rem; color:var(--txtm); text-align:right; max-width:200px; line-height:1.45; }
166
+ .co-progress { position:absolute; bottom:0; left:0; right:0; height:3px; background:rgba(212,168,83,.08); }
167
+ .co-bar {
168
+ height:100%; width:0%;
169
+ background:linear-gradient(90deg, var(--goldd), var(--goldb));
170
+ transition:width .3s ease; position:relative; overflow:hidden;
171
+ }
172
+ .co-bar::after {
173
+ content:''; position:absolute; inset:0;
174
+ background:linear-gradient(90deg,transparent,rgba(255,255,255,.3),transparent);
175
+ animation:shimmer 1.8s infinite;
176
+ }
177
+ @keyframes shimmer { from{transform:translateX(-100%)} to{transform:translateX(200%)} }
178
+ .cam-placeholder {
179
+ position:absolute; inset:0;
180
+ display:flex; flex-direction:column;
181
+ align-items:center; justify-content:center;
182
+ gap:10px; color:var(--txtl); font-size:.8rem; z-index:2; pointer-events:none;
183
+ }
184
+ .cam-placeholder .ph-icon { font-size:2.8rem; opacity:.25; }
185
+
186
+ /* ── Control pane ── */
187
+ #ctrl-pane {
188
+ width:340px; flex-shrink:0; display:flex; flex-direction:column;
189
+ background:var(--surf); border-left:1px solid var(--bdr);
190
+ overflow-y:auto; overflow-x:hidden;
191
+ }
192
+ #ctrl-pane::-webkit-scrollbar { width:4px; }
193
+ #ctrl-pane::-webkit-scrollbar-thumb { background:var(--bdrm); border-radius:4px; }
194
+
195
+ @media (max-width:720px) {
196
+ html,body { overflow:hidden; }
197
+ #app { flex-direction:column; height:100%; }
198
+ #cam-pane { flex:0 0 46vh; min-height:260px; max-height:46vh; width:100%; }
199
+ #ctrl-pane { width:100%; flex:1; border-left:none; border-top:1px solid var(--bdr); overflow-y:auto; }
200
+ .topbar { padding:8px 12px 0; }
201
+ .brand-t { font-size:.95rem; }
202
+ }
203
+
204
+ .cpanel { padding:14px 16px; display:flex; flex-direction:column; gap:12px; }
205
+
206
+ .sec-label {
207
+ font-size:.65rem; font-weight:600; text-transform:uppercase;
208
+ letter-spacing:.08em; color:var(--txtl);
209
+ display:flex; align-items:center; gap:6px; font-family:var(--fM);
210
+ }
211
+
212
+ /* ── Status pill ── */
213
+ .status {
214
+ display:flex; align-items:center; gap:7px; padding:7px 13px;
215
+ border-radius:30px; font-size:.73rem; font-weight:500;
216
+ border:1px solid var(--bdr); background:var(--card); color:var(--txtm);
217
+ transition:all .3s; min-height:34px; font-family:var(--fM);
218
+ }
219
+ .sdot { width:7px; height:7px; border-radius:50%; background:var(--txtl); flex-shrink:0; transition:all .3s; }
220
+ .status.act .sdot { background:var(--sage); box-shadow:0 0 8px var(--sage); animation:pd 1.4s infinite; }
221
+ .status.act { border-color:rgba(126,184,154,.3); color:var(--sage); }
222
+ .status.err .sdot { background:var(--red); }
223
+ .status.err { border-color:rgba(192,97,74,.3); color:var(--red); }
224
+
225
+ /* ── Text display ── */
226
+ .text-display {
227
+ background:var(--card); border:1px solid var(--bdr);
228
+ border-radius:var(--r14); padding:10px 13px; min-height:52px;
229
+ }
230
+ .cur-word {
231
+ font-size:1.15rem; font-weight:700; color:var(--goldb);
232
+ font-family:var(--fD); line-height:1.2; margin-bottom:4px;
233
+ }
234
+ .cur-sentence { font-size:.75rem; color:var(--txtm); line-height:1.5; font-family:var(--fM); }
235
+
236
+ /* ── Buttons ── */
237
+ .btn-row { display:grid; grid-template-columns:repeat(3,1fr); gap:7px; }
238
+ .btn {
239
+ display:flex; align-items:center; justify-content:center; gap:5px;
240
+ padding:9px 6px; border:1px solid transparent; border-radius:var(--r14);
241
+ font-family:var(--fB); font-size:.76rem; font-weight:600;
242
+ cursor:pointer; transition:all .22s; white-space:nowrap; overflow:hidden;
243
+ }
244
+ .btn:active { transform:scale(.96); }
245
+ .btn:disabled { opacity:.4; cursor:not-allowed; }
246
+ .btn-speak {
247
+ background:linear-gradient(135deg,#4e7d66,#7eb89a);
248
+ color:#0d1117; border-color:rgba(126,184,154,.35);
249
+ }
250
+ .btn-speak:hover { box-shadow:0 4px 16px rgba(126,184,154,.3); }
251
+ .btn-back { background:transparent; color:var(--txtm); border-color:var(--bdr); }
252
+ .btn-back:hover { border-color:var(--bdrm); color:var(--txt); }
253
+ .btn-reset { background:transparent; color:var(--red); border-color:rgba(192,97,74,.3); }
254
+ .btn-reset:hover { background:rgba(192,97,74,.08); border-color:rgba(192,97,74,.55); }
255
+
256
+ .divider { height:1px; background:var(--bdr); margin:0 -16px; }
257
+
258
+ /* ── Emotion ── */
259
+ .emotion-header { display:flex; justify-content:space-between; align-items:center; margin-bottom:8px; }
260
+ .emotion-big { font-family:var(--fD); font-size:1.1rem; font-weight:700; color:var(--goldb); }
261
+ .emotion-icon { font-size:1.5rem; animation:emojiPop 2.2s ease infinite; }
262
+ @keyframes emojiPop { 0%,100%{transform:scale(1)} 50%{transform:scale(1.12)} }
263
+
264
+ .emo-bars { display:flex; flex-direction:column; gap:5px; }
265
+ .emo-bar { display:flex; align-items:center; gap:8px; }
266
+ .ebo-label { width:78px; font-size:.72rem; color:var(--txtm); flex-shrink:0; font-family:var(--fB); }
267
+ .ebo-track {
268
+ flex:1; height:20px; background:rgba(26,34,51,.8);
269
+ border-radius:10px; overflow:hidden; border:1px solid var(--bdr);
270
+ }
271
+ .ebo-fill {
272
+ height:100%; transition:width .5s cubic-bezier(.4,0,.2,1);
273
+ display:flex; align-items:center; justify-content:flex-end;
274
+ padding-right:6px; font-size:.62rem; font-weight:700;
275
+ font-family:var(--fM); color:rgba(0,0,0,.75); min-width:28px;
276
+ }
277
+
278
+ /* ── Suggestions ── */
279
+ .sug-detected {
280
+ font-size:.65rem; font-family:var(--fM); color:var(--gold);
281
+ background:rgba(212,168,83,.07); border:1px solid var(--bdr);
282
+ border-radius:var(--r8); padding:4px 10px; margin-bottom:5px; letter-spacing:.04em;
283
+ }
284
+ .sug-list { display:flex; flex-direction:column; gap:4px; }
285
+ .sug-item {
286
+ display:flex; align-items:center; gap:9px; padding:7px 11px;
287
+ border-radius:var(--r8); background:var(--card); border:1px solid var(--bdr);
288
+ cursor:pointer; transition:all .2s; font-size:.82rem; color:var(--txt);
289
+ }
290
+ .sug-item:hover {
291
+ background:rgba(212,168,83,.08); border-color:var(--bdrm);
292
+ transform:translateX(3px); color:var(--goldb);
293
+ }
294
+ .sug-n {
295
+ font-family:var(--fM); font-size:.65rem; font-weight:700; color:var(--gold);
296
+ background:rgba(212,168,83,.12); border:1px solid rgba(212,168,83,.2);
297
+ width:20px; height:20px; display:flex; align-items:center; justify-content:center;
298
+ border-radius:5px; flex-shrink:0;
299
+ }
300
+ .sug-empty { text-align:center; padding:16px 0; color:var(--txtl); font-size:.75rem; line-height:1.7; }
301
+
302
+ /* ── Chart ── */
303
+ #emotionChart { height:160px !important; }
304
+
305
+ /* ── Stats ── */
306
+ .stat-grid { display:grid; grid-template-columns:repeat(3,1fr); gap:6px; }
307
+ .stat-tile {
308
+ background:var(--card); border:1px solid var(--bdr);
309
+ border-radius:var(--r8); padding:9px 10px; text-align:center; transition:border-color .25s;
310
+ }
311
+ .stat-tile:hover { border-color:var(--bdrm); }
312
+ .sl { font-size:.55rem; text-transform:uppercase; letter-spacing:.07em; color:var(--txtl); font-weight:600; margin-bottom:3px; font-family:var(--fM); }
313
+ .sv { font-family:var(--fD); font-size:1.3rem; font-weight:700; color:var(--gold); line-height:1; }
314
+
315
+ /* ── About ── */
316
+ .about-txt { font-size:.75rem; color:var(--txtm); line-height:1.75; }
317
+ .about-txt p { margin-bottom:7px; }
318
+ .alist { list-style:none; margin-bottom:7px; }
319
+ .alist li { padding-left:13px; position:relative; font-size:.73rem; padding-bottom:2px; }
320
+ .alist li::before { content:'β€Ί'; position:absolute; left:0; color:var(--gold); font-weight:700; }
321
+ .acredit { margin-top:10px; padding-top:10px; border-top:1px solid var(--bdr); }
322
+ .aname { font-family:var(--fD); color:var(--gold); font-weight:600; font-size:.88rem; }
323
+ .arole { font-size:.65rem; color:var(--txtl); margin:2px 0 3px; font-family:var(--fM); }
324
+ .aemail { color:var(--sage); font-size:.66px; font-family:var(--fM); }
325
+
326
+ /* ── Nav ── */
327
+ .nav-row { display:flex; gap:7px; padding:0 16px 20px; }
328
+ .nav-a {
329
+ flex:1; display:flex; align-items:center; justify-content:center; gap:6px;
330
+ padding:12px 8px; border-radius:var(--r14); border:1px solid var(--bdr);
331
+ background:var(--card); color:var(--txtm); text-decoration:none;
332
+ font-size:.76rem; font-weight:600; transition:all .25s;
333
+ }
334
+ .nav-a:hover { border-color:var(--bdrm); color:var(--txt); }
335
+ .nav-a.on { background:rgba(212,168,83,.10); border-color:var(--bdrh); color:var(--gold); }
336
+ </style>
337
+ </head>
338
+ <body>
339
+
340
+ <div id="app">
341
+
342
+ <!-- ── Camera pane ── -->
343
+ <div id="cam-pane">
344
+ <div class="topbar">
345
+ <div class="brand">
346
+ <span class="brand-t">ISL Gesture Recognition</span>
347
+ <span class="brand-s">Gesture Β· Emotion Β· AI</span>
348
+ </div>
349
+ <span class="live-badge" id="live-badge">● Ready</span>
350
+ </div>
351
+
352
+ <!-- MJPEG feed β€” displays processed frame from server SharedMemory -->
353
+ <img id="camera-feed"
354
+ src="/video_feed"
355
+ alt="Camera Feed"
356
+ style="display:none;flex:1;width:100%;object-fit:cover;"
357
+ onload="onVidLoad()"
358
+ onerror="document.getElementById('cam-ph').style.display='flex'"/>
359
+
360
+ <div class="cam-placeholder" id="cam-ph">
361
+ <span class="ph-icon">πŸ“·</span>
362
+ <span style="font-family:var(--fM);font-size:.75rem">Connecting to camera…</span>
363
+ </div>
364
+
365
+ <div class="cam-overlay">
366
+ <div>
367
+ <div style="display:flex;align-items:baseline;gap:8px">
368
+ <span id="co-letter" class="co-letter">🀟</span>
369
+ <span id="co-word" class="co-word"></span>
370
+ </div>
371
+ </div>
372
+ <div id="co-desc" class="co-desc">Point camera at your hand</div>
373
+ <div class="co-progress"><div class="co-bar" id="co-bar"></div></div>
374
+ </div>
375
+ </div>
376
+
377
+ <!-- ── Control pane ── -->
378
+ <div id="ctrl-pane">
379
+ <div class="cpanel">
380
+
381
+ <!-- Status -->
382
+ <div class="status" id="status">
383
+ <div class="sdot"></div>
384
+ <span id="stxt">Ready for detection</span>
385
+ </div>
386
+
387
+ <!-- Current word -->
388
+ <div class="text-display">
389
+ <div class="cur-word" id="cur-word">β€”</div>
390
+ <div class="cur-sentence" id="cur-sentence"></div>
391
+ </div>
392
+
393
+ <!-- Buttons -->
394
+ <div class="btn-row">
395
+ <button class="btn btn-speak" onclick="sendCmd('speak')">πŸ”Š Speak</button>
396
+ <button class="btn btn-back" onclick="sendCmd('backspace')">⌫ Back</button>
397
+ <button class="btn btn-reset" onclick="sendCmd('reset')">πŸ”„ Reset</button>
398
+ </div>
399
+
400
+ <div class="divider"></div>
401
+
402
+ <!-- Emotion -->
403
+ <div class="sec-label">😊 Emotion Detection</div>
404
+ <div class="emotion-header">
405
+ <span class="emotion-big" id="emo-label">Neutral</span>
406
+ <span class="emotion-icon" id="emo-icon">😐</span>
407
+ </div>
408
+ <div class="emo-bars">
409
+ <div class="emo-bar">
410
+ <div class="ebo-label">😊 Happy</div>
411
+ <div class="ebo-track"><div class="ebo-fill" id="bar-happy" style="width:0%;background:linear-gradient(90deg,#3a8a5c,#7eb89a)">0%</div></div>
412
+ </div>
413
+ <div class="emo-bar">
414
+ <div class="ebo-label">😒 Sad</div>
415
+ <div class="ebo-track"><div class="ebo-fill" id="bar-sad" style="width:0%;background:linear-gradient(90deg,#3a5a8a,#6090c8)">0%</div></div>
416
+ </div>
417
+ <div class="emo-bar">
418
+ <div class="ebo-label">😠 Angry</div>
419
+ <div class="ebo-track"><div class="ebo-fill" id="bar-angry" style="width:0%;background:linear-gradient(90deg,#8a3a3a,#c06060)">0%</div></div>
420
+ </div>
421
+ <div class="emo-bar">
422
+ <div class="ebo-label">😲 Surprise</div>
423
+ <div class="ebo-track"><div class="ebo-fill" id="bar-surprise" style="width:0%;background:linear-gradient(90deg,#8a6a1a,#d4a853)">0%</div></div>
424
+ </div>
425
+ <div class="emo-bar">
426
+ <div class="ebo-label">😐 Neutral</div>
427
+ <div class="ebo-track"><div class="ebo-fill" id="bar-neutral" style="width:100%;background:linear-gradient(90deg,#3a4455,#5a6878)">100%</div></div>
428
+ </div>
429
+ </div>
430
+
431
+ <div class="divider"></div>
432
+
433
+ <!-- Word Suggestions -->
434
+ <div class="sec-label">πŸ’‘ Word Suggestions</div>
435
+ <div id="sug-area">
436
+ <div class="sug-empty">Detected letters will appear<br>here with word suggestions</div>
437
+ </div>
438
+
439
+ <div class="divider"></div>
440
+
441
+ <!-- Emotion Timeline Chart -->
442
+ <div class="sec-label">πŸ“ˆ Emotion Timeline</div>
443
+ <canvas id="emotionChart"></canvas>
444
+
445
+ <div class="divider"></div>
446
+
447
+ <!-- Statistics -->
448
+ <div class="sec-label">πŸ“Š Statistics</div>
449
+ <div class="stat-grid">
450
+ <div class="stat-tile"><div class="sl">FPS</div><div class="sv" id="s-fps">0</div></div>
451
+ <div class="stat-tile"><div class="sl">Letters</div><div class="sv" id="s-letters">0</div></div>
452
+ <div class="stat-tile"><div class="sl">Words</div><div class="sv" id="s-words">0</div></div>
453
+ <div class="stat-tile"><div class="sl">Sentences</div><div class="sv" id="s-sentences">0</div></div>
454
+ <div class="stat-tile"><div class="sl">Emotions</div><div class="sv" id="s-emotions">0</div></div>
455
+ <div class="stat-tile"><div class="sl">Conf.</div><div class="sv" id="s-conf">β€”</div></div>
456
+ </div>
457
+
458
+ <div class="divider"></div>
459
+
460
+ <!-- About -->
461
+ <div class="sec-label">ℹ️ About</div>
462
+ <div class="about-txt">
463
+ <p>AI Powered ISL Detection is an assistive AI system designed to improve communication between hearing and deaf individuals.</p>
464
+ <p>This module focuses on real-time gesture and facial emotion recognition using computer vision.</p>
465
+ <p>Useful for:</p>
466
+ <ul class="alist">
467
+ <li>Deaf individuals</li>
468
+ <li>Inclusive classrooms</li>
469
+ <li>Assistive communication systems</li>
470
+ <li>Accessibility and AI research</li>
471
+ </ul>
472
+ <div class="acredit">
473
+ <div class="aname">Deepak Roshan</div>
474
+ <div class="arole">AI Engineer</div>
475
+ <div class="aemail">deepakroshan380@gmail.com</div>
476
+ </div>
477
+ </div>
478
+
479
+ </div><!-- cpanel -->
480
+
481
+ <!-- FIX 2: Nav links now built dynamically from window.location -->
482
+ <div class="nav-row">
483
+ <a href="/" class="nav-a on" id="nav-gesture">😊 Gesture Recognition</a>
484
+ <a href="#" class="nav-a" id="nav-avatar">🀟 Speech-to-Sign</a>
485
+ </div>
486
+ </div><!-- ctrl-pane -->
487
+
488
+ </div><!-- app -->
489
+
490
+ <script>
491
+ document.addEventListener('DOMContentLoaded', function () {
492
+
493
+ // FIX 2: Build nav URLs dynamically so they work on any host/domain
494
+ (function() {
495
+ const proto = location.protocol;
496
+ const host = location.hostname;
497
+ const port = location.port;
498
+ if (port && port !== '80' && port !== '443') {
499
+ document.getElementById('nav-gesture').href = proto + '//' + host + ':5000/';
500
+ document.getElementById('nav-avatar').href = proto + '//' + host + ':5001/';
501
+ } else {
502
+ document.getElementById('nav-gesture').href = proto + '//' + host + '/';
503
+ document.getElementById('nav-avatar').href = proto + '//' + host + '/avatar/';
504
+ }
505
+ })();
506
+
507
+ const EMO_ICONS = { happy:'😊', sad:'😒', angry:'😠', surprise:'😲', neutral:'😐' };
508
+ const EMO_KEYS = ['happy','sad','angry','surprise','neutral'];
509
+
510
+ /* ── Chart.js Emotion Timeline ─────────────────────────── */
511
+ const chartCtx = document.getElementById('emotionChart').getContext('2d');
512
+ const chart = new Chart(chartCtx, {
513
+ type: 'line',
514
+ data: {
515
+ labels: [],
516
+ datasets: [
517
+ { label:'Happy', data:[], borderColor:'#7eb89a', borderWidth:1.5, tension:.4, fill:false, pointRadius:0 },
518
+ { label:'Sad', data:[], borderColor:'#6090c8', borderWidth:1.5, tension:.4, fill:false, pointRadius:0 },
519
+ { label:'Angry', data:[], borderColor:'#c06060', borderWidth:1.5, tension:.4, fill:false, pointRadius:0 },
520
+ { label:'Surprise', data:[], borderColor:'#d4a853', borderWidth:1.5, tension:.4, fill:false, pointRadius:0 },
521
+ { label:'Neutral', data:[], borderColor:'#5a6878', borderWidth:1.5, tension:.4, fill:false, pointRadius:0 },
522
+ ]
523
+ },
524
+ options: {
525
+ responsive: true,
526
+ maintainAspectRatio: false,
527
+ plugins: {
528
+ legend: { labels: { color:'#5a5650', font:{ size:9, family:"'DM Mono'" }, boxWidth:10 } }
529
+ },
530
+ scales: {
531
+ y: {
532
+ beginAtZero:true, max:1,
533
+ ticks: { color:'#5a5650', font:{ size:8 } },
534
+ grid: { color:'rgba(212,168,83,.05)' }
535
+ },
536
+ x: { ticks: { display:false }, grid: { color:'rgba(212,168,83,.03)' } }
537
+ },
538
+ animation: false
539
+ }
540
+ });
541
+
542
+ /* ── Browser Camera (FIX 1) ─────────────────────────────── */
543
+ let _camStream = null;
544
+ let _camInterval = null;
545
+ const _capCanvas = document.createElement('canvas');
546
+ const _capCtx = _capCanvas.getContext('2d');
547
+
548
+ async function startBrowserCamera() {
549
+ if (_camStream) return;
550
+ if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
551
+ console.warn('Camera API not available β€” HTTP or unsupported browser');
552
+ return;
553
+ }
554
+ try {
555
+ _camStream = await navigator.mediaDevices.getUserMedia({
556
+ video: { width:{ ideal:640 }, height:{ ideal:480 }, facingMode:'user' },
557
+ audio: false
558
+ });
559
+ const video = document.createElement('video');
560
+ video.srcObject = _camStream;
561
+ video.autoplay = true;
562
+ video.playsInline = true;
563
+ video.addEventListener('loadeddata', () => {
564
+ _capCanvas.width = 320;
565
+ _capCanvas.height = 240;
566
+ _camInterval = setInterval(() => {
567
+ if (document.visibilityState === 'visible' && socket && socket.connected) {
568
+ _capCtx.drawImage(video, 0, 0, 320, 240);
569
+ const frame = _capCanvas.toDataURL('image/jpeg', 0.6);
570
+ socket.emit('browser_frame', { frame: frame.split(',')[1] });
571
+ }
572
+ }, 65); // ~15 fps
573
+ });
574
+ } catch (err) {
575
+ console.error('Camera access error:', err);
576
+ }
577
+ }
578
+
579
+ function stopBrowserCamera() {
580
+ if (_camStream) { _camStream.getTracks().forEach(t => t.stop()); _camStream = null; }
581
+ if (_camInterval) { clearInterval(_camInterval); _camInterval = null; }
582
+ }
583
+
584
+ /* ── Socket.IO ───────────────────────────────────────────── */
585
+ // βœ… FIX 6: transports updated to prefer WebSocket with polling fallback.
586
+ // upgrade:true lets Socket.IO automatically upgrade from polling β†’ websocket
587
+ // once the connection is established. This is required for camera streaming
588
+ // to work reliably on HuggingFace Spaces (HTTPS) and other cloud deployments.
589
+ let socket;
590
+ try {
591
+ socket = io({
592
+ transports: ['websocket', 'polling'],
593
+ upgrade: true,
594
+ reconnectionDelay: 1000
595
+ });
596
+
597
+ socket.on('connect', () => {
598
+ setStat('act', 'Connected β€” detecting…');
599
+ document.getElementById('cam-ph').style.display = 'none';
600
+ setBadge('● Live', 'rec');
601
+ startBrowserCamera();
602
+ });
603
+
604
+ socket.on('disconnect', () => {
605
+ setStat('err', 'Disconnected');
606
+ document.getElementById('cam-ph').style.display = 'flex';
607
+ setBadge('● Offline', 'err');
608
+ stopBrowserCamera();
609
+ });
610
+
611
+ socket.on('state_update', updateUI);
612
+
613
+ } catch (e) {
614
+ console.warn('Socket.IO unavailable β€” demo mode');
615
+ }
616
+
617
+ /* ── Video load callback ──────────────────────────────── */
618
+ window.onVidLoad = function () {
619
+ document.getElementById('cam-ph').style.display = 'none';
620
+ document.getElementById('camera-feed').style.display = 'block';
621
+ setBadge('● Live', 'rec');
622
+ };
623
+
624
+ /* ── Main UI update ───────────────────────────────────── */
625
+ function updateUI(d) {
626
+ if (d.overlay_letter) {
627
+ document.getElementById('co-letter').textContent = d.overlay_letter.toUpperCase();
628
+ }
629
+ if (d.current_word !== undefined) {
630
+ const w = d.current_word || 'β€”';
631
+ document.getElementById('co-word').textContent = w;
632
+ document.getElementById('cur-word').textContent = w;
633
+ }
634
+ if (d.sentence !== undefined) {
635
+ document.getElementById('cur-sentence').textContent = d.sentence || '';
636
+ }
637
+ if (d.overlay_confidence !== undefined) {
638
+ const pct = Math.round(d.overlay_confidence * 100);
639
+ document.getElementById('co-desc').textContent = 'Confidence: ' + pct + '%';
640
+ document.getElementById('s-conf').textContent = pct + '%';
641
+ document.getElementById('co-bar').style.width = pct + '%';
642
+ }
643
+ if (d.hand_detected) {
644
+ setStat('act', d.overlay_letter
645
+ ? 'Detecting: ' + d.overlay_letter + ' (' + Math.round((d.overlay_confidence||0)*100) + '%)'
646
+ : 'Hand in frame β€” signing…');
647
+ } else {
648
+ setStat('', d.current_word ? 'Word so far: ' + d.current_word : 'Ready for detection');
649
+ }
650
+ if (d.fps !== undefined) {
651
+ document.getElementById('s-fps').textContent = Math.round(d.fps);
652
+ }
653
+ if (d.stats) {
654
+ document.getElementById('s-letters').textContent = d.stats.letters_detected || 0;
655
+ document.getElementById('s-words').textContent = d.stats.words_formed || 0;
656
+ document.getElementById('s-sentences').textContent = d.stats.sentences_formed || 0;
657
+ document.getElementById('s-emotions').textContent = d.stats.emotion_changes || 0;
658
+ }
659
+ if (Array.isArray(d.suggestions)) {
660
+ updateSuggestions(d.suggestions, d.current_word);
661
+ }
662
+ if (d.emotion_scores) {
663
+ updateEmotion(d.emotion || 'neutral', d.emotion_scores);
664
+ }
665
+ if (d.emotion_timeline && d.emotion_timeline.length > 0) {
666
+ updateChart(d.emotion_timeline);
667
+ }
668
+ }
669
+
670
+ function updateEmotion(emotion, scores) {
671
+ document.getElementById('emo-label').textContent = emotion.charAt(0).toUpperCase() + emotion.slice(1);
672
+ document.getElementById('emo-icon').textContent = EMO_ICONS[emotion] || '😐';
673
+ EMO_KEYS.forEach(k => {
674
+ const bar = document.getElementById('bar-' + k);
675
+ if (!bar) return;
676
+ const pct = Math.round(Math.max(0, Math.min(1, scores[k] || 0)) * 100);
677
+ bar.style.width = pct + '%';
678
+ bar.textContent = pct + '%';
679
+ });
680
+ }
681
+
682
+ function updateSuggestions(suggestions, currentWord) {
683
+ const area = document.getElementById('sug-area');
684
+ if (!suggestions || !suggestions.length) {
685
+ area.innerHTML = '<div class="sug-empty">Detected letters will appear<br>here with word suggestions</div>';
686
+ return;
687
+ }
688
+ const detectedRow = currentWord
689
+ ? '<div class="sug-detected">Detected: ' + currentWord.toUpperCase() + '</div>' : '';
690
+ area.innerHTML = detectedRow + '<div class="sug-list">' +
691
+ suggestions.map((w, i) =>
692
+ '<div class="sug-item" onclick="window.acceptSug(\'' + w + '\')">' +
693
+ '<span class="sug-n">' + (i+1) + '</span>' + w + '</div>'
694
+ ).join('') + '</div>';
695
+ }
696
+
697
+ function updateChart(timeline) {
698
+ if (!timeline || !timeline.length) return;
699
+ const collected = Object.fromEntries(EMO_KEYS.map(k => [k, []]));
700
+ timeline.forEach(tp => {
701
+ const s = tp.scores || {};
702
+ EMO_KEYS.forEach(k => collected[k].push(s[k] || 0));
703
+ });
704
+ chart.data.labels = timeline.map((_, i) => i);
705
+ chart.data.datasets.forEach((ds, i) => ds.data = collected[EMO_KEYS[i]]);
706
+ chart.update('none');
707
+ }
708
+
709
+ function setStat(cls, msg) {
710
+ document.getElementById('stxt').textContent = msg;
711
+ document.getElementById('status').className = 'status' + (cls ? ' ' + cls : '');
712
+ }
713
+ function setBadge(msg, cls) {
714
+ const b = document.getElementById('live-badge');
715
+ b.textContent = msg;
716
+ b.className = 'live-badge' + (cls ? ' ' + cls : '');
717
+ }
718
+
719
+ window.sendCmd = function (action) {
720
+ if (action === 'speak') {
721
+ const sentence = document.getElementById('cur-sentence').textContent.trim();
722
+ const word = document.getElementById('cur-word').textContent.trim();
723
+ const text = sentence || (word !== 'β€”' ? word : '');
724
+ if (text && socket) socket.emit('command', { action:'speak', text });
725
+ } else {
726
+ if (socket) socket.emit('command', { action });
727
+ }
728
+ };
729
+
730
+ window.acceptSug = function (word) {
731
+ if (socket) socket.emit('command', { action:'accept_suggestion', word });
732
+ };
733
+
734
+ document.addEventListener('keydown', e => {
735
+ if (e.ctrlKey && e.key === 's') { e.preventDefault(); window.sendCmd('speak'); }
736
+ if (e.ctrlKey && e.key === 'r') { e.preventDefault(); window.sendCmd('reset'); }
737
+ if (e.ctrlKey && e.key === 'Backspace') { e.preventDefault(); window.sendCmd('backspace'); }
738
+ if (['1','2','3'].includes(e.key)) {
739
+ const items = document.querySelectorAll('.sug-item');
740
+ const item = items[parseInt(e.key) - 1];
741
+ if (item) item.click();
742
+ }
743
+ });
744
+
745
+ });
746
+ </script>
747
+ </body>
748
+ </html>
749
+ """
750
+
751
+
752
+ # ── MJPEG generator reads directly from SharedMemory ──────────────────────────
753
+ def generate_frames():
754
+ # FIX 5 (part A): If SharedMemory is not configured, stream a blank frame
755
+ # continuously. Removed unreachable `return` after the infinite loop β€”
756
+ # the original code had `return` after `while True` which is dead code and
757
+ # caused a StopIteration on some Python/Flask versions.
758
+ if _shm_name is None or _frame_seq is None:
759
+ blank = np.zeros((FRAME_H, FRAME_W, 3), np.uint8)
760
+ _, enc = cv2.imencode(".jpg", blank)
761
+ chunk = enc.tobytes()
762
+ while True:
763
+ yield (b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" + chunk + b"\r\n")
764
+ time.sleep(0.1)
765
+
766
+ try:
767
+ shm = multiprocessing.shared_memory.SharedMemory(name=_shm_name)
768
+ frame_buf = np.ndarray((FRAME_H, FRAME_W, 3), dtype=np.uint8, buffer=shm.buf)
769
+ except Exception as e:
770
+ print(f"⚠️ MJPEG generator: SharedMemory attach failed: {e}")
771
+ return
772
+
773
+ # FIX 5 (part B): Wait for the detection engine to write the first valid
774
+ # frame before we start streaming. Without this wait, the MJPEG client
775
+ # receives raw uninitialized SharedMemory bytes immediately, which renders
776
+ # as horizontal static/noise stripes on the browser camera feed.
777
+ # Timeout after 15 seconds (150 Γ— 0.1s) to avoid hanging indefinitely.
778
+ print("⏳ MJPEG: waiting for first valid frame from detection engine…")
779
+ timeout_count = 0
780
+ while _frame_seq.value == 0 and timeout_count < 150:
781
+ time.sleep(0.1)
782
+ timeout_count += 1
783
+
784
+ if _frame_seq.value == 0:
785
+ print("⚠️ MJPEG: timed out waiting for first frame β€” streaming anyway")
786
+ else:
787
+ print(f"βœ… MJPEG: first frame ready (seq={_frame_seq.value}) β€” streaming started")
788
+
789
+ last_seq = 0
790
+ try:
791
+ while True:
792
+ if _frame_seq.value != last_seq:
793
+ last_seq = _frame_seq.value
794
+ frame = frame_buf.copy()
795
+ _, buffer = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 80])
796
+ frame_bytes = buffer.tobytes()
797
+ yield (b"--frame\r\n"
798
+ b"Content-Type: image/jpeg\r\n\r\n" +
799
+ frame_bytes + b"\r\n")
800
+ else:
801
+ time.sleep(0.005)
802
+ finally:
803
+ shm.close()
804
+
805
+
806
+ # ── Routes ────────────────────────────────────────────────────────────────────
807
+ @app.route("/")
808
+ def index():
809
+ return render_template_string(HTML)
810
+
811
+
812
+ @app.route("/video_feed")
813
+ def video_feed():
814
+ return Response(
815
+ generate_frames(),
816
+ mimetype="multipart/x-mixed-replace; boundary=frame"
817
+ )
818
+
819
+
820
+ @app.route("/cmd", methods=["POST"])
821
+ def cmd_route():
822
+ data = request.get_json(force=True, silent=True) or {}
823
+ _local_state_q.put(data)
824
+ return jsonify({"ok": True})
825
+
826
+
827
+ @app.route("/health")
828
+ def health():
829
+ return jsonify({"ok": True, "ts": time.time()})
830
+
831
+
832
+ # FIX 4: /avatar route β€” redirects to the speech-to-sign service
833
+ @app.route("/avatar")
834
+ @app.route("/avatar/")
835
+ def avatar_redirect():
836
+ from flask import redirect
837
+ proto = request.headers.get('X-Forwarded-Proto', 'http')
838
+ host = request.host.split(':')[0]
839
+ return redirect(proto + '://' + host + '/avatar/', code=302)
840
+
841
+
842
+ # ── Socket.IO events ──────────────────────────────────────────────────────────
843
+ @socketio.on("connect")
844
+ def on_connect():
845
+ print("πŸ”Œ Browser connected")
846
+
847
+
848
+ @socketio.on("disconnect")
849
+ def on_disconnect():
850
+ print("πŸ”Œ Browser disconnected")
851
+
852
+
853
+ @socketio.on("command")
854
+ def on_cmd(data):
855
+ if data:
856
+ _local_state_q.put(data)
857
+
858
+
859
+ # FIX 1 + FIX 3: browser_frame handler
860
+ @socketio.on("browser_frame")
861
+ def on_browser_frame(data):
862
+ global _shared_state
863
+ if _shared_state is not None and data and "frame" in data:
864
+ _shared_state._cloud_frame = data["frame"]
865
+
866
+
867
+ # ── Dashboard process entry point ─────────────────────────────────────────────
868
+ def dashboard_process(shared_state, shm_name, frame_lock_val, frame_seq):
869
+ global _shm_name, _frame_seq, _shared_state
870
+
871
+ _shm_name = shm_name
872
+ _frame_seq = frame_seq
873
+ _shared_state = shared_state
874
+
875
+ print("=" * 60)
876
+ print("🌐 ISL Web Dashboard Starting...")
877
+ print("=" * 60)
878
+ print("πŸ“Š URL : http://localhost:5000")
879
+ print("πŸ“Ή Video stream : http://localhost:5000/video_feed")
880
+ print(" (MJPEG from SharedMemory β€” no WebSocket for video)")
881
+ print("=" * 60)
882
+
883
+ ui_queue = shared_state.ui_queue
884
+ cmd_queue = shared_state.command_queue
885
+
886
+ def state_drain():
887
+ print("πŸ“₯ State drain thread started")
888
+ while True:
889
+ try:
890
+ packet = ui_queue.get(timeout=1.0)
891
+ try:
892
+ _local_state_q.put_nowait(packet)
893
+ except Exception:
894
+ pass
895
+ except Exception:
896
+ pass
897
+
898
+ threading.Thread(target=state_drain, daemon=True, name="StateDrain").start()
899
+
900
+ def state_emit():
901
+ print("πŸ“‘ State emitter started")
902
+ while True:
903
+ try:
904
+ packet = _local_state_q.get(timeout=1.0)
905
+ except Empty:
906
+ continue
907
+ except Exception:
908
+ continue
909
+
910
+ if "action" in packet:
911
+ try:
912
+ cmd_queue.put_nowait(packet)
913
+ except Exception:
914
+ pass
915
+ continue
916
+
917
+ try:
918
+ socketio.emit("state_update", packet)
919
+ except Exception:
920
+ pass
921
+
922
+ threading.Thread(target=state_emit, daemon=True, name="StateEmit").start()
923
+
924
+ socketio.run(
925
+ app,
926
+ host="0.0.0.0",
927
+ port=5000,
928
+ debug=False,
929
+ use_reloader=False,
930
+ log_output=False,
931
+ )
932
+
933
+
934
+ # ── Compatibility aliases ─────────────────────────────────────────────────────
935
+ def start_ui_server(shared_state_obj, port: int = 5000):
936
+ dashboard_process(shared_state_obj, shm_name=None, frame_lock_val=None, frame_seq=None)
937
+
938
+
939
+ # ── Stand-alone test ──────────────────────────────────────────────────────────
940
+ if __name__ == "__main__":
941
+ socketio.run(app, host="0.0.0.0", port=5000, debug=True, use_reloader=False)
launcher.py ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # launcher.py β€” CLOUD PRODUCTION VERSION
2
+ #
3
+ # ════════════════════════════════════════════════════════════════════════════
4
+ # CLOUD FIX APPLIED (1 change only β€” everything else identical):
5
+ #
6
+ # FIX 1 β€” SharedMemory cleanup on startup:
7
+ # If the previous run crashed without cleanup, "isl_frame_shm"
8
+ # still exists in /dev/shm on Linux. SharedMemory(create=True)
9
+ # then raises FileExistsError and the entire launcher fails.
10
+ # Fixed by attempting to unlink the old block before creating new.
11
+ #
12
+ # ALL original features preserved:
13
+ # βœ… Shared memory block (640Γ—480 BGR, zero-copy frame transfer)
14
+ # βœ… frame_seq counter for dashboard to detect new frames
15
+ # βœ… Auto-restart of crashed child processes (every 2s watch loop)
16
+ # βœ… Clean shutdown with shm.close() + shm.unlink() on Ctrl+C
17
+ # βœ… Core Processor + Web Dashboard spawned as separate processes
18
+ # ════════════════════════════════════════════════════════════════════════════
19
+
20
+ import multiprocessing
21
+ import multiprocessing.shared_memory
22
+ import ctypes
23
+ import time
24
+ import sys
25
+
26
+ from isl_detection import core_processing_engine, SharedState
27
+ from isl_ui_dashboard import dashboard_process
28
+
29
+ FRAME_W, FRAME_H = 640, 480
30
+ FRAME_BYTES = FRAME_W * FRAME_H * 3 # 921 600 bytes for one BGR frame
31
+
32
+
33
+ def _run_core(shared_state, shm_name, frame_lock_val, frame_seq):
34
+ core_processing_engine(shared_state, shm_name, frame_lock_val, frame_seq)
35
+
36
+
37
+ def _run_dashboard(shared_state, shm_name, frame_lock_val, frame_seq):
38
+ dashboard_process(shared_state, shm_name, frame_lock_val, frame_seq)
39
+
40
+
41
+ if __name__ == "__main__":
42
+ multiprocessing.freeze_support() # required on Windows
43
+
44
+ print("=" * 70)
45
+ print(" 🀟 ISL EMOTION TRANSLATOR - MULTI-PROCESS SYSTEM")
46
+ print("=" * 70)
47
+ print()
48
+ print("🎯 Architecture:")
49
+ print(" πŸ“Ή Core Processor β†’ Camera Β· MediaPipe Β· TF model")
50
+ print(" 🌐 Web Dashboard β†’ MJPEG video Β· Socket.IO state")
51
+ print(" πŸ”— SharedMemory β†’ Zero-copy frame transfer (no base64 queue)")
52
+ print()
53
+ print("=" * 70)
54
+
55
+ # FIX 1: Clean up leftover shared memory from a previous crash.
56
+ # On Linux, named shared memory persists in /dev/shm until explicitly
57
+ # unlinked. If the previous run crashed, creating the same name fails.
58
+ try:
59
+ _old_shm = multiprocessing.shared_memory.SharedMemory(name="isl_frame_shm")
60
+ _old_shm.close()
61
+ _old_shm.unlink()
62
+ print("🧹 Cleaned up leftover shared memory from previous run")
63
+ except FileNotFoundError:
64
+ pass # normal β€” no leftover, continue
65
+
66
+ # ── Shared memory block ──────────────────────────────────────────────────
67
+ shm = multiprocessing.shared_memory.SharedMemory(
68
+ create=True, size=FRAME_BYTES, name="isl_frame_shm")
69
+ frame_seq = multiprocessing.Value(ctypes.c_uint64, 0)
70
+ frame_lock = multiprocessing.Value(ctypes.c_bool, False)
71
+
72
+ shared_state = SharedState()
73
+
74
+ # ── Spawn processes ──────────────────────────────────────────────────────
75
+ core_args = (shared_state, shm.name, frame_lock, frame_seq)
76
+ web_args = (shared_state, shm.name, frame_lock, frame_seq)
77
+
78
+ core_proc = multiprocessing.Process(
79
+ target=_run_core, args=core_args, name="CoreProcessor", daemon=True)
80
+ web_proc = multiprocessing.Process(
81
+ target=_run_dashboard, args=web_args, name="WebDashboard", daemon=True)
82
+
83
+ print("πŸš€ Starting Core Processor…")
84
+ core_proc.start()
85
+
86
+ # Give core a moment to open the camera before the dashboard opens
87
+ time.sleep(1.5)
88
+
89
+ print("πŸš€ Starting Web Dashboard on port 5000…")
90
+ web_proc.start()
91
+
92
+ print()
93
+ print("=" * 70)
94
+ print("βœ… SYSTEM READY!")
95
+ print("=" * 70)
96
+ print()
97
+ print("πŸ“Š Dashboard β†’ http://localhost:5000")
98
+ print("πŸ“Ή Video stream β†’ http://localhost:5000/video_feed")
99
+ print()
100
+ print("⌨️ Keyboard shortcuts (in dashboard):")
101
+ print(" Ctrl+R β†’ Reset")
102
+ print(" Ctrl+Backspace β†’ Backspace")
103
+ print(" 1 / 2 / 3 β†’ Accept suggestion")
104
+ print()
105
+ print("πŸ›‘ Press Ctrl+C to stop all processes")
106
+ print("=" * 70)
107
+
108
+ try:
109
+ while True:
110
+ # Auto-restart crashed processes
111
+ if not core_proc.is_alive():
112
+ print("⚠️ Core Processor crashed β€” restarting…")
113
+ core_proc = multiprocessing.Process(
114
+ target=_run_core, args=core_args, name="CoreProcessor", daemon=True)
115
+ core_proc.start()
116
+
117
+ if not web_proc.is_alive():
118
+ print("⚠️ Web Dashboard crashed β€” restarting…")
119
+ web_proc = multiprocessing.Process(
120
+ target=_run_dashboard, args=web_args, name="WebDashboard", daemon=True)
121
+ web_proc.start()
122
+
123
+ time.sleep(2)
124
+
125
+ except KeyboardInterrupt:
126
+ print("\nπŸ›‘ Shutting down…")
127
+ core_proc.terminate()
128
+ web_proc.terminate()
129
+ core_proc.join(timeout=5)
130
+ web_proc.join(timeout=5)
131
+ try:
132
+ shm.close()
133
+ shm.unlink()
134
+ except Exception:
135
+ pass
136
+ print("βœ… All processes stopped")
137
+ print("=" * 70)
138
+ sys.exit(0)
model.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a695177089550ea37eec9e54be2631c28e64ac4e03d666c0e6b86582bfe6a80b
3
+ size 11510976
nginx.conf ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ============================================================
2
+ # nginx.conf β€” Hugging Face Spaces
3
+ # Single port: 7860 (HF requirement)
4
+ # Routes to Flask 5000 and Flask 5001 internally
5
+ # ============================================================
6
+
7
+ worker_processes 1;
8
+ error_log stderr warn;
9
+ pid /tmp/nginx.pid;
10
+
11
+ events {
12
+ worker_connections 256;
13
+ }
14
+
15
+ http {
16
+ include /etc/nginx/mime.types;
17
+ default_type application/octet-stream;
18
+
19
+ # Use /tmp for temp files (HF non-root friendly)
20
+ client_body_temp_path /tmp/nginx_client_body;
21
+ proxy_temp_path /tmp/nginx_proxy;
22
+ fastcgi_temp_path /tmp/nginx_fastcgi;
23
+ uwsgi_temp_path /tmp/nginx_uwsgi;
24
+ scgi_temp_path /tmp/nginx_scgi;
25
+
26
+ sendfile on;
27
+ keepalive_timeout 65;
28
+
29
+ gzip on;
30
+ gzip_types text/plain text/css application/json application/javascript;
31
+
32
+ upstream gesture_app {
33
+ server 127.0.0.1:5000;
34
+ }
35
+
36
+ upstream avatar_app {
37
+ server 127.0.0.1:5001;
38
+ }
39
+
40
+ server {
41
+ listen 7860;
42
+ server_name _;
43
+ client_max_body_size 5M;
44
+
45
+ # ── Health check ──────────────────────────────────────
46
+ location /health {
47
+ proxy_pass http://gesture_app/health;
48
+ proxy_set_header Host $host;
49
+ }
50
+
51
+ # ── Socket.IO β€” gesture app ───────────────────────────
52
+ location /socket.io/ {
53
+ proxy_pass http://gesture_app/socket.io/;
54
+ proxy_http_version 1.1;
55
+ proxy_set_header Upgrade $http_upgrade;
56
+ proxy_set_header Connection "upgrade";
57
+ proxy_set_header Host $host;
58
+ proxy_set_header X-Real-IP $remote_addr;
59
+ proxy_set_header X-Forwarded-Proto $scheme;
60
+ proxy_read_timeout 86400;
61
+ proxy_buffering off;
62
+ proxy_cache off;
63
+ }
64
+
65
+ # ── MJPEG video stream ────────────────────────────────
66
+ location /video_feed {
67
+ proxy_pass http://gesture_app/video_feed;
68
+ proxy_http_version 1.1;
69
+ proxy_set_header Host $host;
70
+ proxy_buffering off;
71
+ proxy_cache off;
72
+ proxy_read_timeout 3600;
73
+ chunked_transfer_encoding on;
74
+ }
75
+
76
+ # ── Avatar Socket.IO ──────────────────────────────────
77
+ location /avatar/socket.io/ {
78
+ proxy_pass http://avatar_app/socket.io/;
79
+ proxy_http_version 1.1;
80
+ proxy_set_header Upgrade $http_upgrade;
81
+ proxy_set_header Connection "upgrade";
82
+ proxy_set_header Host $host;
83
+ proxy_set_header X-Forwarded-Proto $scheme;
84
+ proxy_read_timeout 86400;
85
+ proxy_buffering off;
86
+ }
87
+
88
+ # ── Avatar app ────────────────────────────────────────
89
+ location /avatar/ {
90
+ rewrite ^/avatar/(.*) /$1 break;
91
+ proxy_pass http://avatar_app/;
92
+ proxy_http_version 1.1;
93
+ proxy_set_header Host $host;
94
+ proxy_set_header X-Real-IP $remote_addr;
95
+ proxy_set_header X-Forwarded-Proto $scheme;
96
+ }
97
+
98
+ # ── Gesture app (root) ────────────────────────────────
99
+ location / {
100
+ proxy_pass http://gesture_app/;
101
+ proxy_http_version 1.1;
102
+ proxy_set_header Host $host;
103
+ proxy_set_header X-Real-IP $remote_addr;
104
+ proxy_set_header X-Forwarded-Proto $scheme;
105
+ }
106
+ }
107
+ }
requirements.txt ADDED
Binary file (2.32 kB). View file
 
speech_to_sign_avatar.py ADDED
@@ -0,0 +1,343 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ISL Fingerspelling System - Backend Server
3
+ Full A-Z Letter-by-Letter Speech-to-Sign Conversion
4
+
5
+ Install dependencies:
6
+ pip install SpeechRecognition flask flask-socketio eventlet
7
+
8
+ Usage:
9
+ python speech_to_sign_avatar_fixed.py
10
+
11
+ Then open: http://localhost:5001
12
+ """
13
+
14
+ import eventlet
15
+ eventlet.monkey_patch()
16
+
17
+ import os
18
+ import json
19
+ import time
20
+ import threading
21
+ from datetime import datetime
22
+ from flask import Flask, render_template, request, jsonify, send_from_directory
23
+ from flask_socketio import SocketIO, emit
24
+ import speech_recognition as sr
25
+
26
+ # ======================================
27
+ # RUN MODE CONFIG (LOCAL or CLOUD)
28
+ # ======================================
29
+
30
+ RUN_MODE = os.getenv("RUN_MODE", "LOCAL")
31
+
32
+ # Disable microphone loading in cloud environment
33
+ if RUN_MODE == "CLOUD":
34
+ sr.Microphone = None
35
+
36
+ os.environ["EVENTLET_NO_GREENDNS"] = "yes"
37
+
38
+ # ==================== ISL ALPHABET LIBRARY (A-Z) ====================
39
+
40
+ print("Loading ISL gesture database...")
41
+
42
+ ISL_LIBRARY = {
43
+ 'a': {'name': 'A', 'fingers': {'thumb': 0, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Fist with thumb out'},
44
+ 'b': {'name': 'B', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 0, 'ring': 0, 'pinky': 0}, 'rot': [0, 0, 0], 'desc': 'All fingers straight up'},
45
+ 'c': {'name': 'C', 'fingers': {'thumb': 0.3, 'index': 0.5, 'middle': 0.5, 'ring': 0.5, 'pinky': 0.5}, 'rot': [0, 0, 0], 'desc': 'Curved C shape'},
46
+ 'd': {'name': 'D', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Index up, others closed'},
47
+ 'e': {'name': 'E', 'fingers': {'thumb': 0.8, 'index': 1.2, 'middle': 1.2, 'ring': 1.2, 'pinky': 1.2}, 'rot': [0, 0, 0], 'desc': 'All fingers curled'},
48
+ 'f': {'name': 'F', 'fingers': {'thumb': 0.8, 'index': 0.8, 'middle': 0, 'ring': 0, 'pinky': 0}, 'rot': [0, 0, 0], 'desc': 'OK sign variant'},
49
+ 'g': {'name': 'G', 'fingers': {'thumb': 0, 'index': 0, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 90], 'desc': 'Index and thumb point sideways'},
50
+ 'h': {'name': 'H', 'fingers': {'thumb': 1.5, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 90], 'desc': 'Two fingers horizontal'},
51
+ 'i': {'name': 'I', 'fingers': {'thumb': 1.5, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 0}, 'rot': [0, 0, 0], 'desc': 'Pinky up'},
52
+ 'j': {'name': 'J', 'fingers': {'thumb': 1.5, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 0}, 'rot': [0, 0, 0], 'desc': 'Pinky draws J'},
53
+ 'k': {'name': 'K', 'fingers': {'thumb': 0.5, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 45], 'desc': 'Index and middle V shape'},
54
+ 'l': {'name': 'L', 'fingers': {'thumb': 0, 'index': 0, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'L shape'},
55
+ 'm': {'name': 'M', 'fingers': {'thumb': 0.8, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Three fingers tucked under thumb'},
56
+ 'n': {'name': 'N', 'fingers': {'thumb': 0.8, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Two fingers tucked'},
57
+ 'o': {'name': 'O', 'fingers': {'thumb': 0.5, 'index': 0.8, 'middle': 0.8, 'ring': 0.8, 'pinky': 0.8}, 'rot': [0, 0, 0], 'desc': 'All fingers form O'},
58
+ 'p': {'name': 'P', 'fingers': {'thumb': 0.5, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, -45], 'desc': 'K pointing down'},
59
+ 'q': {'name': 'Q', 'fingers': {'thumb': 0, 'index': 0, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, -90], 'desc': 'G pointing down'},
60
+ 'r': {'name': 'R', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Index and middle crossed'},
61
+ 's': {'name': 'S', 'fingers': {'thumb': 0.5, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Fist with thumb across'},
62
+ 't': {'name': 'T', 'fingers': {'thumb': 0.8, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Thumb between index and middle'},
63
+ 'u': {'name': 'U', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Two fingers together'},
64
+ 'v': {'name': 'V', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Victory/Peace sign'},
65
+ 'w': {'name': 'W', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 0, 'ring': 0, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Three fingers up'},
66
+ 'x': {'name': 'X', 'fingers': {'thumb': 1.5, 'index': 0.8, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Index bent like hook'},
67
+ 'y': {'name': 'Y', 'fingers': {'thumb': 0, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 0}, 'rot': [0, 0, 0], 'desc': 'Thumb and pinky out (hang loose)'},
68
+ 'z': {'name': 'Z', 'fingers': {'thumb': 1.5, 'index': 0, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Index draws Z'}
69
+ }
70
+
71
+ print(f"Loaded {len(ISL_LIBRARY)} gestures")
72
+
73
+ # ==================== Flask Setup ====================
74
+ app = Flask(__name__)
75
+ app.config['SECRET_KEY'] = 'isl-fingerspelling-2024'
76
+ socketio = SocketIO(
77
+ app,
78
+ cors_allowed_origins="*",
79
+ async_mode="eventlet",
80
+ ping_timeout=60,
81
+ ping_interval=25
82
+ )
83
+
84
+ print("=" * 60)
85
+ print("ISL SPEECH TO SIGN AVATAR SYSTEM")
86
+ print("RUN MODE:", RUN_MODE)
87
+ print("=" * 60)
88
+
89
+ # Speech recognizer
90
+ recognizer = sr.Recognizer()
91
+ recognizer.energy_threshold = 3000
92
+ recognizer.dynamic_energy_threshold = True
93
+ recognizer.pause_threshold = 0.8
94
+ recognizer.non_speaking_duration = 0.5
95
+
96
+ # System statistics
97
+ stats = {
98
+ "total_conversions": 0,
99
+ "total_gestures": 0,
100
+ "speech_recognitions": 0,
101
+ "errors": 0,
102
+ "start_time": datetime.now()
103
+ }
104
+
105
+ # ==================== Text to Letter Sequence ====================
106
+ def text_to_letter_sequence(text):
107
+ """Convert text to letter-by-letter fingerspelling sequence"""
108
+ clean_text = text.lower().replace(",", "").replace(".", "").replace("!", "").replace("?", "").replace("-", " ")
109
+ chars = list(clean_text)
110
+
111
+ sequence = []
112
+
113
+ for i, char in enumerate(chars):
114
+ if char == ' ':
115
+ sequence.append({
116
+ 'name': 'SPACE',
117
+ 'desc': 'Word space - pause between words',
118
+ 'duration': 0.6,
119
+ 'isSpace': True,
120
+ 'keyframes': [
121
+ {
122
+ 'time': 0,
123
+ 'hand': 'right',
124
+ 'pos': [0, 0, 0],
125
+ 'rot': [0, 0, 0],
126
+ 'fingers': {'thumb': 0, 'index': 0, 'middle': 0, 'ring': 0, 'pinky': 0}
127
+ },
128
+ {
129
+ 'time': 0.6,
130
+ 'hand': 'right',
131
+ 'pos': [0.2, 0, 0],
132
+ 'rot': [0, 0, 0],
133
+ 'fingers': {'thumb': 0, 'index': 0, 'middle': 0, 'ring': 0, 'pinky': 0}
134
+ }
135
+ ]
136
+ })
137
+ continue
138
+
139
+ if char in ISL_LIBRARY:
140
+ letter_data = ISL_LIBRARY[char]
141
+ rot_rad = [r * 3.14159 / 180 for r in letter_data['rot']]
142
+
143
+ sequence.append({
144
+ 'name': letter_data['name'],
145
+ 'letter': char,
146
+ 'desc': letter_data['desc'],
147
+ 'duration': 0.7,
148
+ 'keyframes': [
149
+ {
150
+ 'time': 0,
151
+ 'hand': 'right',
152
+ 'pos': [0, 0, 0],
153
+ 'rot': rot_rad,
154
+ 'fingers': letter_data['fingers']
155
+ },
156
+ {
157
+ 'time': 0.3,
158
+ 'hand': 'right',
159
+ 'pos': [0, 0.15, 0],
160
+ 'rot': rot_rad,
161
+ 'fingers': letter_data['fingers']
162
+ },
163
+ {
164
+ 'time': 0.7,
165
+ 'hand': 'right',
166
+ 'pos': [0, 0.15, 0],
167
+ 'rot': rot_rad,
168
+ 'fingers': letter_data['fingers']
169
+ }
170
+ ]
171
+ })
172
+
173
+ return sequence
174
+
175
+ # ==================== Routes ====================
176
+ @app.route('/')
177
+ def index():
178
+ """Serve main interface"""
179
+ try:
180
+ return send_from_directory(os.path.join(os.getcwd(), 'templates'), 'avatar_interface.html')
181
+ except Exception:
182
+ return "Avatar UI Loaded. Frontend missing."
183
+
184
+ @app.route('/health')
185
+ def health():
186
+ """Health check endpoint for cloud platforms"""
187
+ return jsonify({"status": "running"})
188
+
189
+ @app.route('/static/<path:filename>')
190
+ def serve_static(filename):
191
+ """Serve static files for cloud platforms"""
192
+ return send_from_directory('static', filename)
193
+
194
+ @app.route('/api/text-to-signs', methods=['POST'])
195
+ def text_to_signs_api():
196
+ """Convert text to letter sequence"""
197
+ try:
198
+ data = request.json
199
+ text = data.get('text', '')
200
+
201
+ if not text:
202
+ return jsonify({"success": False, "error": "No text provided"}), 400
203
+
204
+ print(f'πŸ“ Converting text: "{text}"')
205
+
206
+ sequence = text_to_letter_sequence(text)
207
+
208
+ stats["total_conversions"] += 1
209
+ stats["total_gestures"] += len(sequence)
210
+
211
+ return jsonify({
212
+ "success": True,
213
+ "text": text,
214
+ "sequence": sequence,
215
+ "stats": get_stats()
216
+ })
217
+
218
+ except Exception as e:
219
+ stats["errors"] += 1
220
+ print(f"❌ Error: {e}")
221
+ return jsonify({"success": False, "error": str(e)}), 500
222
+
223
+ @app.route('/api/stats')
224
+ def get_stats_api():
225
+ """Get system statistics"""
226
+ return jsonify(get_stats())
227
+
228
+ def get_stats():
229
+ """Calculate current statistics"""
230
+ uptime = (datetime.now() - stats["start_time"]).total_seconds()
231
+
232
+ return {
233
+ "total_conversions": stats["total_conversions"],
234
+ "total_gestures": stats["total_gestures"],
235
+ "speech_recognitions": stats["speech_recognitions"],
236
+ "errors": stats["errors"],
237
+ "uptime_seconds": int(uptime),
238
+ "alphabet_letters": len(ISL_LIBRARY)
239
+ }
240
+
241
+ # ==================== Socket.IO Events ====================
242
+ @socketio.on('connect')
243
+ def handle_connect():
244
+ """Client connected"""
245
+ print('βœ… Client connected')
246
+ emit('status', {'message': 'Connected', 'stats': get_stats()})
247
+
248
+ @socketio.on('disconnect')
249
+ def handle_disconnect():
250
+ """Client disconnected"""
251
+ print('πŸ‘‹ Client disconnected')
252
+
253
+ @socketio.on('record_speech')
254
+ def handle_record_speech():
255
+ """Record speech and convert to fingerspelling"""
256
+
257
+ # Disable microphone in cloud environment
258
+ if RUN_MODE == "CLOUD":
259
+ emit("error", {"message": "Microphone recording not supported in cloud deployment"})
260
+ return
261
+
262
+ def record_and_convert():
263
+ try:
264
+ print('🎀 Starting speech recognition...')
265
+
266
+ with sr.Microphone() as source:
267
+ print('🎀 Listening... Speak now!')
268
+ socketio.emit('recording_status', {'status': 'listening'})
269
+
270
+ recognizer.adjust_for_ambient_noise(source, duration=0.5)
271
+ audio = recognizer.listen(source, timeout=5, phrase_time_limit=10)
272
+
273
+ print('πŸ”„ Processing speech...')
274
+ socketio.emit('recording_status', {'status': 'processing'})
275
+
276
+ try:
277
+ text = recognizer.recognize_google(audio, language='en-IN')
278
+ print(f'βœ… Recognized: "{text}"')
279
+
280
+ stats["speech_recognitions"] += 1
281
+ stats["total_conversions"] += 1
282
+
283
+ sequence = text_to_letter_sequence(text)
284
+ stats["total_gestures"] += len(sequence)
285
+
286
+ socketio.emit('play_signs', {
287
+ 'text': text,
288
+ 'sequence': sequence,
289
+ 'stats': get_stats()
290
+ })
291
+
292
+ except sr.UnknownValueError:
293
+ print('❌ Could not understand speech')
294
+ stats["errors"] += 1
295
+ socketio.emit('error', {
296
+ 'message': 'Could not understand speech. Please speak clearly and try again.'
297
+ })
298
+
299
+ except sr.RequestError as e:
300
+ print(f'❌ Speech recognition API error: {e}')
301
+ stats["errors"] += 1
302
+ socketio.emit('error', {
303
+ 'message': 'Speech recognition service error. Please check your internet connection.'
304
+ })
305
+
306
+ except sr.WaitTimeoutError:
307
+ print('⏱️ No speech detected (timeout)')
308
+ stats["errors"] += 1
309
+ socketio.emit('error', {
310
+ 'message': 'No speech detected. Please try again.'
311
+ })
312
+
313
+ except Exception as e:
314
+ print(f'❌ Recording error: {e}')
315
+ stats["errors"] += 1
316
+ socketio.emit('error', {
317
+ 'message': f'Recording error: {str(e)}'
318
+ })
319
+
320
+ threading.Thread(target=record_and_convert, daemon=True).start()
321
+
322
+ # ==================== Main ====================
323
+ if __name__ == '__main__':
324
+
325
+ print("\n" + "="*60)
326
+ print("ISL SPEECH TO SIGN AVATAR SYSTEM STARTING")
327
+ print("RUN MODE:", RUN_MODE)
328
+ print("="*60)
329
+
330
+ os.makedirs('templates', exist_ok=True)
331
+
332
+ PORT = int(os.environ.get("PORT", 5001))
333
+
334
+ print("Server starting on port:", PORT)
335
+ print("Health check endpoint: /health")
336
+
337
+ socketio.run(
338
+ app,
339
+ host="0.0.0.0",
340
+ port=PORT,
341
+ debug=False,
342
+ use_reloader=False
343
+ )
supervisord.conf ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ============================================================
2
+ # supervisord.conf β€” Hugging Face Spaces
3
+ # Manages 4 processes in one container:
4
+ # 1. detection_engine β€” MediaPipe + TF inference
5
+ # 2. gesture_app β€” Flask dashboard, port 5000
6
+ # 3. avatar_app β€” Flask avatar, port 5001
7
+ # 4. nginx β€” Reverse proxy, port 7860
8
+ # ============================================================
9
+
10
+ [supervisord]
11
+ nodaemon=true
12
+ logfile=/tmp/supervisord.log
13
+ loglevel=info
14
+ pidfile=/tmp/supervisord.pid
15
+
16
+ [unix_http_server]
17
+ file=/tmp/supervisor.sock
18
+
19
+ [rpcinterface:supervisor]
20
+ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
21
+
22
+ [supervisorctl]
23
+ serverurl=unix:///tmp/supervisor.sock
24
+
25
+ # ── 1. ISL Detection Engine (starts first β€” loads TF model) ──
26
+ [program:detection_engine]
27
+ command=python /app/isl_detection.py
28
+ directory=/app
29
+ autostart=true
30
+ autorestart=true
31
+ priority=1
32
+ startretries=5
33
+ startsecs=20
34
+ stdout_logfile=/tmp/detection.log
35
+ stderr_logfile=/tmp/detection_err.log
36
+ stdout_logfile_maxbytes=5MB
37
+ environment=
38
+ RUN_MODE="CLOUD",
39
+ PYTHONUNBUFFERED="1",
40
+ TF_CPP_MIN_LOG_LEVEL="3",
41
+ CUDA_VISIBLE_DEVICES="-1"
42
+
43
+ # ── 2. Gesture Dashboard (Flask + Socket.IO, port 5000) ───────
44
+ [program:gesture_app]
45
+ command=python /app/isl_ui_dashboard.py
46
+ directory=/app
47
+ autostart=true
48
+ autorestart=true
49
+ priority=10
50
+ startretries=5
51
+ startsecs=15
52
+ stdout_logfile=/tmp/gesture.log
53
+ stderr_logfile=/tmp/gesture_err.log
54
+ stdout_logfile_maxbytes=5MB
55
+ environment=
56
+ RUN_MODE="CLOUD",
57
+ PYTHONUNBUFFERED="1",
58
+ TF_CPP_MIN_LOG_LEVEL="3",
59
+ CUDA_VISIBLE_DEVICES="-1"
60
+
61
+ # ── 3. Avatar App (Flask + eventlet, port 5001) ───────────────
62
+ [program:avatar_app]
63
+ command=python /app/speech_to_sign_avatar.py
64
+ directory=/app
65
+ autostart=true
66
+ autorestart=true
67
+ priority=10
68
+ startretries=5
69
+ startsecs=10
70
+ stdout_logfile=/tmp/avatar.log
71
+ stderr_logfile=/tmp/avatar_err.log
72
+ stdout_logfile_maxbytes=5MB
73
+ environment=
74
+ RUN_MODE="CLOUD",
75
+ PYTHONUNBUFFERED="1",
76
+ PORT="5001"
77
+
78
+ # ── 4. nginx (starts after Flask apps are up) ─────────────────
79
+ [program:nginx]
80
+ command=/usr/sbin/nginx -g "daemon off;" -c /etc/nginx/nginx.conf
81
+ autostart=true
82
+ autorestart=true
83
+ priority=30
84
+ startretries=3
85
+ startsecs=5
86
+ stdout_logfile=/tmp/nginx.log
87
+ stderr_logfile=/tmp/nginx_err.log
88
+ stdout_logfile_maxbytes=5MB
templates/avatar_interface.html ADDED
@@ -0,0 +1,653 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8"/>
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover"/>
6
+ <title>ISL Fingerspelling β€” AI Sign Language</title>
7
+ <link rel="preconnect" href="https://fonts.googleapis.com"/>
8
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin/>
9
+ <link href="https://fonts.googleapis.com/css2?family=Playfair+Display:wght@400;600;700&family=DM+Sans:wght@300;400;500;600&family=DM+Mono:wght@400;500&display=swap" rel="stylesheet"/>
10
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
11
+ <style>
12
+ :root{
13
+ --bg:#0d1117;--surf:#131922;--card:#1a2233;
14
+ --bdr:rgba(212,168,83,.13);--bdrm:rgba(212,168,83,.30);--bdrh:rgba(212,168,83,.58);
15
+ --gold:#d4a853;--goldb:#f0c76a;--goldd:#a07c3a;
16
+ --sage:#7eb89a;--txt:#f0ede6;--txtm:#a8a096;--txtl:#5a5650;--red:#c0614a;
17
+ --r8:8px;--r14:14px;
18
+ --fD:'Playfair Display',Georgia,serif;
19
+ --fB:'DM Sans',system-ui,sans-serif;
20
+ --fM:'DM Mono',monospace;
21
+ }
22
+ *,*::before,*::after{margin:0;padding:0;box-sizing:border-box}
23
+ html,body{height:100%;overflow:hidden}
24
+ body{font-family:var(--fB);background:var(--bg);color:var(--txt);-webkit-font-smoothing:antialiased;display:flex;flex-direction:column}
25
+ #app{display:flex;flex:1;overflow:hidden}
26
+
27
+ /* ── Canvas pane ── */
28
+ #cvp{flex:1;display:flex;flex-direction:column;min-width:0;background:var(--bg);position:relative}
29
+ #hc{flex:1;width:100%;display:block;touch-action:none}
30
+ .topbar{display:flex;align-items:center;justify-content:space-between;padding:10px 16px 0;flex-shrink:0}
31
+ .brand{display:flex;align-items:baseline;gap:8px}
32
+ .brand-t{font-family:var(--fD);font-size:1.1rem;font-weight:700;color:var(--gold)}
33
+ .brand-s{font-size:.65rem;color:var(--txtl);text-transform:uppercase;letter-spacing:.07em}
34
+ .badge{font-size:.65rem;font-family:var(--fM);color:var(--sage);background:rgba(126,184,154,.1);border:1px solid rgba(126,184,154,.25);padding:3px 9px;border-radius:20px;transition:all .3s}
35
+ .vrow{display:flex;gap:4px;padding:6px 16px 0;flex-shrink:0}
36
+ .vb{padding:4px 10px;font-size:.63rem;font-family:var(--fM);background:transparent;border:1px solid var(--bdr);color:var(--txtl);border-radius:6px;cursor:pointer;transition:all .2s;text-transform:uppercase;letter-spacing:.03em}
37
+ .vb:hover{color:var(--txt);border-color:var(--bdrm)}
38
+ .vb.on{background:rgba(212,168,83,.18);color:var(--gold);border-color:var(--bdrm)}
39
+ .gov{position:absolute;bottom:0;left:0;right:0;padding:12px 18px 14px;background:linear-gradient(to top,rgba(13,17,23,1) 55%,transparent);display:flex;align-items:flex-end;justify-content:space-between;gap:10px;pointer-events:none}
40
+ .gl{font-family:var(--fD);font-size:clamp(2rem,5vw,3.2rem);font-weight:700;color:var(--goldb);line-height:1}
41
+ .gd{font-size:.72rem;color:var(--txtm);text-align:right;max-width:180px;line-height:1.45}
42
+ .gp{position:absolute;bottom:0;left:0;right:0;height:3px;background:rgba(212,168,83,.1)}
43
+ .gb{height:100%;width:0%;background:linear-gradient(90deg,var(--goldd),var(--goldb));transition:width .12s linear;position:relative;overflow:hidden}
44
+ .gb::after{content:'';position:absolute;inset:0;background:linear-gradient(90deg,transparent,rgba(255,255,255,.3),transparent);animation:sh 1.8s infinite}
45
+ @keyframes sh{from{transform:translateX(-100%)}to{transform:translateX(200%)}}
46
+
47
+ /* ── Control pane ── */
48
+ #cp{width:340px;flex-shrink:0;display:flex;flex-direction:column;background:var(--surf);border-left:1px solid var(--bdr);overflow-y:auto}
49
+ #cp::-webkit-scrollbar{width:4px}
50
+ #cp::-webkit-scrollbar-thumb{background:var(--bdrm);border-radius:4px}
51
+ @media(max-width:720px){
52
+ #app{flex-direction:column;height:100%}
53
+ #cvp{flex:0 0 46vh;min-height:260px;max-height:46vh}
54
+ #cp{width:100%;flex:1;border-left:none;border-top:1px solid var(--bdr);overflow-y:auto}
55
+ }
56
+ .pan{padding:14px 16px;display:flex;flex-direction:column;gap:12px}
57
+ .slbl{font-size:.65rem;font-weight:600;text-transform:uppercase;letter-spacing:.08em;color:var(--txtl);display:flex;align-items:center;gap:6px}
58
+ textarea{width:100%;height:82px;resize:none;background:var(--card);border:1px solid var(--bdr);border-radius:var(--r14);color:var(--txt);font-family:var(--fB);font-size:.92rem;line-height:1.5;padding:11px 13px 26px;outline:none;transition:border-color .25s}
59
+ textarea::placeholder{color:var(--txtl)}
60
+ textarea:focus{border-color:var(--bdrh)}
61
+ .iwrap{position:relative}
62
+ .cc{position:absolute;bottom:8px;right:10px;font-size:.62rem;font-family:var(--fM);color:var(--txtl)}
63
+
64
+ /* ── Speech warning banner ── */
65
+ .mic-warn{background:rgba(212,168,83,.07);border:1px solid rgba(212,168,83,.25);border-radius:var(--r8);padding:10px 13px;font-size:.72rem;color:var(--txtm);line-height:1.7;display:none}
66
+ .mic-warn.show{display:block}
67
+ .mic-warn strong{color:var(--gold)}
68
+ .mic-warn code{background:rgba(212,168,83,.12);border:1px solid rgba(212,168,83,.2);border-radius:4px;padding:1px 5px;font-family:var(--fM);font-size:.68rem;color:var(--goldb)}
69
+
70
+ /* ── Mic status inline ── */
71
+ .mic-status{display:none;align-items:center;gap:7px;padding:6px 11px;border-radius:var(--r8);background:rgba(126,184,154,.07);border:1px solid rgba(126,184,154,.2);font-size:.72rem;color:var(--sage);font-family:var(--fM)}
72
+ .mic-status.show{display:flex}
73
+ .mic-status.listening{background:rgba(192,97,74,.08);border-color:rgba(192,97,74,.3);color:#e07060}
74
+ .mic-dot{width:7px;height:7px;border-radius:50%;background:var(--sage);animation:pd .9s infinite;flex-shrink:0}
75
+ .mic-status.listening .mic-dot{background:#e07060}
76
+ @keyframes pd{0%,100%{opacity:1;transform:scale(1)}50%{opacity:.5;transform:scale(.75)}}
77
+
78
+ /* ── Buttons ── */
79
+ .bg4{display:grid;grid-template-columns:repeat(4,1fr);gap:7px}
80
+ .btn{display:flex;align-items:center;justify-content:center;gap:4px;padding:10px 6px;border:1px solid transparent;border-radius:var(--r14);font-family:var(--fB);font-size:.76rem;font-weight:600;cursor:pointer;transition:all .22s;white-space:nowrap;overflow:hidden}
81
+ .btn:active{transform:scale(.96)}
82
+ .btn:disabled{opacity:.38;cursor:not-allowed}
83
+ .btn:disabled:active{transform:none}
84
+ .bc{background:linear-gradient(135deg,#c49a3a,#d4a853,#e8c06a);color:#0d1117;border-color:rgba(212,168,83,.4);box-shadow:0 2px 10px rgba(212,168,83,.2)}
85
+ .bc:not(:disabled):hover{box-shadow:0 4px 18px rgba(212,168,83,.38)}
86
+ .bm{background:linear-gradient(135deg,#4e7d66,#7eb89a);color:#0d1117;border-color:rgba(126,184,154,.35)}
87
+ .bm:not(:disabled):hover{box-shadow:0 4px 16px rgba(126,184,154,.3)}
88
+ .bm.listening{background:linear-gradient(135deg,#8a3a2a,#c0614a)!important;color:#fff!important;border-color:rgba(192,97,74,.4)!important;animation:pulse .8s infinite}
89
+ @keyframes pulse{0%,100%{box-shadow:0 0 0 0 rgba(192,97,74,.4)}50%{box-shadow:0 0 0 8px rgba(192,97,74,0)}}
90
+ .bs{background:transparent;color:var(--red);border-color:rgba(192,97,74,.3)}
91
+ .bs:hover{background:rgba(192,97,74,.08);border-color:rgba(192,97,74,.55)}
92
+ .bg2{background:transparent;color:var(--txtm);border-color:var(--bdr)}
93
+ .bg2:hover{border-color:var(--bdrm);color:var(--txt)}
94
+
95
+ /* ── Status pill ── */
96
+ .st{display:flex;align-items:center;gap:7px;padding:7px 13px;border-radius:30px;font-size:.73rem;font-weight:500;border:1px solid var(--bdr);background:var(--card);color:var(--txtm);transition:all .3s;min-height:34px}
97
+ .dot{width:7px;height:7px;border-radius:50%;background:var(--txtl);flex-shrink:0;transition:all .3s}
98
+ .st.act .dot{background:var(--sage);box-shadow:0 0 8px var(--sage);animation:pd 1.4s infinite}
99
+ .st.act{border-color:rgba(126,184,154,.3);color:var(--sage)}
100
+ .st.rec .dot{background:var(--gold);box-shadow:0 0 8px var(--gold);animation:pd .9s infinite}
101
+ .st.rec{border-color:var(--bdrm);color:var(--gold)}
102
+ .st.err .dot{background:var(--red)}
103
+ .st.err{border-color:rgba(192,97,74,.3);color:var(--red)}
104
+
105
+ .div{height:1px;background:var(--bdr);margin:0 -16px}
106
+ .sh{display:flex;align-items:center;justify-content:space-between}
107
+ .sbadge{font-family:var(--fM);font-size:.62rem;background:rgba(212,168,83,.12);color:var(--gold);border:1px solid rgba(212,168,83,.2);padding:2px 8px;border-radius:20px}
108
+ .ll{max-height:180px;overflow-y:auto;display:flex;flex-direction:column;gap:4px;padding-right:2px}
109
+ .ll::-webkit-scrollbar{width:3px}
110
+ .ll::-webkit-scrollbar-thumb{background:var(--bdrm);border-radius:3px}
111
+ .li{display:flex;align-items:center;gap:9px;padding:7px 11px;border-radius:var(--r8);background:var(--card);border:1px solid var(--bdr);transition:all .25s;flex-shrink:0}
112
+ .li.on{background:rgba(212,168,83,.1);border-color:var(--bdrh);box-shadow:0 0 14px rgba(212,168,83,.1)}
113
+ .li.sp{background:rgba(126,184,154,.05);border-color:rgba(126,184,154,.15)}
114
+ .lc{font-family:var(--fD);font-size:1.05rem;font-weight:700;color:var(--gold);width:22px;text-align:center;flex-shrink:0}
115
+ .li.on .lc{color:var(--goldb)}
116
+ .li.sp .lc{color:var(--sage);font-size:.75rem}
117
+ .lh{font-size:.68rem;color:var(--txtl);flex:1}
118
+ .li.on .lh{color:var(--txtm)}
119
+ .es{text-align:center;padding:24px 0;color:var(--txtl);font-size:.78rem;line-height:1.7}
120
+ .sg{display:grid;grid-template-columns:1fr 1fr;gap:7px}
121
+ .st2{background:var(--card);border:1px solid var(--bdr);border-radius:var(--r14);padding:11px 13px;transition:border-color .25s}
122
+ .st2:hover{border-color:var(--bdrm)}
123
+ .sl2{font-size:.6rem;text-transform:uppercase;letter-spacing:.07em;color:var(--txtl);font-weight:600;margin-bottom:3px}
124
+ .sv{font-family:var(--fD);font-size:1.6rem;font-weight:700;color:var(--gold);line-height:1}
125
+ .abt{font-size:.77rem;color:var(--txtm);line-height:1.7}
126
+ .abt p{margin-bottom:8px}
127
+ .al{list-style:none;margin-bottom:8px}
128
+ .al li{padding-left:12px;position:relative;font-size:.75rem;padding-bottom:2px}
129
+ .al li::before{content:'β€Ί';position:absolute;left:0;color:var(--gold);font-weight:700}
130
+ .acr{margin-top:12px;padding-top:10px;border-top:1px solid var(--bdr)}
131
+ .an{font-family:var(--fD);color:var(--gold);font-weight:600;font-size:.88rem}
132
+ .ar{font-size:.67rem;color:var(--txtl);margin:2px 0 3px}
133
+ .ae{color:var(--sage);font-size:.68rem;font-family:var(--fM)}
134
+ .nav{display:flex;gap:7px;padding:0 16px 20px}
135
+ .na{flex:1;display:flex;align-items:center;justify-content:center;gap:6px;padding:12px 8px;border-radius:var(--r14);border:1px solid var(--bdr);background:var(--card);color:var(--txtm);text-decoration:none;font-size:.76rem;font-weight:600;transition:all .25s}
136
+ .na:hover{border-color:var(--bdrm);color:var(--txt)}
137
+ .na.on{background:rgba(212,168,83,.1);border-color:var(--bdrh);color:var(--gold)}
138
+ </style>
139
+ </head>
140
+ <body>
141
+ <div id="app">
142
+
143
+ <!-- Canvas side -->
144
+ <div id="cvp">
145
+ <div class="topbar">
146
+ <div class="brand">
147
+ <span class="brand-t">ISL Fingerspelling</span>
148
+ <span class="brand-s">Speech Β· Sign Β· AI</span>
149
+ </div>
150
+ <span class="badge" id="badge">● Ready</span>
151
+ </div>
152
+ <div class="vrow">
153
+ <button class="vb on" onclick="setView('front',this)">Front</button>
154
+ <button class="vb" onclick="setView('side',this)">Side</button>
155
+ <button class="vb" onclick="setView('top',this)">Top</button>
156
+ <button class="vb" onclick="setView('spin',this)">Spin</button>
157
+ </div>
158
+ <canvas id="hc"></canvas>
159
+ <div class="gov">
160
+ <div id="gl" class="gl">🀟</div>
161
+ <div id="gd" class="gd">Type or speak to begin</div>
162
+ <div class="gp"><div class="gb" id="gb"></div></div>
163
+ </div>
164
+ </div>
165
+
166
+ <!-- Control side -->
167
+ <div id="cp">
168
+ <div class="pan">
169
+
170
+ <div class="slbl">✍️ Enter Text</div>
171
+ <div class="iwrap">
172
+ <textarea id="txt" placeholder="Type your message here…" maxlength="200"></textarea>
173
+ <span class="cc" id="cc">0/200</span>
174
+ </div>
175
+
176
+ <div class="mic-warn" id="mwarn"></div>
177
+ <div class="mic-status" id="mstat">
178
+ <div class="mic-dot"></div>
179
+ <span id="mstxt">Listening…</span>
180
+ </div>
181
+
182
+ <div class="bg4">
183
+ <button class="btn bc" onclick="doConvert()">✨ Convert</button>
184
+ <button class="btn bm" onclick="doSpeak()" id="bspk">🎀 Speak</button>
185
+ <button class="btn bs" onclick="doStop()">⏹ Stop</button>
186
+ <button class="btn bg2" onclick="doClear()">βœ• Clear</button>
187
+ </div>
188
+
189
+ <div class="st" id="st"><div class="dot"></div><span id="stxt">Ready to translate</span></div>
190
+
191
+ <div class="div"></div>
192
+
193
+ <div class="sh">
194
+ <div class="slbl">πŸ“‹ Letter Sequence</div>
195
+ <span class="sbadge" id="scnt">0 letters</span>
196
+ </div>
197
+ <div class="ll" id="ll">
198
+ <div class="es">Type text or speak<br>to see the sequence</div>
199
+ </div>
200
+
201
+ <div class="div"></div>
202
+
203
+ <div class="slbl">πŸ“Š Session Stats</div>
204
+ <div class="sg">
205
+ <div class="st2"><div class="sl2">Conversions</div><div class="sv" id="s0">0</div></div>
206
+ <div class="st2"><div class="sl2">Letters Signed</div><div class="sv" id="s1">0</div></div>
207
+ <div class="st2"><div class="sl2">Speech Uses</div><div class="sv" id="s2">0</div></div>
208
+ <div class="st2"><div class="sl2">Words</div><div class="sv" id="s3">0</div></div>
209
+ </div>
210
+
211
+ <div class="div"></div>
212
+
213
+ <div class="slbl">ℹ️ About</div>
214
+ <div class="abt">
215
+ <p>AI Powered ISL converts typed or spoken language into Indian Sign Language 3D animation, letter by letter.</p>
216
+ <ul class="al">
217
+ <li>Deaf &amp; hard-of-hearing individuals</li>
218
+ <li>Inclusive education environments</li>
219
+ <li>Public service &amp; assistive AI</li>
220
+ </ul>
221
+ <div class="acr">
222
+ <div class="an">Deepak Roshan</div>
223
+ <div class="ar">AI Engineer</div>
224
+ <div class="ae">deepakroshan380@gmail.com</div>
225
+ </div>
226
+ </div>
227
+ </div>
228
+
229
+ <!-- FIX: Nav links built dynamically β€” works on localhost AND cloud domain -->
230
+ <div class="nav">
231
+ <a href="#" class="na" id="nav-gesture">😊 Gesture Recognition</a>
232
+ <a href="#" class="na on" id="nav-avatar">🀟 Speech-to-Sign</a>
233
+ </div>
234
+ </div>
235
+ </div>
236
+
237
+ <script>
238
+ // ══════════════════════════════════════════════════════════
239
+ // ISL POSE TABLE A–Z (unchanged)
240
+ // ══════════════════════════════════════════════════════════
241
+ const ISL={
242
+ a:{thumb:.1, index:1.5,middle:1.5,ring:1.5,pinky:1.5,rz:0},
243
+ b:{thumb:1.2,index:0, middle:0, ring:0, pinky:0, rz:0},
244
+ c:{thumb:.3, index:.5, middle:.5, ring:.5, pinky:.5, rz:0},
245
+ d:{thumb:1.2,index:0, middle:1.5,ring:1.5,pinky:1.5,rz:0},
246
+ e:{thumb:.8, index:1.2,middle:1.2,ring:1.2,pinky:1.2,rz:0},
247
+ f:{thumb:.8, index:.8, middle:0, ring:0, pinky:0, rz:0},
248
+ g:{thumb:0, index:0, middle:1.5,ring:1.5,pinky:1.5,rz:1.57},
249
+ h:{thumb:1.5,index:0, middle:0, ring:1.5,pinky:1.5,rz:1.57},
250
+ i:{thumb:1.5,index:1.5,middle:1.5,ring:1.5,pinky:0, rz:0},
251
+ j:{thumb:1.5,index:1.5,middle:1.5,ring:1.5,pinky:0, rz:0},
252
+ k:{thumb:.5, index:0, middle:0, ring:1.5,pinky:1.5,rz:.78},
253
+ l:{thumb:0, index:0, middle:1.5,ring:1.5,pinky:1.5,rz:0},
254
+ m:{thumb:.8, index:1.5,middle:1.5,ring:1.5,pinky:1.5,rz:0},
255
+ n:{thumb:.8, index:1.5,middle:1.5,ring:1.5,pinky:1.5,rz:0},
256
+ o:{thumb:.5, index:.8, middle:.8, ring:.8, pinky:.8, rz:0},
257
+ p:{thumb:.5, index:0, middle:0, ring:1.5,pinky:1.5,rz:-.78},
258
+ q:{thumb:0, index:0, middle:1.5,ring:1.5,pinky:1.5,rz:-1.57},
259
+ r:{thumb:1.2,index:0, middle:0, ring:1.5,pinky:1.5,rz:0},
260
+ s:{thumb:.5, index:1.5,middle:1.5,ring:1.5,pinky:1.5,rz:0},
261
+ t:{thumb:.8, index:1.5,middle:1.5,ring:1.5,pinky:1.5,rz:0},
262
+ u:{thumb:1.2,index:0, middle:0, ring:1.5,pinky:1.5,rz:0},
263
+ v:{thumb:1.2,index:0, middle:0, ring:1.5,pinky:1.5,rz:0},
264
+ w:{thumb:1.2,index:0, middle:0, ring:0, pinky:1.5,rz:0},
265
+ x:{thumb:1.5,index:.8, middle:1.5,ring:1.5,pinky:1.5,rz:0},
266
+ y:{thumb:0, index:1.5,middle:1.5,ring:1.5,pinky:0, rz:0},
267
+ z:{thumb:1.5,index:0, middle:1.5,ring:1.5,pinky:1.5,rz:0},
268
+ };
269
+ const DESC={
270
+ a:'Fist with thumb at side',b:'Four fingers straight up',c:'Curved C shape',
271
+ d:'Index pointing up',e:'All fingers curled',f:'OK sign variant',
272
+ g:'Index + thumb sideways',h:'Two fingers horizontal',i:'Pinky raised',
273
+ j:'Pinky traces J',k:'V with thumb between',l:'L shape',
274
+ m:'Three fingers over thumb',n:'Two fingers over thumb',o:'O ring shape',
275
+ p:'K pointing down',q:'G pointing down',r:'Fingers crossed',
276
+ s:'Fist thumb across',t:'Thumb between fingers',u:'Two fingers together',
277
+ v:'Peace sign',w:'Three fingers wide',x:'Index hooked',
278
+ y:'Hang loose',z:'Index traces Z'
279
+ };
280
+ const REST={thumb:.05,index:.05,middle:.05,ring:.05,pinky:.05,rz:0};
281
+ const SKIN=0xfdb4c4,JOINT=0xf5a3b5;
282
+
283
+ // ══════════════════════════════════════════════════════════
284
+ // THREE.JS (unchanged)
285
+ // ══════════════════════════════════════════════════════════
286
+ let scene,cam,ren,hg,fing={};
287
+ let autoSpin=false;
288
+ let cC={thumb:.05,index:.05,middle:.05,ring:.05,pinky:.05};
289
+ let tC={...cC},cRZ=0,tRZ=0;
290
+ let aRun=false,aCan=false;
291
+
292
+ function initThree(){
293
+ const cv=document.getElementById('hc');
294
+ const cp=document.getElementById('cvp');
295
+ scene=new THREE.Scene();
296
+ scene.background=new THREE.Color(0x0d1117);
297
+ scene.fog=new THREE.FogExp2(0x0d1117,.022);
298
+ const W=cp.clientWidth,H=cp.clientHeight;
299
+ cam=new THREE.PerspectiveCamera(50,W/H,.1,1000);
300
+ cam.position.set(0,0,8);cam.lookAt(0,0,0);
301
+ ren=new THREE.WebGLRenderer({canvas:cv,antialias:true,powerPreference:'low-power'});
302
+ ren.setSize(W,H,false);ren.setPixelRatio(Math.min(devicePixelRatio,1.5));
303
+ ren.shadowMap.enabled=true;ren.info.autoReset=false;
304
+ scene.add(new THREE.AmbientLight(0xffffff,.65));
305
+ const kl=new THREE.DirectionalLight(0xfff4e0,.85);kl.position.set(5,10,7);kl.castShadow=true;scene.add(kl);
306
+ const fl=new THREE.DirectionalLight(0xd0e8ff,.3);fl.position.set(-6,2,4);scene.add(fl);
307
+ const rl=new THREE.DirectionalLight(0xffddbb,.2);rl.position.set(1,-4,-5);scene.add(rl);
308
+ buildHand();loop();
309
+ }
310
+ function mkFinger(cfg){
311
+ const fg=new THREE.Group();fg.userData={segs:[]};
312
+ const sl=cfg.length/3;let cy=0;
313
+ for(let i=0;i<3;i++){
314
+ const sg=new THREE.Group();const r=cfg.thickness*(1-i*.15);
315
+ const seg=new THREE.Mesh(new THREE.CylinderGeometry(r,r*.9,sl,12),new THREE.MeshStandardMaterial({color:SKIN,roughness:.7}));
316
+ seg.position.y=sl/2;seg.castShadow=true;sg.add(seg);
317
+ if(i<2){const jt=new THREE.Mesh(new THREE.SphereGeometry(r*.7,10,10),new THREE.MeshStandardMaterial({color:JOINT}));jt.position.y=sl;sg.add(jt);}
318
+ sg.position.y=cy;cy+=sl;
319
+ i===0?fg.add(sg):fg.userData.segs[i-1].add(sg);
320
+ fg.userData.segs.push(sg);
321
+ }
322
+ return fg;
323
+ }
324
+ function buildHand(){
325
+ hg=new THREE.Group();
326
+ hg.add(new THREE.Mesh(new THREE.BoxGeometry(1.8,2.5,.6),new THREE.MeshStandardMaterial({color:SKIN,roughness:.7})));
327
+ [{name:'thumb',pos:[-1.1,.5,.15],rot:[0,0,-.5],length:1.2,thickness:.35},
328
+ {name:'index',pos:[-.7,1.4,0],rot:[0,0,.1],length:1.8,thickness:.28},
329
+ {name:'middle',pos:[0,1.5,0],rot:[0,0,0],length:2.0,thickness:.28},
330
+ {name:'ring',pos:[.7,1.4,0],rot:[0,0,-.1],length:1.7,thickness:.26},
331
+ {name:'pinky',pos:[1.1,1.0,0],rot:[0,0,-.3],length:1.3,thickness:.24}].forEach(d=>{
332
+ const f=mkFinger(d);f.position.set(...d.pos);f.rotation.set(...d.rot);hg.add(f);fing[d.name]=f;
333
+ });
334
+ scene.add(hg);
335
+ }
336
+ function bend(name,amt){
337
+ const f=fing[name];if(!f)return;
338
+ f.userData.segs.forEach((s,i)=>{if(s)s.rotation.x=[amt*.4,amt*.6,amt*.8][i];});
339
+ }
340
+ function setPose(p){
341
+ ['thumb','index','middle','ring','pinky'].forEach(k=>{tC[k]=p[k]??0.05;});
342
+ tRZ=p.rz||0;
343
+ }
344
+ function loop(){
345
+ requestAnimationFrame(loop);
346
+ const t=Date.now()*.001;
347
+ if(!aRun){hg.position.y=Math.sin(t*.45)*.18;hg.rotation.y=Math.sin(t*.18)*.05;}
348
+ else{hg.rotation.y=0;}
349
+ ['thumb','index','middle','ring','pinky'].forEach(k=>{cC[k]+=(tC[k]-cC[k])*.13;bend(k,cC[k]);});
350
+ cRZ+=(tRZ-cRZ)*.13;hg.rotation.z=cRZ;
351
+ if(autoSpin){cam.position.x=Math.sin(t*.45)*8;cam.position.z=Math.cos(t*.45)*8;cam.lookAt(0,0,0);}
352
+ ren.render(scene,cam);
353
+ }
354
+ function setView(v,btn){
355
+ document.querySelectorAll('.vb').forEach(b=>b.classList.remove('on'));btn.classList.add('on');
356
+ autoSpin=(v==='spin');if(autoSpin)return;
357
+ const tp={front:[0,0,8],side:[8,0,0],top:[0,8,3]}[v]||[0,0,8];
358
+ animCam(...tp);
359
+ }
360
+ function animCam(tx,ty,tz){
361
+ const sx=cam.position.x,sy=cam.position.y,sz=cam.position.z;
362
+ const s0=performance.now();
363
+ function f(n){const e=ease(Math.min((n-s0)/900,1));cam.position.set(lerp(sx,tx,e),lerp(sy,ty,e),lerp(sz,tz,e));cam.lookAt(0,0,0);if(e<1)requestAnimationFrame(f);}
364
+ requestAnimationFrame(f);
365
+ }
366
+ window.addEventListener('resize',()=>{
367
+ if(!ren)return;
368
+ const cp=document.getElementById('cvp');
369
+ cam.aspect=cp.clientWidth/cp.clientHeight;cam.updateProjectionMatrix();
370
+ ren.setSize(cp.clientWidth,cp.clientHeight,false);
371
+ });
372
+ document.addEventListener('visibilitychange',()=>{if(document.hidden)aCan=true;});
373
+
374
+ // ══════════════════════════════════════════════════════════
375
+ // SPEECH RECOGNITION β€” ROBUST LOCAL + CLOUD HANDLER (unchanged)
376
+ // ══════════════════════════════════════════════════════════
377
+ let recog=null,isRec=false,micAllowed=false;
378
+
379
+ function isMicSafe(){
380
+ const isLocalhost=location.hostname==='localhost'||location.hostname==='127.0.0.1';
381
+ const isHttps=location.protocol==='https:';
382
+ return (isLocalhost||isHttps);
383
+ }
384
+
385
+ function showMicWarn(html){
386
+ const w=document.getElementById('mwarn');
387
+ w.innerHTML=html;w.classList.add('show');
388
+ }
389
+
390
+ function initSpeech(){
391
+ const SR=window.SpeechRecognition||window.webkitSpeechRecognition;
392
+ const btn=document.getElementById('bspk');
393
+
394
+ if(!SR){
395
+ btn.disabled=true;
396
+ btn.title='Use Chrome or Edge for voice input';
397
+ showMicWarn(
398
+ 'πŸ”‡ <strong>Voice input requires Chrome or Edge.</strong><br>'+
399
+ 'Firefox and Safari do not support the Web Speech API.<br>'+
400
+ 'You can still <strong>type text</strong> above and press Convert.'
401
+ );
402
+ return;
403
+ }
404
+
405
+ if(!isMicSafe()){
406
+ btn.disabled=true;
407
+ btn.title='Mic blocked on HTTP β€” HTTPS required on cloud';
408
+ showMicWarn(
409
+ 'πŸ”’ <strong>Microphone is blocked on plain HTTP.</strong><br>'+
410
+ 'On your Oracle Cloud server, enable HTTPS to use voice input.<br>'+
411
+ 'Until then, <strong>type your text</strong> above β€” Convert always works.'
412
+ );
413
+ return;
414
+ }
415
+
416
+ recog=new SR();
417
+ recog.lang='en-IN';
418
+ recog.continuous=false;
419
+ recog.interimResults=false;
420
+ recog.maxAlternatives=3;
421
+
422
+ recog.onstart=()=>{
423
+ setBadge('🎀 Listening…',true);
424
+ setSt('rec','Listening… speak clearly in English');
425
+ setMicStat(true,'Listening β€” speak now');
426
+ document.getElementById('bspk').classList.add('listening');
427
+ };
428
+
429
+ recog.onspeechstart=()=>{
430
+ setMicStat(true,'Speech detected…');
431
+ setSt('rec','Processing your speech…');
432
+ };
433
+
434
+ recog.onresult=ev=>{
435
+ let best='',bestConf=0;
436
+ const results=ev.results[0];
437
+ for(let i=0;i<results.length;i++){
438
+ if(results[i].confidence>=bestConf){bestConf=results[i].confidence;best=results[i].transcript;}
439
+ }
440
+ const t=best.trim();
441
+ if(!t){resetSpk();setSt('err','No speech detected β€” try again');return;}
442
+ document.getElementById('txt').value=t;
443
+ document.getElementById('cc').textContent=t.length+'/200';
444
+ setSt('act','Got: "'+t+'" ('+Math.round(bestConf*100)+'% conf)');
445
+ setMicStat(false,'');
446
+ isRec=false;resetSpk();
447
+ ST.sp++;ST.wd+=t.trim().split(/\s+/).length;updSt();
448
+ const seq=buildSeq(t);renderLL(seq);
449
+ ST.cv++;updSt();
450
+ setTimeout(()=>play(seq),300);
451
+ };
452
+
453
+ recog.onnomatch=()=>{
454
+ isRec=false;resetSpk();
455
+ setMicStat(false,'');
456
+ setSt('err','No match β€” speak more clearly or type instead');
457
+ };
458
+
459
+ recog.onerror=e=>{
460
+ isRec=false;resetSpk();setMicStat(false,'');
461
+ const msgs={
462
+ 'not-allowed' :'πŸ”’ Mic permission denied β€” click the πŸ”’ icon in the address bar to allow',
463
+ 'no-speech' :'No speech detected β€” try speaking louder',
464
+ 'network' :'Network error β€” check your internet connection',
465
+ 'aborted' :'Recording cancelled',
466
+ 'audio-capture':'No microphone found β€” check your device settings',
467
+ 'service-not-allowed':'Speech service blocked β€” use HTTPS on cloud',
468
+ };
469
+ setSt('err',msgs[e.error]||'Speech error: '+e.error);
470
+ if(e.error==='not-allowed'){
471
+ showMicWarn(
472
+ 'πŸ”’ <strong>Microphone permission was denied.</strong><br>'+
473
+ 'Click the πŸ”’ padlock icon in your browser address bar,<br>'+
474
+ 'set Microphone to <strong>Allow</strong>, then refresh the page.'
475
+ );
476
+ document.getElementById('bspk').disabled=true;
477
+ }
478
+ };
479
+
480
+ recog.onend=()=>{
481
+ if(isRec){isRec=false;resetSpk();}
482
+ setMicStat(false,'');
483
+ };
484
+ }
485
+
486
+ function setMicStat(show,msg){
487
+ const el=document.getElementById('mstat');
488
+ const tx=document.getElementById('mstxt');
489
+ if(show){el.classList.add('show','listening');tx.textContent=msg;}
490
+ else{el.classList.remove('show','listening');}
491
+ }
492
+
493
+ function resetSpk(){
494
+ const b=document.getElementById('bspk');
495
+ b.disabled=false;b.classList.remove('listening');
496
+ setBadge('● Ready',false);
497
+ }
498
+
499
+ // ══════════════════════════════════════════════════════════
500
+ // SEQUENCE + PLAYBACK (unchanged)
501
+ // ══════════════════════════════════════════════════════════
502
+ function buildSeq(text){
503
+ const out=[];
504
+ for(const ch of text.toLowerCase().replace(/[^a-z\s]/g,'')){
505
+ if(ch===' ')out.push({isSpace:true,name:'SPACE',desc:'Pause between words'});
506
+ else if(ISL[ch])out.push({letter:ch,name:ch.toUpperCase(),pose:ISL[ch],desc:DESC[ch]||''});
507
+ }
508
+ return out;
509
+ }
510
+
511
+ async function play(seq){
512
+ if(aRun)return;
513
+ aRun=true;aCan=false;
514
+ setBadge('● Signing',true);setSt('act','Fingerspelling…');
515
+ for(let i=0;i<seq.length&&!aCan;i++){
516
+ const item=seq[i];
517
+ document.querySelectorAll('.li').forEach(e=>e.classList.remove('on'));
518
+ const el=document.getElementById('li'+i);
519
+ if(el){el.classList.add('on');el.scrollIntoView({behavior:'smooth',block:'nearest'});}
520
+ document.getElementById('gl').textContent=item.isSpace?'Β·':item.name;
521
+ document.getElementById('gd').textContent=item.desc;
522
+ if(item.isSpace){setPose(REST);await slp(500);}
523
+ else{setPose(item.pose);await slp(880);}
524
+ if(aCan)break;
525
+ document.getElementById('gb').style.width=((i+1)/seq.length*100)+'%';
526
+ ST.lt++;updSt();
527
+ }
528
+ aRun=false;
529
+ if(!aCan){
530
+ setBadge('βœ“ Done',false);setSt('act','Complete!');setPose(REST);
531
+ setTimeout(()=>{
532
+ document.getElementById('gb').style.width='0%';
533
+ document.getElementById('gl').textContent='🀟';
534
+ document.getElementById('gd').textContent='Type or speak to begin';
535
+ setBadge('● Ready',false);setSt('','Ready');
536
+ },2000);
537
+ }
538
+ }
539
+
540
+ // ══════════════════════════════════════════════════════════
541
+ // BUTTON ACTIONS (unchanged)
542
+ // ══════════════════════════════════════════════════════════
543
+ const ST={cv:0,lt:0,sp:0,wd:0};
544
+
545
+ function doConvert(){
546
+ if(aRun)doStop();
547
+ const t=document.getElementById('txt').value.trim();
548
+ if(!t){setSt('err','Please enter some text first');return;}
549
+ const seq=buildSeq(t);renderLL(seq);
550
+ ST.cv++;ST.wd+=t.split(/\s+/).length;updSt();
551
+ play(seq);
552
+ }
553
+
554
+ function doSpeak(){
555
+ if(!recog){setSt('err','Voice input not available β€” see instructions above');return;}
556
+ if(isRec)return;
557
+ isRec=true;
558
+ document.getElementById('bspk').disabled=true;
559
+ try{
560
+ recog.start();
561
+ }catch(e){
562
+ isRec=false;resetSpk();
563
+ setSt('err','Please wait a moment and try again');
564
+ }
565
+ }
566
+
567
+ function doStop(){
568
+ aCan=true;aRun=false;
569
+ if(isRec&&recog){try{recog.stop();}catch(e){}}
570
+ isRec=false;resetSpk();setMicStat(false,'');
571
+ document.getElementById('gb').style.width='0%';
572
+ setPose(REST);setBadge('⏹ Stopped',false);setSt('err','Stopped');
573
+ setTimeout(()=>{setSt('','Ready');setBadge('● Ready',false);},1400);
574
+ }
575
+
576
+ function doClear(){
577
+ doStop();
578
+ document.getElementById('txt').value='';
579
+ document.getElementById('cc').textContent='0/200';
580
+ renderLL([]);setSt('','Cleared');
581
+ setTimeout(()=>setSt('','Ready'),1200);
582
+ }
583
+
584
+ // ══════════════════════════════════════════════════════════
585
+ // UI HELPERS (unchanged)
586
+ // ══════════════════════════════════════════════════════════
587
+ function renderLL(seq){
588
+ const el=document.getElementById('ll');
589
+ if(!seq.length){el.innerHTML='<div class="es">Type text or speak<br>to see the sequence</div>';document.getElementById('scnt').textContent='0 letters';return;}
590
+ let n=0;
591
+ el.innerHTML=seq.map((x,i)=>{
592
+ if(x.isSpace)return`<div class="li sp" id="li${i}"><span class="lc">⎡</span><span class="lh">Word space</span></div>`;
593
+ n++;return`<div class="li" id="li${i}"><span class="lc">${x.name}</span><span class="lh">${x.desc}</span></div>`;
594
+ }).join('');
595
+ document.getElementById('scnt').textContent=n+' letter'+(n!==1?'s':'');
596
+ }
597
+ function setSt(cls,msg){
598
+ document.getElementById('stxt').textContent=msg;
599
+ document.getElementById('st').className='st'+(cls?' '+cls:'');
600
+ }
601
+ function setBadge(msg,blink){
602
+ const b=document.getElementById('badge');b.textContent=msg;
603
+ b.style.animation=blink?'pd .9s infinite':'none';
604
+ }
605
+ function updSt(){
606
+ [ST.cv,ST.lt,ST.sp,ST.wd].forEach((v,i)=>document.getElementById('s'+i).textContent=v);
607
+ }
608
+ function lerp(a,b,t){return a+(b-a)*t;}
609
+ function ease(t){return t<.5?4*t*t*t:1-Math.pow(-2*t+2,3)/2;}
610
+ function slp(ms){return new Promise(r=>setTimeout(r,ms));}
611
+
612
+ // ══════════════════════════════════════════════════════════
613
+ // FIX: Nav links β€” dynamic URL so redirect works on any domain
614
+ // Local dev β†’ uses explicit port numbers (:5000 / :5001)
615
+ // Cloud/Nginx β†’ uses path-based routing (/ and /avatar/)
616
+ // ══════════════════════════════════════════════════════════
617
+ (function(){
618
+ const proto = location.protocol;
619
+ const host = location.hostname;
620
+ const port = location.port;
621
+ if (port && port !== '80' && port !== '443') {
622
+ // Local development with explicit ports
623
+ document.getElementById('nav-gesture').href = proto + '//' + host + ':5000/';
624
+ document.getElementById('nav-avatar').href = proto + '//' + host + ':5001/';
625
+ } else {
626
+ // Cloud / Nginx β€” path-based routing
627
+ document.getElementById('nav-gesture').href = proto + '//' + host + '/';
628
+ document.getElementById('nav-avatar').href = proto + '//' + host + '/avatar/';
629
+ }
630
+ })();
631
+
632
+ // Keyboard shortcuts
633
+ document.addEventListener('keydown',e=>{
634
+ if((e.ctrlKey||e.metaKey)&&e.key==='Enter'){e.preventDefault();doConvert();}
635
+ if(e.key==='Escape')doStop();
636
+ if((e.ctrlKey||e.metaKey)&&e.key===' '){e.preventDefault();doSpeak();}
637
+ });
638
+ document.getElementById('txt').addEventListener('input',function(){
639
+ document.getElementById('cc').textContent=this.value.length+'/200';
640
+ });
641
+
642
+ // ══════════════════════════════════════════════════════════
643
+ // BOOT
644
+ // ══════════════════════════════════════════════════════════
645
+ window.addEventListener('DOMContentLoaded',()=>{
646
+ initThree();
647
+ initSpeech();
648
+ updSt();
649
+ setTimeout(()=>setPose(REST),400);
650
+ });
651
+ </script>
652
+ </body>
653
+ </html>
templates/dashboard.html ADDED
@@ -0,0 +1,1163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+
4
+ <head>
5
+ <meta charset="UTF-8" />
6
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
7
+ <title>ISL Gesture Recognition β€” AI Sign Language</title>
8
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.7.2/socket.io.min.js"></script>
9
+ <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
10
+ <link rel="preconnect" href="https://fonts.googleapis.com" />
11
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
12
+ <link
13
+ href="https://fonts.googleapis.com/css2?family=Playfair+Display:wght@400;600;700&family=DM+Sans:wght@300;400;500;600&family=DM+Mono:wght@400;500&display=swap"
14
+ rel="stylesheet" />
15
+ <style>
16
+ :root {
17
+ --bg: #0d1117;
18
+ --surf: #131922;
19
+ --card: #1a2233;
20
+ --bdr: rgba(212, 168, 83, .13);
21
+ --bdrm: rgba(212, 168, 83, .30);
22
+ --bdrh: rgba(212, 168, 83, .58);
23
+ --gold: #d4a853;
24
+ --goldb: #f0c76a;
25
+ --goldd: #a07c3a;
26
+ --sage: #7eb89a;
27
+ --saged: #4e7d66;
28
+ --txt: #f0ede6;
29
+ --txtm: #a8a096;
30
+ --txtl: #5a5650;
31
+ --red: #c0614a;
32
+ --r8: 8px;
33
+ --r14: 14px;
34
+ --fD: 'Playfair Display', Georgia, serif;
35
+ --fB: 'DM Sans', system-ui, sans-serif;
36
+ --fM: 'DM Mono', monospace;
37
+ }
38
+
39
+ *,
40
+ *::before,
41
+ *::after {
42
+ margin: 0;
43
+ padding: 0;
44
+ box-sizing: border-box;
45
+ }
46
+
47
+ html,
48
+ body {
49
+ height: 100%;
50
+ overflow: hidden;
51
+ }
52
+
53
+ body {
54
+ font-family: var(--fB);
55
+ background: var(--bg);
56
+ color: var(--txt);
57
+ -webkit-font-smoothing: antialiased;
58
+ display: flex;
59
+ flex-direction: column;
60
+ }
61
+
62
+ #app {
63
+ display: flex;
64
+ flex: 1;
65
+ overflow: hidden;
66
+ height: 100%;
67
+ }
68
+
69
+ #cam-pane {
70
+ flex: 1;
71
+ display: flex;
72
+ flex-direction: column;
73
+ min-width: 0;
74
+ background: var(--bg);
75
+ position: relative;
76
+ overflow: hidden;
77
+ }
78
+
79
+ .topbar {
80
+ display: flex;
81
+ align-items: center;
82
+ justify-content: space-between;
83
+ padding: 10px 16px 0;
84
+ flex-shrink: 0;
85
+ position: relative;
86
+ z-index: 5;
87
+ }
88
+
89
+ .brand {
90
+ display: flex;
91
+ align-items: baseline;
92
+ gap: 8px;
93
+ }
94
+
95
+ .brand-t {
96
+ font-family: var(--fD);
97
+ font-size: 1.1rem;
98
+ font-weight: 700;
99
+ color: var(--gold);
100
+ }
101
+
102
+ .brand-s {
103
+ font-size: .65rem;
104
+ color: var(--txtl);
105
+ text-transform: uppercase;
106
+ letter-spacing: .07em;
107
+ }
108
+
109
+ .live-badge {
110
+ font-size: .65rem;
111
+ font-family: var(--fM);
112
+ color: var(--sage);
113
+ background: rgba(126, 184, 154, .10);
114
+ border: 1px solid rgba(126, 184, 154, .25);
115
+ padding: 3px 9px;
116
+ border-radius: 20px;
117
+ }
118
+
119
+ @keyframes pd {
120
+
121
+ 0%,
122
+ 100% {
123
+ opacity: 1;
124
+ transform: scale(1)
125
+ }
126
+
127
+ 50% {
128
+ opacity: .6;
129
+ transform: scale(.8)
130
+ }
131
+ }
132
+
133
+ #camera-feed {
134
+ flex: 1;
135
+ width: 100%;
136
+ display: block;
137
+ object-fit: cover;
138
+ background: #000;
139
+ }
140
+
141
+ .cam-overlay {
142
+ position: absolute;
143
+ bottom: 0;
144
+ left: 0;
145
+ right: 0;
146
+ padding: 12px 18px 14px;
147
+ background: linear-gradient(to top, rgba(13, 17, 23, 1) 55%, transparent);
148
+ display: flex;
149
+ align-items: flex-end;
150
+ justify-content: space-between;
151
+ gap: 10px;
152
+ pointer-events: none;
153
+ z-index: 5;
154
+ }
155
+
156
+ .co-letter {
157
+ font-family: var(--fD);
158
+ font-size: clamp(2rem, 5vw, 3.2rem);
159
+ font-weight: 700;
160
+ color: var(--goldb);
161
+ line-height: 1;
162
+ }
163
+
164
+ .co-word {
165
+ font-size: .85rem;
166
+ font-weight: 600;
167
+ color: var(--gold);
168
+ font-family: var(--fM);
169
+ letter-spacing: .05em;
170
+ }
171
+
172
+ .co-desc {
173
+ font-size: .72rem;
174
+ color: var(--txtm);
175
+ text-align: right;
176
+ max-width: 200px;
177
+ line-height: 1.45;
178
+ }
179
+
180
+ .co-progress {
181
+ position: absolute;
182
+ bottom: 0;
183
+ left: 0;
184
+ right: 0;
185
+ height: 3px;
186
+ background: rgba(212, 168, 83, .08);
187
+ }
188
+
189
+ .co-bar {
190
+ height: 100%;
191
+ width: 0%;
192
+ background: linear-gradient(90deg, var(--goldd), var(--goldb));
193
+ transition: width .3s ease;
194
+ position: relative;
195
+ overflow: hidden;
196
+ }
197
+
198
+ .co-bar::after {
199
+ content: '';
200
+ position: absolute;
201
+ inset: 0;
202
+ background: linear-gradient(90deg, transparent, rgba(255, 255, 255, .3), transparent);
203
+ animation: shimmer 1.8s infinite;
204
+ }
205
+
206
+ @keyframes shimmer {
207
+ from {
208
+ transform: translateX(-100%)
209
+ }
210
+
211
+ to {
212
+ transform: translateX(200%)
213
+ }
214
+ }
215
+
216
+ .cam-placeholder {
217
+ position: absolute;
218
+ inset: 0;
219
+ display: flex;
220
+ flex-direction: column;
221
+ align-items: center;
222
+ justify-content: center;
223
+ gap: 10px;
224
+ color: var(--txtl);
225
+ font-size: .8rem;
226
+ z-index: 2;
227
+ pointer-events: none;
228
+ }
229
+
230
+ .cam-placeholder .ph-icon {
231
+ font-size: 2.8rem;
232
+ opacity: .25;
233
+ }
234
+
235
+ #ctrl-pane {
236
+ width: 340px;
237
+ flex-shrink: 0;
238
+ display: flex;
239
+ flex-direction: column;
240
+ background: var(--surf);
241
+ border-left: 1px solid var(--bdr);
242
+ overflow-y: auto;
243
+ overflow-x: hidden;
244
+ }
245
+
246
+ #ctrl-pane::-webkit-scrollbar {
247
+ width: 4px;
248
+ }
249
+
250
+ #ctrl-pane::-webkit-scrollbar-thumb {
251
+ background: var(--bdrm);
252
+ border-radius: 4px;
253
+ }
254
+
255
+ @media (max-width: 720px) {
256
+
257
+ html,
258
+ body {
259
+ overflow: hidden;
260
+ }
261
+
262
+ #app {
263
+ flex-direction: column;
264
+ height: 100%;
265
+ }
266
+
267
+ #cam-pane {
268
+ flex: 0 0 46vh;
269
+ min-height: 260px;
270
+ max-height: 46vh;
271
+ width: 100%;
272
+ }
273
+
274
+ #ctrl-pane {
275
+ width: 100%;
276
+ flex: 1;
277
+ border-left: none;
278
+ border-top: 1px solid var(--bdr);
279
+ overflow-y: auto;
280
+ }
281
+
282
+ .topbar {
283
+ padding: 8px 12px 0;
284
+ }
285
+
286
+ .brand-t {
287
+ font-size: .95rem;
288
+ }
289
+ }
290
+
291
+ .cpanel {
292
+ padding: 14px 16px;
293
+ display: flex;
294
+ flex-direction: column;
295
+ gap: 12px;
296
+ }
297
+
298
+ .sec-label {
299
+ font-size: .65rem;
300
+ font-weight: 600;
301
+ text-transform: uppercase;
302
+ letter-spacing: .08em;
303
+ color: var(--txtl);
304
+ display: flex;
305
+ align-items: center;
306
+ gap: 6px;
307
+ font-family: var(--fM);
308
+ }
309
+
310
+ .status {
311
+ display: flex;
312
+ align-items: center;
313
+ gap: 7px;
314
+ padding: 7px 13px;
315
+ border-radius: 30px;
316
+ font-size: .73rem;
317
+ font-weight: 500;
318
+ border: 1px solid var(--bdr);
319
+ background: var(--card);
320
+ color: var(--txtm);
321
+ transition: all .3s;
322
+ min-height: 34px;
323
+ font-family: var(--fM);
324
+ }
325
+
326
+ .sdot {
327
+ width: 7px;
328
+ height: 7px;
329
+ border-radius: 50%;
330
+ background: var(--txtl);
331
+ flex-shrink: 0;
332
+ transition: all .3s;
333
+ }
334
+
335
+ .status.act .sdot {
336
+ background: var(--sage);
337
+ box-shadow: 0 0 8px var(--sage);
338
+ animation: pd 1.4s infinite;
339
+ }
340
+
341
+ .status.act {
342
+ border-color: rgba(126, 184, 154, .3);
343
+ color: var(--sage);
344
+ }
345
+
346
+ .status.err .sdot {
347
+ background: var(--red);
348
+ }
349
+
350
+ .status.err {
351
+ border-color: rgba(192, 97, 74, .3);
352
+ color: var(--red);
353
+ }
354
+
355
+ .text-display {
356
+ background: var(--card);
357
+ border: 1px solid var(--bdr);
358
+ border-radius: var(--r14);
359
+ padding: 10px 13px;
360
+ min-height: 52px;
361
+ }
362
+
363
+ .cur-word {
364
+ font-size: 1.15rem;
365
+ font-weight: 700;
366
+ color: var(--goldb);
367
+ font-family: var(--fD);
368
+ line-height: 1.2;
369
+ margin-bottom: 4px;
370
+ }
371
+
372
+ .cur-sentence {
373
+ font-size: .75rem;
374
+ color: var(--txtm);
375
+ line-height: 1.5;
376
+ font-family: var(--fM);
377
+ }
378
+
379
+ .btn-row {
380
+ display: grid;
381
+ grid-template-columns: repeat(3, 1fr);
382
+ gap: 7px;
383
+ }
384
+
385
+ .btn {
386
+ display: flex;
387
+ align-items: center;
388
+ justify-content: center;
389
+ gap: 5px;
390
+ padding: 9px 6px;
391
+ border: 1px solid transparent;
392
+ border-radius: var(--r14);
393
+ font-family: var(--fB);
394
+ font-size: .76rem;
395
+ font-weight: 600;
396
+ cursor: pointer;
397
+ transition: all .22s;
398
+ white-space: nowrap;
399
+ overflow: hidden;
400
+ }
401
+
402
+ .btn:active {
403
+ transform: scale(.96);
404
+ }
405
+
406
+ .btn:disabled {
407
+ opacity: .4;
408
+ cursor: not-allowed;
409
+ }
410
+
411
+ .btn-speak {
412
+ background: linear-gradient(135deg, #4e7d66, #7eb89a);
413
+ color: #0d1117;
414
+ border-color: rgba(126, 184, 154, .35);
415
+ }
416
+
417
+ .btn-speak:hover {
418
+ box-shadow: 0 4px 16px rgba(126, 184, 154, .3);
419
+ }
420
+
421
+ .btn-back {
422
+ background: transparent;
423
+ color: var(--txtm);
424
+ border-color: var(--bdr);
425
+ }
426
+
427
+ .btn-back:hover {
428
+ border-color: var(--bdrm);
429
+ color: var(--txt);
430
+ }
431
+
432
+ .btn-reset {
433
+ background: transparent;
434
+ color: var(--red);
435
+ border-color: rgba(192, 97, 74, .3);
436
+ }
437
+
438
+ .btn-reset:hover {
439
+ background: rgba(192, 97, 74, .08);
440
+ border-color: rgba(192, 97, 74, .55);
441
+ }
442
+
443
+ .divider {
444
+ height: 1px;
445
+ background: var(--bdr);
446
+ margin: 0 -16px;
447
+ }
448
+
449
+ .emotion-header {
450
+ display: flex;
451
+ justify-content: space-between;
452
+ align-items: center;
453
+ margin-bottom: 8px;
454
+ }
455
+
456
+ .emotion-big {
457
+ font-family: var(--fD);
458
+ font-size: 1.1rem;
459
+ font-weight: 700;
460
+ color: var(--goldb);
461
+ }
462
+
463
+ .emotion-icon {
464
+ font-size: 1.5rem;
465
+ animation: emojiPop 2.2s ease infinite;
466
+ }
467
+
468
+ @keyframes emojiPop {
469
+
470
+ 0%,
471
+ 100% {
472
+ transform: scale(1)
473
+ }
474
+
475
+ 50% {
476
+ transform: scale(1.12)
477
+ }
478
+ }
479
+
480
+ .emo-bars {
481
+ display: flex;
482
+ flex-direction: column;
483
+ gap: 5px;
484
+ }
485
+
486
+ .emo-bar {
487
+ display: flex;
488
+ align-items: center;
489
+ gap: 8px;
490
+ }
491
+
492
+ .ebo-label {
493
+ width: 78px;
494
+ font-size: .72rem;
495
+ color: var(--txtm);
496
+ flex-shrink: 0;
497
+ }
498
+
499
+ .ebo-track {
500
+ flex: 1;
501
+ height: 20px;
502
+ background: rgba(26, 34, 51, .8);
503
+ border-radius: 10px;
504
+ overflow: hidden;
505
+ border: 1px solid var(--bdr);
506
+ }
507
+
508
+ .ebo-fill {
509
+ height: 100%;
510
+ transition: width .5s cubic-bezier(.4, 0, .2, 1);
511
+ display: flex;
512
+ align-items: center;
513
+ justify-content: flex-end;
514
+ padding-right: 6px;
515
+ font-size: .62rem;
516
+ font-weight: 700;
517
+ font-family: var(--fM);
518
+ color: rgba(0, 0, 0, .75);
519
+ min-width: 28px;
520
+ }
521
+
522
+ .sug-detected {
523
+ font-size: .65rem;
524
+ font-family: var(--fM);
525
+ color: var(--gold);
526
+ background: rgba(212, 168, 83, .07);
527
+ border: 1px solid var(--bdr);
528
+ border-radius: var(--r8);
529
+ padding: 4px 10px;
530
+ margin-bottom: 5px;
531
+ letter-spacing: .04em;
532
+ }
533
+
534
+ .sug-list {
535
+ display: flex;
536
+ flex-direction: column;
537
+ gap: 4px;
538
+ }
539
+
540
+ .sug-item {
541
+ display: flex;
542
+ align-items: center;
543
+ gap: 9px;
544
+ padding: 7px 11px;
545
+ border-radius: var(--r8);
546
+ background: var(--card);
547
+ border: 1px solid var(--bdr);
548
+ cursor: pointer;
549
+ transition: all .2s;
550
+ font-size: .82rem;
551
+ color: var(--txt);
552
+ }
553
+
554
+ .sug-item:hover {
555
+ background: rgba(212, 168, 83, .08);
556
+ border-color: var(--bdrm);
557
+ transform: translateX(3px);
558
+ color: var(--goldb);
559
+ }
560
+
561
+ .sug-n {
562
+ font-family: var(--fM);
563
+ font-size: .65rem;
564
+ font-weight: 700;
565
+ color: var(--gold);
566
+ background: rgba(212, 168, 83, .12);
567
+ border: 1px solid rgba(212, 168, 83, .2);
568
+ width: 20px;
569
+ height: 20px;
570
+ display: flex;
571
+ align-items: center;
572
+ justify-content: center;
573
+ border-radius: 5px;
574
+ flex-shrink: 0;
575
+ }
576
+
577
+ .sug-empty {
578
+ text-align: center;
579
+ padding: 16px 0;
580
+ color: var(--txtl);
581
+ font-size: .75rem;
582
+ line-height: 1.7;
583
+ }
584
+
585
+ #emotionChart {
586
+ height: 160px !important;
587
+ }
588
+
589
+ .stat-grid {
590
+ display: grid;
591
+ grid-template-columns: repeat(3, 1fr);
592
+ gap: 6px;
593
+ }
594
+
595
+ .stat-tile {
596
+ background: var(--card);
597
+ border: 1px solid var(--bdr);
598
+ border-radius: var(--r8);
599
+ padding: 9px 10px;
600
+ text-align: center;
601
+ transition: border-color .25s;
602
+ }
603
+
604
+ .stat-tile:hover {
605
+ border-color: var(--bdrm);
606
+ }
607
+
608
+ .sl {
609
+ font-size: .55rem;
610
+ text-transform: uppercase;
611
+ letter-spacing: .07em;
612
+ color: var(--txtl);
613
+ font-weight: 600;
614
+ margin-bottom: 3px;
615
+ font-family: var(--fM);
616
+ }
617
+
618
+ .sv {
619
+ font-family: var(--fD);
620
+ font-size: 1.3rem;
621
+ font-weight: 700;
622
+ color: var(--gold);
623
+ line-height: 1;
624
+ }
625
+
626
+ .about-txt {
627
+ font-size: .75rem;
628
+ color: var(--txtm);
629
+ line-height: 1.75;
630
+ }
631
+
632
+ .about-txt p {
633
+ margin-bottom: 7px;
634
+ }
635
+
636
+ .alist {
637
+ list-style: none;
638
+ margin-bottom: 7px;
639
+ }
640
+
641
+ .alist li {
642
+ padding-left: 13px;
643
+ position: relative;
644
+ font-size: .73rem;
645
+ padding-bottom: 2px;
646
+ }
647
+
648
+ .alist li::before {
649
+ content: 'β€Ί';
650
+ position: absolute;
651
+ left: 0;
652
+ color: var(--gold);
653
+ font-weight: 700;
654
+ }
655
+
656
+ .acredit {
657
+ margin-top: 10px;
658
+ padding-top: 10px;
659
+ border-top: 1px solid var(--bdr);
660
+ }
661
+
662
+ .aname {
663
+ font-family: var(--fD);
664
+ color: var(--gold);
665
+ font-weight: 600;
666
+ font-size: .88rem;
667
+ }
668
+
669
+ .arole {
670
+ font-size: .65rem;
671
+ color: var(--txtl);
672
+ margin: 2px 0 3px;
673
+ font-family: var(--fM);
674
+ }
675
+
676
+ .aemail {
677
+ color: var(--sage);
678
+ font-size: .66rem;
679
+ font-family: var(--fM);
680
+ }
681
+
682
+ .nav-row {
683
+ display: flex;
684
+ gap: 7px;
685
+ padding: 0 16px 20px;
686
+ }
687
+
688
+ .nav-a {
689
+ flex: 1;
690
+ display: flex;
691
+ align-items: center;
692
+ justify-content: center;
693
+ gap: 6px;
694
+ padding: 12px 8px;
695
+ border-radius: var(--r14);
696
+ border: 1px solid var(--bdr);
697
+ background: var(--card);
698
+ color: var(--txtm);
699
+ text-decoration: none;
700
+ font-size: .76rem;
701
+ font-weight: 600;
702
+ transition: all .25s;
703
+ }
704
+
705
+ .nav-a:hover {
706
+ border-color: var(--bdrm);
707
+ color: var(--txt);
708
+ }
709
+
710
+ .nav-a.on {
711
+ background: rgba(212, 168, 83, .10);
712
+ border-color: var(--bdrh);
713
+ color: var(--gold);
714
+ }
715
+ </style>
716
+ </head>
717
+
718
+ <body>
719
+
720
+ <div id="app">
721
+
722
+ <div id="cam-pane">
723
+ <div class="topbar">
724
+ <div class="brand">
725
+ <span class="brand-t">ISL Gesture Recognition</span>
726
+ <span class="brand-s">Gesture Β· Emotion Β· AI</span>
727
+ </div>
728
+ <span class="live-badge" id="live-badge">● Ready</span>
729
+ </div>
730
+
731
+ <img id="camera-feed" src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"
732
+ alt="Camera Feed" />
733
+
734
+ <div class="cam-placeholder" id="cam-ph">
735
+ <span class="ph-icon">πŸ“·</span>
736
+ <span style="font-family:var(--fM);font-size:.75rem">Connecting to camera…</span>
737
+ </div>
738
+
739
+ <div class="cam-overlay">
740
+ <div>
741
+ <div style="display:flex;align-items:baseline;gap:8px">
742
+ <span id="co-letter" class="co-letter">🀟</span>
743
+ <span id="co-word" class="co-word"></span>
744
+ </div>
745
+ </div>
746
+ <div id="co-desc" class="co-desc">Point camera at your hand</div>
747
+ <div class="co-progress">
748
+ <div class="co-bar" id="co-bar"></div>
749
+ </div>
750
+ </div>
751
+ </div>
752
+
753
+ <div id="ctrl-pane">
754
+ <div class="cpanel">
755
+
756
+ <div class="status" id="status">
757
+ <div class="sdot"></div>
758
+ <span id="stxt">Ready for detection</span>
759
+ </div>
760
+
761
+ <div class="text-display">
762
+ <div class="cur-word" id="cur-word">β€”</div>
763
+ <div class="cur-sentence" id="cur-sentence"></div>
764
+ </div>
765
+
766
+ <div class="btn-row">
767
+ <button class="btn btn-speak" onclick="sendCmd('speak')">πŸ”Š Speak</button>
768
+ <button class="btn btn-back" onclick="sendCmd('backspace')">⌫ Back</button>
769
+ <button class="btn btn-reset" onclick="sendCmd('reset')">πŸ”„ Reset</button>
770
+ </div>
771
+
772
+ <div class="divider"></div>
773
+
774
+ <div class="sec-label">😊 Emotion Detection</div>
775
+ <div class="emotion-header">
776
+ <span class="emotion-big" id="emo-label">Neutral</span>
777
+ <span class="emotion-icon" id="emo-icon">😐</span>
778
+ </div>
779
+
780
+ <div class="emo-bars">
781
+ <div class="emo-bar">
782
+ <div class="ebo-label">😊 Happy</div>
783
+ <div class="ebo-track">
784
+ <div class="ebo-fill" id="bar-happy" style="width:0%;background:linear-gradient(90deg,#3a8a5c,#7eb89a)">0%
785
+ </div>
786
+ </div>
787
+ </div>
788
+ <div class="emo-bar">
789
+ <div class="ebo-label">😒 Sad</div>
790
+ <div class="ebo-track">
791
+ <div class="ebo-fill" id="bar-sad" style="width:0%;background:linear-gradient(90deg,#3a5a8a,#6090c8)">0%
792
+ </div>
793
+ </div>
794
+ </div>
795
+ <div class="emo-bar">
796
+ <div class="ebo-label">😠 Angry</div>
797
+ <div class="ebo-track">
798
+ <div class="ebo-fill" id="bar-angry" style="width:0%;background:linear-gradient(90deg,#8a3a3a,#c06060)">0%
799
+ </div>
800
+ </div>
801
+ </div>
802
+ <div class="emo-bar">
803
+ <div class="ebo-label">😲 Surprise</div>
804
+ <div class="ebo-track">
805
+ <div class="ebo-fill" id="bar-surprise"
806
+ style="width:0%;background:linear-gradient(90deg,#8a6a1a,#d4a853)">0%</div>
807
+ </div>
808
+ </div>
809
+ <div class="emo-bar">
810
+ <div class="ebo-label">😐 Neutral</div>
811
+ <div class="ebo-track">
812
+ <div class="ebo-fill" id="bar-neutral"
813
+ style="width:100%;background:linear-gradient(90deg,#3a4455,#5a6878)">100%</div>
814
+ </div>
815
+ </div>
816
+ </div>
817
+
818
+ <div class="divider"></div>
819
+
820
+ <div class="sec-label">πŸ’‘ Word Suggestions</div>
821
+ <div id="sug-area">
822
+ <div class="sug-empty">Detected letters will appear<br>here with word suggestions</div>
823
+ </div>
824
+
825
+ <div class="divider"></div>
826
+
827
+ <div class="sec-label">πŸ“ˆ Emotion Timeline</div>
828
+ <canvas id="emotionChart"></canvas>
829
+
830
+ <div class="divider"></div>
831
+
832
+ <div class="sec-label">πŸ“Š Statistics</div>
833
+ <div class="stat-grid">
834
+ <div class="stat-tile">
835
+ <div class="sl">FPS</div>
836
+ <div class="sv" id="s-fps">0</div>
837
+ </div>
838
+ <div class="stat-tile">
839
+ <div class="sl">Letters</div>
840
+ <div class="sv" id="s-letters">0</div>
841
+ </div>
842
+ <div class="stat-tile">
843
+ <div class="sv" id="s-words">0</div>
844
+ <div class="sl">Words</div>
845
+ </div>
846
+ <div class="stat-tile">
847
+ <div class="sl">Sentences</div>
848
+ <div class="sv" id="s-sentences">0</div>
849
+ </div>
850
+ <div class="stat-tile">
851
+ <div class="sl">Emotions</div>
852
+ <div class="sv" id="s-emotions">0</div>
853
+ </div>
854
+ <div class="stat-tile">
855
+ <div class="sl">Conf.</div>
856
+ <div class="sv" id="s-conf">β€”</div>
857
+ </div>
858
+ </div>
859
+
860
+ <div class="divider"></div>
861
+
862
+ <div class="sec-label">ℹ️ About</div>
863
+ <div class="about-txt">
864
+ <p>AI Powered ISL Detection is an assistive AI system designed to improve communication between hearing and
865
+ deaf individuals.</p>
866
+ <p>This module focuses on real-time gesture and facial emotion recognition using computer vision.</p>
867
+ <p>Useful for:</p>
868
+ <ul class="alist">
869
+ <li>Deaf individuals</li>
870
+ <li>Inclusive classrooms</li>
871
+ <li>Assistive communication systems</li>
872
+ <li>Accessibility and AI research</li>
873
+ </ul>
874
+ <div class="acredit">
875
+ <div class="aname">Deepak Roshan</div>
876
+ <div class="arole">AI Engineer</div>
877
+ <div class="aemail">deepakroshan380@gmail.com</div>
878
+ </div>
879
+ </div>
880
+
881
+ </div>
882
+
883
+ <div class="nav-row">
884
+ <a href="#" class="nav-a on" id="nav-gesture">😊 Gesture Recognition</a>
885
+ <a href="#" class="nav-a" id="nav-avatar">🀟 Speech-to-Sign</a>
886
+ </div>
887
+ </div>
888
+
889
+ </div>
890
+
891
+ <script>
892
+ document.addEventListener('DOMContentLoaded', function () {
893
+
894
+ (function () {
895
+ var proto = location.protocol;
896
+ var host = location.hostname;
897
+ var port = location.port;
898
+ if (port && port !== '80' && port !== '443') {
899
+ document.getElementById('nav-gesture').href = proto + '//' + host + ':5000/';
900
+ document.getElementById('nav-avatar').href = proto + '//' + host + ':5001/';
901
+ } else {
902
+ document.getElementById('nav-gesture').href = proto + '//' + host + '/';
903
+ document.getElementById('nav-avatar').href = proto + '//' + host + '/avatar/';
904
+ }
905
+ })();
906
+
907
+ const EMO_ICONS = { happy: '😊', sad: '😒', angry: '😠', surprise: '😲', neutral: '😐' };
908
+ const EMO_KEYS = ['happy', 'sad', 'angry', 'surprise', 'neutral'];
909
+
910
+ const chartCtx = document.getElementById('emotionChart').getContext('2d');
911
+ const chart = new Chart(chartCtx, {
912
+ type: 'line',
913
+ data: {
914
+ labels: [],
915
+ datasets: [
916
+ { label: 'Happy', data: [], borderColor: '#7eb89a', borderWidth: 1.5, tension: .4, fill: false, pointRadius: 0 },
917
+ { label: 'Sad', data: [], borderColor: '#6090c8', borderWidth: 1.5, tension: .4, fill: false, pointRadius: 0 },
918
+ { label: 'Angry', data: [], borderColor: '#c06060', borderWidth: 1.5, tension: .4, fill: false, pointRadius: 0 },
919
+ { label: 'Surprise', data: [], borderColor: '#d4a853', borderWidth: 1.5, tension: .4, fill: false, pointRadius: 0 },
920
+ { label: 'Neutral', data: [], borderColor: '#5a6878', borderWidth: 1.5, tension: .4, fill: false, pointRadius: 0 },
921
+ ]
922
+ },
923
+ options: {
924
+ responsive: true,
925
+ maintainAspectRatio: false,
926
+ plugins: {
927
+ legend: { labels: { color: '#5a5650', font: { size: 9, family: "'DM Mono'" }, boxWidth: 10 } }
928
+ },
929
+ scales: {
930
+ y: {
931
+ beginAtZero: true, max: 1,
932
+ ticks: { color: '#5a5650', font: { size: 8 } },
933
+ grid: { color: 'rgba(212,168,83,.05)' }
934
+ },
935
+ x: {
936
+ ticks: { display: false },
937
+ grid: { color: 'rgba(212,168,83,.03)' }
938
+ }
939
+ },
940
+ animation: false
941
+ }
942
+ });
943
+
944
+ /* ── Browser Camera Streaming ──────────────────────────── */
945
+ let cameraStream = null;
946
+ const captureCanvas = document.createElement('canvas');
947
+ const captureCtx = captureCanvas.getContext('2d');
948
+
949
+ async function startBrowserCamera() {
950
+ if (cameraStream) return;
951
+ if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
952
+ console.error('Camera API not supported β€” requires HTTPS on cloud');
953
+ return;
954
+ }
955
+ try {
956
+ cameraStream = await navigator.mediaDevices.getUserMedia({
957
+ video: { width: { ideal: 640 }, height: { ideal: 480 }, facingMode: 'user' },
958
+ audio: false
959
+ });
960
+ const video = document.createElement('video');
961
+ video.srcObject = cameraStream;
962
+ video.autoplay = true;
963
+ video.playsInline = true;
964
+ video.addEventListener('loadeddata', () => {
965
+ captureCanvas.width = 320;
966
+ captureCanvas.height = 240;
967
+ if (!window.cameraInterval) {
968
+ window.cameraInterval = setInterval(() => {
969
+ if (document.visibilityState === 'visible' && socket && socket.connected) {
970
+ captureCtx.drawImage(video, 0, 0, 320, 240);
971
+ const frame = captureCanvas.toDataURL('image/jpeg', 0.6);
972
+ socket.emit('browser_frame', { frame: frame.split(',')[1] });
973
+ }
974
+ }, 65); // ~15 FPS
975
+ }
976
+ });
977
+ } catch (err) {
978
+ console.error('Camera access error:', err);
979
+ }
980
+ }
981
+
982
+ function stopBrowserCamera() {
983
+ if (cameraStream) {
984
+ cameraStream.getTracks().forEach(track => track.stop());
985
+ cameraStream = null;
986
+ }
987
+ if (window.cameraInterval) {
988
+ clearInterval(window.cameraInterval);
989
+ window.cameraInterval = null;
990
+ }
991
+ }
992
+
993
+ /* ── Socket.IO ──────────────────────────────────────────── */
994
+ // βœ… FIX: transports updated to prefer WebSocket with polling fallback.
995
+ // upgrade:true lets Socket.IO automatically upgrade from polling β†’ websocket
996
+ // once the connection is established. This is required for camera streaming
997
+ // to work reliably on HuggingFace Spaces (HTTPS) and other cloud deployments.
998
+ let socket;
999
+ try {
1000
+ socket = io({
1001
+ transports: ['websocket', 'polling'],
1002
+ upgrade: true,
1003
+ reconnectionDelay: 1000
1004
+ });
1005
+
1006
+ socket.on('connect', () => {
1007
+ setStat('act', 'Connected β€” detecting…');
1008
+ document.getElementById('cam-ph').style.display = 'none';
1009
+ document.getElementById('live-badge').textContent = '● Live';
1010
+ startBrowserCamera();
1011
+ });
1012
+
1013
+ socket.on('disconnect', () => {
1014
+ setStat('', 'Disconnected');
1015
+ document.getElementById('cam-ph').style.display = 'flex';
1016
+ document.getElementById('live-badge').textContent = '● Offline';
1017
+ stopBrowserCamera();
1018
+ });
1019
+
1020
+ socket.on('state_update', (data) => {
1021
+ try {
1022
+ if (data.frame) {
1023
+ document.getElementById('camera-feed').src = 'data:image/jpeg;base64,' + data.frame;
1024
+ document.getElementById('cam-ph').style.display = 'none';
1025
+ }
1026
+
1027
+ const letterEl = document.getElementById('co-letter');
1028
+ const wordEl = document.getElementById('co-word');
1029
+ const descEl = document.getElementById('co-desc');
1030
+
1031
+ if (data.overlay_letter) {
1032
+ letterEl.textContent = data.overlay_letter.toUpperCase();
1033
+ }
1034
+
1035
+ if (data.current_word !== undefined) {
1036
+ const w = data.current_word || 'β€”';
1037
+ wordEl.textContent = w;
1038
+ document.getElementById('cur-word').textContent = w;
1039
+ }
1040
+
1041
+ if (data.sentence !== undefined) {
1042
+ document.getElementById('cur-sentence').textContent = data.sentence || '';
1043
+ }
1044
+
1045
+ if (data.overlay_confidence !== undefined) {
1046
+ const pct = Math.round(data.overlay_confidence * 100);
1047
+ descEl.textContent = 'Confidence: ' + pct + '%';
1048
+ document.getElementById('s-conf').textContent = pct + '%';
1049
+ document.getElementById('co-bar').style.width = pct + '%';
1050
+ }
1051
+
1052
+ if (data.fps !== undefined) {
1053
+ document.getElementById('s-fps').textContent = Math.round(data.fps);
1054
+ }
1055
+
1056
+ if (data.stats) {
1057
+ document.getElementById('s-letters').textContent = data.stats.letters_detected || 0;
1058
+ document.getElementById('s-words').textContent = data.stats.words_formed || 0;
1059
+ document.getElementById('s-sentences').textContent = data.stats.sentences_formed || 0;
1060
+ document.getElementById('s-emotions').textContent = data.stats.emotion_changes || 0;
1061
+ }
1062
+
1063
+ if (Array.isArray(data.suggestions)) {
1064
+ updateSuggestions(data.suggestions, data.current_word);
1065
+ }
1066
+
1067
+ if (data.emotion_scores) {
1068
+ updateEmotion(data.emotion || 'neutral', data.emotion_scores);
1069
+ }
1070
+
1071
+ if (data.emotion_timeline && data.emotion_timeline.length > 0) {
1072
+ updateChart(data.emotion_timeline);
1073
+ }
1074
+
1075
+ } catch (e) { console.error('Update error:', e); }
1076
+ });
1077
+
1078
+ } catch (e) {
1079
+ console.warn('Socket.IO unavailable β€” demo mode');
1080
+ }
1081
+
1082
+ /* ── Update emotion bars ────────────────────────────────── */
1083
+ function updateEmotion(emotion, scores) {
1084
+ document.getElementById('emo-label').textContent = emotion.charAt(0).toUpperCase() + emotion.slice(1);
1085
+ document.getElementById('emo-icon').textContent = EMO_ICONS[emotion] || '😐';
1086
+ EMO_KEYS.forEach(k => {
1087
+ const bar = document.getElementById('bar-' + k);
1088
+ if (!bar) return;
1089
+ let s = Math.max(0, Math.min(1, scores[k] || 0));
1090
+ const pct = Math.round(s * 100);
1091
+ bar.style.width = pct + '%';
1092
+ bar.textContent = pct + '%';
1093
+ });
1094
+ }
1095
+
1096
+ /* ── Update suggestions ─────────────────────────────────── */
1097
+ function updateSuggestions(suggestions, currentWord) {
1098
+ const area = document.getElementById('sug-area');
1099
+ if (!suggestions || !suggestions.length) {
1100
+ area.innerHTML = '<div class="sug-empty">Detected letters will appear<br>here with word suggestions</div>';
1101
+ return;
1102
+ }
1103
+ const detectedRow = currentWord
1104
+ ? '<div class="sug-detected">Detected: ' + currentWord.toUpperCase() + '</div>' : '';
1105
+ area.innerHTML = detectedRow + '<div class="sug-list">' +
1106
+ suggestions.map((w, i) =>
1107
+ '<div class="sug-item" onclick="window.acceptSug(\'' + w + '\')">' +
1108
+ '<span class="sug-n">' + (i + 1) + '</span>' + w + '</div>'
1109
+ ).join('') + '</div>';
1110
+ }
1111
+
1112
+ /* ── Update chart ───────────────────────────────────────── */
1113
+ function updateChart(timeline) {
1114
+ if (!timeline || !timeline.length) return;
1115
+ const collected = Object.fromEntries(EMO_KEYS.map(k => [k, []]));
1116
+ timeline.forEach(tp => {
1117
+ const s = tp.scores || {};
1118
+ EMO_KEYS.forEach(k => collected[k].push(s[k] || 0));
1119
+ });
1120
+ chart.data.labels = timeline.map((_, i) => i);
1121
+ chart.data.datasets.forEach((ds, i) => ds.data = collected[EMO_KEYS[i]]);
1122
+ chart.update('none');
1123
+ }
1124
+
1125
+ /* ── Status helper ──────────────────────────────────────── */
1126
+ function setStat(cls, msg) {
1127
+ document.getElementById('stxt').textContent = msg;
1128
+ document.getElementById('status').className = 'status' + (cls ? ' ' + cls : '');
1129
+ }
1130
+
1131
+ /* ── Global handlers ────────────────────────────────────── */
1132
+ window.sendCmd = function (action) {
1133
+ if (action === 'speak') {
1134
+ const sentence = document.getElementById('cur-sentence').textContent.trim();
1135
+ const word = document.getElementById('cur-word').textContent.trim();
1136
+ const text = sentence || (word !== 'β€”' ? word : '');
1137
+ if (text && socket) socket.emit('command', { action: 'speak', text });
1138
+ } else {
1139
+ if (socket) socket.emit('command', { action });
1140
+ }
1141
+ };
1142
+
1143
+ window.acceptSug = function (word) {
1144
+ if (socket) socket.emit('command', { action: 'accept_suggestion', word });
1145
+ };
1146
+
1147
+ /* ── Keyboard shortcuts ─────────────────────────────────── */
1148
+ document.addEventListener('keydown', e => {
1149
+ if (e.ctrlKey && e.key === 's') { e.preventDefault(); window.sendCmd('speak'); }
1150
+ if (e.ctrlKey && e.key === 'r') { e.preventDefault(); window.sendCmd('reset'); }
1151
+ if (e.ctrlKey && e.key === 'Backspace') { e.preventDefault(); window.sendCmd('backspace'); }
1152
+ if (['1', '2', '3'].includes(e.key)) {
1153
+ const items = document.querySelectorAll('.sug-item');
1154
+ const item = items[parseInt(e.key) - 1];
1155
+ if (item) item.click();
1156
+ }
1157
+ });
1158
+
1159
+ });
1160
+ </script>
1161
+ </body>
1162
+
1163
+ </html>
translation.py ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ translation.py - Rule-based Sign-glish to English translation
3
+
4
+ Simple rule‑based mapper that converts a list of sign tokens into natural English.
5
+ This implementation is deliberately lightweight and does not require any external services.
6
+ """
7
+
8
+ from typing import List
9
+
10
+ # ==================== TRANSLATION MAP ====================
11
+ # Comprehensive mapping dictionary for ISL to English
12
+
13
+ _TRANSLATION_MAP = {
14
+ # Greetings
15
+ "HELLO": "Hello",
16
+ "HI": "Hi",
17
+ "GOODBYE": "Goodbye",
18
+ "BYE": "Bye",
19
+
20
+ # Politeness
21
+ "HELP": "Help",
22
+ "PLEASE": "Please",
23
+ "THANK": "Thank you",
24
+ "THANKS": "Thank you",
25
+ "SORRY": "Sorry",
26
+ "WELCOME": "Welcome",
27
+
28
+ # Pronouns
29
+ "I": "I",
30
+ "ME": "me",
31
+ "YOU": "you",
32
+ "WE": "we",
33
+ "THEY": "they",
34
+
35
+ # Basic responses
36
+ "YES": "yes",
37
+ "NO": "no",
38
+ "OK": "okay",
39
+ "GOOD": "good",
40
+ "BAD": "bad",
41
+
42
+ # Actions
43
+ "WANT": "want",
44
+ "NEED": "need",
45
+ "GO": "go",
46
+ "COME": "come",
47
+ "STOP": "stop",
48
+ "SIT": "sit",
49
+ "STAND": "stand",
50
+ "WALK": "walk",
51
+ "RUN": "run",
52
+ "EAT": "eat",
53
+ "DRINK": "drink",
54
+ "SLEEP": "sleep",
55
+
56
+ # Objects
57
+ "WATER": "water",
58
+ "FOOD": "food",
59
+ "HOME": "home",
60
+ "SCHOOL": "school",
61
+ "WORK": "work",
62
+
63
+ # Time
64
+ "MORNING": "good morning",
65
+ "AFTERNOON": "good afternoon",
66
+ "EVENING": "good evening",
67
+ "NIGHT": "good night",
68
+ "TODAY": "today",
69
+ "TOMORROW": "tomorrow",
70
+ "YESTERDAY": "yesterday",
71
+ "NOW": "now",
72
+
73
+ # Emotions
74
+ "HAPPY": "happy",
75
+ "SAD": "sad",
76
+ "ANGRY": "angry",
77
+ "SURPRISED": "surprised",
78
+
79
+ # Questions
80
+ "HOW": "how",
81
+ "WHAT": "what",
82
+ "WHERE": "where",
83
+ "WHEN": "when",
84
+ "WHY": "why",
85
+ "WHO": "who",
86
+
87
+ # Misc
88
+ "ARE": "are",
89
+ "IS": "is",
90
+ "LOVE": "love",
91
+ }
92
+
93
+ def translate_signs(sign_list: List[str]) -> str:
94
+ """Translate a list of sign tokens into a readable English sentence.
95
+
96
+ Args:
97
+ sign_list: List of uppercase sign tokens.
98
+
99
+ Returns:
100
+ A string with the translated sentence.
101
+
102
+ Example:
103
+ >>> translate_signs(["HELLO", "I", "WANT", "WATER"])
104
+ "Hello I want water"
105
+ """
106
+ if not sign_list:
107
+ return ""
108
+
109
+ translated_words = []
110
+
111
+ for token in sign_list:
112
+ # Handle fingerspelled words
113
+ if token.startswith("SPELL_"):
114
+ # Extract the spelled word
115
+ word = token.replace("SPELL_", "").lower()
116
+ translated_words.append(word)
117
+ else:
118
+ # Look up in translation map
119
+ translated = _TRANSLATION_MAP.get(token.upper(), token.lower())
120
+ translated_words.append(translated)
121
+
122
+ # Join words
123
+ sentence = " ".join(translated_words).strip()
124
+
125
+ # Capitalize first letter
126
+ if sentence:
127
+ sentence = sentence[0].upper() + sentence[1:]
128
+
129
+ return sentence
130
+
131
+ def add_translation(sign_token: str, english_text: str):
132
+ """Add a new translation to the map.
133
+
134
+ Args:
135
+ sign_token: The ISL sign token (uppercase)
136
+ english_text: The English translation
137
+ """
138
+ _TRANSLATION_MAP[sign_token.upper()] = english_text
139
+
140
+ def get_all_translations() -> dict:
141
+ """Get all available translations.
142
+
143
+ Returns:
144
+ Dictionary of all sign-to-English mappings
145
+ """
146
+ return _TRANSLATION_MAP.copy()
147
+
148
+ def translate_with_grammar(sign_list: List[str]) -> str:
149
+ """Translate with basic grammar improvements.
150
+
151
+ Args:
152
+ sign_list: List of sign tokens
153
+
154
+ Returns:
155
+ Grammatically improved sentence
156
+ """
157
+ sentence = translate_signs(sign_list)
158
+
159
+ # Basic grammar improvements
160
+ # Add articles
161
+ sentence = sentence.replace(" water", " some water")
162
+ sentence = sentence.replace(" food", " some food")
163
+
164
+ # Fix common patterns
165
+ sentence = sentence.replace("I want", "I would like")
166
+ sentence = sentence.replace("I need", "I need to")
167
+
168
+ return sentence
169
+
170
+ # ==================== MAIN ====================
171
+
172
+ if __name__ == "__main__":
173
+ # Demo examples
174
+ print("=" * 60)
175
+ print("ISL Sign-to-English Translation Demo")
176
+ print("=" * 60)
177
+ print()
178
+
179
+ # Test cases
180
+ test_cases = [
181
+ ["HELLO", "HOW", "ARE", "YOU"],
182
+ ["I", "WANT", "WATER", "PLEASE"],
183
+ ["THANK", "YOU"],
184
+ ["GOOD", "MORNING"],
185
+ ["I", "NEED", "HELP"],
186
+ ["SORRY", "I", "AM", "SAD"],
187
+ ]
188
+
189
+ for signs in test_cases:
190
+ translation = translate_signs(signs)
191
+ print(f"Signs: {' '.join(signs)}")
192
+ print(f"Translation: {translation}")
193
+ print()
194
+
195
+ print("=" * 60)
unified_launcher.py ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # unified_launcher.py
2
+ # ═══════════════════════════════════════════════════════════════════════════
3
+ # ISL FULL SYSTEM β€” SINGLE COMMAND LAUNCHER
4
+ # Starts BOTH services from ONE terminal:
5
+ #
6
+ # LOCAL β†’ python unified_launcher.py
7
+ # CLOUD β†’ RUN_MODE=CLOUD python unified_launcher.py
8
+ #
9
+ # Port 5000 β†’ ISL Gesture Recognition + Emotion Detection (launcher.py)
10
+ # Port 5001 β†’ Speech-to-Sign / Fingerspelling Avatar (speech_to_sign_avatar.py)
11
+ # OR speech_to_sign_avatar_fixed.py (auto-detected)
12
+ #
13
+ # ⚠️ WHY subprocess.Popen (not multiprocessing)?
14
+ # speech_to_sign_avatar*.py calls eventlet.monkey_patch() at import-time,
15
+ # which corrupts the multiprocessing runtime. Running each script as a
16
+ # fully separate OS process avoids this entirely.
17
+ # ═══════════════════════════════════════════════════════════════════════════
18
+
19
+ import subprocess
20
+ import sys
21
+ import os
22
+ import time
23
+ import multiprocessing
24
+
25
+ # ── Resolve paths ─────────────────────────────────────────────────────────────
26
+ PYTHON = sys.executable # same venv / interpreter that started us
27
+ BASE_DIR = os.path.dirname(os.path.abspath(__file__))
28
+
29
+ RUN_MODE = os.getenv("RUN_MODE", "LOCAL").upper()
30
+
31
+ # ── Auto-detect which avatar script exists ────────────────────────────────────
32
+ # Prefer the fixed/fingerspelling version if present, fall back to original.
33
+ AVATAR_CANDIDATES = [
34
+ "speech_to_sign_avatar_fixed.py",
35
+ "speech_to_sign_avatar.py",
36
+ ]
37
+ AVATAR_SCRIPT = None
38
+ for _candidate in AVATAR_CANDIDATES:
39
+ _path = os.path.join(BASE_DIR, _candidate)
40
+ if os.path.exists(_path):
41
+ AVATAR_SCRIPT = _path
42
+ break
43
+
44
+ GESTURE_SCRIPT = os.path.join(BASE_DIR, "launcher.py")
45
+
46
+ # ── Environment for child processes ──────────────────────────────────────────
47
+ CHILD_ENV = {
48
+ **os.environ,
49
+ "RUN_MODE" : RUN_MODE,
50
+ "PYTHONUNBUFFERED" : "1", # immediate stdout/stderr in logs
51
+ }
52
+
53
+ # ── Oracle Cloud: open firewall ports (run once manually) ─────────────────────
54
+ # sudo firewall-cmd --permanent --add-port=5000/tcp
55
+ # sudo firewall-cmd --permanent --add-port=5001/tcp
56
+ # sudo firewall-cmd --reload
57
+ # Also open in OCI Console β†’ VCN β†’ Security List β†’ Ingress Rules
58
+
59
+
60
+ # ─────────────────────────────────────────────────────────────────────────────
61
+ def start_process(script_path, label):
62
+ """Spawn a child process and return the Popen handle."""
63
+ print(f" πŸš€ Starting {label}")
64
+ print(f" Script : {script_path}")
65
+ proc = subprocess.Popen(
66
+ [PYTHON, script_path],
67
+ env=CHILD_ENV,
68
+ cwd=BASE_DIR,
69
+ stdout=None, # inherit β€” all logs appear in the same terminal
70
+ stderr=None,
71
+ )
72
+ print(f" PID : {proc.pid}")
73
+ return proc
74
+
75
+
76
+ def stop_all(processes):
77
+ """Gracefully terminate all child processes, then force-kill if needed."""
78
+ print("\nπŸ›‘ Shutting down all processes…")
79
+ for label, proc in processes.items():
80
+ if proc and proc.poll() is None:
81
+ print(f" β†’ Terminating {label} (PID {proc.pid})")
82
+ proc.terminate()
83
+
84
+ deadline = time.time() + 6
85
+ for label, proc in processes.items():
86
+ if proc:
87
+ remaining = max(0.1, deadline - time.time())
88
+ try:
89
+ proc.wait(timeout=remaining)
90
+ print(f" βœ… {label} stopped cleanly")
91
+ except subprocess.TimeoutExpired:
92
+ print(f" ⚑ Force-killing {label} (PID {proc.pid})")
93
+ proc.kill()
94
+
95
+ print("βœ… All processes stopped")
96
+ print("=" * 70)
97
+
98
+
99
+ def preflight_check():
100
+ """Verify required scripts exist before starting anything."""
101
+ ok = True
102
+
103
+ if not os.path.exists(GESTURE_SCRIPT):
104
+ print(f"❌ MISSING: launcher.py not found at:\n {GESTURE_SCRIPT}")
105
+ ok = False
106
+
107
+ if AVATAR_SCRIPT is None:
108
+ print("❌ MISSING: No avatar script found.")
109
+ print(" Expected one of:", AVATAR_CANDIDATES)
110
+ ok = False
111
+
112
+ if not ok:
113
+ print("\n Make sure unified_launcher.py is in the SAME folder as the other scripts.")
114
+ sys.exit(1)
115
+
116
+
117
+ # ─────────────────────────────────────────────────────────────────────────────
118
+ def main():
119
+ print()
120
+ print("=" * 70)
121
+ print(" 🀟 ISL FULL SYSTEM β€” UNIFIED LAUNCHER")
122
+ print("=" * 70)
123
+ print(f" RUN_MODE : {RUN_MODE}")
124
+ print(f" Python : {PYTHON}")
125
+ print(f" Base dir : {BASE_DIR}")
126
+ print(f" Gesture : {os.path.basename(GESTURE_SCRIPT)} β†’ port 5000")
127
+ if AVATAR_SCRIPT:
128
+ print(f" Avatar : {os.path.basename(AVATAR_SCRIPT)} β†’ port 5001")
129
+ print("=" * 70)
130
+ print()
131
+
132
+ preflight_check()
133
+
134
+ processes = {
135
+ "ISL Gesture (5000)" : None,
136
+ "Avatar Server (5001)": None,
137
+ }
138
+
139
+ restart_map = {
140
+ "ISL Gesture (5000)" : GESTURE_SCRIPT,
141
+ "Avatar Server (5001)": AVATAR_SCRIPT,
142
+ }
143
+
144
+ # ── Start gesture system first; give camera time to initialise ────────────
145
+ processes["ISL Gesture (5000)"] = start_process(
146
+ GESTURE_SCRIPT, "ISL Gesture Recognition (port 5000)")
147
+ print()
148
+ print(" ⏳ Waiting 2 s for camera to initialise…")
149
+ time.sleep(2)
150
+ print()
151
+
152
+ # ── Start avatar server ───────────────────────────────────────────────────
153
+ processes["Avatar Server (5001)"] = start_process(
154
+ AVATAR_SCRIPT, "Speech-to-Sign Avatar (port 5001)")
155
+ time.sleep(1)
156
+
157
+ # ── Ready banner ──────────────────────────────────────────────────────────
158
+ print()
159
+ print("=" * 70)
160
+ print(" βœ… ALL SYSTEMS RUNNING!")
161
+ print("=" * 70)
162
+
163
+ if RUN_MODE == "CLOUD":
164
+ public_ip = os.getenv("PUBLIC_IP", "<YOUR_ORACLE_PUBLIC_IP>")
165
+ print(f" πŸ“Š ISL Dashboard β†’ http://{public_ip}:5000")
166
+ print(f" 🎭 Avatar System β†’ http://{public_ip}:5001")
167
+ print()
168
+ print(" πŸ’‘ Oracle Cloud checklist:")
169
+ print(" 1. Set env var: export PUBLIC_IP=<your-instance-ip>")
170
+ print(" 2. OCI Console β†’ VCN β†’ Security List β†’ add Ingress rules")
171
+ print(" for TCP port 5000 and TCP port 5001 (source 0.0.0.0/0)")
172
+ print(" 3. On the VM (run once):")
173
+ print(" sudo firewall-cmd --permanent --add-port=5000/tcp")
174
+ print(" sudo firewall-cmd --permanent --add-port=5001/tcp")
175
+ print(" sudo firewall-cmd --reload")
176
+ print(" 4. To auto-start on reboot β†’ see isl.service below")
177
+ else:
178
+ print(" πŸ“Š ISL Dashboard β†’ http://localhost:5000")
179
+ print(" 🎭 Avatar System β†’ http://localhost:5001")
180
+
181
+ print()
182
+ print(" πŸ›‘ Press Ctrl+C to stop everything")
183
+ print("=" * 70)
184
+ print()
185
+
186
+ # ── Watch-dog loop β€” auto-restarts any crashed process ───────────────────
187
+ try:
188
+ while True:
189
+ for label, script in restart_map.items():
190
+ proc = processes[label]
191
+ if proc is not None and proc.poll() is not None:
192
+ exit_code = proc.returncode
193
+ print(f"\n⚠️ [{label}] exited with code {exit_code}")
194
+ print(" Restarting in 3 s…")
195
+ time.sleep(3)
196
+ processes[label] = start_process(script, label)
197
+ print()
198
+ time.sleep(3)
199
+
200
+ except KeyboardInterrupt:
201
+ stop_all(processes)
202
+ sys.exit(0)
203
+
204
+
205
+ # ─────────────────────────────────────────────────────────────────────────────
206
+ if __name__ == "__main__":
207
+ multiprocessing.freeze_support() # no-op on Linux/Mac, required on Win
208
+ main()