Spaces:
Running
Running
Deepak Roshan commited on
Commit Β·
f124d78
0
Parent(s):
fresh clean upload
Browse files- .gitattributes +35 -0
- .gitignore +168 -0
- Dockerfile +67 -0
- README.md +29 -0
- isl_detection.py +918 -0
- isl_ui_dashboard.py +941 -0
- launcher.py +138 -0
- model.h5 +3 -0
- nginx.conf +107 -0
- requirements.txt +0 -0
- speech_to_sign_avatar.py +343 -0
- supervisord.conf +88 -0
- templates/avatar_interface.html +653 -0
- templates/dashboard.html +1163 -0
- translation.py +195 -0
- unified_launcher.py +208 -0
.gitattributes
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
#images
|
| 3 |
+
images/
|
| 4 |
+
|
| 5 |
+
# Byte-compiled / optimized / DLL files
|
| 6 |
+
__pycache__/
|
| 7 |
+
*.py[cod]
|
| 8 |
+
*$py.class
|
| 9 |
+
|
| 10 |
+
# C extensions
|
| 11 |
+
*.so
|
| 12 |
+
|
| 13 |
+
# Distribution / packaging
|
| 14 |
+
.Python
|
| 15 |
+
build/
|
| 16 |
+
develop-eggs/
|
| 17 |
+
dist/
|
| 18 |
+
downloads/
|
| 19 |
+
eggs/
|
| 20 |
+
.eggs/
|
| 21 |
+
lib/
|
| 22 |
+
lib64/
|
| 23 |
+
parts/
|
| 24 |
+
sdist/
|
| 25 |
+
var/
|
| 26 |
+
wheels/
|
| 27 |
+
share/python-wheels/
|
| 28 |
+
*.egg-info/
|
| 29 |
+
.installed.cfg
|
| 30 |
+
*.egg
|
| 31 |
+
MANIFEST
|
| 32 |
+
|
| 33 |
+
# PyInstaller
|
| 34 |
+
# Usually these files are written by a python script from a template
|
| 35 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
| 36 |
+
*.manifest
|
| 37 |
+
*.spec
|
| 38 |
+
|
| 39 |
+
# Installer logs
|
| 40 |
+
pip-log.txt
|
| 41 |
+
pip-delete-this-directory.txt
|
| 42 |
+
|
| 43 |
+
# Unit test / coverage reports
|
| 44 |
+
htmlcov/
|
| 45 |
+
.tox/
|
| 46 |
+
.nox/
|
| 47 |
+
.coverage
|
| 48 |
+
.coverage.*
|
| 49 |
+
.cache
|
| 50 |
+
nosetests.xml
|
| 51 |
+
coverage.xml
|
| 52 |
+
*.cover
|
| 53 |
+
*.py,cover
|
| 54 |
+
.hypothesis/
|
| 55 |
+
.pytest_cache/
|
| 56 |
+
cover/
|
| 57 |
+
|
| 58 |
+
# Translations
|
| 59 |
+
*.mo
|
| 60 |
+
*.pot
|
| 61 |
+
|
| 62 |
+
# Django stuff:
|
| 63 |
+
*.log
|
| 64 |
+
local_settings.py
|
| 65 |
+
db.sqlite3
|
| 66 |
+
db.sqlite3-journal
|
| 67 |
+
|
| 68 |
+
# Flask stuff:
|
| 69 |
+
instance/
|
| 70 |
+
.webassets-cache
|
| 71 |
+
|
| 72 |
+
# Scrapy stuff:
|
| 73 |
+
.scrapy
|
| 74 |
+
|
| 75 |
+
# Sphinx documentation
|
| 76 |
+
docs/_build/
|
| 77 |
+
|
| 78 |
+
# PyBuilder
|
| 79 |
+
.pybuilder/
|
| 80 |
+
target/
|
| 81 |
+
|
| 82 |
+
# Jupyter Notebook
|
| 83 |
+
.ipynb_checkpoints
|
| 84 |
+
|
| 85 |
+
# IPython
|
| 86 |
+
profile_default/
|
| 87 |
+
ipython_config.py
|
| 88 |
+
|
| 89 |
+
# pyenv
|
| 90 |
+
# For a library or package, you might want to ignore these files since the code is
|
| 91 |
+
# intended to run in multiple environments; otherwise, check them in:
|
| 92 |
+
# .python-version
|
| 93 |
+
|
| 94 |
+
# pipenv
|
| 95 |
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
| 96 |
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
| 97 |
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
| 98 |
+
# install all needed dependencies.
|
| 99 |
+
#Pipfile.lock
|
| 100 |
+
|
| 101 |
+
# poetry
|
| 102 |
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
| 103 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
| 104 |
+
# commonly ignored for libraries.
|
| 105 |
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
| 106 |
+
#poetry.lock
|
| 107 |
+
|
| 108 |
+
# pdm
|
| 109 |
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
| 110 |
+
#pdm.lock
|
| 111 |
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
| 112 |
+
# in version control.
|
| 113 |
+
# https://pdm.fming.dev/#use-with-ide
|
| 114 |
+
.pdm.toml
|
| 115 |
+
|
| 116 |
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
| 117 |
+
__pypackages__/
|
| 118 |
+
|
| 119 |
+
# Celery stuff
|
| 120 |
+
celerybeat-schedule
|
| 121 |
+
celerybeat.pid
|
| 122 |
+
|
| 123 |
+
# SageMath parsed files
|
| 124 |
+
*.sage.py
|
| 125 |
+
|
| 126 |
+
# Environments
|
| 127 |
+
.env
|
| 128 |
+
.venv
|
| 129 |
+
env/
|
| 130 |
+
venv/
|
| 131 |
+
ENV/
|
| 132 |
+
env.bak/
|
| 133 |
+
venv.bak/
|
| 134 |
+
mp_env/
|
| 135 |
+
|
| 136 |
+
# Spyder project settings
|
| 137 |
+
.spyderproject
|
| 138 |
+
.spyproject
|
| 139 |
+
|
| 140 |
+
# Rope project settings
|
| 141 |
+
.ropeproject
|
| 142 |
+
|
| 143 |
+
# mkdocs documentation
|
| 144 |
+
/site
|
| 145 |
+
|
| 146 |
+
# mypy
|
| 147 |
+
.mypy_cache/
|
| 148 |
+
.dmypy.json
|
| 149 |
+
dmypy.json
|
| 150 |
+
|
| 151 |
+
# Pyre type checker
|
| 152 |
+
.pyre/
|
| 153 |
+
|
| 154 |
+
# pytype static type analyzer
|
| 155 |
+
.pytype/
|
| 156 |
+
|
| 157 |
+
# Cython debug symbols
|
| 158 |
+
cython_debug/
|
| 159 |
+
|
| 160 |
+
# PyCharm
|
| 161 |
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
| 162 |
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
| 163 |
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
| 164 |
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
| 165 |
+
#.idea/
|
| 166 |
+
voice_*.mp3
|
| 167 |
+
tmp_audio/
|
| 168 |
+
*.mp3
|
Dockerfile
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ============================================================
|
| 2 |
+
# Dockerfile β Hugging Face Spaces (Docker SDK)
|
| 3 |
+
# Free tier: 16GB RAM, 2 vCPU
|
| 4 |
+
# HF exposes only port 7860 β nginx routes both Flask apps
|
| 5 |
+
#
|
| 6 |
+
# / β Flask 5000 (gesture + emotion)
|
| 7 |
+
# /avatar/ β Flask 5001 (speech-to-sign)
|
| 8 |
+
# ============================================================
|
| 9 |
+
|
| 10 |
+
FROM python:3.10-slim
|
| 11 |
+
|
| 12 |
+
# HF Spaces runs as non-root user 1000
|
| 13 |
+
RUN useradd -m -u 1000 user
|
| 14 |
+
USER root
|
| 15 |
+
|
| 16 |
+
# ββ System deps βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 17 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 18 |
+
libgl1 \
|
| 19 |
+
libglib2.0-0 \
|
| 20 |
+
libsm6 \
|
| 21 |
+
libxext6 \
|
| 22 |
+
libxrender-dev \
|
| 23 |
+
libgomp1 \
|
| 24 |
+
nginx \
|
| 25 |
+
supervisor \
|
| 26 |
+
curl \
|
| 27 |
+
git-lfs \
|
| 28 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 29 |
+
|
| 30 |
+
WORKDIR /app
|
| 31 |
+
|
| 32 |
+
# ββ Python deps βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 33 |
+
COPY requirements.txt .
|
| 34 |
+
RUN pip install --no-cache-dir --upgrade pip && \
|
| 35 |
+
pip install --no-cache-dir -r requirements.txt
|
| 36 |
+
|
| 37 |
+
# ββ App files βββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 38 |
+
COPY isl_detection.py .
|
| 39 |
+
COPY isl_ui_dashboard.py .
|
| 40 |
+
COPY launcher.py .
|
| 41 |
+
COPY speech_to_sign_avatar.py .
|
| 42 |
+
COPY translation.py .
|
| 43 |
+
COPY unified_launcher.py .
|
| 44 |
+
COPY model.h5 .
|
| 45 |
+
COPY templates/ templates/
|
| 46 |
+
|
| 47 |
+
# ββ Configs βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 48 |
+
COPY nginx.conf /etc/nginx/nginx.conf
|
| 49 |
+
COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
|
| 50 |
+
|
| 51 |
+
# ββ Nginx log dir permissions βββββββββββββββββββββββββββββββββ
|
| 52 |
+
RUN mkdir -p /var/log/nginx /var/log/supervisor /var/run \
|
| 53 |
+
&& chmod -R 777 /var/log/nginx /var/log/supervisor /var/run \
|
| 54 |
+
&& chown -R user:user /app
|
| 55 |
+
|
| 56 |
+
# ββ Environment βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 57 |
+
ENV RUN_MODE=CLOUD
|
| 58 |
+
ENV PYTHONUNBUFFERED=1
|
| 59 |
+
ENV TF_CPP_MIN_LOG_LEVEL=3
|
| 60 |
+
ENV CUDA_VISIBLE_DEVICES=-1
|
| 61 |
+
|
| 62 |
+
# HF Spaces MUST use port 7860
|
| 63 |
+
EXPOSE 7860
|
| 64 |
+
|
| 65 |
+
# ββ Start supervisor as root (nginx needs it) βββββββββββββββββ
|
| 66 |
+
USER root
|
| 67 |
+
CMD ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
|
README.md
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: ISL Indian Sign Language Detection
|
| 3 |
+
emoji: "π€"
|
| 4 |
+
colorFrom: yellow
|
| 5 |
+
colorTo: green
|
| 6 |
+
sdk: docker
|
| 7 |
+
pinned: false
|
| 8 |
+
license: mit
|
| 9 |
+
app_port: 7860
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# π€ ISL Indian Sign Language Detection System
|
| 13 |
+
|
| 14 |
+
Real-time Indian Sign Language detection with emotion recognition and speech-to-sign avatar.
|
| 15 |
+
|
| 16 |
+
## Features
|
| 17 |
+
- Gesture Recognition β Real-time ISL A-Z letter detection via MediaPipe + TensorFlow
|
| 18 |
+
- Emotion Detection β Happy, Sad, Angry, Surprise, Neutral via facial landmarks
|
| 19 |
+
- Speech-to-Sign Avatar β 3D animated hand avatar for fingerspelling
|
| 20 |
+
|
| 21 |
+
## Usage
|
| 22 |
+
- Main dashboard: `/`
|
| 23 |
+
- Speech-to-Sign avatar: `/avatar/`
|
| 24 |
+
|
| 25 |
+
## Built with
|
| 26 |
+
- Flask + Socket.IO
|
| 27 |
+
- TensorFlow CNN model
|
| 28 |
+
- MediaPipe Hand Tracking
|
| 29 |
+
- Three.js 3D Avatar
|
isl_detection.py
ADDED
|
@@ -0,0 +1,918 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# isl_detection.py β CLOUD PRODUCTION VERSION
|
| 2 |
+
#
|
| 3 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 4 |
+
# CLOUD FIXES APPLIED (3 changes only β everything else identical):
|
| 5 |
+
#
|
| 6 |
+
# FIX 1 β pygame TTS: init only in LOCAL mode (crashes on headless server)
|
| 7 |
+
# FIX 2 β cv2.CAP_DSHOW: Windows-only flag removed for Linux cloud server
|
| 8 |
+
# FIX 3 β SharedMemory cleanup on startup (prevents crash on restart)
|
| 9 |
+
#
|
| 10 |
+
# LOCAL CAMERA FIXES (built-in laptop webcam static/noise):
|
| 11 |
+
# FIX 4 β MJPG codec + 30-frame flush to clear DirectShow buffer on Windows
|
| 12 |
+
# FIX 5 β CAP_PROP_BUFFERSIZE set before flush (not after)
|
| 13 |
+
#
|
| 14 |
+
# ALL original features preserved:
|
| 15 |
+
# β
Emotion detection v3 (happy/sad/angry/surprise/neutral)
|
| 16 |
+
# β
Letter hold timer, same-letter cooldown, confidence buffer
|
| 17 |
+
# β
SharedMemory zero-copy frame transfer
|
| 18 |
+
# β
CLOUD mode browser frame ingestion (_cloud_frame)
|
| 19 |
+
# β
gTTS speak button (disabled silently in cloud β browser TTS handles it)
|
| 20 |
+
# β
Word suggestions, auto-TTS, reset, backspace, accept_suggestion
|
| 21 |
+
# β
Oracle Free Tier safe: TARGET_FPS=15, EMOTION_PROCESS_INTERVAL=5
|
| 22 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 23 |
+
|
| 24 |
+
import cv2
|
| 25 |
+
import mediapipe as mp
|
| 26 |
+
import numpy as np
|
| 27 |
+
import string
|
| 28 |
+
import time
|
| 29 |
+
import copy
|
| 30 |
+
import itertools
|
| 31 |
+
import os
|
| 32 |
+
import base64
|
| 33 |
+
import platform
|
| 34 |
+
import threading
|
| 35 |
+
from collections import deque, defaultdict, Counter
|
| 36 |
+
from multiprocessing import Manager, Value, shared_memory
|
| 37 |
+
import ctypes
|
| 38 |
+
|
| 39 |
+
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
| 40 |
+
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
|
| 41 |
+
|
| 42 |
+
from tensorflow.keras.models import load_model
|
| 43 |
+
from gtts import gTTS
|
| 44 |
+
import pygame
|
| 45 |
+
|
| 46 |
+
try:
|
| 47 |
+
import translation
|
| 48 |
+
TRANSLATION_AVAILABLE = True
|
| 49 |
+
except ImportError:
|
| 50 |
+
TRANSLATION_AVAILABLE = False
|
| 51 |
+
|
| 52 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 53 |
+
# CONFIG
|
| 54 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 55 |
+
RUN_MODE = os.getenv("RUN_MODE", "LOCAL")
|
| 56 |
+
ISL_MODEL_PATH = "model.h5"
|
| 57 |
+
CAM_INDEX = 0
|
| 58 |
+
|
| 59 |
+
# ββ Letter timing ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 60 |
+
LETTER_HOLD_SEC = 0.9
|
| 61 |
+
SAME_LETTER_COOLDOWN_SEC = 1.2
|
| 62 |
+
|
| 63 |
+
# ββ Emotion thresholds ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 64 |
+
SMOOTHING_FRAMES = 2
|
| 65 |
+
MIN_CONF_TO_SHOW = 0.06
|
| 66 |
+
EMOTIONS = ["happy", "sad", "angry", "surprise", "neutral"]
|
| 67 |
+
|
| 68 |
+
CONF_THRESHOLD = 0.32
|
| 69 |
+
SPACE_THRESHOLD = 18
|
| 70 |
+
ENABLE_AUTO_TTS = True
|
| 71 |
+
TTS_LANGUAGE = "en"
|
| 72 |
+
|
| 73 |
+
SHOW_LETTER_OVERLAY = True
|
| 74 |
+
OVERLAY_COLOR_LETTER = (0, 255, 255)
|
| 75 |
+
OVERLAY_COLOR_CONF = (255, 255, 0)
|
| 76 |
+
OVERLAY_POSITION = (20, 50)
|
| 77 |
+
OVERLAY_FONT_SCALE = 1.5
|
| 78 |
+
OVERLAY_THICKNESS = 3
|
| 79 |
+
|
| 80 |
+
COMMON_WORDS = [
|
| 81 |
+
"HELLO", "HELP", "PLEASE", "THANK", "YOU", "YES", "NO", "GOOD", "BAD",
|
| 82 |
+
"MORNING", "AFTERNOON", "EVENING", "NIGHT", "TODAY", "TOMORROW", "WATER",
|
| 83 |
+
"FOOD", "HOME", "SCHOOL", "WORK", "HAPPY", "SAD", "SORRY", "WELCOME"
|
| 84 |
+
]
|
| 85 |
+
|
| 86 |
+
TARGET_FPS = 15
|
| 87 |
+
CAMERA_RESOLUTION = (640, 480)
|
| 88 |
+
EMOTION_PROCESS_INTERVAL = 5
|
| 89 |
+
FRAME_W, FRAME_H = 640, 480
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 93 |
+
# SHARED STATE
|
| 94 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 95 |
+
class SharedState:
|
| 96 |
+
def __init__(self):
|
| 97 |
+
manager = Manager()
|
| 98 |
+
self.ui_queue = manager.Queue(200)
|
| 99 |
+
self.command_queue = manager.Queue(20)
|
| 100 |
+
self.processing_fps = Value(ctypes.c_double, 0.0)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 104 |
+
# TTS ENGINE
|
| 105 |
+
# FIX 1: pygame.mixer.init() only runs in LOCAL mode.
|
| 106 |
+
# On cloud (headless Linux) there is no audio device β
|
| 107 |
+
# init() would crash the entire process at import time.
|
| 108 |
+
# In CLOUD mode TTS is silently disabled here; the browser
|
| 109 |
+
# handles speech via the Web Speech API on the client side.
|
| 110 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 111 |
+
TTS_ENABLED = False
|
| 112 |
+
if RUN_MODE == "LOCAL":
|
| 113 |
+
try:
|
| 114 |
+
pygame.mixer.init()
|
| 115 |
+
TTS_ENABLED = True
|
| 116 |
+
print("β
TTS engine initialized")
|
| 117 |
+
except Exception as e:
|
| 118 |
+
print(f"β οΈ TTS init warning: {e}")
|
| 119 |
+
else:
|
| 120 |
+
print("βΉοΈ TTS disabled (CLOUD/headless mode) β browser handles speech")
|
| 121 |
+
|
| 122 |
+
speaking_word = ""
|
| 123 |
+
pause_audio = False
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def speak_text(text, lang=TTS_LANGUAGE):
|
| 127 |
+
global speaking_word
|
| 128 |
+
|
| 129 |
+
# FIX 1 continued: guard every call β silently skip on cloud
|
| 130 |
+
if not TTS_ENABLED or not text or not text.strip():
|
| 131 |
+
return
|
| 132 |
+
|
| 133 |
+
def _speak():
|
| 134 |
+
global pause_audio, speaking_word
|
| 135 |
+
try:
|
| 136 |
+
fname = f"voice_{int(time.time()*1000)}.mp3"
|
| 137 |
+
gTTS(text=text, lang=lang).save(fname)
|
| 138 |
+
speaking_word = text
|
| 139 |
+
pygame.mixer.music.load(fname)
|
| 140 |
+
pygame.mixer.music.play()
|
| 141 |
+
while pygame.mixer.music.get_busy():
|
| 142 |
+
if pause_audio:
|
| 143 |
+
pygame.mixer.music.pause()
|
| 144 |
+
while pause_audio:
|
| 145 |
+
time.sleep(0.1)
|
| 146 |
+
pygame.mixer.music.unpause()
|
| 147 |
+
pygame.time.Clock().tick(10)
|
| 148 |
+
speaking_word = ""
|
| 149 |
+
pygame.mixer.music.stop()
|
| 150 |
+
pygame.mixer.music.unload()
|
| 151 |
+
time.sleep(0.15)
|
| 152 |
+
for _ in range(3):
|
| 153 |
+
try:
|
| 154 |
+
if os.path.exists(fname):
|
| 155 |
+
os.remove(fname)
|
| 156 |
+
break
|
| 157 |
+
except PermissionError:
|
| 158 |
+
time.sleep(0.2)
|
| 159 |
+
except Exception as e:
|
| 160 |
+
print(f"TTS error: {e}")
|
| 161 |
+
speaking_word = ""
|
| 162 |
+
|
| 163 |
+
threading.Thread(target=_speak, daemon=True).start()
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 167 |
+
# LANDMARK UTILITIES
|
| 168 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 169 |
+
def calc_landmark_list(image, landmarks):
|
| 170 |
+
w, h = image.shape[1], image.shape[0]
|
| 171 |
+
return [
|
| 172 |
+
[min(int(lm.x * w), w - 1), min(int(lm.y * h), h - 1)]
|
| 173 |
+
for lm in landmarks.landmark
|
| 174 |
+
]
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def pre_process_landmark(landmark_list):
|
| 178 |
+
temp = copy.deepcopy(landmark_list)
|
| 179 |
+
bx, by = temp[0]
|
| 180 |
+
for i in range(len(temp)):
|
| 181 |
+
temp[i][0] -= bx
|
| 182 |
+
temp[i][1] -= by
|
| 183 |
+
flat = list(itertools.chain.from_iterable(temp))
|
| 184 |
+
mv = max(map(abs, flat)) if flat else 1
|
| 185 |
+
return [n / mv for n in flat]
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def apply_clahe(frame):
|
| 189 |
+
"""Enhance low-light / low-contrast frames for mobile cameras."""
|
| 190 |
+
try:
|
| 191 |
+
lab = cv2.cvtColor(frame, cv2.COLOR_BGR2LAB)
|
| 192 |
+
l, a, b = cv2.split(lab)
|
| 193 |
+
clahe = cv2.createCLAHE(clipLimit=2.5, tileGridSize=(8, 8))
|
| 194 |
+
return cv2.cvtColor(cv2.merge((clahe.apply(l), a, b)), cv2.COLOR_LAB2BGR)
|
| 195 |
+
except Exception:
|
| 196 |
+
return frame
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 200 |
+
# EMOTION DETECTION β v3
|
| 201 |
+
# Redesigned SAD / ANGRY / SURPRISE with external facial cues.
|
| 202 |
+
# HAPPY unchanged.
|
| 203 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 204 |
+
|
| 205 |
+
def _dist(a, b):
|
| 206 |
+
"""Euclidean distance between two 2-D landmark points."""
|
| 207 |
+
return float(np.linalg.norm(
|
| 208 |
+
np.array(a, dtype=np.float32) - np.array(b, dtype=np.float32)
|
| 209 |
+
))
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def detect_emotion_from_landmarks(face_landmarks, img_shape):
|
| 213 |
+
"""
|
| 214 |
+
Derive emotion scores from 468 MediaPipe Face Mesh landmarks.
|
| 215 |
+
Returns {emotion: float} summing to 1.0.
|
| 216 |
+
|
| 217 |
+
Key landmark reference:
|
| 218 |
+
Mouth corners : 61 (L), 291 (R)
|
| 219 |
+
Lip inner : 13 (top), 14 (bottom)
|
| 220 |
+
Lip outer : 0 (top), 17 (bottom)
|
| 221 |
+
L eye lids : 159 (upper), 145 (lower)
|
| 222 |
+
R eye lids : 386 (upper), 374 (lower)
|
| 223 |
+
L eye corners : 33 (outer), 133 (inner)
|
| 224 |
+
R eye corners : 362 (outer), 263 (inner)
|
| 225 |
+
L brow : 70 (inner), 105 (outer), 52 (arch)
|
| 226 |
+
R brow : 300 (inner), 334 (outer), 282 (arch)
|
| 227 |
+
Nose tip : 4
|
| 228 |
+
Nostrils : 129 (L ala), 358 (R ala)
|
| 229 |
+
Cheeks : 50 (L), 280 (R)
|
| 230 |
+
"""
|
| 231 |
+
try:
|
| 232 |
+
pts = np.array(face_landmarks, dtype=np.float32)
|
| 233 |
+
face_w = max(float(np.max(pts[:, 0]) - np.min(pts[:, 0])), 1.0)
|
| 234 |
+
face_h = max(float(np.max(pts[:, 1]) - np.min(pts[:, 1])), 1.0)
|
| 235 |
+
|
| 236 |
+
def p(i):
|
| 237 |
+
return pts[i]
|
| 238 |
+
|
| 239 |
+
def nw(a, b):
|
| 240 |
+
return _dist(p(a), p(b)) / face_w
|
| 241 |
+
|
| 242 |
+
def nh(a, b):
|
| 243 |
+
return _dist(p(a), p(b)) / face_h
|
| 244 |
+
|
| 245 |
+
def clamp01(v):
|
| 246 |
+
return max(0.0, min(1.0, float(v)))
|
| 247 |
+
|
| 248 |
+
# ββ Mouth ββββββββββββββββββββββββββββββββββββββββ
|
| 249 |
+
lc, rc = p(61), p(291)
|
| 250 |
+
tl, bl = p(13), p(14)
|
| 251 |
+
ut, lb = p(0), p(17)
|
| 252 |
+
|
| 253 |
+
mid_y = (tl[1] + bl[1]) / 2.0
|
| 254 |
+
mouth_curve = ((lc[1] - mid_y) + (rc[1] - mid_y)) / (2.0 * face_h)
|
| 255 |
+
mouth_open = _dist(ut, lb) / face_h
|
| 256 |
+
inner_gap = _dist(tl, bl) / face_h
|
| 257 |
+
mouth_w = _dist(lc, rc) / face_w
|
| 258 |
+
o_ratio = (mouth_open / (mouth_w + 1e-6)) if mouth_w > 0.05 else 0.0
|
| 259 |
+
|
| 260 |
+
# ββ Eyes βββββββββββββββββββββββββββββββββββββββββ
|
| 261 |
+
leu, led = p(159), p(145)
|
| 262 |
+
reu, red = p(386), p(374)
|
| 263 |
+
lel, ler = p(33), p(133)
|
| 264 |
+
rel, rer = p(362), p(263)
|
| 265 |
+
|
| 266 |
+
l_eye_h = _dist(leu, led)
|
| 267 |
+
r_eye_h = _dist(reu, red)
|
| 268 |
+
l_eye_w = _dist(lel, ler)
|
| 269 |
+
r_eye_w = _dist(rel, rer)
|
| 270 |
+
|
| 271 |
+
l_ear = l_eye_h / (l_eye_w + 1e-6)
|
| 272 |
+
r_ear = r_eye_h / (r_eye_w + 1e-6)
|
| 273 |
+
avg_ear = (l_ear + r_ear) / 2.0
|
| 274 |
+
eye_open = ((l_eye_h + r_eye_h) / 2.0) / face_h
|
| 275 |
+
|
| 276 |
+
# ββ Eyebrows ββββββββββββββββββββββββββββββββββββββ
|
| 277 |
+
lib, rib = p(70), p(300)
|
| 278 |
+
lob, rob = p(105), p(334)
|
| 279 |
+
lab, rab = p(52), p(282)
|
| 280 |
+
|
| 281 |
+
l_brow_in_h = (leu[1] - lib[1]) / face_h
|
| 282 |
+
r_brow_in_h = (reu[1] - rib[1]) / face_h
|
| 283 |
+
avg_brow_in_h = (l_brow_in_h + r_brow_in_h) / 2.0
|
| 284 |
+
|
| 285 |
+
l_brow_out_h = (leu[1] - lob[1]) / face_h
|
| 286 |
+
r_brow_out_h = (reu[1] - rob[1]) / face_h
|
| 287 |
+
avg_brow_out_h = (l_brow_out_h + r_brow_out_h) / 2.0
|
| 288 |
+
|
| 289 |
+
brow_inner_gap = nw(70, 300)
|
| 290 |
+
brow_oblique = avg_brow_in_h - avg_brow_out_h
|
| 291 |
+
|
| 292 |
+
l_arch_h = (leu[1] - lab[1]) / face_h
|
| 293 |
+
r_arch_h = (reu[1] - rab[1]) / face_h
|
| 294 |
+
avg_arch_h = (l_arch_h + r_arch_h) / 2.0
|
| 295 |
+
|
| 296 |
+
# ββ Nose βββββββββββββββββββββββββββββββββββββββββ
|
| 297 |
+
nostril_w = nw(129, 358)
|
| 298 |
+
nose_flare = max(0.0, nostril_w - 0.26)
|
| 299 |
+
|
| 300 |
+
# ββ Cheeks βββββββββββββββββββββββββββββββββββββββ
|
| 301 |
+
lch, rch, nb = p(50), p(280), p(4)
|
| 302 |
+
cheek_raise = ((nb[1] - lch[1]) + (nb[1] - rch[1])) / (2.0 * face_h)
|
| 303 |
+
|
| 304 |
+
# ββ HAPPY ββββββββββββββββββββββββββββββββββββββββ
|
| 305 |
+
happy = 0.0
|
| 306 |
+
if mouth_curve < -0.003:
|
| 307 |
+
happy += clamp01(abs(mouth_curve) / 0.018) * 0.55
|
| 308 |
+
if mouth_w > 0.36:
|
| 309 |
+
happy += clamp01((mouth_w - 0.36) / 0.07) * 0.28
|
| 310 |
+
if cheek_raise > 0.007:
|
| 311 |
+
happy += clamp01(cheek_raise / 0.018) * 0.26
|
| 312 |
+
if avg_ear < 0.27 and eye_open > 0.04:
|
| 313 |
+
happy += 0.12
|
| 314 |
+
if mouth_curve > 0.004: happy *= 0.15
|
| 315 |
+
if avg_brow_in_h < -0.05: happy *= 0.40
|
| 316 |
+
if mouth_open > 0.08: happy *= 0.55
|
| 317 |
+
happy = clamp01(happy * 1.30)
|
| 318 |
+
|
| 319 |
+
# ββ SAD β v3 ββββββββββββββββββββββββββββββββββββββ
|
| 320 |
+
sad = 0.0
|
| 321 |
+
if brow_oblique > 0.02:
|
| 322 |
+
sad += clamp01((brow_oblique - 0.02) / 0.055) * 0.45
|
| 323 |
+
if 0.08 < avg_ear < 0.23:
|
| 324 |
+
peak_dist = abs(avg_ear - 0.16)
|
| 325 |
+
sad += clamp01(1.0 - peak_dist / 0.08) * 0.30
|
| 326 |
+
if 0.02 < eye_open < 0.065:
|
| 327 |
+
sad += clamp01((0.065 - eye_open) / 0.025) * 0.18
|
| 328 |
+
if mouth_curve > 0.002:
|
| 329 |
+
sad += clamp01(mouth_curve / 0.020) * 0.40
|
| 330 |
+
if mouth_w < 0.36:
|
| 331 |
+
sad += clamp01((0.36 - mouth_w) / 0.055) * 0.18
|
| 332 |
+
if avg_brow_in_h > 0.04:
|
| 333 |
+
sad += clamp01((avg_brow_in_h - 0.04) / 0.040) * 0.20
|
| 334 |
+
if mouth_curve < -0.004: sad *= 0.08
|
| 335 |
+
if cheek_raise > 0.014: sad *= 0.20
|
| 336 |
+
if avg_ear > 0.30: sad *= 0.25
|
| 337 |
+
if avg_brow_in_h < -0.06 and brow_inner_gap < 0.14: sad *= 0.15
|
| 338 |
+
sad = clamp01(sad * 1.50)
|
| 339 |
+
|
| 340 |
+
# ββ ANGRY β v3 ββββββββββββββββββββββββββββββββββββ
|
| 341 |
+
angry = 0.0
|
| 342 |
+
if nose_flare > 0.005:
|
| 343 |
+
angry += clamp01(nose_flare / 0.055) * 0.35
|
| 344 |
+
if avg_brow_in_h < -0.04:
|
| 345 |
+
angry += clamp01(abs(avg_brow_in_h + 0.04) / 0.045) * 0.40
|
| 346 |
+
if brow_inner_gap < 0.155:
|
| 347 |
+
angry += clamp01((0.155 - brow_inner_gap) / 0.045) * 0.35
|
| 348 |
+
if avg_brow_out_h < -0.03:
|
| 349 |
+
angry += clamp01(abs(avg_brow_out_h + 0.03) / 0.040) * 0.20
|
| 350 |
+
if avg_ear < 0.20 and eye_open < 0.055:
|
| 351 |
+
angry += clamp01((0.055 - eye_open) / 0.025) * 0.25
|
| 352 |
+
if inner_gap < 0.030:
|
| 353 |
+
angry += clamp01((0.030 - inner_gap) / 0.025) * 0.22
|
| 354 |
+
if brow_oblique < 0.015:
|
| 355 |
+
angry += 0.08
|
| 356 |
+
if mouth_curve < -0.005: angry *= 0.15
|
| 357 |
+
if cheek_raise > 0.014: angry *= 0.20
|
| 358 |
+
if avg_brow_in_h > -0.01: angry *= 0.20
|
| 359 |
+
if mouth_open > 0.07: angry *= 0.40
|
| 360 |
+
if brow_oblique > 0.04: angry *= 0.35
|
| 361 |
+
angry = clamp01(angry * 1.45)
|
| 362 |
+
|
| 363 |
+
# ββ SURPRISE β v3 βββββββββββββββββββββββββββββββββ
|
| 364 |
+
surprise = 0.0
|
| 365 |
+
if 0.40 < o_ratio < 1.60:
|
| 366 |
+
peak_dist = abs(o_ratio - 0.90)
|
| 367 |
+
surprise += clamp01(1.0 - peak_dist / 0.55) * 0.50
|
| 368 |
+
if 0.03 < mouth_open < 0.11:
|
| 369 |
+
surprise += clamp01((mouth_open - 0.03) / 0.06) * 0.25
|
| 370 |
+
if mouth_w > 0.40:
|
| 371 |
+
surprise *= max(0.10, 1.0 - (mouth_w - 0.40) / 0.10)
|
| 372 |
+
if avg_ear > 0.27:
|
| 373 |
+
surprise += clamp01((avg_ear - 0.27) / 0.10) * 0.40
|
| 374 |
+
if eye_open > 0.07:
|
| 375 |
+
surprise += clamp01((eye_open - 0.07) / 0.055) * 0.28
|
| 376 |
+
if avg_arch_h > 0.05:
|
| 377 |
+
surprise += clamp01((avg_arch_h - 0.05) / 0.055) * 0.35
|
| 378 |
+
if avg_brow_in_h > 0.04 and avg_brow_out_h > 0.02:
|
| 379 |
+
surprise += 0.15
|
| 380 |
+
if abs(brow_oblique) < 0.025:
|
| 381 |
+
surprise += 0.08
|
| 382 |
+
if mouth_curve < -0.005: surprise *= 0.30
|
| 383 |
+
if avg_brow_in_h < -0.03: surprise *= 0.20
|
| 384 |
+
if avg_ear < 0.18: surprise *= 0.25
|
| 385 |
+
if brow_inner_gap < 0.13: surprise *= 0.25
|
| 386 |
+
if mouth_w > 0.44: surprise *= 0.25
|
| 387 |
+
if mouth_open > 0.12 and o_ratio < 0.40:
|
| 388 |
+
surprise *= 0.30
|
| 389 |
+
surprise = clamp01(surprise * 1.35)
|
| 390 |
+
|
| 391 |
+
# ββ NEUTRAL βββββββββββββββββββββββββββββββββββββββ
|
| 392 |
+
neutral = max(0.0, 1.0 - (happy + sad + angry + surprise) * 1.05)
|
| 393 |
+
|
| 394 |
+
total = happy + sad + angry + surprise + neutral
|
| 395 |
+
if total > 1e-6:
|
| 396 |
+
happy /= total
|
| 397 |
+
sad /= total
|
| 398 |
+
angry /= total
|
| 399 |
+
surprise /= total
|
| 400 |
+
neutral /= total
|
| 401 |
+
|
| 402 |
+
return {k: round(float(v), 4)
|
| 403 |
+
for k, v in zip(EMOTIONS, [happy, sad, angry, surprise, neutral])}
|
| 404 |
+
|
| 405 |
+
except Exception as ex:
|
| 406 |
+
print(f"β οΈ Emotion detection error: {ex}")
|
| 407 |
+
return {e: (1.0 if e == "neutral" else 0.0) for e in EMOTIONS}
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 411 |
+
# DISPLAY HELPERS
|
| 412 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 413 |
+
def get_word_suggestions(partial, word_freq):
|
| 414 |
+
if not partial:
|
| 415 |
+
return []
|
| 416 |
+
p_up = partial.upper()
|
| 417 |
+
seen = {}
|
| 418 |
+
out = []
|
| 419 |
+
for word, freq in word_freq.most_common(20):
|
| 420 |
+
if word.startswith(p_up) and word != p_up:
|
| 421 |
+
out.append((word, freq)); seen[word] = True
|
| 422 |
+
for word in COMMON_WORDS:
|
| 423 |
+
if word.startswith(p_up) and word not in seen:
|
| 424 |
+
out.append((word, 0))
|
| 425 |
+
out.sort(key=lambda x: (-x[1], x[0]))
|
| 426 |
+
return [m[0] for m in out[:3]]
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
def draw_hand_landmarks(frame, hand_landmarks):
|
| 430 |
+
mp.solutions.drawing_utils.draw_landmarks(
|
| 431 |
+
frame, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS,
|
| 432 |
+
mp.solutions.drawing_utils.DrawingSpec(
|
| 433 |
+
color=(0, 255, 0), thickness=2, circle_radius=3),
|
| 434 |
+
mp.solutions.drawing_utils.DrawingSpec(
|
| 435 |
+
color=(0, 255, 255), thickness=2),
|
| 436 |
+
)
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def draw_letter_overlay(frame, letter, confidence):
|
| 440 |
+
if not SHOW_LETTER_OVERLAY or not letter or confidence <= 0:
|
| 441 |
+
return frame
|
| 442 |
+
x, y = OVERLAY_POSITION
|
| 443 |
+
overlay = frame.copy()
|
| 444 |
+
cv2.rectangle(overlay, (x - 10, y - 40), (x + 220, y + 20), (0, 0, 0), -1)
|
| 445 |
+
frame = cv2.addWeighted(overlay, 0.5, frame, 0.5, 0)
|
| 446 |
+
cv2.putText(frame, letter, (x, y),
|
| 447 |
+
cv2.FONT_HERSHEY_SIMPLEX, OVERLAY_FONT_SCALE,
|
| 448 |
+
OVERLAY_COLOR_LETTER, OVERLAY_THICKNESS, cv2.LINE_AA)
|
| 449 |
+
tw = cv2.getTextSize(letter, cv2.FONT_HERSHEY_SIMPLEX,
|
| 450 |
+
OVERLAY_FONT_SCALE, OVERLAY_THICKNESS)[0][0]
|
| 451 |
+
cv2.putText(frame, f"({confidence:.2f})", (x + tw + 10, y),
|
| 452 |
+
cv2.FONT_HERSHEY_SIMPLEX, OVERLAY_FONT_SCALE * 0.7,
|
| 453 |
+
OVERLAY_COLOR_CONF, OVERLAY_THICKNESS - 1, cv2.LINE_AA)
|
| 454 |
+
return frame
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 458 |
+
# CORE PROCESSING ENGINE
|
| 459 |
+
# ββββββββββββββββββββββββββοΏ½οΏ½οΏ½ββββββββββββββββββββββββββββββββββ
|
| 460 |
+
def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, frame_seq=None):
|
| 461 |
+
print("=" * 70)
|
| 462 |
+
print("π ISL Detection System Startingβ¦ (emotion engine v3)")
|
| 463 |
+
print(f" RUN_MODE = {RUN_MODE}")
|
| 464 |
+
print(f" CONF_THRESHOLD = {CONF_THRESHOLD}")
|
| 465 |
+
print(f" LETTER_HOLD_SEC = {LETTER_HOLD_SEC}s")
|
| 466 |
+
print(f" SharedMemory = {'enabled' if shm_name else 'disabled (base64 fallback)'}")
|
| 467 |
+
print("=" * 70)
|
| 468 |
+
|
| 469 |
+
# ββ SharedMemory init ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 470 |
+
shm = None
|
| 471 |
+
shm_array = None
|
| 472 |
+
|
| 473 |
+
if shm_name is not None:
|
| 474 |
+
try:
|
| 475 |
+
shm = shared_memory.SharedMemory(name=shm_name)
|
| 476 |
+
shm_array = np.ndarray((480, 640, 3), dtype=np.uint8, buffer=shm.buf)
|
| 477 |
+
print("β
SharedMemory connected (Zero-copy mode)")
|
| 478 |
+
except Exception as e:
|
| 479 |
+
print(f"β SharedMemory init failed: {e}")
|
| 480 |
+
return
|
| 481 |
+
|
| 482 |
+
# ββ Load ISL model βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 483 |
+
try:
|
| 484 |
+
model = load_model(ISL_MODEL_PATH)
|
| 485 |
+
print("β
ISL model loaded")
|
| 486 |
+
print(f" Input shape : {model.input_shape}")
|
| 487 |
+
print(f" Output shape : {model.output_shape}")
|
| 488 |
+
except Exception as e:
|
| 489 |
+
print(f"β Failed to load model: {e}")
|
| 490 |
+
if shm:
|
| 491 |
+
shm.close()
|
| 492 |
+
return
|
| 493 |
+
|
| 494 |
+
try:
|
| 495 |
+
model.predict(np.zeros((1, 42), np.float32), verbose=0)
|
| 496 |
+
print("β
Model warmed up")
|
| 497 |
+
except Exception as e:
|
| 498 |
+
print(f"β οΈ Warmup: {e}")
|
| 499 |
+
|
| 500 |
+
alphabet = list(string.ascii_uppercase) + [str(i) for i in range(1, 10)]
|
| 501 |
+
print(f" Alphabet : {len(alphabet)} classes (model outputs {model.output_shape[1]})")
|
| 502 |
+
|
| 503 |
+
# ββ Detection state ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 504 |
+
current_word = []
|
| 505 |
+
sentence = []
|
| 506 |
+
no_hand_frames = 0
|
| 507 |
+
|
| 508 |
+
candidate_letter = ""
|
| 509 |
+
candidate_since = 0.0
|
| 510 |
+
last_accepted_letter = ""
|
| 511 |
+
last_accepted_time = 0.0
|
| 512 |
+
|
| 513 |
+
letter_confidence_buffer = deque(maxlen=4)
|
| 514 |
+
word_frequency = Counter()
|
| 515 |
+
current_detected_letter = ""
|
| 516 |
+
current_confidence = 0.0
|
| 517 |
+
|
| 518 |
+
emotion_history = deque(maxlen=SMOOTHING_FRAMES)
|
| 519 |
+
current_emotion = "neutral"
|
| 520 |
+
emotion_scores = {e: (1.0 if e == "neutral" else 0.0) for e in EMOTIONS}
|
| 521 |
+
emotion_timeline = deque(maxlen=100)
|
| 522 |
+
emotion_frame_counter = 0
|
| 523 |
+
|
| 524 |
+
frame_count = 0
|
| 525 |
+
fps_frame_times = deque(maxlen=30)
|
| 526 |
+
|
| 527 |
+
stats = {
|
| 528 |
+
"letters_detected": 0,
|
| 529 |
+
"words_formed": 0,
|
| 530 |
+
"sentences_formed": 0,
|
| 531 |
+
"emotion_changes": 0,
|
| 532 |
+
"session_start": time.time(),
|
| 533 |
+
}
|
| 534 |
+
|
| 535 |
+
cmd_queue = shared_state.command_queue
|
| 536 |
+
ui_queue = shared_state.ui_queue
|
| 537 |
+
|
| 538 |
+
# ββ Camera init ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 539 |
+
print("π₯ Initializing cameraβ¦")
|
| 540 |
+
if RUN_MODE == "LOCAL":
|
| 541 |
+
# FIX 2: cv2.CAP_DSHOW is Windows-only β use platform check
|
| 542 |
+
if platform.system() == "Windows":
|
| 543 |
+
cap = cv2.VideoCapture(CAM_INDEX, cv2.CAP_DSHOW)
|
| 544 |
+
else:
|
| 545 |
+
cap = cv2.VideoCapture(CAM_INDEX) # Linux/Mac β no CAP_DSHOW
|
| 546 |
+
|
| 547 |
+
if not cap.isOpened():
|
| 548 |
+
cap = cv2.VideoCapture(CAM_INDEX) # fallback without backend flag
|
| 549 |
+
if not cap.isOpened():
|
| 550 |
+
print("β Camera not found β is another app using it?")
|
| 551 |
+
if shm:
|
| 552 |
+
shm.close()
|
| 553 |
+
return
|
| 554 |
+
|
| 555 |
+
# FIX 4: Set MJPG codec BEFORE resolution β forces hardware MJPEG path
|
| 556 |
+
# on built-in laptop webcams, avoiding the raw uncompressed buffer that
|
| 557 |
+
# causes the horizontal static/noise stripes seen in the MJPEG stream.
|
| 558 |
+
# Must be set before width/height or it has no effect on some drivers.
|
| 559 |
+
if platform.system() == "Windows":
|
| 560 |
+
cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'))
|
| 561 |
+
|
| 562 |
+
cap.set(cv2.CAP_PROP_FRAME_WIDTH, CAMERA_RESOLUTION[0])
|
| 563 |
+
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, CAMERA_RESOLUTION[1])
|
| 564 |
+
cap.set(cv2.CAP_PROP_FPS, TARGET_FPS)
|
| 565 |
+
|
| 566 |
+
# FIX 5: Buffer size must be 1 to prevent stale frame accumulation.
|
| 567 |
+
# Set AFTER resolution so the driver doesn't reset it.
|
| 568 |
+
cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
|
| 569 |
+
|
| 570 |
+
# FIX 4 continued: Flush 30 frames (was 10) to drain the DirectShow
|
| 571 |
+
# ring buffer. Built-in webcams pre-fill ~20 frames before stabilizing.
|
| 572 |
+
print("οΏ½οΏ½οΏ½οΏ½ Flushing camera buffer (30 frames)β¦")
|
| 573 |
+
for _ in range(30):
|
| 574 |
+
cap.read()
|
| 575 |
+
|
| 576 |
+
# Verify the camera is actually producing valid frames
|
| 577 |
+
ret, test_frame = cap.read()
|
| 578 |
+
if not ret or test_frame is None:
|
| 579 |
+
print("β Camera flush failed β no valid frame received")
|
| 580 |
+
cap.release()
|
| 581 |
+
if shm:
|
| 582 |
+
shm.close()
|
| 583 |
+
return
|
| 584 |
+
|
| 585 |
+
actual_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
| 586 |
+
actual_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
| 587 |
+
print(f"β
Camera: {actual_w}x{actual_h}")
|
| 588 |
+
print("β
Camera ready!")
|
| 589 |
+
else:
|
| 590 |
+
cap = None
|
| 591 |
+
print("βοΈ CLOUD mode β waiting for browser frames via socket")
|
| 592 |
+
|
| 593 |
+
print("=" * 70)
|
| 594 |
+
|
| 595 |
+
mp_hands = mp.solutions.hands
|
| 596 |
+
mp_face = mp.solutions.face_mesh
|
| 597 |
+
|
| 598 |
+
with mp_hands.Hands(
|
| 599 |
+
static_image_mode=False,
|
| 600 |
+
model_complexity=0,
|
| 601 |
+
max_num_hands=1,
|
| 602 |
+
min_detection_confidence=0.6,
|
| 603 |
+
min_tracking_confidence=0.6,
|
| 604 |
+
) as hands, \
|
| 605 |
+
mp_face.FaceMesh(
|
| 606 |
+
static_image_mode=False,
|
| 607 |
+
max_num_faces=1,
|
| 608 |
+
refine_landmarks=True,
|
| 609 |
+
min_detection_confidence=0.6,
|
| 610 |
+
min_tracking_confidence=0.6,
|
| 611 |
+
) as face_mesh:
|
| 612 |
+
|
| 613 |
+
print("β
MediaPipe initialized β entering main loop")
|
| 614 |
+
|
| 615 |
+
while True:
|
| 616 |
+
t0 = time.time()
|
| 617 |
+
now = t0
|
| 618 |
+
|
| 619 |
+
# ββ Commands βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 620 |
+
try:
|
| 621 |
+
while not cmd_queue.empty():
|
| 622 |
+
cmd = cmd_queue.get_nowait()
|
| 623 |
+
action = cmd.get("action", "")
|
| 624 |
+
|
| 625 |
+
if action == "reset":
|
| 626 |
+
current_word = []
|
| 627 |
+
sentence = []
|
| 628 |
+
no_hand_frames = 0
|
| 629 |
+
candidate_letter = ""
|
| 630 |
+
candidate_since = 0.0
|
| 631 |
+
last_accepted_letter = ""
|
| 632 |
+
last_accepted_time = 0.0
|
| 633 |
+
letter_confidence_buffer.clear()
|
| 634 |
+
current_detected_letter = ""
|
| 635 |
+
current_confidence = 0.0
|
| 636 |
+
print("π Reset")
|
| 637 |
+
|
| 638 |
+
elif action == "backspace":
|
| 639 |
+
if current_word:
|
| 640 |
+
current_word.pop()
|
| 641 |
+
elif sentence:
|
| 642 |
+
sentence.pop()
|
| 643 |
+
|
| 644 |
+
elif action == "accept_suggestion":
|
| 645 |
+
w = cmd.get("word", "")
|
| 646 |
+
if w:
|
| 647 |
+
current_word = list(w)
|
| 648 |
+
|
| 649 |
+
elif action == "speak":
|
| 650 |
+
txt = (cmd.get("text", "")
|
| 651 |
+
or (" ".join(sentence) if sentence
|
| 652 |
+
else "".join(current_word)))
|
| 653 |
+
if txt:
|
| 654 |
+
speak_text(txt) # silently skipped in CLOUD mode
|
| 655 |
+
|
| 656 |
+
elif action == "stop":
|
| 657 |
+
if cap:
|
| 658 |
+
cap.release()
|
| 659 |
+
if shm:
|
| 660 |
+
shm.close()
|
| 661 |
+
return
|
| 662 |
+
except Exception:
|
| 663 |
+
pass
|
| 664 |
+
|
| 665 |
+
# ββ Capture ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 666 |
+
if RUN_MODE == "LOCAL":
|
| 667 |
+
ret, raw = cap.read()
|
| 668 |
+
if not ret or raw is None:
|
| 669 |
+
print("β οΈ Camera read failed β retrying")
|
| 670 |
+
time.sleep(0.02)
|
| 671 |
+
continue
|
| 672 |
+
else:
|
| 673 |
+
fd = getattr(shared_state, "_cloud_frame", None)
|
| 674 |
+
if fd is None:
|
| 675 |
+
time.sleep(0.03)
|
| 676 |
+
continue
|
| 677 |
+
shared_state._cloud_frame = None
|
| 678 |
+
try:
|
| 679 |
+
raw = cv2.imdecode(
|
| 680 |
+
np.frombuffer(base64.b64decode(fd), np.uint8),
|
| 681 |
+
cv2.IMREAD_COLOR)
|
| 682 |
+
if raw is None:
|
| 683 |
+
continue
|
| 684 |
+
except Exception:
|
| 685 |
+
continue
|
| 686 |
+
|
| 687 |
+
frame = cv2.resize(raw, (FRAME_W, FRAME_H))
|
| 688 |
+
frame = cv2.flip(frame, 1)
|
| 689 |
+
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 690 |
+
frame_rgb.flags.writeable = False
|
| 691 |
+
|
| 692 |
+
# ββ Hand detection βββββββββββββββββββββββββββββββββββββββββββββ
|
| 693 |
+
hand_results = hands.process(frame_rgb)
|
| 694 |
+
frame_rgb.flags.writeable = True
|
| 695 |
+
|
| 696 |
+
detected_letter = ""
|
| 697 |
+
hand_detected = False
|
| 698 |
+
|
| 699 |
+
if hand_results and hand_results.multi_hand_landmarks:
|
| 700 |
+
hand_detected = True
|
| 701 |
+
no_hand_frames = 0
|
| 702 |
+
|
| 703 |
+
for hlm in hand_results.multi_hand_landmarks:
|
| 704 |
+
draw_hand_landmarks(frame, hlm)
|
| 705 |
+
try:
|
| 706 |
+
lm_list = calc_landmark_list(frame, hlm)
|
| 707 |
+
pre_list = pre_process_landmark(lm_list)
|
| 708 |
+
|
| 709 |
+
if model is not None and len(pre_list) == 42:
|
| 710 |
+
features = np.array(pre_list, dtype=np.float32).reshape(1, 42)
|
| 711 |
+
pred = model.predict(features, verbose=0)
|
| 712 |
+
prob = float(np.max(pred))
|
| 713 |
+
raw_letter = alphabet[int(np.argmax(pred))]
|
| 714 |
+
|
| 715 |
+
current_detected_letter = raw_letter
|
| 716 |
+
current_confidence = prob
|
| 717 |
+
|
| 718 |
+
print(f"DEBUG β {raw_letter} {prob:.4f}")
|
| 719 |
+
|
| 720 |
+
if prob >= CONF_THRESHOLD:
|
| 721 |
+
if raw_letter == candidate_letter:
|
| 722 |
+
letter_confidence_buffer.append(prob)
|
| 723 |
+
else:
|
| 724 |
+
candidate_letter = raw_letter
|
| 725 |
+
candidate_since = now
|
| 726 |
+
letter_confidence_buffer.clear()
|
| 727 |
+
letter_confidence_buffer.append(prob)
|
| 728 |
+
|
| 729 |
+
avg_conf = float(np.mean(letter_confidence_buffer))
|
| 730 |
+
hold_time = now - candidate_since
|
| 731 |
+
|
| 732 |
+
if hold_time >= LETTER_HOLD_SEC and avg_conf >= CONF_THRESHOLD:
|
| 733 |
+
same_letter_ok = (
|
| 734 |
+
raw_letter != last_accepted_letter or
|
| 735 |
+
(now - last_accepted_time) >= SAME_LETTER_COOLDOWN_SEC
|
| 736 |
+
)
|
| 737 |
+
if same_letter_ok:
|
| 738 |
+
detected_letter = raw_letter
|
| 739 |
+
last_accepted_letter = raw_letter
|
| 740 |
+
last_accepted_time = now
|
| 741 |
+
current_word.append(raw_letter)
|
| 742 |
+
stats["letters_detected"] += 1
|
| 743 |
+
print(f"βοΈ Letter: {raw_letter} "
|
| 744 |
+
f"(held {hold_time:.2f}s, conf {avg_conf:.3f})")
|
| 745 |
+
candidate_letter = ""
|
| 746 |
+
candidate_since = 0.0
|
| 747 |
+
letter_confidence_buffer.clear()
|
| 748 |
+
else:
|
| 749 |
+
candidate_letter = ""
|
| 750 |
+
candidate_since = 0.0
|
| 751 |
+
letter_confidence_buffer.clear()
|
| 752 |
+
|
| 753 |
+
except Exception as e:
|
| 754 |
+
print(f"β οΈ Prediction error: {e}")
|
| 755 |
+
|
| 756 |
+
else:
|
| 757 |
+
no_hand_frames += 1
|
| 758 |
+
letter_confidence_buffer.clear()
|
| 759 |
+
candidate_letter = ""
|
| 760 |
+
candidate_since = 0.0
|
| 761 |
+
current_detected_letter = ""
|
| 762 |
+
current_confidence = 0.0
|
| 763 |
+
|
| 764 |
+
if no_hand_frames >= SPACE_THRESHOLD and current_word:
|
| 765 |
+
word = "".join(current_word)
|
| 766 |
+
sentence.append(word)
|
| 767 |
+
word_frequency[word] += 1
|
| 768 |
+
stats["words_formed"] += 1
|
| 769 |
+
print(f"π Word committed: {word}")
|
| 770 |
+
if ENABLE_AUTO_TTS:
|
| 771 |
+
speak_text(word) # silently skipped in CLOUD mode
|
| 772 |
+
current_word = []
|
| 773 |
+
last_accepted_letter = ""
|
| 774 |
+
last_accepted_time = 0.0
|
| 775 |
+
no_hand_frames = 0
|
| 776 |
+
|
| 777 |
+
frame = draw_letter_overlay(frame, current_detected_letter, current_confidence)
|
| 778 |
+
|
| 779 |
+
# ββ Emotion (every N frames) βββββββββββββββββββββββββββββββββββ
|
| 780 |
+
emotion_frame_counter += 1
|
| 781 |
+
if emotion_frame_counter % EMOTION_PROCESS_INTERVAL == 0:
|
| 782 |
+
ns = {e: (1.0 if e == "neutral" else 0.0) for e in EMOTIONS}
|
| 783 |
+
try:
|
| 784 |
+
cr = cv2.cvtColor(apply_clahe(frame), cv2.COLOR_BGR2RGB)
|
| 785 |
+
cr.flags.writeable = False
|
| 786 |
+
mr = face_mesh.process(cr)
|
| 787 |
+
cr.flags.writeable = True
|
| 788 |
+
if mr and mr.multi_face_landmarks:
|
| 789 |
+
ns = detect_emotion_from_landmarks(
|
| 790 |
+
calc_landmark_list(frame, mr.multi_face_landmarks[0]),
|
| 791 |
+
frame.shape)
|
| 792 |
+
except Exception:
|
| 793 |
+
pass
|
| 794 |
+
|
| 795 |
+
emotion_history.append(ns)
|
| 796 |
+
avg_e = defaultdict(float)
|
| 797 |
+
for d in emotion_history:
|
| 798 |
+
for k, v in d.items():
|
| 799 |
+
avg_e[k] += v
|
| 800 |
+
for k in avg_e:
|
| 801 |
+
avg_e[k] /= len(emotion_history)
|
| 802 |
+
|
| 803 |
+
dominant = (max(avg_e, key=avg_e.get)
|
| 804 |
+
if avg_e and max(avg_e.values()) >= MIN_CONF_TO_SHOW
|
| 805 |
+
else "neutral")
|
| 806 |
+
|
| 807 |
+
if current_emotion != dominant:
|
| 808 |
+
stats["emotion_changes"] += 1
|
| 809 |
+
scores_str = " | ".join(
|
| 810 |
+
[f"{k.upper()}:{avg_e[k]:.3f}" for k in EMOTIONS]
|
| 811 |
+
)
|
| 812 |
+
print(f"π Emotion: {current_emotion.upper()} β {dominant.upper()}")
|
| 813 |
+
print(f" Scores : {scores_str}")
|
| 814 |
+
|
| 815 |
+
current_emotion = dominant
|
| 816 |
+
emotion_scores = dict(avg_e)
|
| 817 |
+
emotion_timeline.append({
|
| 818 |
+
"time": time.time(),
|
| 819 |
+
"emotion": dominant,
|
| 820 |
+
"scores": dict(avg_e),
|
| 821 |
+
})
|
| 822 |
+
|
| 823 |
+
# ββ FPS ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 824 |
+
frame_count += 1
|
| 825 |
+
fps_frame_times.append(time.time())
|
| 826 |
+
if len(fps_frame_times) > 1:
|
| 827 |
+
td = fps_frame_times[-1] - fps_frame_times[0]
|
| 828 |
+
if td > 0:
|
| 829 |
+
shared_state.processing_fps.value = (len(fps_frame_times) - 1) / td
|
| 830 |
+
|
| 831 |
+
# ββ Frame output βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 832 |
+
if shm_array is not None:
|
| 833 |
+
shm_array[:] = frame
|
| 834 |
+
if frame_lock_val is not None:
|
| 835 |
+
frame_lock_val.value = 1
|
| 836 |
+
if frame_seq is not None:
|
| 837 |
+
frame_seq.value += 1
|
| 838 |
+
frame_base64 = None
|
| 839 |
+
else:
|
| 840 |
+
_, buffer_img = cv2.imencode(
|
| 841 |
+
".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 65])
|
| 842 |
+
frame_base64 = base64.b64encode(buffer_img.tobytes()).decode("utf-8")
|
| 843 |
+
|
| 844 |
+
# ββ State packet βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 845 |
+
state_packet = {
|
| 846 |
+
"frame": frame_base64 if frame_base64 else "",
|
| 847 |
+
"current_word": "".join(current_word),
|
| 848 |
+
"sentence": " ".join(sentence),
|
| 849 |
+
"suggestions": get_word_suggestions(
|
| 850 |
+
"".join(current_word), word_frequency),
|
| 851 |
+
"emotion": current_emotion,
|
| 852 |
+
"emotion_scores": emotion_scores,
|
| 853 |
+
"emotion_timeline": list(emotion_timeline)[-20:],
|
| 854 |
+
"stats": stats.copy(),
|
| 855 |
+
"detected_letter": detected_letter,
|
| 856 |
+
"hand_detected": hand_detected,
|
| 857 |
+
"fps": shared_state.processing_fps.value,
|
| 858 |
+
"timestamp": time.time(),
|
| 859 |
+
"speaking": speaking_word,
|
| 860 |
+
"overlay_letter": current_detected_letter,
|
| 861 |
+
"overlay_confidence": current_confidence,
|
| 862 |
+
"hold_progress": min(1.0,
|
| 863 |
+
(now - candidate_since) / LETTER_HOLD_SEC)
|
| 864 |
+
if candidate_letter else 0.0,
|
| 865 |
+
}
|
| 866 |
+
|
| 867 |
+
try:
|
| 868 |
+
ui_queue.put_nowait(state_packet)
|
| 869 |
+
except Exception:
|
| 870 |
+
try:
|
| 871 |
+
ui_queue.get_nowait()
|
| 872 |
+
ui_queue.put_nowait(state_packet)
|
| 873 |
+
except Exception:
|
| 874 |
+
pass
|
| 875 |
+
|
| 876 |
+
if frame_count == 1:
|
| 877 |
+
print("π€ First frame sent β dashboard can read now")
|
| 878 |
+
if shm_array is not None:
|
| 879 |
+
print("π€ SharedMemory active β zero-copy mode")
|
| 880 |
+
|
| 881 |
+
# ββ Frame-rate limiter βββββββββββββββββββββββββββββββββββββββββ
|
| 882 |
+
wait = (1.0 / TARGET_FPS) - (time.time() - t0)
|
| 883 |
+
if wait > 0:
|
| 884 |
+
time.sleep(wait)
|
| 885 |
+
|
| 886 |
+
if cap:
|
| 887 |
+
cap.release()
|
| 888 |
+
if shm is not None:
|
| 889 |
+
shm.close()
|
| 890 |
+
|
| 891 |
+
print("β
System stopped")
|
| 892 |
+
|
| 893 |
+
|
| 894 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 895 |
+
# STANDALONE ENTRY POINT
|
| 896 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 897 |
+
if __name__ == "__main__":
|
| 898 |
+
FRAME_BYTES = FRAME_W * FRAME_H * 3
|
| 899 |
+
|
| 900 |
+
# FIX 3: Clean up leftover shared memory from a previous crash
|
| 901 |
+
# Without this, restarting after a crash raises FileExistsError
|
| 902 |
+
try:
|
| 903 |
+
_old_shm = shared_memory.SharedMemory(name="isl_frame_shm")
|
| 904 |
+
_old_shm.close()
|
| 905 |
+
_old_shm.unlink()
|
| 906 |
+
print("π§Ή Cleaned up leftover shared memory from previous run")
|
| 907 |
+
except FileNotFoundError:
|
| 908 |
+
pass # normal β no leftover
|
| 909 |
+
|
| 910 |
+
shm = shared_memory.SharedMemory(create=True, size=FRAME_BYTES, name="isl_frame_shm")
|
| 911 |
+
seq = Value(ctypes.c_uint64, 0)
|
| 912 |
+
lock = Value(ctypes.c_bool, False)
|
| 913 |
+
ss = SharedState()
|
| 914 |
+
try:
|
| 915 |
+
core_processing_engine(ss, shm_name=shm.name, frame_lock_val=lock, frame_seq=seq)
|
| 916 |
+
finally:
|
| 917 |
+
shm.close()
|
| 918 |
+
shm.unlink()
|
isl_ui_dashboard.py
ADDED
|
@@ -0,0 +1,941 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# isl_ui_dashboard.py β CLOUD PRODUCTION VERSION
|
| 2 |
+
#
|
| 3 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 4 |
+
# CLOUD FIXES APPLIED (4 changes only β everything else identical):
|
| 5 |
+
#
|
| 6 |
+
# FIX 1 β Browser camera: HTML now captures + emits browser_frame via socket
|
| 7 |
+
# FIX 2 β Nav links: localhost:5000/5001 β dynamic window.location URLs
|
| 8 |
+
# FIX 3 β _cloud_frame scope: shared_state stored as module-level variable
|
| 9 |
+
# FIX 4 β /avatar route added
|
| 10 |
+
#
|
| 11 |
+
# LOCAL CAMERA FIX:
|
| 12 |
+
# FIX 5 β generate_frames() now waits for frame_seq > 0 before streaming.
|
| 13 |
+
# Without this, the MJPEG client receives raw uninitialized
|
| 14 |
+
# SharedMemory bytes (all zeros or garbage) before the detection
|
| 15 |
+
# engine writes the first valid frame β causing static/noise on
|
| 16 |
+
# the built-in laptop webcam display.
|
| 17 |
+
# Also removed unreachable `return` after infinite fallback loop.
|
| 18 |
+
#
|
| 19 |
+
# TRANSPORT FIX:
|
| 20 |
+
# FIX 6 β transports updated to ['websocket', 'polling'] with upgrade:true
|
| 21 |
+
# in the embedded HTML Socket.IO client. Polling-only breaks camera
|
| 22 |
+
# streaming on HuggingFace Spaces (HTTPS) and other cloud hosts.
|
| 23 |
+
# WebSocket is tried first; polling is the automatic fallback.
|
| 24 |
+
#
|
| 25 |
+
# ALL original features preserved:
|
| 26 |
+
# β
MJPEG /video_feed from SharedMemory (zero-copy)
|
| 27 |
+
# β
Socket.IO state_update events (emotion, word, suggestions, stats)
|
| 28 |
+
# β
Speak, Backspace, Reset commands
|
| 29 |
+
# β
Emotion bars, Chart.js timeline, word suggestions
|
| 30 |
+
# β
async_mode='threading', transports=['websocket','polling']
|
| 31 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 32 |
+
|
| 33 |
+
import time
|
| 34 |
+
import threading
|
| 35 |
+
import numpy as np
|
| 36 |
+
import cv2
|
| 37 |
+
import multiprocessing.shared_memory
|
| 38 |
+
|
| 39 |
+
from queue import Queue, Empty
|
| 40 |
+
from flask import Flask, render_template_string, Response, jsonify, request
|
| 41 |
+
from flask_socketio import SocketIO
|
| 42 |
+
|
| 43 |
+
# ββ App & SocketIO ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 44 |
+
app = Flask(__name__)
|
| 45 |
+
app.config["SECRET_KEY"] = "isl_secret_2024"
|
| 46 |
+
|
| 47 |
+
socketio = SocketIO(
|
| 48 |
+
app,
|
| 49 |
+
async_mode="threading",
|
| 50 |
+
cors_allowed_origins="*",
|
| 51 |
+
ping_timeout=60,
|
| 52 |
+
ping_interval=25,
|
| 53 |
+
logger=False,
|
| 54 |
+
engineio_logger=False,
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
# ββ Module-level state ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 58 |
+
_local_state_q: Queue = Queue(100)
|
| 59 |
+
FRAME_W, FRAME_H = 640, 480
|
| 60 |
+
_shm_name = None
|
| 61 |
+
_frame_seq = None
|
| 62 |
+
|
| 63 |
+
# FIX 3: shared_state stored at module level so on_browser_frame() can
|
| 64 |
+
# set _cloud_frame without being inside dashboard_process() scope.
|
| 65 |
+
_shared_state = None
|
| 66 |
+
|
| 67 |
+
# ββ HTML ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 68 |
+
HTML = r"""
|
| 69 |
+
<!DOCTYPE html>
|
| 70 |
+
<html lang="en">
|
| 71 |
+
<head>
|
| 72 |
+
<meta charset="UTF-8"/>
|
| 73 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover"/>
|
| 74 |
+
<title>ISL Gesture Recognition β AI Sign Language</title>
|
| 75 |
+
<link rel="preconnect" href="https://fonts.googleapis.com"/>
|
| 76 |
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin/>
|
| 77 |
+
<link href="https://fonts.googleapis.com/css2?family=Playfair+Display:wght@400;600;700&family=DM+Sans:wght@300;400;500;600&family=DM+Mono:wght@400;500&display=swap" rel="stylesheet"/>
|
| 78 |
+
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.7.2/socket.io.min.js"></script>
|
| 79 |
+
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
|
| 80 |
+
<style>
|
| 81 |
+
:root {
|
| 82 |
+
--bg: #0d1117;
|
| 83 |
+
--surf: #131922;
|
| 84 |
+
--card: #1a2233;
|
| 85 |
+
--bdr: rgba(212,168,83,.13);
|
| 86 |
+
--bdrm: rgba(212,168,83,.30);
|
| 87 |
+
--bdrh: rgba(212,168,83,.58);
|
| 88 |
+
--gold: #d4a853;
|
| 89 |
+
--goldb: #f0c76a;
|
| 90 |
+
--goldd: #a07c3a;
|
| 91 |
+
--sage: #7eb89a;
|
| 92 |
+
--saged: #4e7d66;
|
| 93 |
+
--txt: #f0ede6;
|
| 94 |
+
--txtm: #a8a096;
|
| 95 |
+
--txtl: #5a5650;
|
| 96 |
+
--red: #c0614a;
|
| 97 |
+
--r8: 8px;
|
| 98 |
+
--r14: 14px;
|
| 99 |
+
--fD: 'Playfair Display', Georgia, serif;
|
| 100 |
+
--fB: 'DM Sans', system-ui, sans-serif;
|
| 101 |
+
--fM: 'DM Mono', monospace;
|
| 102 |
+
}
|
| 103 |
+
*, *::before, *::after { margin:0; padding:0; box-sizing:border-box; }
|
| 104 |
+
html, body { height:100%; overflow:hidden; }
|
| 105 |
+
body {
|
| 106 |
+
font-family: var(--fB);
|
| 107 |
+
background: var(--bg);
|
| 108 |
+
color: var(--txt);
|
| 109 |
+
-webkit-font-smoothing: antialiased;
|
| 110 |
+
display: flex;
|
| 111 |
+
flex-direction: column;
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
#app { display:flex; flex:1; overflow:hidden; height:100%; }
|
| 115 |
+
|
| 116 |
+
/* ββ Camera pane ββ */
|
| 117 |
+
#cam-pane {
|
| 118 |
+
flex: 1;
|
| 119 |
+
display: flex;
|
| 120 |
+
flex-direction: column;
|
| 121 |
+
min-width: 0;
|
| 122 |
+
background: var(--bg);
|
| 123 |
+
position: relative;
|
| 124 |
+
overflow: hidden;
|
| 125 |
+
}
|
| 126 |
+
.topbar {
|
| 127 |
+
display: flex;
|
| 128 |
+
align-items: center;
|
| 129 |
+
justify-content: space-between;
|
| 130 |
+
padding: 10px 16px 0;
|
| 131 |
+
flex-shrink: 0;
|
| 132 |
+
position: relative;
|
| 133 |
+
z-index: 5;
|
| 134 |
+
}
|
| 135 |
+
.brand { display:flex; align-items:baseline; gap:8px; }
|
| 136 |
+
.brand-t { font-family:var(--fD); font-size:1.1rem; font-weight:700; color:var(--gold); }
|
| 137 |
+
.brand-s { font-size:.65rem; color:var(--txtl); text-transform:uppercase; letter-spacing:.07em; }
|
| 138 |
+
.live-badge {
|
| 139 |
+
font-size:.65rem; font-family:var(--fM); color:var(--sage);
|
| 140 |
+
background:rgba(126,184,154,.10); border:1px solid rgba(126,184,154,.25);
|
| 141 |
+
padding:3px 9px; border-radius:20px; transition:all .3s;
|
| 142 |
+
}
|
| 143 |
+
.live-badge.rec { color:var(--gold); background:rgba(212,168,83,.1); border-color:var(--bdrm); animation:pd .9s infinite; }
|
| 144 |
+
.live-badge.err { color:var(--red); background:rgba(192,97,74,.1); border-color:rgba(192,97,74,.3); }
|
| 145 |
+
@keyframes pd { 0%,100%{opacity:1} 50%{opacity:.55} }
|
| 146 |
+
|
| 147 |
+
#camera-feed {
|
| 148 |
+
flex: 1; width:100%; display:block; object-fit:cover; background:#000;
|
| 149 |
+
}
|
| 150 |
+
.cam-overlay {
|
| 151 |
+
position:absolute; bottom:0; left:0; right:0;
|
| 152 |
+
padding:12px 18px 14px;
|
| 153 |
+
background: linear-gradient(to top, rgba(13,17,23,1) 55%, transparent);
|
| 154 |
+
display:flex; align-items:flex-end; justify-content:space-between;
|
| 155 |
+
gap:10px; pointer-events:none; z-index:5;
|
| 156 |
+
}
|
| 157 |
+
.co-letter {
|
| 158 |
+
font-family:var(--fD); font-size:clamp(2rem,5vw,3.2rem);
|
| 159 |
+
font-weight:700; color:var(--goldb); line-height:1;
|
| 160 |
+
}
|
| 161 |
+
.co-word {
|
| 162 |
+
font-size:.85rem; font-weight:600; color:var(--gold);
|
| 163 |
+
font-family:var(--fM); letter-spacing:.05em;
|
| 164 |
+
}
|
| 165 |
+
.co-desc { font-size:.72rem; color:var(--txtm); text-align:right; max-width:200px; line-height:1.45; }
|
| 166 |
+
.co-progress { position:absolute; bottom:0; left:0; right:0; height:3px; background:rgba(212,168,83,.08); }
|
| 167 |
+
.co-bar {
|
| 168 |
+
height:100%; width:0%;
|
| 169 |
+
background:linear-gradient(90deg, var(--goldd), var(--goldb));
|
| 170 |
+
transition:width .3s ease; position:relative; overflow:hidden;
|
| 171 |
+
}
|
| 172 |
+
.co-bar::after {
|
| 173 |
+
content:''; position:absolute; inset:0;
|
| 174 |
+
background:linear-gradient(90deg,transparent,rgba(255,255,255,.3),transparent);
|
| 175 |
+
animation:shimmer 1.8s infinite;
|
| 176 |
+
}
|
| 177 |
+
@keyframes shimmer { from{transform:translateX(-100%)} to{transform:translateX(200%)} }
|
| 178 |
+
.cam-placeholder {
|
| 179 |
+
position:absolute; inset:0;
|
| 180 |
+
display:flex; flex-direction:column;
|
| 181 |
+
align-items:center; justify-content:center;
|
| 182 |
+
gap:10px; color:var(--txtl); font-size:.8rem; z-index:2; pointer-events:none;
|
| 183 |
+
}
|
| 184 |
+
.cam-placeholder .ph-icon { font-size:2.8rem; opacity:.25; }
|
| 185 |
+
|
| 186 |
+
/* ββ Control pane ββ */
|
| 187 |
+
#ctrl-pane {
|
| 188 |
+
width:340px; flex-shrink:0; display:flex; flex-direction:column;
|
| 189 |
+
background:var(--surf); border-left:1px solid var(--bdr);
|
| 190 |
+
overflow-y:auto; overflow-x:hidden;
|
| 191 |
+
}
|
| 192 |
+
#ctrl-pane::-webkit-scrollbar { width:4px; }
|
| 193 |
+
#ctrl-pane::-webkit-scrollbar-thumb { background:var(--bdrm); border-radius:4px; }
|
| 194 |
+
|
| 195 |
+
@media (max-width:720px) {
|
| 196 |
+
html,body { overflow:hidden; }
|
| 197 |
+
#app { flex-direction:column; height:100%; }
|
| 198 |
+
#cam-pane { flex:0 0 46vh; min-height:260px; max-height:46vh; width:100%; }
|
| 199 |
+
#ctrl-pane { width:100%; flex:1; border-left:none; border-top:1px solid var(--bdr); overflow-y:auto; }
|
| 200 |
+
.topbar { padding:8px 12px 0; }
|
| 201 |
+
.brand-t { font-size:.95rem; }
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
.cpanel { padding:14px 16px; display:flex; flex-direction:column; gap:12px; }
|
| 205 |
+
|
| 206 |
+
.sec-label {
|
| 207 |
+
font-size:.65rem; font-weight:600; text-transform:uppercase;
|
| 208 |
+
letter-spacing:.08em; color:var(--txtl);
|
| 209 |
+
display:flex; align-items:center; gap:6px; font-family:var(--fM);
|
| 210 |
+
}
|
| 211 |
+
|
| 212 |
+
/* ββ Status pill ββ */
|
| 213 |
+
.status {
|
| 214 |
+
display:flex; align-items:center; gap:7px; padding:7px 13px;
|
| 215 |
+
border-radius:30px; font-size:.73rem; font-weight:500;
|
| 216 |
+
border:1px solid var(--bdr); background:var(--card); color:var(--txtm);
|
| 217 |
+
transition:all .3s; min-height:34px; font-family:var(--fM);
|
| 218 |
+
}
|
| 219 |
+
.sdot { width:7px; height:7px; border-radius:50%; background:var(--txtl); flex-shrink:0; transition:all .3s; }
|
| 220 |
+
.status.act .sdot { background:var(--sage); box-shadow:0 0 8px var(--sage); animation:pd 1.4s infinite; }
|
| 221 |
+
.status.act { border-color:rgba(126,184,154,.3); color:var(--sage); }
|
| 222 |
+
.status.err .sdot { background:var(--red); }
|
| 223 |
+
.status.err { border-color:rgba(192,97,74,.3); color:var(--red); }
|
| 224 |
+
|
| 225 |
+
/* ββ Text display ββ */
|
| 226 |
+
.text-display {
|
| 227 |
+
background:var(--card); border:1px solid var(--bdr);
|
| 228 |
+
border-radius:var(--r14); padding:10px 13px; min-height:52px;
|
| 229 |
+
}
|
| 230 |
+
.cur-word {
|
| 231 |
+
font-size:1.15rem; font-weight:700; color:var(--goldb);
|
| 232 |
+
font-family:var(--fD); line-height:1.2; margin-bottom:4px;
|
| 233 |
+
}
|
| 234 |
+
.cur-sentence { font-size:.75rem; color:var(--txtm); line-height:1.5; font-family:var(--fM); }
|
| 235 |
+
|
| 236 |
+
/* ββ Buttons ββ */
|
| 237 |
+
.btn-row { display:grid; grid-template-columns:repeat(3,1fr); gap:7px; }
|
| 238 |
+
.btn {
|
| 239 |
+
display:flex; align-items:center; justify-content:center; gap:5px;
|
| 240 |
+
padding:9px 6px; border:1px solid transparent; border-radius:var(--r14);
|
| 241 |
+
font-family:var(--fB); font-size:.76rem; font-weight:600;
|
| 242 |
+
cursor:pointer; transition:all .22s; white-space:nowrap; overflow:hidden;
|
| 243 |
+
}
|
| 244 |
+
.btn:active { transform:scale(.96); }
|
| 245 |
+
.btn:disabled { opacity:.4; cursor:not-allowed; }
|
| 246 |
+
.btn-speak {
|
| 247 |
+
background:linear-gradient(135deg,#4e7d66,#7eb89a);
|
| 248 |
+
color:#0d1117; border-color:rgba(126,184,154,.35);
|
| 249 |
+
}
|
| 250 |
+
.btn-speak:hover { box-shadow:0 4px 16px rgba(126,184,154,.3); }
|
| 251 |
+
.btn-back { background:transparent; color:var(--txtm); border-color:var(--bdr); }
|
| 252 |
+
.btn-back:hover { border-color:var(--bdrm); color:var(--txt); }
|
| 253 |
+
.btn-reset { background:transparent; color:var(--red); border-color:rgba(192,97,74,.3); }
|
| 254 |
+
.btn-reset:hover { background:rgba(192,97,74,.08); border-color:rgba(192,97,74,.55); }
|
| 255 |
+
|
| 256 |
+
.divider { height:1px; background:var(--bdr); margin:0 -16px; }
|
| 257 |
+
|
| 258 |
+
/* ββ Emotion ββ */
|
| 259 |
+
.emotion-header { display:flex; justify-content:space-between; align-items:center; margin-bottom:8px; }
|
| 260 |
+
.emotion-big { font-family:var(--fD); font-size:1.1rem; font-weight:700; color:var(--goldb); }
|
| 261 |
+
.emotion-icon { font-size:1.5rem; animation:emojiPop 2.2s ease infinite; }
|
| 262 |
+
@keyframes emojiPop { 0%,100%{transform:scale(1)} 50%{transform:scale(1.12)} }
|
| 263 |
+
|
| 264 |
+
.emo-bars { display:flex; flex-direction:column; gap:5px; }
|
| 265 |
+
.emo-bar { display:flex; align-items:center; gap:8px; }
|
| 266 |
+
.ebo-label { width:78px; font-size:.72rem; color:var(--txtm); flex-shrink:0; font-family:var(--fB); }
|
| 267 |
+
.ebo-track {
|
| 268 |
+
flex:1; height:20px; background:rgba(26,34,51,.8);
|
| 269 |
+
border-radius:10px; overflow:hidden; border:1px solid var(--bdr);
|
| 270 |
+
}
|
| 271 |
+
.ebo-fill {
|
| 272 |
+
height:100%; transition:width .5s cubic-bezier(.4,0,.2,1);
|
| 273 |
+
display:flex; align-items:center; justify-content:flex-end;
|
| 274 |
+
padding-right:6px; font-size:.62rem; font-weight:700;
|
| 275 |
+
font-family:var(--fM); color:rgba(0,0,0,.75); min-width:28px;
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
/* ββ Suggestions ββ */
|
| 279 |
+
.sug-detected {
|
| 280 |
+
font-size:.65rem; font-family:var(--fM); color:var(--gold);
|
| 281 |
+
background:rgba(212,168,83,.07); border:1px solid var(--bdr);
|
| 282 |
+
border-radius:var(--r8); padding:4px 10px; margin-bottom:5px; letter-spacing:.04em;
|
| 283 |
+
}
|
| 284 |
+
.sug-list { display:flex; flex-direction:column; gap:4px; }
|
| 285 |
+
.sug-item {
|
| 286 |
+
display:flex; align-items:center; gap:9px; padding:7px 11px;
|
| 287 |
+
border-radius:var(--r8); background:var(--card); border:1px solid var(--bdr);
|
| 288 |
+
cursor:pointer; transition:all .2s; font-size:.82rem; color:var(--txt);
|
| 289 |
+
}
|
| 290 |
+
.sug-item:hover {
|
| 291 |
+
background:rgba(212,168,83,.08); border-color:var(--bdrm);
|
| 292 |
+
transform:translateX(3px); color:var(--goldb);
|
| 293 |
+
}
|
| 294 |
+
.sug-n {
|
| 295 |
+
font-family:var(--fM); font-size:.65rem; font-weight:700; color:var(--gold);
|
| 296 |
+
background:rgba(212,168,83,.12); border:1px solid rgba(212,168,83,.2);
|
| 297 |
+
width:20px; height:20px; display:flex; align-items:center; justify-content:center;
|
| 298 |
+
border-radius:5px; flex-shrink:0;
|
| 299 |
+
}
|
| 300 |
+
.sug-empty { text-align:center; padding:16px 0; color:var(--txtl); font-size:.75rem; line-height:1.7; }
|
| 301 |
+
|
| 302 |
+
/* ββ Chart ββ */
|
| 303 |
+
#emotionChart { height:160px !important; }
|
| 304 |
+
|
| 305 |
+
/* ββ Stats ββ */
|
| 306 |
+
.stat-grid { display:grid; grid-template-columns:repeat(3,1fr); gap:6px; }
|
| 307 |
+
.stat-tile {
|
| 308 |
+
background:var(--card); border:1px solid var(--bdr);
|
| 309 |
+
border-radius:var(--r8); padding:9px 10px; text-align:center; transition:border-color .25s;
|
| 310 |
+
}
|
| 311 |
+
.stat-tile:hover { border-color:var(--bdrm); }
|
| 312 |
+
.sl { font-size:.55rem; text-transform:uppercase; letter-spacing:.07em; color:var(--txtl); font-weight:600; margin-bottom:3px; font-family:var(--fM); }
|
| 313 |
+
.sv { font-family:var(--fD); font-size:1.3rem; font-weight:700; color:var(--gold); line-height:1; }
|
| 314 |
+
|
| 315 |
+
/* ββ About ββ */
|
| 316 |
+
.about-txt { font-size:.75rem; color:var(--txtm); line-height:1.75; }
|
| 317 |
+
.about-txt p { margin-bottom:7px; }
|
| 318 |
+
.alist { list-style:none; margin-bottom:7px; }
|
| 319 |
+
.alist li { padding-left:13px; position:relative; font-size:.73rem; padding-bottom:2px; }
|
| 320 |
+
.alist li::before { content:'βΊ'; position:absolute; left:0; color:var(--gold); font-weight:700; }
|
| 321 |
+
.acredit { margin-top:10px; padding-top:10px; border-top:1px solid var(--bdr); }
|
| 322 |
+
.aname { font-family:var(--fD); color:var(--gold); font-weight:600; font-size:.88rem; }
|
| 323 |
+
.arole { font-size:.65rem; color:var(--txtl); margin:2px 0 3px; font-family:var(--fM); }
|
| 324 |
+
.aemail { color:var(--sage); font-size:.66px; font-family:var(--fM); }
|
| 325 |
+
|
| 326 |
+
/* ββ Nav ββ */
|
| 327 |
+
.nav-row { display:flex; gap:7px; padding:0 16px 20px; }
|
| 328 |
+
.nav-a {
|
| 329 |
+
flex:1; display:flex; align-items:center; justify-content:center; gap:6px;
|
| 330 |
+
padding:12px 8px; border-radius:var(--r14); border:1px solid var(--bdr);
|
| 331 |
+
background:var(--card); color:var(--txtm); text-decoration:none;
|
| 332 |
+
font-size:.76rem; font-weight:600; transition:all .25s;
|
| 333 |
+
}
|
| 334 |
+
.nav-a:hover { border-color:var(--bdrm); color:var(--txt); }
|
| 335 |
+
.nav-a.on { background:rgba(212,168,83,.10); border-color:var(--bdrh); color:var(--gold); }
|
| 336 |
+
</style>
|
| 337 |
+
</head>
|
| 338 |
+
<body>
|
| 339 |
+
|
| 340 |
+
<div id="app">
|
| 341 |
+
|
| 342 |
+
<!-- ββ Camera pane ββ -->
|
| 343 |
+
<div id="cam-pane">
|
| 344 |
+
<div class="topbar">
|
| 345 |
+
<div class="brand">
|
| 346 |
+
<span class="brand-t">ISL Gesture Recognition</span>
|
| 347 |
+
<span class="brand-s">Gesture Β· Emotion Β· AI</span>
|
| 348 |
+
</div>
|
| 349 |
+
<span class="live-badge" id="live-badge">β Ready</span>
|
| 350 |
+
</div>
|
| 351 |
+
|
| 352 |
+
<!-- MJPEG feed β displays processed frame from server SharedMemory -->
|
| 353 |
+
<img id="camera-feed"
|
| 354 |
+
src="/video_feed"
|
| 355 |
+
alt="Camera Feed"
|
| 356 |
+
style="display:none;flex:1;width:100%;object-fit:cover;"
|
| 357 |
+
onload="onVidLoad()"
|
| 358 |
+
onerror="document.getElementById('cam-ph').style.display='flex'"/>
|
| 359 |
+
|
| 360 |
+
<div class="cam-placeholder" id="cam-ph">
|
| 361 |
+
<span class="ph-icon">π·</span>
|
| 362 |
+
<span style="font-family:var(--fM);font-size:.75rem">Connecting to cameraβ¦</span>
|
| 363 |
+
</div>
|
| 364 |
+
|
| 365 |
+
<div class="cam-overlay">
|
| 366 |
+
<div>
|
| 367 |
+
<div style="display:flex;align-items:baseline;gap:8px">
|
| 368 |
+
<span id="co-letter" class="co-letter">π€</span>
|
| 369 |
+
<span id="co-word" class="co-word"></span>
|
| 370 |
+
</div>
|
| 371 |
+
</div>
|
| 372 |
+
<div id="co-desc" class="co-desc">Point camera at your hand</div>
|
| 373 |
+
<div class="co-progress"><div class="co-bar" id="co-bar"></div></div>
|
| 374 |
+
</div>
|
| 375 |
+
</div>
|
| 376 |
+
|
| 377 |
+
<!-- ββ Control pane ββ -->
|
| 378 |
+
<div id="ctrl-pane">
|
| 379 |
+
<div class="cpanel">
|
| 380 |
+
|
| 381 |
+
<!-- Status -->
|
| 382 |
+
<div class="status" id="status">
|
| 383 |
+
<div class="sdot"></div>
|
| 384 |
+
<span id="stxt">Ready for detection</span>
|
| 385 |
+
</div>
|
| 386 |
+
|
| 387 |
+
<!-- Current word -->
|
| 388 |
+
<div class="text-display">
|
| 389 |
+
<div class="cur-word" id="cur-word">β</div>
|
| 390 |
+
<div class="cur-sentence" id="cur-sentence"></div>
|
| 391 |
+
</div>
|
| 392 |
+
|
| 393 |
+
<!-- Buttons -->
|
| 394 |
+
<div class="btn-row">
|
| 395 |
+
<button class="btn btn-speak" onclick="sendCmd('speak')">π Speak</button>
|
| 396 |
+
<button class="btn btn-back" onclick="sendCmd('backspace')">β« Back</button>
|
| 397 |
+
<button class="btn btn-reset" onclick="sendCmd('reset')">π Reset</button>
|
| 398 |
+
</div>
|
| 399 |
+
|
| 400 |
+
<div class="divider"></div>
|
| 401 |
+
|
| 402 |
+
<!-- Emotion -->
|
| 403 |
+
<div class="sec-label">π Emotion Detection</div>
|
| 404 |
+
<div class="emotion-header">
|
| 405 |
+
<span class="emotion-big" id="emo-label">Neutral</span>
|
| 406 |
+
<span class="emotion-icon" id="emo-icon">π</span>
|
| 407 |
+
</div>
|
| 408 |
+
<div class="emo-bars">
|
| 409 |
+
<div class="emo-bar">
|
| 410 |
+
<div class="ebo-label">π Happy</div>
|
| 411 |
+
<div class="ebo-track"><div class="ebo-fill" id="bar-happy" style="width:0%;background:linear-gradient(90deg,#3a8a5c,#7eb89a)">0%</div></div>
|
| 412 |
+
</div>
|
| 413 |
+
<div class="emo-bar">
|
| 414 |
+
<div class="ebo-label">π’ Sad</div>
|
| 415 |
+
<div class="ebo-track"><div class="ebo-fill" id="bar-sad" style="width:0%;background:linear-gradient(90deg,#3a5a8a,#6090c8)">0%</div></div>
|
| 416 |
+
</div>
|
| 417 |
+
<div class="emo-bar">
|
| 418 |
+
<div class="ebo-label">π Angry</div>
|
| 419 |
+
<div class="ebo-track"><div class="ebo-fill" id="bar-angry" style="width:0%;background:linear-gradient(90deg,#8a3a3a,#c06060)">0%</div></div>
|
| 420 |
+
</div>
|
| 421 |
+
<div class="emo-bar">
|
| 422 |
+
<div class="ebo-label">π² Surprise</div>
|
| 423 |
+
<div class="ebo-track"><div class="ebo-fill" id="bar-surprise" style="width:0%;background:linear-gradient(90deg,#8a6a1a,#d4a853)">0%</div></div>
|
| 424 |
+
</div>
|
| 425 |
+
<div class="emo-bar">
|
| 426 |
+
<div class="ebo-label">π Neutral</div>
|
| 427 |
+
<div class="ebo-track"><div class="ebo-fill" id="bar-neutral" style="width:100%;background:linear-gradient(90deg,#3a4455,#5a6878)">100%</div></div>
|
| 428 |
+
</div>
|
| 429 |
+
</div>
|
| 430 |
+
|
| 431 |
+
<div class="divider"></div>
|
| 432 |
+
|
| 433 |
+
<!-- Word Suggestions -->
|
| 434 |
+
<div class="sec-label">π‘ Word Suggestions</div>
|
| 435 |
+
<div id="sug-area">
|
| 436 |
+
<div class="sug-empty">Detected letters will appear<br>here with word suggestions</div>
|
| 437 |
+
</div>
|
| 438 |
+
|
| 439 |
+
<div class="divider"></div>
|
| 440 |
+
|
| 441 |
+
<!-- Emotion Timeline Chart -->
|
| 442 |
+
<div class="sec-label">π Emotion Timeline</div>
|
| 443 |
+
<canvas id="emotionChart"></canvas>
|
| 444 |
+
|
| 445 |
+
<div class="divider"></div>
|
| 446 |
+
|
| 447 |
+
<!-- Statistics -->
|
| 448 |
+
<div class="sec-label">π Statistics</div>
|
| 449 |
+
<div class="stat-grid">
|
| 450 |
+
<div class="stat-tile"><div class="sl">FPS</div><div class="sv" id="s-fps">0</div></div>
|
| 451 |
+
<div class="stat-tile"><div class="sl">Letters</div><div class="sv" id="s-letters">0</div></div>
|
| 452 |
+
<div class="stat-tile"><div class="sl">Words</div><div class="sv" id="s-words">0</div></div>
|
| 453 |
+
<div class="stat-tile"><div class="sl">Sentences</div><div class="sv" id="s-sentences">0</div></div>
|
| 454 |
+
<div class="stat-tile"><div class="sl">Emotions</div><div class="sv" id="s-emotions">0</div></div>
|
| 455 |
+
<div class="stat-tile"><div class="sl">Conf.</div><div class="sv" id="s-conf">β</div></div>
|
| 456 |
+
</div>
|
| 457 |
+
|
| 458 |
+
<div class="divider"></div>
|
| 459 |
+
|
| 460 |
+
<!-- About -->
|
| 461 |
+
<div class="sec-label">βΉοΈ About</div>
|
| 462 |
+
<div class="about-txt">
|
| 463 |
+
<p>AI Powered ISL Detection is an assistive AI system designed to improve communication between hearing and deaf individuals.</p>
|
| 464 |
+
<p>This module focuses on real-time gesture and facial emotion recognition using computer vision.</p>
|
| 465 |
+
<p>Useful for:</p>
|
| 466 |
+
<ul class="alist">
|
| 467 |
+
<li>Deaf individuals</li>
|
| 468 |
+
<li>Inclusive classrooms</li>
|
| 469 |
+
<li>Assistive communication systems</li>
|
| 470 |
+
<li>Accessibility and AI research</li>
|
| 471 |
+
</ul>
|
| 472 |
+
<div class="acredit">
|
| 473 |
+
<div class="aname">Deepak Roshan</div>
|
| 474 |
+
<div class="arole">AI Engineer</div>
|
| 475 |
+
<div class="aemail">deepakroshan380@gmail.com</div>
|
| 476 |
+
</div>
|
| 477 |
+
</div>
|
| 478 |
+
|
| 479 |
+
</div><!-- cpanel -->
|
| 480 |
+
|
| 481 |
+
<!-- FIX 2: Nav links now built dynamically from window.location -->
|
| 482 |
+
<div class="nav-row">
|
| 483 |
+
<a href="/" class="nav-a on" id="nav-gesture">π Gesture Recognition</a>
|
| 484 |
+
<a href="#" class="nav-a" id="nav-avatar">π€ Speech-to-Sign</a>
|
| 485 |
+
</div>
|
| 486 |
+
</div><!-- ctrl-pane -->
|
| 487 |
+
|
| 488 |
+
</div><!-- app -->
|
| 489 |
+
|
| 490 |
+
<script>
|
| 491 |
+
document.addEventListener('DOMContentLoaded', function () {
|
| 492 |
+
|
| 493 |
+
// FIX 2: Build nav URLs dynamically so they work on any host/domain
|
| 494 |
+
(function() {
|
| 495 |
+
const proto = location.protocol;
|
| 496 |
+
const host = location.hostname;
|
| 497 |
+
const port = location.port;
|
| 498 |
+
if (port && port !== '80' && port !== '443') {
|
| 499 |
+
document.getElementById('nav-gesture').href = proto + '//' + host + ':5000/';
|
| 500 |
+
document.getElementById('nav-avatar').href = proto + '//' + host + ':5001/';
|
| 501 |
+
} else {
|
| 502 |
+
document.getElementById('nav-gesture').href = proto + '//' + host + '/';
|
| 503 |
+
document.getElementById('nav-avatar').href = proto + '//' + host + '/avatar/';
|
| 504 |
+
}
|
| 505 |
+
})();
|
| 506 |
+
|
| 507 |
+
const EMO_ICONS = { happy:'π', sad:'π’', angry:'π ', surprise:'π²', neutral:'π' };
|
| 508 |
+
const EMO_KEYS = ['happy','sad','angry','surprise','neutral'];
|
| 509 |
+
|
| 510 |
+
/* ββ Chart.js Emotion Timeline βββββββββββββββββββββββββββ */
|
| 511 |
+
const chartCtx = document.getElementById('emotionChart').getContext('2d');
|
| 512 |
+
const chart = new Chart(chartCtx, {
|
| 513 |
+
type: 'line',
|
| 514 |
+
data: {
|
| 515 |
+
labels: [],
|
| 516 |
+
datasets: [
|
| 517 |
+
{ label:'Happy', data:[], borderColor:'#7eb89a', borderWidth:1.5, tension:.4, fill:false, pointRadius:0 },
|
| 518 |
+
{ label:'Sad', data:[], borderColor:'#6090c8', borderWidth:1.5, tension:.4, fill:false, pointRadius:0 },
|
| 519 |
+
{ label:'Angry', data:[], borderColor:'#c06060', borderWidth:1.5, tension:.4, fill:false, pointRadius:0 },
|
| 520 |
+
{ label:'Surprise', data:[], borderColor:'#d4a853', borderWidth:1.5, tension:.4, fill:false, pointRadius:0 },
|
| 521 |
+
{ label:'Neutral', data:[], borderColor:'#5a6878', borderWidth:1.5, tension:.4, fill:false, pointRadius:0 },
|
| 522 |
+
]
|
| 523 |
+
},
|
| 524 |
+
options: {
|
| 525 |
+
responsive: true,
|
| 526 |
+
maintainAspectRatio: false,
|
| 527 |
+
plugins: {
|
| 528 |
+
legend: { labels: { color:'#5a5650', font:{ size:9, family:"'DM Mono'" }, boxWidth:10 } }
|
| 529 |
+
},
|
| 530 |
+
scales: {
|
| 531 |
+
y: {
|
| 532 |
+
beginAtZero:true, max:1,
|
| 533 |
+
ticks: { color:'#5a5650', font:{ size:8 } },
|
| 534 |
+
grid: { color:'rgba(212,168,83,.05)' }
|
| 535 |
+
},
|
| 536 |
+
x: { ticks: { display:false }, grid: { color:'rgba(212,168,83,.03)' } }
|
| 537 |
+
},
|
| 538 |
+
animation: false
|
| 539 |
+
}
|
| 540 |
+
});
|
| 541 |
+
|
| 542 |
+
/* ββ Browser Camera (FIX 1) βββββββββββββββββββββββββββββββ */
|
| 543 |
+
let _camStream = null;
|
| 544 |
+
let _camInterval = null;
|
| 545 |
+
const _capCanvas = document.createElement('canvas');
|
| 546 |
+
const _capCtx = _capCanvas.getContext('2d');
|
| 547 |
+
|
| 548 |
+
async function startBrowserCamera() {
|
| 549 |
+
if (_camStream) return;
|
| 550 |
+
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
| 551 |
+
console.warn('Camera API not available β HTTP or unsupported browser');
|
| 552 |
+
return;
|
| 553 |
+
}
|
| 554 |
+
try {
|
| 555 |
+
_camStream = await navigator.mediaDevices.getUserMedia({
|
| 556 |
+
video: { width:{ ideal:640 }, height:{ ideal:480 }, facingMode:'user' },
|
| 557 |
+
audio: false
|
| 558 |
+
});
|
| 559 |
+
const video = document.createElement('video');
|
| 560 |
+
video.srcObject = _camStream;
|
| 561 |
+
video.autoplay = true;
|
| 562 |
+
video.playsInline = true;
|
| 563 |
+
video.addEventListener('loadeddata', () => {
|
| 564 |
+
_capCanvas.width = 320;
|
| 565 |
+
_capCanvas.height = 240;
|
| 566 |
+
_camInterval = setInterval(() => {
|
| 567 |
+
if (document.visibilityState === 'visible' && socket && socket.connected) {
|
| 568 |
+
_capCtx.drawImage(video, 0, 0, 320, 240);
|
| 569 |
+
const frame = _capCanvas.toDataURL('image/jpeg', 0.6);
|
| 570 |
+
socket.emit('browser_frame', { frame: frame.split(',')[1] });
|
| 571 |
+
}
|
| 572 |
+
}, 65); // ~15 fps
|
| 573 |
+
});
|
| 574 |
+
} catch (err) {
|
| 575 |
+
console.error('Camera access error:', err);
|
| 576 |
+
}
|
| 577 |
+
}
|
| 578 |
+
|
| 579 |
+
function stopBrowserCamera() {
|
| 580 |
+
if (_camStream) { _camStream.getTracks().forEach(t => t.stop()); _camStream = null; }
|
| 581 |
+
if (_camInterval) { clearInterval(_camInterval); _camInterval = null; }
|
| 582 |
+
}
|
| 583 |
+
|
| 584 |
+
/* ββ Socket.IO βββββββββββββββββββββββββββββββββββββββββββββ */
|
| 585 |
+
// β
FIX 6: transports updated to prefer WebSocket with polling fallback.
|
| 586 |
+
// upgrade:true lets Socket.IO automatically upgrade from polling β websocket
|
| 587 |
+
// once the connection is established. This is required for camera streaming
|
| 588 |
+
// to work reliably on HuggingFace Spaces (HTTPS) and other cloud deployments.
|
| 589 |
+
let socket;
|
| 590 |
+
try {
|
| 591 |
+
socket = io({
|
| 592 |
+
transports: ['websocket', 'polling'],
|
| 593 |
+
upgrade: true,
|
| 594 |
+
reconnectionDelay: 1000
|
| 595 |
+
});
|
| 596 |
+
|
| 597 |
+
socket.on('connect', () => {
|
| 598 |
+
setStat('act', 'Connected β detectingβ¦');
|
| 599 |
+
document.getElementById('cam-ph').style.display = 'none';
|
| 600 |
+
setBadge('β Live', 'rec');
|
| 601 |
+
startBrowserCamera();
|
| 602 |
+
});
|
| 603 |
+
|
| 604 |
+
socket.on('disconnect', () => {
|
| 605 |
+
setStat('err', 'Disconnected');
|
| 606 |
+
document.getElementById('cam-ph').style.display = 'flex';
|
| 607 |
+
setBadge('β Offline', 'err');
|
| 608 |
+
stopBrowserCamera();
|
| 609 |
+
});
|
| 610 |
+
|
| 611 |
+
socket.on('state_update', updateUI);
|
| 612 |
+
|
| 613 |
+
} catch (e) {
|
| 614 |
+
console.warn('Socket.IO unavailable β demo mode');
|
| 615 |
+
}
|
| 616 |
+
|
| 617 |
+
/* ββ Video load callback ββββββββββββββββββββββββββββββββ */
|
| 618 |
+
window.onVidLoad = function () {
|
| 619 |
+
document.getElementById('cam-ph').style.display = 'none';
|
| 620 |
+
document.getElementById('camera-feed').style.display = 'block';
|
| 621 |
+
setBadge('β Live', 'rec');
|
| 622 |
+
};
|
| 623 |
+
|
| 624 |
+
/* ββ Main UI update βββββββββββββββββββββββββββββββββββββ */
|
| 625 |
+
function updateUI(d) {
|
| 626 |
+
if (d.overlay_letter) {
|
| 627 |
+
document.getElementById('co-letter').textContent = d.overlay_letter.toUpperCase();
|
| 628 |
+
}
|
| 629 |
+
if (d.current_word !== undefined) {
|
| 630 |
+
const w = d.current_word || 'β';
|
| 631 |
+
document.getElementById('co-word').textContent = w;
|
| 632 |
+
document.getElementById('cur-word').textContent = w;
|
| 633 |
+
}
|
| 634 |
+
if (d.sentence !== undefined) {
|
| 635 |
+
document.getElementById('cur-sentence').textContent = d.sentence || '';
|
| 636 |
+
}
|
| 637 |
+
if (d.overlay_confidence !== undefined) {
|
| 638 |
+
const pct = Math.round(d.overlay_confidence * 100);
|
| 639 |
+
document.getElementById('co-desc').textContent = 'Confidence: ' + pct + '%';
|
| 640 |
+
document.getElementById('s-conf').textContent = pct + '%';
|
| 641 |
+
document.getElementById('co-bar').style.width = pct + '%';
|
| 642 |
+
}
|
| 643 |
+
if (d.hand_detected) {
|
| 644 |
+
setStat('act', d.overlay_letter
|
| 645 |
+
? 'Detecting: ' + d.overlay_letter + ' (' + Math.round((d.overlay_confidence||0)*100) + '%)'
|
| 646 |
+
: 'Hand in frame β signingβ¦');
|
| 647 |
+
} else {
|
| 648 |
+
setStat('', d.current_word ? 'Word so far: ' + d.current_word : 'Ready for detection');
|
| 649 |
+
}
|
| 650 |
+
if (d.fps !== undefined) {
|
| 651 |
+
document.getElementById('s-fps').textContent = Math.round(d.fps);
|
| 652 |
+
}
|
| 653 |
+
if (d.stats) {
|
| 654 |
+
document.getElementById('s-letters').textContent = d.stats.letters_detected || 0;
|
| 655 |
+
document.getElementById('s-words').textContent = d.stats.words_formed || 0;
|
| 656 |
+
document.getElementById('s-sentences').textContent = d.stats.sentences_formed || 0;
|
| 657 |
+
document.getElementById('s-emotions').textContent = d.stats.emotion_changes || 0;
|
| 658 |
+
}
|
| 659 |
+
if (Array.isArray(d.suggestions)) {
|
| 660 |
+
updateSuggestions(d.suggestions, d.current_word);
|
| 661 |
+
}
|
| 662 |
+
if (d.emotion_scores) {
|
| 663 |
+
updateEmotion(d.emotion || 'neutral', d.emotion_scores);
|
| 664 |
+
}
|
| 665 |
+
if (d.emotion_timeline && d.emotion_timeline.length > 0) {
|
| 666 |
+
updateChart(d.emotion_timeline);
|
| 667 |
+
}
|
| 668 |
+
}
|
| 669 |
+
|
| 670 |
+
function updateEmotion(emotion, scores) {
|
| 671 |
+
document.getElementById('emo-label').textContent = emotion.charAt(0).toUpperCase() + emotion.slice(1);
|
| 672 |
+
document.getElementById('emo-icon').textContent = EMO_ICONS[emotion] || 'π';
|
| 673 |
+
EMO_KEYS.forEach(k => {
|
| 674 |
+
const bar = document.getElementById('bar-' + k);
|
| 675 |
+
if (!bar) return;
|
| 676 |
+
const pct = Math.round(Math.max(0, Math.min(1, scores[k] || 0)) * 100);
|
| 677 |
+
bar.style.width = pct + '%';
|
| 678 |
+
bar.textContent = pct + '%';
|
| 679 |
+
});
|
| 680 |
+
}
|
| 681 |
+
|
| 682 |
+
function updateSuggestions(suggestions, currentWord) {
|
| 683 |
+
const area = document.getElementById('sug-area');
|
| 684 |
+
if (!suggestions || !suggestions.length) {
|
| 685 |
+
area.innerHTML = '<div class="sug-empty">Detected letters will appear<br>here with word suggestions</div>';
|
| 686 |
+
return;
|
| 687 |
+
}
|
| 688 |
+
const detectedRow = currentWord
|
| 689 |
+
? '<div class="sug-detected">Detected: ' + currentWord.toUpperCase() + '</div>' : '';
|
| 690 |
+
area.innerHTML = detectedRow + '<div class="sug-list">' +
|
| 691 |
+
suggestions.map((w, i) =>
|
| 692 |
+
'<div class="sug-item" onclick="window.acceptSug(\'' + w + '\')">' +
|
| 693 |
+
'<span class="sug-n">' + (i+1) + '</span>' + w + '</div>'
|
| 694 |
+
).join('') + '</div>';
|
| 695 |
+
}
|
| 696 |
+
|
| 697 |
+
function updateChart(timeline) {
|
| 698 |
+
if (!timeline || !timeline.length) return;
|
| 699 |
+
const collected = Object.fromEntries(EMO_KEYS.map(k => [k, []]));
|
| 700 |
+
timeline.forEach(tp => {
|
| 701 |
+
const s = tp.scores || {};
|
| 702 |
+
EMO_KEYS.forEach(k => collected[k].push(s[k] || 0));
|
| 703 |
+
});
|
| 704 |
+
chart.data.labels = timeline.map((_, i) => i);
|
| 705 |
+
chart.data.datasets.forEach((ds, i) => ds.data = collected[EMO_KEYS[i]]);
|
| 706 |
+
chart.update('none');
|
| 707 |
+
}
|
| 708 |
+
|
| 709 |
+
function setStat(cls, msg) {
|
| 710 |
+
document.getElementById('stxt').textContent = msg;
|
| 711 |
+
document.getElementById('status').className = 'status' + (cls ? ' ' + cls : '');
|
| 712 |
+
}
|
| 713 |
+
function setBadge(msg, cls) {
|
| 714 |
+
const b = document.getElementById('live-badge');
|
| 715 |
+
b.textContent = msg;
|
| 716 |
+
b.className = 'live-badge' + (cls ? ' ' + cls : '');
|
| 717 |
+
}
|
| 718 |
+
|
| 719 |
+
window.sendCmd = function (action) {
|
| 720 |
+
if (action === 'speak') {
|
| 721 |
+
const sentence = document.getElementById('cur-sentence').textContent.trim();
|
| 722 |
+
const word = document.getElementById('cur-word').textContent.trim();
|
| 723 |
+
const text = sentence || (word !== 'β' ? word : '');
|
| 724 |
+
if (text && socket) socket.emit('command', { action:'speak', text });
|
| 725 |
+
} else {
|
| 726 |
+
if (socket) socket.emit('command', { action });
|
| 727 |
+
}
|
| 728 |
+
};
|
| 729 |
+
|
| 730 |
+
window.acceptSug = function (word) {
|
| 731 |
+
if (socket) socket.emit('command', { action:'accept_suggestion', word });
|
| 732 |
+
};
|
| 733 |
+
|
| 734 |
+
document.addEventListener('keydown', e => {
|
| 735 |
+
if (e.ctrlKey && e.key === 's') { e.preventDefault(); window.sendCmd('speak'); }
|
| 736 |
+
if (e.ctrlKey && e.key === 'r') { e.preventDefault(); window.sendCmd('reset'); }
|
| 737 |
+
if (e.ctrlKey && e.key === 'Backspace') { e.preventDefault(); window.sendCmd('backspace'); }
|
| 738 |
+
if (['1','2','3'].includes(e.key)) {
|
| 739 |
+
const items = document.querySelectorAll('.sug-item');
|
| 740 |
+
const item = items[parseInt(e.key) - 1];
|
| 741 |
+
if (item) item.click();
|
| 742 |
+
}
|
| 743 |
+
});
|
| 744 |
+
|
| 745 |
+
});
|
| 746 |
+
</script>
|
| 747 |
+
</body>
|
| 748 |
+
</html>
|
| 749 |
+
"""
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
# ββ MJPEG generator reads directly from SharedMemory ββββββββββββββββββββββββββ
|
| 753 |
+
def generate_frames():
|
| 754 |
+
# FIX 5 (part A): If SharedMemory is not configured, stream a blank frame
|
| 755 |
+
# continuously. Removed unreachable `return` after the infinite loop β
|
| 756 |
+
# the original code had `return` after `while True` which is dead code and
|
| 757 |
+
# caused a StopIteration on some Python/Flask versions.
|
| 758 |
+
if _shm_name is None or _frame_seq is None:
|
| 759 |
+
blank = np.zeros((FRAME_H, FRAME_W, 3), np.uint8)
|
| 760 |
+
_, enc = cv2.imencode(".jpg", blank)
|
| 761 |
+
chunk = enc.tobytes()
|
| 762 |
+
while True:
|
| 763 |
+
yield (b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" + chunk + b"\r\n")
|
| 764 |
+
time.sleep(0.1)
|
| 765 |
+
|
| 766 |
+
try:
|
| 767 |
+
shm = multiprocessing.shared_memory.SharedMemory(name=_shm_name)
|
| 768 |
+
frame_buf = np.ndarray((FRAME_H, FRAME_W, 3), dtype=np.uint8, buffer=shm.buf)
|
| 769 |
+
except Exception as e:
|
| 770 |
+
print(f"β οΈ MJPEG generator: SharedMemory attach failed: {e}")
|
| 771 |
+
return
|
| 772 |
+
|
| 773 |
+
# FIX 5 (part B): Wait for the detection engine to write the first valid
|
| 774 |
+
# frame before we start streaming. Without this wait, the MJPEG client
|
| 775 |
+
# receives raw uninitialized SharedMemory bytes immediately, which renders
|
| 776 |
+
# as horizontal static/noise stripes on the browser camera feed.
|
| 777 |
+
# Timeout after 15 seconds (150 Γ 0.1s) to avoid hanging indefinitely.
|
| 778 |
+
print("β³ MJPEG: waiting for first valid frame from detection engineβ¦")
|
| 779 |
+
timeout_count = 0
|
| 780 |
+
while _frame_seq.value == 0 and timeout_count < 150:
|
| 781 |
+
time.sleep(0.1)
|
| 782 |
+
timeout_count += 1
|
| 783 |
+
|
| 784 |
+
if _frame_seq.value == 0:
|
| 785 |
+
print("β οΈ MJPEG: timed out waiting for first frame β streaming anyway")
|
| 786 |
+
else:
|
| 787 |
+
print(f"β
MJPEG: first frame ready (seq={_frame_seq.value}) β streaming started")
|
| 788 |
+
|
| 789 |
+
last_seq = 0
|
| 790 |
+
try:
|
| 791 |
+
while True:
|
| 792 |
+
if _frame_seq.value != last_seq:
|
| 793 |
+
last_seq = _frame_seq.value
|
| 794 |
+
frame = frame_buf.copy()
|
| 795 |
+
_, buffer = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 80])
|
| 796 |
+
frame_bytes = buffer.tobytes()
|
| 797 |
+
yield (b"--frame\r\n"
|
| 798 |
+
b"Content-Type: image/jpeg\r\n\r\n" +
|
| 799 |
+
frame_bytes + b"\r\n")
|
| 800 |
+
else:
|
| 801 |
+
time.sleep(0.005)
|
| 802 |
+
finally:
|
| 803 |
+
shm.close()
|
| 804 |
+
|
| 805 |
+
|
| 806 |
+
# ββ Routes ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 807 |
+
@app.route("/")
|
| 808 |
+
def index():
|
| 809 |
+
return render_template_string(HTML)
|
| 810 |
+
|
| 811 |
+
|
| 812 |
+
@app.route("/video_feed")
|
| 813 |
+
def video_feed():
|
| 814 |
+
return Response(
|
| 815 |
+
generate_frames(),
|
| 816 |
+
mimetype="multipart/x-mixed-replace; boundary=frame"
|
| 817 |
+
)
|
| 818 |
+
|
| 819 |
+
|
| 820 |
+
@app.route("/cmd", methods=["POST"])
|
| 821 |
+
def cmd_route():
|
| 822 |
+
data = request.get_json(force=True, silent=True) or {}
|
| 823 |
+
_local_state_q.put(data)
|
| 824 |
+
return jsonify({"ok": True})
|
| 825 |
+
|
| 826 |
+
|
| 827 |
+
@app.route("/health")
|
| 828 |
+
def health():
|
| 829 |
+
return jsonify({"ok": True, "ts": time.time()})
|
| 830 |
+
|
| 831 |
+
|
| 832 |
+
# FIX 4: /avatar route β redirects to the speech-to-sign service
|
| 833 |
+
@app.route("/avatar")
|
| 834 |
+
@app.route("/avatar/")
|
| 835 |
+
def avatar_redirect():
|
| 836 |
+
from flask import redirect
|
| 837 |
+
proto = request.headers.get('X-Forwarded-Proto', 'http')
|
| 838 |
+
host = request.host.split(':')[0]
|
| 839 |
+
return redirect(proto + '://' + host + '/avatar/', code=302)
|
| 840 |
+
|
| 841 |
+
|
| 842 |
+
# ββ Socket.IO events ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 843 |
+
@socketio.on("connect")
|
| 844 |
+
def on_connect():
|
| 845 |
+
print("π Browser connected")
|
| 846 |
+
|
| 847 |
+
|
| 848 |
+
@socketio.on("disconnect")
|
| 849 |
+
def on_disconnect():
|
| 850 |
+
print("π Browser disconnected")
|
| 851 |
+
|
| 852 |
+
|
| 853 |
+
@socketio.on("command")
|
| 854 |
+
def on_cmd(data):
|
| 855 |
+
if data:
|
| 856 |
+
_local_state_q.put(data)
|
| 857 |
+
|
| 858 |
+
|
| 859 |
+
# FIX 1 + FIX 3: browser_frame handler
|
| 860 |
+
@socketio.on("browser_frame")
|
| 861 |
+
def on_browser_frame(data):
|
| 862 |
+
global _shared_state
|
| 863 |
+
if _shared_state is not None and data and "frame" in data:
|
| 864 |
+
_shared_state._cloud_frame = data["frame"]
|
| 865 |
+
|
| 866 |
+
|
| 867 |
+
# ββ Dashboard process entry point βββββββββββββββββββββββββββββββββββββββββββββ
|
| 868 |
+
def dashboard_process(shared_state, shm_name, frame_lock_val, frame_seq):
|
| 869 |
+
global _shm_name, _frame_seq, _shared_state
|
| 870 |
+
|
| 871 |
+
_shm_name = shm_name
|
| 872 |
+
_frame_seq = frame_seq
|
| 873 |
+
_shared_state = shared_state
|
| 874 |
+
|
| 875 |
+
print("=" * 60)
|
| 876 |
+
print("π ISL Web Dashboard Starting...")
|
| 877 |
+
print("=" * 60)
|
| 878 |
+
print("π URL : http://localhost:5000")
|
| 879 |
+
print("πΉ Video stream : http://localhost:5000/video_feed")
|
| 880 |
+
print(" (MJPEG from SharedMemory β no WebSocket for video)")
|
| 881 |
+
print("=" * 60)
|
| 882 |
+
|
| 883 |
+
ui_queue = shared_state.ui_queue
|
| 884 |
+
cmd_queue = shared_state.command_queue
|
| 885 |
+
|
| 886 |
+
def state_drain():
|
| 887 |
+
print("π₯ State drain thread started")
|
| 888 |
+
while True:
|
| 889 |
+
try:
|
| 890 |
+
packet = ui_queue.get(timeout=1.0)
|
| 891 |
+
try:
|
| 892 |
+
_local_state_q.put_nowait(packet)
|
| 893 |
+
except Exception:
|
| 894 |
+
pass
|
| 895 |
+
except Exception:
|
| 896 |
+
pass
|
| 897 |
+
|
| 898 |
+
threading.Thread(target=state_drain, daemon=True, name="StateDrain").start()
|
| 899 |
+
|
| 900 |
+
def state_emit():
|
| 901 |
+
print("π‘ State emitter started")
|
| 902 |
+
while True:
|
| 903 |
+
try:
|
| 904 |
+
packet = _local_state_q.get(timeout=1.0)
|
| 905 |
+
except Empty:
|
| 906 |
+
continue
|
| 907 |
+
except Exception:
|
| 908 |
+
continue
|
| 909 |
+
|
| 910 |
+
if "action" in packet:
|
| 911 |
+
try:
|
| 912 |
+
cmd_queue.put_nowait(packet)
|
| 913 |
+
except Exception:
|
| 914 |
+
pass
|
| 915 |
+
continue
|
| 916 |
+
|
| 917 |
+
try:
|
| 918 |
+
socketio.emit("state_update", packet)
|
| 919 |
+
except Exception:
|
| 920 |
+
pass
|
| 921 |
+
|
| 922 |
+
threading.Thread(target=state_emit, daemon=True, name="StateEmit").start()
|
| 923 |
+
|
| 924 |
+
socketio.run(
|
| 925 |
+
app,
|
| 926 |
+
host="0.0.0.0",
|
| 927 |
+
port=5000,
|
| 928 |
+
debug=False,
|
| 929 |
+
use_reloader=False,
|
| 930 |
+
log_output=False,
|
| 931 |
+
)
|
| 932 |
+
|
| 933 |
+
|
| 934 |
+
# ββ Compatibility aliases βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 935 |
+
def start_ui_server(shared_state_obj, port: int = 5000):
|
| 936 |
+
dashboard_process(shared_state_obj, shm_name=None, frame_lock_val=None, frame_seq=None)
|
| 937 |
+
|
| 938 |
+
|
| 939 |
+
# ββ Stand-alone test ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 940 |
+
if __name__ == "__main__":
|
| 941 |
+
socketio.run(app, host="0.0.0.0", port=5000, debug=True, use_reloader=False)
|
launcher.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# launcher.py β CLOUD PRODUCTION VERSION
|
| 2 |
+
#
|
| 3 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 4 |
+
# CLOUD FIX APPLIED (1 change only β everything else identical):
|
| 5 |
+
#
|
| 6 |
+
# FIX 1 β SharedMemory cleanup on startup:
|
| 7 |
+
# If the previous run crashed without cleanup, "isl_frame_shm"
|
| 8 |
+
# still exists in /dev/shm on Linux. SharedMemory(create=True)
|
| 9 |
+
# then raises FileExistsError and the entire launcher fails.
|
| 10 |
+
# Fixed by attempting to unlink the old block before creating new.
|
| 11 |
+
#
|
| 12 |
+
# ALL original features preserved:
|
| 13 |
+
# β
Shared memory block (640Γ480 BGR, zero-copy frame transfer)
|
| 14 |
+
# β
frame_seq counter for dashboard to detect new frames
|
| 15 |
+
# β
Auto-restart of crashed child processes (every 2s watch loop)
|
| 16 |
+
# β
Clean shutdown with shm.close() + shm.unlink() on Ctrl+C
|
| 17 |
+
# β
Core Processor + Web Dashboard spawned as separate processes
|
| 18 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 19 |
+
|
| 20 |
+
import multiprocessing
|
| 21 |
+
import multiprocessing.shared_memory
|
| 22 |
+
import ctypes
|
| 23 |
+
import time
|
| 24 |
+
import sys
|
| 25 |
+
|
| 26 |
+
from isl_detection import core_processing_engine, SharedState
|
| 27 |
+
from isl_ui_dashboard import dashboard_process
|
| 28 |
+
|
| 29 |
+
FRAME_W, FRAME_H = 640, 480
|
| 30 |
+
FRAME_BYTES = FRAME_W * FRAME_H * 3 # 921 600 bytes for one BGR frame
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _run_core(shared_state, shm_name, frame_lock_val, frame_seq):
|
| 34 |
+
core_processing_engine(shared_state, shm_name, frame_lock_val, frame_seq)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _run_dashboard(shared_state, shm_name, frame_lock_val, frame_seq):
|
| 38 |
+
dashboard_process(shared_state, shm_name, frame_lock_val, frame_seq)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
if __name__ == "__main__":
|
| 42 |
+
multiprocessing.freeze_support() # required on Windows
|
| 43 |
+
|
| 44 |
+
print("=" * 70)
|
| 45 |
+
print(" π€ ISL EMOTION TRANSLATOR - MULTI-PROCESS SYSTEM")
|
| 46 |
+
print("=" * 70)
|
| 47 |
+
print()
|
| 48 |
+
print("π― Architecture:")
|
| 49 |
+
print(" πΉ Core Processor β Camera Β· MediaPipe Β· TF model")
|
| 50 |
+
print(" π Web Dashboard β MJPEG video Β· Socket.IO state")
|
| 51 |
+
print(" π SharedMemory β Zero-copy frame transfer (no base64 queue)")
|
| 52 |
+
print()
|
| 53 |
+
print("=" * 70)
|
| 54 |
+
|
| 55 |
+
# FIX 1: Clean up leftover shared memory from a previous crash.
|
| 56 |
+
# On Linux, named shared memory persists in /dev/shm until explicitly
|
| 57 |
+
# unlinked. If the previous run crashed, creating the same name fails.
|
| 58 |
+
try:
|
| 59 |
+
_old_shm = multiprocessing.shared_memory.SharedMemory(name="isl_frame_shm")
|
| 60 |
+
_old_shm.close()
|
| 61 |
+
_old_shm.unlink()
|
| 62 |
+
print("π§Ή Cleaned up leftover shared memory from previous run")
|
| 63 |
+
except FileNotFoundError:
|
| 64 |
+
pass # normal β no leftover, continue
|
| 65 |
+
|
| 66 |
+
# ββ Shared memory block ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 67 |
+
shm = multiprocessing.shared_memory.SharedMemory(
|
| 68 |
+
create=True, size=FRAME_BYTES, name="isl_frame_shm")
|
| 69 |
+
frame_seq = multiprocessing.Value(ctypes.c_uint64, 0)
|
| 70 |
+
frame_lock = multiprocessing.Value(ctypes.c_bool, False)
|
| 71 |
+
|
| 72 |
+
shared_state = SharedState()
|
| 73 |
+
|
| 74 |
+
# ββ Spawn processes ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 75 |
+
core_args = (shared_state, shm.name, frame_lock, frame_seq)
|
| 76 |
+
web_args = (shared_state, shm.name, frame_lock, frame_seq)
|
| 77 |
+
|
| 78 |
+
core_proc = multiprocessing.Process(
|
| 79 |
+
target=_run_core, args=core_args, name="CoreProcessor", daemon=True)
|
| 80 |
+
web_proc = multiprocessing.Process(
|
| 81 |
+
target=_run_dashboard, args=web_args, name="WebDashboard", daemon=True)
|
| 82 |
+
|
| 83 |
+
print("π Starting Core Processorβ¦")
|
| 84 |
+
core_proc.start()
|
| 85 |
+
|
| 86 |
+
# Give core a moment to open the camera before the dashboard opens
|
| 87 |
+
time.sleep(1.5)
|
| 88 |
+
|
| 89 |
+
print("π Starting Web Dashboard on port 5000β¦")
|
| 90 |
+
web_proc.start()
|
| 91 |
+
|
| 92 |
+
print()
|
| 93 |
+
print("=" * 70)
|
| 94 |
+
print("β
SYSTEM READY!")
|
| 95 |
+
print("=" * 70)
|
| 96 |
+
print()
|
| 97 |
+
print("π Dashboard β http://localhost:5000")
|
| 98 |
+
print("πΉ Video stream β http://localhost:5000/video_feed")
|
| 99 |
+
print()
|
| 100 |
+
print("β¨οΈ Keyboard shortcuts (in dashboard):")
|
| 101 |
+
print(" Ctrl+R β Reset")
|
| 102 |
+
print(" Ctrl+Backspace β Backspace")
|
| 103 |
+
print(" 1 / 2 / 3 β Accept suggestion")
|
| 104 |
+
print()
|
| 105 |
+
print("π Press Ctrl+C to stop all processes")
|
| 106 |
+
print("=" * 70)
|
| 107 |
+
|
| 108 |
+
try:
|
| 109 |
+
while True:
|
| 110 |
+
# Auto-restart crashed processes
|
| 111 |
+
if not core_proc.is_alive():
|
| 112 |
+
print("β οΈ Core Processor crashed β restartingβ¦")
|
| 113 |
+
core_proc = multiprocessing.Process(
|
| 114 |
+
target=_run_core, args=core_args, name="CoreProcessor", daemon=True)
|
| 115 |
+
core_proc.start()
|
| 116 |
+
|
| 117 |
+
if not web_proc.is_alive():
|
| 118 |
+
print("β οΈ Web Dashboard crashed β restartingβ¦")
|
| 119 |
+
web_proc = multiprocessing.Process(
|
| 120 |
+
target=_run_dashboard, args=web_args, name="WebDashboard", daemon=True)
|
| 121 |
+
web_proc.start()
|
| 122 |
+
|
| 123 |
+
time.sleep(2)
|
| 124 |
+
|
| 125 |
+
except KeyboardInterrupt:
|
| 126 |
+
print("\nπ Shutting downβ¦")
|
| 127 |
+
core_proc.terminate()
|
| 128 |
+
web_proc.terminate()
|
| 129 |
+
core_proc.join(timeout=5)
|
| 130 |
+
web_proc.join(timeout=5)
|
| 131 |
+
try:
|
| 132 |
+
shm.close()
|
| 133 |
+
shm.unlink()
|
| 134 |
+
except Exception:
|
| 135 |
+
pass
|
| 136 |
+
print("β
All processes stopped")
|
| 137 |
+
print("=" * 70)
|
| 138 |
+
sys.exit(0)
|
model.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a695177089550ea37eec9e54be2631c28e64ac4e03d666c0e6b86582bfe6a80b
|
| 3 |
+
size 11510976
|
nginx.conf
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ============================================================
|
| 2 |
+
# nginx.conf β Hugging Face Spaces
|
| 3 |
+
# Single port: 7860 (HF requirement)
|
| 4 |
+
# Routes to Flask 5000 and Flask 5001 internally
|
| 5 |
+
# ============================================================
|
| 6 |
+
|
| 7 |
+
worker_processes 1;
|
| 8 |
+
error_log stderr warn;
|
| 9 |
+
pid /tmp/nginx.pid;
|
| 10 |
+
|
| 11 |
+
events {
|
| 12 |
+
worker_connections 256;
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
http {
|
| 16 |
+
include /etc/nginx/mime.types;
|
| 17 |
+
default_type application/octet-stream;
|
| 18 |
+
|
| 19 |
+
# Use /tmp for temp files (HF non-root friendly)
|
| 20 |
+
client_body_temp_path /tmp/nginx_client_body;
|
| 21 |
+
proxy_temp_path /tmp/nginx_proxy;
|
| 22 |
+
fastcgi_temp_path /tmp/nginx_fastcgi;
|
| 23 |
+
uwsgi_temp_path /tmp/nginx_uwsgi;
|
| 24 |
+
scgi_temp_path /tmp/nginx_scgi;
|
| 25 |
+
|
| 26 |
+
sendfile on;
|
| 27 |
+
keepalive_timeout 65;
|
| 28 |
+
|
| 29 |
+
gzip on;
|
| 30 |
+
gzip_types text/plain text/css application/json application/javascript;
|
| 31 |
+
|
| 32 |
+
upstream gesture_app {
|
| 33 |
+
server 127.0.0.1:5000;
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
upstream avatar_app {
|
| 37 |
+
server 127.0.0.1:5001;
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
server {
|
| 41 |
+
listen 7860;
|
| 42 |
+
server_name _;
|
| 43 |
+
client_max_body_size 5M;
|
| 44 |
+
|
| 45 |
+
# ββ Health check ββββββββββββββββββββββββββββββββββββββ
|
| 46 |
+
location /health {
|
| 47 |
+
proxy_pass http://gesture_app/health;
|
| 48 |
+
proxy_set_header Host $host;
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
# ββ Socket.IO β gesture app βββββββββββββββββββββββββββ
|
| 52 |
+
location /socket.io/ {
|
| 53 |
+
proxy_pass http://gesture_app/socket.io/;
|
| 54 |
+
proxy_http_version 1.1;
|
| 55 |
+
proxy_set_header Upgrade $http_upgrade;
|
| 56 |
+
proxy_set_header Connection "upgrade";
|
| 57 |
+
proxy_set_header Host $host;
|
| 58 |
+
proxy_set_header X-Real-IP $remote_addr;
|
| 59 |
+
proxy_set_header X-Forwarded-Proto $scheme;
|
| 60 |
+
proxy_read_timeout 86400;
|
| 61 |
+
proxy_buffering off;
|
| 62 |
+
proxy_cache off;
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
# ββ MJPEG video stream ββββββββββββββββββββββββββββββββ
|
| 66 |
+
location /video_feed {
|
| 67 |
+
proxy_pass http://gesture_app/video_feed;
|
| 68 |
+
proxy_http_version 1.1;
|
| 69 |
+
proxy_set_header Host $host;
|
| 70 |
+
proxy_buffering off;
|
| 71 |
+
proxy_cache off;
|
| 72 |
+
proxy_read_timeout 3600;
|
| 73 |
+
chunked_transfer_encoding on;
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
# ββ Avatar Socket.IO ββββββββββββββββββββββββββββββββββ
|
| 77 |
+
location /avatar/socket.io/ {
|
| 78 |
+
proxy_pass http://avatar_app/socket.io/;
|
| 79 |
+
proxy_http_version 1.1;
|
| 80 |
+
proxy_set_header Upgrade $http_upgrade;
|
| 81 |
+
proxy_set_header Connection "upgrade";
|
| 82 |
+
proxy_set_header Host $host;
|
| 83 |
+
proxy_set_header X-Forwarded-Proto $scheme;
|
| 84 |
+
proxy_read_timeout 86400;
|
| 85 |
+
proxy_buffering off;
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
# ββ Avatar app ββββββββββββββββββββββββββββββββββββββββ
|
| 89 |
+
location /avatar/ {
|
| 90 |
+
rewrite ^/avatar/(.*) /$1 break;
|
| 91 |
+
proxy_pass http://avatar_app/;
|
| 92 |
+
proxy_http_version 1.1;
|
| 93 |
+
proxy_set_header Host $host;
|
| 94 |
+
proxy_set_header X-Real-IP $remote_addr;
|
| 95 |
+
proxy_set_header X-Forwarded-Proto $scheme;
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
# ββ Gesture app (root) ββββββββββββββββββββββββββββββββ
|
| 99 |
+
location / {
|
| 100 |
+
proxy_pass http://gesture_app/;
|
| 101 |
+
proxy_http_version 1.1;
|
| 102 |
+
proxy_set_header Host $host;
|
| 103 |
+
proxy_set_header X-Real-IP $remote_addr;
|
| 104 |
+
proxy_set_header X-Forwarded-Proto $scheme;
|
| 105 |
+
}
|
| 106 |
+
}
|
| 107 |
+
}
|
requirements.txt
ADDED
|
Binary file (2.32 kB). View file
|
|
|
speech_to_sign_avatar.py
ADDED
|
@@ -0,0 +1,343 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ISL Fingerspelling System - Backend Server
|
| 3 |
+
Full A-Z Letter-by-Letter Speech-to-Sign Conversion
|
| 4 |
+
|
| 5 |
+
Install dependencies:
|
| 6 |
+
pip install SpeechRecognition flask flask-socketio eventlet
|
| 7 |
+
|
| 8 |
+
Usage:
|
| 9 |
+
python speech_to_sign_avatar_fixed.py
|
| 10 |
+
|
| 11 |
+
Then open: http://localhost:5001
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import eventlet
|
| 15 |
+
eventlet.monkey_patch()
|
| 16 |
+
|
| 17 |
+
import os
|
| 18 |
+
import json
|
| 19 |
+
import time
|
| 20 |
+
import threading
|
| 21 |
+
from datetime import datetime
|
| 22 |
+
from flask import Flask, render_template, request, jsonify, send_from_directory
|
| 23 |
+
from flask_socketio import SocketIO, emit
|
| 24 |
+
import speech_recognition as sr
|
| 25 |
+
|
| 26 |
+
# ======================================
|
| 27 |
+
# RUN MODE CONFIG (LOCAL or CLOUD)
|
| 28 |
+
# ======================================
|
| 29 |
+
|
| 30 |
+
RUN_MODE = os.getenv("RUN_MODE", "LOCAL")
|
| 31 |
+
|
| 32 |
+
# Disable microphone loading in cloud environment
|
| 33 |
+
if RUN_MODE == "CLOUD":
|
| 34 |
+
sr.Microphone = None
|
| 35 |
+
|
| 36 |
+
os.environ["EVENTLET_NO_GREENDNS"] = "yes"
|
| 37 |
+
|
| 38 |
+
# ==================== ISL ALPHABET LIBRARY (A-Z) ====================
|
| 39 |
+
|
| 40 |
+
print("Loading ISL gesture database...")
|
| 41 |
+
|
| 42 |
+
ISL_LIBRARY = {
|
| 43 |
+
'a': {'name': 'A', 'fingers': {'thumb': 0, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Fist with thumb out'},
|
| 44 |
+
'b': {'name': 'B', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 0, 'ring': 0, 'pinky': 0}, 'rot': [0, 0, 0], 'desc': 'All fingers straight up'},
|
| 45 |
+
'c': {'name': 'C', 'fingers': {'thumb': 0.3, 'index': 0.5, 'middle': 0.5, 'ring': 0.5, 'pinky': 0.5}, 'rot': [0, 0, 0], 'desc': 'Curved C shape'},
|
| 46 |
+
'd': {'name': 'D', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Index up, others closed'},
|
| 47 |
+
'e': {'name': 'E', 'fingers': {'thumb': 0.8, 'index': 1.2, 'middle': 1.2, 'ring': 1.2, 'pinky': 1.2}, 'rot': [0, 0, 0], 'desc': 'All fingers curled'},
|
| 48 |
+
'f': {'name': 'F', 'fingers': {'thumb': 0.8, 'index': 0.8, 'middle': 0, 'ring': 0, 'pinky': 0}, 'rot': [0, 0, 0], 'desc': 'OK sign variant'},
|
| 49 |
+
'g': {'name': 'G', 'fingers': {'thumb': 0, 'index': 0, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 90], 'desc': 'Index and thumb point sideways'},
|
| 50 |
+
'h': {'name': 'H', 'fingers': {'thumb': 1.5, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 90], 'desc': 'Two fingers horizontal'},
|
| 51 |
+
'i': {'name': 'I', 'fingers': {'thumb': 1.5, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 0}, 'rot': [0, 0, 0], 'desc': 'Pinky up'},
|
| 52 |
+
'j': {'name': 'J', 'fingers': {'thumb': 1.5, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 0}, 'rot': [0, 0, 0], 'desc': 'Pinky draws J'},
|
| 53 |
+
'k': {'name': 'K', 'fingers': {'thumb': 0.5, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 45], 'desc': 'Index and middle V shape'},
|
| 54 |
+
'l': {'name': 'L', 'fingers': {'thumb': 0, 'index': 0, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'L shape'},
|
| 55 |
+
'm': {'name': 'M', 'fingers': {'thumb': 0.8, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Three fingers tucked under thumb'},
|
| 56 |
+
'n': {'name': 'N', 'fingers': {'thumb': 0.8, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Two fingers tucked'},
|
| 57 |
+
'o': {'name': 'O', 'fingers': {'thumb': 0.5, 'index': 0.8, 'middle': 0.8, 'ring': 0.8, 'pinky': 0.8}, 'rot': [0, 0, 0], 'desc': 'All fingers form O'},
|
| 58 |
+
'p': {'name': 'P', 'fingers': {'thumb': 0.5, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, -45], 'desc': 'K pointing down'},
|
| 59 |
+
'q': {'name': 'Q', 'fingers': {'thumb': 0, 'index': 0, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, -90], 'desc': 'G pointing down'},
|
| 60 |
+
'r': {'name': 'R', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Index and middle crossed'},
|
| 61 |
+
's': {'name': 'S', 'fingers': {'thumb': 0.5, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Fist with thumb across'},
|
| 62 |
+
't': {'name': 'T', 'fingers': {'thumb': 0.8, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Thumb between index and middle'},
|
| 63 |
+
'u': {'name': 'U', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Two fingers together'},
|
| 64 |
+
'v': {'name': 'V', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 0, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Victory/Peace sign'},
|
| 65 |
+
'w': {'name': 'W', 'fingers': {'thumb': 1.2, 'index': 0, 'middle': 0, 'ring': 0, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Three fingers up'},
|
| 66 |
+
'x': {'name': 'X', 'fingers': {'thumb': 1.5, 'index': 0.8, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Index bent like hook'},
|
| 67 |
+
'y': {'name': 'Y', 'fingers': {'thumb': 0, 'index': 1.5, 'middle': 1.5, 'ring': 1.5, 'pinky': 0}, 'rot': [0, 0, 0], 'desc': 'Thumb and pinky out (hang loose)'},
|
| 68 |
+
'z': {'name': 'Z', 'fingers': {'thumb': 1.5, 'index': 0, 'middle': 1.5, 'ring': 1.5, 'pinky': 1.5}, 'rot': [0, 0, 0], 'desc': 'Index draws Z'}
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
print(f"Loaded {len(ISL_LIBRARY)} gestures")
|
| 72 |
+
|
| 73 |
+
# ==================== Flask Setup ====================
|
| 74 |
+
app = Flask(__name__)
|
| 75 |
+
app.config['SECRET_KEY'] = 'isl-fingerspelling-2024'
|
| 76 |
+
socketio = SocketIO(
|
| 77 |
+
app,
|
| 78 |
+
cors_allowed_origins="*",
|
| 79 |
+
async_mode="eventlet",
|
| 80 |
+
ping_timeout=60,
|
| 81 |
+
ping_interval=25
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
print("=" * 60)
|
| 85 |
+
print("ISL SPEECH TO SIGN AVATAR SYSTEM")
|
| 86 |
+
print("RUN MODE:", RUN_MODE)
|
| 87 |
+
print("=" * 60)
|
| 88 |
+
|
| 89 |
+
# Speech recognizer
|
| 90 |
+
recognizer = sr.Recognizer()
|
| 91 |
+
recognizer.energy_threshold = 3000
|
| 92 |
+
recognizer.dynamic_energy_threshold = True
|
| 93 |
+
recognizer.pause_threshold = 0.8
|
| 94 |
+
recognizer.non_speaking_duration = 0.5
|
| 95 |
+
|
| 96 |
+
# System statistics
|
| 97 |
+
stats = {
|
| 98 |
+
"total_conversions": 0,
|
| 99 |
+
"total_gestures": 0,
|
| 100 |
+
"speech_recognitions": 0,
|
| 101 |
+
"errors": 0,
|
| 102 |
+
"start_time": datetime.now()
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
# ==================== Text to Letter Sequence ====================
|
| 106 |
+
def text_to_letter_sequence(text):
|
| 107 |
+
"""Convert text to letter-by-letter fingerspelling sequence"""
|
| 108 |
+
clean_text = text.lower().replace(",", "").replace(".", "").replace("!", "").replace("?", "").replace("-", " ")
|
| 109 |
+
chars = list(clean_text)
|
| 110 |
+
|
| 111 |
+
sequence = []
|
| 112 |
+
|
| 113 |
+
for i, char in enumerate(chars):
|
| 114 |
+
if char == ' ':
|
| 115 |
+
sequence.append({
|
| 116 |
+
'name': 'SPACE',
|
| 117 |
+
'desc': 'Word space - pause between words',
|
| 118 |
+
'duration': 0.6,
|
| 119 |
+
'isSpace': True,
|
| 120 |
+
'keyframes': [
|
| 121 |
+
{
|
| 122 |
+
'time': 0,
|
| 123 |
+
'hand': 'right',
|
| 124 |
+
'pos': [0, 0, 0],
|
| 125 |
+
'rot': [0, 0, 0],
|
| 126 |
+
'fingers': {'thumb': 0, 'index': 0, 'middle': 0, 'ring': 0, 'pinky': 0}
|
| 127 |
+
},
|
| 128 |
+
{
|
| 129 |
+
'time': 0.6,
|
| 130 |
+
'hand': 'right',
|
| 131 |
+
'pos': [0.2, 0, 0],
|
| 132 |
+
'rot': [0, 0, 0],
|
| 133 |
+
'fingers': {'thumb': 0, 'index': 0, 'middle': 0, 'ring': 0, 'pinky': 0}
|
| 134 |
+
}
|
| 135 |
+
]
|
| 136 |
+
})
|
| 137 |
+
continue
|
| 138 |
+
|
| 139 |
+
if char in ISL_LIBRARY:
|
| 140 |
+
letter_data = ISL_LIBRARY[char]
|
| 141 |
+
rot_rad = [r * 3.14159 / 180 for r in letter_data['rot']]
|
| 142 |
+
|
| 143 |
+
sequence.append({
|
| 144 |
+
'name': letter_data['name'],
|
| 145 |
+
'letter': char,
|
| 146 |
+
'desc': letter_data['desc'],
|
| 147 |
+
'duration': 0.7,
|
| 148 |
+
'keyframes': [
|
| 149 |
+
{
|
| 150 |
+
'time': 0,
|
| 151 |
+
'hand': 'right',
|
| 152 |
+
'pos': [0, 0, 0],
|
| 153 |
+
'rot': rot_rad,
|
| 154 |
+
'fingers': letter_data['fingers']
|
| 155 |
+
},
|
| 156 |
+
{
|
| 157 |
+
'time': 0.3,
|
| 158 |
+
'hand': 'right',
|
| 159 |
+
'pos': [0, 0.15, 0],
|
| 160 |
+
'rot': rot_rad,
|
| 161 |
+
'fingers': letter_data['fingers']
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
'time': 0.7,
|
| 165 |
+
'hand': 'right',
|
| 166 |
+
'pos': [0, 0.15, 0],
|
| 167 |
+
'rot': rot_rad,
|
| 168 |
+
'fingers': letter_data['fingers']
|
| 169 |
+
}
|
| 170 |
+
]
|
| 171 |
+
})
|
| 172 |
+
|
| 173 |
+
return sequence
|
| 174 |
+
|
| 175 |
+
# ==================== Routes ====================
|
| 176 |
+
@app.route('/')
|
| 177 |
+
def index():
|
| 178 |
+
"""Serve main interface"""
|
| 179 |
+
try:
|
| 180 |
+
return send_from_directory(os.path.join(os.getcwd(), 'templates'), 'avatar_interface.html')
|
| 181 |
+
except Exception:
|
| 182 |
+
return "Avatar UI Loaded. Frontend missing."
|
| 183 |
+
|
| 184 |
+
@app.route('/health')
|
| 185 |
+
def health():
|
| 186 |
+
"""Health check endpoint for cloud platforms"""
|
| 187 |
+
return jsonify({"status": "running"})
|
| 188 |
+
|
| 189 |
+
@app.route('/static/<path:filename>')
|
| 190 |
+
def serve_static(filename):
|
| 191 |
+
"""Serve static files for cloud platforms"""
|
| 192 |
+
return send_from_directory('static', filename)
|
| 193 |
+
|
| 194 |
+
@app.route('/api/text-to-signs', methods=['POST'])
|
| 195 |
+
def text_to_signs_api():
|
| 196 |
+
"""Convert text to letter sequence"""
|
| 197 |
+
try:
|
| 198 |
+
data = request.json
|
| 199 |
+
text = data.get('text', '')
|
| 200 |
+
|
| 201 |
+
if not text:
|
| 202 |
+
return jsonify({"success": False, "error": "No text provided"}), 400
|
| 203 |
+
|
| 204 |
+
print(f'π Converting text: "{text}"')
|
| 205 |
+
|
| 206 |
+
sequence = text_to_letter_sequence(text)
|
| 207 |
+
|
| 208 |
+
stats["total_conversions"] += 1
|
| 209 |
+
stats["total_gestures"] += len(sequence)
|
| 210 |
+
|
| 211 |
+
return jsonify({
|
| 212 |
+
"success": True,
|
| 213 |
+
"text": text,
|
| 214 |
+
"sequence": sequence,
|
| 215 |
+
"stats": get_stats()
|
| 216 |
+
})
|
| 217 |
+
|
| 218 |
+
except Exception as e:
|
| 219 |
+
stats["errors"] += 1
|
| 220 |
+
print(f"β Error: {e}")
|
| 221 |
+
return jsonify({"success": False, "error": str(e)}), 500
|
| 222 |
+
|
| 223 |
+
@app.route('/api/stats')
|
| 224 |
+
def get_stats_api():
|
| 225 |
+
"""Get system statistics"""
|
| 226 |
+
return jsonify(get_stats())
|
| 227 |
+
|
| 228 |
+
def get_stats():
|
| 229 |
+
"""Calculate current statistics"""
|
| 230 |
+
uptime = (datetime.now() - stats["start_time"]).total_seconds()
|
| 231 |
+
|
| 232 |
+
return {
|
| 233 |
+
"total_conversions": stats["total_conversions"],
|
| 234 |
+
"total_gestures": stats["total_gestures"],
|
| 235 |
+
"speech_recognitions": stats["speech_recognitions"],
|
| 236 |
+
"errors": stats["errors"],
|
| 237 |
+
"uptime_seconds": int(uptime),
|
| 238 |
+
"alphabet_letters": len(ISL_LIBRARY)
|
| 239 |
+
}
|
| 240 |
+
|
| 241 |
+
# ==================== Socket.IO Events ====================
|
| 242 |
+
@socketio.on('connect')
|
| 243 |
+
def handle_connect():
|
| 244 |
+
"""Client connected"""
|
| 245 |
+
print('β
Client connected')
|
| 246 |
+
emit('status', {'message': 'Connected', 'stats': get_stats()})
|
| 247 |
+
|
| 248 |
+
@socketio.on('disconnect')
|
| 249 |
+
def handle_disconnect():
|
| 250 |
+
"""Client disconnected"""
|
| 251 |
+
print('π Client disconnected')
|
| 252 |
+
|
| 253 |
+
@socketio.on('record_speech')
|
| 254 |
+
def handle_record_speech():
|
| 255 |
+
"""Record speech and convert to fingerspelling"""
|
| 256 |
+
|
| 257 |
+
# Disable microphone in cloud environment
|
| 258 |
+
if RUN_MODE == "CLOUD":
|
| 259 |
+
emit("error", {"message": "Microphone recording not supported in cloud deployment"})
|
| 260 |
+
return
|
| 261 |
+
|
| 262 |
+
def record_and_convert():
|
| 263 |
+
try:
|
| 264 |
+
print('π€ Starting speech recognition...')
|
| 265 |
+
|
| 266 |
+
with sr.Microphone() as source:
|
| 267 |
+
print('π€ Listening... Speak now!')
|
| 268 |
+
socketio.emit('recording_status', {'status': 'listening'})
|
| 269 |
+
|
| 270 |
+
recognizer.adjust_for_ambient_noise(source, duration=0.5)
|
| 271 |
+
audio = recognizer.listen(source, timeout=5, phrase_time_limit=10)
|
| 272 |
+
|
| 273 |
+
print('π Processing speech...')
|
| 274 |
+
socketio.emit('recording_status', {'status': 'processing'})
|
| 275 |
+
|
| 276 |
+
try:
|
| 277 |
+
text = recognizer.recognize_google(audio, language='en-IN')
|
| 278 |
+
print(f'β
Recognized: "{text}"')
|
| 279 |
+
|
| 280 |
+
stats["speech_recognitions"] += 1
|
| 281 |
+
stats["total_conversions"] += 1
|
| 282 |
+
|
| 283 |
+
sequence = text_to_letter_sequence(text)
|
| 284 |
+
stats["total_gestures"] += len(sequence)
|
| 285 |
+
|
| 286 |
+
socketio.emit('play_signs', {
|
| 287 |
+
'text': text,
|
| 288 |
+
'sequence': sequence,
|
| 289 |
+
'stats': get_stats()
|
| 290 |
+
})
|
| 291 |
+
|
| 292 |
+
except sr.UnknownValueError:
|
| 293 |
+
print('β Could not understand speech')
|
| 294 |
+
stats["errors"] += 1
|
| 295 |
+
socketio.emit('error', {
|
| 296 |
+
'message': 'Could not understand speech. Please speak clearly and try again.'
|
| 297 |
+
})
|
| 298 |
+
|
| 299 |
+
except sr.RequestError as e:
|
| 300 |
+
print(f'β Speech recognition API error: {e}')
|
| 301 |
+
stats["errors"] += 1
|
| 302 |
+
socketio.emit('error', {
|
| 303 |
+
'message': 'Speech recognition service error. Please check your internet connection.'
|
| 304 |
+
})
|
| 305 |
+
|
| 306 |
+
except sr.WaitTimeoutError:
|
| 307 |
+
print('β±οΈ No speech detected (timeout)')
|
| 308 |
+
stats["errors"] += 1
|
| 309 |
+
socketio.emit('error', {
|
| 310 |
+
'message': 'No speech detected. Please try again.'
|
| 311 |
+
})
|
| 312 |
+
|
| 313 |
+
except Exception as e:
|
| 314 |
+
print(f'β Recording error: {e}')
|
| 315 |
+
stats["errors"] += 1
|
| 316 |
+
socketio.emit('error', {
|
| 317 |
+
'message': f'Recording error: {str(e)}'
|
| 318 |
+
})
|
| 319 |
+
|
| 320 |
+
threading.Thread(target=record_and_convert, daemon=True).start()
|
| 321 |
+
|
| 322 |
+
# ==================== Main ====================
|
| 323 |
+
if __name__ == '__main__':
|
| 324 |
+
|
| 325 |
+
print("\n" + "="*60)
|
| 326 |
+
print("ISL SPEECH TO SIGN AVATAR SYSTEM STARTING")
|
| 327 |
+
print("RUN MODE:", RUN_MODE)
|
| 328 |
+
print("="*60)
|
| 329 |
+
|
| 330 |
+
os.makedirs('templates', exist_ok=True)
|
| 331 |
+
|
| 332 |
+
PORT = int(os.environ.get("PORT", 5001))
|
| 333 |
+
|
| 334 |
+
print("Server starting on port:", PORT)
|
| 335 |
+
print("Health check endpoint: /health")
|
| 336 |
+
|
| 337 |
+
socketio.run(
|
| 338 |
+
app,
|
| 339 |
+
host="0.0.0.0",
|
| 340 |
+
port=PORT,
|
| 341 |
+
debug=False,
|
| 342 |
+
use_reloader=False
|
| 343 |
+
)
|
supervisord.conf
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ============================================================
|
| 2 |
+
# supervisord.conf β Hugging Face Spaces
|
| 3 |
+
# Manages 4 processes in one container:
|
| 4 |
+
# 1. detection_engine β MediaPipe + TF inference
|
| 5 |
+
# 2. gesture_app β Flask dashboard, port 5000
|
| 6 |
+
# 3. avatar_app β Flask avatar, port 5001
|
| 7 |
+
# 4. nginx β Reverse proxy, port 7860
|
| 8 |
+
# ============================================================
|
| 9 |
+
|
| 10 |
+
[supervisord]
|
| 11 |
+
nodaemon=true
|
| 12 |
+
logfile=/tmp/supervisord.log
|
| 13 |
+
loglevel=info
|
| 14 |
+
pidfile=/tmp/supervisord.pid
|
| 15 |
+
|
| 16 |
+
[unix_http_server]
|
| 17 |
+
file=/tmp/supervisor.sock
|
| 18 |
+
|
| 19 |
+
[rpcinterface:supervisor]
|
| 20 |
+
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
| 21 |
+
|
| 22 |
+
[supervisorctl]
|
| 23 |
+
serverurl=unix:///tmp/supervisor.sock
|
| 24 |
+
|
| 25 |
+
# ββ 1. ISL Detection Engine (starts first β loads TF model) ββ
|
| 26 |
+
[program:detection_engine]
|
| 27 |
+
command=python /app/isl_detection.py
|
| 28 |
+
directory=/app
|
| 29 |
+
autostart=true
|
| 30 |
+
autorestart=true
|
| 31 |
+
priority=1
|
| 32 |
+
startretries=5
|
| 33 |
+
startsecs=20
|
| 34 |
+
stdout_logfile=/tmp/detection.log
|
| 35 |
+
stderr_logfile=/tmp/detection_err.log
|
| 36 |
+
stdout_logfile_maxbytes=5MB
|
| 37 |
+
environment=
|
| 38 |
+
RUN_MODE="CLOUD",
|
| 39 |
+
PYTHONUNBUFFERED="1",
|
| 40 |
+
TF_CPP_MIN_LOG_LEVEL="3",
|
| 41 |
+
CUDA_VISIBLE_DEVICES="-1"
|
| 42 |
+
|
| 43 |
+
# ββ 2. Gesture Dashboard (Flask + Socket.IO, port 5000) βββββββ
|
| 44 |
+
[program:gesture_app]
|
| 45 |
+
command=python /app/isl_ui_dashboard.py
|
| 46 |
+
directory=/app
|
| 47 |
+
autostart=true
|
| 48 |
+
autorestart=true
|
| 49 |
+
priority=10
|
| 50 |
+
startretries=5
|
| 51 |
+
startsecs=15
|
| 52 |
+
stdout_logfile=/tmp/gesture.log
|
| 53 |
+
stderr_logfile=/tmp/gesture_err.log
|
| 54 |
+
stdout_logfile_maxbytes=5MB
|
| 55 |
+
environment=
|
| 56 |
+
RUN_MODE="CLOUD",
|
| 57 |
+
PYTHONUNBUFFERED="1",
|
| 58 |
+
TF_CPP_MIN_LOG_LEVEL="3",
|
| 59 |
+
CUDA_VISIBLE_DEVICES="-1"
|
| 60 |
+
|
| 61 |
+
# ββ 3. Avatar App (Flask + eventlet, port 5001) βββββββββββββββ
|
| 62 |
+
[program:avatar_app]
|
| 63 |
+
command=python /app/speech_to_sign_avatar.py
|
| 64 |
+
directory=/app
|
| 65 |
+
autostart=true
|
| 66 |
+
autorestart=true
|
| 67 |
+
priority=10
|
| 68 |
+
startretries=5
|
| 69 |
+
startsecs=10
|
| 70 |
+
stdout_logfile=/tmp/avatar.log
|
| 71 |
+
stderr_logfile=/tmp/avatar_err.log
|
| 72 |
+
stdout_logfile_maxbytes=5MB
|
| 73 |
+
environment=
|
| 74 |
+
RUN_MODE="CLOUD",
|
| 75 |
+
PYTHONUNBUFFERED="1",
|
| 76 |
+
PORT="5001"
|
| 77 |
+
|
| 78 |
+
# ββ 4. nginx (starts after Flask apps are up) βββββββββββββββββ
|
| 79 |
+
[program:nginx]
|
| 80 |
+
command=/usr/sbin/nginx -g "daemon off;" -c /etc/nginx/nginx.conf
|
| 81 |
+
autostart=true
|
| 82 |
+
autorestart=true
|
| 83 |
+
priority=30
|
| 84 |
+
startretries=3
|
| 85 |
+
startsecs=5
|
| 86 |
+
stdout_logfile=/tmp/nginx.log
|
| 87 |
+
stderr_logfile=/tmp/nginx_err.log
|
| 88 |
+
stdout_logfile_maxbytes=5MB
|
templates/avatar_interface.html
ADDED
|
@@ -0,0 +1,653 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="UTF-8"/>
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover"/>
|
| 6 |
+
<title>ISL Fingerspelling β AI Sign Language</title>
|
| 7 |
+
<link rel="preconnect" href="https://fonts.googleapis.com"/>
|
| 8 |
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin/>
|
| 9 |
+
<link href="https://fonts.googleapis.com/css2?family=Playfair+Display:wght@400;600;700&family=DM+Sans:wght@300;400;500;600&family=DM+Mono:wght@400;500&display=swap" rel="stylesheet"/>
|
| 10 |
+
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
|
| 11 |
+
<style>
|
| 12 |
+
:root{
|
| 13 |
+
--bg:#0d1117;--surf:#131922;--card:#1a2233;
|
| 14 |
+
--bdr:rgba(212,168,83,.13);--bdrm:rgba(212,168,83,.30);--bdrh:rgba(212,168,83,.58);
|
| 15 |
+
--gold:#d4a853;--goldb:#f0c76a;--goldd:#a07c3a;
|
| 16 |
+
--sage:#7eb89a;--txt:#f0ede6;--txtm:#a8a096;--txtl:#5a5650;--red:#c0614a;
|
| 17 |
+
--r8:8px;--r14:14px;
|
| 18 |
+
--fD:'Playfair Display',Georgia,serif;
|
| 19 |
+
--fB:'DM Sans',system-ui,sans-serif;
|
| 20 |
+
--fM:'DM Mono',monospace;
|
| 21 |
+
}
|
| 22 |
+
*,*::before,*::after{margin:0;padding:0;box-sizing:border-box}
|
| 23 |
+
html,body{height:100%;overflow:hidden}
|
| 24 |
+
body{font-family:var(--fB);background:var(--bg);color:var(--txt);-webkit-font-smoothing:antialiased;display:flex;flex-direction:column}
|
| 25 |
+
#app{display:flex;flex:1;overflow:hidden}
|
| 26 |
+
|
| 27 |
+
/* ββ Canvas pane ββ */
|
| 28 |
+
#cvp{flex:1;display:flex;flex-direction:column;min-width:0;background:var(--bg);position:relative}
|
| 29 |
+
#hc{flex:1;width:100%;display:block;touch-action:none}
|
| 30 |
+
.topbar{display:flex;align-items:center;justify-content:space-between;padding:10px 16px 0;flex-shrink:0}
|
| 31 |
+
.brand{display:flex;align-items:baseline;gap:8px}
|
| 32 |
+
.brand-t{font-family:var(--fD);font-size:1.1rem;font-weight:700;color:var(--gold)}
|
| 33 |
+
.brand-s{font-size:.65rem;color:var(--txtl);text-transform:uppercase;letter-spacing:.07em}
|
| 34 |
+
.badge{font-size:.65rem;font-family:var(--fM);color:var(--sage);background:rgba(126,184,154,.1);border:1px solid rgba(126,184,154,.25);padding:3px 9px;border-radius:20px;transition:all .3s}
|
| 35 |
+
.vrow{display:flex;gap:4px;padding:6px 16px 0;flex-shrink:0}
|
| 36 |
+
.vb{padding:4px 10px;font-size:.63rem;font-family:var(--fM);background:transparent;border:1px solid var(--bdr);color:var(--txtl);border-radius:6px;cursor:pointer;transition:all .2s;text-transform:uppercase;letter-spacing:.03em}
|
| 37 |
+
.vb:hover{color:var(--txt);border-color:var(--bdrm)}
|
| 38 |
+
.vb.on{background:rgba(212,168,83,.18);color:var(--gold);border-color:var(--bdrm)}
|
| 39 |
+
.gov{position:absolute;bottom:0;left:0;right:0;padding:12px 18px 14px;background:linear-gradient(to top,rgba(13,17,23,1) 55%,transparent);display:flex;align-items:flex-end;justify-content:space-between;gap:10px;pointer-events:none}
|
| 40 |
+
.gl{font-family:var(--fD);font-size:clamp(2rem,5vw,3.2rem);font-weight:700;color:var(--goldb);line-height:1}
|
| 41 |
+
.gd{font-size:.72rem;color:var(--txtm);text-align:right;max-width:180px;line-height:1.45}
|
| 42 |
+
.gp{position:absolute;bottom:0;left:0;right:0;height:3px;background:rgba(212,168,83,.1)}
|
| 43 |
+
.gb{height:100%;width:0%;background:linear-gradient(90deg,var(--goldd),var(--goldb));transition:width .12s linear;position:relative;overflow:hidden}
|
| 44 |
+
.gb::after{content:'';position:absolute;inset:0;background:linear-gradient(90deg,transparent,rgba(255,255,255,.3),transparent);animation:sh 1.8s infinite}
|
| 45 |
+
@keyframes sh{from{transform:translateX(-100%)}to{transform:translateX(200%)}}
|
| 46 |
+
|
| 47 |
+
/* ββ Control pane ββ */
|
| 48 |
+
#cp{width:340px;flex-shrink:0;display:flex;flex-direction:column;background:var(--surf);border-left:1px solid var(--bdr);overflow-y:auto}
|
| 49 |
+
#cp::-webkit-scrollbar{width:4px}
|
| 50 |
+
#cp::-webkit-scrollbar-thumb{background:var(--bdrm);border-radius:4px}
|
| 51 |
+
@media(max-width:720px){
|
| 52 |
+
#app{flex-direction:column;height:100%}
|
| 53 |
+
#cvp{flex:0 0 46vh;min-height:260px;max-height:46vh}
|
| 54 |
+
#cp{width:100%;flex:1;border-left:none;border-top:1px solid var(--bdr);overflow-y:auto}
|
| 55 |
+
}
|
| 56 |
+
.pan{padding:14px 16px;display:flex;flex-direction:column;gap:12px}
|
| 57 |
+
.slbl{font-size:.65rem;font-weight:600;text-transform:uppercase;letter-spacing:.08em;color:var(--txtl);display:flex;align-items:center;gap:6px}
|
| 58 |
+
textarea{width:100%;height:82px;resize:none;background:var(--card);border:1px solid var(--bdr);border-radius:var(--r14);color:var(--txt);font-family:var(--fB);font-size:.92rem;line-height:1.5;padding:11px 13px 26px;outline:none;transition:border-color .25s}
|
| 59 |
+
textarea::placeholder{color:var(--txtl)}
|
| 60 |
+
textarea:focus{border-color:var(--bdrh)}
|
| 61 |
+
.iwrap{position:relative}
|
| 62 |
+
.cc{position:absolute;bottom:8px;right:10px;font-size:.62rem;font-family:var(--fM);color:var(--txtl)}
|
| 63 |
+
|
| 64 |
+
/* ββ Speech warning banner ββ */
|
| 65 |
+
.mic-warn{background:rgba(212,168,83,.07);border:1px solid rgba(212,168,83,.25);border-radius:var(--r8);padding:10px 13px;font-size:.72rem;color:var(--txtm);line-height:1.7;display:none}
|
| 66 |
+
.mic-warn.show{display:block}
|
| 67 |
+
.mic-warn strong{color:var(--gold)}
|
| 68 |
+
.mic-warn code{background:rgba(212,168,83,.12);border:1px solid rgba(212,168,83,.2);border-radius:4px;padding:1px 5px;font-family:var(--fM);font-size:.68rem;color:var(--goldb)}
|
| 69 |
+
|
| 70 |
+
/* ββ Mic status inline ββ */
|
| 71 |
+
.mic-status{display:none;align-items:center;gap:7px;padding:6px 11px;border-radius:var(--r8);background:rgba(126,184,154,.07);border:1px solid rgba(126,184,154,.2);font-size:.72rem;color:var(--sage);font-family:var(--fM)}
|
| 72 |
+
.mic-status.show{display:flex}
|
| 73 |
+
.mic-status.listening{background:rgba(192,97,74,.08);border-color:rgba(192,97,74,.3);color:#e07060}
|
| 74 |
+
.mic-dot{width:7px;height:7px;border-radius:50%;background:var(--sage);animation:pd .9s infinite;flex-shrink:0}
|
| 75 |
+
.mic-status.listening .mic-dot{background:#e07060}
|
| 76 |
+
@keyframes pd{0%,100%{opacity:1;transform:scale(1)}50%{opacity:.5;transform:scale(.75)}}
|
| 77 |
+
|
| 78 |
+
/* ββ Buttons ββ */
|
| 79 |
+
.bg4{display:grid;grid-template-columns:repeat(4,1fr);gap:7px}
|
| 80 |
+
.btn{display:flex;align-items:center;justify-content:center;gap:4px;padding:10px 6px;border:1px solid transparent;border-radius:var(--r14);font-family:var(--fB);font-size:.76rem;font-weight:600;cursor:pointer;transition:all .22s;white-space:nowrap;overflow:hidden}
|
| 81 |
+
.btn:active{transform:scale(.96)}
|
| 82 |
+
.btn:disabled{opacity:.38;cursor:not-allowed}
|
| 83 |
+
.btn:disabled:active{transform:none}
|
| 84 |
+
.bc{background:linear-gradient(135deg,#c49a3a,#d4a853,#e8c06a);color:#0d1117;border-color:rgba(212,168,83,.4);box-shadow:0 2px 10px rgba(212,168,83,.2)}
|
| 85 |
+
.bc:not(:disabled):hover{box-shadow:0 4px 18px rgba(212,168,83,.38)}
|
| 86 |
+
.bm{background:linear-gradient(135deg,#4e7d66,#7eb89a);color:#0d1117;border-color:rgba(126,184,154,.35)}
|
| 87 |
+
.bm:not(:disabled):hover{box-shadow:0 4px 16px rgba(126,184,154,.3)}
|
| 88 |
+
.bm.listening{background:linear-gradient(135deg,#8a3a2a,#c0614a)!important;color:#fff!important;border-color:rgba(192,97,74,.4)!important;animation:pulse .8s infinite}
|
| 89 |
+
@keyframes pulse{0%,100%{box-shadow:0 0 0 0 rgba(192,97,74,.4)}50%{box-shadow:0 0 0 8px rgba(192,97,74,0)}}
|
| 90 |
+
.bs{background:transparent;color:var(--red);border-color:rgba(192,97,74,.3)}
|
| 91 |
+
.bs:hover{background:rgba(192,97,74,.08);border-color:rgba(192,97,74,.55)}
|
| 92 |
+
.bg2{background:transparent;color:var(--txtm);border-color:var(--bdr)}
|
| 93 |
+
.bg2:hover{border-color:var(--bdrm);color:var(--txt)}
|
| 94 |
+
|
| 95 |
+
/* ββ Status pill ββ */
|
| 96 |
+
.st{display:flex;align-items:center;gap:7px;padding:7px 13px;border-radius:30px;font-size:.73rem;font-weight:500;border:1px solid var(--bdr);background:var(--card);color:var(--txtm);transition:all .3s;min-height:34px}
|
| 97 |
+
.dot{width:7px;height:7px;border-radius:50%;background:var(--txtl);flex-shrink:0;transition:all .3s}
|
| 98 |
+
.st.act .dot{background:var(--sage);box-shadow:0 0 8px var(--sage);animation:pd 1.4s infinite}
|
| 99 |
+
.st.act{border-color:rgba(126,184,154,.3);color:var(--sage)}
|
| 100 |
+
.st.rec .dot{background:var(--gold);box-shadow:0 0 8px var(--gold);animation:pd .9s infinite}
|
| 101 |
+
.st.rec{border-color:var(--bdrm);color:var(--gold)}
|
| 102 |
+
.st.err .dot{background:var(--red)}
|
| 103 |
+
.st.err{border-color:rgba(192,97,74,.3);color:var(--red)}
|
| 104 |
+
|
| 105 |
+
.div{height:1px;background:var(--bdr);margin:0 -16px}
|
| 106 |
+
.sh{display:flex;align-items:center;justify-content:space-between}
|
| 107 |
+
.sbadge{font-family:var(--fM);font-size:.62rem;background:rgba(212,168,83,.12);color:var(--gold);border:1px solid rgba(212,168,83,.2);padding:2px 8px;border-radius:20px}
|
| 108 |
+
.ll{max-height:180px;overflow-y:auto;display:flex;flex-direction:column;gap:4px;padding-right:2px}
|
| 109 |
+
.ll::-webkit-scrollbar{width:3px}
|
| 110 |
+
.ll::-webkit-scrollbar-thumb{background:var(--bdrm);border-radius:3px}
|
| 111 |
+
.li{display:flex;align-items:center;gap:9px;padding:7px 11px;border-radius:var(--r8);background:var(--card);border:1px solid var(--bdr);transition:all .25s;flex-shrink:0}
|
| 112 |
+
.li.on{background:rgba(212,168,83,.1);border-color:var(--bdrh);box-shadow:0 0 14px rgba(212,168,83,.1)}
|
| 113 |
+
.li.sp{background:rgba(126,184,154,.05);border-color:rgba(126,184,154,.15)}
|
| 114 |
+
.lc{font-family:var(--fD);font-size:1.05rem;font-weight:700;color:var(--gold);width:22px;text-align:center;flex-shrink:0}
|
| 115 |
+
.li.on .lc{color:var(--goldb)}
|
| 116 |
+
.li.sp .lc{color:var(--sage);font-size:.75rem}
|
| 117 |
+
.lh{font-size:.68rem;color:var(--txtl);flex:1}
|
| 118 |
+
.li.on .lh{color:var(--txtm)}
|
| 119 |
+
.es{text-align:center;padding:24px 0;color:var(--txtl);font-size:.78rem;line-height:1.7}
|
| 120 |
+
.sg{display:grid;grid-template-columns:1fr 1fr;gap:7px}
|
| 121 |
+
.st2{background:var(--card);border:1px solid var(--bdr);border-radius:var(--r14);padding:11px 13px;transition:border-color .25s}
|
| 122 |
+
.st2:hover{border-color:var(--bdrm)}
|
| 123 |
+
.sl2{font-size:.6rem;text-transform:uppercase;letter-spacing:.07em;color:var(--txtl);font-weight:600;margin-bottom:3px}
|
| 124 |
+
.sv{font-family:var(--fD);font-size:1.6rem;font-weight:700;color:var(--gold);line-height:1}
|
| 125 |
+
.abt{font-size:.77rem;color:var(--txtm);line-height:1.7}
|
| 126 |
+
.abt p{margin-bottom:8px}
|
| 127 |
+
.al{list-style:none;margin-bottom:8px}
|
| 128 |
+
.al li{padding-left:12px;position:relative;font-size:.75rem;padding-bottom:2px}
|
| 129 |
+
.al li::before{content:'βΊ';position:absolute;left:0;color:var(--gold);font-weight:700}
|
| 130 |
+
.acr{margin-top:12px;padding-top:10px;border-top:1px solid var(--bdr)}
|
| 131 |
+
.an{font-family:var(--fD);color:var(--gold);font-weight:600;font-size:.88rem}
|
| 132 |
+
.ar{font-size:.67rem;color:var(--txtl);margin:2px 0 3px}
|
| 133 |
+
.ae{color:var(--sage);font-size:.68rem;font-family:var(--fM)}
|
| 134 |
+
.nav{display:flex;gap:7px;padding:0 16px 20px}
|
| 135 |
+
.na{flex:1;display:flex;align-items:center;justify-content:center;gap:6px;padding:12px 8px;border-radius:var(--r14);border:1px solid var(--bdr);background:var(--card);color:var(--txtm);text-decoration:none;font-size:.76rem;font-weight:600;transition:all .25s}
|
| 136 |
+
.na:hover{border-color:var(--bdrm);color:var(--txt)}
|
| 137 |
+
.na.on{background:rgba(212,168,83,.1);border-color:var(--bdrh);color:var(--gold)}
|
| 138 |
+
</style>
|
| 139 |
+
</head>
|
| 140 |
+
<body>
|
| 141 |
+
<div id="app">
|
| 142 |
+
|
| 143 |
+
<!-- Canvas side -->
|
| 144 |
+
<div id="cvp">
|
| 145 |
+
<div class="topbar">
|
| 146 |
+
<div class="brand">
|
| 147 |
+
<span class="brand-t">ISL Fingerspelling</span>
|
| 148 |
+
<span class="brand-s">Speech Β· Sign Β· AI</span>
|
| 149 |
+
</div>
|
| 150 |
+
<span class="badge" id="badge">β Ready</span>
|
| 151 |
+
</div>
|
| 152 |
+
<div class="vrow">
|
| 153 |
+
<button class="vb on" onclick="setView('front',this)">Front</button>
|
| 154 |
+
<button class="vb" onclick="setView('side',this)">Side</button>
|
| 155 |
+
<button class="vb" onclick="setView('top',this)">Top</button>
|
| 156 |
+
<button class="vb" onclick="setView('spin',this)">Spin</button>
|
| 157 |
+
</div>
|
| 158 |
+
<canvas id="hc"></canvas>
|
| 159 |
+
<div class="gov">
|
| 160 |
+
<div id="gl" class="gl">π€</div>
|
| 161 |
+
<div id="gd" class="gd">Type or speak to begin</div>
|
| 162 |
+
<div class="gp"><div class="gb" id="gb"></div></div>
|
| 163 |
+
</div>
|
| 164 |
+
</div>
|
| 165 |
+
|
| 166 |
+
<!-- Control side -->
|
| 167 |
+
<div id="cp">
|
| 168 |
+
<div class="pan">
|
| 169 |
+
|
| 170 |
+
<div class="slbl">βοΈ Enter Text</div>
|
| 171 |
+
<div class="iwrap">
|
| 172 |
+
<textarea id="txt" placeholder="Type your message hereβ¦" maxlength="200"></textarea>
|
| 173 |
+
<span class="cc" id="cc">0/200</span>
|
| 174 |
+
</div>
|
| 175 |
+
|
| 176 |
+
<div class="mic-warn" id="mwarn"></div>
|
| 177 |
+
<div class="mic-status" id="mstat">
|
| 178 |
+
<div class="mic-dot"></div>
|
| 179 |
+
<span id="mstxt">Listeningβ¦</span>
|
| 180 |
+
</div>
|
| 181 |
+
|
| 182 |
+
<div class="bg4">
|
| 183 |
+
<button class="btn bc" onclick="doConvert()">β¨ Convert</button>
|
| 184 |
+
<button class="btn bm" onclick="doSpeak()" id="bspk">π€ Speak</button>
|
| 185 |
+
<button class="btn bs" onclick="doStop()">βΉ Stop</button>
|
| 186 |
+
<button class="btn bg2" onclick="doClear()">β Clear</button>
|
| 187 |
+
</div>
|
| 188 |
+
|
| 189 |
+
<div class="st" id="st"><div class="dot"></div><span id="stxt">Ready to translate</span></div>
|
| 190 |
+
|
| 191 |
+
<div class="div"></div>
|
| 192 |
+
|
| 193 |
+
<div class="sh">
|
| 194 |
+
<div class="slbl">π Letter Sequence</div>
|
| 195 |
+
<span class="sbadge" id="scnt">0 letters</span>
|
| 196 |
+
</div>
|
| 197 |
+
<div class="ll" id="ll">
|
| 198 |
+
<div class="es">Type text or speak<br>to see the sequence</div>
|
| 199 |
+
</div>
|
| 200 |
+
|
| 201 |
+
<div class="div"></div>
|
| 202 |
+
|
| 203 |
+
<div class="slbl">π Session Stats</div>
|
| 204 |
+
<div class="sg">
|
| 205 |
+
<div class="st2"><div class="sl2">Conversions</div><div class="sv" id="s0">0</div></div>
|
| 206 |
+
<div class="st2"><div class="sl2">Letters Signed</div><div class="sv" id="s1">0</div></div>
|
| 207 |
+
<div class="st2"><div class="sl2">Speech Uses</div><div class="sv" id="s2">0</div></div>
|
| 208 |
+
<div class="st2"><div class="sl2">Words</div><div class="sv" id="s3">0</div></div>
|
| 209 |
+
</div>
|
| 210 |
+
|
| 211 |
+
<div class="div"></div>
|
| 212 |
+
|
| 213 |
+
<div class="slbl">βΉοΈ About</div>
|
| 214 |
+
<div class="abt">
|
| 215 |
+
<p>AI Powered ISL converts typed or spoken language into Indian Sign Language 3D animation, letter by letter.</p>
|
| 216 |
+
<ul class="al">
|
| 217 |
+
<li>Deaf & hard-of-hearing individuals</li>
|
| 218 |
+
<li>Inclusive education environments</li>
|
| 219 |
+
<li>Public service & assistive AI</li>
|
| 220 |
+
</ul>
|
| 221 |
+
<div class="acr">
|
| 222 |
+
<div class="an">Deepak Roshan</div>
|
| 223 |
+
<div class="ar">AI Engineer</div>
|
| 224 |
+
<div class="ae">deepakroshan380@gmail.com</div>
|
| 225 |
+
</div>
|
| 226 |
+
</div>
|
| 227 |
+
</div>
|
| 228 |
+
|
| 229 |
+
<!-- FIX: Nav links built dynamically β works on localhost AND cloud domain -->
|
| 230 |
+
<div class="nav">
|
| 231 |
+
<a href="#" class="na" id="nav-gesture">π Gesture Recognition</a>
|
| 232 |
+
<a href="#" class="na on" id="nav-avatar">π€ Speech-to-Sign</a>
|
| 233 |
+
</div>
|
| 234 |
+
</div>
|
| 235 |
+
</div>
|
| 236 |
+
|
| 237 |
+
<script>
|
| 238 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 239 |
+
// ISL POSE TABLE AβZ (unchanged)
|
| 240 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 241 |
+
const ISL={
|
| 242 |
+
a:{thumb:.1, index:1.5,middle:1.5,ring:1.5,pinky:1.5,rz:0},
|
| 243 |
+
b:{thumb:1.2,index:0, middle:0, ring:0, pinky:0, rz:0},
|
| 244 |
+
c:{thumb:.3, index:.5, middle:.5, ring:.5, pinky:.5, rz:0},
|
| 245 |
+
d:{thumb:1.2,index:0, middle:1.5,ring:1.5,pinky:1.5,rz:0},
|
| 246 |
+
e:{thumb:.8, index:1.2,middle:1.2,ring:1.2,pinky:1.2,rz:0},
|
| 247 |
+
f:{thumb:.8, index:.8, middle:0, ring:0, pinky:0, rz:0},
|
| 248 |
+
g:{thumb:0, index:0, middle:1.5,ring:1.5,pinky:1.5,rz:1.57},
|
| 249 |
+
h:{thumb:1.5,index:0, middle:0, ring:1.5,pinky:1.5,rz:1.57},
|
| 250 |
+
i:{thumb:1.5,index:1.5,middle:1.5,ring:1.5,pinky:0, rz:0},
|
| 251 |
+
j:{thumb:1.5,index:1.5,middle:1.5,ring:1.5,pinky:0, rz:0},
|
| 252 |
+
k:{thumb:.5, index:0, middle:0, ring:1.5,pinky:1.5,rz:.78},
|
| 253 |
+
l:{thumb:0, index:0, middle:1.5,ring:1.5,pinky:1.5,rz:0},
|
| 254 |
+
m:{thumb:.8, index:1.5,middle:1.5,ring:1.5,pinky:1.5,rz:0},
|
| 255 |
+
n:{thumb:.8, index:1.5,middle:1.5,ring:1.5,pinky:1.5,rz:0},
|
| 256 |
+
o:{thumb:.5, index:.8, middle:.8, ring:.8, pinky:.8, rz:0},
|
| 257 |
+
p:{thumb:.5, index:0, middle:0, ring:1.5,pinky:1.5,rz:-.78},
|
| 258 |
+
q:{thumb:0, index:0, middle:1.5,ring:1.5,pinky:1.5,rz:-1.57},
|
| 259 |
+
r:{thumb:1.2,index:0, middle:0, ring:1.5,pinky:1.5,rz:0},
|
| 260 |
+
s:{thumb:.5, index:1.5,middle:1.5,ring:1.5,pinky:1.5,rz:0},
|
| 261 |
+
t:{thumb:.8, index:1.5,middle:1.5,ring:1.5,pinky:1.5,rz:0},
|
| 262 |
+
u:{thumb:1.2,index:0, middle:0, ring:1.5,pinky:1.5,rz:0},
|
| 263 |
+
v:{thumb:1.2,index:0, middle:0, ring:1.5,pinky:1.5,rz:0},
|
| 264 |
+
w:{thumb:1.2,index:0, middle:0, ring:0, pinky:1.5,rz:0},
|
| 265 |
+
x:{thumb:1.5,index:.8, middle:1.5,ring:1.5,pinky:1.5,rz:0},
|
| 266 |
+
y:{thumb:0, index:1.5,middle:1.5,ring:1.5,pinky:0, rz:0},
|
| 267 |
+
z:{thumb:1.5,index:0, middle:1.5,ring:1.5,pinky:1.5,rz:0},
|
| 268 |
+
};
|
| 269 |
+
const DESC={
|
| 270 |
+
a:'Fist with thumb at side',b:'Four fingers straight up',c:'Curved C shape',
|
| 271 |
+
d:'Index pointing up',e:'All fingers curled',f:'OK sign variant',
|
| 272 |
+
g:'Index + thumb sideways',h:'Two fingers horizontal',i:'Pinky raised',
|
| 273 |
+
j:'Pinky traces J',k:'V with thumb between',l:'L shape',
|
| 274 |
+
m:'Three fingers over thumb',n:'Two fingers over thumb',o:'O ring shape',
|
| 275 |
+
p:'K pointing down',q:'G pointing down',r:'Fingers crossed',
|
| 276 |
+
s:'Fist thumb across',t:'Thumb between fingers',u:'Two fingers together',
|
| 277 |
+
v:'Peace sign',w:'Three fingers wide',x:'Index hooked',
|
| 278 |
+
y:'Hang loose',z:'Index traces Z'
|
| 279 |
+
};
|
| 280 |
+
const REST={thumb:.05,index:.05,middle:.05,ring:.05,pinky:.05,rz:0};
|
| 281 |
+
const SKIN=0xfdb4c4,JOINT=0xf5a3b5;
|
| 282 |
+
|
| 283 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 284 |
+
// THREE.JS (unchanged)
|
| 285 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 286 |
+
let scene,cam,ren,hg,fing={};
|
| 287 |
+
let autoSpin=false;
|
| 288 |
+
let cC={thumb:.05,index:.05,middle:.05,ring:.05,pinky:.05};
|
| 289 |
+
let tC={...cC},cRZ=0,tRZ=0;
|
| 290 |
+
let aRun=false,aCan=false;
|
| 291 |
+
|
| 292 |
+
function initThree(){
|
| 293 |
+
const cv=document.getElementById('hc');
|
| 294 |
+
const cp=document.getElementById('cvp');
|
| 295 |
+
scene=new THREE.Scene();
|
| 296 |
+
scene.background=new THREE.Color(0x0d1117);
|
| 297 |
+
scene.fog=new THREE.FogExp2(0x0d1117,.022);
|
| 298 |
+
const W=cp.clientWidth,H=cp.clientHeight;
|
| 299 |
+
cam=new THREE.PerspectiveCamera(50,W/H,.1,1000);
|
| 300 |
+
cam.position.set(0,0,8);cam.lookAt(0,0,0);
|
| 301 |
+
ren=new THREE.WebGLRenderer({canvas:cv,antialias:true,powerPreference:'low-power'});
|
| 302 |
+
ren.setSize(W,H,false);ren.setPixelRatio(Math.min(devicePixelRatio,1.5));
|
| 303 |
+
ren.shadowMap.enabled=true;ren.info.autoReset=false;
|
| 304 |
+
scene.add(new THREE.AmbientLight(0xffffff,.65));
|
| 305 |
+
const kl=new THREE.DirectionalLight(0xfff4e0,.85);kl.position.set(5,10,7);kl.castShadow=true;scene.add(kl);
|
| 306 |
+
const fl=new THREE.DirectionalLight(0xd0e8ff,.3);fl.position.set(-6,2,4);scene.add(fl);
|
| 307 |
+
const rl=new THREE.DirectionalLight(0xffddbb,.2);rl.position.set(1,-4,-5);scene.add(rl);
|
| 308 |
+
buildHand();loop();
|
| 309 |
+
}
|
| 310 |
+
function mkFinger(cfg){
|
| 311 |
+
const fg=new THREE.Group();fg.userData={segs:[]};
|
| 312 |
+
const sl=cfg.length/3;let cy=0;
|
| 313 |
+
for(let i=0;i<3;i++){
|
| 314 |
+
const sg=new THREE.Group();const r=cfg.thickness*(1-i*.15);
|
| 315 |
+
const seg=new THREE.Mesh(new THREE.CylinderGeometry(r,r*.9,sl,12),new THREE.MeshStandardMaterial({color:SKIN,roughness:.7}));
|
| 316 |
+
seg.position.y=sl/2;seg.castShadow=true;sg.add(seg);
|
| 317 |
+
if(i<2){const jt=new THREE.Mesh(new THREE.SphereGeometry(r*.7,10,10),new THREE.MeshStandardMaterial({color:JOINT}));jt.position.y=sl;sg.add(jt);}
|
| 318 |
+
sg.position.y=cy;cy+=sl;
|
| 319 |
+
i===0?fg.add(sg):fg.userData.segs[i-1].add(sg);
|
| 320 |
+
fg.userData.segs.push(sg);
|
| 321 |
+
}
|
| 322 |
+
return fg;
|
| 323 |
+
}
|
| 324 |
+
function buildHand(){
|
| 325 |
+
hg=new THREE.Group();
|
| 326 |
+
hg.add(new THREE.Mesh(new THREE.BoxGeometry(1.8,2.5,.6),new THREE.MeshStandardMaterial({color:SKIN,roughness:.7})));
|
| 327 |
+
[{name:'thumb',pos:[-1.1,.5,.15],rot:[0,0,-.5],length:1.2,thickness:.35},
|
| 328 |
+
{name:'index',pos:[-.7,1.4,0],rot:[0,0,.1],length:1.8,thickness:.28},
|
| 329 |
+
{name:'middle',pos:[0,1.5,0],rot:[0,0,0],length:2.0,thickness:.28},
|
| 330 |
+
{name:'ring',pos:[.7,1.4,0],rot:[0,0,-.1],length:1.7,thickness:.26},
|
| 331 |
+
{name:'pinky',pos:[1.1,1.0,0],rot:[0,0,-.3],length:1.3,thickness:.24}].forEach(d=>{
|
| 332 |
+
const f=mkFinger(d);f.position.set(...d.pos);f.rotation.set(...d.rot);hg.add(f);fing[d.name]=f;
|
| 333 |
+
});
|
| 334 |
+
scene.add(hg);
|
| 335 |
+
}
|
| 336 |
+
function bend(name,amt){
|
| 337 |
+
const f=fing[name];if(!f)return;
|
| 338 |
+
f.userData.segs.forEach((s,i)=>{if(s)s.rotation.x=[amt*.4,amt*.6,amt*.8][i];});
|
| 339 |
+
}
|
| 340 |
+
function setPose(p){
|
| 341 |
+
['thumb','index','middle','ring','pinky'].forEach(k=>{tC[k]=p[k]??0.05;});
|
| 342 |
+
tRZ=p.rz||0;
|
| 343 |
+
}
|
| 344 |
+
function loop(){
|
| 345 |
+
requestAnimationFrame(loop);
|
| 346 |
+
const t=Date.now()*.001;
|
| 347 |
+
if(!aRun){hg.position.y=Math.sin(t*.45)*.18;hg.rotation.y=Math.sin(t*.18)*.05;}
|
| 348 |
+
else{hg.rotation.y=0;}
|
| 349 |
+
['thumb','index','middle','ring','pinky'].forEach(k=>{cC[k]+=(tC[k]-cC[k])*.13;bend(k,cC[k]);});
|
| 350 |
+
cRZ+=(tRZ-cRZ)*.13;hg.rotation.z=cRZ;
|
| 351 |
+
if(autoSpin){cam.position.x=Math.sin(t*.45)*8;cam.position.z=Math.cos(t*.45)*8;cam.lookAt(0,0,0);}
|
| 352 |
+
ren.render(scene,cam);
|
| 353 |
+
}
|
| 354 |
+
function setView(v,btn){
|
| 355 |
+
document.querySelectorAll('.vb').forEach(b=>b.classList.remove('on'));btn.classList.add('on');
|
| 356 |
+
autoSpin=(v==='spin');if(autoSpin)return;
|
| 357 |
+
const tp={front:[0,0,8],side:[8,0,0],top:[0,8,3]}[v]||[0,0,8];
|
| 358 |
+
animCam(...tp);
|
| 359 |
+
}
|
| 360 |
+
function animCam(tx,ty,tz){
|
| 361 |
+
const sx=cam.position.x,sy=cam.position.y,sz=cam.position.z;
|
| 362 |
+
const s0=performance.now();
|
| 363 |
+
function f(n){const e=ease(Math.min((n-s0)/900,1));cam.position.set(lerp(sx,tx,e),lerp(sy,ty,e),lerp(sz,tz,e));cam.lookAt(0,0,0);if(e<1)requestAnimationFrame(f);}
|
| 364 |
+
requestAnimationFrame(f);
|
| 365 |
+
}
|
| 366 |
+
window.addEventListener('resize',()=>{
|
| 367 |
+
if(!ren)return;
|
| 368 |
+
const cp=document.getElementById('cvp');
|
| 369 |
+
cam.aspect=cp.clientWidth/cp.clientHeight;cam.updateProjectionMatrix();
|
| 370 |
+
ren.setSize(cp.clientWidth,cp.clientHeight,false);
|
| 371 |
+
});
|
| 372 |
+
document.addEventListener('visibilitychange',()=>{if(document.hidden)aCan=true;});
|
| 373 |
+
|
| 374 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 375 |
+
// SPEECH RECOGNITION β ROBUST LOCAL + CLOUD HANDLER (unchanged)
|
| 376 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 377 |
+
let recog=null,isRec=false,micAllowed=false;
|
| 378 |
+
|
| 379 |
+
function isMicSafe(){
|
| 380 |
+
const isLocalhost=location.hostname==='localhost'||location.hostname==='127.0.0.1';
|
| 381 |
+
const isHttps=location.protocol==='https:';
|
| 382 |
+
return (isLocalhost||isHttps);
|
| 383 |
+
}
|
| 384 |
+
|
| 385 |
+
function showMicWarn(html){
|
| 386 |
+
const w=document.getElementById('mwarn');
|
| 387 |
+
w.innerHTML=html;w.classList.add('show');
|
| 388 |
+
}
|
| 389 |
+
|
| 390 |
+
function initSpeech(){
|
| 391 |
+
const SR=window.SpeechRecognition||window.webkitSpeechRecognition;
|
| 392 |
+
const btn=document.getElementById('bspk');
|
| 393 |
+
|
| 394 |
+
if(!SR){
|
| 395 |
+
btn.disabled=true;
|
| 396 |
+
btn.title='Use Chrome or Edge for voice input';
|
| 397 |
+
showMicWarn(
|
| 398 |
+
'π <strong>Voice input requires Chrome or Edge.</strong><br>'+
|
| 399 |
+
'Firefox and Safari do not support the Web Speech API.<br>'+
|
| 400 |
+
'You can still <strong>type text</strong> above and press Convert.'
|
| 401 |
+
);
|
| 402 |
+
return;
|
| 403 |
+
}
|
| 404 |
+
|
| 405 |
+
if(!isMicSafe()){
|
| 406 |
+
btn.disabled=true;
|
| 407 |
+
btn.title='Mic blocked on HTTP β HTTPS required on cloud';
|
| 408 |
+
showMicWarn(
|
| 409 |
+
'π <strong>Microphone is blocked on plain HTTP.</strong><br>'+
|
| 410 |
+
'On your Oracle Cloud server, enable HTTPS to use voice input.<br>'+
|
| 411 |
+
'Until then, <strong>type your text</strong> above β Convert always works.'
|
| 412 |
+
);
|
| 413 |
+
return;
|
| 414 |
+
}
|
| 415 |
+
|
| 416 |
+
recog=new SR();
|
| 417 |
+
recog.lang='en-IN';
|
| 418 |
+
recog.continuous=false;
|
| 419 |
+
recog.interimResults=false;
|
| 420 |
+
recog.maxAlternatives=3;
|
| 421 |
+
|
| 422 |
+
recog.onstart=()=>{
|
| 423 |
+
setBadge('π€ Listeningβ¦',true);
|
| 424 |
+
setSt('rec','Listening⦠speak clearly in English');
|
| 425 |
+
setMicStat(true,'Listening β speak now');
|
| 426 |
+
document.getElementById('bspk').classList.add('listening');
|
| 427 |
+
};
|
| 428 |
+
|
| 429 |
+
recog.onspeechstart=()=>{
|
| 430 |
+
setMicStat(true,'Speech detectedβ¦');
|
| 431 |
+
setSt('rec','Processing your speechβ¦');
|
| 432 |
+
};
|
| 433 |
+
|
| 434 |
+
recog.onresult=ev=>{
|
| 435 |
+
let best='',bestConf=0;
|
| 436 |
+
const results=ev.results[0];
|
| 437 |
+
for(let i=0;i<results.length;i++){
|
| 438 |
+
if(results[i].confidence>=bestConf){bestConf=results[i].confidence;best=results[i].transcript;}
|
| 439 |
+
}
|
| 440 |
+
const t=best.trim();
|
| 441 |
+
if(!t){resetSpk();setSt('err','No speech detected β try again');return;}
|
| 442 |
+
document.getElementById('txt').value=t;
|
| 443 |
+
document.getElementById('cc').textContent=t.length+'/200';
|
| 444 |
+
setSt('act','Got: "'+t+'" ('+Math.round(bestConf*100)+'% conf)');
|
| 445 |
+
setMicStat(false,'');
|
| 446 |
+
isRec=false;resetSpk();
|
| 447 |
+
ST.sp++;ST.wd+=t.trim().split(/\s+/).length;updSt();
|
| 448 |
+
const seq=buildSeq(t);renderLL(seq);
|
| 449 |
+
ST.cv++;updSt();
|
| 450 |
+
setTimeout(()=>play(seq),300);
|
| 451 |
+
};
|
| 452 |
+
|
| 453 |
+
recog.onnomatch=()=>{
|
| 454 |
+
isRec=false;resetSpk();
|
| 455 |
+
setMicStat(false,'');
|
| 456 |
+
setSt('err','No match β speak more clearly or type instead');
|
| 457 |
+
};
|
| 458 |
+
|
| 459 |
+
recog.onerror=e=>{
|
| 460 |
+
isRec=false;resetSpk();setMicStat(false,'');
|
| 461 |
+
const msgs={
|
| 462 |
+
'not-allowed' :'π Mic permission denied β click the π icon in the address bar to allow',
|
| 463 |
+
'no-speech' :'No speech detected β try speaking louder',
|
| 464 |
+
'network' :'Network error β check your internet connection',
|
| 465 |
+
'aborted' :'Recording cancelled',
|
| 466 |
+
'audio-capture':'No microphone found β check your device settings',
|
| 467 |
+
'service-not-allowed':'Speech service blocked β use HTTPS on cloud',
|
| 468 |
+
};
|
| 469 |
+
setSt('err',msgs[e.error]||'Speech error: '+e.error);
|
| 470 |
+
if(e.error==='not-allowed'){
|
| 471 |
+
showMicWarn(
|
| 472 |
+
'π <strong>Microphone permission was denied.</strong><br>'+
|
| 473 |
+
'Click the π padlock icon in your browser address bar,<br>'+
|
| 474 |
+
'set Microphone to <strong>Allow</strong>, then refresh the page.'
|
| 475 |
+
);
|
| 476 |
+
document.getElementById('bspk').disabled=true;
|
| 477 |
+
}
|
| 478 |
+
};
|
| 479 |
+
|
| 480 |
+
recog.onend=()=>{
|
| 481 |
+
if(isRec){isRec=false;resetSpk();}
|
| 482 |
+
setMicStat(false,'');
|
| 483 |
+
};
|
| 484 |
+
}
|
| 485 |
+
|
| 486 |
+
function setMicStat(show,msg){
|
| 487 |
+
const el=document.getElementById('mstat');
|
| 488 |
+
const tx=document.getElementById('mstxt');
|
| 489 |
+
if(show){el.classList.add('show','listening');tx.textContent=msg;}
|
| 490 |
+
else{el.classList.remove('show','listening');}
|
| 491 |
+
}
|
| 492 |
+
|
| 493 |
+
function resetSpk(){
|
| 494 |
+
const b=document.getElementById('bspk');
|
| 495 |
+
b.disabled=false;b.classList.remove('listening');
|
| 496 |
+
setBadge('β Ready',false);
|
| 497 |
+
}
|
| 498 |
+
|
| 499 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 500 |
+
// SEQUENCE + PLAYBACK (unchanged)
|
| 501 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 502 |
+
function buildSeq(text){
|
| 503 |
+
const out=[];
|
| 504 |
+
for(const ch of text.toLowerCase().replace(/[^a-z\s]/g,'')){
|
| 505 |
+
if(ch===' ')out.push({isSpace:true,name:'SPACE',desc:'Pause between words'});
|
| 506 |
+
else if(ISL[ch])out.push({letter:ch,name:ch.toUpperCase(),pose:ISL[ch],desc:DESC[ch]||''});
|
| 507 |
+
}
|
| 508 |
+
return out;
|
| 509 |
+
}
|
| 510 |
+
|
| 511 |
+
async function play(seq){
|
| 512 |
+
if(aRun)return;
|
| 513 |
+
aRun=true;aCan=false;
|
| 514 |
+
setBadge('β Signing',true);setSt('act','Fingerspellingβ¦');
|
| 515 |
+
for(let i=0;i<seq.length&&!aCan;i++){
|
| 516 |
+
const item=seq[i];
|
| 517 |
+
document.querySelectorAll('.li').forEach(e=>e.classList.remove('on'));
|
| 518 |
+
const el=document.getElementById('li'+i);
|
| 519 |
+
if(el){el.classList.add('on');el.scrollIntoView({behavior:'smooth',block:'nearest'});}
|
| 520 |
+
document.getElementById('gl').textContent=item.isSpace?'Β·':item.name;
|
| 521 |
+
document.getElementById('gd').textContent=item.desc;
|
| 522 |
+
if(item.isSpace){setPose(REST);await slp(500);}
|
| 523 |
+
else{setPose(item.pose);await slp(880);}
|
| 524 |
+
if(aCan)break;
|
| 525 |
+
document.getElementById('gb').style.width=((i+1)/seq.length*100)+'%';
|
| 526 |
+
ST.lt++;updSt();
|
| 527 |
+
}
|
| 528 |
+
aRun=false;
|
| 529 |
+
if(!aCan){
|
| 530 |
+
setBadge('β Done',false);setSt('act','Complete!');setPose(REST);
|
| 531 |
+
setTimeout(()=>{
|
| 532 |
+
document.getElementById('gb').style.width='0%';
|
| 533 |
+
document.getElementById('gl').textContent='π€';
|
| 534 |
+
document.getElementById('gd').textContent='Type or speak to begin';
|
| 535 |
+
setBadge('β Ready',false);setSt('','Ready');
|
| 536 |
+
},2000);
|
| 537 |
+
}
|
| 538 |
+
}
|
| 539 |
+
|
| 540 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 541 |
+
// BUTTON ACTIONS (unchanged)
|
| 542 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 543 |
+
const ST={cv:0,lt:0,sp:0,wd:0};
|
| 544 |
+
|
| 545 |
+
function doConvert(){
|
| 546 |
+
if(aRun)doStop();
|
| 547 |
+
const t=document.getElementById('txt').value.trim();
|
| 548 |
+
if(!t){setSt('err','Please enter some text first');return;}
|
| 549 |
+
const seq=buildSeq(t);renderLL(seq);
|
| 550 |
+
ST.cv++;ST.wd+=t.split(/\s+/).length;updSt();
|
| 551 |
+
play(seq);
|
| 552 |
+
}
|
| 553 |
+
|
| 554 |
+
function doSpeak(){
|
| 555 |
+
if(!recog){setSt('err','Voice input not available β see instructions above');return;}
|
| 556 |
+
if(isRec)return;
|
| 557 |
+
isRec=true;
|
| 558 |
+
document.getElementById('bspk').disabled=true;
|
| 559 |
+
try{
|
| 560 |
+
recog.start();
|
| 561 |
+
}catch(e){
|
| 562 |
+
isRec=false;resetSpk();
|
| 563 |
+
setSt('err','Please wait a moment and try again');
|
| 564 |
+
}
|
| 565 |
+
}
|
| 566 |
+
|
| 567 |
+
function doStop(){
|
| 568 |
+
aCan=true;aRun=false;
|
| 569 |
+
if(isRec&&recog){try{recog.stop();}catch(e){}}
|
| 570 |
+
isRec=false;resetSpk();setMicStat(false,'');
|
| 571 |
+
document.getElementById('gb').style.width='0%';
|
| 572 |
+
setPose(REST);setBadge('βΉ Stopped',false);setSt('err','Stopped');
|
| 573 |
+
setTimeout(()=>{setSt('','Ready');setBadge('β Ready',false);},1400);
|
| 574 |
+
}
|
| 575 |
+
|
| 576 |
+
function doClear(){
|
| 577 |
+
doStop();
|
| 578 |
+
document.getElementById('txt').value='';
|
| 579 |
+
document.getElementById('cc').textContent='0/200';
|
| 580 |
+
renderLL([]);setSt('','Cleared');
|
| 581 |
+
setTimeout(()=>setSt('','Ready'),1200);
|
| 582 |
+
}
|
| 583 |
+
|
| 584 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 585 |
+
// UI HELPERS (unchanged)
|
| 586 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 587 |
+
function renderLL(seq){
|
| 588 |
+
const el=document.getElementById('ll');
|
| 589 |
+
if(!seq.length){el.innerHTML='<div class="es">Type text or speak<br>to see the sequence</div>';document.getElementById('scnt').textContent='0 letters';return;}
|
| 590 |
+
let n=0;
|
| 591 |
+
el.innerHTML=seq.map((x,i)=>{
|
| 592 |
+
if(x.isSpace)return`<div class="li sp" id="li${i}"><span class="lc">β΅</span><span class="lh">Word space</span></div>`;
|
| 593 |
+
n++;return`<div class="li" id="li${i}"><span class="lc">${x.name}</span><span class="lh">${x.desc}</span></div>`;
|
| 594 |
+
}).join('');
|
| 595 |
+
document.getElementById('scnt').textContent=n+' letter'+(n!==1?'s':'');
|
| 596 |
+
}
|
| 597 |
+
function setSt(cls,msg){
|
| 598 |
+
document.getElementById('stxt').textContent=msg;
|
| 599 |
+
document.getElementById('st').className='st'+(cls?' '+cls:'');
|
| 600 |
+
}
|
| 601 |
+
function setBadge(msg,blink){
|
| 602 |
+
const b=document.getElementById('badge');b.textContent=msg;
|
| 603 |
+
b.style.animation=blink?'pd .9s infinite':'none';
|
| 604 |
+
}
|
| 605 |
+
function updSt(){
|
| 606 |
+
[ST.cv,ST.lt,ST.sp,ST.wd].forEach((v,i)=>document.getElementById('s'+i).textContent=v);
|
| 607 |
+
}
|
| 608 |
+
function lerp(a,b,t){return a+(b-a)*t;}
|
| 609 |
+
function ease(t){return t<.5?4*t*t*t:1-Math.pow(-2*t+2,3)/2;}
|
| 610 |
+
function slp(ms){return new Promise(r=>setTimeout(r,ms));}
|
| 611 |
+
|
| 612 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 613 |
+
// FIX: Nav links β dynamic URL so redirect works on any domain
|
| 614 |
+
// Local dev β uses explicit port numbers (:5000 / :5001)
|
| 615 |
+
// Cloud/Nginx β uses path-based routing (/ and /avatar/)
|
| 616 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 617 |
+
(function(){
|
| 618 |
+
const proto = location.protocol;
|
| 619 |
+
const host = location.hostname;
|
| 620 |
+
const port = location.port;
|
| 621 |
+
if (port && port !== '80' && port !== '443') {
|
| 622 |
+
// Local development with explicit ports
|
| 623 |
+
document.getElementById('nav-gesture').href = proto + '//' + host + ':5000/';
|
| 624 |
+
document.getElementById('nav-avatar').href = proto + '//' + host + ':5001/';
|
| 625 |
+
} else {
|
| 626 |
+
// Cloud / Nginx β path-based routing
|
| 627 |
+
document.getElementById('nav-gesture').href = proto + '//' + host + '/';
|
| 628 |
+
document.getElementById('nav-avatar').href = proto + '//' + host + '/avatar/';
|
| 629 |
+
}
|
| 630 |
+
})();
|
| 631 |
+
|
| 632 |
+
// Keyboard shortcuts
|
| 633 |
+
document.addEventListener('keydown',e=>{
|
| 634 |
+
if((e.ctrlKey||e.metaKey)&&e.key==='Enter'){e.preventDefault();doConvert();}
|
| 635 |
+
if(e.key==='Escape')doStop();
|
| 636 |
+
if((e.ctrlKey||e.metaKey)&&e.key===' '){e.preventDefault();doSpeak();}
|
| 637 |
+
});
|
| 638 |
+
document.getElementById('txt').addEventListener('input',function(){
|
| 639 |
+
document.getElementById('cc').textContent=this.value.length+'/200';
|
| 640 |
+
});
|
| 641 |
+
|
| 642 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 643 |
+
// BOOT
|
| 644 |
+
// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 645 |
+
window.addEventListener('DOMContentLoaded',()=>{
|
| 646 |
+
initThree();
|
| 647 |
+
initSpeech();
|
| 648 |
+
updSt();
|
| 649 |
+
setTimeout(()=>setPose(REST),400);
|
| 650 |
+
});
|
| 651 |
+
</script>
|
| 652 |
+
</body>
|
| 653 |
+
</html>
|
templates/dashboard.html
ADDED
|
@@ -0,0 +1,1163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
|
| 4 |
+
<head>
|
| 5 |
+
<meta charset="UTF-8" />
|
| 6 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
|
| 7 |
+
<title>ISL Gesture Recognition β AI Sign Language</title>
|
| 8 |
+
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.7.2/socket.io.min.js"></script>
|
| 9 |
+
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
|
| 10 |
+
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
| 11 |
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
| 12 |
+
<link
|
| 13 |
+
href="https://fonts.googleapis.com/css2?family=Playfair+Display:wght@400;600;700&family=DM+Sans:wght@300;400;500;600&family=DM+Mono:wght@400;500&display=swap"
|
| 14 |
+
rel="stylesheet" />
|
| 15 |
+
<style>
|
| 16 |
+
:root {
|
| 17 |
+
--bg: #0d1117;
|
| 18 |
+
--surf: #131922;
|
| 19 |
+
--card: #1a2233;
|
| 20 |
+
--bdr: rgba(212, 168, 83, .13);
|
| 21 |
+
--bdrm: rgba(212, 168, 83, .30);
|
| 22 |
+
--bdrh: rgba(212, 168, 83, .58);
|
| 23 |
+
--gold: #d4a853;
|
| 24 |
+
--goldb: #f0c76a;
|
| 25 |
+
--goldd: #a07c3a;
|
| 26 |
+
--sage: #7eb89a;
|
| 27 |
+
--saged: #4e7d66;
|
| 28 |
+
--txt: #f0ede6;
|
| 29 |
+
--txtm: #a8a096;
|
| 30 |
+
--txtl: #5a5650;
|
| 31 |
+
--red: #c0614a;
|
| 32 |
+
--r8: 8px;
|
| 33 |
+
--r14: 14px;
|
| 34 |
+
--fD: 'Playfair Display', Georgia, serif;
|
| 35 |
+
--fB: 'DM Sans', system-ui, sans-serif;
|
| 36 |
+
--fM: 'DM Mono', monospace;
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
*,
|
| 40 |
+
*::before,
|
| 41 |
+
*::after {
|
| 42 |
+
margin: 0;
|
| 43 |
+
padding: 0;
|
| 44 |
+
box-sizing: border-box;
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
html,
|
| 48 |
+
body {
|
| 49 |
+
height: 100%;
|
| 50 |
+
overflow: hidden;
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
body {
|
| 54 |
+
font-family: var(--fB);
|
| 55 |
+
background: var(--bg);
|
| 56 |
+
color: var(--txt);
|
| 57 |
+
-webkit-font-smoothing: antialiased;
|
| 58 |
+
display: flex;
|
| 59 |
+
flex-direction: column;
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
#app {
|
| 63 |
+
display: flex;
|
| 64 |
+
flex: 1;
|
| 65 |
+
overflow: hidden;
|
| 66 |
+
height: 100%;
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
#cam-pane {
|
| 70 |
+
flex: 1;
|
| 71 |
+
display: flex;
|
| 72 |
+
flex-direction: column;
|
| 73 |
+
min-width: 0;
|
| 74 |
+
background: var(--bg);
|
| 75 |
+
position: relative;
|
| 76 |
+
overflow: hidden;
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
.topbar {
|
| 80 |
+
display: flex;
|
| 81 |
+
align-items: center;
|
| 82 |
+
justify-content: space-between;
|
| 83 |
+
padding: 10px 16px 0;
|
| 84 |
+
flex-shrink: 0;
|
| 85 |
+
position: relative;
|
| 86 |
+
z-index: 5;
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
.brand {
|
| 90 |
+
display: flex;
|
| 91 |
+
align-items: baseline;
|
| 92 |
+
gap: 8px;
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
.brand-t {
|
| 96 |
+
font-family: var(--fD);
|
| 97 |
+
font-size: 1.1rem;
|
| 98 |
+
font-weight: 700;
|
| 99 |
+
color: var(--gold);
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
.brand-s {
|
| 103 |
+
font-size: .65rem;
|
| 104 |
+
color: var(--txtl);
|
| 105 |
+
text-transform: uppercase;
|
| 106 |
+
letter-spacing: .07em;
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
.live-badge {
|
| 110 |
+
font-size: .65rem;
|
| 111 |
+
font-family: var(--fM);
|
| 112 |
+
color: var(--sage);
|
| 113 |
+
background: rgba(126, 184, 154, .10);
|
| 114 |
+
border: 1px solid rgba(126, 184, 154, .25);
|
| 115 |
+
padding: 3px 9px;
|
| 116 |
+
border-radius: 20px;
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
@keyframes pd {
|
| 120 |
+
|
| 121 |
+
0%,
|
| 122 |
+
100% {
|
| 123 |
+
opacity: 1;
|
| 124 |
+
transform: scale(1)
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
50% {
|
| 128 |
+
opacity: .6;
|
| 129 |
+
transform: scale(.8)
|
| 130 |
+
}
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
#camera-feed {
|
| 134 |
+
flex: 1;
|
| 135 |
+
width: 100%;
|
| 136 |
+
display: block;
|
| 137 |
+
object-fit: cover;
|
| 138 |
+
background: #000;
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
.cam-overlay {
|
| 142 |
+
position: absolute;
|
| 143 |
+
bottom: 0;
|
| 144 |
+
left: 0;
|
| 145 |
+
right: 0;
|
| 146 |
+
padding: 12px 18px 14px;
|
| 147 |
+
background: linear-gradient(to top, rgba(13, 17, 23, 1) 55%, transparent);
|
| 148 |
+
display: flex;
|
| 149 |
+
align-items: flex-end;
|
| 150 |
+
justify-content: space-between;
|
| 151 |
+
gap: 10px;
|
| 152 |
+
pointer-events: none;
|
| 153 |
+
z-index: 5;
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
+
.co-letter {
|
| 157 |
+
font-family: var(--fD);
|
| 158 |
+
font-size: clamp(2rem, 5vw, 3.2rem);
|
| 159 |
+
font-weight: 700;
|
| 160 |
+
color: var(--goldb);
|
| 161 |
+
line-height: 1;
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
.co-word {
|
| 165 |
+
font-size: .85rem;
|
| 166 |
+
font-weight: 600;
|
| 167 |
+
color: var(--gold);
|
| 168 |
+
font-family: var(--fM);
|
| 169 |
+
letter-spacing: .05em;
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
.co-desc {
|
| 173 |
+
font-size: .72rem;
|
| 174 |
+
color: var(--txtm);
|
| 175 |
+
text-align: right;
|
| 176 |
+
max-width: 200px;
|
| 177 |
+
line-height: 1.45;
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
.co-progress {
|
| 181 |
+
position: absolute;
|
| 182 |
+
bottom: 0;
|
| 183 |
+
left: 0;
|
| 184 |
+
right: 0;
|
| 185 |
+
height: 3px;
|
| 186 |
+
background: rgba(212, 168, 83, .08);
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
.co-bar {
|
| 190 |
+
height: 100%;
|
| 191 |
+
width: 0%;
|
| 192 |
+
background: linear-gradient(90deg, var(--goldd), var(--goldb));
|
| 193 |
+
transition: width .3s ease;
|
| 194 |
+
position: relative;
|
| 195 |
+
overflow: hidden;
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
.co-bar::after {
|
| 199 |
+
content: '';
|
| 200 |
+
position: absolute;
|
| 201 |
+
inset: 0;
|
| 202 |
+
background: linear-gradient(90deg, transparent, rgba(255, 255, 255, .3), transparent);
|
| 203 |
+
animation: shimmer 1.8s infinite;
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
@keyframes shimmer {
|
| 207 |
+
from {
|
| 208 |
+
transform: translateX(-100%)
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
to {
|
| 212 |
+
transform: translateX(200%)
|
| 213 |
+
}
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
.cam-placeholder {
|
| 217 |
+
position: absolute;
|
| 218 |
+
inset: 0;
|
| 219 |
+
display: flex;
|
| 220 |
+
flex-direction: column;
|
| 221 |
+
align-items: center;
|
| 222 |
+
justify-content: center;
|
| 223 |
+
gap: 10px;
|
| 224 |
+
color: var(--txtl);
|
| 225 |
+
font-size: .8rem;
|
| 226 |
+
z-index: 2;
|
| 227 |
+
pointer-events: none;
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
.cam-placeholder .ph-icon {
|
| 231 |
+
font-size: 2.8rem;
|
| 232 |
+
opacity: .25;
|
| 233 |
+
}
|
| 234 |
+
|
| 235 |
+
#ctrl-pane {
|
| 236 |
+
width: 340px;
|
| 237 |
+
flex-shrink: 0;
|
| 238 |
+
display: flex;
|
| 239 |
+
flex-direction: column;
|
| 240 |
+
background: var(--surf);
|
| 241 |
+
border-left: 1px solid var(--bdr);
|
| 242 |
+
overflow-y: auto;
|
| 243 |
+
overflow-x: hidden;
|
| 244 |
+
}
|
| 245 |
+
|
| 246 |
+
#ctrl-pane::-webkit-scrollbar {
|
| 247 |
+
width: 4px;
|
| 248 |
+
}
|
| 249 |
+
|
| 250 |
+
#ctrl-pane::-webkit-scrollbar-thumb {
|
| 251 |
+
background: var(--bdrm);
|
| 252 |
+
border-radius: 4px;
|
| 253 |
+
}
|
| 254 |
+
|
| 255 |
+
@media (max-width: 720px) {
|
| 256 |
+
|
| 257 |
+
html,
|
| 258 |
+
body {
|
| 259 |
+
overflow: hidden;
|
| 260 |
+
}
|
| 261 |
+
|
| 262 |
+
#app {
|
| 263 |
+
flex-direction: column;
|
| 264 |
+
height: 100%;
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
+
#cam-pane {
|
| 268 |
+
flex: 0 0 46vh;
|
| 269 |
+
min-height: 260px;
|
| 270 |
+
max-height: 46vh;
|
| 271 |
+
width: 100%;
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
#ctrl-pane {
|
| 275 |
+
width: 100%;
|
| 276 |
+
flex: 1;
|
| 277 |
+
border-left: none;
|
| 278 |
+
border-top: 1px solid var(--bdr);
|
| 279 |
+
overflow-y: auto;
|
| 280 |
+
}
|
| 281 |
+
|
| 282 |
+
.topbar {
|
| 283 |
+
padding: 8px 12px 0;
|
| 284 |
+
}
|
| 285 |
+
|
| 286 |
+
.brand-t {
|
| 287 |
+
font-size: .95rem;
|
| 288 |
+
}
|
| 289 |
+
}
|
| 290 |
+
|
| 291 |
+
.cpanel {
|
| 292 |
+
padding: 14px 16px;
|
| 293 |
+
display: flex;
|
| 294 |
+
flex-direction: column;
|
| 295 |
+
gap: 12px;
|
| 296 |
+
}
|
| 297 |
+
|
| 298 |
+
.sec-label {
|
| 299 |
+
font-size: .65rem;
|
| 300 |
+
font-weight: 600;
|
| 301 |
+
text-transform: uppercase;
|
| 302 |
+
letter-spacing: .08em;
|
| 303 |
+
color: var(--txtl);
|
| 304 |
+
display: flex;
|
| 305 |
+
align-items: center;
|
| 306 |
+
gap: 6px;
|
| 307 |
+
font-family: var(--fM);
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
.status {
|
| 311 |
+
display: flex;
|
| 312 |
+
align-items: center;
|
| 313 |
+
gap: 7px;
|
| 314 |
+
padding: 7px 13px;
|
| 315 |
+
border-radius: 30px;
|
| 316 |
+
font-size: .73rem;
|
| 317 |
+
font-weight: 500;
|
| 318 |
+
border: 1px solid var(--bdr);
|
| 319 |
+
background: var(--card);
|
| 320 |
+
color: var(--txtm);
|
| 321 |
+
transition: all .3s;
|
| 322 |
+
min-height: 34px;
|
| 323 |
+
font-family: var(--fM);
|
| 324 |
+
}
|
| 325 |
+
|
| 326 |
+
.sdot {
|
| 327 |
+
width: 7px;
|
| 328 |
+
height: 7px;
|
| 329 |
+
border-radius: 50%;
|
| 330 |
+
background: var(--txtl);
|
| 331 |
+
flex-shrink: 0;
|
| 332 |
+
transition: all .3s;
|
| 333 |
+
}
|
| 334 |
+
|
| 335 |
+
.status.act .sdot {
|
| 336 |
+
background: var(--sage);
|
| 337 |
+
box-shadow: 0 0 8px var(--sage);
|
| 338 |
+
animation: pd 1.4s infinite;
|
| 339 |
+
}
|
| 340 |
+
|
| 341 |
+
.status.act {
|
| 342 |
+
border-color: rgba(126, 184, 154, .3);
|
| 343 |
+
color: var(--sage);
|
| 344 |
+
}
|
| 345 |
+
|
| 346 |
+
.status.err .sdot {
|
| 347 |
+
background: var(--red);
|
| 348 |
+
}
|
| 349 |
+
|
| 350 |
+
.status.err {
|
| 351 |
+
border-color: rgba(192, 97, 74, .3);
|
| 352 |
+
color: var(--red);
|
| 353 |
+
}
|
| 354 |
+
|
| 355 |
+
.text-display {
|
| 356 |
+
background: var(--card);
|
| 357 |
+
border: 1px solid var(--bdr);
|
| 358 |
+
border-radius: var(--r14);
|
| 359 |
+
padding: 10px 13px;
|
| 360 |
+
min-height: 52px;
|
| 361 |
+
}
|
| 362 |
+
|
| 363 |
+
.cur-word {
|
| 364 |
+
font-size: 1.15rem;
|
| 365 |
+
font-weight: 700;
|
| 366 |
+
color: var(--goldb);
|
| 367 |
+
font-family: var(--fD);
|
| 368 |
+
line-height: 1.2;
|
| 369 |
+
margin-bottom: 4px;
|
| 370 |
+
}
|
| 371 |
+
|
| 372 |
+
.cur-sentence {
|
| 373 |
+
font-size: .75rem;
|
| 374 |
+
color: var(--txtm);
|
| 375 |
+
line-height: 1.5;
|
| 376 |
+
font-family: var(--fM);
|
| 377 |
+
}
|
| 378 |
+
|
| 379 |
+
.btn-row {
|
| 380 |
+
display: grid;
|
| 381 |
+
grid-template-columns: repeat(3, 1fr);
|
| 382 |
+
gap: 7px;
|
| 383 |
+
}
|
| 384 |
+
|
| 385 |
+
.btn {
|
| 386 |
+
display: flex;
|
| 387 |
+
align-items: center;
|
| 388 |
+
justify-content: center;
|
| 389 |
+
gap: 5px;
|
| 390 |
+
padding: 9px 6px;
|
| 391 |
+
border: 1px solid transparent;
|
| 392 |
+
border-radius: var(--r14);
|
| 393 |
+
font-family: var(--fB);
|
| 394 |
+
font-size: .76rem;
|
| 395 |
+
font-weight: 600;
|
| 396 |
+
cursor: pointer;
|
| 397 |
+
transition: all .22s;
|
| 398 |
+
white-space: nowrap;
|
| 399 |
+
overflow: hidden;
|
| 400 |
+
}
|
| 401 |
+
|
| 402 |
+
.btn:active {
|
| 403 |
+
transform: scale(.96);
|
| 404 |
+
}
|
| 405 |
+
|
| 406 |
+
.btn:disabled {
|
| 407 |
+
opacity: .4;
|
| 408 |
+
cursor: not-allowed;
|
| 409 |
+
}
|
| 410 |
+
|
| 411 |
+
.btn-speak {
|
| 412 |
+
background: linear-gradient(135deg, #4e7d66, #7eb89a);
|
| 413 |
+
color: #0d1117;
|
| 414 |
+
border-color: rgba(126, 184, 154, .35);
|
| 415 |
+
}
|
| 416 |
+
|
| 417 |
+
.btn-speak:hover {
|
| 418 |
+
box-shadow: 0 4px 16px rgba(126, 184, 154, .3);
|
| 419 |
+
}
|
| 420 |
+
|
| 421 |
+
.btn-back {
|
| 422 |
+
background: transparent;
|
| 423 |
+
color: var(--txtm);
|
| 424 |
+
border-color: var(--bdr);
|
| 425 |
+
}
|
| 426 |
+
|
| 427 |
+
.btn-back:hover {
|
| 428 |
+
border-color: var(--bdrm);
|
| 429 |
+
color: var(--txt);
|
| 430 |
+
}
|
| 431 |
+
|
| 432 |
+
.btn-reset {
|
| 433 |
+
background: transparent;
|
| 434 |
+
color: var(--red);
|
| 435 |
+
border-color: rgba(192, 97, 74, .3);
|
| 436 |
+
}
|
| 437 |
+
|
| 438 |
+
.btn-reset:hover {
|
| 439 |
+
background: rgba(192, 97, 74, .08);
|
| 440 |
+
border-color: rgba(192, 97, 74, .55);
|
| 441 |
+
}
|
| 442 |
+
|
| 443 |
+
.divider {
|
| 444 |
+
height: 1px;
|
| 445 |
+
background: var(--bdr);
|
| 446 |
+
margin: 0 -16px;
|
| 447 |
+
}
|
| 448 |
+
|
| 449 |
+
.emotion-header {
|
| 450 |
+
display: flex;
|
| 451 |
+
justify-content: space-between;
|
| 452 |
+
align-items: center;
|
| 453 |
+
margin-bottom: 8px;
|
| 454 |
+
}
|
| 455 |
+
|
| 456 |
+
.emotion-big {
|
| 457 |
+
font-family: var(--fD);
|
| 458 |
+
font-size: 1.1rem;
|
| 459 |
+
font-weight: 700;
|
| 460 |
+
color: var(--goldb);
|
| 461 |
+
}
|
| 462 |
+
|
| 463 |
+
.emotion-icon {
|
| 464 |
+
font-size: 1.5rem;
|
| 465 |
+
animation: emojiPop 2.2s ease infinite;
|
| 466 |
+
}
|
| 467 |
+
|
| 468 |
+
@keyframes emojiPop {
|
| 469 |
+
|
| 470 |
+
0%,
|
| 471 |
+
100% {
|
| 472 |
+
transform: scale(1)
|
| 473 |
+
}
|
| 474 |
+
|
| 475 |
+
50% {
|
| 476 |
+
transform: scale(1.12)
|
| 477 |
+
}
|
| 478 |
+
}
|
| 479 |
+
|
| 480 |
+
.emo-bars {
|
| 481 |
+
display: flex;
|
| 482 |
+
flex-direction: column;
|
| 483 |
+
gap: 5px;
|
| 484 |
+
}
|
| 485 |
+
|
| 486 |
+
.emo-bar {
|
| 487 |
+
display: flex;
|
| 488 |
+
align-items: center;
|
| 489 |
+
gap: 8px;
|
| 490 |
+
}
|
| 491 |
+
|
| 492 |
+
.ebo-label {
|
| 493 |
+
width: 78px;
|
| 494 |
+
font-size: .72rem;
|
| 495 |
+
color: var(--txtm);
|
| 496 |
+
flex-shrink: 0;
|
| 497 |
+
}
|
| 498 |
+
|
| 499 |
+
.ebo-track {
|
| 500 |
+
flex: 1;
|
| 501 |
+
height: 20px;
|
| 502 |
+
background: rgba(26, 34, 51, .8);
|
| 503 |
+
border-radius: 10px;
|
| 504 |
+
overflow: hidden;
|
| 505 |
+
border: 1px solid var(--bdr);
|
| 506 |
+
}
|
| 507 |
+
|
| 508 |
+
.ebo-fill {
|
| 509 |
+
height: 100%;
|
| 510 |
+
transition: width .5s cubic-bezier(.4, 0, .2, 1);
|
| 511 |
+
display: flex;
|
| 512 |
+
align-items: center;
|
| 513 |
+
justify-content: flex-end;
|
| 514 |
+
padding-right: 6px;
|
| 515 |
+
font-size: .62rem;
|
| 516 |
+
font-weight: 700;
|
| 517 |
+
font-family: var(--fM);
|
| 518 |
+
color: rgba(0, 0, 0, .75);
|
| 519 |
+
min-width: 28px;
|
| 520 |
+
}
|
| 521 |
+
|
| 522 |
+
.sug-detected {
|
| 523 |
+
font-size: .65rem;
|
| 524 |
+
font-family: var(--fM);
|
| 525 |
+
color: var(--gold);
|
| 526 |
+
background: rgba(212, 168, 83, .07);
|
| 527 |
+
border: 1px solid var(--bdr);
|
| 528 |
+
border-radius: var(--r8);
|
| 529 |
+
padding: 4px 10px;
|
| 530 |
+
margin-bottom: 5px;
|
| 531 |
+
letter-spacing: .04em;
|
| 532 |
+
}
|
| 533 |
+
|
| 534 |
+
.sug-list {
|
| 535 |
+
display: flex;
|
| 536 |
+
flex-direction: column;
|
| 537 |
+
gap: 4px;
|
| 538 |
+
}
|
| 539 |
+
|
| 540 |
+
.sug-item {
|
| 541 |
+
display: flex;
|
| 542 |
+
align-items: center;
|
| 543 |
+
gap: 9px;
|
| 544 |
+
padding: 7px 11px;
|
| 545 |
+
border-radius: var(--r8);
|
| 546 |
+
background: var(--card);
|
| 547 |
+
border: 1px solid var(--bdr);
|
| 548 |
+
cursor: pointer;
|
| 549 |
+
transition: all .2s;
|
| 550 |
+
font-size: .82rem;
|
| 551 |
+
color: var(--txt);
|
| 552 |
+
}
|
| 553 |
+
|
| 554 |
+
.sug-item:hover {
|
| 555 |
+
background: rgba(212, 168, 83, .08);
|
| 556 |
+
border-color: var(--bdrm);
|
| 557 |
+
transform: translateX(3px);
|
| 558 |
+
color: var(--goldb);
|
| 559 |
+
}
|
| 560 |
+
|
| 561 |
+
.sug-n {
|
| 562 |
+
font-family: var(--fM);
|
| 563 |
+
font-size: .65rem;
|
| 564 |
+
font-weight: 700;
|
| 565 |
+
color: var(--gold);
|
| 566 |
+
background: rgba(212, 168, 83, .12);
|
| 567 |
+
border: 1px solid rgba(212, 168, 83, .2);
|
| 568 |
+
width: 20px;
|
| 569 |
+
height: 20px;
|
| 570 |
+
display: flex;
|
| 571 |
+
align-items: center;
|
| 572 |
+
justify-content: center;
|
| 573 |
+
border-radius: 5px;
|
| 574 |
+
flex-shrink: 0;
|
| 575 |
+
}
|
| 576 |
+
|
| 577 |
+
.sug-empty {
|
| 578 |
+
text-align: center;
|
| 579 |
+
padding: 16px 0;
|
| 580 |
+
color: var(--txtl);
|
| 581 |
+
font-size: .75rem;
|
| 582 |
+
line-height: 1.7;
|
| 583 |
+
}
|
| 584 |
+
|
| 585 |
+
#emotionChart {
|
| 586 |
+
height: 160px !important;
|
| 587 |
+
}
|
| 588 |
+
|
| 589 |
+
.stat-grid {
|
| 590 |
+
display: grid;
|
| 591 |
+
grid-template-columns: repeat(3, 1fr);
|
| 592 |
+
gap: 6px;
|
| 593 |
+
}
|
| 594 |
+
|
| 595 |
+
.stat-tile {
|
| 596 |
+
background: var(--card);
|
| 597 |
+
border: 1px solid var(--bdr);
|
| 598 |
+
border-radius: var(--r8);
|
| 599 |
+
padding: 9px 10px;
|
| 600 |
+
text-align: center;
|
| 601 |
+
transition: border-color .25s;
|
| 602 |
+
}
|
| 603 |
+
|
| 604 |
+
.stat-tile:hover {
|
| 605 |
+
border-color: var(--bdrm);
|
| 606 |
+
}
|
| 607 |
+
|
| 608 |
+
.sl {
|
| 609 |
+
font-size: .55rem;
|
| 610 |
+
text-transform: uppercase;
|
| 611 |
+
letter-spacing: .07em;
|
| 612 |
+
color: var(--txtl);
|
| 613 |
+
font-weight: 600;
|
| 614 |
+
margin-bottom: 3px;
|
| 615 |
+
font-family: var(--fM);
|
| 616 |
+
}
|
| 617 |
+
|
| 618 |
+
.sv {
|
| 619 |
+
font-family: var(--fD);
|
| 620 |
+
font-size: 1.3rem;
|
| 621 |
+
font-weight: 700;
|
| 622 |
+
color: var(--gold);
|
| 623 |
+
line-height: 1;
|
| 624 |
+
}
|
| 625 |
+
|
| 626 |
+
.about-txt {
|
| 627 |
+
font-size: .75rem;
|
| 628 |
+
color: var(--txtm);
|
| 629 |
+
line-height: 1.75;
|
| 630 |
+
}
|
| 631 |
+
|
| 632 |
+
.about-txt p {
|
| 633 |
+
margin-bottom: 7px;
|
| 634 |
+
}
|
| 635 |
+
|
| 636 |
+
.alist {
|
| 637 |
+
list-style: none;
|
| 638 |
+
margin-bottom: 7px;
|
| 639 |
+
}
|
| 640 |
+
|
| 641 |
+
.alist li {
|
| 642 |
+
padding-left: 13px;
|
| 643 |
+
position: relative;
|
| 644 |
+
font-size: .73rem;
|
| 645 |
+
padding-bottom: 2px;
|
| 646 |
+
}
|
| 647 |
+
|
| 648 |
+
.alist li::before {
|
| 649 |
+
content: 'βΊ';
|
| 650 |
+
position: absolute;
|
| 651 |
+
left: 0;
|
| 652 |
+
color: var(--gold);
|
| 653 |
+
font-weight: 700;
|
| 654 |
+
}
|
| 655 |
+
|
| 656 |
+
.acredit {
|
| 657 |
+
margin-top: 10px;
|
| 658 |
+
padding-top: 10px;
|
| 659 |
+
border-top: 1px solid var(--bdr);
|
| 660 |
+
}
|
| 661 |
+
|
| 662 |
+
.aname {
|
| 663 |
+
font-family: var(--fD);
|
| 664 |
+
color: var(--gold);
|
| 665 |
+
font-weight: 600;
|
| 666 |
+
font-size: .88rem;
|
| 667 |
+
}
|
| 668 |
+
|
| 669 |
+
.arole {
|
| 670 |
+
font-size: .65rem;
|
| 671 |
+
color: var(--txtl);
|
| 672 |
+
margin: 2px 0 3px;
|
| 673 |
+
font-family: var(--fM);
|
| 674 |
+
}
|
| 675 |
+
|
| 676 |
+
.aemail {
|
| 677 |
+
color: var(--sage);
|
| 678 |
+
font-size: .66rem;
|
| 679 |
+
font-family: var(--fM);
|
| 680 |
+
}
|
| 681 |
+
|
| 682 |
+
.nav-row {
|
| 683 |
+
display: flex;
|
| 684 |
+
gap: 7px;
|
| 685 |
+
padding: 0 16px 20px;
|
| 686 |
+
}
|
| 687 |
+
|
| 688 |
+
.nav-a {
|
| 689 |
+
flex: 1;
|
| 690 |
+
display: flex;
|
| 691 |
+
align-items: center;
|
| 692 |
+
justify-content: center;
|
| 693 |
+
gap: 6px;
|
| 694 |
+
padding: 12px 8px;
|
| 695 |
+
border-radius: var(--r14);
|
| 696 |
+
border: 1px solid var(--bdr);
|
| 697 |
+
background: var(--card);
|
| 698 |
+
color: var(--txtm);
|
| 699 |
+
text-decoration: none;
|
| 700 |
+
font-size: .76rem;
|
| 701 |
+
font-weight: 600;
|
| 702 |
+
transition: all .25s;
|
| 703 |
+
}
|
| 704 |
+
|
| 705 |
+
.nav-a:hover {
|
| 706 |
+
border-color: var(--bdrm);
|
| 707 |
+
color: var(--txt);
|
| 708 |
+
}
|
| 709 |
+
|
| 710 |
+
.nav-a.on {
|
| 711 |
+
background: rgba(212, 168, 83, .10);
|
| 712 |
+
border-color: var(--bdrh);
|
| 713 |
+
color: var(--gold);
|
| 714 |
+
}
|
| 715 |
+
</style>
|
| 716 |
+
</head>
|
| 717 |
+
|
| 718 |
+
<body>
|
| 719 |
+
|
| 720 |
+
<div id="app">
|
| 721 |
+
|
| 722 |
+
<div id="cam-pane">
|
| 723 |
+
<div class="topbar">
|
| 724 |
+
<div class="brand">
|
| 725 |
+
<span class="brand-t">ISL Gesture Recognition</span>
|
| 726 |
+
<span class="brand-s">Gesture Β· Emotion Β· AI</span>
|
| 727 |
+
</div>
|
| 728 |
+
<span class="live-badge" id="live-badge">β Ready</span>
|
| 729 |
+
</div>
|
| 730 |
+
|
| 731 |
+
<img id="camera-feed" src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"
|
| 732 |
+
alt="Camera Feed" />
|
| 733 |
+
|
| 734 |
+
<div class="cam-placeholder" id="cam-ph">
|
| 735 |
+
<span class="ph-icon">π·</span>
|
| 736 |
+
<span style="font-family:var(--fM);font-size:.75rem">Connecting to cameraβ¦</span>
|
| 737 |
+
</div>
|
| 738 |
+
|
| 739 |
+
<div class="cam-overlay">
|
| 740 |
+
<div>
|
| 741 |
+
<div style="display:flex;align-items:baseline;gap:8px">
|
| 742 |
+
<span id="co-letter" class="co-letter">π€</span>
|
| 743 |
+
<span id="co-word" class="co-word"></span>
|
| 744 |
+
</div>
|
| 745 |
+
</div>
|
| 746 |
+
<div id="co-desc" class="co-desc">Point camera at your hand</div>
|
| 747 |
+
<div class="co-progress">
|
| 748 |
+
<div class="co-bar" id="co-bar"></div>
|
| 749 |
+
</div>
|
| 750 |
+
</div>
|
| 751 |
+
</div>
|
| 752 |
+
|
| 753 |
+
<div id="ctrl-pane">
|
| 754 |
+
<div class="cpanel">
|
| 755 |
+
|
| 756 |
+
<div class="status" id="status">
|
| 757 |
+
<div class="sdot"></div>
|
| 758 |
+
<span id="stxt">Ready for detection</span>
|
| 759 |
+
</div>
|
| 760 |
+
|
| 761 |
+
<div class="text-display">
|
| 762 |
+
<div class="cur-word" id="cur-word">β</div>
|
| 763 |
+
<div class="cur-sentence" id="cur-sentence"></div>
|
| 764 |
+
</div>
|
| 765 |
+
|
| 766 |
+
<div class="btn-row">
|
| 767 |
+
<button class="btn btn-speak" onclick="sendCmd('speak')">π Speak</button>
|
| 768 |
+
<button class="btn btn-back" onclick="sendCmd('backspace')">β« Back</button>
|
| 769 |
+
<button class="btn btn-reset" onclick="sendCmd('reset')">π Reset</button>
|
| 770 |
+
</div>
|
| 771 |
+
|
| 772 |
+
<div class="divider"></div>
|
| 773 |
+
|
| 774 |
+
<div class="sec-label">π Emotion Detection</div>
|
| 775 |
+
<div class="emotion-header">
|
| 776 |
+
<span class="emotion-big" id="emo-label">Neutral</span>
|
| 777 |
+
<span class="emotion-icon" id="emo-icon">π</span>
|
| 778 |
+
</div>
|
| 779 |
+
|
| 780 |
+
<div class="emo-bars">
|
| 781 |
+
<div class="emo-bar">
|
| 782 |
+
<div class="ebo-label">π Happy</div>
|
| 783 |
+
<div class="ebo-track">
|
| 784 |
+
<div class="ebo-fill" id="bar-happy" style="width:0%;background:linear-gradient(90deg,#3a8a5c,#7eb89a)">0%
|
| 785 |
+
</div>
|
| 786 |
+
</div>
|
| 787 |
+
</div>
|
| 788 |
+
<div class="emo-bar">
|
| 789 |
+
<div class="ebo-label">π’ Sad</div>
|
| 790 |
+
<div class="ebo-track">
|
| 791 |
+
<div class="ebo-fill" id="bar-sad" style="width:0%;background:linear-gradient(90deg,#3a5a8a,#6090c8)">0%
|
| 792 |
+
</div>
|
| 793 |
+
</div>
|
| 794 |
+
</div>
|
| 795 |
+
<div class="emo-bar">
|
| 796 |
+
<div class="ebo-label">π Angry</div>
|
| 797 |
+
<div class="ebo-track">
|
| 798 |
+
<div class="ebo-fill" id="bar-angry" style="width:0%;background:linear-gradient(90deg,#8a3a3a,#c06060)">0%
|
| 799 |
+
</div>
|
| 800 |
+
</div>
|
| 801 |
+
</div>
|
| 802 |
+
<div class="emo-bar">
|
| 803 |
+
<div class="ebo-label">π² Surprise</div>
|
| 804 |
+
<div class="ebo-track">
|
| 805 |
+
<div class="ebo-fill" id="bar-surprise"
|
| 806 |
+
style="width:0%;background:linear-gradient(90deg,#8a6a1a,#d4a853)">0%</div>
|
| 807 |
+
</div>
|
| 808 |
+
</div>
|
| 809 |
+
<div class="emo-bar">
|
| 810 |
+
<div class="ebo-label">π Neutral</div>
|
| 811 |
+
<div class="ebo-track">
|
| 812 |
+
<div class="ebo-fill" id="bar-neutral"
|
| 813 |
+
style="width:100%;background:linear-gradient(90deg,#3a4455,#5a6878)">100%</div>
|
| 814 |
+
</div>
|
| 815 |
+
</div>
|
| 816 |
+
</div>
|
| 817 |
+
|
| 818 |
+
<div class="divider"></div>
|
| 819 |
+
|
| 820 |
+
<div class="sec-label">π‘ Word Suggestions</div>
|
| 821 |
+
<div id="sug-area">
|
| 822 |
+
<div class="sug-empty">Detected letters will appear<br>here with word suggestions</div>
|
| 823 |
+
</div>
|
| 824 |
+
|
| 825 |
+
<div class="divider"></div>
|
| 826 |
+
|
| 827 |
+
<div class="sec-label">π Emotion Timeline</div>
|
| 828 |
+
<canvas id="emotionChart"></canvas>
|
| 829 |
+
|
| 830 |
+
<div class="divider"></div>
|
| 831 |
+
|
| 832 |
+
<div class="sec-label">π Statistics</div>
|
| 833 |
+
<div class="stat-grid">
|
| 834 |
+
<div class="stat-tile">
|
| 835 |
+
<div class="sl">FPS</div>
|
| 836 |
+
<div class="sv" id="s-fps">0</div>
|
| 837 |
+
</div>
|
| 838 |
+
<div class="stat-tile">
|
| 839 |
+
<div class="sl">Letters</div>
|
| 840 |
+
<div class="sv" id="s-letters">0</div>
|
| 841 |
+
</div>
|
| 842 |
+
<div class="stat-tile">
|
| 843 |
+
<div class="sv" id="s-words">0</div>
|
| 844 |
+
<div class="sl">Words</div>
|
| 845 |
+
</div>
|
| 846 |
+
<div class="stat-tile">
|
| 847 |
+
<div class="sl">Sentences</div>
|
| 848 |
+
<div class="sv" id="s-sentences">0</div>
|
| 849 |
+
</div>
|
| 850 |
+
<div class="stat-tile">
|
| 851 |
+
<div class="sl">Emotions</div>
|
| 852 |
+
<div class="sv" id="s-emotions">0</div>
|
| 853 |
+
</div>
|
| 854 |
+
<div class="stat-tile">
|
| 855 |
+
<div class="sl">Conf.</div>
|
| 856 |
+
<div class="sv" id="s-conf">β</div>
|
| 857 |
+
</div>
|
| 858 |
+
</div>
|
| 859 |
+
|
| 860 |
+
<div class="divider"></div>
|
| 861 |
+
|
| 862 |
+
<div class="sec-label">βΉοΈ About</div>
|
| 863 |
+
<div class="about-txt">
|
| 864 |
+
<p>AI Powered ISL Detection is an assistive AI system designed to improve communication between hearing and
|
| 865 |
+
deaf individuals.</p>
|
| 866 |
+
<p>This module focuses on real-time gesture and facial emotion recognition using computer vision.</p>
|
| 867 |
+
<p>Useful for:</p>
|
| 868 |
+
<ul class="alist">
|
| 869 |
+
<li>Deaf individuals</li>
|
| 870 |
+
<li>Inclusive classrooms</li>
|
| 871 |
+
<li>Assistive communication systems</li>
|
| 872 |
+
<li>Accessibility and AI research</li>
|
| 873 |
+
</ul>
|
| 874 |
+
<div class="acredit">
|
| 875 |
+
<div class="aname">Deepak Roshan</div>
|
| 876 |
+
<div class="arole">AI Engineer</div>
|
| 877 |
+
<div class="aemail">deepakroshan380@gmail.com</div>
|
| 878 |
+
</div>
|
| 879 |
+
</div>
|
| 880 |
+
|
| 881 |
+
</div>
|
| 882 |
+
|
| 883 |
+
<div class="nav-row">
|
| 884 |
+
<a href="#" class="nav-a on" id="nav-gesture">π Gesture Recognition</a>
|
| 885 |
+
<a href="#" class="nav-a" id="nav-avatar">π€ Speech-to-Sign</a>
|
| 886 |
+
</div>
|
| 887 |
+
</div>
|
| 888 |
+
|
| 889 |
+
</div>
|
| 890 |
+
|
| 891 |
+
<script>
|
| 892 |
+
document.addEventListener('DOMContentLoaded', function () {
|
| 893 |
+
|
| 894 |
+
(function () {
|
| 895 |
+
var proto = location.protocol;
|
| 896 |
+
var host = location.hostname;
|
| 897 |
+
var port = location.port;
|
| 898 |
+
if (port && port !== '80' && port !== '443') {
|
| 899 |
+
document.getElementById('nav-gesture').href = proto + '//' + host + ':5000/';
|
| 900 |
+
document.getElementById('nav-avatar').href = proto + '//' + host + ':5001/';
|
| 901 |
+
} else {
|
| 902 |
+
document.getElementById('nav-gesture').href = proto + '//' + host + '/';
|
| 903 |
+
document.getElementById('nav-avatar').href = proto + '//' + host + '/avatar/';
|
| 904 |
+
}
|
| 905 |
+
})();
|
| 906 |
+
|
| 907 |
+
const EMO_ICONS = { happy: 'π', sad: 'π’', angry: 'π ', surprise: 'π²', neutral: 'π' };
|
| 908 |
+
const EMO_KEYS = ['happy', 'sad', 'angry', 'surprise', 'neutral'];
|
| 909 |
+
|
| 910 |
+
const chartCtx = document.getElementById('emotionChart').getContext('2d');
|
| 911 |
+
const chart = new Chart(chartCtx, {
|
| 912 |
+
type: 'line',
|
| 913 |
+
data: {
|
| 914 |
+
labels: [],
|
| 915 |
+
datasets: [
|
| 916 |
+
{ label: 'Happy', data: [], borderColor: '#7eb89a', borderWidth: 1.5, tension: .4, fill: false, pointRadius: 0 },
|
| 917 |
+
{ label: 'Sad', data: [], borderColor: '#6090c8', borderWidth: 1.5, tension: .4, fill: false, pointRadius: 0 },
|
| 918 |
+
{ label: 'Angry', data: [], borderColor: '#c06060', borderWidth: 1.5, tension: .4, fill: false, pointRadius: 0 },
|
| 919 |
+
{ label: 'Surprise', data: [], borderColor: '#d4a853', borderWidth: 1.5, tension: .4, fill: false, pointRadius: 0 },
|
| 920 |
+
{ label: 'Neutral', data: [], borderColor: '#5a6878', borderWidth: 1.5, tension: .4, fill: false, pointRadius: 0 },
|
| 921 |
+
]
|
| 922 |
+
},
|
| 923 |
+
options: {
|
| 924 |
+
responsive: true,
|
| 925 |
+
maintainAspectRatio: false,
|
| 926 |
+
plugins: {
|
| 927 |
+
legend: { labels: { color: '#5a5650', font: { size: 9, family: "'DM Mono'" }, boxWidth: 10 } }
|
| 928 |
+
},
|
| 929 |
+
scales: {
|
| 930 |
+
y: {
|
| 931 |
+
beginAtZero: true, max: 1,
|
| 932 |
+
ticks: { color: '#5a5650', font: { size: 8 } },
|
| 933 |
+
grid: { color: 'rgba(212,168,83,.05)' }
|
| 934 |
+
},
|
| 935 |
+
x: {
|
| 936 |
+
ticks: { display: false },
|
| 937 |
+
grid: { color: 'rgba(212,168,83,.03)' }
|
| 938 |
+
}
|
| 939 |
+
},
|
| 940 |
+
animation: false
|
| 941 |
+
}
|
| 942 |
+
});
|
| 943 |
+
|
| 944 |
+
/* ββ Browser Camera Streaming ββββββββββββββββββββββββββββ */
|
| 945 |
+
let cameraStream = null;
|
| 946 |
+
const captureCanvas = document.createElement('canvas');
|
| 947 |
+
const captureCtx = captureCanvas.getContext('2d');
|
| 948 |
+
|
| 949 |
+
async function startBrowserCamera() {
|
| 950 |
+
if (cameraStream) return;
|
| 951 |
+
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
| 952 |
+
console.error('Camera API not supported β requires HTTPS on cloud');
|
| 953 |
+
return;
|
| 954 |
+
}
|
| 955 |
+
try {
|
| 956 |
+
cameraStream = await navigator.mediaDevices.getUserMedia({
|
| 957 |
+
video: { width: { ideal: 640 }, height: { ideal: 480 }, facingMode: 'user' },
|
| 958 |
+
audio: false
|
| 959 |
+
});
|
| 960 |
+
const video = document.createElement('video');
|
| 961 |
+
video.srcObject = cameraStream;
|
| 962 |
+
video.autoplay = true;
|
| 963 |
+
video.playsInline = true;
|
| 964 |
+
video.addEventListener('loadeddata', () => {
|
| 965 |
+
captureCanvas.width = 320;
|
| 966 |
+
captureCanvas.height = 240;
|
| 967 |
+
if (!window.cameraInterval) {
|
| 968 |
+
window.cameraInterval = setInterval(() => {
|
| 969 |
+
if (document.visibilityState === 'visible' && socket && socket.connected) {
|
| 970 |
+
captureCtx.drawImage(video, 0, 0, 320, 240);
|
| 971 |
+
const frame = captureCanvas.toDataURL('image/jpeg', 0.6);
|
| 972 |
+
socket.emit('browser_frame', { frame: frame.split(',')[1] });
|
| 973 |
+
}
|
| 974 |
+
}, 65); // ~15 FPS
|
| 975 |
+
}
|
| 976 |
+
});
|
| 977 |
+
} catch (err) {
|
| 978 |
+
console.error('Camera access error:', err);
|
| 979 |
+
}
|
| 980 |
+
}
|
| 981 |
+
|
| 982 |
+
function stopBrowserCamera() {
|
| 983 |
+
if (cameraStream) {
|
| 984 |
+
cameraStream.getTracks().forEach(track => track.stop());
|
| 985 |
+
cameraStream = null;
|
| 986 |
+
}
|
| 987 |
+
if (window.cameraInterval) {
|
| 988 |
+
clearInterval(window.cameraInterval);
|
| 989 |
+
window.cameraInterval = null;
|
| 990 |
+
}
|
| 991 |
+
}
|
| 992 |
+
|
| 993 |
+
/* ββ Socket.IO ββββββββββββββββββββββββββββββββββββββββββββ */
|
| 994 |
+
// β
FIX: transports updated to prefer WebSocket with polling fallback.
|
| 995 |
+
// upgrade:true lets Socket.IO automatically upgrade from polling β websocket
|
| 996 |
+
// once the connection is established. This is required for camera streaming
|
| 997 |
+
// to work reliably on HuggingFace Spaces (HTTPS) and other cloud deployments.
|
| 998 |
+
let socket;
|
| 999 |
+
try {
|
| 1000 |
+
socket = io({
|
| 1001 |
+
transports: ['websocket', 'polling'],
|
| 1002 |
+
upgrade: true,
|
| 1003 |
+
reconnectionDelay: 1000
|
| 1004 |
+
});
|
| 1005 |
+
|
| 1006 |
+
socket.on('connect', () => {
|
| 1007 |
+
setStat('act', 'Connected β detectingβ¦');
|
| 1008 |
+
document.getElementById('cam-ph').style.display = 'none';
|
| 1009 |
+
document.getElementById('live-badge').textContent = 'β Live';
|
| 1010 |
+
startBrowserCamera();
|
| 1011 |
+
});
|
| 1012 |
+
|
| 1013 |
+
socket.on('disconnect', () => {
|
| 1014 |
+
setStat('', 'Disconnected');
|
| 1015 |
+
document.getElementById('cam-ph').style.display = 'flex';
|
| 1016 |
+
document.getElementById('live-badge').textContent = 'β Offline';
|
| 1017 |
+
stopBrowserCamera();
|
| 1018 |
+
});
|
| 1019 |
+
|
| 1020 |
+
socket.on('state_update', (data) => {
|
| 1021 |
+
try {
|
| 1022 |
+
if (data.frame) {
|
| 1023 |
+
document.getElementById('camera-feed').src = 'data:image/jpeg;base64,' + data.frame;
|
| 1024 |
+
document.getElementById('cam-ph').style.display = 'none';
|
| 1025 |
+
}
|
| 1026 |
+
|
| 1027 |
+
const letterEl = document.getElementById('co-letter');
|
| 1028 |
+
const wordEl = document.getElementById('co-word');
|
| 1029 |
+
const descEl = document.getElementById('co-desc');
|
| 1030 |
+
|
| 1031 |
+
if (data.overlay_letter) {
|
| 1032 |
+
letterEl.textContent = data.overlay_letter.toUpperCase();
|
| 1033 |
+
}
|
| 1034 |
+
|
| 1035 |
+
if (data.current_word !== undefined) {
|
| 1036 |
+
const w = data.current_word || 'β';
|
| 1037 |
+
wordEl.textContent = w;
|
| 1038 |
+
document.getElementById('cur-word').textContent = w;
|
| 1039 |
+
}
|
| 1040 |
+
|
| 1041 |
+
if (data.sentence !== undefined) {
|
| 1042 |
+
document.getElementById('cur-sentence').textContent = data.sentence || '';
|
| 1043 |
+
}
|
| 1044 |
+
|
| 1045 |
+
if (data.overlay_confidence !== undefined) {
|
| 1046 |
+
const pct = Math.round(data.overlay_confidence * 100);
|
| 1047 |
+
descEl.textContent = 'Confidence: ' + pct + '%';
|
| 1048 |
+
document.getElementById('s-conf').textContent = pct + '%';
|
| 1049 |
+
document.getElementById('co-bar').style.width = pct + '%';
|
| 1050 |
+
}
|
| 1051 |
+
|
| 1052 |
+
if (data.fps !== undefined) {
|
| 1053 |
+
document.getElementById('s-fps').textContent = Math.round(data.fps);
|
| 1054 |
+
}
|
| 1055 |
+
|
| 1056 |
+
if (data.stats) {
|
| 1057 |
+
document.getElementById('s-letters').textContent = data.stats.letters_detected || 0;
|
| 1058 |
+
document.getElementById('s-words').textContent = data.stats.words_formed || 0;
|
| 1059 |
+
document.getElementById('s-sentences').textContent = data.stats.sentences_formed || 0;
|
| 1060 |
+
document.getElementById('s-emotions').textContent = data.stats.emotion_changes || 0;
|
| 1061 |
+
}
|
| 1062 |
+
|
| 1063 |
+
if (Array.isArray(data.suggestions)) {
|
| 1064 |
+
updateSuggestions(data.suggestions, data.current_word);
|
| 1065 |
+
}
|
| 1066 |
+
|
| 1067 |
+
if (data.emotion_scores) {
|
| 1068 |
+
updateEmotion(data.emotion || 'neutral', data.emotion_scores);
|
| 1069 |
+
}
|
| 1070 |
+
|
| 1071 |
+
if (data.emotion_timeline && data.emotion_timeline.length > 0) {
|
| 1072 |
+
updateChart(data.emotion_timeline);
|
| 1073 |
+
}
|
| 1074 |
+
|
| 1075 |
+
} catch (e) { console.error('Update error:', e); }
|
| 1076 |
+
});
|
| 1077 |
+
|
| 1078 |
+
} catch (e) {
|
| 1079 |
+
console.warn('Socket.IO unavailable β demo mode');
|
| 1080 |
+
}
|
| 1081 |
+
|
| 1082 |
+
/* ββ Update emotion bars ββββββββββββββββββββββββββββββββββ */
|
| 1083 |
+
function updateEmotion(emotion, scores) {
|
| 1084 |
+
document.getElementById('emo-label').textContent = emotion.charAt(0).toUpperCase() + emotion.slice(1);
|
| 1085 |
+
document.getElementById('emo-icon').textContent = EMO_ICONS[emotion] || 'π';
|
| 1086 |
+
EMO_KEYS.forEach(k => {
|
| 1087 |
+
const bar = document.getElementById('bar-' + k);
|
| 1088 |
+
if (!bar) return;
|
| 1089 |
+
let s = Math.max(0, Math.min(1, scores[k] || 0));
|
| 1090 |
+
const pct = Math.round(s * 100);
|
| 1091 |
+
bar.style.width = pct + '%';
|
| 1092 |
+
bar.textContent = pct + '%';
|
| 1093 |
+
});
|
| 1094 |
+
}
|
| 1095 |
+
|
| 1096 |
+
/* ββ Update suggestions βββββββββββββββββββββββββββββββββββ */
|
| 1097 |
+
function updateSuggestions(suggestions, currentWord) {
|
| 1098 |
+
const area = document.getElementById('sug-area');
|
| 1099 |
+
if (!suggestions || !suggestions.length) {
|
| 1100 |
+
area.innerHTML = '<div class="sug-empty">Detected letters will appear<br>here with word suggestions</div>';
|
| 1101 |
+
return;
|
| 1102 |
+
}
|
| 1103 |
+
const detectedRow = currentWord
|
| 1104 |
+
? '<div class="sug-detected">Detected: ' + currentWord.toUpperCase() + '</div>' : '';
|
| 1105 |
+
area.innerHTML = detectedRow + '<div class="sug-list">' +
|
| 1106 |
+
suggestions.map((w, i) =>
|
| 1107 |
+
'<div class="sug-item" onclick="window.acceptSug(\'' + w + '\')">' +
|
| 1108 |
+
'<span class="sug-n">' + (i + 1) + '</span>' + w + '</div>'
|
| 1109 |
+
).join('') + '</div>';
|
| 1110 |
+
}
|
| 1111 |
+
|
| 1112 |
+
/* ββ Update chart βββββββββββββββββββββββββββββββββββββββββ */
|
| 1113 |
+
function updateChart(timeline) {
|
| 1114 |
+
if (!timeline || !timeline.length) return;
|
| 1115 |
+
const collected = Object.fromEntries(EMO_KEYS.map(k => [k, []]));
|
| 1116 |
+
timeline.forEach(tp => {
|
| 1117 |
+
const s = tp.scores || {};
|
| 1118 |
+
EMO_KEYS.forEach(k => collected[k].push(s[k] || 0));
|
| 1119 |
+
});
|
| 1120 |
+
chart.data.labels = timeline.map((_, i) => i);
|
| 1121 |
+
chart.data.datasets.forEach((ds, i) => ds.data = collected[EMO_KEYS[i]]);
|
| 1122 |
+
chart.update('none');
|
| 1123 |
+
}
|
| 1124 |
+
|
| 1125 |
+
/* ββ Status helper ββββββββββββββββββββββββββββββββββββββββ */
|
| 1126 |
+
function setStat(cls, msg) {
|
| 1127 |
+
document.getElementById('stxt').textContent = msg;
|
| 1128 |
+
document.getElementById('status').className = 'status' + (cls ? ' ' + cls : '');
|
| 1129 |
+
}
|
| 1130 |
+
|
| 1131 |
+
/* ββ Global handlers ββββββββββββββββββββββββββββββββββββββ */
|
| 1132 |
+
window.sendCmd = function (action) {
|
| 1133 |
+
if (action === 'speak') {
|
| 1134 |
+
const sentence = document.getElementById('cur-sentence').textContent.trim();
|
| 1135 |
+
const word = document.getElementById('cur-word').textContent.trim();
|
| 1136 |
+
const text = sentence || (word !== 'β' ? word : '');
|
| 1137 |
+
if (text && socket) socket.emit('command', { action: 'speak', text });
|
| 1138 |
+
} else {
|
| 1139 |
+
if (socket) socket.emit('command', { action });
|
| 1140 |
+
}
|
| 1141 |
+
};
|
| 1142 |
+
|
| 1143 |
+
window.acceptSug = function (word) {
|
| 1144 |
+
if (socket) socket.emit('command', { action: 'accept_suggestion', word });
|
| 1145 |
+
};
|
| 1146 |
+
|
| 1147 |
+
/* ββ Keyboard shortcuts βββββββββββββββββββββββββββββββββββ */
|
| 1148 |
+
document.addEventListener('keydown', e => {
|
| 1149 |
+
if (e.ctrlKey && e.key === 's') { e.preventDefault(); window.sendCmd('speak'); }
|
| 1150 |
+
if (e.ctrlKey && e.key === 'r') { e.preventDefault(); window.sendCmd('reset'); }
|
| 1151 |
+
if (e.ctrlKey && e.key === 'Backspace') { e.preventDefault(); window.sendCmd('backspace'); }
|
| 1152 |
+
if (['1', '2', '3'].includes(e.key)) {
|
| 1153 |
+
const items = document.querySelectorAll('.sug-item');
|
| 1154 |
+
const item = items[parseInt(e.key) - 1];
|
| 1155 |
+
if (item) item.click();
|
| 1156 |
+
}
|
| 1157 |
+
});
|
| 1158 |
+
|
| 1159 |
+
});
|
| 1160 |
+
</script>
|
| 1161 |
+
</body>
|
| 1162 |
+
|
| 1163 |
+
</html>
|
translation.py
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
translation.py - Rule-based Sign-glish to English translation
|
| 3 |
+
|
| 4 |
+
Simple ruleβbased mapper that converts a list of sign tokens into natural English.
|
| 5 |
+
This implementation is deliberately lightweight and does not require any external services.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import List
|
| 9 |
+
|
| 10 |
+
# ==================== TRANSLATION MAP ====================
|
| 11 |
+
# Comprehensive mapping dictionary for ISL to English
|
| 12 |
+
|
| 13 |
+
_TRANSLATION_MAP = {
|
| 14 |
+
# Greetings
|
| 15 |
+
"HELLO": "Hello",
|
| 16 |
+
"HI": "Hi",
|
| 17 |
+
"GOODBYE": "Goodbye",
|
| 18 |
+
"BYE": "Bye",
|
| 19 |
+
|
| 20 |
+
# Politeness
|
| 21 |
+
"HELP": "Help",
|
| 22 |
+
"PLEASE": "Please",
|
| 23 |
+
"THANK": "Thank you",
|
| 24 |
+
"THANKS": "Thank you",
|
| 25 |
+
"SORRY": "Sorry",
|
| 26 |
+
"WELCOME": "Welcome",
|
| 27 |
+
|
| 28 |
+
# Pronouns
|
| 29 |
+
"I": "I",
|
| 30 |
+
"ME": "me",
|
| 31 |
+
"YOU": "you",
|
| 32 |
+
"WE": "we",
|
| 33 |
+
"THEY": "they",
|
| 34 |
+
|
| 35 |
+
# Basic responses
|
| 36 |
+
"YES": "yes",
|
| 37 |
+
"NO": "no",
|
| 38 |
+
"OK": "okay",
|
| 39 |
+
"GOOD": "good",
|
| 40 |
+
"BAD": "bad",
|
| 41 |
+
|
| 42 |
+
# Actions
|
| 43 |
+
"WANT": "want",
|
| 44 |
+
"NEED": "need",
|
| 45 |
+
"GO": "go",
|
| 46 |
+
"COME": "come",
|
| 47 |
+
"STOP": "stop",
|
| 48 |
+
"SIT": "sit",
|
| 49 |
+
"STAND": "stand",
|
| 50 |
+
"WALK": "walk",
|
| 51 |
+
"RUN": "run",
|
| 52 |
+
"EAT": "eat",
|
| 53 |
+
"DRINK": "drink",
|
| 54 |
+
"SLEEP": "sleep",
|
| 55 |
+
|
| 56 |
+
# Objects
|
| 57 |
+
"WATER": "water",
|
| 58 |
+
"FOOD": "food",
|
| 59 |
+
"HOME": "home",
|
| 60 |
+
"SCHOOL": "school",
|
| 61 |
+
"WORK": "work",
|
| 62 |
+
|
| 63 |
+
# Time
|
| 64 |
+
"MORNING": "good morning",
|
| 65 |
+
"AFTERNOON": "good afternoon",
|
| 66 |
+
"EVENING": "good evening",
|
| 67 |
+
"NIGHT": "good night",
|
| 68 |
+
"TODAY": "today",
|
| 69 |
+
"TOMORROW": "tomorrow",
|
| 70 |
+
"YESTERDAY": "yesterday",
|
| 71 |
+
"NOW": "now",
|
| 72 |
+
|
| 73 |
+
# Emotions
|
| 74 |
+
"HAPPY": "happy",
|
| 75 |
+
"SAD": "sad",
|
| 76 |
+
"ANGRY": "angry",
|
| 77 |
+
"SURPRISED": "surprised",
|
| 78 |
+
|
| 79 |
+
# Questions
|
| 80 |
+
"HOW": "how",
|
| 81 |
+
"WHAT": "what",
|
| 82 |
+
"WHERE": "where",
|
| 83 |
+
"WHEN": "when",
|
| 84 |
+
"WHY": "why",
|
| 85 |
+
"WHO": "who",
|
| 86 |
+
|
| 87 |
+
# Misc
|
| 88 |
+
"ARE": "are",
|
| 89 |
+
"IS": "is",
|
| 90 |
+
"LOVE": "love",
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
def translate_signs(sign_list: List[str]) -> str:
|
| 94 |
+
"""Translate a list of sign tokens into a readable English sentence.
|
| 95 |
+
|
| 96 |
+
Args:
|
| 97 |
+
sign_list: List of uppercase sign tokens.
|
| 98 |
+
|
| 99 |
+
Returns:
|
| 100 |
+
A string with the translated sentence.
|
| 101 |
+
|
| 102 |
+
Example:
|
| 103 |
+
>>> translate_signs(["HELLO", "I", "WANT", "WATER"])
|
| 104 |
+
"Hello I want water"
|
| 105 |
+
"""
|
| 106 |
+
if not sign_list:
|
| 107 |
+
return ""
|
| 108 |
+
|
| 109 |
+
translated_words = []
|
| 110 |
+
|
| 111 |
+
for token in sign_list:
|
| 112 |
+
# Handle fingerspelled words
|
| 113 |
+
if token.startswith("SPELL_"):
|
| 114 |
+
# Extract the spelled word
|
| 115 |
+
word = token.replace("SPELL_", "").lower()
|
| 116 |
+
translated_words.append(word)
|
| 117 |
+
else:
|
| 118 |
+
# Look up in translation map
|
| 119 |
+
translated = _TRANSLATION_MAP.get(token.upper(), token.lower())
|
| 120 |
+
translated_words.append(translated)
|
| 121 |
+
|
| 122 |
+
# Join words
|
| 123 |
+
sentence = " ".join(translated_words).strip()
|
| 124 |
+
|
| 125 |
+
# Capitalize first letter
|
| 126 |
+
if sentence:
|
| 127 |
+
sentence = sentence[0].upper() + sentence[1:]
|
| 128 |
+
|
| 129 |
+
return sentence
|
| 130 |
+
|
| 131 |
+
def add_translation(sign_token: str, english_text: str):
|
| 132 |
+
"""Add a new translation to the map.
|
| 133 |
+
|
| 134 |
+
Args:
|
| 135 |
+
sign_token: The ISL sign token (uppercase)
|
| 136 |
+
english_text: The English translation
|
| 137 |
+
"""
|
| 138 |
+
_TRANSLATION_MAP[sign_token.upper()] = english_text
|
| 139 |
+
|
| 140 |
+
def get_all_translations() -> dict:
|
| 141 |
+
"""Get all available translations.
|
| 142 |
+
|
| 143 |
+
Returns:
|
| 144 |
+
Dictionary of all sign-to-English mappings
|
| 145 |
+
"""
|
| 146 |
+
return _TRANSLATION_MAP.copy()
|
| 147 |
+
|
| 148 |
+
def translate_with_grammar(sign_list: List[str]) -> str:
|
| 149 |
+
"""Translate with basic grammar improvements.
|
| 150 |
+
|
| 151 |
+
Args:
|
| 152 |
+
sign_list: List of sign tokens
|
| 153 |
+
|
| 154 |
+
Returns:
|
| 155 |
+
Grammatically improved sentence
|
| 156 |
+
"""
|
| 157 |
+
sentence = translate_signs(sign_list)
|
| 158 |
+
|
| 159 |
+
# Basic grammar improvements
|
| 160 |
+
# Add articles
|
| 161 |
+
sentence = sentence.replace(" water", " some water")
|
| 162 |
+
sentence = sentence.replace(" food", " some food")
|
| 163 |
+
|
| 164 |
+
# Fix common patterns
|
| 165 |
+
sentence = sentence.replace("I want", "I would like")
|
| 166 |
+
sentence = sentence.replace("I need", "I need to")
|
| 167 |
+
|
| 168 |
+
return sentence
|
| 169 |
+
|
| 170 |
+
# ==================== MAIN ====================
|
| 171 |
+
|
| 172 |
+
if __name__ == "__main__":
|
| 173 |
+
# Demo examples
|
| 174 |
+
print("=" * 60)
|
| 175 |
+
print("ISL Sign-to-English Translation Demo")
|
| 176 |
+
print("=" * 60)
|
| 177 |
+
print()
|
| 178 |
+
|
| 179 |
+
# Test cases
|
| 180 |
+
test_cases = [
|
| 181 |
+
["HELLO", "HOW", "ARE", "YOU"],
|
| 182 |
+
["I", "WANT", "WATER", "PLEASE"],
|
| 183 |
+
["THANK", "YOU"],
|
| 184 |
+
["GOOD", "MORNING"],
|
| 185 |
+
["I", "NEED", "HELP"],
|
| 186 |
+
["SORRY", "I", "AM", "SAD"],
|
| 187 |
+
]
|
| 188 |
+
|
| 189 |
+
for signs in test_cases:
|
| 190 |
+
translation = translate_signs(signs)
|
| 191 |
+
print(f"Signs: {' '.join(signs)}")
|
| 192 |
+
print(f"Translation: {translation}")
|
| 193 |
+
print()
|
| 194 |
+
|
| 195 |
+
print("=" * 60)
|
unified_launcher.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# unified_launcher.py
|
| 2 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 3 |
+
# ISL FULL SYSTEM β SINGLE COMMAND LAUNCHER
|
| 4 |
+
# Starts BOTH services from ONE terminal:
|
| 5 |
+
#
|
| 6 |
+
# LOCAL β python unified_launcher.py
|
| 7 |
+
# CLOUD β RUN_MODE=CLOUD python unified_launcher.py
|
| 8 |
+
#
|
| 9 |
+
# Port 5000 β ISL Gesture Recognition + Emotion Detection (launcher.py)
|
| 10 |
+
# Port 5001 β Speech-to-Sign / Fingerspelling Avatar (speech_to_sign_avatar.py)
|
| 11 |
+
# OR speech_to_sign_avatar_fixed.py (auto-detected)
|
| 12 |
+
#
|
| 13 |
+
# β οΈ WHY subprocess.Popen (not multiprocessing)?
|
| 14 |
+
# speech_to_sign_avatar*.py calls eventlet.monkey_patch() at import-time,
|
| 15 |
+
# which corrupts the multiprocessing runtime. Running each script as a
|
| 16 |
+
# fully separate OS process avoids this entirely.
|
| 17 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 18 |
+
|
| 19 |
+
import subprocess
|
| 20 |
+
import sys
|
| 21 |
+
import os
|
| 22 |
+
import time
|
| 23 |
+
import multiprocessing
|
| 24 |
+
|
| 25 |
+
# ββ Resolve paths βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 26 |
+
PYTHON = sys.executable # same venv / interpreter that started us
|
| 27 |
+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 28 |
+
|
| 29 |
+
RUN_MODE = os.getenv("RUN_MODE", "LOCAL").upper()
|
| 30 |
+
|
| 31 |
+
# ββ Auto-detect which avatar script exists ββββββββββββββββββββββββββββββββββββ
|
| 32 |
+
# Prefer the fixed/fingerspelling version if present, fall back to original.
|
| 33 |
+
AVATAR_CANDIDATES = [
|
| 34 |
+
"speech_to_sign_avatar_fixed.py",
|
| 35 |
+
"speech_to_sign_avatar.py",
|
| 36 |
+
]
|
| 37 |
+
AVATAR_SCRIPT = None
|
| 38 |
+
for _candidate in AVATAR_CANDIDATES:
|
| 39 |
+
_path = os.path.join(BASE_DIR, _candidate)
|
| 40 |
+
if os.path.exists(_path):
|
| 41 |
+
AVATAR_SCRIPT = _path
|
| 42 |
+
break
|
| 43 |
+
|
| 44 |
+
GESTURE_SCRIPT = os.path.join(BASE_DIR, "launcher.py")
|
| 45 |
+
|
| 46 |
+
# ββ Environment for child processes ββββββββββββββββββββββββββββββββββββββββββ
|
| 47 |
+
CHILD_ENV = {
|
| 48 |
+
**os.environ,
|
| 49 |
+
"RUN_MODE" : RUN_MODE,
|
| 50 |
+
"PYTHONUNBUFFERED" : "1", # immediate stdout/stderr in logs
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
# ββ Oracle Cloud: open firewall ports (run once manually) βββββββββββββββββββββ
|
| 54 |
+
# sudo firewall-cmd --permanent --add-port=5000/tcp
|
| 55 |
+
# sudo firewall-cmd --permanent --add-port=5001/tcp
|
| 56 |
+
# sudo firewall-cmd --reload
|
| 57 |
+
# Also open in OCI Console β VCN β Security List β Ingress Rules
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 61 |
+
def start_process(script_path, label):
|
| 62 |
+
"""Spawn a child process and return the Popen handle."""
|
| 63 |
+
print(f" π Starting {label}")
|
| 64 |
+
print(f" Script : {script_path}")
|
| 65 |
+
proc = subprocess.Popen(
|
| 66 |
+
[PYTHON, script_path],
|
| 67 |
+
env=CHILD_ENV,
|
| 68 |
+
cwd=BASE_DIR,
|
| 69 |
+
stdout=None, # inherit β all logs appear in the same terminal
|
| 70 |
+
stderr=None,
|
| 71 |
+
)
|
| 72 |
+
print(f" PID : {proc.pid}")
|
| 73 |
+
return proc
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def stop_all(processes):
|
| 77 |
+
"""Gracefully terminate all child processes, then force-kill if needed."""
|
| 78 |
+
print("\nπ Shutting down all processesβ¦")
|
| 79 |
+
for label, proc in processes.items():
|
| 80 |
+
if proc and proc.poll() is None:
|
| 81 |
+
print(f" β Terminating {label} (PID {proc.pid})")
|
| 82 |
+
proc.terminate()
|
| 83 |
+
|
| 84 |
+
deadline = time.time() + 6
|
| 85 |
+
for label, proc in processes.items():
|
| 86 |
+
if proc:
|
| 87 |
+
remaining = max(0.1, deadline - time.time())
|
| 88 |
+
try:
|
| 89 |
+
proc.wait(timeout=remaining)
|
| 90 |
+
print(f" β
{label} stopped cleanly")
|
| 91 |
+
except subprocess.TimeoutExpired:
|
| 92 |
+
print(f" β‘ Force-killing {label} (PID {proc.pid})")
|
| 93 |
+
proc.kill()
|
| 94 |
+
|
| 95 |
+
print("β
All processes stopped")
|
| 96 |
+
print("=" * 70)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def preflight_check():
|
| 100 |
+
"""Verify required scripts exist before starting anything."""
|
| 101 |
+
ok = True
|
| 102 |
+
|
| 103 |
+
if not os.path.exists(GESTURE_SCRIPT):
|
| 104 |
+
print(f"β MISSING: launcher.py not found at:\n {GESTURE_SCRIPT}")
|
| 105 |
+
ok = False
|
| 106 |
+
|
| 107 |
+
if AVATAR_SCRIPT is None:
|
| 108 |
+
print("β MISSING: No avatar script found.")
|
| 109 |
+
print(" Expected one of:", AVATAR_CANDIDATES)
|
| 110 |
+
ok = False
|
| 111 |
+
|
| 112 |
+
if not ok:
|
| 113 |
+
print("\n Make sure unified_launcher.py is in the SAME folder as the other scripts.")
|
| 114 |
+
sys.exit(1)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 118 |
+
def main():
|
| 119 |
+
print()
|
| 120 |
+
print("=" * 70)
|
| 121 |
+
print(" π€ ISL FULL SYSTEM β UNIFIED LAUNCHER")
|
| 122 |
+
print("=" * 70)
|
| 123 |
+
print(f" RUN_MODE : {RUN_MODE}")
|
| 124 |
+
print(f" Python : {PYTHON}")
|
| 125 |
+
print(f" Base dir : {BASE_DIR}")
|
| 126 |
+
print(f" Gesture : {os.path.basename(GESTURE_SCRIPT)} β port 5000")
|
| 127 |
+
if AVATAR_SCRIPT:
|
| 128 |
+
print(f" Avatar : {os.path.basename(AVATAR_SCRIPT)} β port 5001")
|
| 129 |
+
print("=" * 70)
|
| 130 |
+
print()
|
| 131 |
+
|
| 132 |
+
preflight_check()
|
| 133 |
+
|
| 134 |
+
processes = {
|
| 135 |
+
"ISL Gesture (5000)" : None,
|
| 136 |
+
"Avatar Server (5001)": None,
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
restart_map = {
|
| 140 |
+
"ISL Gesture (5000)" : GESTURE_SCRIPT,
|
| 141 |
+
"Avatar Server (5001)": AVATAR_SCRIPT,
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
# ββ Start gesture system first; give camera time to initialise ββββββββββββ
|
| 145 |
+
processes["ISL Gesture (5000)"] = start_process(
|
| 146 |
+
GESTURE_SCRIPT, "ISL Gesture Recognition (port 5000)")
|
| 147 |
+
print()
|
| 148 |
+
print(" β³ Waiting 2 s for camera to initialiseβ¦")
|
| 149 |
+
time.sleep(2)
|
| 150 |
+
print()
|
| 151 |
+
|
| 152 |
+
# ββ Start avatar server βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 153 |
+
processes["Avatar Server (5001)"] = start_process(
|
| 154 |
+
AVATAR_SCRIPT, "Speech-to-Sign Avatar (port 5001)")
|
| 155 |
+
time.sleep(1)
|
| 156 |
+
|
| 157 |
+
# ββ Ready banner ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 158 |
+
print()
|
| 159 |
+
print("=" * 70)
|
| 160 |
+
print(" β
ALL SYSTEMS RUNNING!")
|
| 161 |
+
print("=" * 70)
|
| 162 |
+
|
| 163 |
+
if RUN_MODE == "CLOUD":
|
| 164 |
+
public_ip = os.getenv("PUBLIC_IP", "<YOUR_ORACLE_PUBLIC_IP>")
|
| 165 |
+
print(f" π ISL Dashboard β http://{public_ip}:5000")
|
| 166 |
+
print(f" π Avatar System β http://{public_ip}:5001")
|
| 167 |
+
print()
|
| 168 |
+
print(" π‘ Oracle Cloud checklist:")
|
| 169 |
+
print(" 1. Set env var: export PUBLIC_IP=<your-instance-ip>")
|
| 170 |
+
print(" 2. OCI Console β VCN β Security List β add Ingress rules")
|
| 171 |
+
print(" for TCP port 5000 and TCP port 5001 (source 0.0.0.0/0)")
|
| 172 |
+
print(" 3. On the VM (run once):")
|
| 173 |
+
print(" sudo firewall-cmd --permanent --add-port=5000/tcp")
|
| 174 |
+
print(" sudo firewall-cmd --permanent --add-port=5001/tcp")
|
| 175 |
+
print(" sudo firewall-cmd --reload")
|
| 176 |
+
print(" 4. To auto-start on reboot β see isl.service below")
|
| 177 |
+
else:
|
| 178 |
+
print(" π ISL Dashboard β http://localhost:5000")
|
| 179 |
+
print(" π Avatar System β http://localhost:5001")
|
| 180 |
+
|
| 181 |
+
print()
|
| 182 |
+
print(" π Press Ctrl+C to stop everything")
|
| 183 |
+
print("=" * 70)
|
| 184 |
+
print()
|
| 185 |
+
|
| 186 |
+
# ββ Watch-dog loop β auto-restarts any crashed process βββββββββββββββββββ
|
| 187 |
+
try:
|
| 188 |
+
while True:
|
| 189 |
+
for label, script in restart_map.items():
|
| 190 |
+
proc = processes[label]
|
| 191 |
+
if proc is not None and proc.poll() is not None:
|
| 192 |
+
exit_code = proc.returncode
|
| 193 |
+
print(f"\nβ οΈ [{label}] exited with code {exit_code}")
|
| 194 |
+
print(" Restarting in 3 sβ¦")
|
| 195 |
+
time.sleep(3)
|
| 196 |
+
processes[label] = start_process(script, label)
|
| 197 |
+
print()
|
| 198 |
+
time.sleep(3)
|
| 199 |
+
|
| 200 |
+
except KeyboardInterrupt:
|
| 201 |
+
stop_all(processes)
|
| 202 |
+
sys.exit(0)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 206 |
+
if __name__ == "__main__":
|
| 207 |
+
multiprocessing.freeze_support() # no-op on Linux/Mac, required on Win
|
| 208 |
+
main()
|