Deploy SignLanguageTranslate with Transformer evaluation
Browse files- .dockerignore +8 -0
- .gitattributes +3 -35
- Dockerfile +23 -14
- ISL_Model_parameter.py +662 -0
- README.md +9 -10
- app.py +694 -0
- categories_processed.png +0 -0
- data/include_validation.json.gz +3 -0
- data/transformer_contract.json +278 -0
- eda/train_test_validation_split-1.png +3 -0
- eda/train_test_validation_split-2.png +3 -0
- expression_mapping.py +168 -0
- model.py +407 -0
- packages.txt +6 -0
- requirements-docker.txt +12 -0
- requirements.txt +18 -2
- transformer_comparison.py +186 -0
- transformer_inference.py +317 -0
- transformer_runtime.py +123 -0
- util.py +455 -0
- validation_playback.html +78 -0
.dockerignore
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.git
|
| 2 |
+
**/__pycache__
|
| 3 |
+
**/*.pyc
|
| 4 |
+
.env
|
| 5 |
+
.env.*
|
| 6 |
+
private-assets
|
| 7 |
+
*.safetensors
|
| 8 |
+
*.mobileprovision
|
.gitattributes
CHANGED
|
@@ -1,35 +1,3 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
eda/train_test_validation_split-1.png filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
eda/train_test_validation_split-2.png filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
data/include_validation.json.gz filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Dockerfile
CHANGED
|
@@ -1,20 +1,29 @@
|
|
| 1 |
-
FROM python:3.
|
| 2 |
|
| 3 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
-
RUN apt-get update
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
&& rm -rf /var/lib/apt/lists/*
|
| 10 |
|
| 11 |
-
|
| 12 |
-
|
| 13 |
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
-
|
|
|
|
| 17 |
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"]
|
|
|
|
| 1 |
+
FROM python:3.11-slim-bookworm
|
| 2 |
|
| 3 |
+
ENV PYTHONUNBUFFERED=1 \
|
| 4 |
+
PYTHONDONTWRITEBYTECODE=1 \
|
| 5 |
+
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
| 6 |
+
HOME=/home/user \
|
| 7 |
+
PATH=/home/user/.local/bin:$PATH \
|
| 8 |
+
KERAS_BACKEND=torch \
|
| 9 |
+
OPENBLAS_NUM_THREADS=1 \
|
| 10 |
+
OMP_NUM_THREADS=1
|
| 11 |
|
| 12 |
+
RUN apt-get update \
|
| 13 |
+
&& apt-get install -y --no-install-recommends ffmpeg \
|
| 14 |
+
&& rm -rf /var/lib/apt/lists/* \
|
| 15 |
+
&& useradd -m -u 1000 user
|
|
|
|
| 16 |
|
| 17 |
+
USER user
|
| 18 |
+
WORKDIR /home/user/app
|
| 19 |
|
| 20 |
+
COPY --chown=user:user requirements-docker.txt ./
|
| 21 |
+
RUN python -m pip install --user --no-cache-dir \
|
| 22 |
+
--index-url https://download.pytorch.org/whl/cpu torch==2.6.0 \
|
| 23 |
+
&& python -m pip install --user --no-cache-dir -r requirements-docker.txt \
|
| 24 |
+
&& python -m pip check
|
| 25 |
|
| 26 |
+
COPY --chown=user:user . .
|
| 27 |
+
EXPOSE 7860
|
| 28 |
|
| 29 |
+
CMD ["streamlit", "run", "app.py", "--server.address=0.0.0.0", "--server.port=7860", "--server.headless=true", "--browser.gatherUsageStats=false"]
|
|
|
|
|
|
ISL_Model_parameter.py
ADDED
|
@@ -0,0 +1,662 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import keras
|
| 2 |
+
from keras.layers import TorchModuleWrapper
|
| 3 |
+
import numpy as np
|
| 4 |
+
import cv2
|
| 5 |
+
import torch
|
| 6 |
+
from scipy.ndimage.filters import gaussian_filter
|
| 7 |
+
import math
|
| 8 |
+
import os
|
| 9 |
+
import numpy as np
|
| 10 |
+
from skimage.measure import label
|
| 11 |
+
import util as util
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class ISLSignPos(keras.Model):
|
| 15 |
+
def __init__(self,pt_body_model,pt_hand_model):
|
| 16 |
+
super().__init__()
|
| 17 |
+
self.pt_body = TorchModuleWrapper(pt_body_model)
|
| 18 |
+
self.pt_body.trainable=False
|
| 19 |
+
self.pt_hand = TorchModuleWrapper(pt_hand_model)
|
| 20 |
+
self.pt_hand.trainable=False
|
| 21 |
+
self.njoint_body = 26
|
| 22 |
+
self.npaf_body = 52
|
| 23 |
+
|
| 24 |
+
def call(self, oriImg):
|
| 25 |
+
candidate, subset = self.bodypos(oriImg.cpu().numpy())
|
| 26 |
+
hands_list = util.handDetect(candidate, subset, oriImg.cpu().numpy())
|
| 27 |
+
all_hand_peaks = []
|
| 28 |
+
for x, y, w, is_left in hands_list:
|
| 29 |
+
peaks = self.handpos(oriImg.cpu().numpy()[y:y+w, x:x+w, :])
|
| 30 |
+
peaks[:, 0] = np.where(peaks[:, 0]==0, peaks[:, 0], peaks[:, 0]+x)
|
| 31 |
+
peaks[:, 1] = np.where(peaks[:, 1]==0, peaks[:, 1], peaks[:, 1]+y)
|
| 32 |
+
all_hand_peaks.append(peaks)
|
| 33 |
+
return (candidate, subset,all_hand_peaks)
|
| 34 |
+
|
| 35 |
+
def bodypos(self, oriImg):
|
| 36 |
+
model_type = 'body25'
|
| 37 |
+
scale_search = [0.5]
|
| 38 |
+
boxsize = 368
|
| 39 |
+
stride = 8
|
| 40 |
+
padValue = 128
|
| 41 |
+
thre1 = 0.1
|
| 42 |
+
thre2 = 0.05
|
| 43 |
+
multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search]
|
| 44 |
+
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], self.njoint_body))
|
| 45 |
+
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], self.npaf_body))
|
| 46 |
+
|
| 47 |
+
for m in range(len(multiplier)):
|
| 48 |
+
scale = multiplier[m]
|
| 49 |
+
imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
|
| 50 |
+
imageToTest_padded, pad = util.padRightDownCorner(imageToTest, stride, padValue)
|
| 51 |
+
im = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
|
| 52 |
+
im = np.ascontiguousarray(im)
|
| 53 |
+
|
| 54 |
+
data = torch.from_numpy(im).float()
|
| 55 |
+
if torch.cuda.is_available():
|
| 56 |
+
data = data.cuda()
|
| 57 |
+
with torch.no_grad():
|
| 58 |
+
Mconv7_stage6_L1, Mconv7_stage6_L2 = self.pt_body(data)
|
| 59 |
+
Mconv7_stage6_L1 = Mconv7_stage6_L1.cpu().numpy()
|
| 60 |
+
Mconv7_stage6_L2 = Mconv7_stage6_L2.cpu().numpy()
|
| 61 |
+
|
| 62 |
+
heatmap = np.transpose(np.squeeze(Mconv7_stage6_L2), (1, 2, 0)) # output 1 is heatmaps
|
| 63 |
+
heatmap = cv2.resize(heatmap, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
|
| 64 |
+
heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
|
| 65 |
+
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
|
| 66 |
+
|
| 67 |
+
paf = np.transpose(np.squeeze(Mconv7_stage6_L1), (1, 2, 0)) # output 0 is PAFs
|
| 68 |
+
paf = cv2.resize(paf, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
|
| 69 |
+
paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
|
| 70 |
+
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
|
| 71 |
+
|
| 72 |
+
heatmap_avg += heatmap_avg + heatmap / len(multiplier)
|
| 73 |
+
paf_avg += + paf / len(multiplier)
|
| 74 |
+
|
| 75 |
+
all_peaks = []
|
| 76 |
+
peak_counter = 0
|
| 77 |
+
|
| 78 |
+
for part in range(self.njoint_body-1):
|
| 79 |
+
map_ori = heatmap_avg[:, :, part]
|
| 80 |
+
one_heatmap = gaussian_filter(map_ori, sigma=3)
|
| 81 |
+
|
| 82 |
+
map_left = np.zeros(one_heatmap.shape)
|
| 83 |
+
map_left[1:, :] = one_heatmap[:-1, :]
|
| 84 |
+
map_right = np.zeros(one_heatmap.shape)
|
| 85 |
+
map_right[:-1, :] = one_heatmap[1:, :]
|
| 86 |
+
map_up = np.zeros(one_heatmap.shape)
|
| 87 |
+
map_up[:, 1:] = one_heatmap[:, :-1]
|
| 88 |
+
map_down = np.zeros(one_heatmap.shape)
|
| 89 |
+
map_down[:, :-1] = one_heatmap[:, 1:]
|
| 90 |
+
|
| 91 |
+
peaks_binary = np.logical_and.reduce(
|
| 92 |
+
(one_heatmap >= map_left, one_heatmap >= map_right, one_heatmap >= map_up, one_heatmap >= map_down, one_heatmap > thre1))
|
| 93 |
+
peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
|
| 94 |
+
peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks]
|
| 95 |
+
peak_id = range(peak_counter, peak_counter + len(peaks))
|
| 96 |
+
peaks_with_score_and_id = [peaks_with_score[i] + (peak_id[i],) for i in range(len(peak_id))]
|
| 97 |
+
|
| 98 |
+
all_peaks.append(peaks_with_score_and_id)
|
| 99 |
+
peak_counter += len(peaks)
|
| 100 |
+
|
| 101 |
+
if model_type=='body25':
|
| 102 |
+
# find connection in the specified sequence, center 29 is in the position 15
|
| 103 |
+
limbSeq = [[1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],\
|
| 104 |
+
[10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],\
|
| 105 |
+
[11,24],[11,22],[14,21],[14,19],[22,23],[19,20]]
|
| 106 |
+
# the middle joints heatmap correpondence
|
| 107 |
+
mapIdx = [[30, 31],[14, 15],[16, 17],[18, 19],[22, 23],[24, 25],[26, 27],[0, 1],[6, 7],\
|
| 108 |
+
[2, 3],[4, 5], [8, 9],[10, 11],[12, 13],[32, 33],[34, 35],[36,37],[38,39],\
|
| 109 |
+
[50,51],[46,47],[44,45],[40,41],[48,49],[42,43]]
|
| 110 |
+
else:
|
| 111 |
+
# find connection in the specified sequence, center 29 is in the position 15
|
| 112 |
+
limbSeq = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], \
|
| 113 |
+
[9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], \
|
| 114 |
+
[0, 15], [15, 17], [2, 16], [5, 17]]
|
| 115 |
+
# the middle joints heatmap correpondence
|
| 116 |
+
mapIdx = [[12, 13], [20, 21], [14, 15], [16, 17], [22, 23], [24, 25], [0, 1], [2, 3], \
|
| 117 |
+
[4, 5], [6, 7], [8, 9], [10, 11], [28, 29], [30, 31], [34, 35], [32, 33], \
|
| 118 |
+
[36, 37], [18, 19], [26, 27]]
|
| 119 |
+
|
| 120 |
+
connection_all = []
|
| 121 |
+
special_k = []
|
| 122 |
+
mid_num = 10
|
| 123 |
+
|
| 124 |
+
for k in range(len(mapIdx)):
|
| 125 |
+
score_mid = paf_avg[:, :, mapIdx[k]]
|
| 126 |
+
candA = all_peaks[limbSeq[k][0]]
|
| 127 |
+
candB = all_peaks[limbSeq[k][1]]
|
| 128 |
+
|
| 129 |
+
nA = len(candA)
|
| 130 |
+
nB = len(candB)
|
| 131 |
+
indexA, indexB = limbSeq[k]
|
| 132 |
+
if (nA != 0 and nB != 0):
|
| 133 |
+
connection_candidate = []
|
| 134 |
+
for i in range(nA):
|
| 135 |
+
for j in range(nB):
|
| 136 |
+
vec = np.subtract(candB[j][:2], candA[i][:2])
|
| 137 |
+
norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
|
| 138 |
+
norm = max(0.001, norm)
|
| 139 |
+
vec = np.divide(vec, norm)
|
| 140 |
+
|
| 141 |
+
startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
|
| 142 |
+
np.linspace(candA[i][1], candB[j][1], num=mid_num)))
|
| 143 |
+
|
| 144 |
+
vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
|
| 145 |
+
for I in range(len(startend))])
|
| 146 |
+
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
|
| 147 |
+
for I in range(len(startend))])
|
| 148 |
+
|
| 149 |
+
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
|
| 150 |
+
score_with_dist_prior = sum(score_midpts) / len(score_midpts) + min(
|
| 151 |
+
0.5 * oriImg.shape[0] / norm - 1, 0)
|
| 152 |
+
criterion1 = len(np.nonzero(score_midpts > thre2)[0]) > 0.8 * len(score_midpts)
|
| 153 |
+
criterion2 = score_with_dist_prior > 0
|
| 154 |
+
if criterion1 and criterion2:
|
| 155 |
+
connection_candidate.append(
|
| 156 |
+
[i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2]])
|
| 157 |
+
|
| 158 |
+
connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
|
| 159 |
+
connection = np.zeros((0, 5))
|
| 160 |
+
for c in range(len(connection_candidate)):
|
| 161 |
+
i, j, s = connection_candidate[c][0:3]
|
| 162 |
+
if (i not in connection[:, 3] and j not in connection[:, 4]):
|
| 163 |
+
connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
|
| 164 |
+
if (len(connection) >= min(nA, nB)):
|
| 165 |
+
break
|
| 166 |
+
|
| 167 |
+
connection_all.append(connection)
|
| 168 |
+
else:
|
| 169 |
+
special_k.append(k)
|
| 170 |
+
connection_all.append([])
|
| 171 |
+
|
| 172 |
+
# last number in each row is the total parts number of that person
|
| 173 |
+
# the second last number in each row is the score of the overall configuration
|
| 174 |
+
subset = -1 * np.ones((0, self.njoint_body+1))
|
| 175 |
+
candidate = np.array([item for sublist in all_peaks for item in sublist])
|
| 176 |
+
|
| 177 |
+
for k in range(len(mapIdx)):
|
| 178 |
+
if k not in special_k:
|
| 179 |
+
partAs = connection_all[k][:, 0]
|
| 180 |
+
partBs = connection_all[k][:, 1]
|
| 181 |
+
indexA, indexB = np.array(limbSeq[k])
|
| 182 |
+
|
| 183 |
+
for i in range(len(connection_all[k])): # = 1:size(temp,1)
|
| 184 |
+
found = 0
|
| 185 |
+
subset_idx = [-1, -1]
|
| 186 |
+
for j in range(len(subset)): # 1:size(subset,1):
|
| 187 |
+
if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
|
| 188 |
+
subset_idx[found] = j
|
| 189 |
+
found += 1
|
| 190 |
+
|
| 191 |
+
if found == 1:
|
| 192 |
+
j = subset_idx[0]
|
| 193 |
+
if subset[j][indexB] != partBs[i]:
|
| 194 |
+
subset[j][indexB] = partBs[i]
|
| 195 |
+
subset[j][-1] += 1
|
| 196 |
+
subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
|
| 197 |
+
elif found == 2: # if found 2 and disjoint, merge them
|
| 198 |
+
j1, j2 = subset_idx
|
| 199 |
+
membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2]
|
| 200 |
+
if len(np.nonzero(membership == 2)[0]) == 0: # merge
|
| 201 |
+
subset[j1][:-2] += (subset[j2][:-2] + 1)
|
| 202 |
+
subset[j1][-2:] += subset[j2][-2:]
|
| 203 |
+
subset[j1][-2] += connection_all[k][i][2]
|
| 204 |
+
subset = np.delete(subset, j2, 0)
|
| 205 |
+
else: # as like found == 1
|
| 206 |
+
subset[j1][indexB] = partBs[i]
|
| 207 |
+
subset[j1][-1] += 1
|
| 208 |
+
subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
|
| 209 |
+
|
| 210 |
+
# if find no partA in the subset, create a new subset
|
| 211 |
+
elif not found and k < self.njoint_body-2:
|
| 212 |
+
row = -1 * np.ones(self.njoint_body+1)
|
| 213 |
+
row[indexA] = partAs[i]
|
| 214 |
+
row[indexB] = partBs[i]
|
| 215 |
+
row[-1] = 2
|
| 216 |
+
row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2]
|
| 217 |
+
subset = np.vstack([subset, row])
|
| 218 |
+
# delete some rows of subset which has few parts occur
|
| 219 |
+
deleteIdx = []
|
| 220 |
+
for i in range(len(subset)):
|
| 221 |
+
if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
|
| 222 |
+
deleteIdx.append(i)
|
| 223 |
+
subset = np.delete(subset, deleteIdx, axis=0)
|
| 224 |
+
|
| 225 |
+
# subset: n*20 array, 0-17 is the index in candidate, 18 is the total score, 19 is the total parts
|
| 226 |
+
# candidate: x, y, score, id
|
| 227 |
+
return candidate, subset
|
| 228 |
+
|
| 229 |
+
def handpos(self, oriImg):
|
| 230 |
+
scale_search = [0.5, 1.0, 1.5, 2.0]
|
| 231 |
+
# scale_search = [0.5]
|
| 232 |
+
boxsize = 368
|
| 233 |
+
stride = 8
|
| 234 |
+
padValue = 128
|
| 235 |
+
thre = 0.05
|
| 236 |
+
multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search]
|
| 237 |
+
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 22))
|
| 238 |
+
# paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
|
| 239 |
+
|
| 240 |
+
for m in range(len(multiplier)):
|
| 241 |
+
scale = multiplier[m]
|
| 242 |
+
imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
|
| 243 |
+
imageToTest_padded, pad = util.padRightDownCorner(imageToTest, stride, padValue)
|
| 244 |
+
im = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
|
| 245 |
+
im = np.ascontiguousarray(im)
|
| 246 |
+
|
| 247 |
+
data = torch.from_numpy(im).float()
|
| 248 |
+
if torch.cuda.is_available():
|
| 249 |
+
data = data.cuda()
|
| 250 |
+
# data = data.permute([2, 0, 1]).unsqueeze(0).float()
|
| 251 |
+
with torch.no_grad():
|
| 252 |
+
output = self.pt_hand(data).cpu().numpy()
|
| 253 |
+
# output = self.model(data).numpy()q
|
| 254 |
+
|
| 255 |
+
# extract outputs, resize, and remove padding
|
| 256 |
+
heatmap = np.transpose(np.squeeze(output), (1, 2, 0)) # output 1 is heatmaps
|
| 257 |
+
heatmap = cv2.resize(heatmap, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
|
| 258 |
+
heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
|
| 259 |
+
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
|
| 260 |
+
|
| 261 |
+
heatmap_avg += heatmap / len(multiplier)
|
| 262 |
+
|
| 263 |
+
all_peaks = []
|
| 264 |
+
for part in range(21):
|
| 265 |
+
map_ori = heatmap_avg[:, :, part]
|
| 266 |
+
one_heatmap = gaussian_filter(map_ori, sigma=3)
|
| 267 |
+
binary = np.ascontiguousarray(one_heatmap > thre, dtype=np.uint8)
|
| 268 |
+
# 全部小于阈值
|
| 269 |
+
if np.sum(binary) == 0:
|
| 270 |
+
all_peaks.append([0, 0])
|
| 271 |
+
continue
|
| 272 |
+
label_img, label_numbers = label(binary, return_num=True, connectivity=binary.ndim)
|
| 273 |
+
max_index = np.argmax([np.sum(map_ori[label_img == i]) for i in range(1, label_numbers + 1)]) + 1
|
| 274 |
+
label_img[label_img != max_index] = 0
|
| 275 |
+
map_ori[label_img == 0] = 0
|
| 276 |
+
|
| 277 |
+
y, x = util.npmax(map_ori)
|
| 278 |
+
all_peaks.append([x, y])
|
| 279 |
+
return np.array(all_peaks)
|
| 280 |
+
|
| 281 |
+
class ISLSignPosTranslator(keras.Model):
|
| 282 |
+
|
| 283 |
+
def __init__(self,body_model,hand_model, translation_model):
|
| 284 |
+
super().__init__()
|
| 285 |
+
self.pt_body = TorchModuleWrapper(body_model)
|
| 286 |
+
self.pt_body.trainable=False
|
| 287 |
+
self.pt_hand = TorchModuleWrapper(hand_model)
|
| 288 |
+
self.pt_hand.trainable=False
|
| 289 |
+
|
| 290 |
+
self.njoint_body = 26
|
| 291 |
+
self.npaf_body = 52
|
| 292 |
+
self.model_type='body25'
|
| 293 |
+
self.translation_layer=translation_model
|
| 294 |
+
|
| 295 |
+
def call(self, window):
|
| 296 |
+
window_size=20
|
| 297 |
+
window_features=[]
|
| 298 |
+
blank_frame=np.zeros((1,156))
|
| 299 |
+
for idx,frame in enumerate(window.cpu()):
|
| 300 |
+
|
| 301 |
+
# frame=frame.cpu().numpy()[:, :, ::-1]
|
| 302 |
+
candidate, subset = self.bodypos(frame.cpu().numpy())
|
| 303 |
+
hands_list = util.handDetect(candidate, subset, frame.cpu().numpy())
|
| 304 |
+
all_hand_peaks = []
|
| 305 |
+
for x, y, w, is_left in hands_list:
|
| 306 |
+
peaks = self.handpos(frame.cpu().numpy()[y:y+w, x:x+w, :])
|
| 307 |
+
peaks[:, 0] = np.where(peaks[:, 0]==0, peaks[:, 0], peaks[:, 0]+x)
|
| 308 |
+
peaks[:, 1] = np.where(peaks[:, 1]==0, peaks[:, 1], peaks[:, 1]+y)
|
| 309 |
+
all_hand_peaks.append(peaks)
|
| 310 |
+
|
| 311 |
+
(bodypose_circles,bodypose_sticks,)=util.get_bodypose(candidate, subset, self.model_type)
|
| 312 |
+
(handpose_edges,handpose_peaks)=util.get_handpose(all_hand_peaks,)
|
| 313 |
+
|
| 314 |
+
feature=self.populate_features(bodypose_circles,handpose_peaks)
|
| 315 |
+
window_features.append(feature)
|
| 316 |
+
|
| 317 |
+
if len(window_features)<window_size:
|
| 318 |
+
for _ in range(0,(window_size-window_features.shape[0])):
|
| 319 |
+
window_features.append(blank_frame)
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
# self.writer.close()
|
| 323 |
+
# timeseries=self.create_timeseries_data(feature)
|
| 324 |
+
# self.frame_to_window(feature)
|
| 325 |
+
# np.savetxt('C:/Users/spsar/OneDrive/Desktop/test/MVI_9590.MOV.window1.numpy',np.array(window_features))
|
| 326 |
+
return self.translation_layer(np.array(window_features).reshape(1,20,156))
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def frame_to_window(self,frame):
|
| 330 |
+
"""
|
| 331 |
+
Converts a single frame to a rolling window array with zero padding.
|
| 332 |
+
|
| 333 |
+
Args:
|
| 334 |
+
frame: A numpy array representing a video frame.
|
| 335 |
+
window_size: The size of the rolling window (default: 20).
|
| 336 |
+
window (optional): An existing window array to add the frame to
|
| 337 |
+
(useful for maintaining rolling window state).
|
| 338 |
+
|
| 339 |
+
Returns:
|
| 340 |
+
A numpy array representing the rolling window with the added frame.
|
| 341 |
+
"""
|
| 342 |
+
|
| 343 |
+
# Shift the window elements by 1 (oldest frame is dropped)
|
| 344 |
+
self.window[:-1] = self.window[1:]
|
| 345 |
+
|
| 346 |
+
# Add the new frame to the end of the window
|
| 347 |
+
self.window[-1] = frame
|
| 348 |
+
|
| 349 |
+
def populate_features(self,bodypose_circles,handpose_peaks):
|
| 350 |
+
# X_body_test = [f'bodypeaks_x_{i}' for i in range(15)] + [f'bodypeaks_y_{i}' for i in range(15)]
|
| 351 |
+
# X_hand0_test = [f'hand0peaks_x_{i}' for i in range(21)] + [f'hand0peaks_y_{i}' for i in range(21)] + [f'hand0peaks_peaktxt{i}' for i in range(21)]
|
| 352 |
+
# X_hand1_test = [f'hand1peaks_x_{i}' for i in range(21)] + [f'hand1peaks_y_{i}' for i in range(21)] + [f'hand1peaks_peaktxt{i}' for i in range(21)]
|
| 353 |
+
|
| 354 |
+
# feature_columns_new = X_body_test + X_hand0_test + X_hand1_test
|
| 355 |
+
feature=[]
|
| 356 |
+
for idx in range(15):
|
| 357 |
+
if(idx<len(bodypose_circles)):
|
| 358 |
+
feature.append(bodypose_circles[idx][0])
|
| 359 |
+
else:
|
| 360 |
+
feature.append(0)
|
| 361 |
+
|
| 362 |
+
for idx in range(15):
|
| 363 |
+
if(idx<len(bodypose_circles)):
|
| 364 |
+
feature.append(bodypose_circles[idx][1])
|
| 365 |
+
else:
|
| 366 |
+
feature.append(0)
|
| 367 |
+
|
| 368 |
+
for hand_idx in range(2):
|
| 369 |
+
for idx in range(21):
|
| 370 |
+
if(idx<len(handpose_peaks[hand_idx])):
|
| 371 |
+
feature.append(float(handpose_peaks[hand_idx][idx][0]))
|
| 372 |
+
else:
|
| 373 |
+
feature.append(0)
|
| 374 |
+
|
| 375 |
+
for idx in range(21):
|
| 376 |
+
if(idx<len(handpose_peaks[hand_idx])):
|
| 377 |
+
feature.append(float(handpose_peaks[hand_idx][idx][1]))
|
| 378 |
+
else:
|
| 379 |
+
feature.append(0)
|
| 380 |
+
|
| 381 |
+
for idx in range(21):
|
| 382 |
+
if(idx<len(handpose_peaks[hand_idx])):
|
| 383 |
+
feature.append(float(handpose_peaks[hand_idx][idx][2]))
|
| 384 |
+
else:
|
| 385 |
+
feature.append(0)
|
| 386 |
+
|
| 387 |
+
# for idx in range(21):
|
| 388 |
+
# if(idx<len(handpose_peaks[1])):
|
| 389 |
+
# feature.append(handpose_peaks[1][idx][0])
|
| 390 |
+
# else:
|
| 391 |
+
# feature.append(0)
|
| 392 |
+
|
| 393 |
+
# for idx in range(21):
|
| 394 |
+
# if(idx<len(handpose_peaks[1])):
|
| 395 |
+
# feature.append(handpose_peaks[1][idx][1])
|
| 396 |
+
# else:
|
| 397 |
+
# feature.append(0)
|
| 398 |
+
|
| 399 |
+
# for idx in range(21):
|
| 400 |
+
# if(idx<len(handpose_peaks[1])):
|
| 401 |
+
# feature.append(handpose_peaks[1][idx][2])
|
| 402 |
+
# else:
|
| 403 |
+
# feature.append(0)
|
| 404 |
+
|
| 405 |
+
# for idx,handedges in enumerate(handpose_edges):
|
| 406 |
+
# for (peaktxt, (handedge_x1, handedge_y1), (handedge_x2, handedge_y2)) in handedges:
|
| 407 |
+
# feature[f'hand{idx}edge_x1_{peaktxt}']=handedge_x1
|
| 408 |
+
# feature[f'hand{idx}edge_y1_{peaktxt}']=handedge_y1
|
| 409 |
+
# feature[f'hand{idx}edge_x2_{peaktxt}']=handedge_x2
|
| 410 |
+
# feature[f'hand{idx}edge_y2_{peaktxt}']=handedge_y2
|
| 411 |
+
|
| 412 |
+
X=np.array(feature)
|
| 413 |
+
# time_steps = 12 # Number of time steps
|
| 414 |
+
# num_features = X.shape[0] // time_steps # Number of features per time step
|
| 415 |
+
# X_reshaped = X.reshape(1,time_steps, num_features)
|
| 416 |
+
return X
|
| 417 |
+
|
| 418 |
+
def bodypos(self, oriImg):
|
| 419 |
+
model_type = 'body25'
|
| 420 |
+
scale_search = [0.5]
|
| 421 |
+
boxsize = 368
|
| 422 |
+
stride = 8
|
| 423 |
+
padValue = 128
|
| 424 |
+
thre1 = 0.1
|
| 425 |
+
thre2 = 0.05
|
| 426 |
+
multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search]
|
| 427 |
+
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], self.njoint_body))
|
| 428 |
+
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], self.npaf_body))
|
| 429 |
+
|
| 430 |
+
for m in range(len(multiplier)):
|
| 431 |
+
scale = multiplier[m]
|
| 432 |
+
imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
|
| 433 |
+
imageToTest_padded, pad = util.padRightDownCorner(imageToTest, stride, padValue)
|
| 434 |
+
im = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
|
| 435 |
+
im = np.ascontiguousarray(im)
|
| 436 |
+
|
| 437 |
+
data = torch.from_numpy(im).float()
|
| 438 |
+
if torch.cuda.is_available():
|
| 439 |
+
data = data.cuda()
|
| 440 |
+
with torch.no_grad():
|
| 441 |
+
Mconv7_stage6_L1, Mconv7_stage6_L2 = self.pt_body(data)
|
| 442 |
+
Mconv7_stage6_L1 = Mconv7_stage6_L1.cpu().numpy()
|
| 443 |
+
Mconv7_stage6_L2 = Mconv7_stage6_L2.cpu().numpy()
|
| 444 |
+
|
| 445 |
+
heatmap = np.transpose(np.squeeze(Mconv7_stage6_L2), (1, 2, 0)) # output 1 is heatmaps
|
| 446 |
+
heatmap = cv2.resize(heatmap, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
|
| 447 |
+
heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
|
| 448 |
+
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
|
| 449 |
+
|
| 450 |
+
paf = np.transpose(np.squeeze(Mconv7_stage6_L1), (1, 2, 0)) # output 0 is PAFs
|
| 451 |
+
paf = cv2.resize(paf, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
|
| 452 |
+
paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
|
| 453 |
+
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
|
| 454 |
+
|
| 455 |
+
heatmap_avg += heatmap_avg + heatmap / len(multiplier)
|
| 456 |
+
paf_avg += + paf / len(multiplier)
|
| 457 |
+
|
| 458 |
+
all_peaks = []
|
| 459 |
+
peak_counter = 0
|
| 460 |
+
|
| 461 |
+
for part in range(self.njoint_body-1):
|
| 462 |
+
map_ori = heatmap_avg[:, :, part]
|
| 463 |
+
one_heatmap = gaussian_filter(map_ori, sigma=3)
|
| 464 |
+
|
| 465 |
+
map_left = np.zeros(one_heatmap.shape)
|
| 466 |
+
map_left[1:, :] = one_heatmap[:-1, :]
|
| 467 |
+
map_right = np.zeros(one_heatmap.shape)
|
| 468 |
+
map_right[:-1, :] = one_heatmap[1:, :]
|
| 469 |
+
map_up = np.zeros(one_heatmap.shape)
|
| 470 |
+
map_up[:, 1:] = one_heatmap[:, :-1]
|
| 471 |
+
map_down = np.zeros(one_heatmap.shape)
|
| 472 |
+
map_down[:, :-1] = one_heatmap[:, 1:]
|
| 473 |
+
|
| 474 |
+
peaks_binary = np.logical_and.reduce(
|
| 475 |
+
(one_heatmap >= map_left, one_heatmap >= map_right, one_heatmap >= map_up, one_heatmap >= map_down, one_heatmap > thre1))
|
| 476 |
+
peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
|
| 477 |
+
peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks]
|
| 478 |
+
peak_id = range(peak_counter, peak_counter + len(peaks))
|
| 479 |
+
peaks_with_score_and_id = [peaks_with_score[i] + (peak_id[i],) for i in range(len(peak_id))]
|
| 480 |
+
|
| 481 |
+
all_peaks.append(peaks_with_score_and_id)
|
| 482 |
+
peak_counter += len(peaks)
|
| 483 |
+
|
| 484 |
+
if model_type=='body25':
|
| 485 |
+
# find connection in the specified sequence, center 29 is in the position 15
|
| 486 |
+
limbSeq = [[1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],\
|
| 487 |
+
[10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],\
|
| 488 |
+
[11,24],[11,22],[14,21],[14,19],[22,23],[19,20]]
|
| 489 |
+
# the middle joints heatmap correpondence
|
| 490 |
+
mapIdx = [[30, 31],[14, 15],[16, 17],[18, 19],[22, 23],[24, 25],[26, 27],[0, 1],[6, 7],\
|
| 491 |
+
[2, 3],[4, 5], [8, 9],[10, 11],[12, 13],[32, 33],[34, 35],[36,37],[38,39],\
|
| 492 |
+
[50,51],[46,47],[44,45],[40,41],[48,49],[42,43]]
|
| 493 |
+
else:
|
| 494 |
+
# find connection in the specified sequence, center 29 is in the position 15
|
| 495 |
+
limbSeq = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], \
|
| 496 |
+
[9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], \
|
| 497 |
+
[0, 15], [15, 17], [2, 16], [5, 17]]
|
| 498 |
+
# the middle joints heatmap correpondence
|
| 499 |
+
mapIdx = [[12, 13], [20, 21], [14, 15], [16, 17], [22, 23], [24, 25], [0, 1], [2, 3], \
|
| 500 |
+
[4, 5], [6, 7], [8, 9], [10, 11], [28, 29], [30, 31], [34, 35], [32, 33], \
|
| 501 |
+
[36, 37], [18, 19], [26, 27]]
|
| 502 |
+
|
| 503 |
+
connection_all = []
|
| 504 |
+
special_k = []
|
| 505 |
+
mid_num = 10
|
| 506 |
+
|
| 507 |
+
for k in range(len(mapIdx)):
|
| 508 |
+
score_mid = paf_avg[:, :, mapIdx[k]]
|
| 509 |
+
candA = all_peaks[limbSeq[k][0]]
|
| 510 |
+
candB = all_peaks[limbSeq[k][1]]
|
| 511 |
+
|
| 512 |
+
nA = len(candA)
|
| 513 |
+
nB = len(candB)
|
| 514 |
+
indexA, indexB = limbSeq[k]
|
| 515 |
+
if (nA != 0 and nB != 0):
|
| 516 |
+
connection_candidate = []
|
| 517 |
+
for i in range(nA):
|
| 518 |
+
for j in range(nB):
|
| 519 |
+
vec = np.subtract(candB[j][:2], candA[i][:2])
|
| 520 |
+
norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
|
| 521 |
+
norm = max(0.001, norm)
|
| 522 |
+
vec = np.divide(vec, norm)
|
| 523 |
+
|
| 524 |
+
startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
|
| 525 |
+
np.linspace(candA[i][1], candB[j][1], num=mid_num)))
|
| 526 |
+
|
| 527 |
+
vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
|
| 528 |
+
for I in range(len(startend))])
|
| 529 |
+
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
|
| 530 |
+
for I in range(len(startend))])
|
| 531 |
+
|
| 532 |
+
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
|
| 533 |
+
score_with_dist_prior = sum(score_midpts) / len(score_midpts) + min(
|
| 534 |
+
0.5 * oriImg.shape[0] / norm - 1, 0)
|
| 535 |
+
criterion1 = len(np.nonzero(score_midpts > thre2)[0]) > 0.8 * len(score_midpts)
|
| 536 |
+
criterion2 = score_with_dist_prior > 0
|
| 537 |
+
if criterion1 and criterion2:
|
| 538 |
+
connection_candidate.append(
|
| 539 |
+
[i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2]])
|
| 540 |
+
|
| 541 |
+
connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
|
| 542 |
+
connection = np.zeros((0, 5))
|
| 543 |
+
for c in range(len(connection_candidate)):
|
| 544 |
+
i, j, s = connection_candidate[c][0:3]
|
| 545 |
+
if (i not in connection[:, 3] and j not in connection[:, 4]):
|
| 546 |
+
connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
|
| 547 |
+
if (len(connection) >= min(nA, nB)):
|
| 548 |
+
break
|
| 549 |
+
|
| 550 |
+
connection_all.append(connection)
|
| 551 |
+
else:
|
| 552 |
+
special_k.append(k)
|
| 553 |
+
connection_all.append([])
|
| 554 |
+
|
| 555 |
+
# last number in each row is the total parts number of that person
|
| 556 |
+
# the second last number in each row is the score of the overall configuration
|
| 557 |
+
subset = -1 * np.ones((0, self.njoint_body+1))
|
| 558 |
+
candidate = np.array([item for sublist in all_peaks for item in sublist])
|
| 559 |
+
|
| 560 |
+
for k in range(len(mapIdx)):
|
| 561 |
+
if k not in special_k:
|
| 562 |
+
partAs = connection_all[k][:, 0]
|
| 563 |
+
partBs = connection_all[k][:, 1]
|
| 564 |
+
indexA, indexB = np.array(limbSeq[k])
|
| 565 |
+
|
| 566 |
+
for i in range(len(connection_all[k])): # = 1:size(temp,1)
|
| 567 |
+
found = 0
|
| 568 |
+
subset_idx = [-1, -1]
|
| 569 |
+
for j in range(len(subset)): # 1:size(subset,1):
|
| 570 |
+
if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
|
| 571 |
+
subset_idx[found] = j
|
| 572 |
+
found += 1
|
| 573 |
+
|
| 574 |
+
if found == 1:
|
| 575 |
+
j = subset_idx[0]
|
| 576 |
+
if subset[j][indexB] != partBs[i]:
|
| 577 |
+
subset[j][indexB] = partBs[i]
|
| 578 |
+
subset[j][-1] += 1
|
| 579 |
+
subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
|
| 580 |
+
elif found == 2: # if found 2 and disjoint, merge them
|
| 581 |
+
j1, j2 = subset_idx
|
| 582 |
+
membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2]
|
| 583 |
+
if len(np.nonzero(membership == 2)[0]) == 0: # merge
|
| 584 |
+
subset[j1][:-2] += (subset[j2][:-2] + 1)
|
| 585 |
+
subset[j1][-2:] += subset[j2][-2:]
|
| 586 |
+
subset[j1][-2] += connection_all[k][i][2]
|
| 587 |
+
subset = np.delete(subset, j2, 0)
|
| 588 |
+
else: # as like found == 1
|
| 589 |
+
subset[j1][indexB] = partBs[i]
|
| 590 |
+
subset[j1][-1] += 1
|
| 591 |
+
subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
|
| 592 |
+
|
| 593 |
+
# if find no partA in the subset, create a new subset
|
| 594 |
+
elif not found and k < self.njoint_body-2:
|
| 595 |
+
row = -1 * np.ones(self.njoint_body+1)
|
| 596 |
+
row[indexA] = partAs[i]
|
| 597 |
+
row[indexB] = partBs[i]
|
| 598 |
+
row[-1] = 2
|
| 599 |
+
row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2]
|
| 600 |
+
subset = np.vstack([subset, row])
|
| 601 |
+
# delete some rows of subset which has few parts occur
|
| 602 |
+
deleteIdx = []
|
| 603 |
+
for i in range(len(subset)):
|
| 604 |
+
if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
|
| 605 |
+
deleteIdx.append(i)
|
| 606 |
+
subset = np.delete(subset, deleteIdx, axis=0)
|
| 607 |
+
|
| 608 |
+
# subset: n*20 array, 0-17 is the index in candidate, 18 is the total score, 19 is the total parts
|
| 609 |
+
# candidate: x, y, score, id
|
| 610 |
+
return candidate, subset
|
| 611 |
+
|
| 612 |
+
def handpos(self, oriImg):
|
| 613 |
+
scale_search = [0.5, 1.0, 1.5, 2.0]
|
| 614 |
+
# scale_search = [0.5]
|
| 615 |
+
boxsize = 368
|
| 616 |
+
stride = 8
|
| 617 |
+
padValue = 128
|
| 618 |
+
thre = 0.05
|
| 619 |
+
multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search]
|
| 620 |
+
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 22))
|
| 621 |
+
# paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
|
| 622 |
+
|
| 623 |
+
for m in range(len(multiplier)):
|
| 624 |
+
scale = multiplier[m]
|
| 625 |
+
imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
|
| 626 |
+
imageToTest_padded, pad = util.padRightDownCorner(imageToTest, stride, padValue)
|
| 627 |
+
im = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
|
| 628 |
+
im = np.ascontiguousarray(im)
|
| 629 |
+
|
| 630 |
+
data = torch.from_numpy(im).float()
|
| 631 |
+
if torch.cuda.is_available():
|
| 632 |
+
data = data.cuda()
|
| 633 |
+
# data = data.permute([2, 0, 1]).unsqueeze(0).float()
|
| 634 |
+
with torch.no_grad():
|
| 635 |
+
output = self.pt_hand(data).cpu().numpy()
|
| 636 |
+
# output = self.model(data).numpy()q
|
| 637 |
+
|
| 638 |
+
# extract outputs, resize, and remove padding
|
| 639 |
+
heatmap = np.transpose(np.squeeze(output), (1, 2, 0)) # output 1 is heatmaps
|
| 640 |
+
heatmap = cv2.resize(heatmap, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
|
| 641 |
+
heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
|
| 642 |
+
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
|
| 643 |
+
|
| 644 |
+
heatmap_avg += heatmap / len(multiplier)
|
| 645 |
+
|
| 646 |
+
all_peaks = []
|
| 647 |
+
for part in range(21):
|
| 648 |
+
map_ori = heatmap_avg[:, :, part]
|
| 649 |
+
one_heatmap = gaussian_filter(map_ori, sigma=3)
|
| 650 |
+
binary = np.ascontiguousarray(one_heatmap > thre, dtype=np.uint8)
|
| 651 |
+
# 全部小于阈值
|
| 652 |
+
if np.sum(binary) == 0:
|
| 653 |
+
all_peaks.append([0, 0])
|
| 654 |
+
continue
|
| 655 |
+
label_img, label_numbers = label(binary, return_num=True, connectivity=binary.ndim)
|
| 656 |
+
max_index = np.argmax([np.sum(map_ori[label_img == i]) for i in range(1, label_numbers + 1)]) + 1
|
| 657 |
+
label_img[label_img != max_index] = 0
|
| 658 |
+
map_ori[label_img == 0] = 0
|
| 659 |
+
|
| 660 |
+
y, x = util.npmax(map_ori)
|
| 661 |
+
all_peaks.append([x, y])
|
| 662 |
+
return np.array(all_peaks)
|
README.md
CHANGED
|
@@ -1,19 +1,18 @@
|
|
| 1 |
---
|
| 2 |
title: SignLanguageTranslate
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: red
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
| 7 |
-
app_port:
|
| 8 |
-
tags:
|
| 9 |
-
- streamlit
|
| 10 |
pinned: false
|
| 11 |
-
|
| 12 |
---
|
| 13 |
|
| 14 |
-
#
|
| 15 |
|
| 16 |
-
|
| 17 |
|
| 18 |
-
|
| 19 |
-
|
|
|
|
|
|
| 1 |
---
|
| 2 |
title: SignLanguageTranslate
|
| 3 |
+
emoji: 👁
|
| 4 |
colorFrom: red
|
| 5 |
+
colorTo: gray
|
| 6 |
sdk: docker
|
| 7 |
+
app_port: 7860
|
|
|
|
|
|
|
| 8 |
pinned: false
|
| 9 |
+
license: mit
|
| 10 |
---
|
| 11 |
|
| 12 |
+
# SignLanguageTranslate
|
| 13 |
|
| 14 |
+
SignLanguageTranslate aims to support sign languages around the world. This demo currently evaluates Indian Sign Language on isolated clips from INCLUDE.
|
| 15 |
|
| 16 |
+
The Transformer comparison uses archived predictions from the rolling Transformer at epoch 31 and the causal streaming Transformer at epoch 102. The accuracy table uses 720 isolated INCLUDE clips from the saved iPad validation split. Both models use the same clips. The original LSTM demo uses its historical test manifest.
|
| 17 |
+
|
| 18 |
+
Public source videos are matched by category, class and filename to a pinned revision of the [public INCLUDE mirror](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-dataset). The app labels that repository's split separately. The new comparison credits the original [AI4Bharat INCLUDE dataset](https://huggingface.co/datasets/ai4bharat/INCLUDE). The [keypoint dataset card](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-keypoints) describes the Apple Vision input used for the comparison.
|
app.py
ADDED
|
@@ -0,0 +1,694 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#Modified by Augmented Startups 2021
|
| 2 |
+
#Face Landmark User Interface with StreamLit
|
| 3 |
+
#Watch Computer Vision Tutorials at www.augmentedstartups.info/YouTube
|
| 4 |
+
import streamlit as st
|
| 5 |
+
|
| 6 |
+
from transformer_comparison import render_transformer_comparison
|
| 7 |
+
|
| 8 |
+
app_mode = st.sidebar.selectbox(
|
| 9 |
+
"Choose the App mode",
|
| 10 |
+
["Transformer comparison", "Run on Test Videos", "About App"],
|
| 11 |
+
)
|
| 12 |
+
if app_mode == "Transformer comparison":
|
| 13 |
+
render_transformer_comparison()
|
| 14 |
+
st.stop()
|
| 15 |
+
|
| 16 |
+
import os
|
| 17 |
+
os.environ["KERAS_BACKEND"] = "torch"
|
| 18 |
+
import keras
|
| 19 |
+
|
| 20 |
+
import cv2
|
| 21 |
+
import numpy as np
|
| 22 |
+
import tempfile
|
| 23 |
+
import time
|
| 24 |
+
from PIL import Image
|
| 25 |
+
from keras.models import Sequential
|
| 26 |
+
import os
|
| 27 |
+
from keras.models import Sequential
|
| 28 |
+
import pickle
|
| 29 |
+
import keras
|
| 30 |
+
from keras.models import Sequential
|
| 31 |
+
import os
|
| 32 |
+
from keras.layers import LSTM, Dense, Bidirectional, Dropout,Input,BatchNormalization
|
| 33 |
+
from model import handpose_model, bodypose_25_model
|
| 34 |
+
from expression_mapping import expression_mapping
|
| 35 |
+
from ISL_Model_parameter import ISLSignPosTranslator
|
| 36 |
+
import pandas as pd
|
| 37 |
+
import numpy as np
|
| 38 |
+
import ffmpeg
|
| 39 |
+
import subprocess
|
| 40 |
+
from typing import NamedTuple
|
| 41 |
+
import json
|
| 42 |
+
import util
|
| 43 |
+
from huggingface_hub import hf_hub_download
|
| 44 |
+
|
| 45 |
+
import shutil, platform, subprocess
|
| 46 |
+
import uuid
|
| 47 |
+
|
| 48 |
+
st.write("Python:", platform.python_version())
|
| 49 |
+
st.write("FFmpeg exists:", shutil.which("ffmpeg"), "FFprobe:", shutil.which("ffprobe"))
|
| 50 |
+
|
| 51 |
+
try:
|
| 52 |
+
import cv2
|
| 53 |
+
st.write("OpenCV version:", cv2.__version__)
|
| 54 |
+
except Exception as e:
|
| 55 |
+
st.error(f"OpenCV import failed: {e}")
|
| 56 |
+
|
| 57 |
+
try:
|
| 58 |
+
import torch, keras
|
| 59 |
+
st.write("Torch:", torch.__version__, "Keras:", keras.__version__)
|
| 60 |
+
except Exception as e:
|
| 61 |
+
st.error(f"Torch/Keras import failed: {e}")
|
| 62 |
+
|
| 63 |
+
class FFProbeResult(NamedTuple):
|
| 64 |
+
return_code: int
|
| 65 |
+
json: str
|
| 66 |
+
error: str
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def ffprobe(file_path) -> FFProbeResult:
|
| 70 |
+
command_array = ["ffprobe",
|
| 71 |
+
"-v", "quiet",
|
| 72 |
+
"-print_format", "json",
|
| 73 |
+
"-show_format",
|
| 74 |
+
"-show_streams",
|
| 75 |
+
file_path]
|
| 76 |
+
result = subprocess.run(command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
|
| 77 |
+
return FFProbeResult(return_code=result.returncode,
|
| 78 |
+
json=result.stdout,
|
| 79 |
+
error=result.stderr)
|
| 80 |
+
X_body_test = [f'bodypeaks_x_{i}' for i in range(15)] + [f'bodypeaks_y_{i}' for i in range(15)]
|
| 81 |
+
X_hand0_test = [f'hand0peaks_x_{i}' for i in range(21)] + [f'hand0peaks_y_{i}' for i in range(21)] + [f'hand0peaks_peaktxt{i}' for i in range(21)]
|
| 82 |
+
X_hand1_test = [f'hand1peaks_x_{i}' for i in range(21)] + [f'hand1peaks_y_{i}' for i in range(21)] + [f'hand1peaks_peaktxt{i}' for i in range(21)]
|
| 83 |
+
|
| 84 |
+
feature_columns_new = X_body_test + X_hand0_test + X_hand1_test
|
| 85 |
+
label_columns = ['Expression_encoded']
|
| 86 |
+
|
| 87 |
+
@st.cache_resource
|
| 88 |
+
def create_timeseries_data(isl_data,feature_columns,label_columns, window_size=20):
|
| 89 |
+
"""
|
| 90 |
+
Creates timeseries data from a DataFrame with a specified window size
|
| 91 |
+
and padding at the end.
|
| 92 |
+
|
| 93 |
+
Args:
|
| 94 |
+
df (pandas.DataFrame): The input DataFrame.
|
| 95 |
+
window_size (int, optional): The window size for creating timeseries data. Defaults to 20.
|
| 96 |
+
pad_value (any, optional): The value to use for padding at the end. Defaults to None.
|
| 97 |
+
|
| 98 |
+
Returns:
|
| 99 |
+
list: A list of lists, where each inner list represents a window of timeseries data.
|
| 100 |
+
"""
|
| 101 |
+
|
| 102 |
+
# Handle empty DataFrame
|
| 103 |
+
if isl_data.empty:
|
| 104 |
+
return [],[]
|
| 105 |
+
|
| 106 |
+
X=[]
|
| 107 |
+
y=[]
|
| 108 |
+
i=0
|
| 109 |
+
for group, file_df in isl_data.groupby(['Type','Expression_encoded','FileName']):
|
| 110 |
+
expr_types,exprs,filepaths=group
|
| 111 |
+
# print('expr_types,exprs,filepaths',(expr_types,exprs,filepaths))
|
| 112 |
+
# print(type(name))
|
| 113 |
+
# Get the rolling window iterator with padding
|
| 114 |
+
first_frame=np.zeros((1,156))
|
| 115 |
+
for idx,x in enumerate([file_df[i:i+window_size] for i in range(0,file_df.shape[0],1)]):#enumerate(file_df.rolling(window=20, step=20,min_periods=1)):
|
| 116 |
+
# print(f'records processed {idx} of {file_df.shape[0]}')
|
| 117 |
+
# print(f"{filepaths}-Frame#{x['Frame'].values}/{file_df['Frame'].max()}")
|
| 118 |
+
if x.shape[0]<window_size:
|
| 119 |
+
X.append(np.concatenate((np.repeat(first_frame, (window_size-x.shape[0]), axis=0),x[feature_columns].values), axis=0))
|
| 120 |
+
y.append(exprs)
|
| 121 |
+
# print('len(X)',len(X))
|
| 122 |
+
# print('len(y)',len(y))
|
| 123 |
+
continue
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
X.append(x[feature_columns].values)
|
| 127 |
+
y.append(exprs)
|
| 128 |
+
# print('len(X)',len(X))
|
| 129 |
+
# print('len(y)',len(y))
|
| 130 |
+
# if idx>4:
|
| 131 |
+
# break
|
| 132 |
+
|
| 133 |
+
# i=i+1
|
| 134 |
+
# if i>4:
|
| 135 |
+
# break
|
| 136 |
+
|
| 137 |
+
return X,y
|
| 138 |
+
|
| 139 |
+
translation_model=None
|
| 140 |
+
|
| 141 |
+
@st.cache_resource
|
| 142 |
+
def get_translator_model():
|
| 143 |
+
translation_model = Sequential()
|
| 144 |
+
translation_model.add(Input(shape=((20, 156))))
|
| 145 |
+
translation_model.add(keras.layers.Masking(mask_value=0.))
|
| 146 |
+
translation_model.add(BatchNormalization())
|
| 147 |
+
translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2, return_sequences=True)))
|
| 148 |
+
|
| 149 |
+
translation_model.add(Dropout(0.2))
|
| 150 |
+
translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2)))
|
| 151 |
+
|
| 152 |
+
translation_model.add(keras.layers.Activation('elu'))
|
| 153 |
+
translation_model.add(Dense(32, use_bias=False, kernel_initializer='he_normal'))
|
| 154 |
+
|
| 155 |
+
translation_model.add(BatchNormalization())
|
| 156 |
+
translation_model.add(Dropout(0.2))
|
| 157 |
+
translation_model.add(keras.layers.Activation('elu'))
|
| 158 |
+
translation_model.add(Dense(32, kernel_initializer='he_normal',use_bias=False))
|
| 159 |
+
|
| 160 |
+
translation_model.add(BatchNormalization())
|
| 161 |
+
translation_model.add(keras.layers.Activation('elu'))
|
| 162 |
+
translation_model.add(Dropout(0.2))
|
| 163 |
+
translation_model.add(Dense(len(list(expression_mapping.keys())), activation='softmax'))
|
| 164 |
+
model_file = hf_hub_download(
|
| 165 |
+
repo_id="sunilsarolkar/isl-translation-model",
|
| 166 |
+
filename="isl_model_final.keras"
|
| 167 |
+
)
|
| 168 |
+
translation_model.load_weights(model_file)
|
| 169 |
+
translation_model.load_weights(model_file)
|
| 170 |
+
return translation_model
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
testing_cleaned = hf_hub_download(
|
| 174 |
+
repo_id="sunilsarolkar/isl-test-data",
|
| 175 |
+
filename="testing_cleaned.csv",
|
| 176 |
+
repo_type="dataset"
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
testing_df=pd.read_csv(testing_cleaned)
|
| 180 |
+
test_files = hf_hub_download(
|
| 181 |
+
repo_id="sunilsarolkar/isl-test-data",
|
| 182 |
+
filename="test_files.csv",
|
| 183 |
+
repo_type="dataset"
|
| 184 |
+
)
|
| 185 |
+
# test_statistic_df=pd.read_csv('test_statistic.csv')
|
| 186 |
+
test_files_df=pd.read_csv(test_files)
|
| 187 |
+
# mp_drawing = mp.solutions.drawing_utils
|
| 188 |
+
# mp_face_mesh = mp.solutions.face_mesh
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
class Writer():
|
| 192 |
+
def __init__(self, output_file, input_fps, input_framesize, input_pix_fmt,
|
| 193 |
+
input_vcodec):
|
| 194 |
+
# if os.path.exists(output_file):
|
| 195 |
+
# os.remove(output_file)
|
| 196 |
+
self.ff_proc = (
|
| 197 |
+
ffmpeg
|
| 198 |
+
.input('pipe:',
|
| 199 |
+
format='rawvideo',
|
| 200 |
+
pix_fmt="bgr24",
|
| 201 |
+
s='%sx%s'%(input_framesize[1],input_framesize[0]),
|
| 202 |
+
r=input_fps)
|
| 203 |
+
.output(output_file, pix_fmt=input_pix_fmt, vcodec=input_vcodec)
|
| 204 |
+
.overwrite_output()
|
| 205 |
+
.run_async(pipe_stdin=True)
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
def __call__(self, frame):
|
| 209 |
+
self.ff_proc.stdin.write(frame.tobytes())
|
| 210 |
+
|
| 211 |
+
def close(self):
|
| 212 |
+
self.ff_proc.stdin.close()
|
| 213 |
+
self.ff_proc.wait()
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
st.title("SignLanguageTranslate")
|
| 217 |
+
st.header("Indian Sign Language · legacy LSTM")
|
| 218 |
+
|
| 219 |
+
st.markdown(
|
| 220 |
+
"""
|
| 221 |
+
<style>
|
| 222 |
+
[data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
|
| 223 |
+
width: 350px;
|
| 224 |
+
}
|
| 225 |
+
[data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
|
| 226 |
+
width: 350px;
|
| 227 |
+
margin-left: -350px;
|
| 228 |
+
}
|
| 229 |
+
</style>
|
| 230 |
+
""",
|
| 231 |
+
unsafe_allow_html=True,
|
| 232 |
+
)
|
| 233 |
+
|
| 234 |
+
st.sidebar.title('ISL Sign Language Translation using Openpose')
|
| 235 |
+
st.sidebar.subheader('Parameters')
|
| 236 |
+
frame_wise_outputs={}
|
| 237 |
+
|
| 238 |
+
def weighted_average(nums, weights):
|
| 239 |
+
if sum(weights)==0:
|
| 240 |
+
return 0
|
| 241 |
+
return sum(x * y for x, y in zip(nums, weights)) / sum(weights)
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
@st.cache_data
|
| 245 |
+
def image_resize(image, width=None, height=None, inter=cv2.INTER_AREA):
|
| 246 |
+
# initialize the dimensions of the image to be resized and
|
| 247 |
+
# grab the image size
|
| 248 |
+
dim = None
|
| 249 |
+
(h, w) = image.shape[:2]
|
| 250 |
+
|
| 251 |
+
# if both the width and height are None, then return the
|
| 252 |
+
# original image
|
| 253 |
+
if width is None and height is None:
|
| 254 |
+
return image
|
| 255 |
+
|
| 256 |
+
# check to see if the width is None
|
| 257 |
+
if width is None:
|
| 258 |
+
# calculate the ratio of the height and construct the
|
| 259 |
+
# dimensions
|
| 260 |
+
r = height / float(h)
|
| 261 |
+
dim = (int(w * r), height)
|
| 262 |
+
|
| 263 |
+
# otherwise, the height is None
|
| 264 |
+
else:
|
| 265 |
+
# calculate the ratio of the width and construct the
|
| 266 |
+
# dimensions
|
| 267 |
+
r = width / float(w)
|
| 268 |
+
dim = (width, int(h * r))
|
| 269 |
+
|
| 270 |
+
# resize the image
|
| 271 |
+
resized = cv2.resize(image, dim, interpolation=inter)
|
| 272 |
+
|
| 273 |
+
# return the resized image
|
| 274 |
+
return resized
|
| 275 |
+
|
| 276 |
+
if app_mode =='About App':
|
| 277 |
+
st.markdown('In this application we are demonstrating model developed for translating the Indian Sign Language(ISL) using LSTM')
|
| 278 |
+
st.markdown(
|
| 279 |
+
"""
|
| 280 |
+
<style>
|
| 281 |
+
[data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
|
| 282 |
+
width: 400px;
|
| 283 |
+
}
|
| 284 |
+
[data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
|
| 285 |
+
width: 400px;
|
| 286 |
+
margin-left: -400px;
|
| 287 |
+
}
|
| 288 |
+
</style>
|
| 289 |
+
""",
|
| 290 |
+
unsafe_allow_html=True,
|
| 291 |
+
)
|
| 292 |
+
# st.video('https://www.youtube.com/watch?v=FMaNNXgB_5c&ab_channel=AugmentedStartups')
|
| 293 |
+
|
| 294 |
+
tab1, tab2, tab3, tab4 = st.tabs(["Dataset Overview", "Data Pipeline", "Model Architecture", "Training Details"])
|
| 295 |
+
|
| 296 |
+
with tab1:
|
| 297 |
+
st.markdown('''
|
| 298 |
+
# Dataset Used \n
|
| 299 |
+
This model is trained using [INCLUDE](https://zenodo.org/records/4010759) dataset. \n
|
| 300 |
+
|
| 301 |
+
### Key Statistics for the dataset is as follows-
|
| 302 |
+
|
| 303 |
+
+-----------------------+-----------------+
|
| 304 |
+
| Characteristics | INCLUDE-DATASET |
|
| 305 |
+
+-----------------------+-----------------+
|
| 306 |
+
| Categories | 15 |
|
| 307 |
+
| Words | 263 |
|
| 308 |
+
| Videos | 4257 |
|
| 309 |
+
| Avg Videos per class | 16.3 |
|
| 310 |
+
| Avg Video Length | 2.57s |
|
| 311 |
+
| Min Video Length | 1.28s |
|
| 312 |
+
| Max Video Length | 6.16s |
|
| 313 |
+
| Frame Rate | 25fps |
|
| 314 |
+
| Resolution | 1920x1080 |
|
| 315 |
+
+-----------------------+-----------------+
|
| 316 |
+
|
| 317 |
+
#### Size of each category
|
| 318 |
+
|
| 319 |
+
+--------------------+-------------------+------------------+
|
| 320 |
+
| Category | Number of Classes | Number of Videos |
|
| 321 |
+
+--------------------+-------------------+------------------+
|
| 322 |
+
| Adjectives | 59 | 791 |
|
| 323 |
+
| Animals | 8 | 166 |
|
| 324 |
+
| Clothes | 10 | 198 |
|
| 325 |
+
| Colours | 11 | 222 |
|
| 326 |
+
| Days and Time | 22 | 306 |
|
| 327 |
+
| Electronics | 10 | 140 |
|
| 328 |
+
| Greetings | 9 | 185 |
|
| 329 |
+
| Means of Transport | 9 | 186 |
|
| 330 |
+
| Objects at Home | 27 | 379 |
|
| 331 |
+
| Occupations | 16 | 225 |
|
| 332 |
+
| People | 26 | 513 |
|
| 333 |
+
| Places | 19 | 399 |
|
| 334 |
+
| Pronouns | 8 | 168 |
|
| 335 |
+
| Seasons | 6 | 85 |
|
| 336 |
+
| Society | 23 | 324 |
|
| 337 |
+
| | Categories# 263 | Total Videos-4287|
|
| 338 |
+
+--------------------+-------------------+------------------+
|
| 339 |
+
''')
|
| 340 |
+
st.info("💡 **Note:** The dataset used for training contains **1986 processed videos out of 4287**. We processed a limited set of records due to time and compute constraints.")
|
| 341 |
+
|
| 342 |
+
with tab2:
|
| 343 |
+
st.markdown('''
|
| 344 |
+
### Data Pipeline
|
| 345 |
+
The pipeline processes video frames to extract pose and hand landmarks using an OpenPose-like approach.
|
| 346 |
+
|
| 347 |
+
For each frame, the feature extraction process produces a vector of **156 features**, consisting of:
|
| 348 |
+
- **Body Pose**: X and Y coordinates for 15 body keypoints, along with edge lengths and angles.
|
| 349 |
+
- **Hand Pose**: X and Y coordinates for 21 keypoints on each hand (left and right).
|
| 350 |
+
|
| 351 |
+
This structured tabular data is then grouped sequentially into sliding windows to capture the temporal motion of the signs.
|
| 352 |
+
''')
|
| 353 |
+
|
| 354 |
+
with tab3:
|
| 355 |
+
st.markdown('''
|
| 356 |
+
### Model Structure
|
| 357 |
+
The translation model utilizes a sequence-to-sequence architecture based on Bidirectional LSTMs.
|
| 358 |
+
|
| 359 |
+
```python
|
| 360 |
+
translation_model = Sequential()
|
| 361 |
+
translation_model.add(Input(shape=((20, 156))))
|
| 362 |
+
translation_model.add(keras.layers.Masking(mask_value=0.))
|
| 363 |
+
translation_model.add(BatchNormalization())
|
| 364 |
+
translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2, return_sequences=True)))
|
| 365 |
+
|
| 366 |
+
translation_model.add(Dropout(0.2))
|
| 367 |
+
translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2)))
|
| 368 |
+
|
| 369 |
+
translation_model.add(keras.layers.Activation('elu'))
|
| 370 |
+
translation_model.add(Dense(32, use_bias=False, kernel_initializer='he_normal'))
|
| 371 |
+
|
| 372 |
+
translation_model.add(BatchNormalization())
|
| 373 |
+
translation_model.add(Dropout(0.2))
|
| 374 |
+
translation_model.add(keras.layers.Activation('elu'))
|
| 375 |
+
translation_model.add(Dense(32, kernel_initializer='he_normal',use_bias=False))
|
| 376 |
+
|
| 377 |
+
translation_model.add(BatchNormalization())
|
| 378 |
+
translation_model.add(keras.layers.Activation('elu'))
|
| 379 |
+
translation_model.add(Dropout(0.2))
|
| 380 |
+
translation_model.add(Dense(len(list(expression_mapping.keys())), activation='softmax'))
|
| 381 |
+
isl_translator=ISLSignPosTranslator(bodypose_25_model(),handpose_model(), translation_model)
|
| 382 |
+
```
|
| 383 |
+
|
| 384 |
+
**Parameters:**
|
| 385 |
+
- Total params: 82,679 (322.96 KB)
|
| 386 |
+
- Trainable params: 82,239 (321.25 KB)
|
| 387 |
+
- Non-trainable params: 440 (1.72 KB)
|
| 388 |
+
''')
|
| 389 |
+
|
| 390 |
+
with tab4:
|
| 391 |
+
st.markdown('''
|
| 392 |
+
### Training Details
|
| 393 |
+
The model was trained using the **Keras 3 API with a PyTorch backend**.
|
| 394 |
+
|
| 395 |
+
**Dataset Size:**
|
| 396 |
+
- **123,743 individual frames** were extracted from the processed videos and used as the training set.
|
| 397 |
+
|
| 398 |
+
**Sequence Windowing:**
|
| 399 |
+
- The temporal data is structured into windows of **20 frames** per sequence (`shape=(20, 156)`), allowing the Bidirectional LSTMs to learn the motion context of the signs.
|
| 400 |
+
|
| 401 |
+
**Metrics & Logs:**
|
| 402 |
+
- You can view the detailed training progression on [Tensorboard](https://huggingface.co/cdsteameight/ISL-SignLanguageTranslation/tensorboard).
|
| 403 |
+
''')
|
| 404 |
+
|
| 405 |
+
elif app_mode =='Run on Test Videos':
|
| 406 |
+
st.markdown('''
|
| 407 |
+
## How to use the Test Videos
|
| 408 |
+
Follow these steps to run a prediction on a test video:
|
| 409 |
+
1. Open the **sidebar** on the left.
|
| 410 |
+
2. Select a **Category** from the dropdown.
|
| 411 |
+
3. Select a specific **Class** (the sign language word you want to test).
|
| 412 |
+
4. Select a specific **File** from the available test videos.
|
| 413 |
+
5. Click the **"Start"** button at the bottom of the sidebar to run the prediction model on the video.
|
| 414 |
+
|
| 415 |
+
The app will load the frames, process them, and display the predicted translation!
|
| 416 |
+
---
|
| 417 |
+
''')
|
| 418 |
+
# placeholder = st.empty()
|
| 419 |
+
category = st.sidebar.selectbox('Choose Category',
|
| 420 |
+
np.sort(test_files_df['Category'].unique(), axis=-1, kind='mergesort'))
|
| 421 |
+
# print(category)
|
| 422 |
+
mask = (test_files_df['Category']==category)
|
| 423 |
+
test_files_df_category=test_files_df[mask]
|
| 424 |
+
cls = st.sidebar.selectbox('Choose Class',
|
| 425 |
+
np.sort(test_files_df_category['Class'].unique(), axis=-1, kind='mergesort')
|
| 426 |
+
)
|
| 427 |
+
mask = (test_files_df['Class']==cls)
|
| 428 |
+
filename = st.sidebar.selectbox('Choose File',
|
| 429 |
+
np.sort(test_files_df_category[mask]['Filename'].unique(), axis=-1, kind='mergesort')
|
| 430 |
+
)
|
| 431 |
+
# print(f'test/{category}/{cls}/{filename}')
|
| 432 |
+
# mask = (include_df['Filepath'].str.contains(key[0])) & (include_df['type']==key[2]) & (include_df['expression']==key[1])
|
| 433 |
+
# stframe = st.empty()
|
| 434 |
+
|
| 435 |
+
if st.sidebar.button("Start", type="primary"):
|
| 436 |
+
mask = (testing_df['FileName'] == filename) & (testing_df['Type']==category)& (testing_df['Expression']==cls)
|
| 437 |
+
# filtered_df = current_test_df.sort_
|
| 438 |
+
|
| 439 |
+
window_size=20
|
| 440 |
+
current_test_df=testing_df[mask]
|
| 441 |
+
|
| 442 |
+
if current_test_df.empty:
|
| 443 |
+
st.error(f"⚠️ No matching rows in testing_df for this selection.{filename} {category} {cls}")
|
| 444 |
+
st.stop()
|
| 445 |
+
else:
|
| 446 |
+
st.success(f"✅ Loaded {current_test_df.shape[0]} frames for this selection.")
|
| 447 |
+
X_test_filtered,y_test_filtered = create_timeseries_data(current_test_df,feature_columns_new,label_columns,window_size=window_size)
|
| 448 |
+
# y_filtered_encoded=to_categorical(y_test_filtered, num_classes=len(df['Expression_encoded'].unique()))
|
| 449 |
+
X_test_filtered=np.array(X_test_filtered)
|
| 450 |
+
|
| 451 |
+
# encoded_translation=model(frame.reshape(1,frame.shape[0],frame.shape[1]))
|
| 452 |
+
st.set_option('deprecation.showfileUploaderEncoding', False)
|
| 453 |
+
|
| 454 |
+
# use_webcam = st.sidebar.button('Use Webcam')
|
| 455 |
+
# record = st.sidebar.checkbox("Record Video")
|
| 456 |
+
# if record:
|
| 457 |
+
# st.checkbox("Recording", value=True)
|
| 458 |
+
|
| 459 |
+
st.sidebar.markdown('---')
|
| 460 |
+
st.markdown(
|
| 461 |
+
"""
|
| 462 |
+
<style>
|
| 463 |
+
[data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
|
| 464 |
+
width: 400px;
|
| 465 |
+
}
|
| 466 |
+
[data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
|
| 467 |
+
width: 400px;
|
| 468 |
+
margin-left: -400px;
|
| 469 |
+
}
|
| 470 |
+
</style>
|
| 471 |
+
""",
|
| 472 |
+
unsafe_allow_html=True,
|
| 473 |
+
)
|
| 474 |
+
|
| 475 |
+
st.sidebar.markdown('---')
|
| 476 |
+
|
| 477 |
+
st.markdown(' ## Output')
|
| 478 |
+
|
| 479 |
+
runtime_progress = st.empty()
|
| 480 |
+
|
| 481 |
+
with runtime_progress.container():
|
| 482 |
+
df1 = pd.DataFrame([['--','--']], columns=['Frames Processed','Detected Class'])
|
| 483 |
+
|
| 484 |
+
my_table = st.table(df1)
|
| 485 |
+
# kpi1, kpi2 = st.columns(2)
|
| 486 |
+
|
| 487 |
+
# with kpi1:
|
| 488 |
+
# st.markdown("**Frames Processed**")
|
| 489 |
+
# kpi1_text = st.markdown(f'0/{current_test_df.shape[0]}')
|
| 490 |
+
|
| 491 |
+
# with kpi2:
|
| 492 |
+
# st.markdown("**Detected Class**")
|
| 493 |
+
# kpi2_text = st.markdown("--")
|
| 494 |
+
|
| 495 |
+
view = st.empty()
|
| 496 |
+
|
| 497 |
+
st.markdown("<hr/>", unsafe_allow_html=True)
|
| 498 |
+
stframes = st.empty()#[st.empty() for _ in range(20)]
|
| 499 |
+
# video_file_buffer = st.sidebar.file_uploader("Upload a video", type=[ "mp4", "mov",'avi','asf', 'm4v' ])
|
| 500 |
+
# tfflie = tempfile.NamedTemporaryFile(delete=False)
|
| 501 |
+
|
| 502 |
+
vid_file = hf_hub_download(
|
| 503 |
+
repo_id="sunilsarolkar/isl-test-data",
|
| 504 |
+
filename=f'test/{category}/{cls}/{filename}',
|
| 505 |
+
repo_type="dataset"
|
| 506 |
+
)
|
| 507 |
+
|
| 508 |
+
if not os.path.exists(vid_file):
|
| 509 |
+
st.error(f"⚠️ Video file not found: {f'test/{category}/{cls}/{filename}'}")
|
| 510 |
+
st.stop()
|
| 511 |
+
|
| 512 |
+
vid = cv2.VideoCapture(vid_file)
|
| 513 |
+
|
| 514 |
+
ffprobe_result = ffprobe(vid_file)
|
| 515 |
+
info = json.loads(ffprobe_result.json)
|
| 516 |
+
videoinfo = [i for i in info["streams"] if i["codec_type"] == "video"][0]
|
| 517 |
+
input_fps = videoinfo["avg_frame_rate"]
|
| 518 |
+
# input_fps = float(input_fps[0])/float(input_fps[1])
|
| 519 |
+
input_pix_fmt = videoinfo["pix_fmt"]
|
| 520 |
+
input_vcodec = videoinfo["codec_name"]
|
| 521 |
+
postfix = info["format"]["format_name"].split(",")[0]
|
| 522 |
+
# print(f'input_vcodec-{input_vcodec}')
|
| 523 |
+
|
| 524 |
+
width = int(vid.get(cv2.CAP_PROP_FRAME_WIDTH))
|
| 525 |
+
height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
| 526 |
+
fps_input = int(vid.get(cv2.CAP_PROP_FPS))
|
| 527 |
+
|
| 528 |
+
#codec = cv2.VideoWriter_fourcc(*FLAGS.output_format)
|
| 529 |
+
# codec = cv2.VideoWriter_fourcc('V','P','0','9')
|
| 530 |
+
# out = cv2.VideoWriter('output1.mp4', codec, fps_input, (width, height))
|
| 531 |
+
|
| 532 |
+
# st.sidebar.text('Input Video')
|
| 533 |
+
# st.sidebar.video(tfflie.name)
|
| 534 |
+
fps = 0
|
| 535 |
+
i = 0
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
# cap = cv2.VideoCapture(video_file,)
|
| 540 |
+
totalFrames=int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
|
| 541 |
+
window_size=20
|
| 542 |
+
# print('current_test_df',current_test_df)
|
| 543 |
+
# print('totalFrames',totalFrames)
|
| 544 |
+
window=[]
|
| 545 |
+
|
| 546 |
+
|
| 547 |
+
prevTime = 0
|
| 548 |
+
postfix = info["format"]["format_name"].split(",")[0]
|
| 549 |
+
|
| 550 |
+
|
| 551 |
+
output_file = f"/tmp/output_{uuid.uuid4().hex}.{postfix}"
|
| 552 |
+
|
| 553 |
+
# height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
| 554 |
+
fps_input = int(vid.get(cv2.CAP_PROP_FPS))
|
| 555 |
+
|
| 556 |
+
#codec = cv2.VideoWriter_fourcc(*FLAGS.output_format)
|
| 557 |
+
|
| 558 |
+
# codec = cv2.VideoWriter_fourcc('m','p','4','v')
|
| 559 |
+
out = None
|
| 560 |
+
writer=None
|
| 561 |
+
weighted_avg_dict={}
|
| 562 |
+
|
| 563 |
+
idx=0
|
| 564 |
+
try:
|
| 565 |
+
for _, row in current_test_df.iterrows():#enumerate(file_df.rolling(window=20, step=20,min_periods=1)):
|
| 566 |
+
# print(f'captured frame#{idx}')
|
| 567 |
+
if not vid.isOpened():
|
| 568 |
+
st.error(f"Could not open video: {vid_file}")
|
| 569 |
+
else:
|
| 570 |
+
if(vid.isOpened()):
|
| 571 |
+
ret, frame = vid.read()
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
if len(window)<window_size:
|
| 575 |
+
canvas=util.drawStickmodel(frame,eval(row['bodypose_circles']),eval(row['bodypose_sticks']),eval(row['handpose_edges']),eval(row['handpose_peaks']))
|
| 576 |
+
canvas_with_plot=util.draw_bar_plot_below_image(canvas,{}, f'Prediction bar plot - Frame number {idx+1} [** no predictions]',canvas)
|
| 577 |
+
canvas_with_plot=util.draw_bar_plot_below_image(canvas_with_plot,weighted_avg_dict, f'Weighted avg - Frame number {idx+1} [** no predictions]',canvas)
|
| 578 |
+
canvas_with_plot=util.add_padding_to_bottom(canvas_with_plot,(255,255,255),100)# Adds padding at bottom
|
| 579 |
+
|
| 580 |
+
if writer is None:
|
| 581 |
+
input_framesize = canvas_with_plot.shape[:2]
|
| 582 |
+
writer = Writer(output_file, input_fps, input_framesize, input_pix_fmt,
|
| 583 |
+
input_vcodec)
|
| 584 |
+
|
| 585 |
+
# if out is None:
|
| 586 |
+
# out=cv2.VideoWriter(output_file, codec, fps_input, frame.shape[:2])
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
writer(canvas_with_plot)
|
| 590 |
+
# out.write(canvas)
|
| 591 |
+
with runtime_progress.container():
|
| 592 |
+
df1 = pd.DataFrame([[f'{idx+1}/{current_test_df.shape[0]}','<model will output after 20 frames>']], columns=['Frames Processed','Detected Class'])
|
| 593 |
+
|
| 594 |
+
my_table = st.table(df1)
|
| 595 |
+
window.append(frame)
|
| 596 |
+
# kpi1_text.write(f"<h1 style='text-align: center; color: red;'>{idx+1}/{current_test_df.shape[0]}</h1>", unsafe_allow_html=True)
|
| 597 |
+
# kpi2_text.write(f"<h1 style='text-align: center; color: red;'>--</h1>", unsafe_allow_html=True)
|
| 598 |
+
view.image(cv2.imencode('.jpg', canvas_with_plot)[1].tobytes(), use_column_width=True)
|
| 599 |
+
else:
|
| 600 |
+
|
| 601 |
+
window[:-1] = window[1:]
|
| 602 |
+
window[-1]=frame
|
| 603 |
+
translation_model=get_translator_model()
|
| 604 |
+
# testing_df[]
|
| 605 |
+
|
| 606 |
+
encoded_translation = translation_model(X_test_filtered[idx-20].reshape(1,X_test_filtered[idx-20].shape[0],X_test_filtered[idx-20].shape[1]))
|
| 607 |
+
encoded_translation=encoded_translation[0].cpu().detach().numpy()
|
| 608 |
+
sorted_index=np.argsort(encoded_translation)[::-1]
|
| 609 |
+
maxindex=np.argmax(encoded_translation)
|
| 610 |
+
|
| 611 |
+
top_3_probs = encoded_translation.argsort()[-3:][::-1] # Get indices of top 3 probabilities (descending order)
|
| 612 |
+
top_3_categories = [expression_mapping[i] for i in top_3_probs] # Convert indices to category names (assuming class_names list exists)
|
| 613 |
+
top_3_values = encoded_translation[top_3_probs] # Get corresponding probabilities
|
| 614 |
+
# print(f'{idx} {encoded_translation[maxindex]:0.4f} {maxindex}-{expression_mapping[maxindex]} ')#{[(pi,encoded_translation[pi],expression_mapping[pi]) for pi in sorted_index]}
|
| 615 |
+
for category, prob in zip(top_3_categories, top_3_values):
|
| 616 |
+
if category not in frame_wise_outputs:
|
| 617 |
+
frame_wise_outputs[category]=[]
|
| 618 |
+
frame_wise_outputs[category].append(prob)
|
| 619 |
+
current_prob={}
|
| 620 |
+
|
| 621 |
+
for category, prob in zip(top_3_categories, top_3_values):
|
| 622 |
+
current_prob[category]=prob
|
| 623 |
+
|
| 624 |
+
for key in frame_wise_outputs:
|
| 625 |
+
weighted_avg_dict[key]=weighted_average(frame_wise_outputs[key],[len(frame_wise_outputs[key]) for i in range(len(frame_wise_outputs[key]))])
|
| 626 |
+
|
| 627 |
+
sorted_dict = dict(sorted(weighted_avg_dict.items(), key=lambda item: item[1], reverse=True))
|
| 628 |
+
canvas=util.drawStickmodel(frame,eval(row['bodypose_circles']),eval(row['bodypose_sticks']),eval(row['handpose_edges']),eval(row['handpose_peaks']))
|
| 629 |
+
canvas_with_plot=util.draw_bar_plot_below_image(canvas,current_prob, f'Prediction at frame window({idx-20+1}-{idx+1})',canvas)
|
| 630 |
+
canvas_with_plot=util.draw_bar_plot_below_image(canvas_with_plot,weighted_avg_dict, f'Weighted avg till window {idx+1}',canvas)
|
| 631 |
+
canvas_with_plot=util.add_padding_to_bottom(canvas_with_plot,(255,255,255),100)
|
| 632 |
+
writer(canvas_with_plot)
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
currTime = time.time()
|
| 636 |
+
fps = 1 / (currTime - prevTime)
|
| 637 |
+
prevTime = currTime
|
| 638 |
+
# out.write(frame)
|
| 639 |
+
# if record:
|
| 640 |
+
# #st.checkbox("Recording", value=True)
|
| 641 |
+
# out.write(frame)
|
| 642 |
+
#Dashboard
|
| 643 |
+
|
| 644 |
+
max_prob = float('-inf') # Initialize with negative infinity
|
| 645 |
+
max_key = None
|
| 646 |
+
|
| 647 |
+
for exp, prob in weighted_avg_dict.items():
|
| 648 |
+
if prob > max_prob:
|
| 649 |
+
max_prob = prob
|
| 650 |
+
max_key = exp
|
| 651 |
+
with runtime_progress.container():
|
| 652 |
+
df1 = pd.DataFrame([[f'{idx+1}/{current_test_df.shape[0]}',f'{max_key} ({max_prob*100:.2f}%)']], columns=['Frames Processed','Detected Class'])
|
| 653 |
+
my_table = st.table(df1)
|
| 654 |
+
# kpi1_text.write(f"<h1 style='text-align: center; color: red;'>{idx+1}/{current_test_df.shape[0]}</h1>", unsafe_allow_html=True)
|
| 655 |
+
# kpi2_text.write(f"<h1 style='text-align: center; color: red;'>{max_key} ({max_prob*100:.2f}%)</h1>", unsafe_allow_html=True)
|
| 656 |
+
# with placeholder.container():
|
| 657 |
+
# # st.write(weighted_avg_dict)
|
| 658 |
+
# # data = {
|
| 659 |
+
# # "I": 0.7350964583456516,
|
| 660 |
+
# # "Hello": 0.1078806109726429,
|
| 661 |
+
# # "you": 0.11776176246348768,
|
| 662 |
+
# # "you (plural)": 0.12685142129916568
|
| 663 |
+
# # }
|
| 664 |
+
|
| 665 |
+
# # Convert the dictionary to a Pandas DataFrame for easier plotting
|
| 666 |
+
# df = pd.DataFrame.from_dict(weighted_avg_dict, orient='index', columns=['Values'])
|
| 667 |
+
|
| 668 |
+
# # Create a bar chart with Streamlit
|
| 669 |
+
# st.bar_chart(df)
|
| 670 |
+
# frame = cv2.resize(frame,(0,0),fx = 0.8 , fy = 0.8)
|
| 671 |
+
# frame = image_resize(image = frame, width = 640)
|
| 672 |
+
view.image(cv2.imencode('.jpg', canvas_with_plot)[1].tobytes(), use_column_width=True)
|
| 673 |
+
|
| 674 |
+
idx=idx+1
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
# st.text('Video Processed')
|
| 678 |
+
with view.container():
|
| 679 |
+
if writer is not None: # ✅ safeguard
|
| 680 |
+
writer.close()
|
| 681 |
+
output_video = open(output_file,'rb')
|
| 682 |
+
out_bytes = output_video.read()
|
| 683 |
+
st.video(out_bytes)
|
| 684 |
+
print(f'Output file - {output_file}')
|
| 685 |
+
else:
|
| 686 |
+
st.warning("No video was processed, writer is empty.")
|
| 687 |
+
# out.release()
|
| 688 |
+
|
| 689 |
+
print(f'Output file - {output_file}')
|
| 690 |
+
finally:
|
| 691 |
+
vid.release()
|
| 692 |
+
if writer is not None:
|
| 693 |
+
writer.close()
|
| 694 |
+
cv2.destroyAllWindows()
|
categories_processed.png
ADDED
|
data/include_validation.json.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ff6d428b43244589ded2237dd1d4cd82253baf1c96afcd20e925312c421782f4
|
| 3 |
+
size 142317
|
data/transformer_contract.json
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": {
|
| 3 |
+
"id": "spsarolkar/AI4Bharat-INCLUDE-keypoints",
|
| 4 |
+
"revision": "8a17a31ce8803cc1d5689f6249d088236d684943",
|
| 5 |
+
"files": {
|
| 6 |
+
"data/apple-vision/validation/validation-00000.jsonl.gz": "0c5370dbaa88f5fadfaaaf14f465176370e9324e6f243cec3c2ad5b0f924df25",
|
| 7 |
+
"data/apple-vision/validation/validation-00001.jsonl.gz": "784a39a1f1cfcbe5f2e2863455caaee08d132e104da3d2229f9335f18c27b4c1",
|
| 8 |
+
"data/apple-vision/validation/validation-00002.jsonl.gz": "584ee286a4511993dd90b906d63e552b7c64e06c2d5fe9be4159467f49750713",
|
| 9 |
+
"data/apple-vision/validation/validation-00003.jsonl.gz": "f3db9a68cbbb282856235ed394dd02662a468a25ed36cb6b5b2628b8cd1d7a99",
|
| 10 |
+
"data/apple-vision/validation/validation-00004.jsonl.gz": "f6b11238377ce13a3ab8e3ff01fe1d7aea4e8936abf541c60399bf24e4eb2035",
|
| 11 |
+
"data/apple-vision/validation/validation-00005.jsonl.gz": "0dee5398e79fcdbdb814f17164083a9d832b5e909642d99322ebf7933ca65a45",
|
| 12 |
+
"metadata/vocabulary.json": "9ac2b427fc3f1742c891ad6092740b336d07be19f4b9ad01984687afd9b4813f"
|
| 13 |
+
}
|
| 14 |
+
},
|
| 15 |
+
"modelRepo": "spsarolkar/isl-transformer-mlx",
|
| 16 |
+
"modelFiles": {
|
| 17 |
+
"rolling": "space-inference/rolling-epoch31.safetensors",
|
| 18 |
+
"causal": "space-inference/causal-epoch102.safetensors"
|
| 19 |
+
},
|
| 20 |
+
"classNames": [
|
| 21 |
+
"Actor",
|
| 22 |
+
"Adult",
|
| 23 |
+
"Afternoon",
|
| 24 |
+
"Alright",
|
| 25 |
+
"Animal",
|
| 26 |
+
"Artist",
|
| 27 |
+
"Author",
|
| 28 |
+
"Baby",
|
| 29 |
+
"Bag",
|
| 30 |
+
"Ball",
|
| 31 |
+
"Bank",
|
| 32 |
+
"Bathroom",
|
| 33 |
+
"Beautiful",
|
| 34 |
+
"Bed",
|
| 35 |
+
"Bedroom",
|
| 36 |
+
"Bicycle",
|
| 37 |
+
"Bill",
|
| 38 |
+
"Bird",
|
| 39 |
+
"Black",
|
| 40 |
+
"Blind",
|
| 41 |
+
"Blue",
|
| 42 |
+
"Boat",
|
| 43 |
+
"Book",
|
| 44 |
+
"Box",
|
| 45 |
+
"Boy",
|
| 46 |
+
"Brother",
|
| 47 |
+
"Brown",
|
| 48 |
+
"Bus",
|
| 49 |
+
"Camera",
|
| 50 |
+
"Car",
|
| 51 |
+
"Card",
|
| 52 |
+
"Cat",
|
| 53 |
+
"Cell phone",
|
| 54 |
+
"Chair",
|
| 55 |
+
"Child",
|
| 56 |
+
"City",
|
| 57 |
+
"Clock",
|
| 58 |
+
"Clothing",
|
| 59 |
+
"Computer",
|
| 60 |
+
"Court",
|
| 61 |
+
"Cow",
|
| 62 |
+
"Crowd",
|
| 63 |
+
"Daughter",
|
| 64 |
+
"Deaf",
|
| 65 |
+
"Death",
|
| 66 |
+
"Doctor",
|
| 67 |
+
"Dog",
|
| 68 |
+
"Door",
|
| 69 |
+
"Dream",
|
| 70 |
+
"Dress",
|
| 71 |
+
"Election",
|
| 72 |
+
"Energy",
|
| 73 |
+
"Evening",
|
| 74 |
+
"Ex. Monsoon",
|
| 75 |
+
"Exercise",
|
| 76 |
+
"Fall",
|
| 77 |
+
"Family",
|
| 78 |
+
"Fan",
|
| 79 |
+
"Father",
|
| 80 |
+
"Fish",
|
| 81 |
+
"Friday",
|
| 82 |
+
"Friend",
|
| 83 |
+
"Gift",
|
| 84 |
+
"Girl",
|
| 85 |
+
"God",
|
| 86 |
+
"Good Morning",
|
| 87 |
+
"Good afternoon",
|
| 88 |
+
"Good evening",
|
| 89 |
+
"Good night",
|
| 90 |
+
"Grandfather",
|
| 91 |
+
"Grandmother",
|
| 92 |
+
"Green",
|
| 93 |
+
"Ground",
|
| 94 |
+
"Gun",
|
| 95 |
+
"Hat",
|
| 96 |
+
"Hello",
|
| 97 |
+
"Horse",
|
| 98 |
+
"Hospital",
|
| 99 |
+
"Hour",
|
| 100 |
+
"House",
|
| 101 |
+
"How are you",
|
| 102 |
+
"Husband",
|
| 103 |
+
"I",
|
| 104 |
+
"Job",
|
| 105 |
+
"Key",
|
| 106 |
+
"King",
|
| 107 |
+
"Kitchen",
|
| 108 |
+
"Lamp",
|
| 109 |
+
"Laptop",
|
| 110 |
+
"Lawyer",
|
| 111 |
+
"Letter",
|
| 112 |
+
"Library",
|
| 113 |
+
"Location",
|
| 114 |
+
"Lock",
|
| 115 |
+
"Man",
|
| 116 |
+
"Manager",
|
| 117 |
+
"Market",
|
| 118 |
+
"Marriage",
|
| 119 |
+
"Mean",
|
| 120 |
+
"Medicine",
|
| 121 |
+
"Minute",
|
| 122 |
+
"Monday",
|
| 123 |
+
"Money",
|
| 124 |
+
"Month",
|
| 125 |
+
"Morning",
|
| 126 |
+
"Mother",
|
| 127 |
+
"Mouse",
|
| 128 |
+
"Newspaper",
|
| 129 |
+
"Nice",
|
| 130 |
+
"Night",
|
| 131 |
+
"Office",
|
| 132 |
+
"Orange",
|
| 133 |
+
"Page",
|
| 134 |
+
"Paint",
|
| 135 |
+
"Pant",
|
| 136 |
+
"Paper",
|
| 137 |
+
"Parent",
|
| 138 |
+
"Park",
|
| 139 |
+
"Patient",
|
| 140 |
+
"Peace",
|
| 141 |
+
"Pen",
|
| 142 |
+
"Pencil",
|
| 143 |
+
"Photograph",
|
| 144 |
+
"Pink",
|
| 145 |
+
"Plane",
|
| 146 |
+
"Player",
|
| 147 |
+
"Pleased",
|
| 148 |
+
"Pocket",
|
| 149 |
+
"Police",
|
| 150 |
+
"President",
|
| 151 |
+
"Price",
|
| 152 |
+
"Priest",
|
| 153 |
+
"Queen",
|
| 154 |
+
"Race (ethnicity)",
|
| 155 |
+
"Radio",
|
| 156 |
+
"Red",
|
| 157 |
+
"Religion",
|
| 158 |
+
"Reporter",
|
| 159 |
+
"Restaurant",
|
| 160 |
+
"Ring",
|
| 161 |
+
"Saturday",
|
| 162 |
+
"School",
|
| 163 |
+
"Science",
|
| 164 |
+
"Screen",
|
| 165 |
+
"Season",
|
| 166 |
+
"Second",
|
| 167 |
+
"Secretary",
|
| 168 |
+
"Shirt",
|
| 169 |
+
"Shoes",
|
| 170 |
+
"Sign",
|
| 171 |
+
"Sister",
|
| 172 |
+
"Skirt",
|
| 173 |
+
"Soap",
|
| 174 |
+
"Soldier",
|
| 175 |
+
"Son",
|
| 176 |
+
"Sport",
|
| 177 |
+
"Spring",
|
| 178 |
+
"Store or Shop",
|
| 179 |
+
"Street or Road",
|
| 180 |
+
"Student",
|
| 181 |
+
"Suit",
|
| 182 |
+
"Summer",
|
| 183 |
+
"Sunday",
|
| 184 |
+
"Table",
|
| 185 |
+
"Teacher",
|
| 186 |
+
"Team",
|
| 187 |
+
"Technology",
|
| 188 |
+
"Telephone",
|
| 189 |
+
"Television",
|
| 190 |
+
"Temple",
|
| 191 |
+
"Thank you",
|
| 192 |
+
"Thursday",
|
| 193 |
+
"Time",
|
| 194 |
+
"Today",
|
| 195 |
+
"Tomorrow",
|
| 196 |
+
"Tool",
|
| 197 |
+
"Train",
|
| 198 |
+
"Train Station",
|
| 199 |
+
"Transportation",
|
| 200 |
+
"Truck",
|
| 201 |
+
"Tuesday",
|
| 202 |
+
"Ugly",
|
| 203 |
+
"University",
|
| 204 |
+
"Waiter",
|
| 205 |
+
"War",
|
| 206 |
+
"Wednesday",
|
| 207 |
+
"Week",
|
| 208 |
+
"White",
|
| 209 |
+
"Wife",
|
| 210 |
+
"Window",
|
| 211 |
+
"Winter",
|
| 212 |
+
"Woman",
|
| 213 |
+
"Year",
|
| 214 |
+
"Yellow",
|
| 215 |
+
"Yesterday",
|
| 216 |
+
"alive",
|
| 217 |
+
"bad",
|
| 218 |
+
"big large",
|
| 219 |
+
"cheap",
|
| 220 |
+
"clean",
|
| 221 |
+
"cold",
|
| 222 |
+
"cool",
|
| 223 |
+
"curved",
|
| 224 |
+
"dead",
|
| 225 |
+
"deep",
|
| 226 |
+
"dirty",
|
| 227 |
+
"dry",
|
| 228 |
+
"expensive",
|
| 229 |
+
"famous",
|
| 230 |
+
"fast",
|
| 231 |
+
"female",
|
| 232 |
+
"flat",
|
| 233 |
+
"good",
|
| 234 |
+
"happy",
|
| 235 |
+
"hard",
|
| 236 |
+
"he",
|
| 237 |
+
"healthy",
|
| 238 |
+
"heavy",
|
| 239 |
+
"high",
|
| 240 |
+
"hot",
|
| 241 |
+
"it",
|
| 242 |
+
"light",
|
| 243 |
+
"long",
|
| 244 |
+
"loose",
|
| 245 |
+
"loud",
|
| 246 |
+
"low",
|
| 247 |
+
"male",
|
| 248 |
+
"narrow",
|
| 249 |
+
"new",
|
| 250 |
+
"old",
|
| 251 |
+
"poor",
|
| 252 |
+
"quiet",
|
| 253 |
+
"rich",
|
| 254 |
+
"sad",
|
| 255 |
+
"shallow",
|
| 256 |
+
"she",
|
| 257 |
+
"short",
|
| 258 |
+
"sick",
|
| 259 |
+
"slow",
|
| 260 |
+
"small little",
|
| 261 |
+
"soft",
|
| 262 |
+
"strong",
|
| 263 |
+
"tall",
|
| 264 |
+
"they",
|
| 265 |
+
"thick",
|
| 266 |
+
"thin",
|
| 267 |
+
"tight",
|
| 268 |
+
"train ticket",
|
| 269 |
+
"warm",
|
| 270 |
+
"we",
|
| 271 |
+
"weak",
|
| 272 |
+
"wet",
|
| 273 |
+
"wide",
|
| 274 |
+
"you",
|
| 275 |
+
"you (plural)",
|
| 276 |
+
"young"
|
| 277 |
+
]
|
| 278 |
+
}
|
eda/train_test_validation_split-1.png
ADDED
|
Git LFS Details
|
eda/train_test_validation_split-2.png
ADDED
|
Git LFS Details
|
expression_mapping.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
expression_mapping={107: "alive",
|
| 2 |
+
58: "Nice",
|
| 3 |
+
8: "Beautiful",
|
| 4 |
+
115: "dead",
|
| 5 |
+
120: "famous",
|
| 6 |
+
122: "female",
|
| 7 |
+
51: "Mean",
|
| 8 |
+
21: "Deaf",
|
| 9 |
+
111: "clean",
|
| 10 |
+
117: "dirty",
|
| 11 |
+
123: "flat",
|
| 12 |
+
110: "cheap",
|
| 13 |
+
119: "expensive",
|
| 14 |
+
116: "deep",
|
| 15 |
+
99: "Ugly",
|
| 16 |
+
114: "curved",
|
| 17 |
+
12: "Blind",
|
| 18 |
+
142: "poor",
|
| 19 |
+
138: "male",
|
| 20 |
+
126: "hard",
|
| 21 |
+
133: "light",
|
| 22 |
+
137: "low",
|
| 23 |
+
113: "cool",
|
| 24 |
+
144: "rich",
|
| 25 |
+
109: "big large",
|
| 26 |
+
108: "bad",
|
| 27 |
+
112: "cold",
|
| 28 |
+
135: "loose",
|
| 29 |
+
121: "fast",
|
| 30 |
+
141: "old",
|
| 31 |
+
130: "high",
|
| 32 |
+
118: "dry",
|
| 33 |
+
145: "sad",
|
| 34 |
+
131: "hot",
|
| 35 |
+
125: "happy",
|
| 36 |
+
129: "heavy",
|
| 37 |
+
128: "healthy",
|
| 38 |
+
124: "good",
|
| 39 |
+
146: "shallow",
|
| 40 |
+
153: "strong",
|
| 41 |
+
161: "weak",
|
| 42 |
+
157: "thin",
|
| 43 |
+
158: "tight",
|
| 44 |
+
136: "loud",
|
| 45 |
+
139: "narrow",
|
| 46 |
+
134: "long",
|
| 47 |
+
156: "thick",
|
| 48 |
+
148: "short",
|
| 49 |
+
152: "soft",
|
| 50 |
+
150: "slow",
|
| 51 |
+
151: "small little",
|
| 52 |
+
149: "sick",
|
| 53 |
+
154: "tall",
|
| 54 |
+
140: "new",
|
| 55 |
+
143: "quiet",
|
| 56 |
+
95: "Today",
|
| 57 |
+
163: "wide",
|
| 58 |
+
159: "warm",
|
| 59 |
+
96: "Tomorrow",
|
| 60 |
+
162: "wet",
|
| 61 |
+
1: "Afternoon",
|
| 62 |
+
27: "Evening",
|
| 63 |
+
56: "Morning",
|
| 64 |
+
59: "Night",
|
| 65 |
+
166: "young",
|
| 66 |
+
53: "Minute",
|
| 67 |
+
38: "Hour",
|
| 68 |
+
88: "Sunday",
|
| 69 |
+
55: "Month",
|
| 70 |
+
94: "Time",
|
| 71 |
+
70: "Pleased",
|
| 72 |
+
63: "Paper",
|
| 73 |
+
105: "Year",
|
| 74 |
+
80: "Second",
|
| 75 |
+
32: "Gift",
|
| 76 |
+
102: "Week",
|
| 77 |
+
43: "Key",
|
| 78 |
+
48: "Lock",
|
| 79 |
+
4: "Bag",
|
| 80 |
+
106: "Yesterday",
|
| 81 |
+
7: "Bathroom",
|
| 82 |
+
15: "Card",
|
| 83 |
+
66: "Pen",
|
| 84 |
+
45: "Letter",
|
| 85 |
+
9: "Bed",
|
| 86 |
+
2: "Alright",
|
| 87 |
+
67: "Pencil",
|
| 88 |
+
24: "Dream",
|
| 89 |
+
13: "Book",
|
| 90 |
+
44: "Kitchen",
|
| 91 |
+
92: "Telephone",
|
| 92 |
+
23: "Door",
|
| 93 |
+
36: "Hello",
|
| 94 |
+
61: "Page",
|
| 95 |
+
40: "How are you",
|
| 96 |
+
16: "Chair",
|
| 97 |
+
89: "Table",
|
| 98 |
+
97: "Tool",
|
| 99 |
+
68: "Photograph",
|
| 100 |
+
10: "Bedroom",
|
| 101 |
+
103: "Window",
|
| 102 |
+
62: "Paint",
|
| 103 |
+
14: "Box",
|
| 104 |
+
76: "Ring",
|
| 105 |
+
82: "Soap",
|
| 106 |
+
20: "Crowd",
|
| 107 |
+
75: "Restaurant",
|
| 108 |
+
98: "Train Station",
|
| 109 |
+
31: "Friend",
|
| 110 |
+
17: "Child",
|
| 111 |
+
0: "Adult",
|
| 112 |
+
46: "Library",
|
| 113 |
+
39: "House",
|
| 114 |
+
42: "India",
|
| 115 |
+
86: "Street or Road",
|
| 116 |
+
72: "Queen",
|
| 117 |
+
85: "Store or Shop",
|
| 118 |
+
64: "Park",
|
| 119 |
+
77: "School",
|
| 120 |
+
18: "City",
|
| 121 |
+
49: "Market",
|
| 122 |
+
60: "Office",
|
| 123 |
+
132: "it",
|
| 124 |
+
41: "I",
|
| 125 |
+
6: "Bank",
|
| 126 |
+
69: "Player",
|
| 127 |
+
147: "she",
|
| 128 |
+
19: "Court",
|
| 129 |
+
155: "they",
|
| 130 |
+
104: "Winter",
|
| 131 |
+
93: "Temple",
|
| 132 |
+
33: "God",
|
| 133 |
+
50: "Marriage",
|
| 134 |
+
29: "Exercise",
|
| 135 |
+
37: "Hospital",
|
| 136 |
+
34: "Ground",
|
| 137 |
+
25: "Election",
|
| 138 |
+
73: "Race (ethnicity)",
|
| 139 |
+
11: "Bill",
|
| 140 |
+
87: "Summer",
|
| 141 |
+
160: "we",
|
| 142 |
+
127: "he",
|
| 143 |
+
22: "Death",
|
| 144 |
+
84: "Spring",
|
| 145 |
+
47: "Location",
|
| 146 |
+
26: "Energy",
|
| 147 |
+
54: "Money",
|
| 148 |
+
28: "Ex. Monsoon",
|
| 149 |
+
165: "you (plural)",
|
| 150 |
+
65: "Peace",
|
| 151 |
+
5: "Ball",
|
| 152 |
+
71: "Price",
|
| 153 |
+
35: "Gun",
|
| 154 |
+
30: "Fall",
|
| 155 |
+
164: "you",
|
| 156 |
+
81: "Sign",
|
| 157 |
+
100: "University",
|
| 158 |
+
83: "Sport",
|
| 159 |
+
74: "Religion",
|
| 160 |
+
101: "War",
|
| 161 |
+
57: "Newspaper",
|
| 162 |
+
3: "Attack",
|
| 163 |
+
90: "Team",
|
| 164 |
+
78: "Science",
|
| 165 |
+
79: "Season",
|
| 166 |
+
52: "Medicine",
|
| 167 |
+
91: "Technology",
|
| 168 |
+
}
|
model.py
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from collections import OrderedDict
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
|
| 7 |
+
# def make_layers(block, no_relu_layers):
|
| 8 |
+
# layers = []
|
| 9 |
+
# for layer_name, v in block.items():
|
| 10 |
+
# if 'pool' in layer_name:
|
| 11 |
+
# layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1],
|
| 12 |
+
# padding=v[2])
|
| 13 |
+
# layers.append((layer_name, layer))
|
| 14 |
+
# else:
|
| 15 |
+
# conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1],
|
| 16 |
+
# kernel_size=v[2], stride=v[3],
|
| 17 |
+
# padding=v[4])
|
| 18 |
+
# layers.append((layer_name, conv2d))
|
| 19 |
+
# if layer_name not in no_relu_layers:
|
| 20 |
+
# layers.append(('relu_'+layer_name, nn.ReLU(inplace=True)))
|
| 21 |
+
|
| 22 |
+
# return nn.Sequential(OrderedDict(layers))
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def make_layers(block, no_relu_layers,prelu_layers = []):
|
| 26 |
+
layers = []
|
| 27 |
+
|
| 28 |
+
for layer_name, v in block.items():
|
| 29 |
+
if 'pool' in layer_name:
|
| 30 |
+
layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1],
|
| 31 |
+
padding=v[2])
|
| 32 |
+
layers.append((layer_name, layer))
|
| 33 |
+
else:
|
| 34 |
+
#[3, 64, 3, 1, 1]
|
| 35 |
+
conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1],
|
| 36 |
+
kernel_size=v[2], stride=v[3],
|
| 37 |
+
padding=v[4])
|
| 38 |
+
layers.append((layer_name, conv2d))
|
| 39 |
+
if layer_name not in no_relu_layers:
|
| 40 |
+
if layer_name not in prelu_layers:
|
| 41 |
+
layers.append(('relu_'+layer_name, nn.ReLU(inplace=True)))
|
| 42 |
+
else:
|
| 43 |
+
layers.append(('prelu'+layer_name[4:],nn.PReLU(v[1])))
|
| 44 |
+
|
| 45 |
+
return nn.Sequential(OrderedDict(layers))
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def make_layers_Mconv(block,no_relu_layers):
|
| 49 |
+
modules = []
|
| 50 |
+
for layer_name, v in block.items():
|
| 51 |
+
layers = []
|
| 52 |
+
if 'pool' in layer_name:
|
| 53 |
+
layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1],
|
| 54 |
+
padding=v[2])
|
| 55 |
+
layers.append((layer_name, layer))
|
| 56 |
+
else:
|
| 57 |
+
conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1],
|
| 58 |
+
kernel_size=v[2], stride=v[3],
|
| 59 |
+
padding=v[4])
|
| 60 |
+
layers.append((layer_name, conv2d))
|
| 61 |
+
if layer_name not in no_relu_layers:
|
| 62 |
+
layers.append(('Mprelu'+layer_name[5:], nn.PReLU(v[1])))
|
| 63 |
+
modules.append(nn.Sequential(OrderedDict(layers)))
|
| 64 |
+
return nn.ModuleList(modules)
|
| 65 |
+
|
| 66 |
+
class bodypose_25_model(nn.Module):
|
| 67 |
+
def __init__(self):
|
| 68 |
+
super(bodypose_25_model,self).__init__()
|
| 69 |
+
# these layers have no relu layer
|
| 70 |
+
no_relu_layers = ['Mconv7_stage0_L1','Mconv7_stage0_L2',\
|
| 71 |
+
'Mconv7_stage1_L1', 'Mconv7_stage1_L2',\
|
| 72 |
+
'Mconv7_stage2_L2', 'Mconv7_stage3_L2']
|
| 73 |
+
prelu_layers = ['conv4_2','conv4_3_CPM','conv4_4_CPM']
|
| 74 |
+
blocks = {}
|
| 75 |
+
block0 = OrderedDict([
|
| 76 |
+
('conv1_1', [3, 64, 3, 1, 1]),
|
| 77 |
+
('conv1_2', [64, 64, 3, 1, 1]),
|
| 78 |
+
('pool1_stage1', [2, 2, 0]),
|
| 79 |
+
('conv2_1', [64, 128, 3, 1, 1]),
|
| 80 |
+
('conv2_2', [128, 128, 3, 1, 1]),
|
| 81 |
+
('pool2_stage1', [2, 2, 0]),
|
| 82 |
+
('conv3_1', [128, 256, 3, 1, 1]),
|
| 83 |
+
('conv3_2', [256, 256, 3, 1, 1]),
|
| 84 |
+
('conv3_3', [256, 256, 3, 1, 1]),
|
| 85 |
+
('conv3_4', [256, 256, 3, 1, 1]),
|
| 86 |
+
('pool3_stage1', [2, 2, 0]),
|
| 87 |
+
('conv4_1', [256, 512, 3, 1, 1]),
|
| 88 |
+
('conv4_2', [512, 512, 3, 1, 1]),
|
| 89 |
+
('conv4_3_CPM', [512, 256, 3, 1, 1]),
|
| 90 |
+
('conv4_4_CPM', [256, 128, 3, 1, 1])
|
| 91 |
+
])
|
| 92 |
+
self.model0 = make_layers(block0, no_relu_layers,prelu_layers)
|
| 93 |
+
|
| 94 |
+
#L2
|
| 95 |
+
#stage0
|
| 96 |
+
blocks['Mconv1_stage0_L2'] = OrderedDict([
|
| 97 |
+
('Mconv1_stage0_L2_0',[128,96,3,1,1]),
|
| 98 |
+
('Mconv1_stage0_L2_1',[96,96,3,1,1]),
|
| 99 |
+
('Mconv1_stage0_L2_2',[96,96,3,1,1])
|
| 100 |
+
])
|
| 101 |
+
for i in range(2,6):
|
| 102 |
+
blocks['Mconv%d_stage0_L2' % i] = OrderedDict([
|
| 103 |
+
('Mconv%d_stage0_L2_0' % i,[288,96,3,1,1]),
|
| 104 |
+
('Mconv%d_stage0_L2_1' % i,[96,96,3,1,1]),
|
| 105 |
+
('Mconv%d_stage0_L2_2' % i,[96,96,3,1,1])
|
| 106 |
+
])
|
| 107 |
+
blocks['Mconv6_7_stage0_L2'] = OrderedDict([
|
| 108 |
+
('Mconv6_stage0_L2',[288, 256, 1,1,0]),
|
| 109 |
+
('Mconv7_stage0_L2',[256,52,1,1,0])
|
| 110 |
+
])
|
| 111 |
+
#stage1~3
|
| 112 |
+
for s in range(1,4):
|
| 113 |
+
blocks['Mconv1_stage%d_L2' % s] = OrderedDict([
|
| 114 |
+
('Mconv1_stage%d_L2_0' % s,[180,128,3,1,1]),
|
| 115 |
+
('Mconv1_stage%d_L2_1' % s,[128,128,3,1,1]),
|
| 116 |
+
('Mconv1_stage%d_L2_2' % s,[128,128,3,1,1])
|
| 117 |
+
])
|
| 118 |
+
for i in range(2,6):
|
| 119 |
+
blocks['Mconv%d_stage%d_L2' % (i,s)] = OrderedDict([
|
| 120 |
+
('Mconv%d_stage%d_L2_0' % (i,s) ,[384,128,3,1,1]),
|
| 121 |
+
('Mconv%d_stage%d_L2_1' % (i,s) ,[128,128,3,1,1]),
|
| 122 |
+
('Mconv%d_stage%d_L2_2' % (i,s) ,[128,128,3,1,1])
|
| 123 |
+
])
|
| 124 |
+
blocks['Mconv6_7_stage%d_L2' % s] = OrderedDict([
|
| 125 |
+
('Mconv6_stage%d_L2' % s,[384,512,1,1,0]),
|
| 126 |
+
('Mconv7_stage%d_L2' % s,[512,52,1,1,0])
|
| 127 |
+
])
|
| 128 |
+
|
| 129 |
+
#L1
|
| 130 |
+
#stage0
|
| 131 |
+
blocks['Mconv1_stage0_L1'] = OrderedDict([
|
| 132 |
+
('Mconv1_stage0_L1_0',[180,96,3,1,1]),
|
| 133 |
+
('Mconv1_stage0_L1_1',[96,96,3,1,1]),
|
| 134 |
+
('Mconv1_stage0_L1_2',[96,96,3,1,1])
|
| 135 |
+
])
|
| 136 |
+
for i in range(2,6):
|
| 137 |
+
blocks['Mconv%d_stage0_L1' % i] = OrderedDict([
|
| 138 |
+
('Mconv%d_stage0_L1_0' % i,[288,96,3,1,1]),
|
| 139 |
+
('Mconv%d_stage0_L1_1' % i,[96,96,3,1,1]),
|
| 140 |
+
('Mconv%d_stage0_L1_2' % i,[96,96,3,1,1])
|
| 141 |
+
])
|
| 142 |
+
blocks['Mconv6_7_stage0_L1'] = OrderedDict([
|
| 143 |
+
('Mconv6_stage0_L1',[288, 256, 1,1,0]),
|
| 144 |
+
('Mconv7_stage0_L1',[256,26,1,1,0])
|
| 145 |
+
])
|
| 146 |
+
#stage1
|
| 147 |
+
blocks['Mconv1_stage1_L1'] = OrderedDict([
|
| 148 |
+
('Mconv1_stage1_L1_0',[206,128,3,1,1]),
|
| 149 |
+
('Mconv1_stage1_L1_1',[128,128,3,1,1]),
|
| 150 |
+
('Mconv1_stage1_L1_2',[128,128,3,1,1])
|
| 151 |
+
])
|
| 152 |
+
for i in range(2,6):
|
| 153 |
+
blocks['Mconv%d_stage1_L1' % i] = OrderedDict([
|
| 154 |
+
('Mconv%d_stage1_L1_0' % i,[384,128,3,1,1]),
|
| 155 |
+
('Mconv%d_stage1_L1_1' % i,[128,128,3,1,1]),
|
| 156 |
+
('Mconv%d_stage1_L1_2' % i,[128,128,3,1,1])
|
| 157 |
+
])
|
| 158 |
+
blocks['Mconv6_7_stage1_L1'] = OrderedDict([
|
| 159 |
+
('Mconv6_stage1_L1',[384,512,1,1,0]),
|
| 160 |
+
('Mconv7_stage1_L1',[512,26,1,1,0])
|
| 161 |
+
])
|
| 162 |
+
|
| 163 |
+
for k in blocks.keys():
|
| 164 |
+
blocks[k] = make_layers_Mconv(blocks[k], no_relu_layers)
|
| 165 |
+
self.models = nn.ModuleDict(blocks)
|
| 166 |
+
#self.model_L2_S0_mconv1 = blocks['Mconv1_stage0_L2']
|
| 167 |
+
for param in self.parameters():
|
| 168 |
+
param.requires_grad = False
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def _Mconv_forward(self,x,models):
|
| 172 |
+
outs = []
|
| 173 |
+
out = x
|
| 174 |
+
for m in models:
|
| 175 |
+
out = m(out)
|
| 176 |
+
outs.append(out)
|
| 177 |
+
return torch.cat(outs,1)
|
| 178 |
+
|
| 179 |
+
def forward(self,x):
|
| 180 |
+
out0 = self.model0(x)
|
| 181 |
+
#L2
|
| 182 |
+
tout = out0
|
| 183 |
+
for s in range(4):
|
| 184 |
+
tout = self._Mconv_forward(tout,self.models['Mconv1_stage%d_L2' % s])
|
| 185 |
+
for v in range(2,6):
|
| 186 |
+
tout = self._Mconv_forward(tout,self.models['Mconv%d_stage%d_L2' % (v,s)])
|
| 187 |
+
tout = self.models['Mconv6_7_stage%d_L2' % s][0](tout)
|
| 188 |
+
tout = self.models['Mconv6_7_stage%d_L2' % s][1](tout)
|
| 189 |
+
outL2 = tout
|
| 190 |
+
tout = torch.cat([out0,tout],1)
|
| 191 |
+
#L1 stage0
|
| 192 |
+
#tout = torch.cat([out0,outL2],1)
|
| 193 |
+
tout = self._Mconv_forward(tout, self.models['Mconv1_stage0_L1'])
|
| 194 |
+
for v in range(2,6):
|
| 195 |
+
tout = self._Mconv_forward(tout, self.models['Mconv%d_stage0_L1' % v])
|
| 196 |
+
tout = self.models['Mconv6_7_stage0_L1'][0](tout)
|
| 197 |
+
tout = self.models['Mconv6_7_stage0_L1'][1](tout)
|
| 198 |
+
outS0L1 = tout
|
| 199 |
+
tout = torch.cat([out0,outS0L1,outL2],1)
|
| 200 |
+
#L1 stage1
|
| 201 |
+
tout = self._Mconv_forward(tout, self.models['Mconv1_stage1_L1'])
|
| 202 |
+
for v in range(2,6):
|
| 203 |
+
tout = self._Mconv_forward(tout, self.models['Mconv%d_stage1_L1' % v])
|
| 204 |
+
tout = self.models['Mconv6_7_stage1_L1'][0](tout)
|
| 205 |
+
outS1L1 = self.models['Mconv6_7_stage1_L1'][1](tout)
|
| 206 |
+
|
| 207 |
+
return outL2, outS1L1
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
class bodypose_model(nn.Module):
|
| 211 |
+
def __init__(self):
|
| 212 |
+
super(bodypose_model, self).__init__()
|
| 213 |
+
|
| 214 |
+
# these layers have no relu layer
|
| 215 |
+
no_relu_layers = ['conv5_5_CPM_L1', 'conv5_5_CPM_L2', 'Mconv7_stage2_L1',\
|
| 216 |
+
'Mconv7_stage2_L2', 'Mconv7_stage3_L1', 'Mconv7_stage3_L2',\
|
| 217 |
+
'Mconv7_stage4_L1', 'Mconv7_stage4_L2', 'Mconv7_stage5_L1',\
|
| 218 |
+
'Mconv7_stage5_L2', 'Mconv7_stage6_L1', 'Mconv7_stage6_L1']
|
| 219 |
+
blocks = {}
|
| 220 |
+
block0 = OrderedDict([
|
| 221 |
+
('conv1_1', [3, 64, 3, 1, 1]),
|
| 222 |
+
('conv1_2', [64, 64, 3, 1, 1]),
|
| 223 |
+
('pool1_stage1', [2, 2, 0]),
|
| 224 |
+
('conv2_1', [64, 128, 3, 1, 1]),
|
| 225 |
+
('conv2_2', [128, 128, 3, 1, 1]),
|
| 226 |
+
('pool2_stage1', [2, 2, 0]),
|
| 227 |
+
('conv3_1', [128, 256, 3, 1, 1]),
|
| 228 |
+
('conv3_2', [256, 256, 3, 1, 1]),
|
| 229 |
+
('conv3_3', [256, 256, 3, 1, 1]),
|
| 230 |
+
('conv3_4', [256, 256, 3, 1, 1]),
|
| 231 |
+
('pool3_stage1', [2, 2, 0]),
|
| 232 |
+
('conv4_1', [256, 512, 3, 1, 1]),
|
| 233 |
+
('conv4_2', [512, 512, 3, 1, 1]),
|
| 234 |
+
('conv4_3_CPM', [512, 256, 3, 1, 1]),
|
| 235 |
+
('conv4_4_CPM', [256, 128, 3, 1, 1])
|
| 236 |
+
])
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
# Stage 1
|
| 240 |
+
block1_1 = OrderedDict([
|
| 241 |
+
('conv5_1_CPM_L1', [128, 128, 3, 1, 1]),
|
| 242 |
+
('conv5_2_CPM_L1', [128, 128, 3, 1, 1]),
|
| 243 |
+
('conv5_3_CPM_L1', [128, 128, 3, 1, 1]),
|
| 244 |
+
('conv5_4_CPM_L1', [128, 512, 1, 1, 0]),
|
| 245 |
+
('conv5_5_CPM_L1', [512, 38, 1, 1, 0])
|
| 246 |
+
])
|
| 247 |
+
|
| 248 |
+
block1_2 = OrderedDict([
|
| 249 |
+
('conv5_1_CPM_L2', [128, 128, 3, 1, 1]),
|
| 250 |
+
('conv5_2_CPM_L2', [128, 128, 3, 1, 1]),
|
| 251 |
+
('conv5_3_CPM_L2', [128, 128, 3, 1, 1]),
|
| 252 |
+
('conv5_4_CPM_L2', [128, 512, 1, 1, 0]),
|
| 253 |
+
('conv5_5_CPM_L2', [512, 19, 1, 1, 0])
|
| 254 |
+
])
|
| 255 |
+
blocks['block1_1'] = block1_1
|
| 256 |
+
blocks['block1_2'] = block1_2
|
| 257 |
+
|
| 258 |
+
self.model0 = make_layers(block0, no_relu_layers)
|
| 259 |
+
|
| 260 |
+
# Stages 2 - 6
|
| 261 |
+
for i in range(2, 7):
|
| 262 |
+
blocks['block%d_1' % i] = OrderedDict([
|
| 263 |
+
('Mconv1_stage%d_L1' % i, [185, 128, 7, 1, 3]),
|
| 264 |
+
('Mconv2_stage%d_L1' % i, [128, 128, 7, 1, 3]),
|
| 265 |
+
('Mconv3_stage%d_L1' % i, [128, 128, 7, 1, 3]),
|
| 266 |
+
('Mconv4_stage%d_L1' % i, [128, 128, 7, 1, 3]),
|
| 267 |
+
('Mconv5_stage%d_L1' % i, [128, 128, 7, 1, 3]),
|
| 268 |
+
('Mconv6_stage%d_L1' % i, [128, 128, 1, 1, 0]),
|
| 269 |
+
('Mconv7_stage%d_L1' % i, [128, 38, 1, 1, 0])
|
| 270 |
+
])
|
| 271 |
+
|
| 272 |
+
blocks['block%d_2' % i] = OrderedDict([
|
| 273 |
+
('Mconv1_stage%d_L2' % i, [185, 128, 7, 1, 3]),
|
| 274 |
+
('Mconv2_stage%d_L2' % i, [128, 128, 7, 1, 3]),
|
| 275 |
+
('Mconv3_stage%d_L2' % i, [128, 128, 7, 1, 3]),
|
| 276 |
+
('Mconv4_stage%d_L2' % i, [128, 128, 7, 1, 3]),
|
| 277 |
+
('Mconv5_stage%d_L2' % i, [128, 128, 7, 1, 3]),
|
| 278 |
+
('Mconv6_stage%d_L2' % i, [128, 128, 1, 1, 0]),
|
| 279 |
+
('Mconv7_stage%d_L2' % i, [128, 19, 1, 1, 0])
|
| 280 |
+
])
|
| 281 |
+
|
| 282 |
+
for k in blocks.keys():
|
| 283 |
+
blocks[k] = make_layers(blocks[k], no_relu_layers)
|
| 284 |
+
|
| 285 |
+
self.model1_1 = blocks['block1_1']
|
| 286 |
+
self.model2_1 = blocks['block2_1']
|
| 287 |
+
self.model3_1 = blocks['block3_1']
|
| 288 |
+
self.model4_1 = blocks['block4_1']
|
| 289 |
+
self.model5_1 = blocks['block5_1']
|
| 290 |
+
self.model6_1 = blocks['block6_1']
|
| 291 |
+
|
| 292 |
+
self.model1_2 = blocks['block1_2']
|
| 293 |
+
self.model2_2 = blocks['block2_2']
|
| 294 |
+
self.model3_2 = blocks['block3_2']
|
| 295 |
+
self.model4_2 = blocks['block4_2']
|
| 296 |
+
self.model5_2 = blocks['block5_2']
|
| 297 |
+
self.model6_2 = blocks['block6_2']
|
| 298 |
+
for param in self.parameters():
|
| 299 |
+
param.requires_grad = False
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def forward(self, x):
|
| 303 |
+
|
| 304 |
+
out1 = self.model0(x)
|
| 305 |
+
|
| 306 |
+
out1_1 = self.model1_1(out1)
|
| 307 |
+
out1_2 = self.model1_2(out1)
|
| 308 |
+
out2 = torch.cat([out1_1, out1_2, out1], 1)
|
| 309 |
+
|
| 310 |
+
out2_1 = self.model2_1(out2)
|
| 311 |
+
out2_2 = self.model2_2(out2)
|
| 312 |
+
out3 = torch.cat([out2_1, out2_2, out1], 1)
|
| 313 |
+
|
| 314 |
+
out3_1 = self.model3_1(out3)
|
| 315 |
+
out3_2 = self.model3_2(out3)
|
| 316 |
+
out4 = torch.cat([out3_1, out3_2, out1], 1)
|
| 317 |
+
|
| 318 |
+
out4_1 = self.model4_1(out4)
|
| 319 |
+
out4_2 = self.model4_2(out4)
|
| 320 |
+
out5 = torch.cat([out4_1, out4_2, out1], 1)
|
| 321 |
+
|
| 322 |
+
out5_1 = self.model5_1(out5)
|
| 323 |
+
out5_2 = self.model5_2(out5)
|
| 324 |
+
out6 = torch.cat([out5_1, out5_2, out1], 1)
|
| 325 |
+
|
| 326 |
+
out6_1 = self.model6_1(out6)
|
| 327 |
+
out6_2 = self.model6_2(out6)
|
| 328 |
+
|
| 329 |
+
return out6_1, out6_2
|
| 330 |
+
|
| 331 |
+
class handpose_model(nn.Module):
|
| 332 |
+
def __init__(self):
|
| 333 |
+
super(handpose_model, self).__init__()
|
| 334 |
+
|
| 335 |
+
# these layers have no relu layer
|
| 336 |
+
no_relu_layers = ['conv6_2_CPM', 'Mconv7_stage2', 'Mconv7_stage3',\
|
| 337 |
+
'Mconv7_stage4', 'Mconv7_stage5', 'Mconv7_stage6']
|
| 338 |
+
# stage 1
|
| 339 |
+
block1_0 = OrderedDict([
|
| 340 |
+
('conv1_1', [3, 64, 3, 1, 1]),
|
| 341 |
+
('conv1_2', [64, 64, 3, 1, 1]),
|
| 342 |
+
('pool1_stage1', [2, 2, 0]),
|
| 343 |
+
('conv2_1', [64, 128, 3, 1, 1]),
|
| 344 |
+
('conv2_2', [128, 128, 3, 1, 1]),
|
| 345 |
+
('pool2_stage1', [2, 2, 0]),
|
| 346 |
+
('conv3_1', [128, 256, 3, 1, 1]),
|
| 347 |
+
('conv3_2', [256, 256, 3, 1, 1]),
|
| 348 |
+
('conv3_3', [256, 256, 3, 1, 1]),
|
| 349 |
+
('conv3_4', [256, 256, 3, 1, 1]),
|
| 350 |
+
('pool3_stage1', [2, 2, 0]),
|
| 351 |
+
('conv4_1', [256, 512, 3, 1, 1]),
|
| 352 |
+
('conv4_2', [512, 512, 3, 1, 1]),
|
| 353 |
+
('conv4_3', [512, 512, 3, 1, 1]),
|
| 354 |
+
('conv4_4', [512, 512, 3, 1, 1]),
|
| 355 |
+
('conv5_1', [512, 512, 3, 1, 1]),
|
| 356 |
+
('conv5_2', [512, 512, 3, 1, 1]),
|
| 357 |
+
('conv5_3_CPM', [512, 128, 3, 1, 1])
|
| 358 |
+
])
|
| 359 |
+
|
| 360 |
+
block1_1 = OrderedDict([
|
| 361 |
+
('conv6_1_CPM', [128, 512, 1, 1, 0]),
|
| 362 |
+
('conv6_2_CPM', [512, 22, 1, 1, 0])
|
| 363 |
+
])
|
| 364 |
+
|
| 365 |
+
blocks = {}
|
| 366 |
+
blocks['block1_0'] = block1_0
|
| 367 |
+
blocks['block1_1'] = block1_1
|
| 368 |
+
|
| 369 |
+
# stage 2-6
|
| 370 |
+
for i in range(2, 7):
|
| 371 |
+
blocks['block%d' % i] = OrderedDict([
|
| 372 |
+
('Mconv1_stage%d' % i, [150, 128, 7, 1, 3]),
|
| 373 |
+
('Mconv2_stage%d' % i, [128, 128, 7, 1, 3]),
|
| 374 |
+
('Mconv3_stage%d' % i, [128, 128, 7, 1, 3]),
|
| 375 |
+
('Mconv4_stage%d' % i, [128, 128, 7, 1, 3]),
|
| 376 |
+
('Mconv5_stage%d' % i, [128, 128, 7, 1, 3]),
|
| 377 |
+
('Mconv6_stage%d' % i, [128, 128, 1, 1, 0]),
|
| 378 |
+
('Mconv7_stage%d' % i, [128, 22, 1, 1, 0])
|
| 379 |
+
])
|
| 380 |
+
|
| 381 |
+
for k in blocks.keys():
|
| 382 |
+
blocks[k] = make_layers(blocks[k], no_relu_layers)
|
| 383 |
+
|
| 384 |
+
self.model1_0 = blocks['block1_0']
|
| 385 |
+
self.model1_1 = blocks['block1_1']
|
| 386 |
+
self.model2 = blocks['block2']
|
| 387 |
+
self.model3 = blocks['block3']
|
| 388 |
+
self.model4 = blocks['block4']
|
| 389 |
+
self.model5 = blocks['block5']
|
| 390 |
+
self.model6 = blocks['block6']
|
| 391 |
+
for param in self.parameters():
|
| 392 |
+
param.requires_grad = False
|
| 393 |
+
|
| 394 |
+
def forward(self, x):
|
| 395 |
+
out1_0 = self.model1_0(x)
|
| 396 |
+
out1_1 = self.model1_1(out1_0)
|
| 397 |
+
concat_stage2 = torch.cat([out1_1, out1_0], 1)
|
| 398 |
+
out_stage2 = self.model2(concat_stage2)
|
| 399 |
+
concat_stage3 = torch.cat([out_stage2, out1_0], 1)
|
| 400 |
+
out_stage3 = self.model3(concat_stage3)
|
| 401 |
+
concat_stage4 = torch.cat([out_stage3, out1_0], 1)
|
| 402 |
+
out_stage4 = self.model4(concat_stage4)
|
| 403 |
+
concat_stage5 = torch.cat([out_stage4, out1_0], 1)
|
| 404 |
+
out_stage5 = self.model5(concat_stage5)
|
| 405 |
+
concat_stage6 = torch.cat([out_stage5, out1_0], 1)
|
| 406 |
+
out_stage6 = self.model6(concat_stage6)
|
| 407 |
+
return out_stage6
|
packages.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
ffmpeg
|
| 2 |
+
libgl1
|
| 3 |
+
libglib2.0-0
|
| 4 |
+
libsm6
|
| 5 |
+
libxrender1
|
| 6 |
+
libxext6
|
requirements-docker.txt
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
streamlit==1.34.0
|
| 2 |
+
numpy==1.26.4
|
| 3 |
+
Pillow==11.2.1
|
| 4 |
+
matplotlib==3.10.1
|
| 5 |
+
opencv-python-headless==4.11.0.86
|
| 6 |
+
scipy==1.15.2
|
| 7 |
+
scikit-image==0.25.2
|
| 8 |
+
pandas==2.2.3
|
| 9 |
+
keras==3.9.2
|
| 10 |
+
ffmpeg-python==0.2.0
|
| 11 |
+
seaborn==0.13.2
|
| 12 |
+
huggingface_hub==0.32.4
|
requirements.txt
CHANGED
|
@@ -1,3 +1,19 @@
|
|
| 1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
pandas
|
| 3 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
streamlit
|
| 2 |
+
numpy
|
| 3 |
+
Pillow
|
| 4 |
+
matplotlib==3.5.3
|
| 5 |
+
opencv-python
|
| 6 |
+
scipy
|
| 7 |
+
scikit-image
|
| 8 |
+
tqdm
|
| 9 |
pandas
|
| 10 |
+
torch
|
| 11 |
+
torchaudio
|
| 12 |
+
torchvision
|
| 13 |
+
torchtext
|
| 14 |
+
torchdata
|
| 15 |
+
av
|
| 16 |
+
keras
|
| 17 |
+
ffmpeg-python
|
| 18 |
+
seaborn[stats]
|
| 19 |
+
huggingface_hub
|
transformer_comparison.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Streamlit comparison of the archived Transformer predictions on one INCLUDE panel."""
|
| 2 |
+
|
| 3 |
+
import csv
|
| 4 |
+
import gzip
|
| 5 |
+
import io
|
| 6 |
+
import json
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from urllib.parse import quote
|
| 9 |
+
|
| 10 |
+
import streamlit as st
|
| 11 |
+
import streamlit.components.v1 as components
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
ARMS = {
|
| 15 |
+
"rolling": "Rolling Transformer · epoch 31",
|
| 16 |
+
"causal": "Causal Transformer · epoch 102",
|
| 17 |
+
}
|
| 18 |
+
DATA = Path(__file__).with_name("data") / "include_validation.json.gz"
|
| 19 |
+
VIDEO_REPO = "spsarolkar/AI4Bharat-INCLUDE-dataset"
|
| 20 |
+
VIDEO_REVISION = "d12293ab05cb5dc2cb6a342c364132a18adc49c9"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@st.cache_data
|
| 24 |
+
def load_panel():
|
| 25 |
+
with gzip.open(DATA, "rt", encoding="utf-8") as source:
|
| 26 |
+
return json.load(source)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def outcome_counts(clips, arm):
|
| 30 |
+
correct = wrong_early = correct_early = unconfirmed = 0
|
| 31 |
+
for clip in clips:
|
| 32 |
+
prediction = clip[arm]
|
| 33 |
+
correct += (prediction["last"] or {}).get("label") == clip["label"]
|
| 34 |
+
early = prediction["early"]
|
| 35 |
+
if early is None:
|
| 36 |
+
unconfirmed += 1
|
| 37 |
+
elif early["label"] == clip["label"]:
|
| 38 |
+
correct_early += 1
|
| 39 |
+
else:
|
| 40 |
+
wrong_early += 1
|
| 41 |
+
return correct, wrong_early, correct_early, unconfirmed
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def score(value, total):
|
| 45 |
+
return f"{value / total:.2%} ({value}/{total})"
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def export_scores(clips):
|
| 49 |
+
buffer = io.StringIO()
|
| 50 |
+
writer = csv.writer(buffer)
|
| 51 |
+
writer.writerow(("case", "expected", "model", "last_guess", "last_guess_seconds",
|
| 52 |
+
"first_confirmation", "confirmation_seconds", "outcome"))
|
| 53 |
+
for clip in clips:
|
| 54 |
+
for arm, model in ARMS.items():
|
| 55 |
+
prediction = clip[arm]
|
| 56 |
+
last, early = prediction["last"] or {}, prediction["early"] or {}
|
| 57 |
+
outcome = ("Unconfirmed" if not early else
|
| 58 |
+
"Correct" if early["label"] == clip["label"] else "Wrong")
|
| 59 |
+
writer.writerow((clip["id"], clip["label"], model, last.get("label"), last.get("time"),
|
| 60 |
+
early.get("label"), early.get("time"), outcome))
|
| 61 |
+
return buffer.getvalue()
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def public_video(path):
|
| 65 |
+
if not path:
|
| 66 |
+
return None
|
| 67 |
+
parts = path.split("/")
|
| 68 |
+
if len(parts) != 5 or parts[0] != "videos" or parts[1] not in {"train", "val", "test"}:
|
| 69 |
+
raise ValueError("Unexpected public video path")
|
| 70 |
+
if any(part in {"", ".", ".."} for part in parts):
|
| 71 |
+
raise ValueError("Invalid public video path")
|
| 72 |
+
return f"https://huggingface.co/datasets/{VIDEO_REPO}/resolve/{VIDEO_REVISION}/{quote(path)}"
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def render_playback(video_url, clip):
|
| 76 |
+
# Inline script data is escaped before interpolation into the iframe document.
|
| 77 |
+
payload = json.dumps({
|
| 78 |
+
"video": video_url, "arms": ARMS, "duration": clip["duration"],
|
| 79 |
+
"predictions": {arm: clip[arm] for arm in ARMS},
|
| 80 |
+
}, ensure_ascii=True).replace("<", "\\u003c")
|
| 81 |
+
template = Path(__file__).with_name("validation_playback.html").read_text(encoding="utf-8")
|
| 82 |
+
components.html(template.replace("/* PAYLOAD */", payload), height=480, scrolling=True)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def render_transformer_comparison():
|
| 86 |
+
st.title("SignLanguageTranslate")
|
| 87 |
+
st.header("Indian Sign Language model comparison")
|
| 88 |
+
try:
|
| 89 |
+
panel = load_panel()
|
| 90 |
+
except (OSError, ValueError):
|
| 91 |
+
st.error("The archived evaluation could not be loaded.")
|
| 92 |
+
return
|
| 93 |
+
|
| 94 |
+
if panel.get("videoSource") != {"repoId": VIDEO_REPO, "revision": VIDEO_REVISION}:
|
| 95 |
+
st.error("The archived evaluation points to a different video dataset revision.")
|
| 96 |
+
return
|
| 97 |
+
clips = panel["clips"]
|
| 98 |
+
st.caption("720 isolated INCLUDE clips from the saved iPad validation split. "
|
| 99 |
+
"Both models were evaluated on the same Apple Vision keypoints.")
|
| 100 |
+
from transformer_runtime import evaluate_validation, load_assets, replay_clip, settings
|
| 101 |
+
|
| 102 |
+
revision = settings()
|
| 103 |
+
live = False
|
| 104 |
+
if revision:
|
| 105 |
+
if st.button("Run both models on all 720 validation clips"):
|
| 106 |
+
try:
|
| 107 |
+
with st.spinner("Running both models on the frozen validation clips…"):
|
| 108 |
+
st.session_state["transformer_evaluation"] = evaluate_validation(revision, clips)
|
| 109 |
+
except (OSError, ValueError, RuntimeError):
|
| 110 |
+
st.error("Live evaluation could not load the pinned artifacts or finish inference.")
|
| 111 |
+
results = st.session_state.get("transformer_evaluation")
|
| 112 |
+
if results is not None:
|
| 113 |
+
clips = results
|
| 114 |
+
live = True
|
| 115 |
+
else:
|
| 116 |
+
st.info("The server is showing the archived validation results. Live inference requires the Space's private model and dataset configuration.")
|
| 117 |
+
st.subheader("Validation accuracy")
|
| 118 |
+
st.caption("Computed by the Space" if live else "Archived evaluation on the Mac")
|
| 119 |
+
table = []
|
| 120 |
+
for arm, name in ARMS.items():
|
| 121 |
+
correct, wrong_early, correct_early, unconfirmed = outcome_counts(clips, arm)
|
| 122 |
+
table.append({
|
| 123 |
+
"Model": name,
|
| 124 |
+
"Final accuracy": score(correct, len(clips)),
|
| 125 |
+
"Wrong early confirmations": score(wrong_early, len(clips)),
|
| 126 |
+
"Correct early confirmations": correct_early,
|
| 127 |
+
"Unconfirmed clips": unconfirmed,
|
| 128 |
+
})
|
| 129 |
+
st.dataframe(table, use_container_width=True, hide_index=True)
|
| 130 |
+
st.caption("Final accuracy uses the last scheduled prediction before the clip ends. "
|
| 131 |
+
"An early confirmation is the first label repeated across three consecutive updates.")
|
| 132 |
+
st.download_button("Download validation results (CSV)", export_scores(clips),
|
| 133 |
+
file_name="include-validation-results.csv", mime="text/csv")
|
| 134 |
+
|
| 135 |
+
st.subheader("Replay predictions")
|
| 136 |
+
labels = sorted({clip["label"] for clip in clips}, key=str.casefold)
|
| 137 |
+
selected_label = st.selectbox("Expected sign", labels, index=labels.index("Woman") if "Woman" in labels else 0)
|
| 138 |
+
matching = [clip for clip in clips if clip["label"] == selected_label]
|
| 139 |
+
default_clip = next((i for i, item in enumerate(matching) if item["id"] == "include-004"), 0)
|
| 140 |
+
clip = st.selectbox("Validation clip", matching, index=default_clip,
|
| 141 |
+
format_func=lambda item: f"{item['id']} · {item['duration']:.2f} s"
|
| 142 |
+
+ (" · source video available" if item["videoPath"] else ""))
|
| 143 |
+
if revision and not live:
|
| 144 |
+
if st.button("Run both models on this clip", key=f"predict-{clip['id']}"):
|
| 145 |
+
try:
|
| 146 |
+
with st.spinner("Running both models on the saved poses…"):
|
| 147 |
+
inputs, models = load_assets(revision)
|
| 148 |
+
index = next(i for i, item in enumerate(clips) if item["id"] == clip["id"])
|
| 149 |
+
st.session_state["transformer_clip"] = replay_clip(inputs, models, index, clip)
|
| 150 |
+
except (OSError, ValueError, RuntimeError):
|
| 151 |
+
st.error("Live inference could not load the pinned artifacts or predict this clip.")
|
| 152 |
+
result = st.session_state.get("transformer_clip")
|
| 153 |
+
if result is not None and result["id"] == clip["id"]:
|
| 154 |
+
clip = result
|
| 155 |
+
live = True
|
| 156 |
+
video_url = public_video(clip["videoPath"])
|
| 157 |
+
if video_url:
|
| 158 |
+
split = {"val": "validation", "train": "train", "test": "test"}[clip["videoPath"].split("/")[1]]
|
| 159 |
+
st.caption(f"Public video repository split: {split}.")
|
| 160 |
+
st.markdown(f"[Open source video]({video_url})")
|
| 161 |
+
else:
|
| 162 |
+
st.caption("Use the timeline slider to review this clip’s archived predictions.")
|
| 163 |
+
st.caption(("Predictions computed by the Space" if live else "Archived model predictions") + " at 250 ms intervals.")
|
| 164 |
+
render_playback(video_url, clip)
|
| 165 |
+
with st.expander("All prediction updates"):
|
| 166 |
+
updates = {}
|
| 167 |
+
for arm, name in ARMS.items():
|
| 168 |
+
for tick in clip[arm]["ticks"]:
|
| 169 |
+
updates.setdefault(tick["time"], {"Time (s)": tick["time"]})[name] = tick["label"]
|
| 170 |
+
st.dataframe(list(updates.values()), use_container_width=True, hide_index=True)
|
| 171 |
+
|
| 172 |
+
with st.expander("Dataset splits and model sources"):
|
| 173 |
+
st.markdown(
|
| 174 |
+
"- This evaluation uses 720 of 743 clips in the saved iPad validation split. "
|
| 175 |
+
"The models share 256 output labels. This panel covers 254 labels, and 23 clips "
|
| 176 |
+
"were excluded because their labels are outside the model vocabulary. "
|
| 177 |
+
"Epoch 102 was selected using this validation panel. Signer independence is unverified.\n"
|
| 178 |
+
"- The legacy LSTM uses 167 output labels and a custom test manifest. Of its 412 clips, "
|
| 179 |
+
"245 match the public INCLUDE train split, 25 validation, 66 test, and 76 have no exact match. "
|
| 180 |
+
"Training membership for the published legacy weights remains unverified.\n"
|
| 181 |
+
"- [INCLUDE source and attribution](https://huggingface.co/datasets/ai4bharat/INCLUDE) · "
|
| 182 |
+
"[Public video dataset](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-dataset) · "
|
| 183 |
+
"[Apple Vision keypoint dataset card](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-keypoints) · "
|
| 184 |
+
"[Native Transformer checkpoint card](https://huggingface.co/spsarolkar/isl-transformer-mlx) · "
|
| 185 |
+
"[Legacy LSTM model card](https://huggingface.co/sunilsarolkar/isl-translation-model)"
|
| 186 |
+
)
|
transformer_inference.py
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CPU replay of the two frozen Apple Vision INCLUDE Transformer checkpoints."""
|
| 2 |
+
|
| 3 |
+
from bisect import bisect_right
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
import struct
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
WEIGHTS_SHA256 = {
|
| 14 |
+
"rolling": "5df06195356eb8b61aff409d556cebcfb81accfe50fce23a3fb8dbb532a8e3d6",
|
| 15 |
+
"causal": "6ed79c31aec5c232141aa9cfccf2fa7eec5265ace50fce3a73a8b75471e658f8",
|
| 16 |
+
}
|
| 17 |
+
VOCABULARY_SHA256 = "caaf0d329a7de21ffc170a19e539d1fe197e9ef7344f8bf57ea752812de43963"
|
| 18 |
+
FEATURES = 180
|
| 19 |
+
HIDDEN = 256
|
| 20 |
+
HEADS = 4
|
| 21 |
+
EMBEDDING = 384
|
| 22 |
+
CONTEXT = 48
|
| 23 |
+
TICK_SECONDS = 0.25
|
| 24 |
+
GROUPS = ("body", "leftHand", "rightHand")
|
| 25 |
+
GROUP_SIZES = (18, 21, 21)
|
| 26 |
+
FREQUENCIES = np.exp(np.arange(0, HIDDEN, 2, dtype=np.float32) * np.float32(-np.log(10_000) / HIDDEN))
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _weights(path, arm):
|
| 30 |
+
path = Path(path)
|
| 31 |
+
if not 0 < path.stat().st_size <= 20_000_000:
|
| 32 |
+
raise ValueError("Unexpected checkpoint length")
|
| 33 |
+
data = path.read_bytes()
|
| 34 |
+
if hashlib.sha256(data).hexdigest() != WEIGHTS_SHA256[arm]:
|
| 35 |
+
raise ValueError(f"Unexpected {arm} checkpoint digest")
|
| 36 |
+
header_size, = struct.unpack_from("<Q", data)
|
| 37 |
+
header = json.loads(data[8:8 + header_size])
|
| 38 |
+
buffer = memoryview(data)[8 + header_size:]
|
| 39 |
+
tensors = {}
|
| 40 |
+
rolling_prefixes = {0: "inputProjection", 1: "inputNorm", 2: "outputNorm", 3: "outputProjection", 5: "pool"}
|
| 41 |
+
for name, metadata in header.items():
|
| 42 |
+
if name == "__metadata__":
|
| 43 |
+
continue
|
| 44 |
+
if arm == "rolling":
|
| 45 |
+
_, layer, suffix = name.split(".", 2)
|
| 46 |
+
layer = int(layer)
|
| 47 |
+
prefix = rolling_prefixes[layer] if layer < 6 else f"encoderLayers.{layer - 6}"
|
| 48 |
+
name = f"{prefix}.{suffix}"
|
| 49 |
+
start, end = metadata["data_offsets"]
|
| 50 |
+
if metadata["dtype"] != "F32" or not 0 <= start < end <= len(buffer):
|
| 51 |
+
raise ValueError("Invalid checkpoint tensor")
|
| 52 |
+
tensors[name] = np.frombuffer(buffer[start:end], dtype="<f4").reshape(metadata["shape"])
|
| 53 |
+
if len(tensors) != (69 if arm == "rolling" else 64):
|
| 54 |
+
raise ValueError("Unexpected checkpoint tensor count")
|
| 55 |
+
return tensors
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _vocabulary(rows):
|
| 59 |
+
if len(rows) != 256 or len({row["word"] for row in rows}) != 256:
|
| 60 |
+
raise ValueError("Expected the frozen 256-sign vocabulary")
|
| 61 |
+
digest = hashlib.sha256()
|
| 62 |
+
for row in rows:
|
| 63 |
+
vector = np.asarray(row["vector"], dtype="<f4")
|
| 64 |
+
if vector.shape != (EMBEDDING,) or not np.isfinite(vector).all():
|
| 65 |
+
raise ValueError("Invalid sign embedding")
|
| 66 |
+
digest.update(row["word"].encode("utf-8") + b"\0")
|
| 67 |
+
digest.update(vector.tobytes())
|
| 68 |
+
if digest.hexdigest() != VOCABULARY_SHA256:
|
| 69 |
+
raise ValueError("Unexpected ordered vocabulary digest")
|
| 70 |
+
rows = sorted(rows, key=lambda row: row["word"])
|
| 71 |
+
vectors = np.asarray([row["vector"] for row in rows], dtype=np.float32)
|
| 72 |
+
return [row["word"] for row in rows], vectors, np.linalg.norm(vectors, axis=1)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _softmax(x):
|
| 76 |
+
x = np.exp(x - x.max(axis=-1, keepdims=True))
|
| 77 |
+
return x / x.sum(axis=-1, keepdims=True)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _gelu(x):
|
| 81 |
+
# Abramowitz and Stegun 7.1.26 approximates erf to within 1.5e-7 in
|
| 82 |
+
# real arithmetic. Float32 output is checked against the native replay.
|
| 83 |
+
scaled = x * np.float32(1 / np.sqrt(2))
|
| 84 |
+
t = np.float32(1) / (np.float32(1) + np.float32(0.3275911) * np.abs(scaled))
|
| 85 |
+
polynomial = np.float32(1.061405429)
|
| 86 |
+
for coefficient in (-1.453152027, 1.421413741, -0.284496736, 0.254829592):
|
| 87 |
+
polynomial = polynomial * t + np.float32(coefficient)
|
| 88 |
+
erf = np.copysign(np.float32(1) - polynomial * t * np.exp(-scaled * scaled), scaled)
|
| 89 |
+
return (x * (np.float32(1) + erf)) / np.float32(2)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
@dataclass(frozen=True)
|
| 93 |
+
class _Group:
|
| 94 |
+
names: tuple
|
| 95 |
+
values: np.ndarray
|
| 96 |
+
|
| 97 |
+
@classmethod
|
| 98 |
+
def decode(cls, points):
|
| 99 |
+
if points is None:
|
| 100 |
+
return None
|
| 101 |
+
return cls(tuple(p["id"] for p in points),
|
| 102 |
+
np.asarray([(p["x"], p["y"], p["confidence"]) for p in points], dtype=np.float32).reshape(-1, 3))
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _interpolate(a, b, fraction):
|
| 106 |
+
if a is None or b is None or len(a.names) != len(b.names):
|
| 107 |
+
return a if fraction < 0.5 else b
|
| 108 |
+
values = a.values + (b.values - a.values) * fraction
|
| 109 |
+
if a.names != b.names:
|
| 110 |
+
mismatched = [i for i, (name_a, name_b) in enumerate(zip(a.names, b.names)) if name_a != name_b]
|
| 111 |
+
values[mismatched] = a.values[mismatched]
|
| 112 |
+
return _Group(a.names, values)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _serialize(groups):
|
| 116 |
+
body = groups[0]
|
| 117 |
+
offset = np.zeros(2, dtype=np.float32)
|
| 118 |
+
for name in ("neck", "nose"):
|
| 119 |
+
if name in body.names:
|
| 120 |
+
offset = body.values[body.names.index(name), :2]
|
| 121 |
+
break
|
| 122 |
+
result = np.zeros((FEATURES // 3, 3), dtype=np.float32)
|
| 123 |
+
start = 0
|
| 124 |
+
for group, minimum_size in zip(groups, GROUP_SIZES):
|
| 125 |
+
if group is not None:
|
| 126 |
+
length = min(len(group.names), len(result) - start)
|
| 127 |
+
result[start:start + length] = group.values[:length]
|
| 128 |
+
result[start:start + length, :2] -= offset
|
| 129 |
+
start += max(minimum_size, len(group.names))
|
| 130 |
+
else:
|
| 131 |
+
start += minimum_size
|
| 132 |
+
if start >= len(result):
|
| 133 |
+
break
|
| 134 |
+
return result.reshape(FEATURES)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def _resample(groups, serialized, start, end):
|
| 138 |
+
count = end - start
|
| 139 |
+
if count == 60:
|
| 140 |
+
return serialized[start:end]
|
| 141 |
+
result = np.empty((60, FEATURES), dtype=np.float32)
|
| 142 |
+
for row in range(60):
|
| 143 |
+
position = row / 59 * (count - 1)
|
| 144 |
+
lower = int(position)
|
| 145 |
+
upper = min(lower + 1, count - 1)
|
| 146 |
+
if lower == upper:
|
| 147 |
+
result[row] = serialized[start + lower]
|
| 148 |
+
continue
|
| 149 |
+
fraction = np.float32(position - lower)
|
| 150 |
+
result[row] = _serialize([
|
| 151 |
+
_interpolate(a, b, fraction) for a, b in zip(groups[start + lower], groups[start + upper])
|
| 152 |
+
])
|
| 153 |
+
return result
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def _position(time):
|
| 157 |
+
estimate = int(time * 12)
|
| 158 |
+
if estimate / 12 > time:
|
| 159 |
+
return estimate - 1
|
| 160 |
+
if (estimate + 1) / 12 <= time:
|
| 161 |
+
return estimate + 1
|
| 162 |
+
return estimate
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
class _PoseSequence:
|
| 166 |
+
def __init__(self, frames):
|
| 167 |
+
self.times = [float(frame["timestamp"]) for frame in frames]
|
| 168 |
+
if (not self.times or any(not np.isfinite(t) or t < 0 for t in self.times)
|
| 169 |
+
or any(a > b for a, b in zip(self.times, self.times[1:]))):
|
| 170 |
+
raise ValueError("Expected saved poses with finite chronological timestamps")
|
| 171 |
+
self.groups = [tuple(_Group.decode(frame.get(key)) for key in GROUPS) for frame in frames]
|
| 172 |
+
if any(groups[0] is None for groups in self.groups):
|
| 173 |
+
raise ValueError("Missing body group in saved pose")
|
| 174 |
+
self.rows = np.asarray([_serialize(groups) for groups in self.groups])
|
| 175 |
+
self.usable = np.isfinite(self.rows).all(axis=1) & (self.rows[:, 2::3] > 0).any(axis=1)
|
| 176 |
+
self.next_frame = 0
|
| 177 |
+
self.last_rolling_time = -np.inf
|
| 178 |
+
|
| 179 |
+
def rolling(self, time):
|
| 180 |
+
start = bisect_right(self.times, time - 2)
|
| 181 |
+
end = bisect_right(self.times, time, lo=start)
|
| 182 |
+
if end - start < 2 or self.times[end - 1] <= self.last_rolling_time:
|
| 183 |
+
return None
|
| 184 |
+
self.last_rolling_time = self.times[end - 1]
|
| 185 |
+
return _resample(self.groups, self.rows, start, end)
|
| 186 |
+
|
| 187 |
+
def causal(self, time):
|
| 188 |
+
horizon = _position(time) / 12
|
| 189 |
+
positions, values = [], []
|
| 190 |
+
while self.next_frame < len(self.times) and self.times[self.next_frame] <= horizon:
|
| 191 |
+
first = self.next_frame
|
| 192 |
+
frame_time = self.times[first]
|
| 193 |
+
position = _position(frame_time)
|
| 194 |
+
if position / 12 < frame_time:
|
| 195 |
+
position += 1
|
| 196 |
+
grid_time = position / 12
|
| 197 |
+
self.next_frame = bisect_right(self.times, grid_time, lo=first)
|
| 198 |
+
for i in range(self.next_frame - 1, first - 1, -1):
|
| 199 |
+
if grid_time - self.times[i] > 0.25:
|
| 200 |
+
break
|
| 201 |
+
if self.usable[i]:
|
| 202 |
+
positions.append(position)
|
| 203 |
+
values.append(self.rows[i])
|
| 204 |
+
break
|
| 205 |
+
return np.asarray(positions, dtype=np.int32), np.asarray(values, dtype=np.float32)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
class Transformer:
|
| 209 |
+
"""Load immutable model weights; each replay creates its own temporal state."""
|
| 210 |
+
|
| 211 |
+
def __init__(self, weights_path, vocabulary, arm):
|
| 212 |
+
if arm not in WEIGHTS_SHA256:
|
| 213 |
+
raise ValueError("Unknown Transformer mode")
|
| 214 |
+
self.arm = arm
|
| 215 |
+
self.weights = _weights(weights_path, arm)
|
| 216 |
+
self.labels, self.vectors, self.vector_norms = _vocabulary(vocabulary)
|
| 217 |
+
|
| 218 |
+
def _linear(self, x, name):
|
| 219 |
+
x = x @ self.weights[name + ".weight"].T
|
| 220 |
+
bias = self.weights.get(name + ".bias")
|
| 221 |
+
return x if bias is None else x + bias
|
| 222 |
+
|
| 223 |
+
def _norm(self, x, name):
|
| 224 |
+
mean = x.mean(axis=-1, keepdims=True)
|
| 225 |
+
centered = x - mean
|
| 226 |
+
variance = (centered * centered).mean(axis=-1, keepdims=True)
|
| 227 |
+
return centered / np.sqrt(variance + np.float32(1e-5)) * self.weights[name + ".weight"] + self.weights[name + ".bias"]
|
| 228 |
+
|
| 229 |
+
def _embed(self, rows, positions, cache=None):
|
| 230 |
+
x = self._norm(self._linear(rows, "inputProjection"), "inputNorm")
|
| 231 |
+
phase = positions.astype(np.float32)[:, None] * FREQUENCIES
|
| 232 |
+
x += np.concatenate((np.sin(phase), np.cos(phase)), axis=-1)
|
| 233 |
+
if cache is not None:
|
| 234 |
+
old_positions, old_layers = cache
|
| 235 |
+
key_positions = np.concatenate((old_positions, positions))
|
| 236 |
+
recent = positions[-1] - (CONTEXT - 1)
|
| 237 |
+
retain = int(np.searchsorted(key_positions, recent))
|
| 238 |
+
mask = (key_positions[None, :] <= positions[:, None]) & (key_positions[None, :] >= positions[:, None] - (CONTEXT - 1))
|
| 239 |
+
retained = []
|
| 240 |
+
for block in range(4):
|
| 241 |
+
name = f"encoderLayers.{block}"
|
| 242 |
+
z = self._norm(x, name + ".norm1")
|
| 243 |
+
attention = name + ".attention."
|
| 244 |
+
queries, keys, values = [
|
| 245 |
+
self._linear(z, attention + layer).reshape(len(x), HEADS, -1).transpose(1, 0, 2)
|
| 246 |
+
for layer in ("query_proj", "key_proj", "value_proj")
|
| 247 |
+
]
|
| 248 |
+
if cache is not None and old_layers:
|
| 249 |
+
old_keys, old_values = old_layers[block]
|
| 250 |
+
keys = np.concatenate((old_keys, keys), axis=1)
|
| 251 |
+
values = np.concatenate((old_values, values), axis=1)
|
| 252 |
+
scores = queries @ keys.swapaxes(-1, -2) * np.float32(1 / 8)
|
| 253 |
+
if cache is not None:
|
| 254 |
+
scores = np.where(mask, scores, np.float32(-np.inf))
|
| 255 |
+
retained.append((keys[:, retain:].copy(), values[:, retain:].copy()))
|
| 256 |
+
joined = (_softmax(scores) @ values).transpose(1, 0, 2).reshape(len(x), HIDDEN)
|
| 257 |
+
x += self._linear(joined, attention + "out_proj") * self.weights[name + ".gamma1"]
|
| 258 |
+
z = self._linear(self._norm(x, name + ".norm2"), name + ".mlp.layers.0")
|
| 259 |
+
z = _gelu(z)
|
| 260 |
+
x += self._linear(z, name + ".mlp.layers.3") * self.weights[name + ".gamma2"]
|
| 261 |
+
x = self._norm(x, "outputNorm")
|
| 262 |
+
if cache is None:
|
| 263 |
+
keys = self._linear(x, "pool.keyProj")
|
| 264 |
+
values = self._linear(x, "pool.valueProj")
|
| 265 |
+
scores = self.weights["pool.query"].reshape(1, HIDDEN) @ keys.T * np.float32(1 / 16)
|
| 266 |
+
x = _softmax(scores) @ values
|
| 267 |
+
next_cache = None
|
| 268 |
+
else:
|
| 269 |
+
next_cache = key_positions[retain:].copy(), retained
|
| 270 |
+
x = self._linear(x, "outputProjection")
|
| 271 |
+
x /= np.sqrt((x * x).sum(axis=-1, keepdims=True) + np.float32(1e-6))
|
| 272 |
+
if not np.isfinite(x).all():
|
| 273 |
+
raise ValueError("Transformer produced non-finite output")
|
| 274 |
+
return x, next_cache
|
| 275 |
+
|
| 276 |
+
def _predict(self, embedding):
|
| 277 |
+
norms = np.linalg.norm(embedding) * self.vector_norms
|
| 278 |
+
scores = np.divide(self.vectors @ embedding, norms, out=np.zeros_like(norms), where=norms > 0)
|
| 279 |
+
best = int(scores.argmax())
|
| 280 |
+
return self.labels[best], float(max(0, scores[best]) * 100)
|
| 281 |
+
|
| 282 |
+
def replay(self, frames, duration):
|
| 283 |
+
"""Yield scheduled predictions and the first latched confirmation."""
|
| 284 |
+
duration = float(duration)
|
| 285 |
+
if not np.isfinite(duration) or not 0 < duration <= 120:
|
| 286 |
+
raise ValueError("Expected a finite clip duration up to 120 seconds")
|
| 287 |
+
poses = _PoseSequence(frames)
|
| 288 |
+
rolling_positions = np.arange(60, dtype=np.int32)
|
| 289 |
+
cache = np.empty(0, dtype=np.int32), []
|
| 290 |
+
previous = confirmed = confirmed_at = None
|
| 291 |
+
streak = 0
|
| 292 |
+
index = 2
|
| 293 |
+
while index * TICK_SECONDS < duration:
|
| 294 |
+
time = index * TICK_SECONDS
|
| 295 |
+
label = score = None
|
| 296 |
+
if self.arm == "rolling":
|
| 297 |
+
rows = poses.rolling(time)
|
| 298 |
+
if rows is not None:
|
| 299 |
+
output, _ = self._embed(rows, rolling_positions)
|
| 300 |
+
label, score = self._predict(output[-1])
|
| 301 |
+
else:
|
| 302 |
+
positions, rows = poses.causal(time)
|
| 303 |
+
for start in range(0, len(positions), CONTEXT):
|
| 304 |
+
stop = start + CONTEXT
|
| 305 |
+
output, cache = self._embed(rows[start:stop], positions[start:stop], cache)
|
| 306 |
+
if len(positions):
|
| 307 |
+
label, score = self._predict(output[-1])
|
| 308 |
+
if label is None:
|
| 309 |
+
streak = 0
|
| 310 |
+
else:
|
| 311 |
+
streak = streak + 1 if label == previous else 1
|
| 312 |
+
if confirmed is None and streak == 3:
|
| 313 |
+
confirmed, confirmed_at = label, time
|
| 314 |
+
previous = label
|
| 315 |
+
yield {"index": index, "time": time, "label": label, "score": score,
|
| 316 |
+
"confirmationLabel": confirmed, "confirmationTime": confirmed_at}
|
| 317 |
+
index += 1
|
transformer_runtime.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Read the pinned checkpoints and Apple Vision validation data on the server."""
|
| 2 |
+
|
| 3 |
+
import gzip
|
| 4 |
+
import hashlib
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
import re
|
| 9 |
+
|
| 10 |
+
import streamlit as st
|
| 11 |
+
|
| 12 |
+
from transformer_inference import Transformer
|
| 13 |
+
|
| 14 |
+
CONTRACT_PATH = Path(__file__).with_name("data") / "transformer_contract.json"
|
| 15 |
+
CONTRACT = json.loads(CONTRACT_PATH.read_text(encoding="utf-8"))
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def read_verified(path, expected_hash):
|
| 19 |
+
payload = Path(path).read_bytes()
|
| 20 |
+
if hashlib.sha256(payload).hexdigest() != expected_hash:
|
| 21 |
+
raise ValueError("A frozen validation file failed its checksum")
|
| 22 |
+
return payload
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def load_validation(download):
|
| 26 |
+
"""Return only the approved validation rows, ordered by native clip ID."""
|
| 27 |
+
files = CONTRACT["dataset"]["files"]
|
| 28 |
+
vocab_path = "metadata/vocabulary.json"
|
| 29 |
+
vocabulary_rows = json.loads(read_verified(download(vocab_path), files[vocab_path]))
|
| 30 |
+
vectors = {row["name"]: row["embedding"] for row in vocabulary_rows if row["type"] == "word"}
|
| 31 |
+
labels = CONTRACT["classNames"]
|
| 32 |
+
vocabulary = [{"word": label, "vector": vectors[label]} for label in labels]
|
| 33 |
+
allowed = set(labels)
|
| 34 |
+
clips = {}
|
| 35 |
+
seen = set()
|
| 36 |
+
for name, expected in files.items():
|
| 37 |
+
if name == vocab_path:
|
| 38 |
+
continue
|
| 39 |
+
payload = read_verified(download(name), expected)
|
| 40 |
+
for line in gzip.decompress(payload).splitlines():
|
| 41 |
+
row = json.loads(line)
|
| 42 |
+
identity = row["video_id"]
|
| 43 |
+
words = [label["name"] for label in row["labels"] if label["type"] == "word"]
|
| 44 |
+
if (identity in seen or row["split"] != "validation" or row["dataset"] != "INCLUDE"
|
| 45 |
+
or row["extractor"] != "AppleVision" or row["native_format_version"] != 1
|
| 46 |
+
or len(words) != 1):
|
| 47 |
+
raise ValueError("Unexpected validation record")
|
| 48 |
+
seen.add(identity)
|
| 49 |
+
if words[0] not in allowed:
|
| 50 |
+
continue
|
| 51 |
+
frames = []
|
| 52 |
+
for index, frame in enumerate(row["frames"]):
|
| 53 |
+
if frame["frame_index"] != index or frame["source_model"] != "AppleVision":
|
| 54 |
+
raise ValueError("Unexpected frame schema")
|
| 55 |
+
frames.append({
|
| 56 |
+
"timestamp": frame["timestamp_s"],
|
| 57 |
+
"body": frame["body"], "leftHand": frame["left_hand"], "rightHand": frame["right_hand"],
|
| 58 |
+
})
|
| 59 |
+
if not frames or len(frames) != row["frame_count"]:
|
| 60 |
+
raise ValueError("Missing validation frames")
|
| 61 |
+
clips[identity] = (words[0], row["duration_s"], frames)
|
| 62 |
+
if len(seen) != 743 or len(clips) != 720:
|
| 63 |
+
raise ValueError("Frozen validation cohort size differs")
|
| 64 |
+
return vocabulary, [clips[key] for key in sorted(clips)]
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def settings():
|
| 68 |
+
if os.environ.get("ISL_PREVIEW_ASSETS"):
|
| 69 |
+
return "local"
|
| 70 |
+
revision = os.environ.get("ISL_MODEL_REVISION", "")
|
| 71 |
+
if os.environ.get("ISL_ARTIFACT_READ_TOKEN") and re.fullmatch(r"[0-9a-f]{40}", revision):
|
| 72 |
+
return revision
|
| 73 |
+
return None
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
@st.cache_resource
|
| 77 |
+
def load_assets(revision):
|
| 78 |
+
dataset = CONTRACT["dataset"]
|
| 79 |
+
if revision == "local":
|
| 80 |
+
# The local preview accepts a server-configured directory with symlinked
|
| 81 |
+
# assets. It never accepts file paths from visitors.
|
| 82 |
+
root = Path(os.environ["ISL_PREVIEW_ASSETS"])
|
| 83 |
+
vocabulary, clips = load_validation(lambda name: root / "dataset" / name)
|
| 84 |
+
paths = {arm: root / filename for arm, filename in CONTRACT["modelFiles"].items()}
|
| 85 |
+
else:
|
| 86 |
+
from huggingface_hub import hf_hub_download
|
| 87 |
+
token = os.environ["ISL_ARTIFACT_READ_TOKEN"]
|
| 88 |
+
def download(name):
|
| 89 |
+
return hf_hub_download(dataset["id"], name, repo_type="dataset",
|
| 90 |
+
revision=dataset["revision"], token=token)
|
| 91 |
+
vocabulary, clips = load_validation(download)
|
| 92 |
+
paths = {arm: hf_hub_download(CONTRACT["modelRepo"], name, revision=revision, token=token)
|
| 93 |
+
for arm, name in CONTRACT["modelFiles"].items()}
|
| 94 |
+
models = {arm: Transformer(path, vocabulary, arm) for arm, path in paths.items()}
|
| 95 |
+
return clips, models
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def replay_clip(clips, models, index, expected):
|
| 99 |
+
label, duration, frames = clips[index]
|
| 100 |
+
if (label, duration) != (expected["label"], expected["duration"]):
|
| 101 |
+
raise ValueError("Live validation clip differs from the archived panel")
|
| 102 |
+
output = {"id": expected["id"], "label": label, "duration": duration,
|
| 103 |
+
"videoPath": expected["videoPath"]}
|
| 104 |
+
for arm, model in models.items():
|
| 105 |
+
rows = list(model.replay(frames, duration))
|
| 106 |
+
ticks = [{key: row[key] for key in ("time", "label", "score")} for row in rows]
|
| 107 |
+
last = rows[-1]
|
| 108 |
+
early = next((row for row in rows if row["confirmationTime"] is not None), None)
|
| 109 |
+
output[arm] = {
|
| 110 |
+
"last": ticks[-1],
|
| 111 |
+
"early": None if early is None else {
|
| 112 |
+
"time": last["confirmationTime"], "label": last["confirmationLabel"], "score": early["score"],
|
| 113 |
+
},
|
| 114 |
+
"ticks": ticks,
|
| 115 |
+
}
|
| 116 |
+
return output
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
@st.cache_data(max_entries=1)
|
| 120 |
+
def evaluate_validation(revision, expected_clips):
|
| 121 |
+
clips, models = load_assets(revision)
|
| 122 |
+
return [replay_clip(clips, models, index, expected)
|
| 123 |
+
for index, expected in enumerate(expected_clips)]
|
util.py
ADDED
|
@@ -0,0 +1,455 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import math
|
| 3 |
+
import cv2
|
| 4 |
+
import matplotlib
|
| 5 |
+
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
|
| 6 |
+
from matplotlib.figure import Figure
|
| 7 |
+
import numpy as np
|
| 8 |
+
import matplotlib.pyplot as plt
|
| 9 |
+
import cv2
|
| 10 |
+
import copy
|
| 11 |
+
import seaborn as sns
|
| 12 |
+
|
| 13 |
+
def padRightDownCorner(img, stride, padValue):
|
| 14 |
+
h = img.shape[0]
|
| 15 |
+
w = img.shape[1]
|
| 16 |
+
|
| 17 |
+
pad = 4 * [None]
|
| 18 |
+
pad[0] = 0 # up
|
| 19 |
+
pad[1] = 0 # left
|
| 20 |
+
pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down
|
| 21 |
+
pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right
|
| 22 |
+
|
| 23 |
+
img_padded = img
|
| 24 |
+
pad_up = np.tile(img_padded[0:1, :, :]*0 + padValue, (pad[0], 1, 1))
|
| 25 |
+
img_padded = np.concatenate((pad_up, img_padded), axis=0)
|
| 26 |
+
pad_left = np.tile(img_padded[:, 0:1, :]*0 + padValue, (1, pad[1], 1))
|
| 27 |
+
img_padded = np.concatenate((pad_left, img_padded), axis=1)
|
| 28 |
+
pad_down = np.tile(img_padded[-2:-1, :, :]*0 + padValue, (pad[2], 1, 1))
|
| 29 |
+
img_padded = np.concatenate((img_padded, pad_down), axis=0)
|
| 30 |
+
pad_right = np.tile(img_padded[:, -2:-1, :]*0 + padValue, (1, pad[3], 1))
|
| 31 |
+
img_padded = np.concatenate((img_padded, pad_right), axis=1)
|
| 32 |
+
|
| 33 |
+
return img_padded, pad
|
| 34 |
+
|
| 35 |
+
# transfer caffe model to pytorch which will match the layer name
|
| 36 |
+
def transfer(model, model_weights):
|
| 37 |
+
transfered_model_weights = {}
|
| 38 |
+
for weights_name in model.state_dict().keys():
|
| 39 |
+
if len(weights_name.split('.'))>4: # body25
|
| 40 |
+
transfered_model_weights[weights_name] = model_weights['.'.join(
|
| 41 |
+
weights_name.split('.')[3:])]
|
| 42 |
+
else:
|
| 43 |
+
transfered_model_weights[weights_name] = model_weights['.'.join(
|
| 44 |
+
weights_name.split('.')[1:])]
|
| 45 |
+
return transfered_model_weights
|
| 46 |
+
|
| 47 |
+
# draw the body keypoint and lims
|
| 48 |
+
def draw_bodypose(canvas, candidate, subset, model_type='body25'):
|
| 49 |
+
stickwidth = 4
|
| 50 |
+
if model_type == 'body25':
|
| 51 |
+
limbSeq = [[1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],\
|
| 52 |
+
[10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],\
|
| 53 |
+
[11,24],[11,22],[14,21],[14,19],[22,23],[19,20]]
|
| 54 |
+
njoint = 25
|
| 55 |
+
else:
|
| 56 |
+
limbSeq = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], \
|
| 57 |
+
[9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], \
|
| 58 |
+
[0, 15], [15, 17], [2, 16], [5, 17]]
|
| 59 |
+
njoint = 18
|
| 60 |
+
|
| 61 |
+
# colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
|
| 62 |
+
# [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
|
| 63 |
+
# [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
|
| 64 |
+
|
| 65 |
+
colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
|
| 66 |
+
[0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
|
| 67 |
+
[170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85], [255,255,0], [255,255,85], [255,255,170],\
|
| 68 |
+
[255,255,255],[170,255,255],[85,255,255],[0,255,255]]
|
| 69 |
+
|
| 70 |
+
for i in range(njoint):
|
| 71 |
+
for n in range(len(subset)):
|
| 72 |
+
index = int(subset[n][i])
|
| 73 |
+
if index == -1:
|
| 74 |
+
continue
|
| 75 |
+
x, y = candidate[index][0:2]
|
| 76 |
+
cv2.circle(canvas, (int(x), int(y)), 4, colors[i], thickness=-1)
|
| 77 |
+
for i in range(njoint-1):
|
| 78 |
+
for n in range(len(subset)):
|
| 79 |
+
index = subset[n][np.array(limbSeq[i])]
|
| 80 |
+
if -1 in index:
|
| 81 |
+
continue
|
| 82 |
+
cur_canvas = canvas.copy()
|
| 83 |
+
Y = candidate[index.astype(int), 0]
|
| 84 |
+
X = candidate[index.astype(int), 1]
|
| 85 |
+
mX = np.mean(X)
|
| 86 |
+
mY = np.mean(Y)
|
| 87 |
+
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
|
| 88 |
+
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
|
| 89 |
+
# print('original (mX,mY,length,angle)',(mX,mY,length,angle))
|
| 90 |
+
# print(f'original cv2.ellipse2Poly((int({mY}), int({mX})), (int({length} / 2), {stickwidth}), int({angle}), 0, 360, 1)')
|
| 91 |
+
polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
|
| 92 |
+
# print(f'cv2.fillConvexPoly(cur_canvas, polygon, colors[i])')
|
| 93 |
+
cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
|
| 94 |
+
canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
|
| 95 |
+
# plt.imsave("preview.jpg", canvas[:, :, [2, 1, 0]])
|
| 96 |
+
# plt.imshow(canvas[:, :, [2, 1, 0]])
|
| 97 |
+
return canvas
|
| 98 |
+
#subsets [[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, -1.0, 11.0, 12.0, -1.0, 13.0, 14.0, 15.0, 16.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 26.650803712300775, 17.0]]
|
| 99 |
+
#candidates [[983.0, 172.0, 0.8991263508796692, 0.0], [980.0, 352.0, 0.930037796497345, 1.0], [848.0, 342.0, 0.8652207255363464, 2.0], [811.0, 598.0, 0.8107873797416687, 3.0], [806.0, 817.0, 0.7464589476585388, 4.0], [1120.0, 361.0, 0.8538270592689514, 5.0], [1148.0, 601.0, 0.6797391176223755, 6.0], [1149.0, 834.0, 0.5189468264579773, 7.0], [968.0, 757.0, 0.6468111276626587, 8.0], [876.0, 756.0, 0.6387956142425537, 9.0], [854.0, 1072.0, 0.4211728572845459, 10.0], [1057.0, 759.0, 0.6311940550804138, 11.0], [1038.0, 1072.0, 0.38531172275543213, 12.0], [955.0, 146.0, 0.925083339214325, 13.0], [1016.0, 151.0, 0.9023998379707336, 14.0], [909.0, 167.0, 0.9096773862838745, 15.0], [1057.0, 173.0, 0.8605436086654663, 16.0]]
|
| 100 |
+
def get_bodypose(candidate, subset, model_type='coco'):
|
| 101 |
+
stickwidth = 4
|
| 102 |
+
if model_type == 'body25':
|
| 103 |
+
limbSeq = [[1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],\
|
| 104 |
+
[10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],\
|
| 105 |
+
[11,24],[11,22],[14,21],[14,19],[22,23],[19,20]]
|
| 106 |
+
njoint = 25
|
| 107 |
+
else:
|
| 108 |
+
limbSeq = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], \
|
| 109 |
+
[9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], \
|
| 110 |
+
[0, 15], [15, 17], [2, 16], [5, 17]]
|
| 111 |
+
njoint = 18
|
| 112 |
+
|
| 113 |
+
# colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
|
| 114 |
+
# [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
|
| 115 |
+
# [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
|
| 116 |
+
|
| 117 |
+
colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
|
| 118 |
+
[0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
|
| 119 |
+
[170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85], [255,255,0], [255,255,85], [255,255,170],\
|
| 120 |
+
[255,255,255],[170,255,255],[85,255,255],[0,255,255]]
|
| 121 |
+
|
| 122 |
+
x_y_circles=[]
|
| 123 |
+
for i in range(njoint):
|
| 124 |
+
for n in range(len(subset)):
|
| 125 |
+
index = int(subset[n][i])
|
| 126 |
+
if index == -1:
|
| 127 |
+
continue
|
| 128 |
+
x, y = candidate[index][0:2] # 983.0, 172.0
|
| 129 |
+
x_y_circles.append((x, y))
|
| 130 |
+
# cv2.circle(canvas, (int(x), int(y)), 4, colors[i], thickness=-1)
|
| 131 |
+
|
| 132 |
+
x_y_sticks=[]
|
| 133 |
+
for i in range(njoint-1):
|
| 134 |
+
for n in range(len(subset)):
|
| 135 |
+
index = subset[n][np.array(limbSeq[i])] #0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, -1.0, 11.0, 12.0, -1.0, 13.0, 14.0, 15.0, 16.0, -1.0, -1.0, -1.0, -1.0, -1.0
|
| 136 |
+
if -1 in index:
|
| 137 |
+
continue
|
| 138 |
+
# cur_canvas = canvas.copy()
|
| 139 |
+
Y = candidate[index.astype(int), 0]
|
| 140 |
+
X = candidate[index.astype(int), 1]
|
| 141 |
+
mX = np.mean(X)
|
| 142 |
+
mY = np.mean(Y)
|
| 143 |
+
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
|
| 144 |
+
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
|
| 145 |
+
x_y_sticks.append((mY, mX,angle,length))
|
| 146 |
+
# print('new (mX,mY,length,angle)',(mX,mY,length,angle))
|
| 147 |
+
# polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
|
| 148 |
+
# cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
|
| 149 |
+
# canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
|
| 150 |
+
# plt.imsave("preview.jpg", canvas[:, :, [2, 1, 0]])
|
| 151 |
+
# plt.imshow(canvas[:, :, [2, 1, 0]])
|
| 152 |
+
return (x_y_circles,x_y_sticks,)
|
| 153 |
+
|
| 154 |
+
#all_hands_peaks[[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [1100, 858], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0]], [[0, 0], [858, 859], [868, 894], [873, 938], [0, 0], [802, 920], [807, 961], [821, 977], [836, 992], [0, 0], [781, 955], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]]
|
| 155 |
+
def draw_handpose(canvas, all_hand_peaks, show_number=False):
|
| 156 |
+
edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], \
|
| 157 |
+
[10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]]
|
| 158 |
+
fig = Figure(figsize=plt.figaspect(canvas))
|
| 159 |
+
|
| 160 |
+
fig.subplots_adjust(0, 0, 1, 1)
|
| 161 |
+
fig.subplots_adjust(bottom=0, top=1, left=0, right=1)
|
| 162 |
+
bg = FigureCanvas(fig)
|
| 163 |
+
ax = fig.subplots()
|
| 164 |
+
ax.axis('off')
|
| 165 |
+
ax.imshow(canvas)
|
| 166 |
+
|
| 167 |
+
width, height = ax.figure.get_size_inches() * ax.figure.get_dpi()
|
| 168 |
+
|
| 169 |
+
for peaks in all_hand_peaks:
|
| 170 |
+
for ie, e in enumerate(edges):
|
| 171 |
+
if np.sum(np.all(peaks[e], axis=1)==0)==0:
|
| 172 |
+
x1, y1 = peaks[e[0]]
|
| 173 |
+
x2, y2 = peaks[e[1]]
|
| 174 |
+
# print(f'original ax.plot([{x1}, {x2}], [{y1}, {y2}], color=matplotlib.colors.hsv_to_rgb([ie/float({len(edges)}), 1.0, 1.0]))')
|
| 175 |
+
ax.plot([x1, x2], [y1, y2], color=matplotlib.colors.hsv_to_rgb([ie/float(len(edges)), 1.0, 1.0]))
|
| 176 |
+
|
| 177 |
+
for i, keyponit in enumerate(peaks):
|
| 178 |
+
x, y = keyponit
|
| 179 |
+
# print(f"original ax.plot({x}, {y}, 'r.')")
|
| 180 |
+
ax.plot(x, y, 'r.')
|
| 181 |
+
if show_number:
|
| 182 |
+
ax.text(x, y, str(i))
|
| 183 |
+
# print(f'width = {width}, height={height}')
|
| 184 |
+
bg.draw()
|
| 185 |
+
canvas = np.fromstring(bg.tostring_rgb(), dtype='uint8').reshape(int(height), int(width), 3)
|
| 186 |
+
return canvas
|
| 187 |
+
|
| 188 |
+
def get_handpose(all_hand_peaks, show_number=False):
|
| 189 |
+
edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], \
|
| 190 |
+
[10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]]
|
| 191 |
+
# fig = Figure(figsize=plt.figaspect(canvas))
|
| 192 |
+
|
| 193 |
+
# fig.subplots_adjust(0, 0, 1, 1)
|
| 194 |
+
# fig.subplots_adjust(bottom=0, top=1, left=0, right=1)
|
| 195 |
+
# bg = FigureCanvas(fig)
|
| 196 |
+
# ax = fig.subplots()
|
| 197 |
+
# ax.axis('off')
|
| 198 |
+
# ax.imshow(canvas)
|
| 199 |
+
|
| 200 |
+
# width, height = ax.figure.get_size_inches() * ax.figure.get_dpi()
|
| 201 |
+
export_edges=[[],[]]
|
| 202 |
+
export_peaks=[[],[]]
|
| 203 |
+
for idx,peaks in enumerate(all_hand_peaks):
|
| 204 |
+
for ie, e in enumerate(edges):
|
| 205 |
+
if np.sum(np.all(peaks[e], axis=1)==0)==0:
|
| 206 |
+
x1, y1 = peaks[e[0]]
|
| 207 |
+
x2, y2 = peaks[e[1]]
|
| 208 |
+
export_edges[idx].append((ie,(x1, y1),(x2, y2)))
|
| 209 |
+
# ax.plot([x1, x2], [y1, y2], color=matplotlib.colors.hsv_to_rgb([ie/float(len(edges)), 1.0, 1.0]))
|
| 210 |
+
|
| 211 |
+
for i, keyponit in enumerate(peaks):
|
| 212 |
+
x, y = keyponit
|
| 213 |
+
# ax.plot(x, y, 'r.')
|
| 214 |
+
# if show_number:
|
| 215 |
+
# ax.text(x, y, str(i))
|
| 216 |
+
|
| 217 |
+
export_peaks[idx].append((x,y,str(i)))
|
| 218 |
+
# bg.draw()
|
| 219 |
+
# canvas = np.fromstring(bg.tostring_rgb(), dtype='uint8').reshape(int(height), int(width), 3)
|
| 220 |
+
return (export_edges,export_peaks)
|
| 221 |
+
|
| 222 |
+
# image drawed by opencv is not good.
|
| 223 |
+
def draw_handpose_by_opencv(canvas, peaks, show_number=False):
|
| 224 |
+
edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], \
|
| 225 |
+
[10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]]
|
| 226 |
+
# cv2.rectangle(canvas, (x, y), (x+w, y+w), (0, 255, 0), 2, lineType=cv2.LINE_AA)
|
| 227 |
+
# cv2.putText(canvas, 'left' if is_left else 'right', (x, y), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
|
| 228 |
+
for ie, e in enumerate(edges):
|
| 229 |
+
if np.sum(np.all(peaks[e], axis=1)==0)==0:
|
| 230 |
+
x1, y1 = peaks[e[0]]
|
| 231 |
+
x2, y2 = peaks[e[1]]
|
| 232 |
+
cv2.line(canvas, (x1, y1), (x2, y2), matplotlib.colors.hsv_to_rgb([ie/float(len(edges)), 1.0, 1.0])*255, thickness=2)
|
| 233 |
+
|
| 234 |
+
for i, keyponit in enumerate(peaks):
|
| 235 |
+
x, y = keyponit
|
| 236 |
+
cv2.circle(canvas, (x, y), 4, (0, 0, 255), thickness=-1)
|
| 237 |
+
if show_number:
|
| 238 |
+
cv2.putText(canvas, str(i), (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 0), lineType=cv2.LINE_AA)
|
| 239 |
+
return canvas
|
| 240 |
+
|
| 241 |
+
# detect hand according to body pose keypoints
|
| 242 |
+
# please refer to https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/hand/handDetector.cpp
|
| 243 |
+
def handDetect(candidate, subset, oriImg):
|
| 244 |
+
# right hand: wrist 4, elbow 3, shoulder 2
|
| 245 |
+
# left hand: wrist 7, elbow 6, shoulder 5
|
| 246 |
+
ratioWristElbow = 0.33
|
| 247 |
+
detect_result = []
|
| 248 |
+
|
| 249 |
+
image_height, image_width = oriImg.shape[0:2]
|
| 250 |
+
#print(f'handDetect ---------- {image_height}, {image_width}')
|
| 251 |
+
for person in subset.astype(int):
|
| 252 |
+
# if any of three not detected
|
| 253 |
+
has_left = np.sum(person[[5, 6, 7]] == -1) == 0
|
| 254 |
+
has_right = np.sum(person[[2, 3, 4]] == -1) == 0
|
| 255 |
+
if not (has_left or has_right):
|
| 256 |
+
continue
|
| 257 |
+
hands = []
|
| 258 |
+
#left hand
|
| 259 |
+
if has_left:
|
| 260 |
+
left_shoulder_index, left_elbow_index, left_wrist_index = person[[5, 6, 7]]
|
| 261 |
+
x1, y1 = candidate[left_shoulder_index][:2]
|
| 262 |
+
x2, y2 = candidate[left_elbow_index][:2]
|
| 263 |
+
x3, y3 = candidate[left_wrist_index][:2]
|
| 264 |
+
hands.append([x1, y1, x2, y2, x3, y3, True])
|
| 265 |
+
# right hand
|
| 266 |
+
if has_right:
|
| 267 |
+
right_shoulder_index, right_elbow_index, right_wrist_index = person[[2, 3, 4]]
|
| 268 |
+
x1, y1 = candidate[right_shoulder_index][:2]
|
| 269 |
+
x2, y2 = candidate[right_elbow_index][:2]
|
| 270 |
+
x3, y3 = candidate[right_wrist_index][:2]
|
| 271 |
+
hands.append([x1, y1, x2, y2, x3, y3, False])
|
| 272 |
+
|
| 273 |
+
for x1, y1, x2, y2, x3, y3, is_left in hands:
|
| 274 |
+
# pos_hand = pos_wrist + ratio * (pos_wrist - pos_elbox) = (1 + ratio) * pos_wrist - ratio * pos_elbox
|
| 275 |
+
# handRectangle.x = posePtr[wrist*3] + ratioWristElbow * (posePtr[wrist*3] - posePtr[elbow*3]);
|
| 276 |
+
# handRectangle.y = posePtr[wrist*3+1] + ratioWristElbow * (posePtr[wrist*3+1] - posePtr[elbow*3+1]);
|
| 277 |
+
# const auto distanceWristElbow = getDistance(poseKeypoints, person, wrist, elbow);
|
| 278 |
+
# const auto distanceElbowShoulder = getDistance(poseKeypoints, person, elbow, shoulder);
|
| 279 |
+
# handRectangle.width = 1.5f * fastMax(distanceWristElbow, 0.9f * distanceElbowShoulder);
|
| 280 |
+
x = x3 + ratioWristElbow * (x3 - x2)
|
| 281 |
+
y = y3 + ratioWristElbow * (y3 - y2)
|
| 282 |
+
distanceWristElbow = math.sqrt((x3 - x2) ** 2 + (y3 - y2) ** 2)
|
| 283 |
+
distanceElbowShoulder = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)
|
| 284 |
+
width = 1.5 * max(distanceWristElbow, 0.9 * distanceElbowShoulder)
|
| 285 |
+
# x-y refers to the center --> offset to topLeft point
|
| 286 |
+
# handRectangle.x -= handRectangle.width / 2.f;
|
| 287 |
+
# handRectangle.y -= handRectangle.height / 2.f;
|
| 288 |
+
x -= width / 2
|
| 289 |
+
y -= width / 2 # width = height
|
| 290 |
+
# overflow the image
|
| 291 |
+
if x < 0: x = 0
|
| 292 |
+
if y < 0: y = 0
|
| 293 |
+
width1 = width
|
| 294 |
+
width2 = width
|
| 295 |
+
if x + width > image_width: width1 = image_width - x
|
| 296 |
+
if y + width > image_height: width2 = image_height - y
|
| 297 |
+
width = min(width1, width2)
|
| 298 |
+
# the max hand box value is 20 pixels
|
| 299 |
+
if width >= 20:
|
| 300 |
+
detect_result.append([int(x), int(y), int(width), is_left])
|
| 301 |
+
|
| 302 |
+
'''
|
| 303 |
+
return value: [[x, y, w, True if left hand else False]].
|
| 304 |
+
width=height since the network require squared input.
|
| 305 |
+
x, y is the coordinate of top left
|
| 306 |
+
'''
|
| 307 |
+
return detect_result
|
| 308 |
+
|
| 309 |
+
def drawStickmodel(oriImg, x_ytupple, x_y_sticks, export_edges, export_peaks):
|
| 310 |
+
canvas = copy.deepcopy(oriImg)
|
| 311 |
+
|
| 312 |
+
colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0],
|
| 313 |
+
[85, 255, 0], [0, 255, 0], [0, 255, 85], [0, 255, 170], [0, 255, 255],
|
| 314 |
+
[0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], [170, 0, 255],
|
| 315 |
+
[255, 0, 255], [255, 0, 170], [255, 0, 85], [255,255,0], [255,255,85],
|
| 316 |
+
[255,255,170], [255,255,255],[170,255,255],[85,255,255],[0,255,255]]
|
| 317 |
+
stickwidth = 4
|
| 318 |
+
|
| 319 |
+
for idx, (mX, mY, angle, length) in enumerate(x_y_sticks):
|
| 320 |
+
cur_canvas = canvas.copy()
|
| 321 |
+
polygon = cv2.ellipse2Poly((int(mX), int(mY)), (int(length / 2), stickwidth),
|
| 322 |
+
int(angle), 0, 360, 1)
|
| 323 |
+
cv2.fillConvexPoly(cur_canvas, polygon, colors[idx])
|
| 324 |
+
canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
|
| 325 |
+
|
| 326 |
+
for idx, (x, y) in enumerate(x_ytupple):
|
| 327 |
+
cv2.circle(canvas, (int(x), int(y)), 4, colors[idx], thickness=-1)
|
| 328 |
+
|
| 329 |
+
# Handpose
|
| 330 |
+
fig = Figure(figsize=plt.figaspect(canvas))
|
| 331 |
+
fig.subplots_adjust(0, 0, 1, 1)
|
| 332 |
+
ax = fig.subplots()
|
| 333 |
+
ax.axis('off')
|
| 334 |
+
ax.imshow(canvas)
|
| 335 |
+
|
| 336 |
+
edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9],
|
| 337 |
+
[9, 10], [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16],
|
| 338 |
+
[0, 17], [17, 18], [18, 19], [19, 20]]
|
| 339 |
+
|
| 340 |
+
for both_hand_edges in export_edges:
|
| 341 |
+
for (ie, (x1, y1), (x2, y2)) in both_hand_edges:
|
| 342 |
+
ax.plot([x1, x2], [y1, y2],
|
| 343 |
+
color=matplotlib.colors.hsv_to_rgb([ie/float(len(edges)), 1.0, 1.0]))
|
| 344 |
+
|
| 345 |
+
for both_hand_peaks in export_peaks:
|
| 346 |
+
for (x, y, text) in both_hand_peaks:
|
| 347 |
+
ax.plot(x, y, 'r.')
|
| 348 |
+
|
| 349 |
+
# Convert figure to numpy array
|
| 350 |
+
bg = FigureCanvas(fig)
|
| 351 |
+
bg.draw()
|
| 352 |
+
|
| 353 |
+
width, height = fig.get_size_inches() * fig.get_dpi()
|
| 354 |
+
buf = bg.buffer_rgba()
|
| 355 |
+
canvas = np.frombuffer(buf, dtype=np.uint8).reshape(int(height), int(width), 4)
|
| 356 |
+
canvas = canvas[:, :, :3] # keep only RGB
|
| 357 |
+
|
| 358 |
+
plt.close(fig) # clean up
|
| 359 |
+
return cv2.resize(canvas, (math.ceil(width), math.ceil(height)))
|
| 360 |
+
|
| 361 |
+
def draw_bar_plot_below_image(image, predictions, title, origImg):
|
| 362 |
+
"""
|
| 363 |
+
Draws a bar plot of predictions below an image using OpenCV and Matplotlib.
|
| 364 |
+
|
| 365 |
+
Args:
|
| 366 |
+
image (numpy.ndarray): The image to display.
|
| 367 |
+
predictions (numpy.ndarray): Array containing prediction probabilities.
|
| 368 |
+
"""
|
| 369 |
+
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
fig, ax = plt.subplots(figsize=(origImg.shape[1]/100,origImg.shape[0]/200), dpi=100)
|
| 373 |
+
plt.title(title)
|
| 374 |
+
# Create a figure and plot the bar chart
|
| 375 |
+
labels = list(predictions.keys())
|
| 376 |
+
probabilities = list(predictions.values())
|
| 377 |
+
|
| 378 |
+
# Create a Seaborn bar plot
|
| 379 |
+
sns.barplot(x=labels, y=probabilities,ax=ax) # Default color palette used
|
| 380 |
+
plt.close(fig) # Close plot to avoid memory leaks
|
| 381 |
+
fig.canvas.draw()
|
| 382 |
+
# Convert the plot to a NumPy array for manipulation
|
| 383 |
+
plot_image = np.array(fig.canvas.renderer.buffer_rgba())[:, :, :3] # Remove alpha channel
|
| 384 |
+
|
| 385 |
+
# Resize the plot image to match the width of the original image
|
| 386 |
+
# plot_image = cv2.resize(plot_image, (image.shape[1], math.ceil(image.shape[0] * 0.8))) # Adjust height ratio as needed
|
| 387 |
+
|
| 388 |
+
# Combine the image and plot image vertically (stacking)
|
| 389 |
+
combined_image = np.vstack((image, cv2.resize(plot_image,(image.shape[1],plot_image.shape[0]))))
|
| 390 |
+
|
| 391 |
+
return combined_image
|
| 392 |
+
|
| 393 |
+
def add_padding_to_bottom(image, pad_value, pad_height):
|
| 394 |
+
"""
|
| 395 |
+
Adds padding to the bottom of an image with a specified value.
|
| 396 |
+
|
| 397 |
+
Args:
|
| 398 |
+
image (numpy.ndarray): The input image.
|
| 399 |
+
pad_value (tuple or int): The color value to fill the padding area.
|
| 400 |
+
pad_height (int): The height of the padding to add at the bottom.
|
| 401 |
+
|
| 402 |
+
Returns:
|
| 403 |
+
numpy.ndarray: The image with padding added.
|
| 404 |
+
"""
|
| 405 |
+
|
| 406 |
+
# Get image dimensions
|
| 407 |
+
height, width, channels = image.shape
|
| 408 |
+
padding=np.zeros((pad_height, width, channels), dtype=image.dtype)
|
| 409 |
+
padding[:,:,:]=pad_value
|
| 410 |
+
# # Create a new image with the desired height
|
| 411 |
+
# padded_image = np.zeros((height + pad_height, width, channels), dtype=image.dtype)
|
| 412 |
+
|
| 413 |
+
# # Copy the original image to the top of the padded image
|
| 414 |
+
# padded_image[:height, :, :] = image
|
| 415 |
+
|
| 416 |
+
# # Fill the padding area with the specified value
|
| 417 |
+
# if isinstance(pad_value, tuple): # Check for multiple color values (e.g., BGR)
|
| 418 |
+
# padded_image[height:, :, :] = pad_value
|
| 419 |
+
# else: # Single value for all channels (e.g., black)
|
| 420 |
+
# padded_image[height:, :, :] = np.full((pad_height, width, 1), pad_value, dtype=image.dtype)
|
| 421 |
+
|
| 422 |
+
return np.vstack((image, padding))
|
| 423 |
+
|
| 424 |
+
def crop_to_drawing(image):
|
| 425 |
+
"""
|
| 426 |
+
Crops an image to the tight bounding rectangle of non-zero pixels.
|
| 427 |
+
|
| 428 |
+
Args:
|
| 429 |
+
image: A NumPy array representing the image.
|
| 430 |
+
|
| 431 |
+
Returns:
|
| 432 |
+
A cropped image (NumPy array) containing only the drawing area.
|
| 433 |
+
"""
|
| 434 |
+
image=np.transpose(image, (2, 0, 1))
|
| 435 |
+
united_x,united_h=0,0
|
| 436 |
+
for channel in np.arange(image.shape[0]):
|
| 437 |
+
x, y, w, h = cv2.boundingRect(image[channel])
|
| 438 |
+
if x>united_x:
|
| 439 |
+
united_x=x
|
| 440 |
+
|
| 441 |
+
if h>united_h:
|
| 442 |
+
united_h=h
|
| 443 |
+
|
| 444 |
+
for channel in np.arange(image.shape[0]):
|
| 445 |
+
# Crop the image
|
| 446 |
+
image[channel] = image[channel][y:y+united_h, x:x+united_x]
|
| 447 |
+
return image.transpose(image, (1,2,0))
|
| 448 |
+
|
| 449 |
+
# get max index of 2d array
|
| 450 |
+
def npmax(array):
|
| 451 |
+
arrayindex = array.argmax(1)
|
| 452 |
+
arrayvalue = array.max(1)
|
| 453 |
+
i = arrayvalue.argmax()
|
| 454 |
+
j = arrayindex[i]
|
| 455 |
+
return i, j
|
validation_playback.html
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<style>
|
| 6 |
+
body { margin: 0; font: 15px system-ui, sans-serif; color: #17212d; background: #fff; }
|
| 7 |
+
video { display: block; width: 100%; max-height: 248px; background: #111; border-radius: 8px; }
|
| 8 |
+
label { display: block; margin: 8px 0; }
|
| 9 |
+
input { width: calc(100% - 4px); margin: 0; }
|
| 10 |
+
#predictions { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 8px; }
|
| 11 |
+
article { border: 1px solid #bbc4cf; border-radius: 8px; padding: 10px; }
|
| 12 |
+
h2 { font-size: 14px; margin: 0 0 6px; }
|
| 13 |
+
p { margin: 4px 0; }
|
| 14 |
+
@media (prefers-color-scheme: dark) {
|
| 15 |
+
body { color: #e5e7eb; background: #111827; }
|
| 16 |
+
article { border-color: #667185; }
|
| 17 |
+
}
|
| 18 |
+
</style>
|
| 19 |
+
</head>
|
| 20 |
+
<body>
|
| 21 |
+
<video id="video" controls preload="none" playsinline hidden></video>
|
| 22 |
+
<p id="status" role="status"></p>
|
| 23 |
+
<label for="position">Playback time <output id="seconds" for="position">0.00 s</output></label>
|
| 24 |
+
<input type="range" id="position" min="0" step="0.01">
|
| 25 |
+
<section id="predictions" aria-label="Model predictions"></section>
|
| 26 |
+
<script>
|
| 27 |
+
const data = /* PAYLOAD */;
|
| 28 |
+
const video = document.getElementById("video");
|
| 29 |
+
const slider = document.getElementById("position");
|
| 30 |
+
const status = document.getElementById("status");
|
| 31 |
+
const seconds = document.getElementById("seconds");
|
| 32 |
+
const cards = {};
|
| 33 |
+
slider.max = data.duration;
|
| 34 |
+
if (data.video) {
|
| 35 |
+
video.hidden = false;
|
| 36 |
+
video.src = data.video;
|
| 37 |
+
video.addEventListener("error", () => {
|
| 38 |
+
status.textContent = "This browser could not play the source video. You can still use the timeline slider.";
|
| 39 |
+
});
|
| 40 |
+
}
|
| 41 |
+
for (const [arm, name] of Object.entries(data.arms)) {
|
| 42 |
+
const card = document.createElement("article");
|
| 43 |
+
const title = document.createElement("h2");
|
| 44 |
+
title.textContent = name;
|
| 45 |
+
const guess = document.createElement("p");
|
| 46 |
+
const confirmed = document.createElement("p");
|
| 47 |
+
card.append(title, guess, confirmed);
|
| 48 |
+
document.getElementById("predictions").append(card);
|
| 49 |
+
cards[arm] = {guess, confirmed};
|
| 50 |
+
}
|
| 51 |
+
function update(time) {
|
| 52 |
+
slider.value = time;
|
| 53 |
+
seconds.textContent = time.toFixed(2) + " s";
|
| 54 |
+
for (const [arm, {guess, confirmed}] of Object.entries(cards)) {
|
| 55 |
+
const result = data.predictions[arm];
|
| 56 |
+
let last;
|
| 57 |
+
for (const tick of result.ticks) {
|
| 58 |
+
if (tick.time > time) break;
|
| 59 |
+
last = tick;
|
| 60 |
+
}
|
| 61 |
+
guess.textContent = "Current guess: " + (last?.label ?? "—");
|
| 62 |
+
confirmed.textContent = "Confirmed: " + (
|
| 63 |
+
result.early && result.early.time <= time
|
| 64 |
+
? result.early.label + " at " + result.early.time.toFixed(2) + " s"
|
| 65 |
+
: "—"
|
| 66 |
+
);
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
slider.addEventListener("input", () => {
|
| 70 |
+
const time = Number(slider.value);
|
| 71 |
+
if (data.video && video.readyState >= 1) video.currentTime = time;
|
| 72 |
+
update(time);
|
| 73 |
+
});
|
| 74 |
+
video.addEventListener("timeupdate", () => update(video.currentTime));
|
| 75 |
+
update(0);
|
| 76 |
+
</script>
|
| 77 |
+
</body>
|
| 78 |
+
</html>
|