spsarolkar commited on
Commit
e35b538
·
verified ·
1 Parent(s): c67d0a1

Deploy SignLanguageTranslate with Transformer evaluation

Browse files
.dockerignore ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ .git
2
+ **/__pycache__
3
+ **/*.pyc
4
+ .env
5
+ .env.*
6
+ private-assets
7
+ *.safetensors
8
+ *.mobileprovision
.gitattributes CHANGED
@@ -1,35 +1,3 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ eda/train_test_validation_split-1.png filter=lfs diff=lfs merge=lfs -text
2
+ eda/train_test_validation_split-2.png filter=lfs diff=lfs merge=lfs -text
3
+ data/include_validation.json.gz filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
Dockerfile CHANGED
@@ -1,20 +1,29 @@
1
- FROM python:3.13.5-slim
2
 
3
- WORKDIR /app
 
 
 
 
 
 
 
4
 
5
- RUN apt-get update && apt-get install -y \
6
- build-essential \
7
- curl \
8
- git \
9
- && rm -rf /var/lib/apt/lists/*
10
 
11
- COPY requirements.txt ./
12
- COPY src/ ./src/
13
 
14
- RUN pip3 install -r requirements.txt
 
 
 
 
15
 
16
- EXPOSE 8501
 
17
 
18
- HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
19
-
20
- ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"]
 
1
+ FROM python:3.11-slim-bookworm
2
 
3
+ ENV PYTHONUNBUFFERED=1 \
4
+ PYTHONDONTWRITEBYTECODE=1 \
5
+ PIP_DISABLE_PIP_VERSION_CHECK=1 \
6
+ HOME=/home/user \
7
+ PATH=/home/user/.local/bin:$PATH \
8
+ KERAS_BACKEND=torch \
9
+ OPENBLAS_NUM_THREADS=1 \
10
+ OMP_NUM_THREADS=1
11
 
12
+ RUN apt-get update \
13
+ && apt-get install -y --no-install-recommends ffmpeg \
14
+ && rm -rf /var/lib/apt/lists/* \
15
+ && useradd -m -u 1000 user
 
16
 
17
+ USER user
18
+ WORKDIR /home/user/app
19
 
20
+ COPY --chown=user:user requirements-docker.txt ./
21
+ RUN python -m pip install --user --no-cache-dir \
22
+ --index-url https://download.pytorch.org/whl/cpu torch==2.6.0 \
23
+ && python -m pip install --user --no-cache-dir -r requirements-docker.txt \
24
+ && python -m pip check
25
 
26
+ COPY --chown=user:user . .
27
+ EXPOSE 7860
28
 
29
+ CMD ["streamlit", "run", "app.py", "--server.address=0.0.0.0", "--server.port=7860", "--server.headless=true", "--browser.gatherUsageStats=false"]
 
 
ISL_Model_parameter.py ADDED
@@ -0,0 +1,662 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import keras
2
+ from keras.layers import TorchModuleWrapper
3
+ import numpy as np
4
+ import cv2
5
+ import torch
6
+ from scipy.ndimage.filters import gaussian_filter
7
+ import math
8
+ import os
9
+ import numpy as np
10
+ from skimage.measure import label
11
+ import util as util
12
+
13
+
14
+ class ISLSignPos(keras.Model):
15
+ def __init__(self,pt_body_model,pt_hand_model):
16
+ super().__init__()
17
+ self.pt_body = TorchModuleWrapper(pt_body_model)
18
+ self.pt_body.trainable=False
19
+ self.pt_hand = TorchModuleWrapper(pt_hand_model)
20
+ self.pt_hand.trainable=False
21
+ self.njoint_body = 26
22
+ self.npaf_body = 52
23
+
24
+ def call(self, oriImg):
25
+ candidate, subset = self.bodypos(oriImg.cpu().numpy())
26
+ hands_list = util.handDetect(candidate, subset, oriImg.cpu().numpy())
27
+ all_hand_peaks = []
28
+ for x, y, w, is_left in hands_list:
29
+ peaks = self.handpos(oriImg.cpu().numpy()[y:y+w, x:x+w, :])
30
+ peaks[:, 0] = np.where(peaks[:, 0]==0, peaks[:, 0], peaks[:, 0]+x)
31
+ peaks[:, 1] = np.where(peaks[:, 1]==0, peaks[:, 1], peaks[:, 1]+y)
32
+ all_hand_peaks.append(peaks)
33
+ return (candidate, subset,all_hand_peaks)
34
+
35
+ def bodypos(self, oriImg):
36
+ model_type = 'body25'
37
+ scale_search = [0.5]
38
+ boxsize = 368
39
+ stride = 8
40
+ padValue = 128
41
+ thre1 = 0.1
42
+ thre2 = 0.05
43
+ multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search]
44
+ heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], self.njoint_body))
45
+ paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], self.npaf_body))
46
+
47
+ for m in range(len(multiplier)):
48
+ scale = multiplier[m]
49
+ imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
50
+ imageToTest_padded, pad = util.padRightDownCorner(imageToTest, stride, padValue)
51
+ im = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
52
+ im = np.ascontiguousarray(im)
53
+
54
+ data = torch.from_numpy(im).float()
55
+ if torch.cuda.is_available():
56
+ data = data.cuda()
57
+ with torch.no_grad():
58
+ Mconv7_stage6_L1, Mconv7_stage6_L2 = self.pt_body(data)
59
+ Mconv7_stage6_L1 = Mconv7_stage6_L1.cpu().numpy()
60
+ Mconv7_stage6_L2 = Mconv7_stage6_L2.cpu().numpy()
61
+
62
+ heatmap = np.transpose(np.squeeze(Mconv7_stage6_L2), (1, 2, 0)) # output 1 is heatmaps
63
+ heatmap = cv2.resize(heatmap, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
64
+ heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
65
+ heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
66
+
67
+ paf = np.transpose(np.squeeze(Mconv7_stage6_L1), (1, 2, 0)) # output 0 is PAFs
68
+ paf = cv2.resize(paf, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
69
+ paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
70
+ paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
71
+
72
+ heatmap_avg += heatmap_avg + heatmap / len(multiplier)
73
+ paf_avg += + paf / len(multiplier)
74
+
75
+ all_peaks = []
76
+ peak_counter = 0
77
+
78
+ for part in range(self.njoint_body-1):
79
+ map_ori = heatmap_avg[:, :, part]
80
+ one_heatmap = gaussian_filter(map_ori, sigma=3)
81
+
82
+ map_left = np.zeros(one_heatmap.shape)
83
+ map_left[1:, :] = one_heatmap[:-1, :]
84
+ map_right = np.zeros(one_heatmap.shape)
85
+ map_right[:-1, :] = one_heatmap[1:, :]
86
+ map_up = np.zeros(one_heatmap.shape)
87
+ map_up[:, 1:] = one_heatmap[:, :-1]
88
+ map_down = np.zeros(one_heatmap.shape)
89
+ map_down[:, :-1] = one_heatmap[:, 1:]
90
+
91
+ peaks_binary = np.logical_and.reduce(
92
+ (one_heatmap >= map_left, one_heatmap >= map_right, one_heatmap >= map_up, one_heatmap >= map_down, one_heatmap > thre1))
93
+ peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
94
+ peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks]
95
+ peak_id = range(peak_counter, peak_counter + len(peaks))
96
+ peaks_with_score_and_id = [peaks_with_score[i] + (peak_id[i],) for i in range(len(peak_id))]
97
+
98
+ all_peaks.append(peaks_with_score_and_id)
99
+ peak_counter += len(peaks)
100
+
101
+ if model_type=='body25':
102
+ # find connection in the specified sequence, center 29 is in the position 15
103
+ limbSeq = [[1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],\
104
+ [10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],\
105
+ [11,24],[11,22],[14,21],[14,19],[22,23],[19,20]]
106
+ # the middle joints heatmap correpondence
107
+ mapIdx = [[30, 31],[14, 15],[16, 17],[18, 19],[22, 23],[24, 25],[26, 27],[0, 1],[6, 7],\
108
+ [2, 3],[4, 5], [8, 9],[10, 11],[12, 13],[32, 33],[34, 35],[36,37],[38,39],\
109
+ [50,51],[46,47],[44,45],[40,41],[48,49],[42,43]]
110
+ else:
111
+ # find connection in the specified sequence, center 29 is in the position 15
112
+ limbSeq = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], \
113
+ [9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], \
114
+ [0, 15], [15, 17], [2, 16], [5, 17]]
115
+ # the middle joints heatmap correpondence
116
+ mapIdx = [[12, 13], [20, 21], [14, 15], [16, 17], [22, 23], [24, 25], [0, 1], [2, 3], \
117
+ [4, 5], [6, 7], [8, 9], [10, 11], [28, 29], [30, 31], [34, 35], [32, 33], \
118
+ [36, 37], [18, 19], [26, 27]]
119
+
120
+ connection_all = []
121
+ special_k = []
122
+ mid_num = 10
123
+
124
+ for k in range(len(mapIdx)):
125
+ score_mid = paf_avg[:, :, mapIdx[k]]
126
+ candA = all_peaks[limbSeq[k][0]]
127
+ candB = all_peaks[limbSeq[k][1]]
128
+
129
+ nA = len(candA)
130
+ nB = len(candB)
131
+ indexA, indexB = limbSeq[k]
132
+ if (nA != 0 and nB != 0):
133
+ connection_candidate = []
134
+ for i in range(nA):
135
+ for j in range(nB):
136
+ vec = np.subtract(candB[j][:2], candA[i][:2])
137
+ norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
138
+ norm = max(0.001, norm)
139
+ vec = np.divide(vec, norm)
140
+
141
+ startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
142
+ np.linspace(candA[i][1], candB[j][1], num=mid_num)))
143
+
144
+ vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
145
+ for I in range(len(startend))])
146
+ vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
147
+ for I in range(len(startend))])
148
+
149
+ score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
150
+ score_with_dist_prior = sum(score_midpts) / len(score_midpts) + min(
151
+ 0.5 * oriImg.shape[0] / norm - 1, 0)
152
+ criterion1 = len(np.nonzero(score_midpts > thre2)[0]) > 0.8 * len(score_midpts)
153
+ criterion2 = score_with_dist_prior > 0
154
+ if criterion1 and criterion2:
155
+ connection_candidate.append(
156
+ [i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2]])
157
+
158
+ connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
159
+ connection = np.zeros((0, 5))
160
+ for c in range(len(connection_candidate)):
161
+ i, j, s = connection_candidate[c][0:3]
162
+ if (i not in connection[:, 3] and j not in connection[:, 4]):
163
+ connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
164
+ if (len(connection) >= min(nA, nB)):
165
+ break
166
+
167
+ connection_all.append(connection)
168
+ else:
169
+ special_k.append(k)
170
+ connection_all.append([])
171
+
172
+ # last number in each row is the total parts number of that person
173
+ # the second last number in each row is the score of the overall configuration
174
+ subset = -1 * np.ones((0, self.njoint_body+1))
175
+ candidate = np.array([item for sublist in all_peaks for item in sublist])
176
+
177
+ for k in range(len(mapIdx)):
178
+ if k not in special_k:
179
+ partAs = connection_all[k][:, 0]
180
+ partBs = connection_all[k][:, 1]
181
+ indexA, indexB = np.array(limbSeq[k])
182
+
183
+ for i in range(len(connection_all[k])): # = 1:size(temp,1)
184
+ found = 0
185
+ subset_idx = [-1, -1]
186
+ for j in range(len(subset)): # 1:size(subset,1):
187
+ if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
188
+ subset_idx[found] = j
189
+ found += 1
190
+
191
+ if found == 1:
192
+ j = subset_idx[0]
193
+ if subset[j][indexB] != partBs[i]:
194
+ subset[j][indexB] = partBs[i]
195
+ subset[j][-1] += 1
196
+ subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
197
+ elif found == 2: # if found 2 and disjoint, merge them
198
+ j1, j2 = subset_idx
199
+ membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2]
200
+ if len(np.nonzero(membership == 2)[0]) == 0: # merge
201
+ subset[j1][:-2] += (subset[j2][:-2] + 1)
202
+ subset[j1][-2:] += subset[j2][-2:]
203
+ subset[j1][-2] += connection_all[k][i][2]
204
+ subset = np.delete(subset, j2, 0)
205
+ else: # as like found == 1
206
+ subset[j1][indexB] = partBs[i]
207
+ subset[j1][-1] += 1
208
+ subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
209
+
210
+ # if find no partA in the subset, create a new subset
211
+ elif not found and k < self.njoint_body-2:
212
+ row = -1 * np.ones(self.njoint_body+1)
213
+ row[indexA] = partAs[i]
214
+ row[indexB] = partBs[i]
215
+ row[-1] = 2
216
+ row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2]
217
+ subset = np.vstack([subset, row])
218
+ # delete some rows of subset which has few parts occur
219
+ deleteIdx = []
220
+ for i in range(len(subset)):
221
+ if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
222
+ deleteIdx.append(i)
223
+ subset = np.delete(subset, deleteIdx, axis=0)
224
+
225
+ # subset: n*20 array, 0-17 is the index in candidate, 18 is the total score, 19 is the total parts
226
+ # candidate: x, y, score, id
227
+ return candidate, subset
228
+
229
+ def handpos(self, oriImg):
230
+ scale_search = [0.5, 1.0, 1.5, 2.0]
231
+ # scale_search = [0.5]
232
+ boxsize = 368
233
+ stride = 8
234
+ padValue = 128
235
+ thre = 0.05
236
+ multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search]
237
+ heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 22))
238
+ # paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
239
+
240
+ for m in range(len(multiplier)):
241
+ scale = multiplier[m]
242
+ imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
243
+ imageToTest_padded, pad = util.padRightDownCorner(imageToTest, stride, padValue)
244
+ im = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
245
+ im = np.ascontiguousarray(im)
246
+
247
+ data = torch.from_numpy(im).float()
248
+ if torch.cuda.is_available():
249
+ data = data.cuda()
250
+ # data = data.permute([2, 0, 1]).unsqueeze(0).float()
251
+ with torch.no_grad():
252
+ output = self.pt_hand(data).cpu().numpy()
253
+ # output = self.model(data).numpy()q
254
+
255
+ # extract outputs, resize, and remove padding
256
+ heatmap = np.transpose(np.squeeze(output), (1, 2, 0)) # output 1 is heatmaps
257
+ heatmap = cv2.resize(heatmap, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
258
+ heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
259
+ heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
260
+
261
+ heatmap_avg += heatmap / len(multiplier)
262
+
263
+ all_peaks = []
264
+ for part in range(21):
265
+ map_ori = heatmap_avg[:, :, part]
266
+ one_heatmap = gaussian_filter(map_ori, sigma=3)
267
+ binary = np.ascontiguousarray(one_heatmap > thre, dtype=np.uint8)
268
+ # 全部小于阈值
269
+ if np.sum(binary) == 0:
270
+ all_peaks.append([0, 0])
271
+ continue
272
+ label_img, label_numbers = label(binary, return_num=True, connectivity=binary.ndim)
273
+ max_index = np.argmax([np.sum(map_ori[label_img == i]) for i in range(1, label_numbers + 1)]) + 1
274
+ label_img[label_img != max_index] = 0
275
+ map_ori[label_img == 0] = 0
276
+
277
+ y, x = util.npmax(map_ori)
278
+ all_peaks.append([x, y])
279
+ return np.array(all_peaks)
280
+
281
+ class ISLSignPosTranslator(keras.Model):
282
+
283
+ def __init__(self,body_model,hand_model, translation_model):
284
+ super().__init__()
285
+ self.pt_body = TorchModuleWrapper(body_model)
286
+ self.pt_body.trainable=False
287
+ self.pt_hand = TorchModuleWrapper(hand_model)
288
+ self.pt_hand.trainable=False
289
+
290
+ self.njoint_body = 26
291
+ self.npaf_body = 52
292
+ self.model_type='body25'
293
+ self.translation_layer=translation_model
294
+
295
+ def call(self, window):
296
+ window_size=20
297
+ window_features=[]
298
+ blank_frame=np.zeros((1,156))
299
+ for idx,frame in enumerate(window.cpu()):
300
+
301
+ # frame=frame.cpu().numpy()[:, :, ::-1]
302
+ candidate, subset = self.bodypos(frame.cpu().numpy())
303
+ hands_list = util.handDetect(candidate, subset, frame.cpu().numpy())
304
+ all_hand_peaks = []
305
+ for x, y, w, is_left in hands_list:
306
+ peaks = self.handpos(frame.cpu().numpy()[y:y+w, x:x+w, :])
307
+ peaks[:, 0] = np.where(peaks[:, 0]==0, peaks[:, 0], peaks[:, 0]+x)
308
+ peaks[:, 1] = np.where(peaks[:, 1]==0, peaks[:, 1], peaks[:, 1]+y)
309
+ all_hand_peaks.append(peaks)
310
+
311
+ (bodypose_circles,bodypose_sticks,)=util.get_bodypose(candidate, subset, self.model_type)
312
+ (handpose_edges,handpose_peaks)=util.get_handpose(all_hand_peaks,)
313
+
314
+ feature=self.populate_features(bodypose_circles,handpose_peaks)
315
+ window_features.append(feature)
316
+
317
+ if len(window_features)<window_size:
318
+ for _ in range(0,(window_size-window_features.shape[0])):
319
+ window_features.append(blank_frame)
320
+
321
+
322
+ # self.writer.close()
323
+ # timeseries=self.create_timeseries_data(feature)
324
+ # self.frame_to_window(feature)
325
+ # np.savetxt('C:/Users/spsar/OneDrive/Desktop/test/MVI_9590.MOV.window1.numpy',np.array(window_features))
326
+ return self.translation_layer(np.array(window_features).reshape(1,20,156))
327
+
328
+
329
+ def frame_to_window(self,frame):
330
+ """
331
+ Converts a single frame to a rolling window array with zero padding.
332
+
333
+ Args:
334
+ frame: A numpy array representing a video frame.
335
+ window_size: The size of the rolling window (default: 20).
336
+ window (optional): An existing window array to add the frame to
337
+ (useful for maintaining rolling window state).
338
+
339
+ Returns:
340
+ A numpy array representing the rolling window with the added frame.
341
+ """
342
+
343
+ # Shift the window elements by 1 (oldest frame is dropped)
344
+ self.window[:-1] = self.window[1:]
345
+
346
+ # Add the new frame to the end of the window
347
+ self.window[-1] = frame
348
+
349
+ def populate_features(self,bodypose_circles,handpose_peaks):
350
+ # X_body_test = [f'bodypeaks_x_{i}' for i in range(15)] + [f'bodypeaks_y_{i}' for i in range(15)]
351
+ # X_hand0_test = [f'hand0peaks_x_{i}' for i in range(21)] + [f'hand0peaks_y_{i}' for i in range(21)] + [f'hand0peaks_peaktxt{i}' for i in range(21)]
352
+ # X_hand1_test = [f'hand1peaks_x_{i}' for i in range(21)] + [f'hand1peaks_y_{i}' for i in range(21)] + [f'hand1peaks_peaktxt{i}' for i in range(21)]
353
+
354
+ # feature_columns_new = X_body_test + X_hand0_test + X_hand1_test
355
+ feature=[]
356
+ for idx in range(15):
357
+ if(idx<len(bodypose_circles)):
358
+ feature.append(bodypose_circles[idx][0])
359
+ else:
360
+ feature.append(0)
361
+
362
+ for idx in range(15):
363
+ if(idx<len(bodypose_circles)):
364
+ feature.append(bodypose_circles[idx][1])
365
+ else:
366
+ feature.append(0)
367
+
368
+ for hand_idx in range(2):
369
+ for idx in range(21):
370
+ if(idx<len(handpose_peaks[hand_idx])):
371
+ feature.append(float(handpose_peaks[hand_idx][idx][0]))
372
+ else:
373
+ feature.append(0)
374
+
375
+ for idx in range(21):
376
+ if(idx<len(handpose_peaks[hand_idx])):
377
+ feature.append(float(handpose_peaks[hand_idx][idx][1]))
378
+ else:
379
+ feature.append(0)
380
+
381
+ for idx in range(21):
382
+ if(idx<len(handpose_peaks[hand_idx])):
383
+ feature.append(float(handpose_peaks[hand_idx][idx][2]))
384
+ else:
385
+ feature.append(0)
386
+
387
+ # for idx in range(21):
388
+ # if(idx<len(handpose_peaks[1])):
389
+ # feature.append(handpose_peaks[1][idx][0])
390
+ # else:
391
+ # feature.append(0)
392
+
393
+ # for idx in range(21):
394
+ # if(idx<len(handpose_peaks[1])):
395
+ # feature.append(handpose_peaks[1][idx][1])
396
+ # else:
397
+ # feature.append(0)
398
+
399
+ # for idx in range(21):
400
+ # if(idx<len(handpose_peaks[1])):
401
+ # feature.append(handpose_peaks[1][idx][2])
402
+ # else:
403
+ # feature.append(0)
404
+
405
+ # for idx,handedges in enumerate(handpose_edges):
406
+ # for (peaktxt, (handedge_x1, handedge_y1), (handedge_x2, handedge_y2)) in handedges:
407
+ # feature[f'hand{idx}edge_x1_{peaktxt}']=handedge_x1
408
+ # feature[f'hand{idx}edge_y1_{peaktxt}']=handedge_y1
409
+ # feature[f'hand{idx}edge_x2_{peaktxt}']=handedge_x2
410
+ # feature[f'hand{idx}edge_y2_{peaktxt}']=handedge_y2
411
+
412
+ X=np.array(feature)
413
+ # time_steps = 12 # Number of time steps
414
+ # num_features = X.shape[0] // time_steps # Number of features per time step
415
+ # X_reshaped = X.reshape(1,time_steps, num_features)
416
+ return X
417
+
418
+ def bodypos(self, oriImg):
419
+ model_type = 'body25'
420
+ scale_search = [0.5]
421
+ boxsize = 368
422
+ stride = 8
423
+ padValue = 128
424
+ thre1 = 0.1
425
+ thre2 = 0.05
426
+ multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search]
427
+ heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], self.njoint_body))
428
+ paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], self.npaf_body))
429
+
430
+ for m in range(len(multiplier)):
431
+ scale = multiplier[m]
432
+ imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
433
+ imageToTest_padded, pad = util.padRightDownCorner(imageToTest, stride, padValue)
434
+ im = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
435
+ im = np.ascontiguousarray(im)
436
+
437
+ data = torch.from_numpy(im).float()
438
+ if torch.cuda.is_available():
439
+ data = data.cuda()
440
+ with torch.no_grad():
441
+ Mconv7_stage6_L1, Mconv7_stage6_L2 = self.pt_body(data)
442
+ Mconv7_stage6_L1 = Mconv7_stage6_L1.cpu().numpy()
443
+ Mconv7_stage6_L2 = Mconv7_stage6_L2.cpu().numpy()
444
+
445
+ heatmap = np.transpose(np.squeeze(Mconv7_stage6_L2), (1, 2, 0)) # output 1 is heatmaps
446
+ heatmap = cv2.resize(heatmap, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
447
+ heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
448
+ heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
449
+
450
+ paf = np.transpose(np.squeeze(Mconv7_stage6_L1), (1, 2, 0)) # output 0 is PAFs
451
+ paf = cv2.resize(paf, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
452
+ paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
453
+ paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
454
+
455
+ heatmap_avg += heatmap_avg + heatmap / len(multiplier)
456
+ paf_avg += + paf / len(multiplier)
457
+
458
+ all_peaks = []
459
+ peak_counter = 0
460
+
461
+ for part in range(self.njoint_body-1):
462
+ map_ori = heatmap_avg[:, :, part]
463
+ one_heatmap = gaussian_filter(map_ori, sigma=3)
464
+
465
+ map_left = np.zeros(one_heatmap.shape)
466
+ map_left[1:, :] = one_heatmap[:-1, :]
467
+ map_right = np.zeros(one_heatmap.shape)
468
+ map_right[:-1, :] = one_heatmap[1:, :]
469
+ map_up = np.zeros(one_heatmap.shape)
470
+ map_up[:, 1:] = one_heatmap[:, :-1]
471
+ map_down = np.zeros(one_heatmap.shape)
472
+ map_down[:, :-1] = one_heatmap[:, 1:]
473
+
474
+ peaks_binary = np.logical_and.reduce(
475
+ (one_heatmap >= map_left, one_heatmap >= map_right, one_heatmap >= map_up, one_heatmap >= map_down, one_heatmap > thre1))
476
+ peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
477
+ peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks]
478
+ peak_id = range(peak_counter, peak_counter + len(peaks))
479
+ peaks_with_score_and_id = [peaks_with_score[i] + (peak_id[i],) for i in range(len(peak_id))]
480
+
481
+ all_peaks.append(peaks_with_score_and_id)
482
+ peak_counter += len(peaks)
483
+
484
+ if model_type=='body25':
485
+ # find connection in the specified sequence, center 29 is in the position 15
486
+ limbSeq = [[1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],\
487
+ [10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],\
488
+ [11,24],[11,22],[14,21],[14,19],[22,23],[19,20]]
489
+ # the middle joints heatmap correpondence
490
+ mapIdx = [[30, 31],[14, 15],[16, 17],[18, 19],[22, 23],[24, 25],[26, 27],[0, 1],[6, 7],\
491
+ [2, 3],[4, 5], [8, 9],[10, 11],[12, 13],[32, 33],[34, 35],[36,37],[38,39],\
492
+ [50,51],[46,47],[44,45],[40,41],[48,49],[42,43]]
493
+ else:
494
+ # find connection in the specified sequence, center 29 is in the position 15
495
+ limbSeq = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], \
496
+ [9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], \
497
+ [0, 15], [15, 17], [2, 16], [5, 17]]
498
+ # the middle joints heatmap correpondence
499
+ mapIdx = [[12, 13], [20, 21], [14, 15], [16, 17], [22, 23], [24, 25], [0, 1], [2, 3], \
500
+ [4, 5], [6, 7], [8, 9], [10, 11], [28, 29], [30, 31], [34, 35], [32, 33], \
501
+ [36, 37], [18, 19], [26, 27]]
502
+
503
+ connection_all = []
504
+ special_k = []
505
+ mid_num = 10
506
+
507
+ for k in range(len(mapIdx)):
508
+ score_mid = paf_avg[:, :, mapIdx[k]]
509
+ candA = all_peaks[limbSeq[k][0]]
510
+ candB = all_peaks[limbSeq[k][1]]
511
+
512
+ nA = len(candA)
513
+ nB = len(candB)
514
+ indexA, indexB = limbSeq[k]
515
+ if (nA != 0 and nB != 0):
516
+ connection_candidate = []
517
+ for i in range(nA):
518
+ for j in range(nB):
519
+ vec = np.subtract(candB[j][:2], candA[i][:2])
520
+ norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
521
+ norm = max(0.001, norm)
522
+ vec = np.divide(vec, norm)
523
+
524
+ startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
525
+ np.linspace(candA[i][1], candB[j][1], num=mid_num)))
526
+
527
+ vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
528
+ for I in range(len(startend))])
529
+ vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
530
+ for I in range(len(startend))])
531
+
532
+ score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
533
+ score_with_dist_prior = sum(score_midpts) / len(score_midpts) + min(
534
+ 0.5 * oriImg.shape[0] / norm - 1, 0)
535
+ criterion1 = len(np.nonzero(score_midpts > thre2)[0]) > 0.8 * len(score_midpts)
536
+ criterion2 = score_with_dist_prior > 0
537
+ if criterion1 and criterion2:
538
+ connection_candidate.append(
539
+ [i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2]])
540
+
541
+ connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
542
+ connection = np.zeros((0, 5))
543
+ for c in range(len(connection_candidate)):
544
+ i, j, s = connection_candidate[c][0:3]
545
+ if (i not in connection[:, 3] and j not in connection[:, 4]):
546
+ connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
547
+ if (len(connection) >= min(nA, nB)):
548
+ break
549
+
550
+ connection_all.append(connection)
551
+ else:
552
+ special_k.append(k)
553
+ connection_all.append([])
554
+
555
+ # last number in each row is the total parts number of that person
556
+ # the second last number in each row is the score of the overall configuration
557
+ subset = -1 * np.ones((0, self.njoint_body+1))
558
+ candidate = np.array([item for sublist in all_peaks for item in sublist])
559
+
560
+ for k in range(len(mapIdx)):
561
+ if k not in special_k:
562
+ partAs = connection_all[k][:, 0]
563
+ partBs = connection_all[k][:, 1]
564
+ indexA, indexB = np.array(limbSeq[k])
565
+
566
+ for i in range(len(connection_all[k])): # = 1:size(temp,1)
567
+ found = 0
568
+ subset_idx = [-1, -1]
569
+ for j in range(len(subset)): # 1:size(subset,1):
570
+ if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
571
+ subset_idx[found] = j
572
+ found += 1
573
+
574
+ if found == 1:
575
+ j = subset_idx[0]
576
+ if subset[j][indexB] != partBs[i]:
577
+ subset[j][indexB] = partBs[i]
578
+ subset[j][-1] += 1
579
+ subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
580
+ elif found == 2: # if found 2 and disjoint, merge them
581
+ j1, j2 = subset_idx
582
+ membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2]
583
+ if len(np.nonzero(membership == 2)[0]) == 0: # merge
584
+ subset[j1][:-2] += (subset[j2][:-2] + 1)
585
+ subset[j1][-2:] += subset[j2][-2:]
586
+ subset[j1][-2] += connection_all[k][i][2]
587
+ subset = np.delete(subset, j2, 0)
588
+ else: # as like found == 1
589
+ subset[j1][indexB] = partBs[i]
590
+ subset[j1][-1] += 1
591
+ subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
592
+
593
+ # if find no partA in the subset, create a new subset
594
+ elif not found and k < self.njoint_body-2:
595
+ row = -1 * np.ones(self.njoint_body+1)
596
+ row[indexA] = partAs[i]
597
+ row[indexB] = partBs[i]
598
+ row[-1] = 2
599
+ row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2]
600
+ subset = np.vstack([subset, row])
601
+ # delete some rows of subset which has few parts occur
602
+ deleteIdx = []
603
+ for i in range(len(subset)):
604
+ if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
605
+ deleteIdx.append(i)
606
+ subset = np.delete(subset, deleteIdx, axis=0)
607
+
608
+ # subset: n*20 array, 0-17 is the index in candidate, 18 is the total score, 19 is the total parts
609
+ # candidate: x, y, score, id
610
+ return candidate, subset
611
+
612
+ def handpos(self, oriImg):
613
+ scale_search = [0.5, 1.0, 1.5, 2.0]
614
+ # scale_search = [0.5]
615
+ boxsize = 368
616
+ stride = 8
617
+ padValue = 128
618
+ thre = 0.05
619
+ multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search]
620
+ heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 22))
621
+ # paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
622
+
623
+ for m in range(len(multiplier)):
624
+ scale = multiplier[m]
625
+ imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
626
+ imageToTest_padded, pad = util.padRightDownCorner(imageToTest, stride, padValue)
627
+ im = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
628
+ im = np.ascontiguousarray(im)
629
+
630
+ data = torch.from_numpy(im).float()
631
+ if torch.cuda.is_available():
632
+ data = data.cuda()
633
+ # data = data.permute([2, 0, 1]).unsqueeze(0).float()
634
+ with torch.no_grad():
635
+ output = self.pt_hand(data).cpu().numpy()
636
+ # output = self.model(data).numpy()q
637
+
638
+ # extract outputs, resize, and remove padding
639
+ heatmap = np.transpose(np.squeeze(output), (1, 2, 0)) # output 1 is heatmaps
640
+ heatmap = cv2.resize(heatmap, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
641
+ heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
642
+ heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
643
+
644
+ heatmap_avg += heatmap / len(multiplier)
645
+
646
+ all_peaks = []
647
+ for part in range(21):
648
+ map_ori = heatmap_avg[:, :, part]
649
+ one_heatmap = gaussian_filter(map_ori, sigma=3)
650
+ binary = np.ascontiguousarray(one_heatmap > thre, dtype=np.uint8)
651
+ # 全部小于阈值
652
+ if np.sum(binary) == 0:
653
+ all_peaks.append([0, 0])
654
+ continue
655
+ label_img, label_numbers = label(binary, return_num=True, connectivity=binary.ndim)
656
+ max_index = np.argmax([np.sum(map_ori[label_img == i]) for i in range(1, label_numbers + 1)]) + 1
657
+ label_img[label_img != max_index] = 0
658
+ map_ori[label_img == 0] = 0
659
+
660
+ y, x = util.npmax(map_ori)
661
+ all_peaks.append([x, y])
662
+ return np.array(all_peaks)
README.md CHANGED
@@ -1,19 +1,18 @@
1
  ---
2
  title: SignLanguageTranslate
3
- emoji: 🚀
4
  colorFrom: red
5
- colorTo: red
6
  sdk: docker
7
- app_port: 8501
8
- tags:
9
- - streamlit
10
  pinned: false
11
- short_description: Streamlit template space
12
  ---
13
 
14
- # Welcome to Streamlit!
15
 
16
- Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
17
 
18
- If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
19
- forums](https://discuss.streamlit.io).
 
 
1
  ---
2
  title: SignLanguageTranslate
3
+ emoji: 👁
4
  colorFrom: red
5
+ colorTo: gray
6
  sdk: docker
7
+ app_port: 7860
 
 
8
  pinned: false
9
+ license: mit
10
  ---
11
 
12
+ # SignLanguageTranslate
13
 
14
+ SignLanguageTranslate aims to support sign languages around the world. This demo currently evaluates Indian Sign Language on isolated clips from INCLUDE.
15
 
16
+ The Transformer comparison uses archived predictions from the rolling Transformer at epoch 31 and the causal streaming Transformer at epoch 102. The accuracy table uses 720 isolated INCLUDE clips from the saved iPad validation split. Both models use the same clips. The original LSTM demo uses its historical test manifest.
17
+
18
+ Public source videos are matched by category, class and filename to a pinned revision of the [public INCLUDE mirror](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-dataset). The app labels that repository's split separately. The new comparison credits the original [AI4Bharat INCLUDE dataset](https://huggingface.co/datasets/ai4bharat/INCLUDE). The [keypoint dataset card](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-keypoints) describes the Apple Vision input used for the comparison.
app.py ADDED
@@ -0,0 +1,694 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #Modified by Augmented Startups 2021
2
+ #Face Landmark User Interface with StreamLit
3
+ #Watch Computer Vision Tutorials at www.augmentedstartups.info/YouTube
4
+ import streamlit as st
5
+
6
+ from transformer_comparison import render_transformer_comparison
7
+
8
+ app_mode = st.sidebar.selectbox(
9
+ "Choose the App mode",
10
+ ["Transformer comparison", "Run on Test Videos", "About App"],
11
+ )
12
+ if app_mode == "Transformer comparison":
13
+ render_transformer_comparison()
14
+ st.stop()
15
+
16
+ import os
17
+ os.environ["KERAS_BACKEND"] = "torch"
18
+ import keras
19
+
20
+ import cv2
21
+ import numpy as np
22
+ import tempfile
23
+ import time
24
+ from PIL import Image
25
+ from keras.models import Sequential
26
+ import os
27
+ from keras.models import Sequential
28
+ import pickle
29
+ import keras
30
+ from keras.models import Sequential
31
+ import os
32
+ from keras.layers import LSTM, Dense, Bidirectional, Dropout,Input,BatchNormalization
33
+ from model import handpose_model, bodypose_25_model
34
+ from expression_mapping import expression_mapping
35
+ from ISL_Model_parameter import ISLSignPosTranslator
36
+ import pandas as pd
37
+ import numpy as np
38
+ import ffmpeg
39
+ import subprocess
40
+ from typing import NamedTuple
41
+ import json
42
+ import util
43
+ from huggingface_hub import hf_hub_download
44
+
45
+ import shutil, platform, subprocess
46
+ import uuid
47
+
48
+ st.write("Python:", platform.python_version())
49
+ st.write("FFmpeg exists:", shutil.which("ffmpeg"), "FFprobe:", shutil.which("ffprobe"))
50
+
51
+ try:
52
+ import cv2
53
+ st.write("OpenCV version:", cv2.__version__)
54
+ except Exception as e:
55
+ st.error(f"OpenCV import failed: {e}")
56
+
57
+ try:
58
+ import torch, keras
59
+ st.write("Torch:", torch.__version__, "Keras:", keras.__version__)
60
+ except Exception as e:
61
+ st.error(f"Torch/Keras import failed: {e}")
62
+
63
+ class FFProbeResult(NamedTuple):
64
+ return_code: int
65
+ json: str
66
+ error: str
67
+
68
+
69
+ def ffprobe(file_path) -> FFProbeResult:
70
+ command_array = ["ffprobe",
71
+ "-v", "quiet",
72
+ "-print_format", "json",
73
+ "-show_format",
74
+ "-show_streams",
75
+ file_path]
76
+ result = subprocess.run(command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
77
+ return FFProbeResult(return_code=result.returncode,
78
+ json=result.stdout,
79
+ error=result.stderr)
80
+ X_body_test = [f'bodypeaks_x_{i}' for i in range(15)] + [f'bodypeaks_y_{i}' for i in range(15)]
81
+ X_hand0_test = [f'hand0peaks_x_{i}' for i in range(21)] + [f'hand0peaks_y_{i}' for i in range(21)] + [f'hand0peaks_peaktxt{i}' for i in range(21)]
82
+ X_hand1_test = [f'hand1peaks_x_{i}' for i in range(21)] + [f'hand1peaks_y_{i}' for i in range(21)] + [f'hand1peaks_peaktxt{i}' for i in range(21)]
83
+
84
+ feature_columns_new = X_body_test + X_hand0_test + X_hand1_test
85
+ label_columns = ['Expression_encoded']
86
+
87
+ @st.cache_resource
88
+ def create_timeseries_data(isl_data,feature_columns,label_columns, window_size=20):
89
+ """
90
+ Creates timeseries data from a DataFrame with a specified window size
91
+ and padding at the end.
92
+
93
+ Args:
94
+ df (pandas.DataFrame): The input DataFrame.
95
+ window_size (int, optional): The window size for creating timeseries data. Defaults to 20.
96
+ pad_value (any, optional): The value to use for padding at the end. Defaults to None.
97
+
98
+ Returns:
99
+ list: A list of lists, where each inner list represents a window of timeseries data.
100
+ """
101
+
102
+ # Handle empty DataFrame
103
+ if isl_data.empty:
104
+ return [],[]
105
+
106
+ X=[]
107
+ y=[]
108
+ i=0
109
+ for group, file_df in isl_data.groupby(['Type','Expression_encoded','FileName']):
110
+ expr_types,exprs,filepaths=group
111
+ # print('expr_types,exprs,filepaths',(expr_types,exprs,filepaths))
112
+ # print(type(name))
113
+ # Get the rolling window iterator with padding
114
+ first_frame=np.zeros((1,156))
115
+ for idx,x in enumerate([file_df[i:i+window_size] for i in range(0,file_df.shape[0],1)]):#enumerate(file_df.rolling(window=20, step=20,min_periods=1)):
116
+ # print(f'records processed {idx} of {file_df.shape[0]}')
117
+ # print(f"{filepaths}-Frame#{x['Frame'].values}/{file_df['Frame'].max()}")
118
+ if x.shape[0]<window_size:
119
+ X.append(np.concatenate((np.repeat(first_frame, (window_size-x.shape[0]), axis=0),x[feature_columns].values), axis=0))
120
+ y.append(exprs)
121
+ # print('len(X)',len(X))
122
+ # print('len(y)',len(y))
123
+ continue
124
+
125
+
126
+ X.append(x[feature_columns].values)
127
+ y.append(exprs)
128
+ # print('len(X)',len(X))
129
+ # print('len(y)',len(y))
130
+ # if idx>4:
131
+ # break
132
+
133
+ # i=i+1
134
+ # if i>4:
135
+ # break
136
+
137
+ return X,y
138
+
139
+ translation_model=None
140
+
141
+ @st.cache_resource
142
+ def get_translator_model():
143
+ translation_model = Sequential()
144
+ translation_model.add(Input(shape=((20, 156))))
145
+ translation_model.add(keras.layers.Masking(mask_value=0.))
146
+ translation_model.add(BatchNormalization())
147
+ translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2, return_sequences=True)))
148
+
149
+ translation_model.add(Dropout(0.2))
150
+ translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2)))
151
+
152
+ translation_model.add(keras.layers.Activation('elu'))
153
+ translation_model.add(Dense(32, use_bias=False, kernel_initializer='he_normal'))
154
+
155
+ translation_model.add(BatchNormalization())
156
+ translation_model.add(Dropout(0.2))
157
+ translation_model.add(keras.layers.Activation('elu'))
158
+ translation_model.add(Dense(32, kernel_initializer='he_normal',use_bias=False))
159
+
160
+ translation_model.add(BatchNormalization())
161
+ translation_model.add(keras.layers.Activation('elu'))
162
+ translation_model.add(Dropout(0.2))
163
+ translation_model.add(Dense(len(list(expression_mapping.keys())), activation='softmax'))
164
+ model_file = hf_hub_download(
165
+ repo_id="sunilsarolkar/isl-translation-model",
166
+ filename="isl_model_final.keras"
167
+ )
168
+ translation_model.load_weights(model_file)
169
+ translation_model.load_weights(model_file)
170
+ return translation_model
171
+
172
+
173
+ testing_cleaned = hf_hub_download(
174
+ repo_id="sunilsarolkar/isl-test-data",
175
+ filename="testing_cleaned.csv",
176
+ repo_type="dataset"
177
+ )
178
+
179
+ testing_df=pd.read_csv(testing_cleaned)
180
+ test_files = hf_hub_download(
181
+ repo_id="sunilsarolkar/isl-test-data",
182
+ filename="test_files.csv",
183
+ repo_type="dataset"
184
+ )
185
+ # test_statistic_df=pd.read_csv('test_statistic.csv')
186
+ test_files_df=pd.read_csv(test_files)
187
+ # mp_drawing = mp.solutions.drawing_utils
188
+ # mp_face_mesh = mp.solutions.face_mesh
189
+
190
+
191
+ class Writer():
192
+ def __init__(self, output_file, input_fps, input_framesize, input_pix_fmt,
193
+ input_vcodec):
194
+ # if os.path.exists(output_file):
195
+ # os.remove(output_file)
196
+ self.ff_proc = (
197
+ ffmpeg
198
+ .input('pipe:',
199
+ format='rawvideo',
200
+ pix_fmt="bgr24",
201
+ s='%sx%s'%(input_framesize[1],input_framesize[0]),
202
+ r=input_fps)
203
+ .output(output_file, pix_fmt=input_pix_fmt, vcodec=input_vcodec)
204
+ .overwrite_output()
205
+ .run_async(pipe_stdin=True)
206
+ )
207
+
208
+ def __call__(self, frame):
209
+ self.ff_proc.stdin.write(frame.tobytes())
210
+
211
+ def close(self):
212
+ self.ff_proc.stdin.close()
213
+ self.ff_proc.wait()
214
+
215
+
216
+ st.title("SignLanguageTranslate")
217
+ st.header("Indian Sign Language · legacy LSTM")
218
+
219
+ st.markdown(
220
+ """
221
+ <style>
222
+ [data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
223
+ width: 350px;
224
+ }
225
+ [data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
226
+ width: 350px;
227
+ margin-left: -350px;
228
+ }
229
+ </style>
230
+ """,
231
+ unsafe_allow_html=True,
232
+ )
233
+
234
+ st.sidebar.title('ISL Sign Language Translation using Openpose')
235
+ st.sidebar.subheader('Parameters')
236
+ frame_wise_outputs={}
237
+
238
+ def weighted_average(nums, weights):
239
+ if sum(weights)==0:
240
+ return 0
241
+ return sum(x * y for x, y in zip(nums, weights)) / sum(weights)
242
+
243
+
244
+ @st.cache_data
245
+ def image_resize(image, width=None, height=None, inter=cv2.INTER_AREA):
246
+ # initialize the dimensions of the image to be resized and
247
+ # grab the image size
248
+ dim = None
249
+ (h, w) = image.shape[:2]
250
+
251
+ # if both the width and height are None, then return the
252
+ # original image
253
+ if width is None and height is None:
254
+ return image
255
+
256
+ # check to see if the width is None
257
+ if width is None:
258
+ # calculate the ratio of the height and construct the
259
+ # dimensions
260
+ r = height / float(h)
261
+ dim = (int(w * r), height)
262
+
263
+ # otherwise, the height is None
264
+ else:
265
+ # calculate the ratio of the width and construct the
266
+ # dimensions
267
+ r = width / float(w)
268
+ dim = (width, int(h * r))
269
+
270
+ # resize the image
271
+ resized = cv2.resize(image, dim, interpolation=inter)
272
+
273
+ # return the resized image
274
+ return resized
275
+
276
+ if app_mode =='About App':
277
+ st.markdown('In this application we are demonstrating model developed for translating the Indian Sign Language(ISL) using LSTM')
278
+ st.markdown(
279
+ """
280
+ <style>
281
+ [data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
282
+ width: 400px;
283
+ }
284
+ [data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
285
+ width: 400px;
286
+ margin-left: -400px;
287
+ }
288
+ </style>
289
+ """,
290
+ unsafe_allow_html=True,
291
+ )
292
+ # st.video('https://www.youtube.com/watch?v=FMaNNXgB_5c&ab_channel=AugmentedStartups')
293
+
294
+ tab1, tab2, tab3, tab4 = st.tabs(["Dataset Overview", "Data Pipeline", "Model Architecture", "Training Details"])
295
+
296
+ with tab1:
297
+ st.markdown('''
298
+ # Dataset Used \n
299
+ This model is trained using [INCLUDE](https://zenodo.org/records/4010759) dataset. \n
300
+
301
+ ### Key Statistics for the dataset is as follows-
302
+
303
+ +-----------------------+-----------------+
304
+ | Characteristics | INCLUDE-DATASET |
305
+ +-----------------------+-----------------+
306
+ | Categories | 15 |
307
+ | Words | 263 |
308
+ | Videos | 4257 |
309
+ | Avg Videos per class | 16.3 |
310
+ | Avg Video Length | 2.57s |
311
+ | Min Video Length | 1.28s |
312
+ | Max Video Length | 6.16s |
313
+ | Frame Rate | 25fps |
314
+ | Resolution | 1920x1080 |
315
+ +-----------------------+-----------------+
316
+
317
+ #### Size of each category
318
+
319
+ +--------------------+-------------------+------------------+
320
+ | Category | Number of Classes | Number of Videos |
321
+ +--------------------+-------------------+------------------+
322
+ | Adjectives | 59 | 791 |
323
+ | Animals | 8 | 166 |
324
+ | Clothes | 10 | 198 |
325
+ | Colours | 11 | 222 |
326
+ | Days and Time | 22 | 306 |
327
+ | Electronics | 10 | 140 |
328
+ | Greetings | 9 | 185 |
329
+ | Means of Transport | 9 | 186 |
330
+ | Objects at Home | 27 | 379 |
331
+ | Occupations | 16 | 225 |
332
+ | People | 26 | 513 |
333
+ | Places | 19 | 399 |
334
+ | Pronouns | 8 | 168 |
335
+ | Seasons | 6 | 85 |
336
+ | Society | 23 | 324 |
337
+ | | Categories# 263 | Total Videos-4287|
338
+ +--------------------+-------------------+------------------+
339
+ ''')
340
+ st.info("💡 **Note:** The dataset used for training contains **1986 processed videos out of 4287**. We processed a limited set of records due to time and compute constraints.")
341
+
342
+ with tab2:
343
+ st.markdown('''
344
+ ### Data Pipeline
345
+ The pipeline processes video frames to extract pose and hand landmarks using an OpenPose-like approach.
346
+
347
+ For each frame, the feature extraction process produces a vector of **156 features**, consisting of:
348
+ - **Body Pose**: X and Y coordinates for 15 body keypoints, along with edge lengths and angles.
349
+ - **Hand Pose**: X and Y coordinates for 21 keypoints on each hand (left and right).
350
+
351
+ This structured tabular data is then grouped sequentially into sliding windows to capture the temporal motion of the signs.
352
+ ''')
353
+
354
+ with tab3:
355
+ st.markdown('''
356
+ ### Model Structure
357
+ The translation model utilizes a sequence-to-sequence architecture based on Bidirectional LSTMs.
358
+
359
+ ```python
360
+ translation_model = Sequential()
361
+ translation_model.add(Input(shape=((20, 156))))
362
+ translation_model.add(keras.layers.Masking(mask_value=0.))
363
+ translation_model.add(BatchNormalization())
364
+ translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2, return_sequences=True)))
365
+
366
+ translation_model.add(Dropout(0.2))
367
+ translation_model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2)))
368
+
369
+ translation_model.add(keras.layers.Activation('elu'))
370
+ translation_model.add(Dense(32, use_bias=False, kernel_initializer='he_normal'))
371
+
372
+ translation_model.add(BatchNormalization())
373
+ translation_model.add(Dropout(0.2))
374
+ translation_model.add(keras.layers.Activation('elu'))
375
+ translation_model.add(Dense(32, kernel_initializer='he_normal',use_bias=False))
376
+
377
+ translation_model.add(BatchNormalization())
378
+ translation_model.add(keras.layers.Activation('elu'))
379
+ translation_model.add(Dropout(0.2))
380
+ translation_model.add(Dense(len(list(expression_mapping.keys())), activation='softmax'))
381
+ isl_translator=ISLSignPosTranslator(bodypose_25_model(),handpose_model(), translation_model)
382
+ ```
383
+
384
+ **Parameters:**
385
+ - Total params: 82,679 (322.96 KB)
386
+ - Trainable params: 82,239 (321.25 KB)
387
+ - Non-trainable params: 440 (1.72 KB)
388
+ ''')
389
+
390
+ with tab4:
391
+ st.markdown('''
392
+ ### Training Details
393
+ The model was trained using the **Keras 3 API with a PyTorch backend**.
394
+
395
+ **Dataset Size:**
396
+ - **123,743 individual frames** were extracted from the processed videos and used as the training set.
397
+
398
+ **Sequence Windowing:**
399
+ - The temporal data is structured into windows of **20 frames** per sequence (`shape=(20, 156)`), allowing the Bidirectional LSTMs to learn the motion context of the signs.
400
+
401
+ **Metrics & Logs:**
402
+ - You can view the detailed training progression on [Tensorboard](https://huggingface.co/cdsteameight/ISL-SignLanguageTranslation/tensorboard).
403
+ ''')
404
+
405
+ elif app_mode =='Run on Test Videos':
406
+ st.markdown('''
407
+ ## How to use the Test Videos
408
+ Follow these steps to run a prediction on a test video:
409
+ 1. Open the **sidebar** on the left.
410
+ 2. Select a **Category** from the dropdown.
411
+ 3. Select a specific **Class** (the sign language word you want to test).
412
+ 4. Select a specific **File** from the available test videos.
413
+ 5. Click the **"Start"** button at the bottom of the sidebar to run the prediction model on the video.
414
+
415
+ The app will load the frames, process them, and display the predicted translation!
416
+ ---
417
+ ''')
418
+ # placeholder = st.empty()
419
+ category = st.sidebar.selectbox('Choose Category',
420
+ np.sort(test_files_df['Category'].unique(), axis=-1, kind='mergesort'))
421
+ # print(category)
422
+ mask = (test_files_df['Category']==category)
423
+ test_files_df_category=test_files_df[mask]
424
+ cls = st.sidebar.selectbox('Choose Class',
425
+ np.sort(test_files_df_category['Class'].unique(), axis=-1, kind='mergesort')
426
+ )
427
+ mask = (test_files_df['Class']==cls)
428
+ filename = st.sidebar.selectbox('Choose File',
429
+ np.sort(test_files_df_category[mask]['Filename'].unique(), axis=-1, kind='mergesort')
430
+ )
431
+ # print(f'test/{category}/{cls}/{filename}')
432
+ # mask = (include_df['Filepath'].str.contains(key[0])) & (include_df['type']==key[2]) & (include_df['expression']==key[1])
433
+ # stframe = st.empty()
434
+
435
+ if st.sidebar.button("Start", type="primary"):
436
+ mask = (testing_df['FileName'] == filename) & (testing_df['Type']==category)& (testing_df['Expression']==cls)
437
+ # filtered_df = current_test_df.sort_
438
+
439
+ window_size=20
440
+ current_test_df=testing_df[mask]
441
+
442
+ if current_test_df.empty:
443
+ st.error(f"⚠️ No matching rows in testing_df for this selection.{filename} {category} {cls}")
444
+ st.stop()
445
+ else:
446
+ st.success(f"✅ Loaded {current_test_df.shape[0]} frames for this selection.")
447
+ X_test_filtered,y_test_filtered = create_timeseries_data(current_test_df,feature_columns_new,label_columns,window_size=window_size)
448
+ # y_filtered_encoded=to_categorical(y_test_filtered, num_classes=len(df['Expression_encoded'].unique()))
449
+ X_test_filtered=np.array(X_test_filtered)
450
+
451
+ # encoded_translation=model(frame.reshape(1,frame.shape[0],frame.shape[1]))
452
+ st.set_option('deprecation.showfileUploaderEncoding', False)
453
+
454
+ # use_webcam = st.sidebar.button('Use Webcam')
455
+ # record = st.sidebar.checkbox("Record Video")
456
+ # if record:
457
+ # st.checkbox("Recording", value=True)
458
+
459
+ st.sidebar.markdown('---')
460
+ st.markdown(
461
+ """
462
+ <style>
463
+ [data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
464
+ width: 400px;
465
+ }
466
+ [data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
467
+ width: 400px;
468
+ margin-left: -400px;
469
+ }
470
+ </style>
471
+ """,
472
+ unsafe_allow_html=True,
473
+ )
474
+
475
+ st.sidebar.markdown('---')
476
+
477
+ st.markdown(' ## Output')
478
+
479
+ runtime_progress = st.empty()
480
+
481
+ with runtime_progress.container():
482
+ df1 = pd.DataFrame([['--','--']], columns=['Frames Processed','Detected Class'])
483
+
484
+ my_table = st.table(df1)
485
+ # kpi1, kpi2 = st.columns(2)
486
+
487
+ # with kpi1:
488
+ # st.markdown("**Frames Processed**")
489
+ # kpi1_text = st.markdown(f'0/{current_test_df.shape[0]}')
490
+
491
+ # with kpi2:
492
+ # st.markdown("**Detected Class**")
493
+ # kpi2_text = st.markdown("--")
494
+
495
+ view = st.empty()
496
+
497
+ st.markdown("<hr/>", unsafe_allow_html=True)
498
+ stframes = st.empty()#[st.empty() for _ in range(20)]
499
+ # video_file_buffer = st.sidebar.file_uploader("Upload a video", type=[ "mp4", "mov",'avi','asf', 'm4v' ])
500
+ # tfflie = tempfile.NamedTemporaryFile(delete=False)
501
+
502
+ vid_file = hf_hub_download(
503
+ repo_id="sunilsarolkar/isl-test-data",
504
+ filename=f'test/{category}/{cls}/{filename}',
505
+ repo_type="dataset"
506
+ )
507
+
508
+ if not os.path.exists(vid_file):
509
+ st.error(f"⚠️ Video file not found: {f'test/{category}/{cls}/{filename}'}")
510
+ st.stop()
511
+
512
+ vid = cv2.VideoCapture(vid_file)
513
+
514
+ ffprobe_result = ffprobe(vid_file)
515
+ info = json.loads(ffprobe_result.json)
516
+ videoinfo = [i for i in info["streams"] if i["codec_type"] == "video"][0]
517
+ input_fps = videoinfo["avg_frame_rate"]
518
+ # input_fps = float(input_fps[0])/float(input_fps[1])
519
+ input_pix_fmt = videoinfo["pix_fmt"]
520
+ input_vcodec = videoinfo["codec_name"]
521
+ postfix = info["format"]["format_name"].split(",")[0]
522
+ # print(f'input_vcodec-{input_vcodec}')
523
+
524
+ width = int(vid.get(cv2.CAP_PROP_FRAME_WIDTH))
525
+ height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT))
526
+ fps_input = int(vid.get(cv2.CAP_PROP_FPS))
527
+
528
+ #codec = cv2.VideoWriter_fourcc(*FLAGS.output_format)
529
+ # codec = cv2.VideoWriter_fourcc('V','P','0','9')
530
+ # out = cv2.VideoWriter('output1.mp4', codec, fps_input, (width, height))
531
+
532
+ # st.sidebar.text('Input Video')
533
+ # st.sidebar.video(tfflie.name)
534
+ fps = 0
535
+ i = 0
536
+
537
+
538
+
539
+ # cap = cv2.VideoCapture(video_file,)
540
+ totalFrames=int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
541
+ window_size=20
542
+ # print('current_test_df',current_test_df)
543
+ # print('totalFrames',totalFrames)
544
+ window=[]
545
+
546
+
547
+ prevTime = 0
548
+ postfix = info["format"]["format_name"].split(",")[0]
549
+
550
+
551
+ output_file = f"/tmp/output_{uuid.uuid4().hex}.{postfix}"
552
+
553
+ # height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT))
554
+ fps_input = int(vid.get(cv2.CAP_PROP_FPS))
555
+
556
+ #codec = cv2.VideoWriter_fourcc(*FLAGS.output_format)
557
+
558
+ # codec = cv2.VideoWriter_fourcc('m','p','4','v')
559
+ out = None
560
+ writer=None
561
+ weighted_avg_dict={}
562
+
563
+ idx=0
564
+ try:
565
+ for _, row in current_test_df.iterrows():#enumerate(file_df.rolling(window=20, step=20,min_periods=1)):
566
+ # print(f'captured frame#{idx}')
567
+ if not vid.isOpened():
568
+ st.error(f"Could not open video: {vid_file}")
569
+ else:
570
+ if(vid.isOpened()):
571
+ ret, frame = vid.read()
572
+
573
+
574
+ if len(window)<window_size:
575
+ canvas=util.drawStickmodel(frame,eval(row['bodypose_circles']),eval(row['bodypose_sticks']),eval(row['handpose_edges']),eval(row['handpose_peaks']))
576
+ canvas_with_plot=util.draw_bar_plot_below_image(canvas,{}, f'Prediction bar plot - Frame number {idx+1} [** no predictions]',canvas)
577
+ canvas_with_plot=util.draw_bar_plot_below_image(canvas_with_plot,weighted_avg_dict, f'Weighted avg - Frame number {idx+1} [** no predictions]',canvas)
578
+ canvas_with_plot=util.add_padding_to_bottom(canvas_with_plot,(255,255,255),100)# Adds padding at bottom
579
+
580
+ if writer is None:
581
+ input_framesize = canvas_with_plot.shape[:2]
582
+ writer = Writer(output_file, input_fps, input_framesize, input_pix_fmt,
583
+ input_vcodec)
584
+
585
+ # if out is None:
586
+ # out=cv2.VideoWriter(output_file, codec, fps_input, frame.shape[:2])
587
+
588
+
589
+ writer(canvas_with_plot)
590
+ # out.write(canvas)
591
+ with runtime_progress.container():
592
+ df1 = pd.DataFrame([[f'{idx+1}/{current_test_df.shape[0]}','<model will output after 20 frames>']], columns=['Frames Processed','Detected Class'])
593
+
594
+ my_table = st.table(df1)
595
+ window.append(frame)
596
+ # kpi1_text.write(f"<h1 style='text-align: center; color: red;'>{idx+1}/{current_test_df.shape[0]}</h1>", unsafe_allow_html=True)
597
+ # kpi2_text.write(f"<h1 style='text-align: center; color: red;'>--</h1>", unsafe_allow_html=True)
598
+ view.image(cv2.imencode('.jpg', canvas_with_plot)[1].tobytes(), use_column_width=True)
599
+ else:
600
+
601
+ window[:-1] = window[1:]
602
+ window[-1]=frame
603
+ translation_model=get_translator_model()
604
+ # testing_df[]
605
+
606
+ encoded_translation = translation_model(X_test_filtered[idx-20].reshape(1,X_test_filtered[idx-20].shape[0],X_test_filtered[idx-20].shape[1]))
607
+ encoded_translation=encoded_translation[0].cpu().detach().numpy()
608
+ sorted_index=np.argsort(encoded_translation)[::-1]
609
+ maxindex=np.argmax(encoded_translation)
610
+
611
+ top_3_probs = encoded_translation.argsort()[-3:][::-1] # Get indices of top 3 probabilities (descending order)
612
+ top_3_categories = [expression_mapping[i] for i in top_3_probs] # Convert indices to category names (assuming class_names list exists)
613
+ top_3_values = encoded_translation[top_3_probs] # Get corresponding probabilities
614
+ # print(f'{idx} {encoded_translation[maxindex]:0.4f} {maxindex}-{expression_mapping[maxindex]} ')#{[(pi,encoded_translation[pi],expression_mapping[pi]) for pi in sorted_index]}
615
+ for category, prob in zip(top_3_categories, top_3_values):
616
+ if category not in frame_wise_outputs:
617
+ frame_wise_outputs[category]=[]
618
+ frame_wise_outputs[category].append(prob)
619
+ current_prob={}
620
+
621
+ for category, prob in zip(top_3_categories, top_3_values):
622
+ current_prob[category]=prob
623
+
624
+ for key in frame_wise_outputs:
625
+ weighted_avg_dict[key]=weighted_average(frame_wise_outputs[key],[len(frame_wise_outputs[key]) for i in range(len(frame_wise_outputs[key]))])
626
+
627
+ sorted_dict = dict(sorted(weighted_avg_dict.items(), key=lambda item: item[1], reverse=True))
628
+ canvas=util.drawStickmodel(frame,eval(row['bodypose_circles']),eval(row['bodypose_sticks']),eval(row['handpose_edges']),eval(row['handpose_peaks']))
629
+ canvas_with_plot=util.draw_bar_plot_below_image(canvas,current_prob, f'Prediction at frame window({idx-20+1}-{idx+1})',canvas)
630
+ canvas_with_plot=util.draw_bar_plot_below_image(canvas_with_plot,weighted_avg_dict, f'Weighted avg till window {idx+1}',canvas)
631
+ canvas_with_plot=util.add_padding_to_bottom(canvas_with_plot,(255,255,255),100)
632
+ writer(canvas_with_plot)
633
+
634
+
635
+ currTime = time.time()
636
+ fps = 1 / (currTime - prevTime)
637
+ prevTime = currTime
638
+ # out.write(frame)
639
+ # if record:
640
+ # #st.checkbox("Recording", value=True)
641
+ # out.write(frame)
642
+ #Dashboard
643
+
644
+ max_prob = float('-inf') # Initialize with negative infinity
645
+ max_key = None
646
+
647
+ for exp, prob in weighted_avg_dict.items():
648
+ if prob > max_prob:
649
+ max_prob = prob
650
+ max_key = exp
651
+ with runtime_progress.container():
652
+ df1 = pd.DataFrame([[f'{idx+1}/{current_test_df.shape[0]}',f'{max_key} ({max_prob*100:.2f}%)']], columns=['Frames Processed','Detected Class'])
653
+ my_table = st.table(df1)
654
+ # kpi1_text.write(f"<h1 style='text-align: center; color: red;'>{idx+1}/{current_test_df.shape[0]}</h1>", unsafe_allow_html=True)
655
+ # kpi2_text.write(f"<h1 style='text-align: center; color: red;'>{max_key} ({max_prob*100:.2f}%)</h1>", unsafe_allow_html=True)
656
+ # with placeholder.container():
657
+ # # st.write(weighted_avg_dict)
658
+ # # data = {
659
+ # # "I": 0.7350964583456516,
660
+ # # "Hello": 0.1078806109726429,
661
+ # # "you": 0.11776176246348768,
662
+ # # "you (plural)": 0.12685142129916568
663
+ # # }
664
+
665
+ # # Convert the dictionary to a Pandas DataFrame for easier plotting
666
+ # df = pd.DataFrame.from_dict(weighted_avg_dict, orient='index', columns=['Values'])
667
+
668
+ # # Create a bar chart with Streamlit
669
+ # st.bar_chart(df)
670
+ # frame = cv2.resize(frame,(0,0),fx = 0.8 , fy = 0.8)
671
+ # frame = image_resize(image = frame, width = 640)
672
+ view.image(cv2.imencode('.jpg', canvas_with_plot)[1].tobytes(), use_column_width=True)
673
+
674
+ idx=idx+1
675
+
676
+
677
+ # st.text('Video Processed')
678
+ with view.container():
679
+ if writer is not None: # ✅ safeguard
680
+ writer.close()
681
+ output_video = open(output_file,'rb')
682
+ out_bytes = output_video.read()
683
+ st.video(out_bytes)
684
+ print(f'Output file - {output_file}')
685
+ else:
686
+ st.warning("No video was processed, writer is empty.")
687
+ # out.release()
688
+
689
+ print(f'Output file - {output_file}')
690
+ finally:
691
+ vid.release()
692
+ if writer is not None:
693
+ writer.close()
694
+ cv2.destroyAllWindows()
categories_processed.png ADDED
data/include_validation.json.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff6d428b43244589ded2237dd1d4cd82253baf1c96afcd20e925312c421782f4
3
+ size 142317
data/transformer_contract.json ADDED
@@ -0,0 +1,278 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "id": "spsarolkar/AI4Bharat-INCLUDE-keypoints",
4
+ "revision": "8a17a31ce8803cc1d5689f6249d088236d684943",
5
+ "files": {
6
+ "data/apple-vision/validation/validation-00000.jsonl.gz": "0c5370dbaa88f5fadfaaaf14f465176370e9324e6f243cec3c2ad5b0f924df25",
7
+ "data/apple-vision/validation/validation-00001.jsonl.gz": "784a39a1f1cfcbe5f2e2863455caaee08d132e104da3d2229f9335f18c27b4c1",
8
+ "data/apple-vision/validation/validation-00002.jsonl.gz": "584ee286a4511993dd90b906d63e552b7c64e06c2d5fe9be4159467f49750713",
9
+ "data/apple-vision/validation/validation-00003.jsonl.gz": "f3db9a68cbbb282856235ed394dd02662a468a25ed36cb6b5b2628b8cd1d7a99",
10
+ "data/apple-vision/validation/validation-00004.jsonl.gz": "f6b11238377ce13a3ab8e3ff01fe1d7aea4e8936abf541c60399bf24e4eb2035",
11
+ "data/apple-vision/validation/validation-00005.jsonl.gz": "0dee5398e79fcdbdb814f17164083a9d832b5e909642d99322ebf7933ca65a45",
12
+ "metadata/vocabulary.json": "9ac2b427fc3f1742c891ad6092740b336d07be19f4b9ad01984687afd9b4813f"
13
+ }
14
+ },
15
+ "modelRepo": "spsarolkar/isl-transformer-mlx",
16
+ "modelFiles": {
17
+ "rolling": "space-inference/rolling-epoch31.safetensors",
18
+ "causal": "space-inference/causal-epoch102.safetensors"
19
+ },
20
+ "classNames": [
21
+ "Actor",
22
+ "Adult",
23
+ "Afternoon",
24
+ "Alright",
25
+ "Animal",
26
+ "Artist",
27
+ "Author",
28
+ "Baby",
29
+ "Bag",
30
+ "Ball",
31
+ "Bank",
32
+ "Bathroom",
33
+ "Beautiful",
34
+ "Bed",
35
+ "Bedroom",
36
+ "Bicycle",
37
+ "Bill",
38
+ "Bird",
39
+ "Black",
40
+ "Blind",
41
+ "Blue",
42
+ "Boat",
43
+ "Book",
44
+ "Box",
45
+ "Boy",
46
+ "Brother",
47
+ "Brown",
48
+ "Bus",
49
+ "Camera",
50
+ "Car",
51
+ "Card",
52
+ "Cat",
53
+ "Cell phone",
54
+ "Chair",
55
+ "Child",
56
+ "City",
57
+ "Clock",
58
+ "Clothing",
59
+ "Computer",
60
+ "Court",
61
+ "Cow",
62
+ "Crowd",
63
+ "Daughter",
64
+ "Deaf",
65
+ "Death",
66
+ "Doctor",
67
+ "Dog",
68
+ "Door",
69
+ "Dream",
70
+ "Dress",
71
+ "Election",
72
+ "Energy",
73
+ "Evening",
74
+ "Ex. Monsoon",
75
+ "Exercise",
76
+ "Fall",
77
+ "Family",
78
+ "Fan",
79
+ "Father",
80
+ "Fish",
81
+ "Friday",
82
+ "Friend",
83
+ "Gift",
84
+ "Girl",
85
+ "God",
86
+ "Good Morning",
87
+ "Good afternoon",
88
+ "Good evening",
89
+ "Good night",
90
+ "Grandfather",
91
+ "Grandmother",
92
+ "Green",
93
+ "Ground",
94
+ "Gun",
95
+ "Hat",
96
+ "Hello",
97
+ "Horse",
98
+ "Hospital",
99
+ "Hour",
100
+ "House",
101
+ "How are you",
102
+ "Husband",
103
+ "I",
104
+ "Job",
105
+ "Key",
106
+ "King",
107
+ "Kitchen",
108
+ "Lamp",
109
+ "Laptop",
110
+ "Lawyer",
111
+ "Letter",
112
+ "Library",
113
+ "Location",
114
+ "Lock",
115
+ "Man",
116
+ "Manager",
117
+ "Market",
118
+ "Marriage",
119
+ "Mean",
120
+ "Medicine",
121
+ "Minute",
122
+ "Monday",
123
+ "Money",
124
+ "Month",
125
+ "Morning",
126
+ "Mother",
127
+ "Mouse",
128
+ "Newspaper",
129
+ "Nice",
130
+ "Night",
131
+ "Office",
132
+ "Orange",
133
+ "Page",
134
+ "Paint",
135
+ "Pant",
136
+ "Paper",
137
+ "Parent",
138
+ "Park",
139
+ "Patient",
140
+ "Peace",
141
+ "Pen",
142
+ "Pencil",
143
+ "Photograph",
144
+ "Pink",
145
+ "Plane",
146
+ "Player",
147
+ "Pleased",
148
+ "Pocket",
149
+ "Police",
150
+ "President",
151
+ "Price",
152
+ "Priest",
153
+ "Queen",
154
+ "Race (ethnicity)",
155
+ "Radio",
156
+ "Red",
157
+ "Religion",
158
+ "Reporter",
159
+ "Restaurant",
160
+ "Ring",
161
+ "Saturday",
162
+ "School",
163
+ "Science",
164
+ "Screen",
165
+ "Season",
166
+ "Second",
167
+ "Secretary",
168
+ "Shirt",
169
+ "Shoes",
170
+ "Sign",
171
+ "Sister",
172
+ "Skirt",
173
+ "Soap",
174
+ "Soldier",
175
+ "Son",
176
+ "Sport",
177
+ "Spring",
178
+ "Store or Shop",
179
+ "Street or Road",
180
+ "Student",
181
+ "Suit",
182
+ "Summer",
183
+ "Sunday",
184
+ "Table",
185
+ "Teacher",
186
+ "Team",
187
+ "Technology",
188
+ "Telephone",
189
+ "Television",
190
+ "Temple",
191
+ "Thank you",
192
+ "Thursday",
193
+ "Time",
194
+ "Today",
195
+ "Tomorrow",
196
+ "Tool",
197
+ "Train",
198
+ "Train Station",
199
+ "Transportation",
200
+ "Truck",
201
+ "Tuesday",
202
+ "Ugly",
203
+ "University",
204
+ "Waiter",
205
+ "War",
206
+ "Wednesday",
207
+ "Week",
208
+ "White",
209
+ "Wife",
210
+ "Window",
211
+ "Winter",
212
+ "Woman",
213
+ "Year",
214
+ "Yellow",
215
+ "Yesterday",
216
+ "alive",
217
+ "bad",
218
+ "big large",
219
+ "cheap",
220
+ "clean",
221
+ "cold",
222
+ "cool",
223
+ "curved",
224
+ "dead",
225
+ "deep",
226
+ "dirty",
227
+ "dry",
228
+ "expensive",
229
+ "famous",
230
+ "fast",
231
+ "female",
232
+ "flat",
233
+ "good",
234
+ "happy",
235
+ "hard",
236
+ "he",
237
+ "healthy",
238
+ "heavy",
239
+ "high",
240
+ "hot",
241
+ "it",
242
+ "light",
243
+ "long",
244
+ "loose",
245
+ "loud",
246
+ "low",
247
+ "male",
248
+ "narrow",
249
+ "new",
250
+ "old",
251
+ "poor",
252
+ "quiet",
253
+ "rich",
254
+ "sad",
255
+ "shallow",
256
+ "she",
257
+ "short",
258
+ "sick",
259
+ "slow",
260
+ "small little",
261
+ "soft",
262
+ "strong",
263
+ "tall",
264
+ "they",
265
+ "thick",
266
+ "thin",
267
+ "tight",
268
+ "train ticket",
269
+ "warm",
270
+ "we",
271
+ "weak",
272
+ "wet",
273
+ "wide",
274
+ "you",
275
+ "you (plural)",
276
+ "young"
277
+ ]
278
+ }
eda/train_test_validation_split-1.png ADDED

Git LFS Details

  • SHA256: e7b1fa109930615195557a188bacb77560c5dec94f52a82693d772a56af97643
  • Pointer size: 131 Bytes
  • Size of remote file: 115 kB
eda/train_test_validation_split-2.png ADDED

Git LFS Details

  • SHA256: 9ec1608762a566112bdcd49e82cba4a64405e6beefd9d2cd3fe8913fc6efe871
  • Pointer size: 131 Bytes
  • Size of remote file: 171 kB
expression_mapping.py ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ expression_mapping={107: "alive",
2
+ 58: "Nice",
3
+ 8: "Beautiful",
4
+ 115: "dead",
5
+ 120: "famous",
6
+ 122: "female",
7
+ 51: "Mean",
8
+ 21: "Deaf",
9
+ 111: "clean",
10
+ 117: "dirty",
11
+ 123: "flat",
12
+ 110: "cheap",
13
+ 119: "expensive",
14
+ 116: "deep",
15
+ 99: "Ugly",
16
+ 114: "curved",
17
+ 12: "Blind",
18
+ 142: "poor",
19
+ 138: "male",
20
+ 126: "hard",
21
+ 133: "light",
22
+ 137: "low",
23
+ 113: "cool",
24
+ 144: "rich",
25
+ 109: "big large",
26
+ 108: "bad",
27
+ 112: "cold",
28
+ 135: "loose",
29
+ 121: "fast",
30
+ 141: "old",
31
+ 130: "high",
32
+ 118: "dry",
33
+ 145: "sad",
34
+ 131: "hot",
35
+ 125: "happy",
36
+ 129: "heavy",
37
+ 128: "healthy",
38
+ 124: "good",
39
+ 146: "shallow",
40
+ 153: "strong",
41
+ 161: "weak",
42
+ 157: "thin",
43
+ 158: "tight",
44
+ 136: "loud",
45
+ 139: "narrow",
46
+ 134: "long",
47
+ 156: "thick",
48
+ 148: "short",
49
+ 152: "soft",
50
+ 150: "slow",
51
+ 151: "small little",
52
+ 149: "sick",
53
+ 154: "tall",
54
+ 140: "new",
55
+ 143: "quiet",
56
+ 95: "Today",
57
+ 163: "wide",
58
+ 159: "warm",
59
+ 96: "Tomorrow",
60
+ 162: "wet",
61
+ 1: "Afternoon",
62
+ 27: "Evening",
63
+ 56: "Morning",
64
+ 59: "Night",
65
+ 166: "young",
66
+ 53: "Minute",
67
+ 38: "Hour",
68
+ 88: "Sunday",
69
+ 55: "Month",
70
+ 94: "Time",
71
+ 70: "Pleased",
72
+ 63: "Paper",
73
+ 105: "Year",
74
+ 80: "Second",
75
+ 32: "Gift",
76
+ 102: "Week",
77
+ 43: "Key",
78
+ 48: "Lock",
79
+ 4: "Bag",
80
+ 106: "Yesterday",
81
+ 7: "Bathroom",
82
+ 15: "Card",
83
+ 66: "Pen",
84
+ 45: "Letter",
85
+ 9: "Bed",
86
+ 2: "Alright",
87
+ 67: "Pencil",
88
+ 24: "Dream",
89
+ 13: "Book",
90
+ 44: "Kitchen",
91
+ 92: "Telephone",
92
+ 23: "Door",
93
+ 36: "Hello",
94
+ 61: "Page",
95
+ 40: "How are you",
96
+ 16: "Chair",
97
+ 89: "Table",
98
+ 97: "Tool",
99
+ 68: "Photograph",
100
+ 10: "Bedroom",
101
+ 103: "Window",
102
+ 62: "Paint",
103
+ 14: "Box",
104
+ 76: "Ring",
105
+ 82: "Soap",
106
+ 20: "Crowd",
107
+ 75: "Restaurant",
108
+ 98: "Train Station",
109
+ 31: "Friend",
110
+ 17: "Child",
111
+ 0: "Adult",
112
+ 46: "Library",
113
+ 39: "House",
114
+ 42: "India",
115
+ 86: "Street or Road",
116
+ 72: "Queen",
117
+ 85: "Store or Shop",
118
+ 64: "Park",
119
+ 77: "School",
120
+ 18: "City",
121
+ 49: "Market",
122
+ 60: "Office",
123
+ 132: "it",
124
+ 41: "I",
125
+ 6: "Bank",
126
+ 69: "Player",
127
+ 147: "she",
128
+ 19: "Court",
129
+ 155: "they",
130
+ 104: "Winter",
131
+ 93: "Temple",
132
+ 33: "God",
133
+ 50: "Marriage",
134
+ 29: "Exercise",
135
+ 37: "Hospital",
136
+ 34: "Ground",
137
+ 25: "Election",
138
+ 73: "Race (ethnicity)",
139
+ 11: "Bill",
140
+ 87: "Summer",
141
+ 160: "we",
142
+ 127: "he",
143
+ 22: "Death",
144
+ 84: "Spring",
145
+ 47: "Location",
146
+ 26: "Energy",
147
+ 54: "Money",
148
+ 28: "Ex. Monsoon",
149
+ 165: "you (plural)",
150
+ 65: "Peace",
151
+ 5: "Ball",
152
+ 71: "Price",
153
+ 35: "Gun",
154
+ 30: "Fall",
155
+ 164: "you",
156
+ 81: "Sign",
157
+ 100: "University",
158
+ 83: "Sport",
159
+ 74: "Religion",
160
+ 101: "War",
161
+ 57: "Newspaper",
162
+ 3: "Attack",
163
+ 90: "Team",
164
+ 78: "Science",
165
+ 79: "Season",
166
+ 52: "Medicine",
167
+ 91: "Technology",
168
+ }
model.py ADDED
@@ -0,0 +1,407 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from collections import OrderedDict
3
+
4
+ import torch
5
+ import torch.nn as nn
6
+
7
+ # def make_layers(block, no_relu_layers):
8
+ # layers = []
9
+ # for layer_name, v in block.items():
10
+ # if 'pool' in layer_name:
11
+ # layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1],
12
+ # padding=v[2])
13
+ # layers.append((layer_name, layer))
14
+ # else:
15
+ # conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1],
16
+ # kernel_size=v[2], stride=v[3],
17
+ # padding=v[4])
18
+ # layers.append((layer_name, conv2d))
19
+ # if layer_name not in no_relu_layers:
20
+ # layers.append(('relu_'+layer_name, nn.ReLU(inplace=True)))
21
+
22
+ # return nn.Sequential(OrderedDict(layers))
23
+
24
+
25
+ def make_layers(block, no_relu_layers,prelu_layers = []):
26
+ layers = []
27
+
28
+ for layer_name, v in block.items():
29
+ if 'pool' in layer_name:
30
+ layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1],
31
+ padding=v[2])
32
+ layers.append((layer_name, layer))
33
+ else:
34
+ #[3, 64, 3, 1, 1]
35
+ conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1],
36
+ kernel_size=v[2], stride=v[3],
37
+ padding=v[4])
38
+ layers.append((layer_name, conv2d))
39
+ if layer_name not in no_relu_layers:
40
+ if layer_name not in prelu_layers:
41
+ layers.append(('relu_'+layer_name, nn.ReLU(inplace=True)))
42
+ else:
43
+ layers.append(('prelu'+layer_name[4:],nn.PReLU(v[1])))
44
+
45
+ return nn.Sequential(OrderedDict(layers))
46
+
47
+
48
+ def make_layers_Mconv(block,no_relu_layers):
49
+ modules = []
50
+ for layer_name, v in block.items():
51
+ layers = []
52
+ if 'pool' in layer_name:
53
+ layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1],
54
+ padding=v[2])
55
+ layers.append((layer_name, layer))
56
+ else:
57
+ conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1],
58
+ kernel_size=v[2], stride=v[3],
59
+ padding=v[4])
60
+ layers.append((layer_name, conv2d))
61
+ if layer_name not in no_relu_layers:
62
+ layers.append(('Mprelu'+layer_name[5:], nn.PReLU(v[1])))
63
+ modules.append(nn.Sequential(OrderedDict(layers)))
64
+ return nn.ModuleList(modules)
65
+
66
+ class bodypose_25_model(nn.Module):
67
+ def __init__(self):
68
+ super(bodypose_25_model,self).__init__()
69
+ # these layers have no relu layer
70
+ no_relu_layers = ['Mconv7_stage0_L1','Mconv7_stage0_L2',\
71
+ 'Mconv7_stage1_L1', 'Mconv7_stage1_L2',\
72
+ 'Mconv7_stage2_L2', 'Mconv7_stage3_L2']
73
+ prelu_layers = ['conv4_2','conv4_3_CPM','conv4_4_CPM']
74
+ blocks = {}
75
+ block0 = OrderedDict([
76
+ ('conv1_1', [3, 64, 3, 1, 1]),
77
+ ('conv1_2', [64, 64, 3, 1, 1]),
78
+ ('pool1_stage1', [2, 2, 0]),
79
+ ('conv2_1', [64, 128, 3, 1, 1]),
80
+ ('conv2_2', [128, 128, 3, 1, 1]),
81
+ ('pool2_stage1', [2, 2, 0]),
82
+ ('conv3_1', [128, 256, 3, 1, 1]),
83
+ ('conv3_2', [256, 256, 3, 1, 1]),
84
+ ('conv3_3', [256, 256, 3, 1, 1]),
85
+ ('conv3_4', [256, 256, 3, 1, 1]),
86
+ ('pool3_stage1', [2, 2, 0]),
87
+ ('conv4_1', [256, 512, 3, 1, 1]),
88
+ ('conv4_2', [512, 512, 3, 1, 1]),
89
+ ('conv4_3_CPM', [512, 256, 3, 1, 1]),
90
+ ('conv4_4_CPM', [256, 128, 3, 1, 1])
91
+ ])
92
+ self.model0 = make_layers(block0, no_relu_layers,prelu_layers)
93
+
94
+ #L2
95
+ #stage0
96
+ blocks['Mconv1_stage0_L2'] = OrderedDict([
97
+ ('Mconv1_stage0_L2_0',[128,96,3,1,1]),
98
+ ('Mconv1_stage0_L2_1',[96,96,3,1,1]),
99
+ ('Mconv1_stage0_L2_2',[96,96,3,1,1])
100
+ ])
101
+ for i in range(2,6):
102
+ blocks['Mconv%d_stage0_L2' % i] = OrderedDict([
103
+ ('Mconv%d_stage0_L2_0' % i,[288,96,3,1,1]),
104
+ ('Mconv%d_stage0_L2_1' % i,[96,96,3,1,1]),
105
+ ('Mconv%d_stage0_L2_2' % i,[96,96,3,1,1])
106
+ ])
107
+ blocks['Mconv6_7_stage0_L2'] = OrderedDict([
108
+ ('Mconv6_stage0_L2',[288, 256, 1,1,0]),
109
+ ('Mconv7_stage0_L2',[256,52,1,1,0])
110
+ ])
111
+ #stage1~3
112
+ for s in range(1,4):
113
+ blocks['Mconv1_stage%d_L2' % s] = OrderedDict([
114
+ ('Mconv1_stage%d_L2_0' % s,[180,128,3,1,1]),
115
+ ('Mconv1_stage%d_L2_1' % s,[128,128,3,1,1]),
116
+ ('Mconv1_stage%d_L2_2' % s,[128,128,3,1,1])
117
+ ])
118
+ for i in range(2,6):
119
+ blocks['Mconv%d_stage%d_L2' % (i,s)] = OrderedDict([
120
+ ('Mconv%d_stage%d_L2_0' % (i,s) ,[384,128,3,1,1]),
121
+ ('Mconv%d_stage%d_L2_1' % (i,s) ,[128,128,3,1,1]),
122
+ ('Mconv%d_stage%d_L2_2' % (i,s) ,[128,128,3,1,1])
123
+ ])
124
+ blocks['Mconv6_7_stage%d_L2' % s] = OrderedDict([
125
+ ('Mconv6_stage%d_L2' % s,[384,512,1,1,0]),
126
+ ('Mconv7_stage%d_L2' % s,[512,52,1,1,0])
127
+ ])
128
+
129
+ #L1
130
+ #stage0
131
+ blocks['Mconv1_stage0_L1'] = OrderedDict([
132
+ ('Mconv1_stage0_L1_0',[180,96,3,1,1]),
133
+ ('Mconv1_stage0_L1_1',[96,96,3,1,1]),
134
+ ('Mconv1_stage0_L1_2',[96,96,3,1,1])
135
+ ])
136
+ for i in range(2,6):
137
+ blocks['Mconv%d_stage0_L1' % i] = OrderedDict([
138
+ ('Mconv%d_stage0_L1_0' % i,[288,96,3,1,1]),
139
+ ('Mconv%d_stage0_L1_1' % i,[96,96,3,1,1]),
140
+ ('Mconv%d_stage0_L1_2' % i,[96,96,3,1,1])
141
+ ])
142
+ blocks['Mconv6_7_stage0_L1'] = OrderedDict([
143
+ ('Mconv6_stage0_L1',[288, 256, 1,1,0]),
144
+ ('Mconv7_stage0_L1',[256,26,1,1,0])
145
+ ])
146
+ #stage1
147
+ blocks['Mconv1_stage1_L1'] = OrderedDict([
148
+ ('Mconv1_stage1_L1_0',[206,128,3,1,1]),
149
+ ('Mconv1_stage1_L1_1',[128,128,3,1,1]),
150
+ ('Mconv1_stage1_L1_2',[128,128,3,1,1])
151
+ ])
152
+ for i in range(2,6):
153
+ blocks['Mconv%d_stage1_L1' % i] = OrderedDict([
154
+ ('Mconv%d_stage1_L1_0' % i,[384,128,3,1,1]),
155
+ ('Mconv%d_stage1_L1_1' % i,[128,128,3,1,1]),
156
+ ('Mconv%d_stage1_L1_2' % i,[128,128,3,1,1])
157
+ ])
158
+ blocks['Mconv6_7_stage1_L1'] = OrderedDict([
159
+ ('Mconv6_stage1_L1',[384,512,1,1,0]),
160
+ ('Mconv7_stage1_L1',[512,26,1,1,0])
161
+ ])
162
+
163
+ for k in blocks.keys():
164
+ blocks[k] = make_layers_Mconv(blocks[k], no_relu_layers)
165
+ self.models = nn.ModuleDict(blocks)
166
+ #self.model_L2_S0_mconv1 = blocks['Mconv1_stage0_L2']
167
+ for param in self.parameters():
168
+ param.requires_grad = False
169
+
170
+
171
+ def _Mconv_forward(self,x,models):
172
+ outs = []
173
+ out = x
174
+ for m in models:
175
+ out = m(out)
176
+ outs.append(out)
177
+ return torch.cat(outs,1)
178
+
179
+ def forward(self,x):
180
+ out0 = self.model0(x)
181
+ #L2
182
+ tout = out0
183
+ for s in range(4):
184
+ tout = self._Mconv_forward(tout,self.models['Mconv1_stage%d_L2' % s])
185
+ for v in range(2,6):
186
+ tout = self._Mconv_forward(tout,self.models['Mconv%d_stage%d_L2' % (v,s)])
187
+ tout = self.models['Mconv6_7_stage%d_L2' % s][0](tout)
188
+ tout = self.models['Mconv6_7_stage%d_L2' % s][1](tout)
189
+ outL2 = tout
190
+ tout = torch.cat([out0,tout],1)
191
+ #L1 stage0
192
+ #tout = torch.cat([out0,outL2],1)
193
+ tout = self._Mconv_forward(tout, self.models['Mconv1_stage0_L1'])
194
+ for v in range(2,6):
195
+ tout = self._Mconv_forward(tout, self.models['Mconv%d_stage0_L1' % v])
196
+ tout = self.models['Mconv6_7_stage0_L1'][0](tout)
197
+ tout = self.models['Mconv6_7_stage0_L1'][1](tout)
198
+ outS0L1 = tout
199
+ tout = torch.cat([out0,outS0L1,outL2],1)
200
+ #L1 stage1
201
+ tout = self._Mconv_forward(tout, self.models['Mconv1_stage1_L1'])
202
+ for v in range(2,6):
203
+ tout = self._Mconv_forward(tout, self.models['Mconv%d_stage1_L1' % v])
204
+ tout = self.models['Mconv6_7_stage1_L1'][0](tout)
205
+ outS1L1 = self.models['Mconv6_7_stage1_L1'][1](tout)
206
+
207
+ return outL2, outS1L1
208
+
209
+
210
+ class bodypose_model(nn.Module):
211
+ def __init__(self):
212
+ super(bodypose_model, self).__init__()
213
+
214
+ # these layers have no relu layer
215
+ no_relu_layers = ['conv5_5_CPM_L1', 'conv5_5_CPM_L2', 'Mconv7_stage2_L1',\
216
+ 'Mconv7_stage2_L2', 'Mconv7_stage3_L1', 'Mconv7_stage3_L2',\
217
+ 'Mconv7_stage4_L1', 'Mconv7_stage4_L2', 'Mconv7_stage5_L1',\
218
+ 'Mconv7_stage5_L2', 'Mconv7_stage6_L1', 'Mconv7_stage6_L1']
219
+ blocks = {}
220
+ block0 = OrderedDict([
221
+ ('conv1_1', [3, 64, 3, 1, 1]),
222
+ ('conv1_2', [64, 64, 3, 1, 1]),
223
+ ('pool1_stage1', [2, 2, 0]),
224
+ ('conv2_1', [64, 128, 3, 1, 1]),
225
+ ('conv2_2', [128, 128, 3, 1, 1]),
226
+ ('pool2_stage1', [2, 2, 0]),
227
+ ('conv3_1', [128, 256, 3, 1, 1]),
228
+ ('conv3_2', [256, 256, 3, 1, 1]),
229
+ ('conv3_3', [256, 256, 3, 1, 1]),
230
+ ('conv3_4', [256, 256, 3, 1, 1]),
231
+ ('pool3_stage1', [2, 2, 0]),
232
+ ('conv4_1', [256, 512, 3, 1, 1]),
233
+ ('conv4_2', [512, 512, 3, 1, 1]),
234
+ ('conv4_3_CPM', [512, 256, 3, 1, 1]),
235
+ ('conv4_4_CPM', [256, 128, 3, 1, 1])
236
+ ])
237
+
238
+
239
+ # Stage 1
240
+ block1_1 = OrderedDict([
241
+ ('conv5_1_CPM_L1', [128, 128, 3, 1, 1]),
242
+ ('conv5_2_CPM_L1', [128, 128, 3, 1, 1]),
243
+ ('conv5_3_CPM_L1', [128, 128, 3, 1, 1]),
244
+ ('conv5_4_CPM_L1', [128, 512, 1, 1, 0]),
245
+ ('conv5_5_CPM_L1', [512, 38, 1, 1, 0])
246
+ ])
247
+
248
+ block1_2 = OrderedDict([
249
+ ('conv5_1_CPM_L2', [128, 128, 3, 1, 1]),
250
+ ('conv5_2_CPM_L2', [128, 128, 3, 1, 1]),
251
+ ('conv5_3_CPM_L2', [128, 128, 3, 1, 1]),
252
+ ('conv5_4_CPM_L2', [128, 512, 1, 1, 0]),
253
+ ('conv5_5_CPM_L2', [512, 19, 1, 1, 0])
254
+ ])
255
+ blocks['block1_1'] = block1_1
256
+ blocks['block1_2'] = block1_2
257
+
258
+ self.model0 = make_layers(block0, no_relu_layers)
259
+
260
+ # Stages 2 - 6
261
+ for i in range(2, 7):
262
+ blocks['block%d_1' % i] = OrderedDict([
263
+ ('Mconv1_stage%d_L1' % i, [185, 128, 7, 1, 3]),
264
+ ('Mconv2_stage%d_L1' % i, [128, 128, 7, 1, 3]),
265
+ ('Mconv3_stage%d_L1' % i, [128, 128, 7, 1, 3]),
266
+ ('Mconv4_stage%d_L1' % i, [128, 128, 7, 1, 3]),
267
+ ('Mconv5_stage%d_L1' % i, [128, 128, 7, 1, 3]),
268
+ ('Mconv6_stage%d_L1' % i, [128, 128, 1, 1, 0]),
269
+ ('Mconv7_stage%d_L1' % i, [128, 38, 1, 1, 0])
270
+ ])
271
+
272
+ blocks['block%d_2' % i] = OrderedDict([
273
+ ('Mconv1_stage%d_L2' % i, [185, 128, 7, 1, 3]),
274
+ ('Mconv2_stage%d_L2' % i, [128, 128, 7, 1, 3]),
275
+ ('Mconv3_stage%d_L2' % i, [128, 128, 7, 1, 3]),
276
+ ('Mconv4_stage%d_L2' % i, [128, 128, 7, 1, 3]),
277
+ ('Mconv5_stage%d_L2' % i, [128, 128, 7, 1, 3]),
278
+ ('Mconv6_stage%d_L2' % i, [128, 128, 1, 1, 0]),
279
+ ('Mconv7_stage%d_L2' % i, [128, 19, 1, 1, 0])
280
+ ])
281
+
282
+ for k in blocks.keys():
283
+ blocks[k] = make_layers(blocks[k], no_relu_layers)
284
+
285
+ self.model1_1 = blocks['block1_1']
286
+ self.model2_1 = blocks['block2_1']
287
+ self.model3_1 = blocks['block3_1']
288
+ self.model4_1 = blocks['block4_1']
289
+ self.model5_1 = blocks['block5_1']
290
+ self.model6_1 = blocks['block6_1']
291
+
292
+ self.model1_2 = blocks['block1_2']
293
+ self.model2_2 = blocks['block2_2']
294
+ self.model3_2 = blocks['block3_2']
295
+ self.model4_2 = blocks['block4_2']
296
+ self.model5_2 = blocks['block5_2']
297
+ self.model6_2 = blocks['block6_2']
298
+ for param in self.parameters():
299
+ param.requires_grad = False
300
+
301
+
302
+ def forward(self, x):
303
+
304
+ out1 = self.model0(x)
305
+
306
+ out1_1 = self.model1_1(out1)
307
+ out1_2 = self.model1_2(out1)
308
+ out2 = torch.cat([out1_1, out1_2, out1], 1)
309
+
310
+ out2_1 = self.model2_1(out2)
311
+ out2_2 = self.model2_2(out2)
312
+ out3 = torch.cat([out2_1, out2_2, out1], 1)
313
+
314
+ out3_1 = self.model3_1(out3)
315
+ out3_2 = self.model3_2(out3)
316
+ out4 = torch.cat([out3_1, out3_2, out1], 1)
317
+
318
+ out4_1 = self.model4_1(out4)
319
+ out4_2 = self.model4_2(out4)
320
+ out5 = torch.cat([out4_1, out4_2, out1], 1)
321
+
322
+ out5_1 = self.model5_1(out5)
323
+ out5_2 = self.model5_2(out5)
324
+ out6 = torch.cat([out5_1, out5_2, out1], 1)
325
+
326
+ out6_1 = self.model6_1(out6)
327
+ out6_2 = self.model6_2(out6)
328
+
329
+ return out6_1, out6_2
330
+
331
+ class handpose_model(nn.Module):
332
+ def __init__(self):
333
+ super(handpose_model, self).__init__()
334
+
335
+ # these layers have no relu layer
336
+ no_relu_layers = ['conv6_2_CPM', 'Mconv7_stage2', 'Mconv7_stage3',\
337
+ 'Mconv7_stage4', 'Mconv7_stage5', 'Mconv7_stage6']
338
+ # stage 1
339
+ block1_0 = OrderedDict([
340
+ ('conv1_1', [3, 64, 3, 1, 1]),
341
+ ('conv1_2', [64, 64, 3, 1, 1]),
342
+ ('pool1_stage1', [2, 2, 0]),
343
+ ('conv2_1', [64, 128, 3, 1, 1]),
344
+ ('conv2_2', [128, 128, 3, 1, 1]),
345
+ ('pool2_stage1', [2, 2, 0]),
346
+ ('conv3_1', [128, 256, 3, 1, 1]),
347
+ ('conv3_2', [256, 256, 3, 1, 1]),
348
+ ('conv3_3', [256, 256, 3, 1, 1]),
349
+ ('conv3_4', [256, 256, 3, 1, 1]),
350
+ ('pool3_stage1', [2, 2, 0]),
351
+ ('conv4_1', [256, 512, 3, 1, 1]),
352
+ ('conv4_2', [512, 512, 3, 1, 1]),
353
+ ('conv4_3', [512, 512, 3, 1, 1]),
354
+ ('conv4_4', [512, 512, 3, 1, 1]),
355
+ ('conv5_1', [512, 512, 3, 1, 1]),
356
+ ('conv5_2', [512, 512, 3, 1, 1]),
357
+ ('conv5_3_CPM', [512, 128, 3, 1, 1])
358
+ ])
359
+
360
+ block1_1 = OrderedDict([
361
+ ('conv6_1_CPM', [128, 512, 1, 1, 0]),
362
+ ('conv6_2_CPM', [512, 22, 1, 1, 0])
363
+ ])
364
+
365
+ blocks = {}
366
+ blocks['block1_0'] = block1_0
367
+ blocks['block1_1'] = block1_1
368
+
369
+ # stage 2-6
370
+ for i in range(2, 7):
371
+ blocks['block%d' % i] = OrderedDict([
372
+ ('Mconv1_stage%d' % i, [150, 128, 7, 1, 3]),
373
+ ('Mconv2_stage%d' % i, [128, 128, 7, 1, 3]),
374
+ ('Mconv3_stage%d' % i, [128, 128, 7, 1, 3]),
375
+ ('Mconv4_stage%d' % i, [128, 128, 7, 1, 3]),
376
+ ('Mconv5_stage%d' % i, [128, 128, 7, 1, 3]),
377
+ ('Mconv6_stage%d' % i, [128, 128, 1, 1, 0]),
378
+ ('Mconv7_stage%d' % i, [128, 22, 1, 1, 0])
379
+ ])
380
+
381
+ for k in blocks.keys():
382
+ blocks[k] = make_layers(blocks[k], no_relu_layers)
383
+
384
+ self.model1_0 = blocks['block1_0']
385
+ self.model1_1 = blocks['block1_1']
386
+ self.model2 = blocks['block2']
387
+ self.model3 = blocks['block3']
388
+ self.model4 = blocks['block4']
389
+ self.model5 = blocks['block5']
390
+ self.model6 = blocks['block6']
391
+ for param in self.parameters():
392
+ param.requires_grad = False
393
+
394
+ def forward(self, x):
395
+ out1_0 = self.model1_0(x)
396
+ out1_1 = self.model1_1(out1_0)
397
+ concat_stage2 = torch.cat([out1_1, out1_0], 1)
398
+ out_stage2 = self.model2(concat_stage2)
399
+ concat_stage3 = torch.cat([out_stage2, out1_0], 1)
400
+ out_stage3 = self.model3(concat_stage3)
401
+ concat_stage4 = torch.cat([out_stage3, out1_0], 1)
402
+ out_stage4 = self.model4(concat_stage4)
403
+ concat_stage5 = torch.cat([out_stage4, out1_0], 1)
404
+ out_stage5 = self.model5(concat_stage5)
405
+ concat_stage6 = torch.cat([out_stage5, out1_0], 1)
406
+ out_stage6 = self.model6(concat_stage6)
407
+ return out_stage6
packages.txt ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ ffmpeg
2
+ libgl1
3
+ libglib2.0-0
4
+ libsm6
5
+ libxrender1
6
+ libxext6
requirements-docker.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ streamlit==1.34.0
2
+ numpy==1.26.4
3
+ Pillow==11.2.1
4
+ matplotlib==3.10.1
5
+ opencv-python-headless==4.11.0.86
6
+ scipy==1.15.2
7
+ scikit-image==0.25.2
8
+ pandas==2.2.3
9
+ keras==3.9.2
10
+ ffmpeg-python==0.2.0
11
+ seaborn==0.13.2
12
+ huggingface_hub==0.32.4
requirements.txt CHANGED
@@ -1,3 +1,19 @@
1
- altair
 
 
 
 
 
 
 
2
  pandas
3
- streamlit
 
 
 
 
 
 
 
 
 
 
1
+ streamlit
2
+ numpy
3
+ Pillow
4
+ matplotlib==3.5.3
5
+ opencv-python
6
+ scipy
7
+ scikit-image
8
+ tqdm
9
  pandas
10
+ torch
11
+ torchaudio
12
+ torchvision
13
+ torchtext
14
+ torchdata
15
+ av
16
+ keras
17
+ ffmpeg-python
18
+ seaborn[stats]
19
+ huggingface_hub
transformer_comparison.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Streamlit comparison of the archived Transformer predictions on one INCLUDE panel."""
2
+
3
+ import csv
4
+ import gzip
5
+ import io
6
+ import json
7
+ from pathlib import Path
8
+ from urllib.parse import quote
9
+
10
+ import streamlit as st
11
+ import streamlit.components.v1 as components
12
+
13
+
14
+ ARMS = {
15
+ "rolling": "Rolling Transformer · epoch 31",
16
+ "causal": "Causal Transformer · epoch 102",
17
+ }
18
+ DATA = Path(__file__).with_name("data") / "include_validation.json.gz"
19
+ VIDEO_REPO = "spsarolkar/AI4Bharat-INCLUDE-dataset"
20
+ VIDEO_REVISION = "d12293ab05cb5dc2cb6a342c364132a18adc49c9"
21
+
22
+
23
+ @st.cache_data
24
+ def load_panel():
25
+ with gzip.open(DATA, "rt", encoding="utf-8") as source:
26
+ return json.load(source)
27
+
28
+
29
+ def outcome_counts(clips, arm):
30
+ correct = wrong_early = correct_early = unconfirmed = 0
31
+ for clip in clips:
32
+ prediction = clip[arm]
33
+ correct += (prediction["last"] or {}).get("label") == clip["label"]
34
+ early = prediction["early"]
35
+ if early is None:
36
+ unconfirmed += 1
37
+ elif early["label"] == clip["label"]:
38
+ correct_early += 1
39
+ else:
40
+ wrong_early += 1
41
+ return correct, wrong_early, correct_early, unconfirmed
42
+
43
+
44
+ def score(value, total):
45
+ return f"{value / total:.2%} ({value}/{total})"
46
+
47
+
48
+ def export_scores(clips):
49
+ buffer = io.StringIO()
50
+ writer = csv.writer(buffer)
51
+ writer.writerow(("case", "expected", "model", "last_guess", "last_guess_seconds",
52
+ "first_confirmation", "confirmation_seconds", "outcome"))
53
+ for clip in clips:
54
+ for arm, model in ARMS.items():
55
+ prediction = clip[arm]
56
+ last, early = prediction["last"] or {}, prediction["early"] or {}
57
+ outcome = ("Unconfirmed" if not early else
58
+ "Correct" if early["label"] == clip["label"] else "Wrong")
59
+ writer.writerow((clip["id"], clip["label"], model, last.get("label"), last.get("time"),
60
+ early.get("label"), early.get("time"), outcome))
61
+ return buffer.getvalue()
62
+
63
+
64
+ def public_video(path):
65
+ if not path:
66
+ return None
67
+ parts = path.split("/")
68
+ if len(parts) != 5 or parts[0] != "videos" or parts[1] not in {"train", "val", "test"}:
69
+ raise ValueError("Unexpected public video path")
70
+ if any(part in {"", ".", ".."} for part in parts):
71
+ raise ValueError("Invalid public video path")
72
+ return f"https://huggingface.co/datasets/{VIDEO_REPO}/resolve/{VIDEO_REVISION}/{quote(path)}"
73
+
74
+
75
+ def render_playback(video_url, clip):
76
+ # Inline script data is escaped before interpolation into the iframe document.
77
+ payload = json.dumps({
78
+ "video": video_url, "arms": ARMS, "duration": clip["duration"],
79
+ "predictions": {arm: clip[arm] for arm in ARMS},
80
+ }, ensure_ascii=True).replace("<", "\\u003c")
81
+ template = Path(__file__).with_name("validation_playback.html").read_text(encoding="utf-8")
82
+ components.html(template.replace("/* PAYLOAD */", payload), height=480, scrolling=True)
83
+
84
+
85
+ def render_transformer_comparison():
86
+ st.title("SignLanguageTranslate")
87
+ st.header("Indian Sign Language model comparison")
88
+ try:
89
+ panel = load_panel()
90
+ except (OSError, ValueError):
91
+ st.error("The archived evaluation could not be loaded.")
92
+ return
93
+
94
+ if panel.get("videoSource") != {"repoId": VIDEO_REPO, "revision": VIDEO_REVISION}:
95
+ st.error("The archived evaluation points to a different video dataset revision.")
96
+ return
97
+ clips = panel["clips"]
98
+ st.caption("720 isolated INCLUDE clips from the saved iPad validation split. "
99
+ "Both models were evaluated on the same Apple Vision keypoints.")
100
+ from transformer_runtime import evaluate_validation, load_assets, replay_clip, settings
101
+
102
+ revision = settings()
103
+ live = False
104
+ if revision:
105
+ if st.button("Run both models on all 720 validation clips"):
106
+ try:
107
+ with st.spinner("Running both models on the frozen validation clips…"):
108
+ st.session_state["transformer_evaluation"] = evaluate_validation(revision, clips)
109
+ except (OSError, ValueError, RuntimeError):
110
+ st.error("Live evaluation could not load the pinned artifacts or finish inference.")
111
+ results = st.session_state.get("transformer_evaluation")
112
+ if results is not None:
113
+ clips = results
114
+ live = True
115
+ else:
116
+ st.info("The server is showing the archived validation results. Live inference requires the Space's private model and dataset configuration.")
117
+ st.subheader("Validation accuracy")
118
+ st.caption("Computed by the Space" if live else "Archived evaluation on the Mac")
119
+ table = []
120
+ for arm, name in ARMS.items():
121
+ correct, wrong_early, correct_early, unconfirmed = outcome_counts(clips, arm)
122
+ table.append({
123
+ "Model": name,
124
+ "Final accuracy": score(correct, len(clips)),
125
+ "Wrong early confirmations": score(wrong_early, len(clips)),
126
+ "Correct early confirmations": correct_early,
127
+ "Unconfirmed clips": unconfirmed,
128
+ })
129
+ st.dataframe(table, use_container_width=True, hide_index=True)
130
+ st.caption("Final accuracy uses the last scheduled prediction before the clip ends. "
131
+ "An early confirmation is the first label repeated across three consecutive updates.")
132
+ st.download_button("Download validation results (CSV)", export_scores(clips),
133
+ file_name="include-validation-results.csv", mime="text/csv")
134
+
135
+ st.subheader("Replay predictions")
136
+ labels = sorted({clip["label"] for clip in clips}, key=str.casefold)
137
+ selected_label = st.selectbox("Expected sign", labels, index=labels.index("Woman") if "Woman" in labels else 0)
138
+ matching = [clip for clip in clips if clip["label"] == selected_label]
139
+ default_clip = next((i for i, item in enumerate(matching) if item["id"] == "include-004"), 0)
140
+ clip = st.selectbox("Validation clip", matching, index=default_clip,
141
+ format_func=lambda item: f"{item['id']} · {item['duration']:.2f} s"
142
+ + (" · source video available" if item["videoPath"] else ""))
143
+ if revision and not live:
144
+ if st.button("Run both models on this clip", key=f"predict-{clip['id']}"):
145
+ try:
146
+ with st.spinner("Running both models on the saved poses…"):
147
+ inputs, models = load_assets(revision)
148
+ index = next(i for i, item in enumerate(clips) if item["id"] == clip["id"])
149
+ st.session_state["transformer_clip"] = replay_clip(inputs, models, index, clip)
150
+ except (OSError, ValueError, RuntimeError):
151
+ st.error("Live inference could not load the pinned artifacts or predict this clip.")
152
+ result = st.session_state.get("transformer_clip")
153
+ if result is not None and result["id"] == clip["id"]:
154
+ clip = result
155
+ live = True
156
+ video_url = public_video(clip["videoPath"])
157
+ if video_url:
158
+ split = {"val": "validation", "train": "train", "test": "test"}[clip["videoPath"].split("/")[1]]
159
+ st.caption(f"Public video repository split: {split}.")
160
+ st.markdown(f"[Open source video]({video_url})")
161
+ else:
162
+ st.caption("Use the timeline slider to review this clip’s archived predictions.")
163
+ st.caption(("Predictions computed by the Space" if live else "Archived model predictions") + " at 250 ms intervals.")
164
+ render_playback(video_url, clip)
165
+ with st.expander("All prediction updates"):
166
+ updates = {}
167
+ for arm, name in ARMS.items():
168
+ for tick in clip[arm]["ticks"]:
169
+ updates.setdefault(tick["time"], {"Time (s)": tick["time"]})[name] = tick["label"]
170
+ st.dataframe(list(updates.values()), use_container_width=True, hide_index=True)
171
+
172
+ with st.expander("Dataset splits and model sources"):
173
+ st.markdown(
174
+ "- This evaluation uses 720 of 743 clips in the saved iPad validation split. "
175
+ "The models share 256 output labels. This panel covers 254 labels, and 23 clips "
176
+ "were excluded because their labels are outside the model vocabulary. "
177
+ "Epoch 102 was selected using this validation panel. Signer independence is unverified.\n"
178
+ "- The legacy LSTM uses 167 output labels and a custom test manifest. Of its 412 clips, "
179
+ "245 match the public INCLUDE train split, 25 validation, 66 test, and 76 have no exact match. "
180
+ "Training membership for the published legacy weights remains unverified.\n"
181
+ "- [INCLUDE source and attribution](https://huggingface.co/datasets/ai4bharat/INCLUDE) · "
182
+ "[Public video dataset](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-dataset) · "
183
+ "[Apple Vision keypoint dataset card](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-keypoints) · "
184
+ "[Native Transformer checkpoint card](https://huggingface.co/spsarolkar/isl-transformer-mlx) · "
185
+ "[Legacy LSTM model card](https://huggingface.co/sunilsarolkar/isl-translation-model)"
186
+ )
transformer_inference.py ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CPU replay of the two frozen Apple Vision INCLUDE Transformer checkpoints."""
2
+
3
+ from bisect import bisect_right
4
+ from dataclasses import dataclass
5
+ import hashlib
6
+ import json
7
+ from pathlib import Path
8
+ import struct
9
+
10
+ import numpy as np
11
+
12
+
13
+ WEIGHTS_SHA256 = {
14
+ "rolling": "5df06195356eb8b61aff409d556cebcfb81accfe50fce23a3fb8dbb532a8e3d6",
15
+ "causal": "6ed79c31aec5c232141aa9cfccf2fa7eec5265ace50fce3a73a8b75471e658f8",
16
+ }
17
+ VOCABULARY_SHA256 = "caaf0d329a7de21ffc170a19e539d1fe197e9ef7344f8bf57ea752812de43963"
18
+ FEATURES = 180
19
+ HIDDEN = 256
20
+ HEADS = 4
21
+ EMBEDDING = 384
22
+ CONTEXT = 48
23
+ TICK_SECONDS = 0.25
24
+ GROUPS = ("body", "leftHand", "rightHand")
25
+ GROUP_SIZES = (18, 21, 21)
26
+ FREQUENCIES = np.exp(np.arange(0, HIDDEN, 2, dtype=np.float32) * np.float32(-np.log(10_000) / HIDDEN))
27
+
28
+
29
+ def _weights(path, arm):
30
+ path = Path(path)
31
+ if not 0 < path.stat().st_size <= 20_000_000:
32
+ raise ValueError("Unexpected checkpoint length")
33
+ data = path.read_bytes()
34
+ if hashlib.sha256(data).hexdigest() != WEIGHTS_SHA256[arm]:
35
+ raise ValueError(f"Unexpected {arm} checkpoint digest")
36
+ header_size, = struct.unpack_from("<Q", data)
37
+ header = json.loads(data[8:8 + header_size])
38
+ buffer = memoryview(data)[8 + header_size:]
39
+ tensors = {}
40
+ rolling_prefixes = {0: "inputProjection", 1: "inputNorm", 2: "outputNorm", 3: "outputProjection", 5: "pool"}
41
+ for name, metadata in header.items():
42
+ if name == "__metadata__":
43
+ continue
44
+ if arm == "rolling":
45
+ _, layer, suffix = name.split(".", 2)
46
+ layer = int(layer)
47
+ prefix = rolling_prefixes[layer] if layer < 6 else f"encoderLayers.{layer - 6}"
48
+ name = f"{prefix}.{suffix}"
49
+ start, end = metadata["data_offsets"]
50
+ if metadata["dtype"] != "F32" or not 0 <= start < end <= len(buffer):
51
+ raise ValueError("Invalid checkpoint tensor")
52
+ tensors[name] = np.frombuffer(buffer[start:end], dtype="<f4").reshape(metadata["shape"])
53
+ if len(tensors) != (69 if arm == "rolling" else 64):
54
+ raise ValueError("Unexpected checkpoint tensor count")
55
+ return tensors
56
+
57
+
58
+ def _vocabulary(rows):
59
+ if len(rows) != 256 or len({row["word"] for row in rows}) != 256:
60
+ raise ValueError("Expected the frozen 256-sign vocabulary")
61
+ digest = hashlib.sha256()
62
+ for row in rows:
63
+ vector = np.asarray(row["vector"], dtype="<f4")
64
+ if vector.shape != (EMBEDDING,) or not np.isfinite(vector).all():
65
+ raise ValueError("Invalid sign embedding")
66
+ digest.update(row["word"].encode("utf-8") + b"\0")
67
+ digest.update(vector.tobytes())
68
+ if digest.hexdigest() != VOCABULARY_SHA256:
69
+ raise ValueError("Unexpected ordered vocabulary digest")
70
+ rows = sorted(rows, key=lambda row: row["word"])
71
+ vectors = np.asarray([row["vector"] for row in rows], dtype=np.float32)
72
+ return [row["word"] for row in rows], vectors, np.linalg.norm(vectors, axis=1)
73
+
74
+
75
+ def _softmax(x):
76
+ x = np.exp(x - x.max(axis=-1, keepdims=True))
77
+ return x / x.sum(axis=-1, keepdims=True)
78
+
79
+
80
+ def _gelu(x):
81
+ # Abramowitz and Stegun 7.1.26 approximates erf to within 1.5e-7 in
82
+ # real arithmetic. Float32 output is checked against the native replay.
83
+ scaled = x * np.float32(1 / np.sqrt(2))
84
+ t = np.float32(1) / (np.float32(1) + np.float32(0.3275911) * np.abs(scaled))
85
+ polynomial = np.float32(1.061405429)
86
+ for coefficient in (-1.453152027, 1.421413741, -0.284496736, 0.254829592):
87
+ polynomial = polynomial * t + np.float32(coefficient)
88
+ erf = np.copysign(np.float32(1) - polynomial * t * np.exp(-scaled * scaled), scaled)
89
+ return (x * (np.float32(1) + erf)) / np.float32(2)
90
+
91
+
92
+ @dataclass(frozen=True)
93
+ class _Group:
94
+ names: tuple
95
+ values: np.ndarray
96
+
97
+ @classmethod
98
+ def decode(cls, points):
99
+ if points is None:
100
+ return None
101
+ return cls(tuple(p["id"] for p in points),
102
+ np.asarray([(p["x"], p["y"], p["confidence"]) for p in points], dtype=np.float32).reshape(-1, 3))
103
+
104
+
105
+ def _interpolate(a, b, fraction):
106
+ if a is None or b is None or len(a.names) != len(b.names):
107
+ return a if fraction < 0.5 else b
108
+ values = a.values + (b.values - a.values) * fraction
109
+ if a.names != b.names:
110
+ mismatched = [i for i, (name_a, name_b) in enumerate(zip(a.names, b.names)) if name_a != name_b]
111
+ values[mismatched] = a.values[mismatched]
112
+ return _Group(a.names, values)
113
+
114
+
115
+ def _serialize(groups):
116
+ body = groups[0]
117
+ offset = np.zeros(2, dtype=np.float32)
118
+ for name in ("neck", "nose"):
119
+ if name in body.names:
120
+ offset = body.values[body.names.index(name), :2]
121
+ break
122
+ result = np.zeros((FEATURES // 3, 3), dtype=np.float32)
123
+ start = 0
124
+ for group, minimum_size in zip(groups, GROUP_SIZES):
125
+ if group is not None:
126
+ length = min(len(group.names), len(result) - start)
127
+ result[start:start + length] = group.values[:length]
128
+ result[start:start + length, :2] -= offset
129
+ start += max(minimum_size, len(group.names))
130
+ else:
131
+ start += minimum_size
132
+ if start >= len(result):
133
+ break
134
+ return result.reshape(FEATURES)
135
+
136
+
137
+ def _resample(groups, serialized, start, end):
138
+ count = end - start
139
+ if count == 60:
140
+ return serialized[start:end]
141
+ result = np.empty((60, FEATURES), dtype=np.float32)
142
+ for row in range(60):
143
+ position = row / 59 * (count - 1)
144
+ lower = int(position)
145
+ upper = min(lower + 1, count - 1)
146
+ if lower == upper:
147
+ result[row] = serialized[start + lower]
148
+ continue
149
+ fraction = np.float32(position - lower)
150
+ result[row] = _serialize([
151
+ _interpolate(a, b, fraction) for a, b in zip(groups[start + lower], groups[start + upper])
152
+ ])
153
+ return result
154
+
155
+
156
+ def _position(time):
157
+ estimate = int(time * 12)
158
+ if estimate / 12 > time:
159
+ return estimate - 1
160
+ if (estimate + 1) / 12 <= time:
161
+ return estimate + 1
162
+ return estimate
163
+
164
+
165
+ class _PoseSequence:
166
+ def __init__(self, frames):
167
+ self.times = [float(frame["timestamp"]) for frame in frames]
168
+ if (not self.times or any(not np.isfinite(t) or t < 0 for t in self.times)
169
+ or any(a > b for a, b in zip(self.times, self.times[1:]))):
170
+ raise ValueError("Expected saved poses with finite chronological timestamps")
171
+ self.groups = [tuple(_Group.decode(frame.get(key)) for key in GROUPS) for frame in frames]
172
+ if any(groups[0] is None for groups in self.groups):
173
+ raise ValueError("Missing body group in saved pose")
174
+ self.rows = np.asarray([_serialize(groups) for groups in self.groups])
175
+ self.usable = np.isfinite(self.rows).all(axis=1) & (self.rows[:, 2::3] > 0).any(axis=1)
176
+ self.next_frame = 0
177
+ self.last_rolling_time = -np.inf
178
+
179
+ def rolling(self, time):
180
+ start = bisect_right(self.times, time - 2)
181
+ end = bisect_right(self.times, time, lo=start)
182
+ if end - start < 2 or self.times[end - 1] <= self.last_rolling_time:
183
+ return None
184
+ self.last_rolling_time = self.times[end - 1]
185
+ return _resample(self.groups, self.rows, start, end)
186
+
187
+ def causal(self, time):
188
+ horizon = _position(time) / 12
189
+ positions, values = [], []
190
+ while self.next_frame < len(self.times) and self.times[self.next_frame] <= horizon:
191
+ first = self.next_frame
192
+ frame_time = self.times[first]
193
+ position = _position(frame_time)
194
+ if position / 12 < frame_time:
195
+ position += 1
196
+ grid_time = position / 12
197
+ self.next_frame = bisect_right(self.times, grid_time, lo=first)
198
+ for i in range(self.next_frame - 1, first - 1, -1):
199
+ if grid_time - self.times[i] > 0.25:
200
+ break
201
+ if self.usable[i]:
202
+ positions.append(position)
203
+ values.append(self.rows[i])
204
+ break
205
+ return np.asarray(positions, dtype=np.int32), np.asarray(values, dtype=np.float32)
206
+
207
+
208
+ class Transformer:
209
+ """Load immutable model weights; each replay creates its own temporal state."""
210
+
211
+ def __init__(self, weights_path, vocabulary, arm):
212
+ if arm not in WEIGHTS_SHA256:
213
+ raise ValueError("Unknown Transformer mode")
214
+ self.arm = arm
215
+ self.weights = _weights(weights_path, arm)
216
+ self.labels, self.vectors, self.vector_norms = _vocabulary(vocabulary)
217
+
218
+ def _linear(self, x, name):
219
+ x = x @ self.weights[name + ".weight"].T
220
+ bias = self.weights.get(name + ".bias")
221
+ return x if bias is None else x + bias
222
+
223
+ def _norm(self, x, name):
224
+ mean = x.mean(axis=-1, keepdims=True)
225
+ centered = x - mean
226
+ variance = (centered * centered).mean(axis=-1, keepdims=True)
227
+ return centered / np.sqrt(variance + np.float32(1e-5)) * self.weights[name + ".weight"] + self.weights[name + ".bias"]
228
+
229
+ def _embed(self, rows, positions, cache=None):
230
+ x = self._norm(self._linear(rows, "inputProjection"), "inputNorm")
231
+ phase = positions.astype(np.float32)[:, None] * FREQUENCIES
232
+ x += np.concatenate((np.sin(phase), np.cos(phase)), axis=-1)
233
+ if cache is not None:
234
+ old_positions, old_layers = cache
235
+ key_positions = np.concatenate((old_positions, positions))
236
+ recent = positions[-1] - (CONTEXT - 1)
237
+ retain = int(np.searchsorted(key_positions, recent))
238
+ mask = (key_positions[None, :] <= positions[:, None]) & (key_positions[None, :] >= positions[:, None] - (CONTEXT - 1))
239
+ retained = []
240
+ for block in range(4):
241
+ name = f"encoderLayers.{block}"
242
+ z = self._norm(x, name + ".norm1")
243
+ attention = name + ".attention."
244
+ queries, keys, values = [
245
+ self._linear(z, attention + layer).reshape(len(x), HEADS, -1).transpose(1, 0, 2)
246
+ for layer in ("query_proj", "key_proj", "value_proj")
247
+ ]
248
+ if cache is not None and old_layers:
249
+ old_keys, old_values = old_layers[block]
250
+ keys = np.concatenate((old_keys, keys), axis=1)
251
+ values = np.concatenate((old_values, values), axis=1)
252
+ scores = queries @ keys.swapaxes(-1, -2) * np.float32(1 / 8)
253
+ if cache is not None:
254
+ scores = np.where(mask, scores, np.float32(-np.inf))
255
+ retained.append((keys[:, retain:].copy(), values[:, retain:].copy()))
256
+ joined = (_softmax(scores) @ values).transpose(1, 0, 2).reshape(len(x), HIDDEN)
257
+ x += self._linear(joined, attention + "out_proj") * self.weights[name + ".gamma1"]
258
+ z = self._linear(self._norm(x, name + ".norm2"), name + ".mlp.layers.0")
259
+ z = _gelu(z)
260
+ x += self._linear(z, name + ".mlp.layers.3") * self.weights[name + ".gamma2"]
261
+ x = self._norm(x, "outputNorm")
262
+ if cache is None:
263
+ keys = self._linear(x, "pool.keyProj")
264
+ values = self._linear(x, "pool.valueProj")
265
+ scores = self.weights["pool.query"].reshape(1, HIDDEN) @ keys.T * np.float32(1 / 16)
266
+ x = _softmax(scores) @ values
267
+ next_cache = None
268
+ else:
269
+ next_cache = key_positions[retain:].copy(), retained
270
+ x = self._linear(x, "outputProjection")
271
+ x /= np.sqrt((x * x).sum(axis=-1, keepdims=True) + np.float32(1e-6))
272
+ if not np.isfinite(x).all():
273
+ raise ValueError("Transformer produced non-finite output")
274
+ return x, next_cache
275
+
276
+ def _predict(self, embedding):
277
+ norms = np.linalg.norm(embedding) * self.vector_norms
278
+ scores = np.divide(self.vectors @ embedding, norms, out=np.zeros_like(norms), where=norms > 0)
279
+ best = int(scores.argmax())
280
+ return self.labels[best], float(max(0, scores[best]) * 100)
281
+
282
+ def replay(self, frames, duration):
283
+ """Yield scheduled predictions and the first latched confirmation."""
284
+ duration = float(duration)
285
+ if not np.isfinite(duration) or not 0 < duration <= 120:
286
+ raise ValueError("Expected a finite clip duration up to 120 seconds")
287
+ poses = _PoseSequence(frames)
288
+ rolling_positions = np.arange(60, dtype=np.int32)
289
+ cache = np.empty(0, dtype=np.int32), []
290
+ previous = confirmed = confirmed_at = None
291
+ streak = 0
292
+ index = 2
293
+ while index * TICK_SECONDS < duration:
294
+ time = index * TICK_SECONDS
295
+ label = score = None
296
+ if self.arm == "rolling":
297
+ rows = poses.rolling(time)
298
+ if rows is not None:
299
+ output, _ = self._embed(rows, rolling_positions)
300
+ label, score = self._predict(output[-1])
301
+ else:
302
+ positions, rows = poses.causal(time)
303
+ for start in range(0, len(positions), CONTEXT):
304
+ stop = start + CONTEXT
305
+ output, cache = self._embed(rows[start:stop], positions[start:stop], cache)
306
+ if len(positions):
307
+ label, score = self._predict(output[-1])
308
+ if label is None:
309
+ streak = 0
310
+ else:
311
+ streak = streak + 1 if label == previous else 1
312
+ if confirmed is None and streak == 3:
313
+ confirmed, confirmed_at = label, time
314
+ previous = label
315
+ yield {"index": index, "time": time, "label": label, "score": score,
316
+ "confirmationLabel": confirmed, "confirmationTime": confirmed_at}
317
+ index += 1
transformer_runtime.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Read the pinned checkpoints and Apple Vision validation data on the server."""
2
+
3
+ import gzip
4
+ import hashlib
5
+ import json
6
+ import os
7
+ from pathlib import Path
8
+ import re
9
+
10
+ import streamlit as st
11
+
12
+ from transformer_inference import Transformer
13
+
14
+ CONTRACT_PATH = Path(__file__).with_name("data") / "transformer_contract.json"
15
+ CONTRACT = json.loads(CONTRACT_PATH.read_text(encoding="utf-8"))
16
+
17
+
18
+ def read_verified(path, expected_hash):
19
+ payload = Path(path).read_bytes()
20
+ if hashlib.sha256(payload).hexdigest() != expected_hash:
21
+ raise ValueError("A frozen validation file failed its checksum")
22
+ return payload
23
+
24
+
25
+ def load_validation(download):
26
+ """Return only the approved validation rows, ordered by native clip ID."""
27
+ files = CONTRACT["dataset"]["files"]
28
+ vocab_path = "metadata/vocabulary.json"
29
+ vocabulary_rows = json.loads(read_verified(download(vocab_path), files[vocab_path]))
30
+ vectors = {row["name"]: row["embedding"] for row in vocabulary_rows if row["type"] == "word"}
31
+ labels = CONTRACT["classNames"]
32
+ vocabulary = [{"word": label, "vector": vectors[label]} for label in labels]
33
+ allowed = set(labels)
34
+ clips = {}
35
+ seen = set()
36
+ for name, expected in files.items():
37
+ if name == vocab_path:
38
+ continue
39
+ payload = read_verified(download(name), expected)
40
+ for line in gzip.decompress(payload).splitlines():
41
+ row = json.loads(line)
42
+ identity = row["video_id"]
43
+ words = [label["name"] for label in row["labels"] if label["type"] == "word"]
44
+ if (identity in seen or row["split"] != "validation" or row["dataset"] != "INCLUDE"
45
+ or row["extractor"] != "AppleVision" or row["native_format_version"] != 1
46
+ or len(words) != 1):
47
+ raise ValueError("Unexpected validation record")
48
+ seen.add(identity)
49
+ if words[0] not in allowed:
50
+ continue
51
+ frames = []
52
+ for index, frame in enumerate(row["frames"]):
53
+ if frame["frame_index"] != index or frame["source_model"] != "AppleVision":
54
+ raise ValueError("Unexpected frame schema")
55
+ frames.append({
56
+ "timestamp": frame["timestamp_s"],
57
+ "body": frame["body"], "leftHand": frame["left_hand"], "rightHand": frame["right_hand"],
58
+ })
59
+ if not frames or len(frames) != row["frame_count"]:
60
+ raise ValueError("Missing validation frames")
61
+ clips[identity] = (words[0], row["duration_s"], frames)
62
+ if len(seen) != 743 or len(clips) != 720:
63
+ raise ValueError("Frozen validation cohort size differs")
64
+ return vocabulary, [clips[key] for key in sorted(clips)]
65
+
66
+
67
+ def settings():
68
+ if os.environ.get("ISL_PREVIEW_ASSETS"):
69
+ return "local"
70
+ revision = os.environ.get("ISL_MODEL_REVISION", "")
71
+ if os.environ.get("ISL_ARTIFACT_READ_TOKEN") and re.fullmatch(r"[0-9a-f]{40}", revision):
72
+ return revision
73
+ return None
74
+
75
+
76
+ @st.cache_resource
77
+ def load_assets(revision):
78
+ dataset = CONTRACT["dataset"]
79
+ if revision == "local":
80
+ # The local preview accepts a server-configured directory with symlinked
81
+ # assets. It never accepts file paths from visitors.
82
+ root = Path(os.environ["ISL_PREVIEW_ASSETS"])
83
+ vocabulary, clips = load_validation(lambda name: root / "dataset" / name)
84
+ paths = {arm: root / filename for arm, filename in CONTRACT["modelFiles"].items()}
85
+ else:
86
+ from huggingface_hub import hf_hub_download
87
+ token = os.environ["ISL_ARTIFACT_READ_TOKEN"]
88
+ def download(name):
89
+ return hf_hub_download(dataset["id"], name, repo_type="dataset",
90
+ revision=dataset["revision"], token=token)
91
+ vocabulary, clips = load_validation(download)
92
+ paths = {arm: hf_hub_download(CONTRACT["modelRepo"], name, revision=revision, token=token)
93
+ for arm, name in CONTRACT["modelFiles"].items()}
94
+ models = {arm: Transformer(path, vocabulary, arm) for arm, path in paths.items()}
95
+ return clips, models
96
+
97
+
98
+ def replay_clip(clips, models, index, expected):
99
+ label, duration, frames = clips[index]
100
+ if (label, duration) != (expected["label"], expected["duration"]):
101
+ raise ValueError("Live validation clip differs from the archived panel")
102
+ output = {"id": expected["id"], "label": label, "duration": duration,
103
+ "videoPath": expected["videoPath"]}
104
+ for arm, model in models.items():
105
+ rows = list(model.replay(frames, duration))
106
+ ticks = [{key: row[key] for key in ("time", "label", "score")} for row in rows]
107
+ last = rows[-1]
108
+ early = next((row for row in rows if row["confirmationTime"] is not None), None)
109
+ output[arm] = {
110
+ "last": ticks[-1],
111
+ "early": None if early is None else {
112
+ "time": last["confirmationTime"], "label": last["confirmationLabel"], "score": early["score"],
113
+ },
114
+ "ticks": ticks,
115
+ }
116
+ return output
117
+
118
+
119
+ @st.cache_data(max_entries=1)
120
+ def evaluate_validation(revision, expected_clips):
121
+ clips, models = load_assets(revision)
122
+ return [replay_clip(clips, models, index, expected)
123
+ for index, expected in enumerate(expected_clips)]
util.py ADDED
@@ -0,0 +1,455 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import math
3
+ import cv2
4
+ import matplotlib
5
+ from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
6
+ from matplotlib.figure import Figure
7
+ import numpy as np
8
+ import matplotlib.pyplot as plt
9
+ import cv2
10
+ import copy
11
+ import seaborn as sns
12
+
13
+ def padRightDownCorner(img, stride, padValue):
14
+ h = img.shape[0]
15
+ w = img.shape[1]
16
+
17
+ pad = 4 * [None]
18
+ pad[0] = 0 # up
19
+ pad[1] = 0 # left
20
+ pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down
21
+ pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right
22
+
23
+ img_padded = img
24
+ pad_up = np.tile(img_padded[0:1, :, :]*0 + padValue, (pad[0], 1, 1))
25
+ img_padded = np.concatenate((pad_up, img_padded), axis=0)
26
+ pad_left = np.tile(img_padded[:, 0:1, :]*0 + padValue, (1, pad[1], 1))
27
+ img_padded = np.concatenate((pad_left, img_padded), axis=1)
28
+ pad_down = np.tile(img_padded[-2:-1, :, :]*0 + padValue, (pad[2], 1, 1))
29
+ img_padded = np.concatenate((img_padded, pad_down), axis=0)
30
+ pad_right = np.tile(img_padded[:, -2:-1, :]*0 + padValue, (1, pad[3], 1))
31
+ img_padded = np.concatenate((img_padded, pad_right), axis=1)
32
+
33
+ return img_padded, pad
34
+
35
+ # transfer caffe model to pytorch which will match the layer name
36
+ def transfer(model, model_weights):
37
+ transfered_model_weights = {}
38
+ for weights_name in model.state_dict().keys():
39
+ if len(weights_name.split('.'))>4: # body25
40
+ transfered_model_weights[weights_name] = model_weights['.'.join(
41
+ weights_name.split('.')[3:])]
42
+ else:
43
+ transfered_model_weights[weights_name] = model_weights['.'.join(
44
+ weights_name.split('.')[1:])]
45
+ return transfered_model_weights
46
+
47
+ # draw the body keypoint and lims
48
+ def draw_bodypose(canvas, candidate, subset, model_type='body25'):
49
+ stickwidth = 4
50
+ if model_type == 'body25':
51
+ limbSeq = [[1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],\
52
+ [10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],\
53
+ [11,24],[11,22],[14,21],[14,19],[22,23],[19,20]]
54
+ njoint = 25
55
+ else:
56
+ limbSeq = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], \
57
+ [9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], \
58
+ [0, 15], [15, 17], [2, 16], [5, 17]]
59
+ njoint = 18
60
+
61
+ # colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
62
+ # [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
63
+ # [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
64
+
65
+ colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
66
+ [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
67
+ [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85], [255,255,0], [255,255,85], [255,255,170],\
68
+ [255,255,255],[170,255,255],[85,255,255],[0,255,255]]
69
+
70
+ for i in range(njoint):
71
+ for n in range(len(subset)):
72
+ index = int(subset[n][i])
73
+ if index == -1:
74
+ continue
75
+ x, y = candidate[index][0:2]
76
+ cv2.circle(canvas, (int(x), int(y)), 4, colors[i], thickness=-1)
77
+ for i in range(njoint-1):
78
+ for n in range(len(subset)):
79
+ index = subset[n][np.array(limbSeq[i])]
80
+ if -1 in index:
81
+ continue
82
+ cur_canvas = canvas.copy()
83
+ Y = candidate[index.astype(int), 0]
84
+ X = candidate[index.astype(int), 1]
85
+ mX = np.mean(X)
86
+ mY = np.mean(Y)
87
+ length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
88
+ angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
89
+ # print('original (mX,mY,length,angle)',(mX,mY,length,angle))
90
+ # print(f'original cv2.ellipse2Poly((int({mY}), int({mX})), (int({length} / 2), {stickwidth}), int({angle}), 0, 360, 1)')
91
+ polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
92
+ # print(f'cv2.fillConvexPoly(cur_canvas, polygon, colors[i])')
93
+ cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
94
+ canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
95
+ # plt.imsave("preview.jpg", canvas[:, :, [2, 1, 0]])
96
+ # plt.imshow(canvas[:, :, [2, 1, 0]])
97
+ return canvas
98
+ #subsets [[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, -1.0, 11.0, 12.0, -1.0, 13.0, 14.0, 15.0, 16.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 26.650803712300775, 17.0]]
99
+ #candidates [[983.0, 172.0, 0.8991263508796692, 0.0], [980.0, 352.0, 0.930037796497345, 1.0], [848.0, 342.0, 0.8652207255363464, 2.0], [811.0, 598.0, 0.8107873797416687, 3.0], [806.0, 817.0, 0.7464589476585388, 4.0], [1120.0, 361.0, 0.8538270592689514, 5.0], [1148.0, 601.0, 0.6797391176223755, 6.0], [1149.0, 834.0, 0.5189468264579773, 7.0], [968.0, 757.0, 0.6468111276626587, 8.0], [876.0, 756.0, 0.6387956142425537, 9.0], [854.0, 1072.0, 0.4211728572845459, 10.0], [1057.0, 759.0, 0.6311940550804138, 11.0], [1038.0, 1072.0, 0.38531172275543213, 12.0], [955.0, 146.0, 0.925083339214325, 13.0], [1016.0, 151.0, 0.9023998379707336, 14.0], [909.0, 167.0, 0.9096773862838745, 15.0], [1057.0, 173.0, 0.8605436086654663, 16.0]]
100
+ def get_bodypose(candidate, subset, model_type='coco'):
101
+ stickwidth = 4
102
+ if model_type == 'body25':
103
+ limbSeq = [[1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],\
104
+ [10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],\
105
+ [11,24],[11,22],[14,21],[14,19],[22,23],[19,20]]
106
+ njoint = 25
107
+ else:
108
+ limbSeq = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], \
109
+ [9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16], \
110
+ [0, 15], [15, 17], [2, 16], [5, 17]]
111
+ njoint = 18
112
+
113
+ # colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
114
+ # [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
115
+ # [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
116
+
117
+ colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
118
+ [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
119
+ [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85], [255,255,0], [255,255,85], [255,255,170],\
120
+ [255,255,255],[170,255,255],[85,255,255],[0,255,255]]
121
+
122
+ x_y_circles=[]
123
+ for i in range(njoint):
124
+ for n in range(len(subset)):
125
+ index = int(subset[n][i])
126
+ if index == -1:
127
+ continue
128
+ x, y = candidate[index][0:2] # 983.0, 172.0
129
+ x_y_circles.append((x, y))
130
+ # cv2.circle(canvas, (int(x), int(y)), 4, colors[i], thickness=-1)
131
+
132
+ x_y_sticks=[]
133
+ for i in range(njoint-1):
134
+ for n in range(len(subset)):
135
+ index = subset[n][np.array(limbSeq[i])] #0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, -1.0, 11.0, 12.0, -1.0, 13.0, 14.0, 15.0, 16.0, -1.0, -1.0, -1.0, -1.0, -1.0
136
+ if -1 in index:
137
+ continue
138
+ # cur_canvas = canvas.copy()
139
+ Y = candidate[index.astype(int), 0]
140
+ X = candidate[index.astype(int), 1]
141
+ mX = np.mean(X)
142
+ mY = np.mean(Y)
143
+ length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
144
+ angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
145
+ x_y_sticks.append((mY, mX,angle,length))
146
+ # print('new (mX,mY,length,angle)',(mX,mY,length,angle))
147
+ # polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
148
+ # cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
149
+ # canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
150
+ # plt.imsave("preview.jpg", canvas[:, :, [2, 1, 0]])
151
+ # plt.imshow(canvas[:, :, [2, 1, 0]])
152
+ return (x_y_circles,x_y_sticks,)
153
+
154
+ #all_hands_peaks[[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [1100, 858], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0]], [[0, 0], [858, 859], [868, 894], [873, 938], [0, 0], [802, 920], [807, 961], [821, 977], [836, 992], [0, 0], [781, 955], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]]
155
+ def draw_handpose(canvas, all_hand_peaks, show_number=False):
156
+ edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], \
157
+ [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]]
158
+ fig = Figure(figsize=plt.figaspect(canvas))
159
+
160
+ fig.subplots_adjust(0, 0, 1, 1)
161
+ fig.subplots_adjust(bottom=0, top=1, left=0, right=1)
162
+ bg = FigureCanvas(fig)
163
+ ax = fig.subplots()
164
+ ax.axis('off')
165
+ ax.imshow(canvas)
166
+
167
+ width, height = ax.figure.get_size_inches() * ax.figure.get_dpi()
168
+
169
+ for peaks in all_hand_peaks:
170
+ for ie, e in enumerate(edges):
171
+ if np.sum(np.all(peaks[e], axis=1)==0)==0:
172
+ x1, y1 = peaks[e[0]]
173
+ x2, y2 = peaks[e[1]]
174
+ # print(f'original ax.plot([{x1}, {x2}], [{y1}, {y2}], color=matplotlib.colors.hsv_to_rgb([ie/float({len(edges)}), 1.0, 1.0]))')
175
+ ax.plot([x1, x2], [y1, y2], color=matplotlib.colors.hsv_to_rgb([ie/float(len(edges)), 1.0, 1.0]))
176
+
177
+ for i, keyponit in enumerate(peaks):
178
+ x, y = keyponit
179
+ # print(f"original ax.plot({x}, {y}, 'r.')")
180
+ ax.plot(x, y, 'r.')
181
+ if show_number:
182
+ ax.text(x, y, str(i))
183
+ # print(f'width = {width}, height={height}')
184
+ bg.draw()
185
+ canvas = np.fromstring(bg.tostring_rgb(), dtype='uint8').reshape(int(height), int(width), 3)
186
+ return canvas
187
+
188
+ def get_handpose(all_hand_peaks, show_number=False):
189
+ edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], \
190
+ [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]]
191
+ # fig = Figure(figsize=plt.figaspect(canvas))
192
+
193
+ # fig.subplots_adjust(0, 0, 1, 1)
194
+ # fig.subplots_adjust(bottom=0, top=1, left=0, right=1)
195
+ # bg = FigureCanvas(fig)
196
+ # ax = fig.subplots()
197
+ # ax.axis('off')
198
+ # ax.imshow(canvas)
199
+
200
+ # width, height = ax.figure.get_size_inches() * ax.figure.get_dpi()
201
+ export_edges=[[],[]]
202
+ export_peaks=[[],[]]
203
+ for idx,peaks in enumerate(all_hand_peaks):
204
+ for ie, e in enumerate(edges):
205
+ if np.sum(np.all(peaks[e], axis=1)==0)==0:
206
+ x1, y1 = peaks[e[0]]
207
+ x2, y2 = peaks[e[1]]
208
+ export_edges[idx].append((ie,(x1, y1),(x2, y2)))
209
+ # ax.plot([x1, x2], [y1, y2], color=matplotlib.colors.hsv_to_rgb([ie/float(len(edges)), 1.0, 1.0]))
210
+
211
+ for i, keyponit in enumerate(peaks):
212
+ x, y = keyponit
213
+ # ax.plot(x, y, 'r.')
214
+ # if show_number:
215
+ # ax.text(x, y, str(i))
216
+
217
+ export_peaks[idx].append((x,y,str(i)))
218
+ # bg.draw()
219
+ # canvas = np.fromstring(bg.tostring_rgb(), dtype='uint8').reshape(int(height), int(width), 3)
220
+ return (export_edges,export_peaks)
221
+
222
+ # image drawed by opencv is not good.
223
+ def draw_handpose_by_opencv(canvas, peaks, show_number=False):
224
+ edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], \
225
+ [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]]
226
+ # cv2.rectangle(canvas, (x, y), (x+w, y+w), (0, 255, 0), 2, lineType=cv2.LINE_AA)
227
+ # cv2.putText(canvas, 'left' if is_left else 'right', (x, y), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
228
+ for ie, e in enumerate(edges):
229
+ if np.sum(np.all(peaks[e], axis=1)==0)==0:
230
+ x1, y1 = peaks[e[0]]
231
+ x2, y2 = peaks[e[1]]
232
+ cv2.line(canvas, (x1, y1), (x2, y2), matplotlib.colors.hsv_to_rgb([ie/float(len(edges)), 1.0, 1.0])*255, thickness=2)
233
+
234
+ for i, keyponit in enumerate(peaks):
235
+ x, y = keyponit
236
+ cv2.circle(canvas, (x, y), 4, (0, 0, 255), thickness=-1)
237
+ if show_number:
238
+ cv2.putText(canvas, str(i), (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 0), lineType=cv2.LINE_AA)
239
+ return canvas
240
+
241
+ # detect hand according to body pose keypoints
242
+ # please refer to https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/hand/handDetector.cpp
243
+ def handDetect(candidate, subset, oriImg):
244
+ # right hand: wrist 4, elbow 3, shoulder 2
245
+ # left hand: wrist 7, elbow 6, shoulder 5
246
+ ratioWristElbow = 0.33
247
+ detect_result = []
248
+
249
+ image_height, image_width = oriImg.shape[0:2]
250
+ #print(f'handDetect ---------- {image_height}, {image_width}')
251
+ for person in subset.astype(int):
252
+ # if any of three not detected
253
+ has_left = np.sum(person[[5, 6, 7]] == -1) == 0
254
+ has_right = np.sum(person[[2, 3, 4]] == -1) == 0
255
+ if not (has_left or has_right):
256
+ continue
257
+ hands = []
258
+ #left hand
259
+ if has_left:
260
+ left_shoulder_index, left_elbow_index, left_wrist_index = person[[5, 6, 7]]
261
+ x1, y1 = candidate[left_shoulder_index][:2]
262
+ x2, y2 = candidate[left_elbow_index][:2]
263
+ x3, y3 = candidate[left_wrist_index][:2]
264
+ hands.append([x1, y1, x2, y2, x3, y3, True])
265
+ # right hand
266
+ if has_right:
267
+ right_shoulder_index, right_elbow_index, right_wrist_index = person[[2, 3, 4]]
268
+ x1, y1 = candidate[right_shoulder_index][:2]
269
+ x2, y2 = candidate[right_elbow_index][:2]
270
+ x3, y3 = candidate[right_wrist_index][:2]
271
+ hands.append([x1, y1, x2, y2, x3, y3, False])
272
+
273
+ for x1, y1, x2, y2, x3, y3, is_left in hands:
274
+ # pos_hand = pos_wrist + ratio * (pos_wrist - pos_elbox) = (1 + ratio) * pos_wrist - ratio * pos_elbox
275
+ # handRectangle.x = posePtr[wrist*3] + ratioWristElbow * (posePtr[wrist*3] - posePtr[elbow*3]);
276
+ # handRectangle.y = posePtr[wrist*3+1] + ratioWristElbow * (posePtr[wrist*3+1] - posePtr[elbow*3+1]);
277
+ # const auto distanceWristElbow = getDistance(poseKeypoints, person, wrist, elbow);
278
+ # const auto distanceElbowShoulder = getDistance(poseKeypoints, person, elbow, shoulder);
279
+ # handRectangle.width = 1.5f * fastMax(distanceWristElbow, 0.9f * distanceElbowShoulder);
280
+ x = x3 + ratioWristElbow * (x3 - x2)
281
+ y = y3 + ratioWristElbow * (y3 - y2)
282
+ distanceWristElbow = math.sqrt((x3 - x2) ** 2 + (y3 - y2) ** 2)
283
+ distanceElbowShoulder = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)
284
+ width = 1.5 * max(distanceWristElbow, 0.9 * distanceElbowShoulder)
285
+ # x-y refers to the center --> offset to topLeft point
286
+ # handRectangle.x -= handRectangle.width / 2.f;
287
+ # handRectangle.y -= handRectangle.height / 2.f;
288
+ x -= width / 2
289
+ y -= width / 2 # width = height
290
+ # overflow the image
291
+ if x < 0: x = 0
292
+ if y < 0: y = 0
293
+ width1 = width
294
+ width2 = width
295
+ if x + width > image_width: width1 = image_width - x
296
+ if y + width > image_height: width2 = image_height - y
297
+ width = min(width1, width2)
298
+ # the max hand box value is 20 pixels
299
+ if width >= 20:
300
+ detect_result.append([int(x), int(y), int(width), is_left])
301
+
302
+ '''
303
+ return value: [[x, y, w, True if left hand else False]].
304
+ width=height since the network require squared input.
305
+ x, y is the coordinate of top left
306
+ '''
307
+ return detect_result
308
+
309
+ def drawStickmodel(oriImg, x_ytupple, x_y_sticks, export_edges, export_peaks):
310
+ canvas = copy.deepcopy(oriImg)
311
+
312
+ colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0],
313
+ [85, 255, 0], [0, 255, 0], [0, 255, 85], [0, 255, 170], [0, 255, 255],
314
+ [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], [170, 0, 255],
315
+ [255, 0, 255], [255, 0, 170], [255, 0, 85], [255,255,0], [255,255,85],
316
+ [255,255,170], [255,255,255],[170,255,255],[85,255,255],[0,255,255]]
317
+ stickwidth = 4
318
+
319
+ for idx, (mX, mY, angle, length) in enumerate(x_y_sticks):
320
+ cur_canvas = canvas.copy()
321
+ polygon = cv2.ellipse2Poly((int(mX), int(mY)), (int(length / 2), stickwidth),
322
+ int(angle), 0, 360, 1)
323
+ cv2.fillConvexPoly(cur_canvas, polygon, colors[idx])
324
+ canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
325
+
326
+ for idx, (x, y) in enumerate(x_ytupple):
327
+ cv2.circle(canvas, (int(x), int(y)), 4, colors[idx], thickness=-1)
328
+
329
+ # Handpose
330
+ fig = Figure(figsize=plt.figaspect(canvas))
331
+ fig.subplots_adjust(0, 0, 1, 1)
332
+ ax = fig.subplots()
333
+ ax.axis('off')
334
+ ax.imshow(canvas)
335
+
336
+ edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9],
337
+ [9, 10], [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16],
338
+ [0, 17], [17, 18], [18, 19], [19, 20]]
339
+
340
+ for both_hand_edges in export_edges:
341
+ for (ie, (x1, y1), (x2, y2)) in both_hand_edges:
342
+ ax.plot([x1, x2], [y1, y2],
343
+ color=matplotlib.colors.hsv_to_rgb([ie/float(len(edges)), 1.0, 1.0]))
344
+
345
+ for both_hand_peaks in export_peaks:
346
+ for (x, y, text) in both_hand_peaks:
347
+ ax.plot(x, y, 'r.')
348
+
349
+ # Convert figure to numpy array
350
+ bg = FigureCanvas(fig)
351
+ bg.draw()
352
+
353
+ width, height = fig.get_size_inches() * fig.get_dpi()
354
+ buf = bg.buffer_rgba()
355
+ canvas = np.frombuffer(buf, dtype=np.uint8).reshape(int(height), int(width), 4)
356
+ canvas = canvas[:, :, :3] # keep only RGB
357
+
358
+ plt.close(fig) # clean up
359
+ return cv2.resize(canvas, (math.ceil(width), math.ceil(height)))
360
+
361
+ def draw_bar_plot_below_image(image, predictions, title, origImg):
362
+ """
363
+ Draws a bar plot of predictions below an image using OpenCV and Matplotlib.
364
+
365
+ Args:
366
+ image (numpy.ndarray): The image to display.
367
+ predictions (numpy.ndarray): Array containing prediction probabilities.
368
+ """
369
+
370
+
371
+
372
+ fig, ax = plt.subplots(figsize=(origImg.shape[1]/100,origImg.shape[0]/200), dpi=100)
373
+ plt.title(title)
374
+ # Create a figure and plot the bar chart
375
+ labels = list(predictions.keys())
376
+ probabilities = list(predictions.values())
377
+
378
+ # Create a Seaborn bar plot
379
+ sns.barplot(x=labels, y=probabilities,ax=ax) # Default color palette used
380
+ plt.close(fig) # Close plot to avoid memory leaks
381
+ fig.canvas.draw()
382
+ # Convert the plot to a NumPy array for manipulation
383
+ plot_image = np.array(fig.canvas.renderer.buffer_rgba())[:, :, :3] # Remove alpha channel
384
+
385
+ # Resize the plot image to match the width of the original image
386
+ # plot_image = cv2.resize(plot_image, (image.shape[1], math.ceil(image.shape[0] * 0.8))) # Adjust height ratio as needed
387
+
388
+ # Combine the image and plot image vertically (stacking)
389
+ combined_image = np.vstack((image, cv2.resize(plot_image,(image.shape[1],plot_image.shape[0]))))
390
+
391
+ return combined_image
392
+
393
+ def add_padding_to_bottom(image, pad_value, pad_height):
394
+ """
395
+ Adds padding to the bottom of an image with a specified value.
396
+
397
+ Args:
398
+ image (numpy.ndarray): The input image.
399
+ pad_value (tuple or int): The color value to fill the padding area.
400
+ pad_height (int): The height of the padding to add at the bottom.
401
+
402
+ Returns:
403
+ numpy.ndarray: The image with padding added.
404
+ """
405
+
406
+ # Get image dimensions
407
+ height, width, channels = image.shape
408
+ padding=np.zeros((pad_height, width, channels), dtype=image.dtype)
409
+ padding[:,:,:]=pad_value
410
+ # # Create a new image with the desired height
411
+ # padded_image = np.zeros((height + pad_height, width, channels), dtype=image.dtype)
412
+
413
+ # # Copy the original image to the top of the padded image
414
+ # padded_image[:height, :, :] = image
415
+
416
+ # # Fill the padding area with the specified value
417
+ # if isinstance(pad_value, tuple): # Check for multiple color values (e.g., BGR)
418
+ # padded_image[height:, :, :] = pad_value
419
+ # else: # Single value for all channels (e.g., black)
420
+ # padded_image[height:, :, :] = np.full((pad_height, width, 1), pad_value, dtype=image.dtype)
421
+
422
+ return np.vstack((image, padding))
423
+
424
+ def crop_to_drawing(image):
425
+ """
426
+ Crops an image to the tight bounding rectangle of non-zero pixels.
427
+
428
+ Args:
429
+ image: A NumPy array representing the image.
430
+
431
+ Returns:
432
+ A cropped image (NumPy array) containing only the drawing area.
433
+ """
434
+ image=np.transpose(image, (2, 0, 1))
435
+ united_x,united_h=0,0
436
+ for channel in np.arange(image.shape[0]):
437
+ x, y, w, h = cv2.boundingRect(image[channel])
438
+ if x>united_x:
439
+ united_x=x
440
+
441
+ if h>united_h:
442
+ united_h=h
443
+
444
+ for channel in np.arange(image.shape[0]):
445
+ # Crop the image
446
+ image[channel] = image[channel][y:y+united_h, x:x+united_x]
447
+ return image.transpose(image, (1,2,0))
448
+
449
+ # get max index of 2d array
450
+ def npmax(array):
451
+ arrayindex = array.argmax(1)
452
+ arrayvalue = array.max(1)
453
+ i = arrayvalue.argmax()
454
+ j = arrayindex[i]
455
+ return i, j
validation_playback.html ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <style>
6
+ body { margin: 0; font: 15px system-ui, sans-serif; color: #17212d; background: #fff; }
7
+ video { display: block; width: 100%; max-height: 248px; background: #111; border-radius: 8px; }
8
+ label { display: block; margin: 8px 0; }
9
+ input { width: calc(100% - 4px); margin: 0; }
10
+ #predictions { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 8px; }
11
+ article { border: 1px solid #bbc4cf; border-radius: 8px; padding: 10px; }
12
+ h2 { font-size: 14px; margin: 0 0 6px; }
13
+ p { margin: 4px 0; }
14
+ @media (prefers-color-scheme: dark) {
15
+ body { color: #e5e7eb; background: #111827; }
16
+ article { border-color: #667185; }
17
+ }
18
+ </style>
19
+ </head>
20
+ <body>
21
+ <video id="video" controls preload="none" playsinline hidden></video>
22
+ <p id="status" role="status"></p>
23
+ <label for="position">Playback time <output id="seconds" for="position">0.00 s</output></label>
24
+ <input type="range" id="position" min="0" step="0.01">
25
+ <section id="predictions" aria-label="Model predictions"></section>
26
+ <script>
27
+ const data = /* PAYLOAD */;
28
+ const video = document.getElementById("video");
29
+ const slider = document.getElementById("position");
30
+ const status = document.getElementById("status");
31
+ const seconds = document.getElementById("seconds");
32
+ const cards = {};
33
+ slider.max = data.duration;
34
+ if (data.video) {
35
+ video.hidden = false;
36
+ video.src = data.video;
37
+ video.addEventListener("error", () => {
38
+ status.textContent = "This browser could not play the source video. You can still use the timeline slider.";
39
+ });
40
+ }
41
+ for (const [arm, name] of Object.entries(data.arms)) {
42
+ const card = document.createElement("article");
43
+ const title = document.createElement("h2");
44
+ title.textContent = name;
45
+ const guess = document.createElement("p");
46
+ const confirmed = document.createElement("p");
47
+ card.append(title, guess, confirmed);
48
+ document.getElementById("predictions").append(card);
49
+ cards[arm] = {guess, confirmed};
50
+ }
51
+ function update(time) {
52
+ slider.value = time;
53
+ seconds.textContent = time.toFixed(2) + " s";
54
+ for (const [arm, {guess, confirmed}] of Object.entries(cards)) {
55
+ const result = data.predictions[arm];
56
+ let last;
57
+ for (const tick of result.ticks) {
58
+ if (tick.time > time) break;
59
+ last = tick;
60
+ }
61
+ guess.textContent = "Current guess: " + (last?.label ?? "—");
62
+ confirmed.textContent = "Confirmed: " + (
63
+ result.early && result.early.time <= time
64
+ ? result.early.label + " at " + result.early.time.toFixed(2) + " s"
65
+ : "—"
66
+ );
67
+ }
68
+ }
69
+ slider.addEventListener("input", () => {
70
+ const time = Number(slider.value);
71
+ if (data.video && video.readyState >= 1) video.currentTime = time;
72
+ update(time);
73
+ });
74
+ video.addEventListener("timeupdate", () => update(video.currentTime));
75
+ update(0);
76
+ </script>
77
+ </body>
78
+ </html>