HuggingFace-SK commited on
Commit
9b2eab6
·
1 Parent(s): 717217e

update all android changes

Browse files
exported/model.tflite CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:22598566d9e7879981299c3b6abacbf3d3bfe9e19dba8b63baec40a3e9459451
3
  size 4036817
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:716a92fa6546fd52e64c328521273aeb7827f01cb8335a8d6db00524c2e53aff
3
  size 4036817
main.py CHANGED
@@ -7,8 +7,11 @@ def index():
7
  return render_template('browser-detect.html')
8
 
9
  @app.route('/exported')
10
- def send_report():
11
- return send_from_directory("better_exported", "model.tflite")
 
 
 
12
 
13
  if (__name__ == '__main__'):
14
  app.run( host='0.0.0.0', port=7860)
 
7
  return render_template('browser-detect.html')
8
 
9
  @app.route('/exported')
10
+ def send_asl():
11
+ return send_from_directory("exported", "model.tflite")
12
+ @app.route('/word')
13
+ def send_word():
14
+ return send_from_directory("exported", "word.tflite")
15
 
16
  if (__name__ == '__main__'):
17
  app.run( host='0.0.0.0', port=7860)
static/browser_detect.css CHANGED
@@ -1,33 +1,50 @@
1
  * {
2
- box-sizing: border-box;
3
  }
4
 
5
  body {
6
- font-family: roboto;
7
- color: #3d3d3d;
8
- --mdc-theme-primary: #007f8b;
9
- --mdc-theme-on-primary: #f1f3f4;
10
- margin-left: 0px;
11
  }
12
 
13
  .container {
14
- position: relative;
15
  }
16
 
17
 
18
- #predicted_result{
19
- position: relative;
20
  text-align: left;
21
  padding: 10px;
22
- background-color: #656565;
23
  border-radius: 12.5px;
24
- min-width:calc(10px + 1em);
25
- max-width:100%;
26
  font-weight: bold;
27
- color:#DFDFDF
 
 
28
  }
29
- .wrapper_result{
30
- position: relative;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
  text-align: left;
32
  padding: 10px;
33
  background-color: #F3F3F3;
@@ -35,75 +52,327 @@ body {
35
  margin: 10px;
36
  margin-bottom: 20px;
37
  border-radius: 15px;
 
38
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
 
 
 
40
  }
41
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
 
43
- #text{
44
  text-align: center;
45
  padding: 10px;
46
- background-color:#d9d9d9;
47
-
48
  margin: none;
49
  margin-top: 0px;
50
  border-radius: 12.25px;
51
  resize: none;
52
- display:block; position:relative;
53
- width:100%;
 
54
  border: none;
55
- font-size:15px;
56
 
57
  }
58
- .wrapper_text{
59
- position: relative;
 
60
  text-align: left;
61
  padding: 10px;
62
  background-color: #F3F3F3;
63
  box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.25);
64
  margin: 10px;
65
  margin-top: none;
 
66
  border-radius: 15px;
67
  overflow-y: hidden;
68
  }
69
- .canvas_wrapper{
 
70
  position: relative;
71
- text-align: left;
72
- height:290px;
73
- background-color: #F3F3F3;
74
- box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38);
75
- margin:10px;
76
- margin-bottom: 20px;
77
-
78
- border-radius: 15px;
79
- overflow: hidden;
80
- }
81
- #output_canvas{
82
- display:block; position:absolute;
83
- left:0px;
84
- width:100%
85
- }
86
- #text-to-speech{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  margin-top: 5px;
88
  padding-top: 10px;
89
  padding-bottom: 10px;
90
  border-radius: 12.25px;
91
- width:100%;
92
- background-color: #374e60;
93
- font-size: large;
94
- border: none;
95
- color:#d3d3d3
 
 
 
 
 
 
96
  }
97
- #webcamButton{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  margin: 0;
99
  position: absolute;
100
  top: 50%;
101
  left: 50%;
102
  -ms-transform: translate(-50%, -50%);
103
  transform: translate(-50%, -50%);
104
- padding:10px;
105
- border:none;
106
  border-radius: 12.25px;
107
- background-color: #f2f2f2;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
108
  box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
109
  }
 
1
  * {
2
+ box-sizing: border-box;
3
  }
4
 
5
  body {
6
+ font-family: roboto;
7
+ color: #3d3d3d;
8
+ --mdc-theme-primary: #007f8b;
9
+ --mdc-theme-on-primary: #f1f3f4;
 
10
  }
11
 
12
  .container {
13
+ position: relative;
14
  }
15
 
16
 
17
+ #predicted_result {
18
+ position: relative;
19
  text-align: left;
20
  padding: 10px;
21
+ background: #5d5d5d;
22
  border-radius: 12.5px;
23
+ min-width: calc(10px + 1em);
24
+ max-width: 100%;
25
  font-weight: bold;
26
+ color: #DFDFDF;
27
+ overflow: hidden;
28
+
29
  }
30
+
31
+ .mode_switch_modeBtn {
32
+ position: relative;
33
+ text-align: left;
34
+ padding: 2px;
35
+ border-radius: 12.5px;
36
+ color: #000000;
37
+ font-weight: 600;
38
+
39
+ min-width: 60px;
40
+ height:45px;
41
+ line-height: 40px;
42
+ text-align: center;
43
+
44
+ }
45
+
46
+ .wrapper_result {
47
+ position: relative;
48
  text-align: left;
49
  padding: 10px;
50
  background-color: #F3F3F3;
 
52
  margin: 10px;
53
  margin-bottom: 20px;
54
  border-radius: 15px;
55
+ }
56
 
57
+ @keyframes pulse {
58
+ 0% {
59
+ background-color: #F3F3F3; /* Original background color */
60
+ box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.25);
61
+ }
62
+ 50% {
63
+ background-color: #eeffed; /* Green color for pulse */
64
+ box-shadow: 0px 0px 30px 10px #6eff6e6c; /* Optional: stronger shadow */
65
+ }
66
+ 100% {
67
+ background-color: #F3F3F3; /* Return to original color */
68
+ box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.25);
69
+ }
70
+ }
71
 
72
+ .pulse {
73
+ animation: pulse 0.6s ease-in-out 1; /* Duration, timing function, and iteration count */
74
  }
75
 
76
+ .mode_selector{
77
+ position:absolute;
78
+ width:75px;
79
+ height:35px;
80
+ background: #ff6e7d75;
81
+ border-radius: 15px;
82
+ top:5px;
83
+ left: 5px;
84
+ transition: all 0.25s ease-out;
85
+ z-index: 10;
86
+ }
87
+
88
+ .mode_switch{
89
+ overflow: hidden;
90
+ position: relative;
91
+ text-align: left;
92
+ height:45px;
93
+ background-color: #ffffff2e;
94
+ backdrop-filter: blur(30px);
95
+ box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.25);
96
+
97
+ justify-content: space-evenly;
98
+ border-radius: 15px;
99
+ gap:10px;
100
+ width: 160px;
101
+
102
+ }
103
+
104
+ .right{
105
+ left:80px;
106
+ }
107
+ .mode_switch_wrapper{
108
+ position: absolute;
109
+ bottom:10px;
110
+ width:100%;
111
+ }
112
 
113
+ #text {
114
  text-align: center;
115
  padding: 10px;
116
+ background-color: #d9d9d9;
117
+
118
  margin: none;
119
  margin-top: 0px;
120
  border-radius: 12.25px;
121
  resize: none;
122
+ display: block;
123
+ position: relative;
124
+ width: 100%;
125
  border: none;
126
+ font-size: 15px;
127
 
128
  }
129
+
130
+ .wrapper_text {
131
+ position: relative;
132
  text-align: left;
133
  padding: 10px;
134
  background-color: #F3F3F3;
135
  box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.25);
136
  margin: 10px;
137
  margin-top: none;
138
+ margin-bottom: 65px;
139
  border-radius: 15px;
140
  overflow-y: hidden;
141
  }
142
+
143
+ .canvas_wrapper {
144
  position: relative;
145
+ text-align: left;
146
+ height: 290px;
147
+ background-color: #F3F3F3;
148
+ box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38);
149
+ margin: 10px;
150
+ margin-bottom: 20px;
151
+
152
+ border-radius: 15px;
153
+ overflow: hidden;
154
+ }
155
+
156
+ #output_canvas {
157
+ display: block;
158
+ position: absolute;
159
+ left: 0px;
160
+ width: 100%
161
+ }
162
+
163
+ #speakButton {
164
+ margin-top: 5px;
165
+ padding-top: 10px;
166
+ padding-bottom: 10px;
167
+ border-radius: 12.25px;
168
+ flex:2;
169
+ display: flex;
170
+ align-items: center;
171
+ gap:10px;
172
+ justify-content:center;
173
+ background: rgb(0,27,64);
174
+ background: linear-gradient(45deg, #a8404b 0%, #ff6e7d 33%, #d6505e 66%, #a8404b 100%);
175
+ font-size: large;
176
+ border: none;
177
+ color: #fcfcfc;
178
+ background-size: 200% 200%;
179
+ /*animation: mystical 10s ease infinite; /* Animation */
180
+ }
181
+ #undoButton {
182
  margin-top: 5px;
183
  padding-top: 10px;
184
  padding-bottom: 10px;
185
  border-radius: 12.25px;
186
+ flex:1;
187
+ display: flex;
188
+ align-items: center;
189
+ gap:10px;
190
+ justify-content:center;
191
+ background: #5d5d5d;
192
+ font-size: large;
193
+ border: none;
194
+ color: #fcfcfc;
195
+ background-size: 200% 200%;
196
+ /*animation: mystical 10s ease infinite; /* Animation */
197
  }
198
+ #clearButton {
199
+ margin-top: 5px;
200
+ padding-top: 10px;
201
+ padding-bottom: 10px;
202
+ border-radius: 12.25px;
203
+ flex:1;
204
+ display: flex;
205
+ align-items: center;
206
+ gap:10px;
207
+ justify-content:center;
208
+ background: #5d5d5d;
209
+ font-size: large;
210
+ border: none;
211
+ color: #fcfcfc;
212
+ background-size: 200% 200%;
213
+ /*animation: mystical 10s ease infinite; /* Animation */
214
+ }
215
+
216
+ @keyframes mystical {
217
+ 0% {
218
+ background-position: 0% 50%;
219
+ }
220
+ 25% {
221
+ background-position: 50% 0%;
222
+ }
223
+ 50% {
224
+ background-position: 100% 50%;
225
+ }
226
+ 75% {
227
+ background-position: 50% 100%;
228
+ }
229
+ 100% {
230
+ background-position: 0% 50%;
231
+ }
232
+ }
233
+ #webcamButton {
234
  margin: 0;
235
  position: absolute;
236
  top: 50%;
237
  left: 50%;
238
  -ms-transform: translate(-50%, -50%);
239
  transform: translate(-50%, -50%);
240
+ padding: 10px;
241
+ border: none;
242
  border-radius: 12.25px;
243
+ background: rgb(0,27,64);
244
+ background-size: 400% 400%;
245
+ background: rgb(0,27,64);
246
+ background: linear-gradient(45deg, #a8404b 0%, #ff6e7d 33%, #d6505e 66%, #a8404b 100%);
247
+ color: rgb(233, 251, 255);
248
+ box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38);
249
+ }
250
+
251
+ .bottom-nav {
252
+ position: fixed;
253
+ bottom: 0;
254
+ height: 60px;
255
+ border-top-left-radius: 15px;
256
+ border-top-right-radius: 15px;
257
+ background-color: #fcfcfc;
258
+ box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38);
259
+ width: 100%;
260
+ display:flex;
261
+ align-items: center;
262
+ justify-content: space-evenly;
263
+ flex-direction: row;
264
+ font-family: roboto;
265
+ z-index:10;
266
+ left:0;
267
+ }
268
+ .bottom-nav-btn{
269
+ flex-direction: row;
270
+ display: flex;
271
+ align-items: center;
272
+ justify-content: center;
273
+
274
+ }
275
+
276
+ .center-btn{
277
+ border-radius: 30px;
278
+ box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38);
279
+ background-size: 400% 400%;
280
+ background: linear-gradient(45deg, #a8404b 0%, #ff6e7d 50%, #d6505e 100%);
281
+ opacity: 1;
282
+ height: 60px;
283
+ width: 80px;
284
+ background-position: 0% 50%;
285
+ transition: all 0.1s ease;
286
+ }
287
+
288
+ .about-card{
289
+ padding:10px;
290
+ border-radius: 10px;
291
  box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38);
292
+ display:flex;
293
+ flex-direction: column;
294
+ gap:5x;
295
+ margin-bottom:10px;
296
+ overflow: hidden;
297
+ }
298
+ .about-card span:nth-child(1){
299
+ font-size: 20px;
300
+ font-family: Impact, Haettenschweiler, 'Arial Narrow Bold', sans-serif;
301
+ font-weight: 700;
302
+ }
303
+ .about-card label:nth-child(1){
304
+ font-size: 20px;
305
+ font-family: Impact, Haettenschweiler, 'Arial Narrow Bold', sans-serif;
306
+ font-weight: 700;
307
+ }
308
+ .about-card span:nth-child(2){
309
+ font-family: Impact, Haettenschweiler, 'Arial Narrow Bold', sans-serif;
310
+ font-size: 12px;
311
+ font-weight: 700;
312
+ }
313
+ h1{
314
+ font-size: 30px;
315
+ font-family: Impact, Haettenschweiler, 'Arial Narrow Bold', sans-serif;
316
+ font-weight: 700;
317
+ }
318
+ #info-page{
319
+ margin-bottom: 200px;
320
+ }
321
+ #speechForm{
322
+ font-family: Impact, Haettenschweiler, 'Arial Narrow Bold', sans-serif;
323
+ font-weight: 700;
324
+ padding-left: 10px;
325
+ padding-right: 10px;
326
+ }
327
+ select{
328
+ margin-top: 5px;
329
+ width:100%;
330
+ border-radius: 5px;
331
+ border: none;
332
+ padding:5px;
333
+ background-color: #d9d9d9;
334
+ }
335
+ #speechForm button{
336
+ margin-top: 5px;
337
+ width:100%;
338
+ border-radius: 5px;
339
+ border: none;
340
+ color:#fcfcfc;
341
+ padding:5px;
342
+ background-color: #f96a79;
343
+ }
344
+ #testText{
345
+ margin-top: 5px;
346
+ width:100%;
347
+ border-radius: 5px;
348
+ border: none;
349
+ resize: none;
350
+ padding:5px;
351
+ background-color: #d9d9d9;
352
+ }
353
+
354
+ #settings-page{
355
+ position: relative;
356
+ overflow-x: hidden;
357
+ margin: -8px;
358
+ padding-bottom: 20px;
359
+ margin-bottom: 40px;
360
+ }
361
+ .settings-btn{
362
+ display: inline-flex;
363
+ justify-content: space-between;
364
+ align-items: center;
365
+ }
366
+
367
+ .settings-header{
368
+ font-family: Impact, Haettenschweiler, 'Arial Narrow Bold', sans-serif;
369
+ font-weight: 700;
370
+ font-size: 30px;
371
+ }
372
+ .textControls{
373
+ display: flex;
374
+ gap:8px;
375
+ flex-direction: row;
376
+ justify-content: space-evenly;
377
+ align-items: center;
378
  }
templates/browser-detect.html CHANGED
@@ -1,4 +1,23 @@
1
  <!DOCTYPE html>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  <html lang="en">
3
 
4
  <head></head>
@@ -14,14 +33,14 @@
14
  <!-- For Android
15
  <link rel="stylesheet" type="text/css" href="http://127.0.0.1:8125/assets/static/browser_detect.css" />
16
  -->
17
- <!-- For Web -->
18
  <link rel="stylesheet" type="text/css" href="static/browser_detect.css" />
19
 
20
 
21
  </head>
22
 
23
  <body translate="no">
24
- <!-- For Android
25
  <script src="../assets/ipc/androidjs.js"></script>
26
  <script src="http://127.0.0.1:8125/assets/static/drawing_utils.js" crossorigin="anonymous"></script>
27
  <script src="http://127.0.0.1:8125/assets/static/hands.js" crossorigin="anonymous"></script>
@@ -29,8 +48,7 @@
29
  <script src="http://127.0.0.1:8125/assets/static/tfjs-backend-cpu"></script>
30
  <script src="http://127.0.0.1:8125/assets/static/tf-tflite.min.js"></script>
31
  <script src="http://127.0.0.1:8125/assets/static/vision_wasm_internal.js" crossorigin="anonymous"></script>
32
- -->
33
-
34
  <!-- For Web -->
35
  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js"
36
  crossorigin="anonymous"></script>
@@ -38,586 +56,1140 @@
38
  <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"></script>
39
  <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-cpu"></script>
40
  <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-tflite/dist/tf-tflite.min.js"></script>
41
-
42
-
43
-
44
- <div class="container">
45
-
46
- <video id="webcam" style="display:none" autoplay playsinline muted></video>
47
- <div class="canvas_wrapper" id="canvas_wrapper">
48
- <button id="switch-camera" style="display:none; position: absolute; top:10px; left:10px; padding:5px; height:40px; width:40px; text-align: center; border-radius: 12.25px; font-size: 20px; font-weight: 900; border:none; background-color: #f2f2f2; color:black;
49
- box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38); z-index:100">
50
- <span>⟳</span>
51
- </button>
52
- <canvas class="output_canvas" id="output_canvas" width="100%" height="300%"></canvas>
53
- <center>
54
- <button id="webcamButton" style="font-weight: 600; color:black;">
55
- <span>Enable Webcam</span>
56
  </button>
57
- </center>
58
- </div>
59
- </div>
60
- <center>
61
- <img id="output_image" style="display:none"></img>
62
- <div class="wrapper_result">
63
- <div id="predicted_result">></div>
 
 
 
 
 
 
 
 
64
  </div>
65
- <div class="wrapper_text">
66
- <textarea id="text" onkeyup="set_output_array(this.value)"></textarea>
67
- <button id="text-to-speech" onclick="speak(document.getElementById('text').value)">
68
- <span>Listen 🔊</span>
69
- </button>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
 
71
- <audio id="audioPlayer">-</audio>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
  </div>
73
- <div id="logUI">
74
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
75
  </div>
76
- <center>
77
- <script>
78
- var speechSupported = true
79
- var prevSpeech = ""
80
 
81
- logUI = document.getElementById("logUI")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
 
83
- function logMessage(msg) {
84
- const span = document.createElement('span');
85
- span.textContent = msg;
86
- logUI.appendChild(span);
87
- logUI.appendChild(document.createElement('br')); // Add a line break
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
  }
89
 
90
- const originalFetch = window.fetch;
91
-
92
- // Override the fetch function
93
- window.fetch = async function (input, init) {
94
- // Convert input to URL if it's a Request object
95
- const url = typeof input === 'string' ? input : input.url;
96
- var newUrl = url
97
- if (url == 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm') {
98
- // newUrl = 'http://127.0.0.1:8125/assets/static/vision_wasm_internal.wasm' //For Android
99
- newUrl = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm' // For Web
100
-
101
- }
102
- console.log("This was FETCHED: ", newUrl)
103
- // Call the original fetch function with the new URL
104
- return originalFetch(newUrl, init);
105
- };
106
-
107
-
108
- var synthesis = window.speechSynthesis;
109
-
110
- if ('speechSynthesis' in window) {
111
-
112
- var synthesis = window.speechSynthesis;
113
-
114
- // Get the first `en` language voice in the list
115
- var voice = synthesis.getVoices().filter(function (voice) {
116
- return voice.lang === 'en';
117
- })[0];
118
-
119
- // Create an utterance object
120
-
121
- } else {
122
- speechSupported = false;
123
- console.log('Text-to-speech not supported.');
124
  }
125
 
126
- function speak(text) {
127
- console.log("speech api support", speechSupported)
128
- console.log("condition: ", !speechSupported)
129
- console.log("condition2: ", speechSupported == false)
130
- if (!speechSupported) {
131
- console.log("speech api support", speechSupported)
132
- const audioPlayer = document.getElementById('audioPlayer');
133
- if (prevSpeech != text) {
134
- prevSpeech = text
135
- audioPlayer.src = 'http://127.0.0.1:8125/speech?t=' + text; // Set the audio source
136
- console.log("Set src: ", audioPlayer.src)
137
- }
138
-
139
- audioPlayer.play() // Play the audio
140
- .then(() => {
141
-
142
- console.log('Audio is playing');
143
- })
144
- .catch(error => {
145
- console.error('Error playing audio:', error);
146
- prevSpeech = ''
147
- });
148
- } else
149
- if ('speechSynthesis' in window) {
150
- var utterance = new SpeechSynthesisUtterance(text);
151
- utterance.voice = voice;
152
- utterance.pitch = 0.6;
153
- utterance.rate = 0.8;
154
- utterance.volume = 0.8;
155
- synthesis.speak(utterance);
156
  } else {
157
- console.log("Text to speech is now not supported")
 
 
 
158
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  }
160
- var word_list = []
161
-
162
-
163
- function set_output_array(text) {
164
- console.log(text)
165
- word_list = text.split("");
166
- console.log(word_list)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
  }
168
 
169
- </script>
170
-
171
- <script type="module">
172
 
173
-
174
- //import { HandLandmarker, FilesetResolver } from "http://127.0.0.1:8125/assets/static/tasks-vision@0.10.0" // For Android
175
- import { HandLandmarker, FilesetResolver } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0"; // For Web
176
- let handLandmarker = undefined;
177
- let runningMode = "IMAGE";
178
- let enableWebcamButton;
179
- let webcamRunning = false;
180
- var time_since_letter = 0
181
- var last_letter_time = 0
182
- var is_first_run = 1
183
- // Before we can use HandLandmarker class we must wait for it to finish
184
- // loading. Machine Learning models can be large and take a moment to
185
- // get everything needed to run.
186
- const createHandLandmarker = async () => {
187
- const vision = await FilesetResolver.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm"); // This doesnt really matter as this is already imported somewhere else, and the code runs fine without the request
188
- handLandmarker = await HandLandmarker.createFromOptions(vision, {
189
- baseOptions: {
190
- // modelAssetPath: `http://127.0.0.1:8125/assets/static/hand_landmarker.task`, // For Android
191
- modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`, // For Web
192
- delegate: "GPU"
193
- },
194
- runningMode: runningMode,
195
- numHands: 1
196
  });
197
- };
198
- createHandLandmarker();
199
-
200
- // const MODEL_PATH = "http://127.0.0.1:8125/assets/static/model.tflite" // For Android
201
- const MODEL_PATH = "/exported" // For Web
202
- var objectDetector = tflite.loadTFLiteModel(MODEL_PATH);
203
-
204
- /********************************************************************
205
- // Continuously grab images
206
- ********************************************************************/
207
- var global_res = 0;
208
- const video = document.getElementById("webcam");
209
- const canvasElement = document.getElementById("output_canvas");
210
- const canvasCtx = canvasElement.getContext("2d");
211
- var x_array = []
212
- var y_array = []
213
- var video_facing_mode = "user"
214
- // Check if webcam access is supported.
215
- const hasGetUserMedia = () => { var _a; return !!((_a = navigator.mediaDevices) === null || _a === void 0 ? void 0 : _a.getUserMedia); };
216
- // If webcam supported, add event listener to button for when user
217
- // wants to activate it.
218
- if (hasGetUserMedia()) {
219
- enableWebcamButton = document.getElementById("webcamButton");
220
- enableWebcamButton.addEventListener("click", enableCam);
221
- document.getElementById("switch-camera").addEventListener("click", switch_camera);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
222
  }
223
- else {
224
- console.warn("getUserMedia() is not supported by your browser");
225
- }
226
- async function switch_camera() {
227
- if (video_facing_mode == 'user') {
228
- webcamRunning = false
229
- video_facing_mode = 'environment'
230
- await load_camera()
231
- webcamRunning = true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
232
  }
233
  else {
234
- webcamRunning = false
235
- video_facing_mode = 'user'
236
- await load_camera()
237
- webcamRunning = true
238
  }
239
  }
240
- // Enable the live webcam view and start detection.
241
- function enableCam(event) {
242
- if (!handLandmarker) {
243
- console.log("Wait! objectDetector not loaded yet.");
244
- return;
245
- }
246
- if (webcamRunning === true) {
247
- webcamRunning = false;
248
- enableWebcamButton.innerText = "ENABLE PREDICTIONS";
 
 
 
 
 
 
 
 
249
  }
250
  else {
251
- webcamRunning = true;
252
- enableWebcamButton.style = "display:none"
253
- document.getElementById("switch-camera").style.display = "block"
254
-
255
  }
256
- // getUsermedia parameters.
257
- load_camera()
 
 
258
  }
259
- function load_camera() {
260
- const constraints = {
261
- video: {
262
- facingMode: video_facing_mode
263
- }
264
- };
265
- // Activate the webcam stream.
266
- navigator.mediaDevices.getUserMedia(constraints)
267
- .then((stream) => {
268
- video.srcObject = stream;
269
- video.play();
270
- video.addEventListener("loadeddata", predictWebcam);
271
- })
272
- .catch((error) => {
273
- console.error("Error accessing the camera: ", error.name, error.message, error.code);
274
- });
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
275
  }
276
- let lastVideoTime = -1;
277
- let results = undefined;
278
- console.log(video);
279
- async function predictWebcam() {
280
- if (video.videoHeight == 0) {
281
- return
282
- }
283
- canvasElement.width = window.innerWidth;
284
- // Start detecting the stream.
285
- if (runningMode === "IMAGE") {
286
- runningMode = "VIDEO";
287
- await handLandmarker.setOptions({ runningMode: "VIDEO" });
288
- }
289
- let startTimeMs = performance.now();
290
- if (lastVideoTime !== video.currentTime) {
291
- lastVideoTime = video.currentTime;
292
- results = handLandmarker.detectForVideo(video, startTimeMs);
293
- }
294
- canvasCtx.save();
295
- canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
296
- canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width)
297
- if (is_first_run == 1) {
298
- var elem_rect = document.getElementById("output_canvas").getBoundingClientRect()
299
- console.log(elem_rect.height | 0);
300
- document.getElementById("canvas_wrapper").style.height = (elem_rect.height | 0).toString() + "px"
301
 
302
- is_first_run = 0
303
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
304
 
305
- if (results.landmarks && results.handednesses[0]) {
306
- var current_time = Math.round(Date.now())
307
- document.getElementById("predicted_result").style.width = String((current_time - last_letter_time) / 10) + "%"
308
- if (results.handednesses[0][0].categoryName == "Left") {
309
- annotateImage()
310
- console.log("LEFT")
311
- //detectSign()
312
- } else {
313
- console.log("RIGHT")
314
- var current_result = "_"
315
- var previous_result = document.getElementById("predicted_result").innerText
316
- document.getElementById("predicted_result").innerText = current_result
317
-
318
-
319
- if (previous_result == current_result) {
320
- if (current_time - last_letter_time > 1000) {
321
- last_letter_time = current_time
322
- word_list.push(" ")
323
- console.log(word_list)
324
- document.getElementById("text").value = word_list.join('')
325
- }
326
- }
327
- else {
328
- last_letter_time = current_time
329
- }
330
- }
331
- }
332
- else {
333
- if (30 > calculateCanvasBrightness(canvasElement)) {
334
-
335
- var current_result = "<"
336
- var previous_result = document.getElementById("predicted_result").innerText
337
- document.getElementById("predicted_result").innerText = current_result
338
- var current_time = Math.round(Date.now())
339
- console.log(current_time - last_letter_time)
340
- if (previous_result == current_result) {
341
- if (current_time - last_letter_time > 400) {
342
- last_letter_time = current_time
343
- word_list.pop()
344
- console.log(word_list)
345
- document.getElementById("text").value = word_list.join('')
346
- }
347
- }
348
- else {
349
- last_letter_time = current_time
350
- }
351
- } else {
352
- last_letter_time = Math.round(Date.now())
353
-
354
- document.getElementById("predicted_result").style.width = String(0) + "%"
355
- }
356
- }
357
 
358
- canvasCtx.restore();
359
- // Kepp predicting
360
- if (webcamRunning === true) {
361
- window.requestAnimationFrame(predictWebcam);
362
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
363
  }
364
- function annotateImage() {
365
-
366
- //console.log(results.landmarks)
367
- if (results.landmarks[0]) {
368
- x_array = []
369
- y_array = []
370
- results.landmarks[0].forEach(iterate)
371
- //console.log(x_array)
372
- var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
373
- var image_width = canvasElement.width
374
- var min_x = Math.min(...x_array) * image_width
375
- var min_y = Math.min(...y_array) * image_height
376
- var max_x = Math.max(...x_array) * image_width
377
- var max_y = Math.max(...y_array) * image_height
378
-
379
- var sect_height = max_y - (min_y)
380
- var sect_width = max_x - (min_x)
381
- var center_x = (min_x + max_x) / 2
382
- var center_y = (min_y + max_y) / 2
383
-
384
- var sect_diameter = 50
385
- if (sect_height > sect_width) {
386
- sect_diameter = sect_height
387
- //console.log("sect_height", sect_diameter)
388
- }
389
- if (sect_height < sect_width) {
390
- sect_diameter = sect_width
391
- // console.log("sect_width", sect_diameter)
392
- }
393
 
394
- sect_diameter = sect_diameter + 50
395
- var sect_radius = sect_diameter / 2
396
- var crop_top = center_y - sect_radius
397
- var crop_bottom = center_y + sect_radius
398
- var crop_left = center_x - sect_radius
399
- var crop_right = center_x + sect_radius
400
- if (crop_top < 0) {
401
- crop_top = 0
402
- }
403
- if (crop_left < 0) {
404
- crop_left = 0
405
- }
406
- if (crop_right > image_width) {
407
- crop_right = image_width
408
- }
409
- if (crop_bottom > image_height) {
410
- crop_bottom = image_height
411
- }
412
 
413
- canvasCtx.beginPath();
414
- canvasCtx.rect(crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top);
415
- canvasCtx.stroke();
416
 
417
 
418
- }
419
- /* for (const landmarks of results.multiHandLandmarks) {
420
- drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS, {
421
- color: "#00FF00",
422
- lineWidth: 5
423
- });
424
- drawLandmarks(canvasCtx, landmarks, { color: "#FF0000", lineWidth: 2 });
425
- }*/
426
- // console.log(results)
427
- const landmarks = results.landmarks;
428
- if (landmarks[0]) {
429
- var hand = landmarks[0]
430
-
431
- // Thumb connections
432
- drawConnection(hand[4], hand[3], '#ffe5b4', 5); // 4-3
433
- drawConnection(hand[3], hand[2], '#ffe5b4', 5); // 3-2
434
- drawConnection(hand[2], hand[1], '#ffe5b4', 5); // 2-1
435
-
436
- // Index connections
437
- drawConnection(hand[8], hand[7], '#804080', 5); // 8-7
438
- drawConnection(hand[7], hand[6], '#804080', 5); // 7-6
439
- drawConnection(hand[6], hand[5], '#804080', 5); // 6-5
440
-
441
- // Middle connections
442
- drawConnection(hand[12], hand[11], '#ffcc00', 5); // 12-11
443
- drawConnection(hand[11], hand[10], '#ffcc00', 5); // 11-10
444
- drawConnection(hand[10], hand[9], '#ffcc00', 5); // 10-9
445
-
446
- // Ring connections
447
- drawConnection(hand[16], hand[15], '#30ff30', 5); // 16-15
448
- drawConnection(hand[15], hand[14], '#30ff30', 5); // 15-14
449
- drawConnection(hand[14], hand[13], '#30ff30', 5); // 14-13
450
-
451
- // Pinky connections
452
- drawConnection(hand[20], hand[19], '#1565c0', 5); // 20-19
453
- drawConnection(hand[19], hand[18], '#1565c0', 5); // 19-18
454
- drawConnection(hand[18], hand[17], '#1565c0', 5); // 18-17
455
-
456
- drawConnection(hand[0], hand[1], '#808080', 5); // 0-1
457
- drawConnection(hand[0], hand[5], '#808080', 5); // 0-5
458
- drawConnection(hand[0], hand[17], '#808080', 5); // 0-17
459
- drawConnection(hand[5], hand[9], '#808080', 5); // 5-9
460
- drawConnection(hand[9], hand[13], '#808080', 5); // 9-13
461
- drawConnection(hand[13], hand[17], '#808080', 5); // 13-17
462
-
463
- // Thumb
464
- drawLandmarks(canvasCtx, hand[2], '#ffe5b4'); // Thumb tip (2)
465
- drawLandmarks(canvasCtx, hand[3], '#ffe5b4'); // Thumb base (3)
466
- drawLandmarks(canvasCtx, hand[4], '#ffe5b4'); // Thumb base (4)
467
-
468
- // Index
469
- drawLandmarks(canvasCtx, hand[6], '#804080'); // Index tip (6)
470
- drawLandmarks(canvasCtx, hand[7], '#804080'); // Index base (7)
471
- drawLandmarks(canvasCtx, hand[8], '#804080'); // Index base (8)
472
-
473
- // Middle
474
- drawLandmarks(canvasCtx, hand[10], '#ffcc00'); // Middle tip (10)
475
- drawLandmarks(canvasCtx, hand[11], '#ffcc00'); // Middle base (11)
476
- drawLandmarks(canvasCtx, hand[12], '#ffcc00'); // Middle base (12)
477
-
478
- // Ring
479
- drawLandmarks(canvasCtx, hand[14], '#30ff30'); // Ring tip (14)
480
- drawLandmarks(canvasCtx, hand[15], '#30ff30'); // Ring base (15)
481
- drawLandmarks(canvasCtx, hand[16], '#30ff30'); // Ring base (16)
482
-
483
- // Pinky
484
- drawLandmarks(canvasCtx, hand[18], '#1565c0'); // Pinky tip (18)
485
- drawLandmarks(canvasCtx, hand[19], '#1565c0'); // Pinky base (19)
486
- drawLandmarks(canvasCtx, hand[20], '#1565c0'); // Pinky base (20)
487
-
488
- drawLandmarks(canvasCtx, hand[0], '#ff3030'); // Wrist (0)
489
-
490
- drawLandmarks(canvasCtx, hand[1], '#ff3030'); // Palm base (1)
491
-
492
- drawLandmarks(canvasCtx, hand[5], '#ff3030'); // Index palm (5)
493
-
494
- drawLandmarks(canvasCtx, hand[9], '#ff3030'); // Middle palm (9)
495
-
496
- drawLandmarks(canvasCtx, hand[13], '#ff3030'); // Ring palm (13)
497
-
498
- drawLandmarks(canvasCtx, hand[17], '#ff3030'); // Pinky palm (17)
499
- cropCanvas(canvasElement, crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top)
500
- }
501
- // Add more drawing calls for each landmark collection as needed
502
 
503
 
 
 
 
 
504
 
 
 
 
 
505
 
506
- //# sourceURL=pen.js
507
- }
 
 
 
 
508
 
 
 
 
509
 
510
- function iterate(x, y) {
511
- x_array.push(x.x)
512
- y_array.push(x.y)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
513
  }
 
 
 
 
 
 
 
514
 
515
- const cropCanvas = (sourceCanvas, left, top, width, height) => {
516
- let destCanvas = document.createElement('canvas');
517
- destCanvas.width = 224;
518
- var cropAspectRatio = width / height;
519
 
520
- destCanvas.height = 224 / cropAspectRatio
521
- destCanvas.getContext("2d").drawImage(
522
- sourceCanvas,
523
- left, top, width, height, // source rect with content to crop
524
- 0, 0, 224, destCanvas.height); // newCanvas, same size as source
525
- var predictionInput = tf.browser.fromPixels(destCanvas.getContext("2d").getImageData(0, 0, 224, 224))
526
 
527
- predict(tf.expandDims(predictionInput, 0));
528
- }
529
- async function predict(inputTensor) {
530
-
531
- //console.log("in predict")
532
- var letter_list = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "#"]
533
- objectDetector.then(function (res) {
534
- var prediction = res.predict(inputTensor);
535
- var outputArray = prediction.dataSync(); // Get the output as an array
536
- var predictedClass = outputArray.indexOf(Math.max(...outputArray)); // Get the index
537
- var current_result = letter_list[predictedClass]
538
- var previous_result = document.getElementById("predicted_result").innerText
539
- document.getElementById("predicted_result").innerText = current_result
540
- var current_time = Math.round(Date.now())
541
-
542
- if (previous_result == current_result) {
543
- if (current_time - last_letter_time > 1000) {
544
- last_letter_time = current_time
545
- word_list.push(current_result)
546
- console.log(word_list)
547
- document.getElementById("text").value = word_list.join('')
548
- }
549
- }
550
- else {
551
- last_letter_time = current_time
552
- }
553
- console.log(letter_list[predictedClass]);
554
- }, function (err) {
555
- console.log(err);
556
- });
557
 
558
- }
559
 
560
- function drawLandmarks(canvasCtx, landmarks, color) {
561
- var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
562
- var image_width = canvasElement.width
563
 
564
- canvasCtx.fillStyle = color;
565
- canvasCtx.strokeStyle = 'white';
566
- canvasCtx.lineWidth = 1;
567
- canvasCtx.beginPath();
568
- canvasCtx.arc(landmarks.x * image_width, landmarks.y * image_height, 6, 0, 2 * Math.PI);
569
- canvasCtx.fill();
570
- canvasCtx.stroke();
571
 
572
- }
 
 
 
 
 
 
 
 
573
 
574
- function drawConnection(startNode, endNode, strokeColor, strokeWidth) {
 
 
575
 
576
- var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
577
- var image_width = canvasElement.width
578
 
579
- canvasCtx.strokeStyle = strokeColor;
580
- canvasCtx.lineWidth = strokeWidth;
581
- canvasCtx.beginPath();
582
- canvasCtx.moveTo(startNode.x * image_width, startNode.y * image_height);
583
- canvasCtx.lineTo(endNode.x * image_width, endNode.y * image_height);
584
- canvasCtx.stroke();
585
- }
586
- function calculateCanvasBrightness(canvas) {
587
- const context = canvas.getContext('2d');
588
-
589
- // Get the image data from the canvas
590
- const imageData = context.getImageData(0, 0, canvas.width, canvas.height);
591
- const data = imageData.data;
592
-
593
- let totalBrightness = 0;
594
- let pixelCount = 0;
595
-
596
- // Loop through each pixel
597
- for (let i = 0; i < data.length; i += 4) {
598
- const r = data[i]; // Red
599
- const g = data[i + 1]; // Green
600
- const b = data[i + 2]; // Blue
601
-
602
- // Calculate brightness for this pixel
603
- const brightness = 0.299 * r + 0.587 * g + 0.114 * b;
604
- totalBrightness += brightness;
605
- pixelCount++;
606
- }
 
607
 
608
- // Calculate average brightness
609
- const averageBrightness = totalBrightness / pixelCount;
610
 
611
- return averageBrightness;
612
- }
613
- </script>
614
 
615
- <script src="https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.js"
616
- crossorigin="anonymous"></script>
617
 
618
 
619
 
620
-
621
  </body>
622
 
623
  </html>
 
1
  <!DOCTYPE html>
2
+ <!--
3
+ SignSpeak - A Communicator For Signers
4
+
5
+ Copyright (C) 2025 Shantanu Khedkar
6
+
7
+ This program is free software: you can redistribute it and/or modify
8
+ it under the terms of the GNU General Public License as published by
9
+ the Free Software Foundation, either version 3 of the License, or
10
+ (at your option) any later version.
11
+
12
+ This program is distributed in the hope that it will be useful,
13
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+ GNU General Public License for more details.
16
+
17
+ You should have received a copy of the GNU General Public License
18
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
19
+ -->
20
+
21
  <html lang="en">
22
 
23
  <head></head>
 
33
  <!-- For Android
34
  <link rel="stylesheet" type="text/css" href="http://127.0.0.1:8125/assets/static/browser_detect.css" />
35
  -->
36
+ <!-- For Web-->
37
  <link rel="stylesheet" type="text/css" href="static/browser_detect.css" />
38
 
39
 
40
  </head>
41
 
42
  <body translate="no">
43
+ <!-- For Android
44
  <script src="../assets/ipc/androidjs.js"></script>
45
  <script src="http://127.0.0.1:8125/assets/static/drawing_utils.js" crossorigin="anonymous"></script>
46
  <script src="http://127.0.0.1:8125/assets/static/hands.js" crossorigin="anonymous"></script>
 
48
  <script src="http://127.0.0.1:8125/assets/static/tfjs-backend-cpu"></script>
49
  <script src="http://127.0.0.1:8125/assets/static/tf-tflite.min.js"></script>
50
  <script src="http://127.0.0.1:8125/assets/static/vision_wasm_internal.js" crossorigin="anonymous"></script>
51
+ -->
 
52
  <!-- For Web -->
53
  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js"
54
  crossorigin="anonymous"></script>
 
56
  <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"></script>
57
  <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-cpu"></script>
58
  <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-tflite/dist/tf-tflite.min.js"></script>
59
+ <script src="https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.js"
60
+ crossorigin="anonymous"></script>
61
+
62
+ <section id="home-page">
63
+ <div class="container">
64
+
65
+ <video id="webcam" style="display:none" autoplay playsinline muted></video>
66
+ <div class="canvas_wrapper" id="canvas_wrapper">
67
+ <button id="switch-camera"
68
+ style="display:none; position: absolute; top:10px; left:10px; padding:5px; height:40px; width:40px; text-align: center; border-radius: 12.25px; font-size: 20px; font-weight: 900; border:none; background-color: #ffffff2e; backdrop-filter: blur(30px); color:black; box-shadow: 0px 4px 20px 4px rgba(0, 0, 0, 0.38); z-index:100">
69
+ <span>⟳</span>
 
 
 
 
70
  </button>
71
+ <canvas class="output_canvas" id="output_canvas" width="100%" height="500%"></canvas>
72
+ <center>
73
+ <button id="webcamButton" style="font-weight: 600;">
74
+ <span>Enable Webcam</span>
75
+ </button>
76
+ <div class="mode_switch_wrapper">
77
+ <div class="mode_switch" id="mode-switch" style="display: none;">
78
+ <div class="mode_selector" id="modeSelector"></div>
79
+ <div class="mode_switch_modeBtn">Text</div>
80
+ <div class="mode_switch_modeBtn">Word</div>
81
+ </div>
82
+ </div>
83
+ </center>
84
+
85
+ </div>
86
  </div>
87
+ <center>
88
+ <img id="output_image" style="display:none"></img>
89
+
90
+ <div class="wrapper_result">
91
+ <div id="predicted_result">></div>
92
+ </div>
93
+ <div class="wrapper_text">
94
+ <textarea id="text" onkeyup="set_output_array(this.value)"></textarea>
95
+ <div class="textControls">
96
+ <button id="undoButton" onclick="undo()">
97
+ <svg xmlns="http://www.w3.org/2000/svg" height="16px" width="16px"
98
+ viewBox="0 0 576 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.-->
99
+ <path fill="#fcfcfc"
100
+ d="M576 128c0-35.3-28.7-64-64-64L205.3 64c-17 0-33.3 6.7-45.3 18.7L9.4 233.4c-6 6-9.4 14.1-9.4 22.6s3.4 16.6 9.4 22.6L160 429.3c12 12 28.3 18.7 45.3 18.7L512 448c35.3 0 64-28.7 64-64l0-256zM271 175c9.4-9.4 24.6-9.4 33.9 0l47 47 47-47c9.4-9.4 24.6-9.4 33.9 0s9.4 24.6 0 33.9l-47 47 47 47c9.4 9.4 9.4 24.6 0 33.9s-24.6 9.4-33.9 0l-47-47-47 47c-9.4 9.4-24.6 9.4-33.9 0s-9.4-24.6 0-33.9l47-47-47-47c-9.4-9.4-9.4-24.6 0-33.9z" />
101
+ </svg>
102
+ </button>
103
+ <button id="speakButton" onclick="speak(document.getElementById('text').value)">
104
+ <span>Listen</span>
105
+ <svg xmlns="http://www.w3.org/2000/svg" height="16px" width="16px"
106
+ viewBox="0 0 640 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.-->
107
+ <path fill="#fcfcfc"
108
+ d="M533.6 32.5C598.5 85.2 640 165.8 640 256s-41.5 170.7-106.4 223.5c-10.3 8.4-25.4 6.8-33.8-3.5s-6.8-25.4 3.5-33.8C557.5 398.2 592 331.2 592 256s-34.5-142.2-88.7-186.3c-10.3-8.4-11.8-23.5-3.5-33.8s23.5-11.8 33.8-3.5zM473.1 107c43.2 35.2 70.9 88.9 70.9 149s-27.7 113.8-70.9 149c-10.3 8.4-25.4 6.8-33.8-3.5s-6.8-25.4 3.5-33.8C475.3 341.3 496 301.1 496 256s-20.7-85.3-53.2-111.8c-10.3-8.4-11.8-23.5-3.5-33.8s23.5-11.8 33.8-3.5zm-60.5 74.5C434.1 199.1 448 225.9 448 256s-13.9 56.9-35.4 74.5c-10.3 8.4-25.4 6.8-33.8-3.5s-6.8-25.4 3.5-33.8C393.1 284.4 400 271 400 256s-6.9-28.4-17.7-37.3c-10.3-8.4-11.8-23.5-3.5-33.8s23.5-11.8 33.8-3.5zM301.1 34.8C312.6 40 320 51.4 320 64l0 384c0 12.6-7.4 24-18.9 29.2s-25 3.1-34.4-5.3L131.8 352 64 352c-35.3 0-64-28.7-64-64l0-64c0-35.3 28.7-64 64-64l67.8 0L266.7 40.1c9.4-8.4 22.9-10.4 34.4-5.3z" />
109
+ </svg>
110
+ </button>
111
+ <button id="clearButton" onclick="clear_output_array()">
112
+ <svg xmlns="http://www.w3.org/2000/svg" height="16px" width="16px"
113
+ viewBox="0 0 448 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.-->
114
+ <path fill="#fcfcfc"
115
+ d="M135.2 17.7L128 32 32 32C14.3 32 0 46.3 0 64S14.3 96 32 96l384 0c17.7 0 32-14.3 32-32s-14.3-32-32-32l-96 0-7.2-14.3C307.4 6.8 296.3 0 284.2 0L163.8 0c-12.1 0-23.2 6.8-28.6 17.7zM416 128L32 128 53.2 467c1.6 25.3 22.6 45 47.9 45l245.8 0c25.3 0 46.3-19.7 47.9-45L416 128z" />
116
+ </svg>
117
+ </button>
118
+ </div>
119
+
120
+ <audio id="audioPlayer">-</audio>
121
+ </div>
122
+
123
+ <div id="logUI" style="display:none"></div>
124
+ </center>
125
+ </section>
126
+ <section id="info-page" style="display:none; padding: 10px;">
127
+ <center>
128
 
129
+ <svg width="77.5" height="103" viewBox="0 0 310 412" fill="none" xmlns="http://www.w3.org/2000/svg">
130
+ <g filter="url(#filter0_d_21435_307)">
131
+ <path
132
+ d="M186.81 198.496C176.153 230.73 130.699 230.73 120.042 198.496C118.566 194.034 117.995 189.322 118.361 184.635L129.694 27.7729C130.663 15.3457 140.999 5.75757 153.426 5.75757C165.852 5.75757 176.188 15.3457 177.158 27.7729L188.491 184.635C188.857 189.322 188.285 194.034 186.81 198.496Z"
133
+ fill="#FF6E7D" />
134
+ <path
135
+ d="M129.248 210.461C131.138 242.358 93.345 260.329 69.9261 238.67C65.7586 234.815 62.575 230.013 60.6432 224.667L21.653 116.772C17.3477 104.858 22.7777 91.6137 34.1929 86.1855C45.6081 80.7572 59.2664 84.9246 65.7351 95.8096L124.318 194.388C127.221 199.272 128.911 204.784 129.248 210.461Z"
136
+ fill="#FF6E7D" />
137
+ <path
138
+ d="M92.2772 198.028C93.9837 196.578 95.8085 195.273 97.7326 194.129L171.089 150.508C188.74 140.012 211.322 151.175 213.777 171.611C214.485 177.507 217.012 183.034 221.005 187.418L268.354 239.422C278.948 251.057 281.761 267.839 275.544 282.309L245.615 351.969C239.301 366.663 224.879 376.182 208.928 376.182H97.9241C81.9729 376.182 67.5505 366.663 61.2371 351.969L32.802 285.787C25.8462 269.597 30.2544 250.749 43.6623 239.352L92.2772 198.028Z"
139
+ fill="#FF6E7D" />
140
+ <path
141
+ d="M55.1238 304.632C26.6986 296.743 20.0237 259.356 43.9529 242.062C49.3081 238.192 55.6418 235.913 62.2284 235.486L152.085 229.664C164.75 228.844 176.01 237.693 178.247 250.224C180.485 262.756 172.986 274.975 160.824 278.614L74.5386 304.437C68.2137 306.33 61.4852 306.397 55.1238 304.632Z"
142
+ fill="#D6505E" />
143
+ <path
144
+ d="M239.546 153.001C228.433 125.986 190.518 125.338 178.413 151.957C176.038 157.178 175.042 162.922 175.519 168.637L183.682 266.336C184.68 278.282 194.526 287.521 206.477 287.725C218.427 287.929 228.609 279.033 230.048 267.128L241.82 169.769C242.509 164.074 241.726 158.3 239.546 153.001Z"
145
+ fill="#D6505E" />
146
+ <path
147
+ d="M286.006 191.73C274.39 169.568 243.251 168.174 229.541 189.203C225.491 195.415 223.646 202.808 224.305 210.184L229.849 272.21C230.925 284.253 240.697 293.621 252.745 294.16C264.794 294.699 275.429 286.245 277.666 274.35L289.188 213.088C290.559 205.803 289.437 198.277 286.006 191.73Z"
148
+ fill="#A8404B" />
149
+ </g>
150
+ <defs>
151
+ <filter id="filter0_d_21435_307" x="0.181641" y="0.757568" width="309.576" height="410.424"
152
+ filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
153
+ <feFlood flood-opacity="0" result="BackgroundImageFix" />
154
+ <feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"
155
+ result="hardAlpha" />
156
+ <feOffset dy="15" />
157
+ <feGaussianBlur stdDeviation="10" />
158
+ <feComposite in2="hardAlpha" operator="out" />
159
+ <feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.3 0" />
160
+ <feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow_21435_307" />
161
+ <feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow_21435_307" result="shape" />
162
+ </filter>
163
+ </defs>
164
+ </svg>
165
+ <h1>SpeakSign</h1>
166
+ </center>
167
+ <hr>
168
+ <div class="about-card">
169
+ <span>Version</span>
170
+
171
+ <span>1.2.0</span>
172
  </div>
173
+ <div class="about-card">
174
+ <span>Author</span>
175
+ <span>Shantanu Khedkar</span>
176
+ </div>
177
+ <div class="about-card">
178
+ <!--For Android
179
+ <img src="http://127.0.0.1:8125/assets/static/ASL.png" style="height:auto; width:auto; margin: -5px"></img>
180
+ -->
181
+ <!--For Web-->
182
+ <img src="static/ASL.png" style="height:auto; width:auto; margin: -5px"></img>
183
+ </div>
184
+ <div class="about-card">
185
+ <span>Source</span>
186
+ <span><a href="https://github.com/Shantanu-Khedkar/silangint"
187
+ target="_blank">https://github.com/Shantanu-Khedkar/silangint</a></span>
188
+ </div>
189
+ <div class="about-card">
190
+ <span>License</span>
191
+ <span>GPL-v3.0 - <a href="https://github.com/Shantanu-Khedkar/silangint/blob/master/LICENSE"
192
+ target="_blank">https://github.com/Shantanu-Khedkar/silangint/blob/master/LICENSE</a></span>
193
+ </div>
194
+ <div class="about-card">
195
+ <span>Copyright Notice</span>
196
+ <span>
197
+ SignSpeak - A Communicator For Signers
198
+ <br>
199
+ Copyright (C) 2025 Shantanu Khedkar
200
+ <br>
201
+ This program is free software: you can redistribute it and/or modify
202
+ it under the terms of the GNU General Public License as published by
203
+ the Free Software Foundation, either version 3 of the License, or
204
+ (at your option) any later version.
205
+ <br>
206
+ This program is distributed in the hope that it will be useful,
207
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
208
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
209
+ GNU General Public License for more details.
210
+ <br>
211
+ You should have received a copy of the GNU General Public License
212
+ along with this program. If not, see https://www.gnu.org/licenses.
213
+ </span>
214
  </div>
 
 
 
 
215
 
216
+ </section>
217
+ <section id="settings-page" style="display:none">
218
+ <div
219
+ style="display:inline-flex; align-items:center; gap:10px; padding:12px; margin-bottom: 24px; margin-top: 16px;">
220
+ <div style="padding:10px; display: inherit;" onclick="settingsBack()">
221
+ <svg xmlns="http://www.w3.org/2000/svg" width="30px" height="30px"
222
+ viewBox="0 0 320 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.-->
223
+ <path
224
+ d="M41.4 233.4c-12.5 12.5-12.5 32.8 0 45.3l160 160c12.5 12.5 32.8 12.5 45.3 0s12.5-32.8 0-45.3L109.3 256 246.6 118.6c12.5-12.5 12.5-32.8 0-45.3s-32.8-12.5-45.3 0l-160 160z" />
225
+ </svg>
226
+ </div>
227
+ <span class="settings-header">Settings</span>
228
+ </div>
229
+ <form id="speechForm" style="display:block;">
230
+ <div class="about-card">
231
+ <label for="language">Select Language:</label>
232
+ <select id="language"></select>
233
+ </div>
234
+ <div class="about-card">
235
+ <label for="voice">Select Voice:</label>
236
+ <select id="voice"></select>
237
+ </div>
238
+ <div class="about-card">
239
+ <label for="rate">Rate:</label>
240
+ <input type="range" id="rate" min="0.1" max="2" step="0.1" value="1">
241
+ <span style="display:none" id="rateValue">1</span>
242
+ </div>
243
+ <div class="about-card">
244
+ <label for="pitch">Pitch:</label>
245
+ <input type="range" id="pitch" min="0" max="2" step="0.1" value="1">
246
+ <span style="display:none" id="pitchValue">1</span>
247
+ </div>
248
+ <div class="about-card">
249
+ <label for="testText">Test Voice:</label>
250
+ <input id="testText" value="Hello, this is a test!" placeholder="Enter test sentence..."
251
+ onkeyup="set_output_array(this.value)"></input>
252
+ <button type="button" id="testSpeakButton"
253
+ onclick="document.getElementById('text').value = document.getElementById('testText').value; document.getElementById('speakButton').click()">
254
+ <a>Listen 🔊</a>
255
+ </button>
256
+ </div>
257
+ </form>
258
+ </section>
259
+ <div class="bottom-nav">
260
+ <div id="info-" class="bottom-nav-btn">
261
+ <svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px"
262
+ viewBox="0 0 512 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.-->
263
+ <path fill="#5d5d5d"
264
+ d="M256 512A256 256 0 1 0 256 0a256 256 0 1 0 0 512zM216 336l24 0 0-64-24 0c-13.3 0-24-10.7-24-24s10.7-24 24-24l48 0c13.3 0 24 10.7 24 24l0 88 8 0c13.3 0 24 10.7 24 24s-10.7 24-24 24l-80 0c-13.3 0-24-10.7-24-24s10.7-24 24-24zm40-208a32 32 0 1 1 0 64 32 32 0 1 1 0-64z" />
265
+ </svg>
266
+ </div>
267
+ <div id="home-" class="bottom-nav-btn center-btn">
268
+ <!--For Android
269
+ <img src="http://127.0.0.1:8125/assets/static/logo_sil.svg" height="48" width="48px"></img>
270
+ -->
271
+ <img src="/static/logo_sil.svg" height="48" width="48px"></img>
272
+ </div>
273
+ <div id="settings-" class="bottom-nav-btn">
274
+ <svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px"
275
+ viewBox="0 0 512 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.-->
276
+ <path fill="#5d5d5d"
277
+ d="M495.9 166.6c3.2 8.7 .5 18.4-6.4 24.6l-43.3 39.4c1.1 8.3 1.7 16.8 1.7 25.4s-.6 17.1-1.7 25.4l43.3 39.4c6.9 6.2 9.6 15.9 6.4 24.6c-4.4 11.9-9.7 23.3-15.8 34.3l-4.7 8.1c-6.6 11-14 21.4-22.1 31.2c-5.9 7.2-15.7 9.6-24.5 6.8l-55.7-17.7c-13.4 10.3-28.2 18.9-44 25.4l-12.5 57.1c-2 9.1-9 16.3-18.2 17.8c-13.8 2.3-28 3.5-42.5 3.5s-28.7-1.2-42.5-3.5c-9.2-1.5-16.2-8.7-18.2-17.8l-12.5-57.1c-15.8-6.5-30.6-15.1-44-25.4L83.1 425.9c-8.8 2.8-18.6 .3-24.5-6.8c-8.1-9.8-15.5-20.2-22.1-31.2l-4.7-8.1c-6.1-11-11.4-22.4-15.8-34.3c-3.2-8.7-.5-18.4 6.4-24.6l43.3-39.4C64.6 273.1 64 264.6 64 256s.6-17.1 1.7-25.4L22.4 191.2c-6.9-6.2-9.6-15.9-6.4-24.6c4.4-11.9 9.7-23.3 15.8-34.3l4.7-8.1c6.6-11 14-21.4 22.1-31.2c5.9-7.2 15.7-9.6 24.5-6.8l55.7 17.7c13.4-10.3 28.2-18.9 44-25.4l12.5-57.1c2-9.1 9-16.3 18.2-17.8C227.3 1.2 241.5 0 256 0s28.7 1.2 42.5 3.5c9.2 1.5 16.2 8.7 18.2 17.8l12.5 57.1c15.8 6.5 30.6 15.1 44 25.4l55.7-17.7c8.8-2.8 18.6-.3 24.5 6.8c8.1 9.8 15.5 20.2 22.1 31.2l4.7 8.1c6.1 11 11.4 22.4 15.8 34.3zM256 336a80 80 0 1 0 0-160 80 80 0 1 0 0 160z" />
278
+ </svg>
279
+ </div>
280
 
281
+ </div>
282
+ <script>
283
+
284
+ var speechSupported = true
285
+ var prevSpeech = ""
286
+ var prevSettings = ""
287
+
288
+ logUI = document.getElementById("logUI")
289
+ const speechSynthesis = window.speechSynthesis;
290
+ const voiceSelect = document.getElementById('voice');
291
+ const textInput = document.getElementById('text');
292
+ const languageSelect = document.getElementById('language');
293
+ const speakButton = document.getElementById('speakButton');
294
+ const rateInput = document.getElementById('rate');
295
+ const pitchInput = document.getElementById('pitch');
296
+ const rateValue = document.getElementById('rateValue');
297
+ const pitchValue = document.getElementById('pitchValue');
298
+
299
+ let voices = [];
300
+
301
+ const languageNames = {
302
+ 'en': '- English ',
303
+ 'mra': '- Marathi',
304
+ 'mr': '- Marathi',
305
+ 'hi': '- Hindi',
306
+ 'es': 'Spanish',
307
+ 'fr': 'French',
308
+ 'de': 'German',
309
+ 'it': 'Italian',
310
+ 'af': 'Afrikaans',
311
+ 'am': 'Amharic',
312
+ 'an': 'Aragonese',
313
+ 'ar': 'Arabic',
314
+ 'as': 'Assamese',
315
+ 'az': 'Azerbaijani',
316
+ 'bg': 'Bulgarian',
317
+ 'bn': 'Bengali',
318
+ 'bpy': 'Bishnupriya Manipuri',
319
+ 'bs': 'Bosnian',
320
+ 'ca': 'Catalan',
321
+ 'cs': 'Czech',
322
+ 'cy': 'Welsh',
323
+ 'da': 'Danish',
324
+ 'de': 'German',
325
+ 'el': 'Greek',
326
+ 'eo': 'Esperanto',
327
+ 'es': 'Spanish',
328
+ 'et': 'Estonian',
329
+ 'eu': 'Basque',
330
+ 'fa': 'Persian',
331
+ 'fi': 'Finnish',
332
+ 'fr': 'French',
333
+ 'ga': 'Irish',
334
+ 'gd': 'Scottish Gaelic',
335
+ 'gn': 'Guarani',
336
+ 'grc': 'Ancient Greek',
337
+ 'gu': 'Gujarati',
338
+ 'hak': 'Hakka',
339
+ 'hr': 'Croatian',
340
+ 'ht': 'Haitian Creole',
341
+ 'hu': 'Hungarian',
342
+ 'hy': 'Armenian',
343
+ 'ia': 'Interlingua',
344
+ 'id': 'Indonesian',
345
+ 'is': 'Icelandic',
346
+ 'it': 'Italian',
347
+ 'ja': 'Japanese',
348
+ 'jbo': 'Lojban',
349
+ 'ka': 'Georgian',
350
+ 'kk': 'Kazakh',
351
+ 'kl': 'Kalaallisut',
352
+ 'kn': 'Kannada',
353
+ 'ko': 'Korean',
354
+ 'kok': 'Konkani',
355
+ 'ku': 'Kurdish',
356
+ 'ky': 'Kyrgyz',
357
+ 'la': 'Latin',
358
+ 'lfn': 'Lingua Franca Nova',
359
+ 'lt': 'Lithuanian',
360
+ 'lv': 'Latvian',
361
+ 'mi': 'Māori',
362
+ 'mk': 'Macedonian',
363
+ 'ml': 'Malayalam',
364
+ 'ms': 'Malay',
365
+ 'mt': 'Maltese',
366
+ 'my': 'Burmese',
367
+ 'nci': 'Navajo',
368
+ 'ne': 'Nepali',
369
+ 'nl': 'Dutch',
370
+ 'no': 'Norwegian',
371
+ 'om': 'Oromo',
372
+ 'or': 'Odia',
373
+ 'pa': 'Punjabi',
374
+ 'pap': 'Papiamento',
375
+ 'pl': 'Polish',
376
+ 'pt': 'Portuguese',
377
+ 'quc': 'K’iche’',
378
+ 'ro': 'Romanian',
379
+ 'ru': 'Russian',
380
+ 'sd': 'Sindhi',
381
+ 'shn': 'Shan',
382
+ 'si': 'Sinhala',
383
+ 'sk': 'Slovak',
384
+ 'sl': 'Slovenian',
385
+ 'sq': 'Albanian',
386
+ 'sr': 'Serbian',
387
+ 'sv': 'Swedish',
388
+ 'sw': 'Swahili',
389
+ 'ta': 'Tamil',
390
+ 'te': 'Telugu',
391
+ 'tn': 'Tswana',
392
+ 'tr': 'Turkish',
393
+ 'tt': 'Tatar',
394
+ 'ur': 'Urdu',
395
+ 'vi': 'Vietnamese'
396
+
397
+ };
398
+ function undo() {
399
+ word_list.pop()
400
+ textInput.value = word_list.join('')
401
+ }
402
+ function clear() {
403
+ word_list = []
404
+ textInput.value = ''
405
+ }
406
+ function populateVoiceList() {
407
+ voices = speechSynthesis.getVoices();
408
+ updateLanguageList();
409
+ updateVoiceList();
410
+ }
411
+ async function populateTTWVoices() {
412
+ const url = "http://127.0.0.1:8125/assets/static/voicesList.txt";
413
+ try {
414
+ const response = await fetch(url);
415
+ if (!response.ok) {
416
+ throw new Error(`Response status: ${response.status}`);
417
  }
418
 
419
+ var voicesList = await response.text();
420
+ voicesList = voicesList.split('\n')
421
+ voicesList.pop()
422
+ voicesList.forEach(voice => {
423
+ const option = document.createElement('option');
424
+ option.value = voice;
425
+ option.textContent = voice;
426
+ voiceSelect.appendChild(option);
427
+ })
428
+ } catch (error) {
429
+ console.error(error.message);
430
+ }
431
+ }
432
+ async function populateTTWLangs() {
433
+ const url = "http://127.0.0.1:8125/assets/static/langList.txt";
434
+ try {
435
+ const response = await fetch(url);
436
+ if (!response.ok) {
437
+ throw new Error(`Response status: ${response.status}`);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
438
  }
439
 
440
+ var langsList = await response.text();
441
+ langsList = langsList.split('\n')
442
+ langsList.pop()
443
+ unknownLangs = []
444
+ langsList.forEach(lang => {
445
+ if (!languageNames[lang]) {
446
+ unknownLangs.push(lang)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
447
  } else {
448
+ const option = document.createElement('option');
449
+ option.value = lang;
450
+ option.textContent = languageNames[lang];
451
+ languageSelect.appendChild(option);
452
  }
453
+ })
454
+ unknownLangs.forEach(lang => {
455
+ const option = document.createElement('option');
456
+ option.value = lang;
457
+ option.textContent = lang;
458
+ languageSelect.appendChild(option);
459
+ })
460
+ } catch (error) {
461
+ console.error(error.message);
462
+ }
463
+
464
+ const optionNodes = Array.from(languageSelect.children);
465
+ const comparator = new Intl.Collator('en'.slice(0, 2)).compare;
466
+ optionNodes.sort((a, b) => comparator(a.textContent, b.textContent));
467
+ optionNodes.forEach((option) => languageSelect.appendChild(option));
468
+ languageSelect.children[0].selected = "selected"
469
+ }
470
+
471
+ function getBaseLanguageCode(lang) {
472
+ return lang.split('-').slice(0, 2).join('-');
473
+ }
474
+ function getBaseBaseLanguageCode(lang) {
475
+ return lang.split('-').slice(0, 1).join('-');
476
+ }
477
+ function getModifierLanguageCode(lang) {
478
+ return lang.split('-').slice(1, 3).join('-');
479
+ }
480
+
481
+ function updateLanguageList() {
482
+ const uniqueLanguages = new Set(voices.map(voice => getBaseLanguageCode(voice.lang)));
483
+ languageSelect.innerHTML = '';
484
+
485
+ Object.keys(languageNames).forEach(lang => {
486
+ if (uniqueLanguages.has(lang)) {
487
+ const option = document.createElement('option');
488
+ option.value = lang;
489
+ option.textContent = languageNames[lang];
490
+ languageSelect.appendChild(option);
491
  }
492
+ });
493
+
494
+ uniqueLanguages.forEach(lang => {
495
+ if (!languageNames[lang]) {
496
+ if (!languageNames[getBaseBaseLanguageCode(lang)]) {
497
+ const option = document.createElement('option');
498
+ option.value = lang;
499
+ option.textContent = lang;
500
+ languageSelect.appendChild(option);
501
+ }
502
+ else {
503
+ const option = document.createElement('option');
504
+ option.value = lang;
505
+ option.textContent = `${languageNames[getBaseBaseLanguageCode(lang)]} (${getModifierLanguageCode(lang)})`;
506
+ languageSelect.appendChild(option);
507
+ }
508
+ }
509
+ });
510
+
511
+ const optionNodes = Array.from(languageSelect.children);
512
+ const comparator = new Intl.Collator('en'.slice(0, 2)).compare;
513
+ optionNodes.sort((a, b) => comparator(a.textContent, b.textContent));
514
+ optionNodes.forEach((option) => languageSelect.appendChild(option));
515
+ languageSelect.children[0].selected = "selected"
516
+ }
517
+
518
+ function updateVoiceList() {
519
+ const selectedLanguage = languageSelect.value;
520
+ voiceSelect.innerHTML = '';
521
+
522
+ voices.forEach((voice) => {
523
+ if (getBaseLanguageCode(voice.lang) === selectedLanguage) {
524
+ const option = document.createElement('option');
525
+ option.value = voice.name;
526
+ option.textContent = `${voice.name} (${voice.lang})`;
527
+ voiceSelect.appendChild(option);
528
+ }
529
+ });
530
+ }
531
+ function settingsBack() {
532
+ document.getElementById("home-").click()
533
+ }
534
+ function logMessage(msg) {
535
+ const span = document.createElement('span');
536
+ span.textContent = msg;
537
+ logUI.appendChild(span);
538
+ logUI.appendChild(document.createElement('br')); // Add a line break
539
+ }
540
+
541
+ const originalFetch = window.fetch;
542
+
543
+ // Override the fetch function
544
+ window.fetch = async function (input, init) {
545
+ // Convert input to URL if it's a Request object
546
+ const url = typeof input === 'string' ? input : input.url;
547
+ var newUrl = url
548
+ if (url == 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm') {
549
+ //newUrl = 'http://127.0.0.1:8125/assets/static/vision_wasm_internal.wasm' //For Android
550
+ newUrl = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm' // For Web
551
+
552
+ }
553
+ console.log("This was FETCHED: ", newUrl)
554
+ // Call the original fetch function with the new URL
555
+ return originalFetch(newUrl, init);
556
+ };
557
+
558
+
559
+
560
+
561
+ if ('speechSynthesis' in window) {
562
+ speechSynthesis.onvoiceschanged = () => {
563
+ populateVoiceList();
564
+ };
565
+
566
+ languageSelect.addEventListener('change', updateVoiceList);
567
+
568
+ rateInput.addEventListener('input', () => {
569
+ rateValue.textContent = rateInput.value;
570
+ });
571
+
572
+ pitchInput.addEventListener('input', () => {
573
+ pitchValue.textContent = pitchInput.value;
574
+ });
575
+
576
+ speakButton.addEventListener('click', () => {
577
+ const utterance = new SpeechSynthesisUtterance(textInput.value);
578
+ const selectedVoice = voices.find(voice => voice.name === voiceSelect.value);
579
+ utterance.voice = selectedVoice;
580
+ utterance.rate = rateInput.value;
581
+ utterance.pitch = pitchInput.value;
582
+ speechSynthesis.speak(utterance);
583
+ });
584
+
585
+ populateVoiceList()
586
+ // Create an utterance object ⣿
587
+
588
+ } else {
589
+ speechSupported = false;
590
+ console.log('Text-to-speech not supported.');
591
+
592
+ populateTTWLangs()
593
+ populateTTWVoices()
594
+ }
595
+
596
+ function speak(toSpeak) {
597
+ console.log("speech api support", speechSupported)
598
+ console.log("condition: ", !speechSupported)
599
+ console.log("condition2: ", speechSupported == false)
600
+ console.log("speech api support", speechSupported)
601
+ if (!speechSupported) {
602
+ const audioPlayer = document.getElementById('audioPlayer');
603
+ var currSettings = '&v=' + encodeURIComponent(languageSelect.value + "+" + voiceSelect.value) + '&r=' + Math.round(rateInput.value * 7.5).toString() + '&p=' + Math.round(pitchInput.value * 49.5).toString()
604
+ if ((prevSpeech != toSpeak) || (prevSettings != currSettings)) {
605
+ prevSpeech = toSpeak
606
+ prevSettings = currSettings
607
+ audioPlayer.src = 'http://127.0.0.1:8125/speech?t=' + encodeURIComponent(toSpeak) + currSettings
608
+ console.log("Set src: ", audioPlayer.src)
609
  }
610
 
611
+ audioPlayer.play() // Play the audio
612
+ .then(() => {
 
613
 
614
+ console.log('Audio is playing');
615
+ })
616
+ .catch(error => {
617
+ console.error('Error playing audio:', error);
618
+ prevSpeech = ''
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
619
  });
620
+ } else if ('speechSynthesis' in window) {
621
+ console.log("probably spoken")
622
+ } else {
623
+ console.log("Text to speech is now not supported")
624
+ }
625
+ }
626
+ var word_list = []
627
+
628
+
629
+ function set_output_array(text) {
630
+ console.log(text)
631
+ word_list = text.split("");
632
+ console.log(word_list)
633
+ }
634
+ function clear_output_array() {
635
+ word_list = [];
636
+ textInput.value = ""
637
+ }
638
+
639
+ </script>
640
+
641
+ <script type="module">
642
+ document.getElementById("info-").addEventListener("click", switchPage.bind(null, "info-"));
643
+ document.getElementById("home-").addEventListener("click", switchPage.bind(null, "home-"));
644
+ document.getElementById("settings-").addEventListener("click", switchPage.bind(null, "settings-"));
645
+
646
+ //import { HandLandmarker, FilesetResolver } from "http://127.0.0.1:8125/assets/static/tasks-vision@0.10.0" // For Android
647
+ import { HandLandmarker, FilesetResolver } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0"; // For Web
648
+ let handLandmarker = undefined;
649
+ let runningMode = "IMAGE";
650
+ let enableWebcamButton;
651
+ let webcamRunning = false;
652
+ var time_since_letter = 0
653
+ var last_letter_time = 0
654
+ var is_first_run = 1
655
+ const letter_list = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "#"]
656
+ const phrase_list = [" Good", " Hello", " How're", " I'm", " Thanks", " You"]
657
+ var index_list = letter_list
658
+ // Before we can use HandLandmarker class we must wait for it to finish
659
+ // loading. Machine Learning models can be large and take a moment to
660
+ // get everything needed to run.
661
+ const createHandLandmarker = async () => {
662
+ const vision = await FilesetResolver.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm"); // This doesnt really matter as this is already imported somewhere else, and the code runs fine without the request
663
+ handLandmarker = await HandLandmarker.createFromOptions(vision, {
664
+ baseOptions: {
665
+ //modelAssetPath: `http://127.0.0.1:8125/assets/static/hand_landmarker.task`, // For Android
666
+ modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`, // For Web
667
+ delegate: "GPU"
668
+ },
669
+ runningMode: runningMode,
670
+ numHands: 1
671
+ });
672
+ };
673
+ createHandLandmarker();
674
+
675
+ //const MODEL_PATH = "http://127.0.0.1:8125/assets/static/model.tflite" // For Android
676
+ //const WORD_MODEL = "http://127.0.0.1:8125/assets/static/word.tflite" // For Android
677
+ const MODEL_PATH = "/exported" // For Web
678
+ const WORD_MODEL = "/word" // For Web
679
+
680
+ const letterDetector = tflite.loadTFLiteModel(MODEL_PATH);
681
+ const wordDetector = tflite.loadTFLiteModel(WORD_MODEL);
682
+ var objectDetector = letterDetector
683
+ /********************************************************************
684
+ // Continuously grab images
685
+ ********************************************************************/
686
+ document.getElementById("mode-switch").addEventListener('click', function () {
687
+ document.getElementById('modeSelector').classList.toggle('right')
688
+ var modeState = Array.from(document.getElementById('modeSelector').classList).includes('right')
689
+ console.log("swicth", modeState)
690
+ if (modeState) {
691
+ console.log("word")
692
+ objectDetector = wordDetector
693
+ index_list = phrase_list
694
+ } else {
695
+ console.log("letter")
696
+ objectDetector = letterDetector
697
+ index_list = letter_list
698
+ }
699
+ })
700
+ var global_res = 0;
701
+ const video = document.getElementById("webcam");
702
+ const canvasElement = document.getElementById("output_canvas");
703
+ const canvasCtx = canvasElement.getContext("2d", { willReadFrequently: true });
704
+ var x_array = []
705
+ var y_array = []
706
+ var video_facing_mode = "user"
707
+ // Check if webcam access is supported.
708
+ const hasGetUserMedia = () => { var _a; return !!((_a = navigator.mediaDevices) === null || _a === void 0 ? void 0 : _a.getUserMedia); };
709
+ // If webcam supported, add event listener to button for when user
710
+ // wants to activate it.
711
+ if (hasGetUserMedia()) {
712
+ enableWebcamButton = document.getElementById("webcamButton");
713
+ enableWebcamButton.addEventListener("click", enableCam);
714
+ document.getElementById("switch-camera").addEventListener("click", switch_camera);
715
+ }
716
+ else {
717
+ console.warn("getUserMedia() is not supported by your browser");
718
+ }
719
+ async function switch_camera() {
720
+ if (video_facing_mode == 'user') {
721
+ webcamRunning = false
722
+ video_facing_mode = 'environment'
723
+ await load_camera()
724
+ webcamRunning = true
725
+ }
726
+ else {
727
+ webcamRunning = false
728
+ video_facing_mode = 'user'
729
+ await load_camera()
730
+ webcamRunning = true
731
+ }
732
+ }
733
+ // Enable the live webcam view and start detection.
734
+ function enableCam(event) {
735
+ if (!handLandmarker) {
736
+ console.log("Wait! objectDetector not loaded yet.");
737
+ return;
738
+ }
739
+ if (webcamRunning === true) {
740
+ webcamRunning = false;
741
+ enableWebcamButton.innerText = "ENABLE PREDICTIONS";
742
+ }
743
+ else {
744
+ webcamRunning = true;
745
+ enableWebcamButton.style = "display:none"
746
+ document.getElementById("switch-camera").style.display = "block"
747
+ document.getElementById("mode-switch").style.display = "flex"
748
+
749
+ }
750
+ // getUsermedia parameters.
751
+ load_camera()
752
+ }
753
+
754
+ function switchPage(elem) {
755
+ prevSpeech = ""
756
+
757
+ var pH = document.getElementById("home-page")
758
+ var pI = document.getElementById("info-page")
759
+ var pS = document.getElementById("settings-page")
760
+
761
+ pH.style.display = "none"
762
+ pI.style.display = "none"
763
+ pS.style.display = "none"
764
+
765
+ document.getElementById(elem + "page").style.display = "block"
766
+ if (elem != "home-") {
767
+ webcamRunning = false
768
+ document.getElementById("webcamButton").style = "display:block"
769
+ document.getElementById("switch-camera").style.display = "none"
770
+ document.getElementById("mode-switch").style.display = "none"
771
+ var canvas = document.getElementById("output_canvas")
772
+ const context = canvas.getContext('2d');
773
+ context.clearRect(0, 0, canvas.width, canvas.height);
774
+ } else {
775
+ textInput.value = ""
776
+ clear_output_array()
777
+ var canvas = document.getElementById("output_canvas")
778
+ const context = canvas.getContext('2d');
779
+ context.clearRect(0, 0, canvas.width, canvas.height);
780
+ }
781
+ }
782
+
783
+ function load_camera() {
784
+ try {
785
+ var stream = video.srcObject;
786
+ // now get all tracks
787
+ var tracks = stream.getTracks();
788
+ // now close each track by having forEach loop
789
+ tracks.forEach(function (track) {
790
+ // stopping every track
791
+ track.stop();
792
+ });
793
+ // assign null to srcObject of video
794
+ video.srcObject = null;
795
+ } catch (error) {
796
+ console.error(error.message);
797
+ }
798
+
799
+ const constraints = {
800
+ video: {
801
+ facingMode: video_facing_mode
802
  }
803
+ };
804
+ // Activate the webcam stream.
805
+ navigator.mediaDevices.getUserMedia(constraints)
806
+ .then((stream) => {
807
+ video.srcObject = stream;
808
+ video.play();
809
+ video.addEventListener("loadeddata", predictWebcam);
810
+ })
811
+ .catch((error) => {
812
+ console.error("Error accessing the camera: ", error.name, error.message, error.code);
813
+ });
814
+ }
815
+ let lastVideoTime = -1;
816
+ let results = undefined;
817
+ console.log(video);
818
+ async function predictWebcam() {
819
+ if (video.videoHeight == 0) {
820
+ return
821
+ }
822
+ canvasElement.width = window.innerWidth;
823
+ // Start detecting the stream.
824
+ if (runningMode === "IMAGE") {
825
+ runningMode = "VIDEO";
826
+ await handLandmarker.setOptions({ runningMode: "VIDEO" });
827
+ }
828
+ let startTimeMs = performance.now();
829
+ if (lastVideoTime !== video.currentTime) {
830
+ lastVideoTime = video.currentTime;
831
+ results = handLandmarker.detectForVideo(video, startTimeMs);
832
+ }
833
+ canvasCtx.save();
834
+ canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
835
+
836
+ if (is_first_run == 1) {
837
+ var elem_rect = document.getElementById("output_canvas").getBoundingClientRect()
838
+ console.log(elem_rect.height | 0);
839
+ document.getElementById("canvas_wrapper").style.height = (elem_rect.height | 0).toString() + "px"
840
+
841
+ is_first_run = 0
842
+ }
843
+
844
+ if (results.landmarks && results.handednesses[0]) {
845
+ var current_time = Math.round(Date.now())
846
+ document.getElementById("predicted_result").style.width = String((current_time - last_letter_time) / 7) + "%"
847
+ if (results.handednesses[0][0].categoryName == "Left") {
848
+ if(!Array.from(document.getElementById('modeSelector').classList).includes('right') ){
849
+ annotateImage()
850
+ }
851
+ canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width)
852
+ annotateImage(Array.from(document.getElementById('modeSelector').classList).includes('right') )
853
+ console.log("LEFT")
854
+ //detectSign()
855
+ } else {
856
+ canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width)
857
+ console.log("RIGHT")
858
+ var current_result = "_"
859
+ var previous_result = document.getElementById("predicted_result").innerText
860
+ document.getElementById("predicted_result").innerText = current_result
861
+
862
+
863
+ if (previous_result == current_result) {
864
+ if (current_time - last_letter_time > 700) {
865
+ last_letter_time = current_time
866
+ word_list.push(" ")
867
+ triggerPulse()
868
+ console.log(word_list)
869
+ document.getElementById("text").value = word_list.join('')
870
+ }
871
  }
872
  else {
873
+ last_letter_time = current_time
 
 
 
874
  }
875
  }
876
+ }
877
+ else {
878
+ canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width)
879
+ if (30 > calculateCanvasBrightness(canvasElement)) {
880
+
881
+ var current_result = "<"
882
+ var previous_result = document.getElementById("predicted_result").innerText
883
+ document.getElementById("predicted_result").innerText = current_result
884
+ var current_time = Math.round(Date.now())
885
+ console.log(current_time - last_letter_time)
886
+ if (previous_result == current_result) {
887
+ if (current_time - last_letter_time > 400) {
888
+ last_letter_time = current_time
889
+ word_list.pop()
890
+ console.log(word_list)
891
+ document.getElementById("text").value = word_list.join('')
892
+ }
893
  }
894
  else {
895
+ last_letter_time = current_time
 
 
 
896
  }
897
+ } else {
898
+ last_letter_time = Math.round(Date.now())
899
+
900
+ document.getElementById("predicted_result").style.width = String(0) + "%"
901
  }
902
+ }
903
+
904
+ canvasCtx.restore();
905
+ // Kepp predicting
906
+ if (webcamRunning === true) {
907
+ window.requestAnimationFrame(predictWebcam);
908
+ }
909
+ }
910
+ function annotateImage(firstA=true) {
911
+
912
+ //console.log(results.landmarks)
913
+ if (results.landmarks[0]) {
914
+ x_array = []
915
+ y_array = []
916
+ results.landmarks[0].forEach(iterate)
917
+ //console.log(x_array)
918
+ var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
919
+ var image_width = canvasElement.width
920
+ var min_x = Math.min(...x_array) * image_width
921
+ var min_y = Math.min(...y_array) * image_height
922
+ var max_x = Math.max(...x_array) * image_width
923
+ var max_y = Math.max(...y_array) * image_height
924
+
925
+ var sect_height = max_y - (min_y)
926
+ var sect_width = max_x - (min_x)
927
+ var center_x = (min_x + max_x) / 2
928
+ var center_y = (min_y + max_y) / 2
929
+
930
+ var sect_diameter = 50
931
+ if (sect_height > sect_width) {
932
+ sect_diameter = sect_height
933
+ //console.log("sect_height", sect_diameter)
934
+ }
935
+ if (sect_height < sect_width) {
936
+ sect_diameter = sect_width
937
+ // console.log("sect_width", sect_diameter)
938
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
939
 
940
+ sect_diameter = sect_diameter + 50
941
+ var sect_radius = sect_diameter / 2
942
+ var crop_top = center_y - sect_radius
943
+ var crop_bottom = center_y + sect_radius
944
+ var crop_left = center_x - sect_radius
945
+ var crop_right = center_x + sect_radius
946
+ if (crop_top < 0) {
947
+ crop_top = 0
948
+ }
949
+ if (crop_left < 0) {
950
+ crop_left = 0
951
+ }
952
+ if (crop_right > image_width) {
953
+ crop_right = image_width
954
+ }
955
+ if (crop_bottom > image_height) {
956
+ crop_bottom = image_height
957
+ }
958
+ if(firstA == true){
959
+ canvasCtx.beginPath();
960
+ canvasCtx.rect(crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top);
961
+ canvasCtx.stroke();
962
+ }else{
963
+ canvasCtx.beginPath();
964
+ canvasCtx.rect(crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top);
965
+ canvasCtx.stroke();
966
+ }
967
+
968
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
969
 
970
+ }
971
+ /* for (const landmarks of results.multiHandLandmarks) {
972
+ drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS, {
973
+ color: "#00FF00",
974
+ lineWidth: 5
975
+ });
976
+ drawLandmarks(canvasCtx, landmarks, { color: "#FF0000", lineWidth: 2 });
977
+ }*/
978
+ // console.log(results)
979
+ const landmarks = results.landmarks;
980
+ if (landmarks[0]) {
981
+ var hand = landmarks[0]
982
+ const zValues = hand.map(node => node.z);
983
+ const maxZ = Math.max.apply(null, zValues)
984
+ const minZ = Math.min.apply(null, zValues)
985
+ const zRange = maxZ-minZ
986
+ hand.forEach(node => {
987
+ node.z = ((node.z - minZ) / zRange) * 255
988
+ });
989
+ // Thumb connections
990
+ drawConnection(hand[4], hand[3], 'rgb(0, 0, 255)', 5); // 4-3 (was red)
991
+ drawConnection(hand[3], hand[2], 'rgb(0, 0, 255)', 5); // 3-2 (was red)
992
+ drawConnection(hand[2], hand[1], 'rgb(0, 0, 255)', 5); // 2-1 (was red)
993
+
994
+ // Index connections
995
+ drawConnection(hand[8], hand[7], 'rgb(0, 255, 0)', 5); // 8-7
996
+ drawConnection(hand[7], hand[6], 'rgb(0, 255, 0)', 5); // 7-6
997
+ drawConnection(hand[6], hand[5], 'rgb(0, 255, 0)', 5); // 6-5
998
+
999
+ // Middle connections
1000
+ drawConnection(hand[12], hand[11], 'rgb(255, 0, 0)', 5); // 12-11 (was blue)
1001
+ drawConnection(hand[11], hand[10], 'rgb(255, 0, 0)', 5); // 11-10 (was blue)
1002
+ drawConnection(hand[10], hand[9], 'rgb(255, 0, 0)', 5); // 10-9 (was blue)
1003
+
1004
+ // Ring connections
1005
+ drawConnection(hand[16], hand[15], 'rgb(0, 255, 255)', 5); // 16-15 (was yellow)
1006
+ drawConnection(hand[15], hand[14], 'rgb(0, 255, 255)', 5); // 15-14 (was yellow)
1007
+ drawConnection(hand[14], hand[13], 'rgb(0, 255, 255)', 5); // 14-13 (was yellow)
1008
+
1009
+ // Pinky connections
1010
+ drawConnection(hand[20], hand[19], 'rgb(255, 0, 255)', 5); // 20-19
1011
+ drawConnection(hand[19], hand[18], 'rgb(255, 0, 255)', 5); // 19-18
1012
+ drawConnection(hand[18], hand[17], 'rgb(255, 0, 255)', 5); // 18-17
1013
+
1014
+ drawConnection(hand[0], hand[1], 'rgb(200, 200, 200)', 5); // 0-1
1015
+ drawConnection(hand[0], hand[5], 'rgb(200, 200, 200)', 5); // 0-5
1016
+ drawConnection(hand[0], hand[17], 'rgb(200, 200, 200)', 5); // 0-17
1017
+ drawConnection(hand[5], hand[9], 'rgb(200, 200, 200)', 5); // 5-9
1018
+ drawConnection(hand[9], hand[13], 'rgb(200, 200, 200)', 5); // 9-13
1019
+ drawConnection(hand[13], hand[17], 'rgb(200, 200, 200)', 5); // 13-17
1020
+
1021
+ // Thumb
1022
+ drawLandmarks(canvasCtx, hand[2], '#ffe5b4'); // Thumb tip (2)
1023
+ drawLandmarks(canvasCtx, hand[3], '#ffe5b4'); // Thumb base (3)
1024
+ drawLandmarks(canvasCtx, hand[4], '#ffe5b4'); // Thumb base (4)
1025
+
1026
+ // Index
1027
+ drawLandmarks(canvasCtx, hand[6], '#804080'); // Index tip (6)
1028
+ drawLandmarks(canvasCtx, hand[7], '#804080'); // Index base (7)
1029
+ drawLandmarks(canvasCtx, hand[8], '#804080'); // Index base (8)
1030
+
1031
+ // Middle
1032
+ drawLandmarks(canvasCtx, hand[10], '#ffcc00'); // Middle tip (10)
1033
+ drawLandmarks(canvasCtx, hand[11], '#ffcc00'); // Middle base (11)
1034
+ drawLandmarks(canvasCtx, hand[12], '#ffcc00'); // Middle base (12)
1035
+
1036
+ // Ring
1037
+ drawLandmarks(canvasCtx, hand[14], '#30ff30'); // Ring tip (14)
1038
+ drawLandmarks(canvasCtx, hand[15], '#30ff30'); // Ring base (15)
1039
+ drawLandmarks(canvasCtx, hand[16], '#30ff30'); // Ring base (16)
1040
+
1041
+ // Pinky
1042
+ drawLandmarks(canvasCtx, hand[18], '#1565c0'); // Pinky tip (18)
1043
+ drawLandmarks(canvasCtx, hand[19], '#1565c0'); // Pinky base (19)
1044
+ drawLandmarks(canvasCtx, hand[20], '#1565c0'); // Pinky base (20)
1045
+
1046
+ drawLandmarks(canvasCtx, hand[0], '#ff3030'); // Wrist (0)
1047
+
1048
+ drawLandmarks(canvasCtx, hand[1], '#ff3030'); // Palm base (1)
1049
+
1050
+ drawLandmarks(canvasCtx, hand[5], '#ff3030'); // Index palm (5)
1051
+
1052
+ drawLandmarks(canvasCtx, hand[9], '#ff3030'); // Middle palm (9)
1053
+
1054
+ drawLandmarks(canvasCtx, hand[13], '#ff3030'); // Ring palm (13)
1055
+
1056
+ drawLandmarks(canvasCtx, hand[17], '#ff3030'); // Pinky palm (17)
1057
+
1058
+ // Crop Canvas
1059
+ if(firstA == true){
1060
+ cropCanvas(canvasElement, crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top)
1061
  }
1062
+ }
1063
+ // Add more drawing calls for each landmark collection as needed
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1064
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1065
 
 
 
 
1066
 
1067
 
1068
+ //# sourceURL=pen.js
1069
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1070
 
1071
 
1072
+ function iterate(x, y) {
1073
+ x_array.push(x.x)
1074
+ y_array.push(x.y)
1075
+ }
1076
 
1077
+ const cropCanvas = (sourceCanvas, left, top, width, height) => {
1078
+ let destCanvas = document.createElement('canvas');
1079
+ destCanvas.width = 224;
1080
+ var cropAspectRatio = width / height;
1081
 
1082
+ destCanvas.height = 224 / cropAspectRatio
1083
+ destCanvas.getContext("2d").drawImage(
1084
+ sourceCanvas,
1085
+ left, top, width, height, // source rect with content to crop
1086
+ 0, 0, 224, destCanvas.height); // newCanvas, same size as source
1087
+ var predictionInput = tf.browser.fromPixels(destCanvas.getContext("2d").getImageData(0, 0, 224, 224))
1088
 
1089
+ predict(tf.expandDims(predictionInput, 0));
1090
+ }
1091
+ async function predict(inputTensor) {
1092
 
1093
+ console.log(index_list[0])
1094
+ objectDetector.then(function (res) {
1095
+ var prediction = res.predict(inputTensor);
1096
+ var outputArray = prediction.dataSync(); // Get the output as an array
1097
+ var predictedClass = outputArray.indexOf(Math.max(...outputArray)); // Get the index
1098
+ var current_result = index_list[predictedClass]
1099
+ var previous_result = document.getElementById("predicted_result").innerText
1100
+ document.getElementById("predicted_result").innerText = current_result
1101
+ var current_time = Math.round(Date.now())
1102
+ if (index_list[0] == " Good") {
1103
+ previous_result = " " + previous_result
1104
+ }
1105
+ console.log("p:", previous_result, "c:", current_result)
1106
+ if (previous_result == current_result) {
1107
+ if (current_time - last_letter_time > 700) {
1108
+ last_letter_time = current_time
1109
+ word_list.push(current_result)
1110
+ triggerPulse()
1111
+ console.log(word_list)
1112
+ document.getElementById("text").value = word_list.join('')
1113
+ }
1114
  }
1115
+ else {
1116
+ last_letter_time = current_time
1117
+ }
1118
+ console.log(index_list[predictedClass]);
1119
+ }, function (err) {
1120
+ console.log(err);
1121
+ });
1122
 
1123
+ }
 
 
 
1124
 
1125
+ function drawLandmarks(canvasCtx, landmarks, color) {
1126
+ var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
1127
+ var image_width = canvasElement.width
 
 
 
1128
 
1129
+ canvasCtx.fillStyle = `rgb(${landmarks.z},${landmarks.z},${landmarks.z})`;
1130
+ canvasCtx.beginPath();
1131
+ canvasCtx.arc(landmarks.x * image_width, landmarks.y * image_height, 8, 0, 2 * Math.PI);
1132
+ canvasCtx.fill();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1133
 
1134
+ }
1135
 
1136
+ function drawConnection(startNode, endNode, strokeColor, strokeWidth) {
 
 
1137
 
1138
+ var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
1139
+ var image_width = canvasElement.width
 
 
 
 
 
1140
 
1141
+ canvasCtx.strokeStyle = strokeColor;
1142
+ canvasCtx.lineWidth = strokeWidth-1;
1143
+ canvasCtx.beginPath();
1144
+ canvasCtx.moveTo(startNode.x * image_width, startNode.y * image_height);
1145
+ canvasCtx.lineTo(endNode.x * image_width, endNode.y * image_height);
1146
+ canvasCtx.stroke();
1147
+ }
1148
+ function calculateCanvasBrightness(canvas) {
1149
+ const context = canvas.getContext('2d', { willReadFrequently: true });
1150
 
1151
+ // Get the image data from the canvas
1152
+ const imageData = context.getImageData(0, 0, canvas.width, canvas.height);
1153
+ const data = imageData.data;
1154
 
1155
+ let totalBrightness = 0;
1156
+ let pixelCount = 0;
1157
 
1158
+ // Loop through each pixel
1159
+ for (let i = 0; i < data.length; i += 4) {
1160
+ const r = data[i]; // Red
1161
+ const g = data[i + 1]; // Green
1162
+ const b = data[i + 2]; // Blue
1163
+
1164
+ // Calculate brightness for this pixel
1165
+ const brightness = 0.299 * r + 0.587 * g + 0.114 * b;
1166
+ totalBrightness += brightness;
1167
+ pixelCount++;
1168
+ }
1169
+
1170
+ // Calculate average brightness
1171
+ const averageBrightness = totalBrightness / pixelCount;
1172
+
1173
+ return averageBrightness;
1174
+ }
1175
+ function triggerPulse() {
1176
+ console.log('did nothing')
1177
+ /* Apply after working on pulse css more
1178
+ const resultWrapper = document.querySelector('.wrapper_result');
1179
+ resultWrapper.classList.add('pulse');
1180
+
1181
+ // Remove the class after the animation ends to allow it to be triggered again
1182
+ resultWrapper.addEventListener('animationend', () => {
1183
+ resultWrapper.classList.remove('pulse');
1184
+ }, { once: true });*/
1185
+ }
1186
+ </script>
1187
 
 
 
1188
 
 
 
 
1189
 
 
 
1190
 
1191
 
1192
 
 
1193
  </body>
1194
 
1195
  </html>