Emeritus-21 commited on
Commit
c2f3a36
·
verified ·
1 Parent(s): 15293dd

Update templates/browser-detect.html

Browse files
Files changed (1) hide show
  1. templates/browser-detect.html +141 -35
templates/browser-detect.html CHANGED
@@ -3,14 +3,17 @@
3
  SignSpeak - A Communicator For Signers
4
 
5
  Copyright (C) 2025 Shantanu Khedkar
 
6
  This program is free software: you can redistribute it and/or modify
7
  it under the terms of the GNU General Public License as published by
8
  the Free Software Foundation, either version 3 of the License, or
9
  (at your option) any later version.
 
10
  This program is distributed in the hope that it will be useful,
11
  but WITHOUT ANY WARRANTY; without even the implied warranty of
12
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
  GNU General Public License for more details.
 
14
  You should have received a copy of the GNU General Public License
15
  along with this program. If not, see <https://www.gnu.org/licenses/>.
16
  -->
@@ -272,42 +275,34 @@
272
  </form>
273
  </section>
274
  <div class="bottom-nav">
275
- <div id="info-" class="bottom-nav-btn">
276
- <svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px" viewBox="0 0 512 512">
277
- <path fill="#5d5d5d" d="M256 512A256 256 0 1 0 256 0a256 256 0 1 0 0 512zM216 336l24 0 0-64-24 0c-13.3 0-24-10.7-24-24s10.7-24 24-24l48 0c13.3 0 24 10.7 24 24l0 88 8 0c13.3 0 24 10.7 24 24s-10.7 24-24 24l-80 0c-13.3 0-24-10.7-24-24s10.7-24 24-24zm40-208a32 32 0 1 1 0 64 32 32 0 1 1 0-64z" />
278
- </svg>
279
- </div>
280
-
281
- <div id="home-" class="bottom-nav-btn center-btn">
282
- <img src="/static/logo_sil.svg" height="48" width="48px" alt="Home Icon" />
283
- </div>
284
-
285
- <div id="settings-" class="bottom-nav-btn">
286
- <svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px" viewBox="0 0 512 512">
287
- <path fill="#5d5d5d" d="M495.9 166.6c3.2 8.7 .5 18.4-6.4 24.6l-43.3 39.4c1.1 8.3 1.7 16.8 1.7 25.4s-.6 17.1-1.7 25.4l43.3 39.4c6.9 6.2 9.6 15.9 6.4 24.6c-4.4 11.9-9.7 23.3-15.8 34.3l-4.7 8.1c-6.6 11-14 21.4-22.1 31.2c-5.9 7.2-15.7 9.6-24.5 6.8l-55.7-17.7c-13.4 10.3-28.2 18.9-44 25.4l-12.5 57.1c-2 9.1-9 16.3-18.2 17.8c-13.8 2.3-28 3.5-42.5 3.5s-28.7-1.2-42.5-3.5c-9.2-1.5-16.2-8.7-18.2-17.8l-12.5-57.1c-15.8-6.5-30.6-15.1-44-25.4L83.1 425.9c-8.8 2.8-18.6 .3-24.5-6.8c-8.1-9.8-15.5-20.2-22.1-31.2l-4.7-8.1c-6.1-11-11.4-22.4-15.8-34.3c-3.2-8.7-.5-18.4 6.4-24.6l43.3-39.4C64.6 273.1 64 264.6 64 256s.6-17.1 1.7-25.4L22.4 191.2c-6.9-6.2-9.6-15.9-6.4-24.6c4.4-11.9 9.7-23.3 15.8-34.3l4.7-8.1c6.6-11 14-21.4 22.1-31.2c5.9-7.2 15.7-9.6 24.5-6.8l55.7 17.7c13.4-10.3 28.2-18.9 44-25.4l12.5-57.1c2-9.1 9-16.3 18.2-17.8C227.3 1.2 241.5 0 256 0s28.7 1.2 42.5 3.5c9.2 1.5 16.2 8.7 18.2 17.8l12.5 57.1c15.8 6.5 30.6 15.1 44 25.4l55.7-17.7c8.8-2.8 18.6-.3 24.5 6.8c8.1 9.8 15.5 20.2 22.1 31.2l4.7 8.1c6.1 11 11.4 22.4 15.8 34.3zM256 336a80 80 0 1 0 0-160 80 80 0 1 0 0 160z" />
288
- </svg>
289
- </div>
290
-
291
- <div id="digits-" class="bottom-nav-btn">
292
- <svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px" viewBox="0 0 512 512">
293
- <path fill="#5d5d5d" d="M256 512A256 256 0 1 0 256 0a256 256 0 1 0 0 512zM232 344V280H168c-13.3 0-24-10.7-24-24s10.7-24 24-24h64V168c0-13.3 10.7-24 24-24s24 10.7 24 24v64h64c13.3 0 24 10.7 24 24s-10.7 24-24 24H280v64c0 13.3-10.7 24-24 24s-24-10.7-24-24z" />
294
- </svg>
295
- <button id="goDigitsBtn">Go to Digits</button>
296
- </div>
297
- </div>
298
-
299
- <script>
300
- document.getElementById("goDigitsBtn").addEventListener("click", function() {
301
- window.open("https://huggingface.co/spaces/Emeritus-21/digits-signlang", "_blank");
302
- });
303
- </script>
304
 
305
-
306
  <script>
307
  var word_list = []
308
  var speechSupported = true
309
  var prevSpeech = ""
310
  var prevSettings = ""
 
311
  logUI = document.getElementById("logUI")
312
  const speechSynthesis = window.speechSynthesis;
313
  const voiceSelect = document.getElementById('voice');
@@ -318,7 +313,9 @@ var word_list = []
318
  const pitchInput = document.getElementById('pitch');
319
  const rateValue = document.getElementById('rateValue');
320
  const pitchValue = document.getElementById('pitchValue');
 
321
  let voices = [];
 
322
  const languageNames = {
323
  'en': '- English ',
324
  'mra': '- Marathi',
@@ -414,6 +411,7 @@ var word_list = []
414
  'tt': 'Tatar',
415
  'ur': 'Urdu',
416
  'vi': 'Vietnamese'
 
417
  };
418
  function undo() {
419
  word_list.pop()
@@ -435,6 +433,7 @@ var word_list = []
435
  if (!response.ok) {
436
  throw new Error(`Response status: ${response.status}`);
437
  }
 
438
  var voicesList = await response.text();
439
  voicesList = voicesList.split('\n')
440
  voicesList.pop()
@@ -455,6 +454,7 @@ var word_list = []
455
  if (!response.ok) {
456
  throw new Error(`Response status: ${response.status}`);
457
  }
 
458
  var langsList = await response.text();
459
  langsList = langsList.split('\n')
460
  langsList.pop()
@@ -478,12 +478,14 @@ var word_list = []
478
  } catch (error) {
479
  console.error(error.message);
480
  }
 
481
  const optionNodes = Array.from(languageSelect.children);
482
  const comparator = new Intl.Collator('en'.slice(0, 2)).compare;
483
  optionNodes.sort((a, b) => comparator(a.textContent, b.textContent));
484
  optionNodes.forEach((option) => languageSelect.appendChild(option));
485
  languageSelect.children[0].selected = "selected"
486
  }
 
487
  function getBaseLanguageCode(lang) {
488
  return lang.split('-').slice(0, 2).join('-');
489
  }
@@ -493,9 +495,11 @@ var word_list = []
493
  function getModifierLanguageCode(lang) {
494
  return lang.split('-').slice(1, 3).join('-');
495
  }
 
496
  function updateLanguageList() {
497
  const uniqueLanguages = new Set(voices.map(voice => getBaseLanguageCode(voice.lang)));
498
  languageSelect.innerHTML = '';
 
499
  Object.keys(languageNames).forEach(lang => {
500
  if (uniqueLanguages.has(lang)) {
501
  const option = document.createElement('option');
@@ -504,6 +508,7 @@ var word_list = []
504
  languageSelect.appendChild(option);
505
  }
506
  });
 
507
  uniqueLanguages.forEach(lang => {
508
  if (!languageNames[lang]) {
509
  if (!languageNames[getBaseBaseLanguageCode(lang)]) {
@@ -520,15 +525,18 @@ var word_list = []
520
  }
521
  }
522
  });
 
523
  const optionNodes = Array.from(languageSelect.children);
524
  const comparator = new Intl.Collator('en'.slice(0, 2)).compare;
525
  optionNodes.sort((a, b) => comparator(a.textContent, b.textContent));
526
  optionNodes.forEach((option) => languageSelect.appendChild(option));
527
  languageSelect.children[0].selected = "selected"
528
  }
 
529
  function updateVoiceList() {
530
  const selectedLanguage = languageSelect.value;
531
  voiceSelect.innerHTML = '';
 
532
  voices.forEach((voice) => {
533
  if (getBaseLanguageCode(voice.lang) === selectedLanguage) {
534
  const option = document.createElement('option');
@@ -547,7 +555,9 @@ var word_list = []
547
  logUI.appendChild(span);
548
  logUI.appendChild(document.createElement('br')); // Add a line break
549
  }
 
550
  const originalFetch = window.fetch;
 
551
  // Override the fetch function
552
  window.fetch = async function (input, init) {
553
  // Convert input to URL if it's a Request object
@@ -556,22 +566,31 @@ var word_list = []
556
  if (url == 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm') {
557
  //newUrl = 'http://127.0.0.1:8125/assets/static/vision_wasm_internal.wasm' //For Android
558
  newUrl = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm' // For Web
 
559
  }
560
  console.log("This was FETCHED: ", newUrl)
561
  // Call the original fetch function with the new URL
562
  return originalFetch(newUrl, init);
563
  };
 
 
 
 
564
  if ('speechSynthesis' in window) {
565
  speechSynthesis.onvoiceschanged = () => {
566
  populateVoiceList();
567
  };
 
568
  languageSelect.addEventListener('change', updateVoiceList);
 
569
  rateInput.addEventListener('input', () => {
570
  rateValue.textContent = rateInput.value;
571
  });
 
572
  pitchInput.addEventListener('input', () => {
573
  pitchValue.textContent = pitchInput.value;
574
  });
 
575
  speakButton.addEventListener('click', () => {
576
  const utterance = new SpeechSynthesisUtterance(textInput.value);
577
  const selectedVoice = voices.find(voice => voice.name === voiceSelect.value);
@@ -580,14 +599,18 @@ var word_list = []
580
  utterance.pitch = pitchInput.value;
581
  speechSynthesis.speak(utterance);
582
  });
 
583
  populateVoiceList()
584
  // Create an utterance object ⣿
 
585
  } else {
586
  speechSupported = false;
587
  console.log('Text-to-speech not supported.');
 
588
  populateTTWLangs()
589
  populateTTWVoices()
590
  }
 
591
  function speak(toSpeak) {
592
  console.log("speech api support", speechSupported)
593
  console.log("condition: ", !speechSupported)
@@ -602,8 +625,10 @@ var word_list = []
602
  audioPlayer.src = 'http://127.0.0.1:8125/speech?t=' + encodeURIComponent(toSpeak) + currSettings
603
  console.log("Set src: ", audioPlayer.src)
604
  }
 
605
  audioPlayer.play() // Play the audio
606
  .then(() => {
 
607
  console.log('Audio is playing');
608
  })
609
  .catch(error => {
@@ -616,6 +641,9 @@ var word_list = []
616
  console.log("Text to speech is now not supported")
617
  }
618
  }
 
 
 
619
  function set_output_array(text) {
620
  console.log(text)
621
  word_list = text.split("");
@@ -625,12 +653,14 @@ var word_list = []
625
  word_list = [];
626
  textInput.value = ""
627
  }
 
628
  </script>
629
 
630
  <script type="module">
631
  document.getElementById("info-").addEventListener("click", switchPage.bind(null, "info-"));
632
  document.getElementById("home-").addEventListener("click", switchPage.bind(null, "home-"));
633
  document.getElementById("settings-").addEventListener("click", switchPage.bind(null, "settings-"));
 
634
  //import { HandLandmarker, FilesetResolver } from "http://127.0.0.1:8125/assets/static/tasks-vision@0.10.0" // For Android
635
  import { HandLandmarker, FilesetResolver } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0"; // For Web
636
  let handLandmarker = undefined;
@@ -640,6 +670,7 @@ var word_list = []
640
  var time_since_letter = 0
641
  var last_letter_time = 0
642
  var maxPercentage = 1
 
643
  function inputFrequency() {
644
  return document.getElementById('duration').value
645
  }
@@ -648,10 +679,10 @@ var word_list = []
648
  }
649
  document.getElementById('durationValue').innerText = inputFrequency()
650
  document.getElementById('repeatValue').innerText = repeatBuffer()
651
- var is_first_run = 1
652
- const letter_list = ["A", "B", "", "D", "E", "F", "G", "", "I", "J", "K", "L", "M", "N", "O", "P", "", "R", "S", "T", "U", "V", "W", "", "Y", "Z", ""]
653
- const phrase_list = ["Ere-ọwurọ", "Bọkọ", "Bokọ???", "Ere-Alẹ", "Mo dọkpẹ", "Ẹrẹ"]
654
- // const phrase_list = [" Good-morning", " Hello", " How are you", "Good-evening", " Thanks", " You"]
655
  // Before we can use HandLandmarker class we must wait for it to finish
656
  // loading. Machine Learning models can be large and take a moment to
657
  // get everything needed to run.
@@ -668,10 +699,12 @@ var word_list = []
668
  });
669
  };
670
  createHandLandmarker();
 
671
  //const MODEL_PATH = "http://127.0.0.1:8125/assets/static/model.tflite" // For Android
672
  //const WORD_MODEL = "http://127.0.0.1:8125/assets/static/word.tflite" // For Android
673
  const MODEL_PATH = "/exported" // For Web
674
  const WORD_MODEL = "/word" // For Web
 
675
  const letterDetector = tflite.loadTFLiteModel(MODEL_PATH);
676
  const wordDetector = tflite.loadTFLiteModel(WORD_MODEL);
677
  var objectDetector = letterDetector
@@ -741,18 +774,23 @@ var word_list = []
741
  enableWebcamButton.style = "display:none"
742
  document.getElementById("switch-camera").style.display = "block"
743
  document.getElementById("mode-switch").style.display = "flex"
 
744
  }
745
  // getUsermedia parameters.
746
  load_camera()
747
  }
 
748
  function switchPage(elem) {
749
  prevSpeech = ""
 
750
  var pH = document.getElementById("home-page")
751
  var pI = document.getElementById("info-page")
752
  var pS = document.getElementById("settings-page")
 
753
  pH.style.display = "none"
754
  pI.style.display = "none"
755
  pS.style.display = "none"
 
756
  document.getElementById(elem + "page").style.display = "block"
757
  if (elem != "home-") {
758
  webcamRunning = false
@@ -770,6 +808,7 @@ var word_list = []
770
  context.clearRect(0, 0, canvas.width, canvas.height);
771
  }
772
  }
 
773
  function load_camera() {
774
  try {
775
  var stream = video.srcObject;
@@ -785,6 +824,7 @@ var word_list = []
785
  } catch (error) {
786
  console.error(error.message);
787
  }
 
788
  const constraints = {
789
  video: {
790
  facingMode: video_facing_mode
@@ -821,14 +861,18 @@ var word_list = []
821
  }
822
  canvasCtx.save();
823
  canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
 
824
  if (is_first_run == 1) {
825
  var elem_rect = document.getElementById("output_canvas").getBoundingClientRect()
826
  console.log(elem_rect.height | 0);
827
  document.getElementById("canvas_wrapper").style.height = (elem_rect.height | 0).toString() + "px"
 
828
  is_first_run = 0
829
  }
 
830
  if (results.landmarks && results.handednesses[0]) {
831
  var current_time = Math.round(Date.now())
 
832
  if (results.handednesses[0][0].categoryName == "Left") {
833
  if (!Array.from(document.getElementById('modeSelector').classList).includes('right')) {
834
  annotateImage()
@@ -843,6 +887,8 @@ var word_list = []
843
  var current_result = "_"
844
  var previous_result = document.getElementById("predicted_result").innerText
845
  document.getElementById("predicted_result").innerText = current_result
 
 
846
  if (previous_result == current_result) {
847
  if (current_time - last_letter_time > getNaturalLength(" ")) {
848
  last_letter_time = current_time
@@ -861,6 +907,7 @@ var word_list = []
861
  else {
862
  canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width)
863
  if (30 > calculateCanvasBrightness(canvasElement)) {
 
864
  var current_result = "<"
865
  var previous_result = document.getElementById("predicted_result").innerText
866
  document.getElementById("predicted_result").innerText = current_result
@@ -879,9 +926,11 @@ var word_list = []
879
  }
880
  } else {
881
  last_letter_time = Math.round(Date.now())
 
882
  document.getElementById("predicted_result").style.width = String(0) + "%"
883
  }
884
  }
 
885
  canvasCtx.restore();
886
  // Keep predicting
887
  if (webcamRunning === true) {
@@ -889,6 +938,7 @@ var word_list = []
889
  }
890
  }
891
  function annotateImage(firstA = true) {
 
892
  //console.log(results.landmarks)
893
  if (results.landmarks[0]) {
894
  x_array = []
@@ -901,10 +951,12 @@ var word_list = []
901
  var min_y = Math.min(...y_array) * image_height
902
  var max_x = Math.max(...x_array) * image_width
903
  var max_y = Math.max(...y_array) * image_height
 
904
  var sect_height = max_y - (min_y)
905
  var sect_width = max_x - (min_x)
906
  var center_x = (min_x + max_x) / 2
907
  var center_y = (min_y + max_y) / 2
 
908
  var sect_diameter = 50
909
  if (sect_height > sect_width) {
910
  sect_diameter = sect_height
@@ -914,6 +966,7 @@ var word_list = []
914
  sect_diameter = sect_width
915
  // console.log("sect_width", sect_diameter)
916
  }
 
917
  sect_diameter = sect_diameter + 50
918
  var sect_radius = sect_diameter / 2
919
  var crop_top = center_y - sect_radius
@@ -940,11 +993,15 @@ var word_list = []
940
  canvasCtx.fillStyle = "#ffffff"
941
  canvasCtx.fill()
942
  }
 
943
  } else {
944
  canvasCtx.beginPath();
945
  canvasCtx.rect(crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top);
946
  canvasCtx.stroke();
947
  }
 
 
 
948
  }
949
  /* for (const landmarks of results.multiHandLandmarks) {
950
  drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS, {
@@ -968,76 +1025,102 @@ var word_list = []
968
  drawConnection(hand[4], hand[3], 'rgb(0, 0, 255)', 5); // 4-3 (was red)
969
  drawConnection(hand[3], hand[2], 'rgb(0, 0, 255)', 5); // 3-2 (was red)
970
  drawConnection(hand[2], hand[1], 'rgb(0, 0, 255)', 5); // 2-1 (was red)
 
971
  // Index connections
972
  drawConnection(hand[8], hand[7], 'rgb(0, 255, 0)', 5); // 8-7
973
  drawConnection(hand[7], hand[6], 'rgb(0, 255, 0)', 5); // 7-6
974
  drawConnection(hand[6], hand[5], 'rgb(0, 255, 0)', 5); // 6-5
 
975
  // Middle connections
976
  drawConnection(hand[12], hand[11], 'rgb(255, 0, 0)', 5); // 12-11 (was blue)
977
  drawConnection(hand[11], hand[10], 'rgb(255, 0, 0)', 5); // 11-10 (was blue)
978
  drawConnection(hand[10], hand[9], 'rgb(255, 0, 0)', 5); // 10-9 (was blue)
 
979
  // Ring connections
980
  drawConnection(hand[16], hand[15], 'rgb(0, 255, 255)', 5); // 16-15 (was yellow)
981
  drawConnection(hand[15], hand[14], 'rgb(0, 255, 255)', 5); // 15-14 (was yellow)
982
  drawConnection(hand[14], hand[13], 'rgb(0, 255, 255)', 5); // 14-13 (was yellow)
 
983
  // Pinky connections
984
  drawConnection(hand[20], hand[19], 'rgb(255, 0, 255)', 5); // 20-19
985
  drawConnection(hand[19], hand[18], 'rgb(255, 0, 255)', 5); // 19-18
986
  drawConnection(hand[18], hand[17], 'rgb(255, 0, 255)', 5); // 18-17
 
987
  drawConnection(hand[0], hand[1], 'rgb(200, 200, 200)', 5); // 0-1
988
  drawConnection(hand[0], hand[5], 'rgb(200, 200, 200)', 5); // 0-5
989
  drawConnection(hand[0], hand[17], 'rgb(200, 200, 200)', 5); // 0-17
990
  drawConnection(hand[5], hand[9], 'rgb(200, 200, 200)', 5); // 5-9
991
  drawConnection(hand[9], hand[13], 'rgb(200, 200, 200)', 5); // 9-13
992
  drawConnection(hand[13], hand[17], 'rgb(200, 200, 200)', 5); // 13-17
 
993
  // Thumb
994
  drawLandmarks(canvasCtx, hand[2], '#ffe5b4'); // Thumb tip (2)
995
  drawLandmarks(canvasCtx, hand[3], '#ffe5b4'); // Thumb base (3)
996
  drawLandmarks(canvasCtx, hand[4], '#ffe5b4'); // Thumb base (4)
 
997
  // Index
998
  drawLandmarks(canvasCtx, hand[6], '#804080'); // Index tip (6)
999
  drawLandmarks(canvasCtx, hand[7], '#804080'); // Index base (7)
1000
  drawLandmarks(canvasCtx, hand[8], '#804080'); // Index base (8)
 
1001
  // Middle
1002
  drawLandmarks(canvasCtx, hand[10], '#ffcc00'); // Middle tip (10)
1003
  drawLandmarks(canvasCtx, hand[11], '#ffcc00'); // Middle base (11)
1004
  drawLandmarks(canvasCtx, hand[12], '#ffcc00'); // Middle base (12)
 
1005
  // Ring
1006
  drawLandmarks(canvasCtx, hand[14], '#30ff30'); // Ring tip (14)
1007
  drawLandmarks(canvasCtx, hand[15], '#30ff30'); // Ring base (15)
1008
  drawLandmarks(canvasCtx, hand[16], '#30ff30'); // Ring base (16)
 
1009
  // Pinky
1010
  drawLandmarks(canvasCtx, hand[18], '#1565c0'); // Pinky tip (18)
1011
  drawLandmarks(canvasCtx, hand[19], '#1565c0'); // Pinky base (19)
1012
  drawLandmarks(canvasCtx, hand[20], '#1565c0'); // Pinky base (20)
 
1013
  drawLandmarks(canvasCtx, hand[0], '#ff3030'); // Wrist (0)
 
1014
  drawLandmarks(canvasCtx, hand[1], '#ff3030'); // Palm base (1)
 
1015
  drawLandmarks(canvasCtx, hand[5], '#ff3030'); // Index palm (5)
 
1016
  drawLandmarks(canvasCtx, hand[9], '#ff3030'); // Middle palm (9)
 
1017
  drawLandmarks(canvasCtx, hand[13], '#ff3030'); // Ring palm (13)
 
1018
  drawLandmarks(canvasCtx, hand[17], '#ff3030'); // Pinky palm (17)
 
1019
  // Crop Canvas
1020
  if (firstA == true) {
1021
  cropCanvas(canvasElement, crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top)
1022
  }
1023
  }
1024
  // Add more drawing calls for each landmark collection as needed
 
 
 
 
1025
  //# sourceURL=pen.js
1026
  }
 
 
1027
  function iterate(x, y) {
1028
  x_array.push(x.x)
1029
  y_array.push(x.y)
1030
  }
 
1031
  const cropCanvas = (sourceCanvas, left, top, width, height) => {
1032
  let destCanvas = document.createElement('canvas');
1033
  destCanvas.width = 224;
1034
  var cropAspectRatio = width / height;
 
1035
  destCanvas.height = 224 / cropAspectRatio
1036
  destCanvas.getContext("2d").drawImage(
1037
  sourceCanvas,
1038
  left, top, width, height, // source rect with content to crop
1039
  0, 0, 224, destCanvas.height); // newCanvas, same size as source
1040
  var predictionInput = tf.browser.fromPixels(destCanvas.getContext("2d").getImageData(0, 0, 224, 224))
 
1041
  predict(tf.expandDims(predictionInput, 0));
1042
  }
1043
  function getNaturalLength(letter) {
@@ -1065,19 +1148,28 @@ var word_list = []
1065
  function mapLogitsToPercentage(logits) {
1066
  // Apply ceiling of 260
1067
  const cappedLogits = logits.map(value => Math.min(value, 260));
 
1068
  // Map the values from 0-260 to 0-100
1069
  const mappedPercentages = cappedLogits.map(value => (value / 260) * 100);
 
1070
  return mappedPercentages;
1071
  }
 
 
1072
  async function predict(inputTensor) {
 
1073
  console.log(index_list[0])
1074
  objectDetector.then(function (res) {
 
1075
  var prediction = res.predict(inputTensor);
 
1076
  var outputArray = prediction.dataSync(); // Get the output as an array
1077
  var percentages = mapLogitsToPercentage(outputArray)
1078
  maxPercentage = Math.max(...percentages)
1079
  document.getElementById('predicted_result').style.opacity = (maxPercentage+30)+'%'
 
1080
  var predictedClass = percentages.indexOf(Math.max(...percentages)); // Get the index
 
1081
  var current_result = index_list[predictedClass]
1082
  global_cres = current_result
1083
  var previous_result = document.getElementById("predicted_result").innerText
@@ -1088,6 +1180,7 @@ var word_list = []
1088
  }
1089
  console.log("p:", previous_result, "c:", current_result)
1090
  if (previous_result == current_result) {
 
1091
  if (current_time - last_letter_time > getNaturalLength(current_result)) {
1092
  last_letter_time = current_time
1093
  word_list.push(current_result)
@@ -1103,18 +1196,25 @@ var word_list = []
1103
  }, function (err) {
1104
  console.log(err);
1105
  });
 
1106
  }
 
1107
  function drawLandmarks(canvasCtx, landmarks, color) {
1108
  var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
1109
  var image_width = canvasElement.width
 
1110
  canvasCtx.fillStyle = `rgb(${landmarks.z},${landmarks.z},${landmarks.z})`;
1111
  canvasCtx.beginPath();
1112
  canvasCtx.arc(landmarks.x * image_width, landmarks.y * image_height, 8, 0, 2 * Math.PI);
1113
  canvasCtx.fill();
 
1114
  }
 
1115
  function drawConnection(startNode, endNode, strokeColor, strokeWidth) {
 
1116
  var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
1117
  var image_width = canvasElement.width
 
1118
  canvasCtx.strokeStyle = strokeColor;
1119
  canvasCtx.lineWidth = strokeWidth - 1;
1120
  canvasCtx.beginPath();
@@ -1124,23 +1224,29 @@ var word_list = []
1124
  }
1125
  function calculateCanvasBrightness(canvas) {
1126
  const context = canvas.getContext('2d', { willReadFrequently: true });
 
1127
  // Get the image data from the canvas
1128
  const imageData = context.getImageData(0, 0, canvas.width, canvas.height);
1129
  const data = imageData.data;
 
1130
  let totalBrightness = 0;
1131
  let pixelCount = 0;
 
1132
  // Loop through each pixel
1133
  for (let i = 0; i < data.length; i += 4) {
1134
  const r = data[i]; // Red
1135
  const g = data[i + 1]; // Green
1136
  const b = data[i + 2]; // Blue
 
1137
  // Calculate brightness for this pixel
1138
  const brightness = 0.299 * r + 0.587 * g + 0.114 * b;
1139
  totalBrightness += brightness;
1140
  pixelCount++;
1141
  }
 
1142
  // Calculate average brightness
1143
  const averageBrightness = totalBrightness / pixelCount;
 
1144
  return averageBrightness;
1145
  }
1146
  function triggerPulse() {
@@ -1148,6 +1254,7 @@ var word_list = []
1148
  /* Apply after working on pulse css more
1149
  const resultWrapper = document.querySelector('.wrapper_result');
1150
  resultWrapper.classList.add('pulse');
 
1151
  // Remove the class after the animation ends to allow it to be triggered again
1152
  resultWrapper.addEventListener('animationend', () => {
1153
  resultWrapper.classList.remove('pulse');
@@ -1163,4 +1270,3 @@ var word_list = []
1163
  </body>
1164
 
1165
  </html>
1166
-
 
3
  SignSpeak - A Communicator For Signers
4
 
5
  Copyright (C) 2025 Shantanu Khedkar
6
+
7
  This program is free software: you can redistribute it and/or modify
8
  it under the terms of the GNU General Public License as published by
9
  the Free Software Foundation, either version 3 of the License, or
10
  (at your option) any later version.
11
+
12
  This program is distributed in the hope that it will be useful,
13
  but WITHOUT ANY WARRANTY; without even the implied warranty of
14
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
  GNU General Public License for more details.
16
+
17
  You should have received a copy of the GNU General Public License
18
  along with this program. If not, see <https://www.gnu.org/licenses/>.
19
  -->
 
275
  </form>
276
  </section>
277
  <div class="bottom-nav">
278
+ <div id="info-" class="bottom-nav-btn">
279
+ <svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px"
280
+ viewBox="0 0 512 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.-->
281
+ <path fill="#5d5d5d"
282
+ d="M256 512A256 256 0 1 0 256 0a256 256 0 1 0 0 512zM216 336l24 0 0-64-24 0c-13.3 0-24-10.7-24-24s10.7-24 24-24l48 0c13.3 0 24 10.7 24 24l0 88 8 0c13.3 0 24 10.7 24 24s-10.7 24-24 24l-80 0c-13.3 0-24-10.7-24-24s10.7-24 24-24zm40-208a32 32 0 1 1 0 64 32 32 0 1 1 0-64z" />
283
+ </svg>
284
+ </div>
285
+ <div id="home-" class="bottom-nav-btn center-btn">
286
+ <!--For Android
287
+ <img src="http://127.0.0.1:8125/assets/static/logo_sil.svg" height="48" width="48px"></img>
288
+ -->
289
+ <img src="/static/logo_sil.svg" height="48" width="48px"></img>
290
+ </div>
291
+ <div id="settings-" class="bottom-nav-btn">
292
+ <svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px"
293
+ viewBox="0 0 512 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.-->
294
+ <path fill="#5d5d5d"
295
+ d="M495.9 166.6c3.2 8.7 .5 18.4-6.4 24.6l-43.3 39.4c1.1 8.3 1.7 16.8 1.7 25.4s-.6 17.1-1.7 25.4l43.3 39.4c6.9 6.2 9.6 15.9 6.4 24.6c-4.4 11.9-9.7 23.3-15.8 34.3l-4.7 8.1c-6.6 11-14 21.4-22.1 31.2c-5.9 7.2-15.7 9.6-24.5 6.8l-55.7-17.7c-13.4 10.3-28.2 18.9-44 25.4l-12.5 57.1c-2 9.1-9 16.3-18.2 17.8c-13.8 2.3-28 3.5-42.5 3.5s-28.7-1.2-42.5-3.5c-9.2-1.5-16.2-8.7-18.2-17.8l-12.5-57.1c-15.8-6.5-30.6-15.1-44-25.4L83.1 425.9c-8.8 2.8-18.6 .3-24.5-6.8c-8.1-9.8-15.5-20.2-22.1-31.2l-4.7-8.1c-6.1-11-11.4-22.4-15.8-34.3c-3.2-8.7-.5-18.4 6.4-24.6l43.3-39.4C64.6 273.1 64 264.6 64 256s.6-17.1 1.7-25.4L22.4 191.2c-6.9-6.2-9.6-15.9-6.4-24.6c4.4-11.9 9.7-23.3 15.8-34.3l4.7-8.1c6.6-11 14-21.4 22.1-31.2c5.9-7.2 15.7-9.6 24.5-6.8l55.7 17.7c13.4-10.3 28.2-18.9 44-25.4l12.5-57.1c2-9.1 9-16.3 18.2-17.8C227.3 1.2 241.5 0 256 0s28.7 1.2 42.5 3.5c9.2 1.5 16.2 8.7 18.2 17.8l12.5 57.1c15.8 6.5 30.6 15.1 44 25.4l55.7-17.7c8.8-2.8 18.6-.3 24.5 6.8c8.1 9.8 15.5 20.2 22.1 31.2l4.7 8.1c6.1 11 11.4 22.4 15.8 34.3zM256 336a80 80 0 1 0 0-160 80 80 0 1 0 0 160z" />
296
+ </svg>
297
+ </div>
 
 
 
 
 
 
 
 
 
298
 
299
+ </div>
300
  <script>
301
  var word_list = []
302
  var speechSupported = true
303
  var prevSpeech = ""
304
  var prevSettings = ""
305
+
306
  logUI = document.getElementById("logUI")
307
  const speechSynthesis = window.speechSynthesis;
308
  const voiceSelect = document.getElementById('voice');
 
313
  const pitchInput = document.getElementById('pitch');
314
  const rateValue = document.getElementById('rateValue');
315
  const pitchValue = document.getElementById('pitchValue');
316
+
317
  let voices = [];
318
+
319
  const languageNames = {
320
  'en': '- English ',
321
  'mra': '- Marathi',
 
411
  'tt': 'Tatar',
412
  'ur': 'Urdu',
413
  'vi': 'Vietnamese'
414
+
415
  };
416
  function undo() {
417
  word_list.pop()
 
433
  if (!response.ok) {
434
  throw new Error(`Response status: ${response.status}`);
435
  }
436
+
437
  var voicesList = await response.text();
438
  voicesList = voicesList.split('\n')
439
  voicesList.pop()
 
454
  if (!response.ok) {
455
  throw new Error(`Response status: ${response.status}`);
456
  }
457
+
458
  var langsList = await response.text();
459
  langsList = langsList.split('\n')
460
  langsList.pop()
 
478
  } catch (error) {
479
  console.error(error.message);
480
  }
481
+
482
  const optionNodes = Array.from(languageSelect.children);
483
  const comparator = new Intl.Collator('en'.slice(0, 2)).compare;
484
  optionNodes.sort((a, b) => comparator(a.textContent, b.textContent));
485
  optionNodes.forEach((option) => languageSelect.appendChild(option));
486
  languageSelect.children[0].selected = "selected"
487
  }
488
+
489
  function getBaseLanguageCode(lang) {
490
  return lang.split('-').slice(0, 2).join('-');
491
  }
 
495
  function getModifierLanguageCode(lang) {
496
  return lang.split('-').slice(1, 3).join('-');
497
  }
498
+
499
  function updateLanguageList() {
500
  const uniqueLanguages = new Set(voices.map(voice => getBaseLanguageCode(voice.lang)));
501
  languageSelect.innerHTML = '';
502
+
503
  Object.keys(languageNames).forEach(lang => {
504
  if (uniqueLanguages.has(lang)) {
505
  const option = document.createElement('option');
 
508
  languageSelect.appendChild(option);
509
  }
510
  });
511
+
512
  uniqueLanguages.forEach(lang => {
513
  if (!languageNames[lang]) {
514
  if (!languageNames[getBaseBaseLanguageCode(lang)]) {
 
525
  }
526
  }
527
  });
528
+
529
  const optionNodes = Array.from(languageSelect.children);
530
  const comparator = new Intl.Collator('en'.slice(0, 2)).compare;
531
  optionNodes.sort((a, b) => comparator(a.textContent, b.textContent));
532
  optionNodes.forEach((option) => languageSelect.appendChild(option));
533
  languageSelect.children[0].selected = "selected"
534
  }
535
+
536
  function updateVoiceList() {
537
  const selectedLanguage = languageSelect.value;
538
  voiceSelect.innerHTML = '';
539
+
540
  voices.forEach((voice) => {
541
  if (getBaseLanguageCode(voice.lang) === selectedLanguage) {
542
  const option = document.createElement('option');
 
555
  logUI.appendChild(span);
556
  logUI.appendChild(document.createElement('br')); // Add a line break
557
  }
558
+
559
  const originalFetch = window.fetch;
560
+
561
  // Override the fetch function
562
  window.fetch = async function (input, init) {
563
  // Convert input to URL if it's a Request object
 
566
  if (url == 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm') {
567
  //newUrl = 'http://127.0.0.1:8125/assets/static/vision_wasm_internal.wasm' //For Android
568
  newUrl = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm' // For Web
569
+
570
  }
571
  console.log("This was FETCHED: ", newUrl)
572
  // Call the original fetch function with the new URL
573
  return originalFetch(newUrl, init);
574
  };
575
+
576
+
577
+
578
+
579
  if ('speechSynthesis' in window) {
580
  speechSynthesis.onvoiceschanged = () => {
581
  populateVoiceList();
582
  };
583
+
584
  languageSelect.addEventListener('change', updateVoiceList);
585
+
586
  rateInput.addEventListener('input', () => {
587
  rateValue.textContent = rateInput.value;
588
  });
589
+
590
  pitchInput.addEventListener('input', () => {
591
  pitchValue.textContent = pitchInput.value;
592
  });
593
+
594
  speakButton.addEventListener('click', () => {
595
  const utterance = new SpeechSynthesisUtterance(textInput.value);
596
  const selectedVoice = voices.find(voice => voice.name === voiceSelect.value);
 
599
  utterance.pitch = pitchInput.value;
600
  speechSynthesis.speak(utterance);
601
  });
602
+
603
  populateVoiceList()
604
  // Create an utterance object ⣿
605
+
606
  } else {
607
  speechSupported = false;
608
  console.log('Text-to-speech not supported.');
609
+
610
  populateTTWLangs()
611
  populateTTWVoices()
612
  }
613
+
614
  function speak(toSpeak) {
615
  console.log("speech api support", speechSupported)
616
  console.log("condition: ", !speechSupported)
 
625
  audioPlayer.src = 'http://127.0.0.1:8125/speech?t=' + encodeURIComponent(toSpeak) + currSettings
626
  console.log("Set src: ", audioPlayer.src)
627
  }
628
+
629
  audioPlayer.play() // Play the audio
630
  .then(() => {
631
+
632
  console.log('Audio is playing');
633
  })
634
  .catch(error => {
 
641
  console.log("Text to speech is now not supported")
642
  }
643
  }
644
+
645
+
646
+
647
  function set_output_array(text) {
648
  console.log(text)
649
  word_list = text.split("");
 
653
  word_list = [];
654
  textInput.value = ""
655
  }
656
+
657
  </script>
658
 
659
  <script type="module">
660
  document.getElementById("info-").addEventListener("click", switchPage.bind(null, "info-"));
661
  document.getElementById("home-").addEventListener("click", switchPage.bind(null, "home-"));
662
  document.getElementById("settings-").addEventListener("click", switchPage.bind(null, "settings-"));
663
+
664
  //import { HandLandmarker, FilesetResolver } from "http://127.0.0.1:8125/assets/static/tasks-vision@0.10.0" // For Android
665
  import { HandLandmarker, FilesetResolver } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0"; // For Web
666
  let handLandmarker = undefined;
 
670
  var time_since_letter = 0
671
  var last_letter_time = 0
672
  var maxPercentage = 1
673
+
674
  function inputFrequency() {
675
  return document.getElementById('duration').value
676
  }
 
679
  }
680
  document.getElementById('durationValue').innerText = inputFrequency()
681
  document.getElementById('repeatValue').innerText = repeatBuffer()
682
+ var is_first_run = 1
683
+ const letter_list = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "#"]
684
+ const phrase_list = [" Good", " Hello", " How're", " I'm", " Thanks", " You"]
685
+ var index_list = letter_list
686
  // Before we can use HandLandmarker class we must wait for it to finish
687
  // loading. Machine Learning models can be large and take a moment to
688
  // get everything needed to run.
 
699
  });
700
  };
701
  createHandLandmarker();
702
+
703
  //const MODEL_PATH = "http://127.0.0.1:8125/assets/static/model.tflite" // For Android
704
  //const WORD_MODEL = "http://127.0.0.1:8125/assets/static/word.tflite" // For Android
705
  const MODEL_PATH = "/exported" // For Web
706
  const WORD_MODEL = "/word" // For Web
707
+
708
  const letterDetector = tflite.loadTFLiteModel(MODEL_PATH);
709
  const wordDetector = tflite.loadTFLiteModel(WORD_MODEL);
710
  var objectDetector = letterDetector
 
774
  enableWebcamButton.style = "display:none"
775
  document.getElementById("switch-camera").style.display = "block"
776
  document.getElementById("mode-switch").style.display = "flex"
777
+
778
  }
779
  // getUsermedia parameters.
780
  load_camera()
781
  }
782
+
783
  function switchPage(elem) {
784
  prevSpeech = ""
785
+
786
  var pH = document.getElementById("home-page")
787
  var pI = document.getElementById("info-page")
788
  var pS = document.getElementById("settings-page")
789
+
790
  pH.style.display = "none"
791
  pI.style.display = "none"
792
  pS.style.display = "none"
793
+
794
  document.getElementById(elem + "page").style.display = "block"
795
  if (elem != "home-") {
796
  webcamRunning = false
 
808
  context.clearRect(0, 0, canvas.width, canvas.height);
809
  }
810
  }
811
+
812
  function load_camera() {
813
  try {
814
  var stream = video.srcObject;
 
824
  } catch (error) {
825
  console.error(error.message);
826
  }
827
+
828
  const constraints = {
829
  video: {
830
  facingMode: video_facing_mode
 
861
  }
862
  canvasCtx.save();
863
  canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
864
+
865
  if (is_first_run == 1) {
866
  var elem_rect = document.getElementById("output_canvas").getBoundingClientRect()
867
  console.log(elem_rect.height | 0);
868
  document.getElementById("canvas_wrapper").style.height = (elem_rect.height | 0).toString() + "px"
869
+
870
  is_first_run = 0
871
  }
872
+
873
  if (results.landmarks && results.handednesses[0]) {
874
  var current_time = Math.round(Date.now())
875
+
876
  if (results.handednesses[0][0].categoryName == "Left") {
877
  if (!Array.from(document.getElementById('modeSelector').classList).includes('right')) {
878
  annotateImage()
 
887
  var current_result = "_"
888
  var previous_result = document.getElementById("predicted_result").innerText
889
  document.getElementById("predicted_result").innerText = current_result
890
+
891
+
892
  if (previous_result == current_result) {
893
  if (current_time - last_letter_time > getNaturalLength(" ")) {
894
  last_letter_time = current_time
 
907
  else {
908
  canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width)
909
  if (30 > calculateCanvasBrightness(canvasElement)) {
910
+
911
  var current_result = "<"
912
  var previous_result = document.getElementById("predicted_result").innerText
913
  document.getElementById("predicted_result").innerText = current_result
 
926
  }
927
  } else {
928
  last_letter_time = Math.round(Date.now())
929
+
930
  document.getElementById("predicted_result").style.width = String(0) + "%"
931
  }
932
  }
933
+
934
  canvasCtx.restore();
935
  // Keep predicting
936
  if (webcamRunning === true) {
 
938
  }
939
  }
940
  function annotateImage(firstA = true) {
941
+
942
  //console.log(results.landmarks)
943
  if (results.landmarks[0]) {
944
  x_array = []
 
951
  var min_y = Math.min(...y_array) * image_height
952
  var max_x = Math.max(...x_array) * image_width
953
  var max_y = Math.max(...y_array) * image_height
954
+
955
  var sect_height = max_y - (min_y)
956
  var sect_width = max_x - (min_x)
957
  var center_x = (min_x + max_x) / 2
958
  var center_y = (min_y + max_y) / 2
959
+
960
  var sect_diameter = 50
961
  if (sect_height > sect_width) {
962
  sect_diameter = sect_height
 
966
  sect_diameter = sect_width
967
  // console.log("sect_width", sect_diameter)
968
  }
969
+
970
  sect_diameter = sect_diameter + 50
971
  var sect_radius = sect_diameter / 2
972
  var crop_top = center_y - sect_radius
 
993
  canvasCtx.fillStyle = "#ffffff"
994
  canvasCtx.fill()
995
  }
996
+
997
  } else {
998
  canvasCtx.beginPath();
999
  canvasCtx.rect(crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top);
1000
  canvasCtx.stroke();
1001
  }
1002
+
1003
+
1004
+
1005
  }
1006
  /* for (const landmarks of results.multiHandLandmarks) {
1007
  drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS, {
 
1025
  drawConnection(hand[4], hand[3], 'rgb(0, 0, 255)', 5); // 4-3 (was red)
1026
  drawConnection(hand[3], hand[2], 'rgb(0, 0, 255)', 5); // 3-2 (was red)
1027
  drawConnection(hand[2], hand[1], 'rgb(0, 0, 255)', 5); // 2-1 (was red)
1028
+
1029
  // Index connections
1030
  drawConnection(hand[8], hand[7], 'rgb(0, 255, 0)', 5); // 8-7
1031
  drawConnection(hand[7], hand[6], 'rgb(0, 255, 0)', 5); // 7-6
1032
  drawConnection(hand[6], hand[5], 'rgb(0, 255, 0)', 5); // 6-5
1033
+
1034
  // Middle connections
1035
  drawConnection(hand[12], hand[11], 'rgb(255, 0, 0)', 5); // 12-11 (was blue)
1036
  drawConnection(hand[11], hand[10], 'rgb(255, 0, 0)', 5); // 11-10 (was blue)
1037
  drawConnection(hand[10], hand[9], 'rgb(255, 0, 0)', 5); // 10-9 (was blue)
1038
+
1039
  // Ring connections
1040
  drawConnection(hand[16], hand[15], 'rgb(0, 255, 255)', 5); // 16-15 (was yellow)
1041
  drawConnection(hand[15], hand[14], 'rgb(0, 255, 255)', 5); // 15-14 (was yellow)
1042
  drawConnection(hand[14], hand[13], 'rgb(0, 255, 255)', 5); // 14-13 (was yellow)
1043
+
1044
  // Pinky connections
1045
  drawConnection(hand[20], hand[19], 'rgb(255, 0, 255)', 5); // 20-19
1046
  drawConnection(hand[19], hand[18], 'rgb(255, 0, 255)', 5); // 19-18
1047
  drawConnection(hand[18], hand[17], 'rgb(255, 0, 255)', 5); // 18-17
1048
+
1049
  drawConnection(hand[0], hand[1], 'rgb(200, 200, 200)', 5); // 0-1
1050
  drawConnection(hand[0], hand[5], 'rgb(200, 200, 200)', 5); // 0-5
1051
  drawConnection(hand[0], hand[17], 'rgb(200, 200, 200)', 5); // 0-17
1052
  drawConnection(hand[5], hand[9], 'rgb(200, 200, 200)', 5); // 5-9
1053
  drawConnection(hand[9], hand[13], 'rgb(200, 200, 200)', 5); // 9-13
1054
  drawConnection(hand[13], hand[17], 'rgb(200, 200, 200)', 5); // 13-17
1055
+
1056
  // Thumb
1057
  drawLandmarks(canvasCtx, hand[2], '#ffe5b4'); // Thumb tip (2)
1058
  drawLandmarks(canvasCtx, hand[3], '#ffe5b4'); // Thumb base (3)
1059
  drawLandmarks(canvasCtx, hand[4], '#ffe5b4'); // Thumb base (4)
1060
+
1061
  // Index
1062
  drawLandmarks(canvasCtx, hand[6], '#804080'); // Index tip (6)
1063
  drawLandmarks(canvasCtx, hand[7], '#804080'); // Index base (7)
1064
  drawLandmarks(canvasCtx, hand[8], '#804080'); // Index base (8)
1065
+
1066
  // Middle
1067
  drawLandmarks(canvasCtx, hand[10], '#ffcc00'); // Middle tip (10)
1068
  drawLandmarks(canvasCtx, hand[11], '#ffcc00'); // Middle base (11)
1069
  drawLandmarks(canvasCtx, hand[12], '#ffcc00'); // Middle base (12)
1070
+
1071
  // Ring
1072
  drawLandmarks(canvasCtx, hand[14], '#30ff30'); // Ring tip (14)
1073
  drawLandmarks(canvasCtx, hand[15], '#30ff30'); // Ring base (15)
1074
  drawLandmarks(canvasCtx, hand[16], '#30ff30'); // Ring base (16)
1075
+
1076
  // Pinky
1077
  drawLandmarks(canvasCtx, hand[18], '#1565c0'); // Pinky tip (18)
1078
  drawLandmarks(canvasCtx, hand[19], '#1565c0'); // Pinky base (19)
1079
  drawLandmarks(canvasCtx, hand[20], '#1565c0'); // Pinky base (20)
1080
+
1081
  drawLandmarks(canvasCtx, hand[0], '#ff3030'); // Wrist (0)
1082
+
1083
  drawLandmarks(canvasCtx, hand[1], '#ff3030'); // Palm base (1)
1084
+
1085
  drawLandmarks(canvasCtx, hand[5], '#ff3030'); // Index palm (5)
1086
+
1087
  drawLandmarks(canvasCtx, hand[9], '#ff3030'); // Middle palm (9)
1088
+
1089
  drawLandmarks(canvasCtx, hand[13], '#ff3030'); // Ring palm (13)
1090
+
1091
  drawLandmarks(canvasCtx, hand[17], '#ff3030'); // Pinky palm (17)
1092
+
1093
  // Crop Canvas
1094
  if (firstA == true) {
1095
  cropCanvas(canvasElement, crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top)
1096
  }
1097
  }
1098
  // Add more drawing calls for each landmark collection as needed
1099
+
1100
+
1101
+
1102
+
1103
  //# sourceURL=pen.js
1104
  }
1105
+
1106
+
1107
  function iterate(x, y) {
1108
  x_array.push(x.x)
1109
  y_array.push(x.y)
1110
  }
1111
+
1112
  const cropCanvas = (sourceCanvas, left, top, width, height) => {
1113
  let destCanvas = document.createElement('canvas');
1114
  destCanvas.width = 224;
1115
  var cropAspectRatio = width / height;
1116
+
1117
  destCanvas.height = 224 / cropAspectRatio
1118
  destCanvas.getContext("2d").drawImage(
1119
  sourceCanvas,
1120
  left, top, width, height, // source rect with content to crop
1121
  0, 0, 224, destCanvas.height); // newCanvas, same size as source
1122
  var predictionInput = tf.browser.fromPixels(destCanvas.getContext("2d").getImageData(0, 0, 224, 224))
1123
+
1124
  predict(tf.expandDims(predictionInput, 0));
1125
  }
1126
  function getNaturalLength(letter) {
 
1148
  function mapLogitsToPercentage(logits) {
1149
  // Apply ceiling of 260
1150
  const cappedLogits = logits.map(value => Math.min(value, 260));
1151
+
1152
  // Map the values from 0-260 to 0-100
1153
  const mappedPercentages = cappedLogits.map(value => (value / 260) * 100);
1154
+
1155
  return mappedPercentages;
1156
  }
1157
+
1158
+
1159
  async function predict(inputTensor) {
1160
+
1161
  console.log(index_list[0])
1162
  objectDetector.then(function (res) {
1163
+
1164
  var prediction = res.predict(inputTensor);
1165
+
1166
  var outputArray = prediction.dataSync(); // Get the output as an array
1167
  var percentages = mapLogitsToPercentage(outputArray)
1168
  maxPercentage = Math.max(...percentages)
1169
  document.getElementById('predicted_result').style.opacity = (maxPercentage+30)+'%'
1170
+
1171
  var predictedClass = percentages.indexOf(Math.max(...percentages)); // Get the index
1172
+
1173
  var current_result = index_list[predictedClass]
1174
  global_cres = current_result
1175
  var previous_result = document.getElementById("predicted_result").innerText
 
1180
  }
1181
  console.log("p:", previous_result, "c:", current_result)
1182
  if (previous_result == current_result) {
1183
+
1184
  if (current_time - last_letter_time > getNaturalLength(current_result)) {
1185
  last_letter_time = current_time
1186
  word_list.push(current_result)
 
1196
  }, function (err) {
1197
  console.log(err);
1198
  });
1199
+
1200
  }
1201
+
1202
  function drawLandmarks(canvasCtx, landmarks, color) {
1203
  var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
1204
  var image_width = canvasElement.width
1205
+
1206
  canvasCtx.fillStyle = `rgb(${landmarks.z},${landmarks.z},${landmarks.z})`;
1207
  canvasCtx.beginPath();
1208
  canvasCtx.arc(landmarks.x * image_width, landmarks.y * image_height, 8, 0, 2 * Math.PI);
1209
  canvasCtx.fill();
1210
+
1211
  }
1212
+
1213
  function drawConnection(startNode, endNode, strokeColor, strokeWidth) {
1214
+
1215
  var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
1216
  var image_width = canvasElement.width
1217
+
1218
  canvasCtx.strokeStyle = strokeColor;
1219
  canvasCtx.lineWidth = strokeWidth - 1;
1220
  canvasCtx.beginPath();
 
1224
  }
1225
  function calculateCanvasBrightness(canvas) {
1226
  const context = canvas.getContext('2d', { willReadFrequently: true });
1227
+
1228
  // Get the image data from the canvas
1229
  const imageData = context.getImageData(0, 0, canvas.width, canvas.height);
1230
  const data = imageData.data;
1231
+
1232
  let totalBrightness = 0;
1233
  let pixelCount = 0;
1234
+
1235
  // Loop through each pixel
1236
  for (let i = 0; i < data.length; i += 4) {
1237
  const r = data[i]; // Red
1238
  const g = data[i + 1]; // Green
1239
  const b = data[i + 2]; // Blue
1240
+
1241
  // Calculate brightness for this pixel
1242
  const brightness = 0.299 * r + 0.587 * g + 0.114 * b;
1243
  totalBrightness += brightness;
1244
  pixelCount++;
1245
  }
1246
+
1247
  // Calculate average brightness
1248
  const averageBrightness = totalBrightness / pixelCount;
1249
+
1250
  return averageBrightness;
1251
  }
1252
  function triggerPulse() {
 
1254
  /* Apply after working on pulse css more
1255
  const resultWrapper = document.querySelector('.wrapper_result');
1256
  resultWrapper.classList.add('pulse');
1257
+
1258
  // Remove the class after the animation ends to allow it to be triggered again
1259
  resultWrapper.addEventListener('animationend', () => {
1260
  resultWrapper.classList.remove('pulse');
 
1270
  </body>
1271
 
1272
  </html>