Update templates/browser-detect.html
Browse files- templates/browser-detect.html +141 -35
templates/browser-detect.html
CHANGED
|
@@ -3,14 +3,17 @@
|
|
| 3 |
SignSpeak - A Communicator For Signers
|
| 4 |
|
| 5 |
Copyright (C) 2025 Shantanu Khedkar
|
|
|
|
| 6 |
This program is free software: you can redistribute it and/or modify
|
| 7 |
it under the terms of the GNU General Public License as published by
|
| 8 |
the Free Software Foundation, either version 3 of the License, or
|
| 9 |
(at your option) any later version.
|
|
|
|
| 10 |
This program is distributed in the hope that it will be useful,
|
| 11 |
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
| 12 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
| 13 |
GNU General Public License for more details.
|
|
|
|
| 14 |
You should have received a copy of the GNU General Public License
|
| 15 |
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
| 16 |
-->
|
|
@@ -272,42 +275,34 @@
|
|
| 272 |
</form>
|
| 273 |
</section>
|
| 274 |
<div class="bottom-nav">
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
<button id="goDigitsBtn">Go to Digits</button>
|
| 296 |
-
</div>
|
| 297 |
-
</div>
|
| 298 |
-
|
| 299 |
-
<script>
|
| 300 |
-
document.getElementById("goDigitsBtn").addEventListener("click", function() {
|
| 301 |
-
window.open("https://huggingface.co/spaces/Emeritus-21/digits-signlang", "_blank");
|
| 302 |
-
});
|
| 303 |
-
</script>
|
| 304 |
|
| 305 |
-
|
| 306 |
<script>
|
| 307 |
var word_list = []
|
| 308 |
var speechSupported = true
|
| 309 |
var prevSpeech = ""
|
| 310 |
var prevSettings = ""
|
|
|
|
| 311 |
logUI = document.getElementById("logUI")
|
| 312 |
const speechSynthesis = window.speechSynthesis;
|
| 313 |
const voiceSelect = document.getElementById('voice');
|
|
@@ -318,7 +313,9 @@ var word_list = []
|
|
| 318 |
const pitchInput = document.getElementById('pitch');
|
| 319 |
const rateValue = document.getElementById('rateValue');
|
| 320 |
const pitchValue = document.getElementById('pitchValue');
|
|
|
|
| 321 |
let voices = [];
|
|
|
|
| 322 |
const languageNames = {
|
| 323 |
'en': '- English ',
|
| 324 |
'mra': '- Marathi',
|
|
@@ -414,6 +411,7 @@ var word_list = []
|
|
| 414 |
'tt': 'Tatar',
|
| 415 |
'ur': 'Urdu',
|
| 416 |
'vi': 'Vietnamese'
|
|
|
|
| 417 |
};
|
| 418 |
function undo() {
|
| 419 |
word_list.pop()
|
|
@@ -435,6 +433,7 @@ var word_list = []
|
|
| 435 |
if (!response.ok) {
|
| 436 |
throw new Error(`Response status: ${response.status}`);
|
| 437 |
}
|
|
|
|
| 438 |
var voicesList = await response.text();
|
| 439 |
voicesList = voicesList.split('\n')
|
| 440 |
voicesList.pop()
|
|
@@ -455,6 +454,7 @@ var word_list = []
|
|
| 455 |
if (!response.ok) {
|
| 456 |
throw new Error(`Response status: ${response.status}`);
|
| 457 |
}
|
|
|
|
| 458 |
var langsList = await response.text();
|
| 459 |
langsList = langsList.split('\n')
|
| 460 |
langsList.pop()
|
|
@@ -478,12 +478,14 @@ var word_list = []
|
|
| 478 |
} catch (error) {
|
| 479 |
console.error(error.message);
|
| 480 |
}
|
|
|
|
| 481 |
const optionNodes = Array.from(languageSelect.children);
|
| 482 |
const comparator = new Intl.Collator('en'.slice(0, 2)).compare;
|
| 483 |
optionNodes.sort((a, b) => comparator(a.textContent, b.textContent));
|
| 484 |
optionNodes.forEach((option) => languageSelect.appendChild(option));
|
| 485 |
languageSelect.children[0].selected = "selected"
|
| 486 |
}
|
|
|
|
| 487 |
function getBaseLanguageCode(lang) {
|
| 488 |
return lang.split('-').slice(0, 2).join('-');
|
| 489 |
}
|
|
@@ -493,9 +495,11 @@ var word_list = []
|
|
| 493 |
function getModifierLanguageCode(lang) {
|
| 494 |
return lang.split('-').slice(1, 3).join('-');
|
| 495 |
}
|
|
|
|
| 496 |
function updateLanguageList() {
|
| 497 |
const uniqueLanguages = new Set(voices.map(voice => getBaseLanguageCode(voice.lang)));
|
| 498 |
languageSelect.innerHTML = '';
|
|
|
|
| 499 |
Object.keys(languageNames).forEach(lang => {
|
| 500 |
if (uniqueLanguages.has(lang)) {
|
| 501 |
const option = document.createElement('option');
|
|
@@ -504,6 +508,7 @@ var word_list = []
|
|
| 504 |
languageSelect.appendChild(option);
|
| 505 |
}
|
| 506 |
});
|
|
|
|
| 507 |
uniqueLanguages.forEach(lang => {
|
| 508 |
if (!languageNames[lang]) {
|
| 509 |
if (!languageNames[getBaseBaseLanguageCode(lang)]) {
|
|
@@ -520,15 +525,18 @@ var word_list = []
|
|
| 520 |
}
|
| 521 |
}
|
| 522 |
});
|
|
|
|
| 523 |
const optionNodes = Array.from(languageSelect.children);
|
| 524 |
const comparator = new Intl.Collator('en'.slice(0, 2)).compare;
|
| 525 |
optionNodes.sort((a, b) => comparator(a.textContent, b.textContent));
|
| 526 |
optionNodes.forEach((option) => languageSelect.appendChild(option));
|
| 527 |
languageSelect.children[0].selected = "selected"
|
| 528 |
}
|
|
|
|
| 529 |
function updateVoiceList() {
|
| 530 |
const selectedLanguage = languageSelect.value;
|
| 531 |
voiceSelect.innerHTML = '';
|
|
|
|
| 532 |
voices.forEach((voice) => {
|
| 533 |
if (getBaseLanguageCode(voice.lang) === selectedLanguage) {
|
| 534 |
const option = document.createElement('option');
|
|
@@ -547,7 +555,9 @@ var word_list = []
|
|
| 547 |
logUI.appendChild(span);
|
| 548 |
logUI.appendChild(document.createElement('br')); // Add a line break
|
| 549 |
}
|
|
|
|
| 550 |
const originalFetch = window.fetch;
|
|
|
|
| 551 |
// Override the fetch function
|
| 552 |
window.fetch = async function (input, init) {
|
| 553 |
// Convert input to URL if it's a Request object
|
|
@@ -556,22 +566,31 @@ var word_list = []
|
|
| 556 |
if (url == 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm') {
|
| 557 |
//newUrl = 'http://127.0.0.1:8125/assets/static/vision_wasm_internal.wasm' //For Android
|
| 558 |
newUrl = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm' // For Web
|
|
|
|
| 559 |
}
|
| 560 |
console.log("This was FETCHED: ", newUrl)
|
| 561 |
// Call the original fetch function with the new URL
|
| 562 |
return originalFetch(newUrl, init);
|
| 563 |
};
|
|
|
|
|
|
|
|
|
|
|
|
|
| 564 |
if ('speechSynthesis' in window) {
|
| 565 |
speechSynthesis.onvoiceschanged = () => {
|
| 566 |
populateVoiceList();
|
| 567 |
};
|
|
|
|
| 568 |
languageSelect.addEventListener('change', updateVoiceList);
|
|
|
|
| 569 |
rateInput.addEventListener('input', () => {
|
| 570 |
rateValue.textContent = rateInput.value;
|
| 571 |
});
|
|
|
|
| 572 |
pitchInput.addEventListener('input', () => {
|
| 573 |
pitchValue.textContent = pitchInput.value;
|
| 574 |
});
|
|
|
|
| 575 |
speakButton.addEventListener('click', () => {
|
| 576 |
const utterance = new SpeechSynthesisUtterance(textInput.value);
|
| 577 |
const selectedVoice = voices.find(voice => voice.name === voiceSelect.value);
|
|
@@ -580,14 +599,18 @@ var word_list = []
|
|
| 580 |
utterance.pitch = pitchInput.value;
|
| 581 |
speechSynthesis.speak(utterance);
|
| 582 |
});
|
|
|
|
| 583 |
populateVoiceList()
|
| 584 |
// Create an utterance object ⣿
|
|
|
|
| 585 |
} else {
|
| 586 |
speechSupported = false;
|
| 587 |
console.log('Text-to-speech not supported.');
|
|
|
|
| 588 |
populateTTWLangs()
|
| 589 |
populateTTWVoices()
|
| 590 |
}
|
|
|
|
| 591 |
function speak(toSpeak) {
|
| 592 |
console.log("speech api support", speechSupported)
|
| 593 |
console.log("condition: ", !speechSupported)
|
|
@@ -602,8 +625,10 @@ var word_list = []
|
|
| 602 |
audioPlayer.src = 'http://127.0.0.1:8125/speech?t=' + encodeURIComponent(toSpeak) + currSettings
|
| 603 |
console.log("Set src: ", audioPlayer.src)
|
| 604 |
}
|
|
|
|
| 605 |
audioPlayer.play() // Play the audio
|
| 606 |
.then(() => {
|
|
|
|
| 607 |
console.log('Audio is playing');
|
| 608 |
})
|
| 609 |
.catch(error => {
|
|
@@ -616,6 +641,9 @@ var word_list = []
|
|
| 616 |
console.log("Text to speech is now not supported")
|
| 617 |
}
|
| 618 |
}
|
|
|
|
|
|
|
|
|
|
| 619 |
function set_output_array(text) {
|
| 620 |
console.log(text)
|
| 621 |
word_list = text.split("");
|
|
@@ -625,12 +653,14 @@ var word_list = []
|
|
| 625 |
word_list = [];
|
| 626 |
textInput.value = ""
|
| 627 |
}
|
|
|
|
| 628 |
</script>
|
| 629 |
|
| 630 |
<script type="module">
|
| 631 |
document.getElementById("info-").addEventListener("click", switchPage.bind(null, "info-"));
|
| 632 |
document.getElementById("home-").addEventListener("click", switchPage.bind(null, "home-"));
|
| 633 |
document.getElementById("settings-").addEventListener("click", switchPage.bind(null, "settings-"));
|
|
|
|
| 634 |
//import { HandLandmarker, FilesetResolver } from "http://127.0.0.1:8125/assets/static/tasks-vision@0.10.0" // For Android
|
| 635 |
import { HandLandmarker, FilesetResolver } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0"; // For Web
|
| 636 |
let handLandmarker = undefined;
|
|
@@ -640,6 +670,7 @@ var word_list = []
|
|
| 640 |
var time_since_letter = 0
|
| 641 |
var last_letter_time = 0
|
| 642 |
var maxPercentage = 1
|
|
|
|
| 643 |
function inputFrequency() {
|
| 644 |
return document.getElementById('duration').value
|
| 645 |
}
|
|
@@ -648,10 +679,10 @@ var word_list = []
|
|
| 648 |
}
|
| 649 |
document.getElementById('durationValue').innerText = inputFrequency()
|
| 650 |
document.getElementById('repeatValue').innerText = repeatBuffer()
|
| 651 |
-
|
| 652 |
-
const letter_list = ["A", "B", "", "D", "E", "F", "G", "", "I", "J", "K", "L", "M", "N", "O", "P", "", "R", "S", "T", "U", "V", "W", "", "Y", "Z", ""]
|
| 653 |
-
const phrase_list = ["
|
| 654 |
-
|
| 655 |
// Before we can use HandLandmarker class we must wait for it to finish
|
| 656 |
// loading. Machine Learning models can be large and take a moment to
|
| 657 |
// get everything needed to run.
|
|
@@ -668,10 +699,12 @@ var word_list = []
|
|
| 668 |
});
|
| 669 |
};
|
| 670 |
createHandLandmarker();
|
|
|
|
| 671 |
//const MODEL_PATH = "http://127.0.0.1:8125/assets/static/model.tflite" // For Android
|
| 672 |
//const WORD_MODEL = "http://127.0.0.1:8125/assets/static/word.tflite" // For Android
|
| 673 |
const MODEL_PATH = "/exported" // For Web
|
| 674 |
const WORD_MODEL = "/word" // For Web
|
|
|
|
| 675 |
const letterDetector = tflite.loadTFLiteModel(MODEL_PATH);
|
| 676 |
const wordDetector = tflite.loadTFLiteModel(WORD_MODEL);
|
| 677 |
var objectDetector = letterDetector
|
|
@@ -741,18 +774,23 @@ var word_list = []
|
|
| 741 |
enableWebcamButton.style = "display:none"
|
| 742 |
document.getElementById("switch-camera").style.display = "block"
|
| 743 |
document.getElementById("mode-switch").style.display = "flex"
|
|
|
|
| 744 |
}
|
| 745 |
// getUsermedia parameters.
|
| 746 |
load_camera()
|
| 747 |
}
|
|
|
|
| 748 |
function switchPage(elem) {
|
| 749 |
prevSpeech = ""
|
|
|
|
| 750 |
var pH = document.getElementById("home-page")
|
| 751 |
var pI = document.getElementById("info-page")
|
| 752 |
var pS = document.getElementById("settings-page")
|
|
|
|
| 753 |
pH.style.display = "none"
|
| 754 |
pI.style.display = "none"
|
| 755 |
pS.style.display = "none"
|
|
|
|
| 756 |
document.getElementById(elem + "page").style.display = "block"
|
| 757 |
if (elem != "home-") {
|
| 758 |
webcamRunning = false
|
|
@@ -770,6 +808,7 @@ var word_list = []
|
|
| 770 |
context.clearRect(0, 0, canvas.width, canvas.height);
|
| 771 |
}
|
| 772 |
}
|
|
|
|
| 773 |
function load_camera() {
|
| 774 |
try {
|
| 775 |
var stream = video.srcObject;
|
|
@@ -785,6 +824,7 @@ var word_list = []
|
|
| 785 |
} catch (error) {
|
| 786 |
console.error(error.message);
|
| 787 |
}
|
|
|
|
| 788 |
const constraints = {
|
| 789 |
video: {
|
| 790 |
facingMode: video_facing_mode
|
|
@@ -821,14 +861,18 @@ var word_list = []
|
|
| 821 |
}
|
| 822 |
canvasCtx.save();
|
| 823 |
canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
|
|
|
|
| 824 |
if (is_first_run == 1) {
|
| 825 |
var elem_rect = document.getElementById("output_canvas").getBoundingClientRect()
|
| 826 |
console.log(elem_rect.height | 0);
|
| 827 |
document.getElementById("canvas_wrapper").style.height = (elem_rect.height | 0).toString() + "px"
|
|
|
|
| 828 |
is_first_run = 0
|
| 829 |
}
|
|
|
|
| 830 |
if (results.landmarks && results.handednesses[0]) {
|
| 831 |
var current_time = Math.round(Date.now())
|
|
|
|
| 832 |
if (results.handednesses[0][0].categoryName == "Left") {
|
| 833 |
if (!Array.from(document.getElementById('modeSelector').classList).includes('right')) {
|
| 834 |
annotateImage()
|
|
@@ -843,6 +887,8 @@ var word_list = []
|
|
| 843 |
var current_result = "_"
|
| 844 |
var previous_result = document.getElementById("predicted_result").innerText
|
| 845 |
document.getElementById("predicted_result").innerText = current_result
|
|
|
|
|
|
|
| 846 |
if (previous_result == current_result) {
|
| 847 |
if (current_time - last_letter_time > getNaturalLength(" ")) {
|
| 848 |
last_letter_time = current_time
|
|
@@ -861,6 +907,7 @@ var word_list = []
|
|
| 861 |
else {
|
| 862 |
canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width)
|
| 863 |
if (30 > calculateCanvasBrightness(canvasElement)) {
|
|
|
|
| 864 |
var current_result = "<"
|
| 865 |
var previous_result = document.getElementById("predicted_result").innerText
|
| 866 |
document.getElementById("predicted_result").innerText = current_result
|
|
@@ -879,9 +926,11 @@ var word_list = []
|
|
| 879 |
}
|
| 880 |
} else {
|
| 881 |
last_letter_time = Math.round(Date.now())
|
|
|
|
| 882 |
document.getElementById("predicted_result").style.width = String(0) + "%"
|
| 883 |
}
|
| 884 |
}
|
|
|
|
| 885 |
canvasCtx.restore();
|
| 886 |
// Keep predicting
|
| 887 |
if (webcamRunning === true) {
|
|
@@ -889,6 +938,7 @@ var word_list = []
|
|
| 889 |
}
|
| 890 |
}
|
| 891 |
function annotateImage(firstA = true) {
|
|
|
|
| 892 |
//console.log(results.landmarks)
|
| 893 |
if (results.landmarks[0]) {
|
| 894 |
x_array = []
|
|
@@ -901,10 +951,12 @@ var word_list = []
|
|
| 901 |
var min_y = Math.min(...y_array) * image_height
|
| 902 |
var max_x = Math.max(...x_array) * image_width
|
| 903 |
var max_y = Math.max(...y_array) * image_height
|
|
|
|
| 904 |
var sect_height = max_y - (min_y)
|
| 905 |
var sect_width = max_x - (min_x)
|
| 906 |
var center_x = (min_x + max_x) / 2
|
| 907 |
var center_y = (min_y + max_y) / 2
|
|
|
|
| 908 |
var sect_diameter = 50
|
| 909 |
if (sect_height > sect_width) {
|
| 910 |
sect_diameter = sect_height
|
|
@@ -914,6 +966,7 @@ var word_list = []
|
|
| 914 |
sect_diameter = sect_width
|
| 915 |
// console.log("sect_width", sect_diameter)
|
| 916 |
}
|
|
|
|
| 917 |
sect_diameter = sect_diameter + 50
|
| 918 |
var sect_radius = sect_diameter / 2
|
| 919 |
var crop_top = center_y - sect_radius
|
|
@@ -940,11 +993,15 @@ var word_list = []
|
|
| 940 |
canvasCtx.fillStyle = "#ffffff"
|
| 941 |
canvasCtx.fill()
|
| 942 |
}
|
|
|
|
| 943 |
} else {
|
| 944 |
canvasCtx.beginPath();
|
| 945 |
canvasCtx.rect(crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top);
|
| 946 |
canvasCtx.stroke();
|
| 947 |
}
|
|
|
|
|
|
|
|
|
|
| 948 |
}
|
| 949 |
/* for (const landmarks of results.multiHandLandmarks) {
|
| 950 |
drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS, {
|
|
@@ -968,76 +1025,102 @@ var word_list = []
|
|
| 968 |
drawConnection(hand[4], hand[3], 'rgb(0, 0, 255)', 5); // 4-3 (was red)
|
| 969 |
drawConnection(hand[3], hand[2], 'rgb(0, 0, 255)', 5); // 3-2 (was red)
|
| 970 |
drawConnection(hand[2], hand[1], 'rgb(0, 0, 255)', 5); // 2-1 (was red)
|
|
|
|
| 971 |
// Index connections
|
| 972 |
drawConnection(hand[8], hand[7], 'rgb(0, 255, 0)', 5); // 8-7
|
| 973 |
drawConnection(hand[7], hand[6], 'rgb(0, 255, 0)', 5); // 7-6
|
| 974 |
drawConnection(hand[6], hand[5], 'rgb(0, 255, 0)', 5); // 6-5
|
|
|
|
| 975 |
// Middle connections
|
| 976 |
drawConnection(hand[12], hand[11], 'rgb(255, 0, 0)', 5); // 12-11 (was blue)
|
| 977 |
drawConnection(hand[11], hand[10], 'rgb(255, 0, 0)', 5); // 11-10 (was blue)
|
| 978 |
drawConnection(hand[10], hand[9], 'rgb(255, 0, 0)', 5); // 10-9 (was blue)
|
|
|
|
| 979 |
// Ring connections
|
| 980 |
drawConnection(hand[16], hand[15], 'rgb(0, 255, 255)', 5); // 16-15 (was yellow)
|
| 981 |
drawConnection(hand[15], hand[14], 'rgb(0, 255, 255)', 5); // 15-14 (was yellow)
|
| 982 |
drawConnection(hand[14], hand[13], 'rgb(0, 255, 255)', 5); // 14-13 (was yellow)
|
|
|
|
| 983 |
// Pinky connections
|
| 984 |
drawConnection(hand[20], hand[19], 'rgb(255, 0, 255)', 5); // 20-19
|
| 985 |
drawConnection(hand[19], hand[18], 'rgb(255, 0, 255)', 5); // 19-18
|
| 986 |
drawConnection(hand[18], hand[17], 'rgb(255, 0, 255)', 5); // 18-17
|
|
|
|
| 987 |
drawConnection(hand[0], hand[1], 'rgb(200, 200, 200)', 5); // 0-1
|
| 988 |
drawConnection(hand[0], hand[5], 'rgb(200, 200, 200)', 5); // 0-5
|
| 989 |
drawConnection(hand[0], hand[17], 'rgb(200, 200, 200)', 5); // 0-17
|
| 990 |
drawConnection(hand[5], hand[9], 'rgb(200, 200, 200)', 5); // 5-9
|
| 991 |
drawConnection(hand[9], hand[13], 'rgb(200, 200, 200)', 5); // 9-13
|
| 992 |
drawConnection(hand[13], hand[17], 'rgb(200, 200, 200)', 5); // 13-17
|
|
|
|
| 993 |
// Thumb
|
| 994 |
drawLandmarks(canvasCtx, hand[2], '#ffe5b4'); // Thumb tip (2)
|
| 995 |
drawLandmarks(canvasCtx, hand[3], '#ffe5b4'); // Thumb base (3)
|
| 996 |
drawLandmarks(canvasCtx, hand[4], '#ffe5b4'); // Thumb base (4)
|
|
|
|
| 997 |
// Index
|
| 998 |
drawLandmarks(canvasCtx, hand[6], '#804080'); // Index tip (6)
|
| 999 |
drawLandmarks(canvasCtx, hand[7], '#804080'); // Index base (7)
|
| 1000 |
drawLandmarks(canvasCtx, hand[8], '#804080'); // Index base (8)
|
|
|
|
| 1001 |
// Middle
|
| 1002 |
drawLandmarks(canvasCtx, hand[10], '#ffcc00'); // Middle tip (10)
|
| 1003 |
drawLandmarks(canvasCtx, hand[11], '#ffcc00'); // Middle base (11)
|
| 1004 |
drawLandmarks(canvasCtx, hand[12], '#ffcc00'); // Middle base (12)
|
|
|
|
| 1005 |
// Ring
|
| 1006 |
drawLandmarks(canvasCtx, hand[14], '#30ff30'); // Ring tip (14)
|
| 1007 |
drawLandmarks(canvasCtx, hand[15], '#30ff30'); // Ring base (15)
|
| 1008 |
drawLandmarks(canvasCtx, hand[16], '#30ff30'); // Ring base (16)
|
|
|
|
| 1009 |
// Pinky
|
| 1010 |
drawLandmarks(canvasCtx, hand[18], '#1565c0'); // Pinky tip (18)
|
| 1011 |
drawLandmarks(canvasCtx, hand[19], '#1565c0'); // Pinky base (19)
|
| 1012 |
drawLandmarks(canvasCtx, hand[20], '#1565c0'); // Pinky base (20)
|
|
|
|
| 1013 |
drawLandmarks(canvasCtx, hand[0], '#ff3030'); // Wrist (0)
|
|
|
|
| 1014 |
drawLandmarks(canvasCtx, hand[1], '#ff3030'); // Palm base (1)
|
|
|
|
| 1015 |
drawLandmarks(canvasCtx, hand[5], '#ff3030'); // Index palm (5)
|
|
|
|
| 1016 |
drawLandmarks(canvasCtx, hand[9], '#ff3030'); // Middle palm (9)
|
|
|
|
| 1017 |
drawLandmarks(canvasCtx, hand[13], '#ff3030'); // Ring palm (13)
|
|
|
|
| 1018 |
drawLandmarks(canvasCtx, hand[17], '#ff3030'); // Pinky palm (17)
|
|
|
|
| 1019 |
// Crop Canvas
|
| 1020 |
if (firstA == true) {
|
| 1021 |
cropCanvas(canvasElement, crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top)
|
| 1022 |
}
|
| 1023 |
}
|
| 1024 |
// Add more drawing calls for each landmark collection as needed
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1025 |
//# sourceURL=pen.js
|
| 1026 |
}
|
|
|
|
|
|
|
| 1027 |
function iterate(x, y) {
|
| 1028 |
x_array.push(x.x)
|
| 1029 |
y_array.push(x.y)
|
| 1030 |
}
|
|
|
|
| 1031 |
const cropCanvas = (sourceCanvas, left, top, width, height) => {
|
| 1032 |
let destCanvas = document.createElement('canvas');
|
| 1033 |
destCanvas.width = 224;
|
| 1034 |
var cropAspectRatio = width / height;
|
|
|
|
| 1035 |
destCanvas.height = 224 / cropAspectRatio
|
| 1036 |
destCanvas.getContext("2d").drawImage(
|
| 1037 |
sourceCanvas,
|
| 1038 |
left, top, width, height, // source rect with content to crop
|
| 1039 |
0, 0, 224, destCanvas.height); // newCanvas, same size as source
|
| 1040 |
var predictionInput = tf.browser.fromPixels(destCanvas.getContext("2d").getImageData(0, 0, 224, 224))
|
|
|
|
| 1041 |
predict(tf.expandDims(predictionInput, 0));
|
| 1042 |
}
|
| 1043 |
function getNaturalLength(letter) {
|
|
@@ -1065,19 +1148,28 @@ var word_list = []
|
|
| 1065 |
function mapLogitsToPercentage(logits) {
|
| 1066 |
// Apply ceiling of 260
|
| 1067 |
const cappedLogits = logits.map(value => Math.min(value, 260));
|
|
|
|
| 1068 |
// Map the values from 0-260 to 0-100
|
| 1069 |
const mappedPercentages = cappedLogits.map(value => (value / 260) * 100);
|
|
|
|
| 1070 |
return mappedPercentages;
|
| 1071 |
}
|
|
|
|
|
|
|
| 1072 |
async function predict(inputTensor) {
|
|
|
|
| 1073 |
console.log(index_list[0])
|
| 1074 |
objectDetector.then(function (res) {
|
|
|
|
| 1075 |
var prediction = res.predict(inputTensor);
|
|
|
|
| 1076 |
var outputArray = prediction.dataSync(); // Get the output as an array
|
| 1077 |
var percentages = mapLogitsToPercentage(outputArray)
|
| 1078 |
maxPercentage = Math.max(...percentages)
|
| 1079 |
document.getElementById('predicted_result').style.opacity = (maxPercentage+30)+'%'
|
|
|
|
| 1080 |
var predictedClass = percentages.indexOf(Math.max(...percentages)); // Get the index
|
|
|
|
| 1081 |
var current_result = index_list[predictedClass]
|
| 1082 |
global_cres = current_result
|
| 1083 |
var previous_result = document.getElementById("predicted_result").innerText
|
|
@@ -1088,6 +1180,7 @@ var word_list = []
|
|
| 1088 |
}
|
| 1089 |
console.log("p:", previous_result, "c:", current_result)
|
| 1090 |
if (previous_result == current_result) {
|
|
|
|
| 1091 |
if (current_time - last_letter_time > getNaturalLength(current_result)) {
|
| 1092 |
last_letter_time = current_time
|
| 1093 |
word_list.push(current_result)
|
|
@@ -1103,18 +1196,25 @@ var word_list = []
|
|
| 1103 |
}, function (err) {
|
| 1104 |
console.log(err);
|
| 1105 |
});
|
|
|
|
| 1106 |
}
|
|
|
|
| 1107 |
function drawLandmarks(canvasCtx, landmarks, color) {
|
| 1108 |
var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
|
| 1109 |
var image_width = canvasElement.width
|
|
|
|
| 1110 |
canvasCtx.fillStyle = `rgb(${landmarks.z},${landmarks.z},${landmarks.z})`;
|
| 1111 |
canvasCtx.beginPath();
|
| 1112 |
canvasCtx.arc(landmarks.x * image_width, landmarks.y * image_height, 8, 0, 2 * Math.PI);
|
| 1113 |
canvasCtx.fill();
|
|
|
|
| 1114 |
}
|
|
|
|
| 1115 |
function drawConnection(startNode, endNode, strokeColor, strokeWidth) {
|
|
|
|
| 1116 |
var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
|
| 1117 |
var image_width = canvasElement.width
|
|
|
|
| 1118 |
canvasCtx.strokeStyle = strokeColor;
|
| 1119 |
canvasCtx.lineWidth = strokeWidth - 1;
|
| 1120 |
canvasCtx.beginPath();
|
|
@@ -1124,23 +1224,29 @@ var word_list = []
|
|
| 1124 |
}
|
| 1125 |
function calculateCanvasBrightness(canvas) {
|
| 1126 |
const context = canvas.getContext('2d', { willReadFrequently: true });
|
|
|
|
| 1127 |
// Get the image data from the canvas
|
| 1128 |
const imageData = context.getImageData(0, 0, canvas.width, canvas.height);
|
| 1129 |
const data = imageData.data;
|
|
|
|
| 1130 |
let totalBrightness = 0;
|
| 1131 |
let pixelCount = 0;
|
|
|
|
| 1132 |
// Loop through each pixel
|
| 1133 |
for (let i = 0; i < data.length; i += 4) {
|
| 1134 |
const r = data[i]; // Red
|
| 1135 |
const g = data[i + 1]; // Green
|
| 1136 |
const b = data[i + 2]; // Blue
|
|
|
|
| 1137 |
// Calculate brightness for this pixel
|
| 1138 |
const brightness = 0.299 * r + 0.587 * g + 0.114 * b;
|
| 1139 |
totalBrightness += brightness;
|
| 1140 |
pixelCount++;
|
| 1141 |
}
|
|
|
|
| 1142 |
// Calculate average brightness
|
| 1143 |
const averageBrightness = totalBrightness / pixelCount;
|
|
|
|
| 1144 |
return averageBrightness;
|
| 1145 |
}
|
| 1146 |
function triggerPulse() {
|
|
@@ -1148,6 +1254,7 @@ var word_list = []
|
|
| 1148 |
/* Apply after working on pulse css more
|
| 1149 |
const resultWrapper = document.querySelector('.wrapper_result');
|
| 1150 |
resultWrapper.classList.add('pulse');
|
|
|
|
| 1151 |
// Remove the class after the animation ends to allow it to be triggered again
|
| 1152 |
resultWrapper.addEventListener('animationend', () => {
|
| 1153 |
resultWrapper.classList.remove('pulse');
|
|
@@ -1163,4 +1270,3 @@ var word_list = []
|
|
| 1163 |
</body>
|
| 1164 |
|
| 1165 |
</html>
|
| 1166 |
-
|
|
|
|
| 3 |
SignSpeak - A Communicator For Signers
|
| 4 |
|
| 5 |
Copyright (C) 2025 Shantanu Khedkar
|
| 6 |
+
|
| 7 |
This program is free software: you can redistribute it and/or modify
|
| 8 |
it under the terms of the GNU General Public License as published by
|
| 9 |
the Free Software Foundation, either version 3 of the License, or
|
| 10 |
(at your option) any later version.
|
| 11 |
+
|
| 12 |
This program is distributed in the hope that it will be useful,
|
| 13 |
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
| 14 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
| 15 |
GNU General Public License for more details.
|
| 16 |
+
|
| 17 |
You should have received a copy of the GNU General Public License
|
| 18 |
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
| 19 |
-->
|
|
|
|
| 275 |
</form>
|
| 276 |
</section>
|
| 277 |
<div class="bottom-nav">
|
| 278 |
+
<div id="info-" class="bottom-nav-btn">
|
| 279 |
+
<svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px"
|
| 280 |
+
viewBox="0 0 512 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.-->
|
| 281 |
+
<path fill="#5d5d5d"
|
| 282 |
+
d="M256 512A256 256 0 1 0 256 0a256 256 0 1 0 0 512zM216 336l24 0 0-64-24 0c-13.3 0-24-10.7-24-24s10.7-24 24-24l48 0c13.3 0 24 10.7 24 24l0 88 8 0c13.3 0 24 10.7 24 24s-10.7 24-24 24l-80 0c-13.3 0-24-10.7-24-24s10.7-24 24-24zm40-208a32 32 0 1 1 0 64 32 32 0 1 1 0-64z" />
|
| 283 |
+
</svg>
|
| 284 |
+
</div>
|
| 285 |
+
<div id="home-" class="bottom-nav-btn center-btn">
|
| 286 |
+
<!--For Android
|
| 287 |
+
<img src="http://127.0.0.1:8125/assets/static/logo_sil.svg" height="48" width="48px"></img>
|
| 288 |
+
-->
|
| 289 |
+
<img src="/static/logo_sil.svg" height="48" width="48px"></img>
|
| 290 |
+
</div>
|
| 291 |
+
<div id="settings-" class="bottom-nav-btn">
|
| 292 |
+
<svg xmlns="http://www.w3.org/2000/svg" height="28px" width="28px"
|
| 293 |
+
viewBox="0 0 512 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.-->
|
| 294 |
+
<path fill="#5d5d5d"
|
| 295 |
+
d="M495.9 166.6c3.2 8.7 .5 18.4-6.4 24.6l-43.3 39.4c1.1 8.3 1.7 16.8 1.7 25.4s-.6 17.1-1.7 25.4l43.3 39.4c6.9 6.2 9.6 15.9 6.4 24.6c-4.4 11.9-9.7 23.3-15.8 34.3l-4.7 8.1c-6.6 11-14 21.4-22.1 31.2c-5.9 7.2-15.7 9.6-24.5 6.8l-55.7-17.7c-13.4 10.3-28.2 18.9-44 25.4l-12.5 57.1c-2 9.1-9 16.3-18.2 17.8c-13.8 2.3-28 3.5-42.5 3.5s-28.7-1.2-42.5-3.5c-9.2-1.5-16.2-8.7-18.2-17.8l-12.5-57.1c-15.8-6.5-30.6-15.1-44-25.4L83.1 425.9c-8.8 2.8-18.6 .3-24.5-6.8c-8.1-9.8-15.5-20.2-22.1-31.2l-4.7-8.1c-6.1-11-11.4-22.4-15.8-34.3c-3.2-8.7-.5-18.4 6.4-24.6l43.3-39.4C64.6 273.1 64 264.6 64 256s.6-17.1 1.7-25.4L22.4 191.2c-6.9-6.2-9.6-15.9-6.4-24.6c4.4-11.9 9.7-23.3 15.8-34.3l4.7-8.1c6.6-11 14-21.4 22.1-31.2c5.9-7.2 15.7-9.6 24.5-6.8l55.7 17.7c13.4-10.3 28.2-18.9 44-25.4l12.5-57.1c2-9.1 9-16.3 18.2-17.8C227.3 1.2 241.5 0 256 0s28.7 1.2 42.5 3.5c9.2 1.5 16.2 8.7 18.2 17.8l12.5 57.1c15.8 6.5 30.6 15.1 44 25.4l55.7-17.7c8.8-2.8 18.6-.3 24.5 6.8c8.1 9.8 15.5 20.2 22.1 31.2l4.7 8.1c6.1 11 11.4 22.4 15.8 34.3zM256 336a80 80 0 1 0 0-160 80 80 0 1 0 0 160z" />
|
| 296 |
+
</svg>
|
| 297 |
+
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
|
| 299 |
+
</div>
|
| 300 |
<script>
|
| 301 |
var word_list = []
|
| 302 |
var speechSupported = true
|
| 303 |
var prevSpeech = ""
|
| 304 |
var prevSettings = ""
|
| 305 |
+
|
| 306 |
logUI = document.getElementById("logUI")
|
| 307 |
const speechSynthesis = window.speechSynthesis;
|
| 308 |
const voiceSelect = document.getElementById('voice');
|
|
|
|
| 313 |
const pitchInput = document.getElementById('pitch');
|
| 314 |
const rateValue = document.getElementById('rateValue');
|
| 315 |
const pitchValue = document.getElementById('pitchValue');
|
| 316 |
+
|
| 317 |
let voices = [];
|
| 318 |
+
|
| 319 |
const languageNames = {
|
| 320 |
'en': '- English ',
|
| 321 |
'mra': '- Marathi',
|
|
|
|
| 411 |
'tt': 'Tatar',
|
| 412 |
'ur': 'Urdu',
|
| 413 |
'vi': 'Vietnamese'
|
| 414 |
+
|
| 415 |
};
|
| 416 |
function undo() {
|
| 417 |
word_list.pop()
|
|
|
|
| 433 |
if (!response.ok) {
|
| 434 |
throw new Error(`Response status: ${response.status}`);
|
| 435 |
}
|
| 436 |
+
|
| 437 |
var voicesList = await response.text();
|
| 438 |
voicesList = voicesList.split('\n')
|
| 439 |
voicesList.pop()
|
|
|
|
| 454 |
if (!response.ok) {
|
| 455 |
throw new Error(`Response status: ${response.status}`);
|
| 456 |
}
|
| 457 |
+
|
| 458 |
var langsList = await response.text();
|
| 459 |
langsList = langsList.split('\n')
|
| 460 |
langsList.pop()
|
|
|
|
| 478 |
} catch (error) {
|
| 479 |
console.error(error.message);
|
| 480 |
}
|
| 481 |
+
|
| 482 |
const optionNodes = Array.from(languageSelect.children);
|
| 483 |
const comparator = new Intl.Collator('en'.slice(0, 2)).compare;
|
| 484 |
optionNodes.sort((a, b) => comparator(a.textContent, b.textContent));
|
| 485 |
optionNodes.forEach((option) => languageSelect.appendChild(option));
|
| 486 |
languageSelect.children[0].selected = "selected"
|
| 487 |
}
|
| 488 |
+
|
| 489 |
function getBaseLanguageCode(lang) {
|
| 490 |
return lang.split('-').slice(0, 2).join('-');
|
| 491 |
}
|
|
|
|
| 495 |
function getModifierLanguageCode(lang) {
|
| 496 |
return lang.split('-').slice(1, 3).join('-');
|
| 497 |
}
|
| 498 |
+
|
| 499 |
function updateLanguageList() {
|
| 500 |
const uniqueLanguages = new Set(voices.map(voice => getBaseLanguageCode(voice.lang)));
|
| 501 |
languageSelect.innerHTML = '';
|
| 502 |
+
|
| 503 |
Object.keys(languageNames).forEach(lang => {
|
| 504 |
if (uniqueLanguages.has(lang)) {
|
| 505 |
const option = document.createElement('option');
|
|
|
|
| 508 |
languageSelect.appendChild(option);
|
| 509 |
}
|
| 510 |
});
|
| 511 |
+
|
| 512 |
uniqueLanguages.forEach(lang => {
|
| 513 |
if (!languageNames[lang]) {
|
| 514 |
if (!languageNames[getBaseBaseLanguageCode(lang)]) {
|
|
|
|
| 525 |
}
|
| 526 |
}
|
| 527 |
});
|
| 528 |
+
|
| 529 |
const optionNodes = Array.from(languageSelect.children);
|
| 530 |
const comparator = new Intl.Collator('en'.slice(0, 2)).compare;
|
| 531 |
optionNodes.sort((a, b) => comparator(a.textContent, b.textContent));
|
| 532 |
optionNodes.forEach((option) => languageSelect.appendChild(option));
|
| 533 |
languageSelect.children[0].selected = "selected"
|
| 534 |
}
|
| 535 |
+
|
| 536 |
function updateVoiceList() {
|
| 537 |
const selectedLanguage = languageSelect.value;
|
| 538 |
voiceSelect.innerHTML = '';
|
| 539 |
+
|
| 540 |
voices.forEach((voice) => {
|
| 541 |
if (getBaseLanguageCode(voice.lang) === selectedLanguage) {
|
| 542 |
const option = document.createElement('option');
|
|
|
|
| 555 |
logUI.appendChild(span);
|
| 556 |
logUI.appendChild(document.createElement('br')); // Add a line break
|
| 557 |
}
|
| 558 |
+
|
| 559 |
const originalFetch = window.fetch;
|
| 560 |
+
|
| 561 |
// Override the fetch function
|
| 562 |
window.fetch = async function (input, init) {
|
| 563 |
// Convert input to URL if it's a Request object
|
|
|
|
| 566 |
if (url == 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm') {
|
| 567 |
//newUrl = 'http://127.0.0.1:8125/assets/static/vision_wasm_internal.wasm' //For Android
|
| 568 |
newUrl = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.wasm' // For Web
|
| 569 |
+
|
| 570 |
}
|
| 571 |
console.log("This was FETCHED: ", newUrl)
|
| 572 |
// Call the original fetch function with the new URL
|
| 573 |
return originalFetch(newUrl, init);
|
| 574 |
};
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
|
| 578 |
+
|
| 579 |
if ('speechSynthesis' in window) {
|
| 580 |
speechSynthesis.onvoiceschanged = () => {
|
| 581 |
populateVoiceList();
|
| 582 |
};
|
| 583 |
+
|
| 584 |
languageSelect.addEventListener('change', updateVoiceList);
|
| 585 |
+
|
| 586 |
rateInput.addEventListener('input', () => {
|
| 587 |
rateValue.textContent = rateInput.value;
|
| 588 |
});
|
| 589 |
+
|
| 590 |
pitchInput.addEventListener('input', () => {
|
| 591 |
pitchValue.textContent = pitchInput.value;
|
| 592 |
});
|
| 593 |
+
|
| 594 |
speakButton.addEventListener('click', () => {
|
| 595 |
const utterance = new SpeechSynthesisUtterance(textInput.value);
|
| 596 |
const selectedVoice = voices.find(voice => voice.name === voiceSelect.value);
|
|
|
|
| 599 |
utterance.pitch = pitchInput.value;
|
| 600 |
speechSynthesis.speak(utterance);
|
| 601 |
});
|
| 602 |
+
|
| 603 |
populateVoiceList()
|
| 604 |
// Create an utterance object ⣿
|
| 605 |
+
|
| 606 |
} else {
|
| 607 |
speechSupported = false;
|
| 608 |
console.log('Text-to-speech not supported.');
|
| 609 |
+
|
| 610 |
populateTTWLangs()
|
| 611 |
populateTTWVoices()
|
| 612 |
}
|
| 613 |
+
|
| 614 |
function speak(toSpeak) {
|
| 615 |
console.log("speech api support", speechSupported)
|
| 616 |
console.log("condition: ", !speechSupported)
|
|
|
|
| 625 |
audioPlayer.src = 'http://127.0.0.1:8125/speech?t=' + encodeURIComponent(toSpeak) + currSettings
|
| 626 |
console.log("Set src: ", audioPlayer.src)
|
| 627 |
}
|
| 628 |
+
|
| 629 |
audioPlayer.play() // Play the audio
|
| 630 |
.then(() => {
|
| 631 |
+
|
| 632 |
console.log('Audio is playing');
|
| 633 |
})
|
| 634 |
.catch(error => {
|
|
|
|
| 641 |
console.log("Text to speech is now not supported")
|
| 642 |
}
|
| 643 |
}
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
|
| 647 |
function set_output_array(text) {
|
| 648 |
console.log(text)
|
| 649 |
word_list = text.split("");
|
|
|
|
| 653 |
word_list = [];
|
| 654 |
textInput.value = ""
|
| 655 |
}
|
| 656 |
+
|
| 657 |
</script>
|
| 658 |
|
| 659 |
<script type="module">
|
| 660 |
document.getElementById("info-").addEventListener("click", switchPage.bind(null, "info-"));
|
| 661 |
document.getElementById("home-").addEventListener("click", switchPage.bind(null, "home-"));
|
| 662 |
document.getElementById("settings-").addEventListener("click", switchPage.bind(null, "settings-"));
|
| 663 |
+
|
| 664 |
//import { HandLandmarker, FilesetResolver } from "http://127.0.0.1:8125/assets/static/tasks-vision@0.10.0" // For Android
|
| 665 |
import { HandLandmarker, FilesetResolver } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0"; // For Web
|
| 666 |
let handLandmarker = undefined;
|
|
|
|
| 670 |
var time_since_letter = 0
|
| 671 |
var last_letter_time = 0
|
| 672 |
var maxPercentage = 1
|
| 673 |
+
|
| 674 |
function inputFrequency() {
|
| 675 |
return document.getElementById('duration').value
|
| 676 |
}
|
|
|
|
| 679 |
}
|
| 680 |
document.getElementById('durationValue').innerText = inputFrequency()
|
| 681 |
document.getElementById('repeatValue').innerText = repeatBuffer()
|
| 682 |
+
var is_first_run = 1
|
| 683 |
+
const letter_list = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "#"]
|
| 684 |
+
const phrase_list = [" Good", " Hello", " How're", " I'm", " Thanks", " You"]
|
| 685 |
+
var index_list = letter_list
|
| 686 |
// Before we can use HandLandmarker class we must wait for it to finish
|
| 687 |
// loading. Machine Learning models can be large and take a moment to
|
| 688 |
// get everything needed to run.
|
|
|
|
| 699 |
});
|
| 700 |
};
|
| 701 |
createHandLandmarker();
|
| 702 |
+
|
| 703 |
//const MODEL_PATH = "http://127.0.0.1:8125/assets/static/model.tflite" // For Android
|
| 704 |
//const WORD_MODEL = "http://127.0.0.1:8125/assets/static/word.tflite" // For Android
|
| 705 |
const MODEL_PATH = "/exported" // For Web
|
| 706 |
const WORD_MODEL = "/word" // For Web
|
| 707 |
+
|
| 708 |
const letterDetector = tflite.loadTFLiteModel(MODEL_PATH);
|
| 709 |
const wordDetector = tflite.loadTFLiteModel(WORD_MODEL);
|
| 710 |
var objectDetector = letterDetector
|
|
|
|
| 774 |
enableWebcamButton.style = "display:none"
|
| 775 |
document.getElementById("switch-camera").style.display = "block"
|
| 776 |
document.getElementById("mode-switch").style.display = "flex"
|
| 777 |
+
|
| 778 |
}
|
| 779 |
// getUsermedia parameters.
|
| 780 |
load_camera()
|
| 781 |
}
|
| 782 |
+
|
| 783 |
function switchPage(elem) {
|
| 784 |
prevSpeech = ""
|
| 785 |
+
|
| 786 |
var pH = document.getElementById("home-page")
|
| 787 |
var pI = document.getElementById("info-page")
|
| 788 |
var pS = document.getElementById("settings-page")
|
| 789 |
+
|
| 790 |
pH.style.display = "none"
|
| 791 |
pI.style.display = "none"
|
| 792 |
pS.style.display = "none"
|
| 793 |
+
|
| 794 |
document.getElementById(elem + "page").style.display = "block"
|
| 795 |
if (elem != "home-") {
|
| 796 |
webcamRunning = false
|
|
|
|
| 808 |
context.clearRect(0, 0, canvas.width, canvas.height);
|
| 809 |
}
|
| 810 |
}
|
| 811 |
+
|
| 812 |
function load_camera() {
|
| 813 |
try {
|
| 814 |
var stream = video.srcObject;
|
|
|
|
| 824 |
} catch (error) {
|
| 825 |
console.error(error.message);
|
| 826 |
}
|
| 827 |
+
|
| 828 |
const constraints = {
|
| 829 |
video: {
|
| 830 |
facingMode: video_facing_mode
|
|
|
|
| 861 |
}
|
| 862 |
canvasCtx.save();
|
| 863 |
canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
|
| 864 |
+
|
| 865 |
if (is_first_run == 1) {
|
| 866 |
var elem_rect = document.getElementById("output_canvas").getBoundingClientRect()
|
| 867 |
console.log(elem_rect.height | 0);
|
| 868 |
document.getElementById("canvas_wrapper").style.height = (elem_rect.height | 0).toString() + "px"
|
| 869 |
+
|
| 870 |
is_first_run = 0
|
| 871 |
}
|
| 872 |
+
|
| 873 |
if (results.landmarks && results.handednesses[0]) {
|
| 874 |
var current_time = Math.round(Date.now())
|
| 875 |
+
|
| 876 |
if (results.handednesses[0][0].categoryName == "Left") {
|
| 877 |
if (!Array.from(document.getElementById('modeSelector').classList).includes('right')) {
|
| 878 |
annotateImage()
|
|
|
|
| 887 |
var current_result = "_"
|
| 888 |
var previous_result = document.getElementById("predicted_result").innerText
|
| 889 |
document.getElementById("predicted_result").innerText = current_result
|
| 890 |
+
|
| 891 |
+
|
| 892 |
if (previous_result == current_result) {
|
| 893 |
if (current_time - last_letter_time > getNaturalLength(" ")) {
|
| 894 |
last_letter_time = current_time
|
|
|
|
| 907 |
else {
|
| 908 |
canvasCtx.drawImage(video, 0, 0, canvasElement.width, (video.videoHeight / video.videoWidth) * canvasElement.width)
|
| 909 |
if (30 > calculateCanvasBrightness(canvasElement)) {
|
| 910 |
+
|
| 911 |
var current_result = "<"
|
| 912 |
var previous_result = document.getElementById("predicted_result").innerText
|
| 913 |
document.getElementById("predicted_result").innerText = current_result
|
|
|
|
| 926 |
}
|
| 927 |
} else {
|
| 928 |
last_letter_time = Math.round(Date.now())
|
| 929 |
+
|
| 930 |
document.getElementById("predicted_result").style.width = String(0) + "%"
|
| 931 |
}
|
| 932 |
}
|
| 933 |
+
|
| 934 |
canvasCtx.restore();
|
| 935 |
// Keep predicting
|
| 936 |
if (webcamRunning === true) {
|
|
|
|
| 938 |
}
|
| 939 |
}
|
| 940 |
function annotateImage(firstA = true) {
|
| 941 |
+
|
| 942 |
//console.log(results.landmarks)
|
| 943 |
if (results.landmarks[0]) {
|
| 944 |
x_array = []
|
|
|
|
| 951 |
var min_y = Math.min(...y_array) * image_height
|
| 952 |
var max_x = Math.max(...x_array) * image_width
|
| 953 |
var max_y = Math.max(...y_array) * image_height
|
| 954 |
+
|
| 955 |
var sect_height = max_y - (min_y)
|
| 956 |
var sect_width = max_x - (min_x)
|
| 957 |
var center_x = (min_x + max_x) / 2
|
| 958 |
var center_y = (min_y + max_y) / 2
|
| 959 |
+
|
| 960 |
var sect_diameter = 50
|
| 961 |
if (sect_height > sect_width) {
|
| 962 |
sect_diameter = sect_height
|
|
|
|
| 966 |
sect_diameter = sect_width
|
| 967 |
// console.log("sect_width", sect_diameter)
|
| 968 |
}
|
| 969 |
+
|
| 970 |
sect_diameter = sect_diameter + 50
|
| 971 |
var sect_radius = sect_diameter / 2
|
| 972 |
var crop_top = center_y - sect_radius
|
|
|
|
| 993 |
canvasCtx.fillStyle = "#ffffff"
|
| 994 |
canvasCtx.fill()
|
| 995 |
}
|
| 996 |
+
|
| 997 |
} else {
|
| 998 |
canvasCtx.beginPath();
|
| 999 |
canvasCtx.rect(crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top);
|
| 1000 |
canvasCtx.stroke();
|
| 1001 |
}
|
| 1002 |
+
|
| 1003 |
+
|
| 1004 |
+
|
| 1005 |
}
|
| 1006 |
/* for (const landmarks of results.multiHandLandmarks) {
|
| 1007 |
drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS, {
|
|
|
|
| 1025 |
drawConnection(hand[4], hand[3], 'rgb(0, 0, 255)', 5); // 4-3 (was red)
|
| 1026 |
drawConnection(hand[3], hand[2], 'rgb(0, 0, 255)', 5); // 3-2 (was red)
|
| 1027 |
drawConnection(hand[2], hand[1], 'rgb(0, 0, 255)', 5); // 2-1 (was red)
|
| 1028 |
+
|
| 1029 |
// Index connections
|
| 1030 |
drawConnection(hand[8], hand[7], 'rgb(0, 255, 0)', 5); // 8-7
|
| 1031 |
drawConnection(hand[7], hand[6], 'rgb(0, 255, 0)', 5); // 7-6
|
| 1032 |
drawConnection(hand[6], hand[5], 'rgb(0, 255, 0)', 5); // 6-5
|
| 1033 |
+
|
| 1034 |
// Middle connections
|
| 1035 |
drawConnection(hand[12], hand[11], 'rgb(255, 0, 0)', 5); // 12-11 (was blue)
|
| 1036 |
drawConnection(hand[11], hand[10], 'rgb(255, 0, 0)', 5); // 11-10 (was blue)
|
| 1037 |
drawConnection(hand[10], hand[9], 'rgb(255, 0, 0)', 5); // 10-9 (was blue)
|
| 1038 |
+
|
| 1039 |
// Ring connections
|
| 1040 |
drawConnection(hand[16], hand[15], 'rgb(0, 255, 255)', 5); // 16-15 (was yellow)
|
| 1041 |
drawConnection(hand[15], hand[14], 'rgb(0, 255, 255)', 5); // 15-14 (was yellow)
|
| 1042 |
drawConnection(hand[14], hand[13], 'rgb(0, 255, 255)', 5); // 14-13 (was yellow)
|
| 1043 |
+
|
| 1044 |
// Pinky connections
|
| 1045 |
drawConnection(hand[20], hand[19], 'rgb(255, 0, 255)', 5); // 20-19
|
| 1046 |
drawConnection(hand[19], hand[18], 'rgb(255, 0, 255)', 5); // 19-18
|
| 1047 |
drawConnection(hand[18], hand[17], 'rgb(255, 0, 255)', 5); // 18-17
|
| 1048 |
+
|
| 1049 |
drawConnection(hand[0], hand[1], 'rgb(200, 200, 200)', 5); // 0-1
|
| 1050 |
drawConnection(hand[0], hand[5], 'rgb(200, 200, 200)', 5); // 0-5
|
| 1051 |
drawConnection(hand[0], hand[17], 'rgb(200, 200, 200)', 5); // 0-17
|
| 1052 |
drawConnection(hand[5], hand[9], 'rgb(200, 200, 200)', 5); // 5-9
|
| 1053 |
drawConnection(hand[9], hand[13], 'rgb(200, 200, 200)', 5); // 9-13
|
| 1054 |
drawConnection(hand[13], hand[17], 'rgb(200, 200, 200)', 5); // 13-17
|
| 1055 |
+
|
| 1056 |
// Thumb
|
| 1057 |
drawLandmarks(canvasCtx, hand[2], '#ffe5b4'); // Thumb tip (2)
|
| 1058 |
drawLandmarks(canvasCtx, hand[3], '#ffe5b4'); // Thumb base (3)
|
| 1059 |
drawLandmarks(canvasCtx, hand[4], '#ffe5b4'); // Thumb base (4)
|
| 1060 |
+
|
| 1061 |
// Index
|
| 1062 |
drawLandmarks(canvasCtx, hand[6], '#804080'); // Index tip (6)
|
| 1063 |
drawLandmarks(canvasCtx, hand[7], '#804080'); // Index base (7)
|
| 1064 |
drawLandmarks(canvasCtx, hand[8], '#804080'); // Index base (8)
|
| 1065 |
+
|
| 1066 |
// Middle
|
| 1067 |
drawLandmarks(canvasCtx, hand[10], '#ffcc00'); // Middle tip (10)
|
| 1068 |
drawLandmarks(canvasCtx, hand[11], '#ffcc00'); // Middle base (11)
|
| 1069 |
drawLandmarks(canvasCtx, hand[12], '#ffcc00'); // Middle base (12)
|
| 1070 |
+
|
| 1071 |
// Ring
|
| 1072 |
drawLandmarks(canvasCtx, hand[14], '#30ff30'); // Ring tip (14)
|
| 1073 |
drawLandmarks(canvasCtx, hand[15], '#30ff30'); // Ring base (15)
|
| 1074 |
drawLandmarks(canvasCtx, hand[16], '#30ff30'); // Ring base (16)
|
| 1075 |
+
|
| 1076 |
// Pinky
|
| 1077 |
drawLandmarks(canvasCtx, hand[18], '#1565c0'); // Pinky tip (18)
|
| 1078 |
drawLandmarks(canvasCtx, hand[19], '#1565c0'); // Pinky base (19)
|
| 1079 |
drawLandmarks(canvasCtx, hand[20], '#1565c0'); // Pinky base (20)
|
| 1080 |
+
|
| 1081 |
drawLandmarks(canvasCtx, hand[0], '#ff3030'); // Wrist (0)
|
| 1082 |
+
|
| 1083 |
drawLandmarks(canvasCtx, hand[1], '#ff3030'); // Palm base (1)
|
| 1084 |
+
|
| 1085 |
drawLandmarks(canvasCtx, hand[5], '#ff3030'); // Index palm (5)
|
| 1086 |
+
|
| 1087 |
drawLandmarks(canvasCtx, hand[9], '#ff3030'); // Middle palm (9)
|
| 1088 |
+
|
| 1089 |
drawLandmarks(canvasCtx, hand[13], '#ff3030'); // Ring palm (13)
|
| 1090 |
+
|
| 1091 |
drawLandmarks(canvasCtx, hand[17], '#ff3030'); // Pinky palm (17)
|
| 1092 |
+
|
| 1093 |
// Crop Canvas
|
| 1094 |
if (firstA == true) {
|
| 1095 |
cropCanvas(canvasElement, crop_left, crop_top, crop_right - crop_left, crop_bottom - crop_top)
|
| 1096 |
}
|
| 1097 |
}
|
| 1098 |
// Add more drawing calls for each landmark collection as needed
|
| 1099 |
+
|
| 1100 |
+
|
| 1101 |
+
|
| 1102 |
+
|
| 1103 |
//# sourceURL=pen.js
|
| 1104 |
}
|
| 1105 |
+
|
| 1106 |
+
|
| 1107 |
function iterate(x, y) {
|
| 1108 |
x_array.push(x.x)
|
| 1109 |
y_array.push(x.y)
|
| 1110 |
}
|
| 1111 |
+
|
| 1112 |
const cropCanvas = (sourceCanvas, left, top, width, height) => {
|
| 1113 |
let destCanvas = document.createElement('canvas');
|
| 1114 |
destCanvas.width = 224;
|
| 1115 |
var cropAspectRatio = width / height;
|
| 1116 |
+
|
| 1117 |
destCanvas.height = 224 / cropAspectRatio
|
| 1118 |
destCanvas.getContext("2d").drawImage(
|
| 1119 |
sourceCanvas,
|
| 1120 |
left, top, width, height, // source rect with content to crop
|
| 1121 |
0, 0, 224, destCanvas.height); // newCanvas, same size as source
|
| 1122 |
var predictionInput = tf.browser.fromPixels(destCanvas.getContext("2d").getImageData(0, 0, 224, 224))
|
| 1123 |
+
|
| 1124 |
predict(tf.expandDims(predictionInput, 0));
|
| 1125 |
}
|
| 1126 |
function getNaturalLength(letter) {
|
|
|
|
| 1148 |
function mapLogitsToPercentage(logits) {
|
| 1149 |
// Apply ceiling of 260
|
| 1150 |
const cappedLogits = logits.map(value => Math.min(value, 260));
|
| 1151 |
+
|
| 1152 |
// Map the values from 0-260 to 0-100
|
| 1153 |
const mappedPercentages = cappedLogits.map(value => (value / 260) * 100);
|
| 1154 |
+
|
| 1155 |
return mappedPercentages;
|
| 1156 |
}
|
| 1157 |
+
|
| 1158 |
+
|
| 1159 |
async function predict(inputTensor) {
|
| 1160 |
+
|
| 1161 |
console.log(index_list[0])
|
| 1162 |
objectDetector.then(function (res) {
|
| 1163 |
+
|
| 1164 |
var prediction = res.predict(inputTensor);
|
| 1165 |
+
|
| 1166 |
var outputArray = prediction.dataSync(); // Get the output as an array
|
| 1167 |
var percentages = mapLogitsToPercentage(outputArray)
|
| 1168 |
maxPercentage = Math.max(...percentages)
|
| 1169 |
document.getElementById('predicted_result').style.opacity = (maxPercentage+30)+'%'
|
| 1170 |
+
|
| 1171 |
var predictedClass = percentages.indexOf(Math.max(...percentages)); // Get the index
|
| 1172 |
+
|
| 1173 |
var current_result = index_list[predictedClass]
|
| 1174 |
global_cres = current_result
|
| 1175 |
var previous_result = document.getElementById("predicted_result").innerText
|
|
|
|
| 1180 |
}
|
| 1181 |
console.log("p:", previous_result, "c:", current_result)
|
| 1182 |
if (previous_result == current_result) {
|
| 1183 |
+
|
| 1184 |
if (current_time - last_letter_time > getNaturalLength(current_result)) {
|
| 1185 |
last_letter_time = current_time
|
| 1186 |
word_list.push(current_result)
|
|
|
|
| 1196 |
}, function (err) {
|
| 1197 |
console.log(err);
|
| 1198 |
});
|
| 1199 |
+
|
| 1200 |
}
|
| 1201 |
+
|
| 1202 |
function drawLandmarks(canvasCtx, landmarks, color) {
|
| 1203 |
var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
|
| 1204 |
var image_width = canvasElement.width
|
| 1205 |
+
|
| 1206 |
canvasCtx.fillStyle = `rgb(${landmarks.z},${landmarks.z},${landmarks.z})`;
|
| 1207 |
canvasCtx.beginPath();
|
| 1208 |
canvasCtx.arc(landmarks.x * image_width, landmarks.y * image_height, 8, 0, 2 * Math.PI);
|
| 1209 |
canvasCtx.fill();
|
| 1210 |
+
|
| 1211 |
}
|
| 1212 |
+
|
| 1213 |
function drawConnection(startNode, endNode, strokeColor, strokeWidth) {
|
| 1214 |
+
|
| 1215 |
var image_height = (video.videoHeight / video.videoWidth) * canvasElement.width
|
| 1216 |
var image_width = canvasElement.width
|
| 1217 |
+
|
| 1218 |
canvasCtx.strokeStyle = strokeColor;
|
| 1219 |
canvasCtx.lineWidth = strokeWidth - 1;
|
| 1220 |
canvasCtx.beginPath();
|
|
|
|
| 1224 |
}
|
| 1225 |
function calculateCanvasBrightness(canvas) {
|
| 1226 |
const context = canvas.getContext('2d', { willReadFrequently: true });
|
| 1227 |
+
|
| 1228 |
// Get the image data from the canvas
|
| 1229 |
const imageData = context.getImageData(0, 0, canvas.width, canvas.height);
|
| 1230 |
const data = imageData.data;
|
| 1231 |
+
|
| 1232 |
let totalBrightness = 0;
|
| 1233 |
let pixelCount = 0;
|
| 1234 |
+
|
| 1235 |
// Loop through each pixel
|
| 1236 |
for (let i = 0; i < data.length; i += 4) {
|
| 1237 |
const r = data[i]; // Red
|
| 1238 |
const g = data[i + 1]; // Green
|
| 1239 |
const b = data[i + 2]; // Blue
|
| 1240 |
+
|
| 1241 |
// Calculate brightness for this pixel
|
| 1242 |
const brightness = 0.299 * r + 0.587 * g + 0.114 * b;
|
| 1243 |
totalBrightness += brightness;
|
| 1244 |
pixelCount++;
|
| 1245 |
}
|
| 1246 |
+
|
| 1247 |
// Calculate average brightness
|
| 1248 |
const averageBrightness = totalBrightness / pixelCount;
|
| 1249 |
+
|
| 1250 |
return averageBrightness;
|
| 1251 |
}
|
| 1252 |
function triggerPulse() {
|
|
|
|
| 1254 |
/* Apply after working on pulse css more
|
| 1255 |
const resultWrapper = document.querySelector('.wrapper_result');
|
| 1256 |
resultWrapper.classList.add('pulse');
|
| 1257 |
+
|
| 1258 |
// Remove the class after the animation ends to allow it to be triggered again
|
| 1259 |
resultWrapper.addEventListener('animationend', () => {
|
| 1260 |
resultWrapper.classList.remove('pulse');
|
|
|
|
| 1270 |
</body>
|
| 1271 |
|
| 1272 |
</html>
|
|
|