| function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { |
| try { |
| var info = gen[key](arg); |
| var value = info.value; |
| } catch (error) { |
| reject(error); |
| return; |
| } |
| if (info.done) { |
| resolve(value); |
| } else { |
| Promise.resolve(value).then(_next, _throw); |
| } |
| } |
| function _async_to_generator(fn) { |
| return function() { |
| var self = this, args = arguments; |
| return new Promise(function(resolve, reject) { |
| var gen = fn.apply(self, args); |
| function _next(value) { |
| asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); |
| } |
| function _throw(err) { |
| asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); |
| } |
| _next(undefined); |
| }); |
| }; |
| } |
| function _class_call_check(instance, Constructor) { |
| if (!(instance instanceof Constructor)) { |
| throw new TypeError("Cannot call a class as a function"); |
| } |
| } |
| function _defineProperties(target, props) { |
| for(var i = 0; i < props.length; i++){ |
| var descriptor = props[i]; |
| descriptor.enumerable = descriptor.enumerable || false; |
| descriptor.configurable = true; |
| if ("value" in descriptor) descriptor.writable = true; |
| Object.defineProperty(target, descriptor.key, descriptor); |
| } |
| } |
| function _create_class(Constructor, protoProps, staticProps) { |
| if (protoProps) _defineProperties(Constructor.prototype, protoProps); |
| if (staticProps) _defineProperties(Constructor, staticProps); |
| return Constructor; |
| } |
| function _define_property(obj, key, value) { |
| if (key in obj) { |
| Object.defineProperty(obj, key, { |
| value: value, |
| enumerable: true, |
| configurable: true, |
| writable: true |
| }); |
| } else { |
| obj[key] = value; |
| } |
| return obj; |
| } |
| function _object_spread(target) { |
| for(var i = 1; i < arguments.length; i++){ |
| var source = arguments[i] != null ? arguments[i] : {}; |
| var ownKeys = Object.keys(source); |
| if (typeof Object.getOwnPropertySymbols === "function") { |
| ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) { |
| return Object.getOwnPropertyDescriptor(source, sym).enumerable; |
| })); |
| } |
| ownKeys.forEach(function(key) { |
| _define_property(target, key, source[key]); |
| }); |
| } |
| return target; |
| } |
| function _ts_generator(thisArg, body) { |
| var f, y, t, g, _ = { |
| label: 0, |
| sent: function() { |
| if (t[0] & 1) throw t[1]; |
| return t[1]; |
| }, |
| trys: [], |
| ops: [] |
| }; |
| return g = { |
| next: verb(0), |
| "throw": verb(1), |
| "return": verb(2) |
| }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { |
| return this; |
| }), g; |
| function verb(n) { |
| return function(v) { |
| return step([ |
| n, |
| v |
| ]); |
| }; |
| } |
| function step(op) { |
| if (f) throw new TypeError("Generator is already executing."); |
| while(_)try { |
| if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; |
| if (y = 0, t) op = [ |
| op[0] & 2, |
| t.value |
| ]; |
| switch(op[0]){ |
| case 0: |
| case 1: |
| t = op; |
| break; |
| case 4: |
| _.label++; |
| return { |
| value: op[1], |
| done: false |
| }; |
| case 5: |
| _.label++; |
| y = op[1]; |
| op = [ |
| 0 |
| ]; |
| continue; |
| case 7: |
| op = _.ops.pop(); |
| _.trys.pop(); |
| continue; |
| default: |
| if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { |
| _ = 0; |
| continue; |
| } |
| if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) { |
| _.label = op[1]; |
| break; |
| } |
| if (op[0] === 6 && _.label < t[1]) { |
| _.label = t[1]; |
| t = op; |
| break; |
| } |
| if (t && _.label < t[2]) { |
| _.label = t[2]; |
| _.ops.push(op); |
| break; |
| } |
| if (t[2]) _.ops.pop(); |
| _.trys.pop(); |
| continue; |
| } |
| op = body.call(thisArg, _); |
| } catch (e) { |
| op = [ |
| 6, |
| e |
| ]; |
| y = 0; |
| } finally{ |
| f = t = 0; |
| } |
| if (op[0] & 5) throw op[1]; |
| return { |
| value: op[0] ? op[1] : void 0, |
| done: true |
| }; |
| } |
| } |
| import * as THREE from 'three'; |
| import { GLTFLoader } from 'three/loaders/GLTFLoader.js'; |
| import { HandLandmarker, FilesetResolver } from 'https://esm.sh/@mediapipe/tasks-vision@0.10.14'; |
| import { AudioManager } from './audioManager.js'; |
| import { SpeechManager } from './SpeechManager.js'; |
| export var Game = function() { |
| "use strict"; |
| function Game(renderDiv) { |
| var _this = this; |
| _class_call_check(this, Game); |
| this.renderDiv = renderDiv; |
| this.scene = null; |
| this.camera = null; |
| this.renderer = null; |
| this.videoElement = null; |
| this.handLandmarker = null; |
| this.lastVideoTime = -1; |
| this.hands = []; |
| this.handLineMaterial = null; |
| this.fingertipMaterialHand1 = null; |
| this.fingertipMaterialHand2 = null; |
| this.fingertipLandmarkIndices = [ |
| 0, |
| 4, |
| 8, |
| 12, |
| 16, |
| 20 |
| ]; |
| this.handConnections = null; |
| |
| this.gameState = 'loading'; |
| this.gameOverText = null; |
| this.clock = new THREE.Clock(); |
| this.audioManager = new AudioManager(); |
| this.lastLandmarkPositions = [ |
| [], |
| [] |
| ]; |
| this.smoothingFactor = 0.4; |
| this.loadedModels = {}; |
| this.pandaModel = null; |
| this.animationMixer = null; |
| this.animationClips = []; |
| this.animationActions = {}; |
| this.currentAction = null; |
| this.speechManager = null; |
| this.speechBubble = null; |
| this.speechBubbleTimeout = null; |
| this.isSpeechActive = false; |
| this.grabbingHandIndex = -1; |
| this.pickedUpModel = null; |
| this.modelDragOffset = new THREE.Vector3(); |
| this.modelGrabStartDepth = 0; |
| this.interactionMode = 'drag'; |
| this.interactionModeButtons = {}; |
| this.loadedDroppedModelData = null; |
| this.interactionModeColors = { |
| drag: { |
| base: '#00FFFF', |
| text: '#000000', |
| hand: new THREE.Color('#00FFFF') |
| }, |
| rotate: { |
| base: '#FF00FF', |
| text: '#FFFFFF', |
| hand: new THREE.Color('#FF00FF') |
| }, |
| scale: { |
| base: '#FFFF00', |
| text: '#000000', |
| hand: new THREE.Color('#FFFF00') |
| }, |
| animate: { |
| base: '#FFA500', |
| text: '#000000', |
| hand: new THREE.Color('#FFA500') |
| } |
| }; |
| this.rotateLastHandX = null; |
| this.rotateSensitivity = 0.02; |
| this.scaleInitialPinchDistance = null; |
| this.scaleInitialModelScale = null; |
| this.scaleSensitivity = 0.05; |
| this.grabbingPulseSpeed = 8; |
| this.grabbingPulseAmplitude = 0.5; |
| this.pulseBaseScale = 1.0; |
| this.fingertipDefaultOpacity = 0.3; |
| this.fingertipGrabOpacity = 1.0; |
| this.instructionTextElement = document.querySelector("#instruction-text"); |
| this.interactionModeInstructions = { |
| drag: "Pinch to grab and move the model", |
| rotate: "Pinch and move hand left/right to rotate", |
| scale: "Use two hands. Pinch with both and move hands closer/farther", |
| animate: "Pinch and move hand up/down to cycle animations" |
| }; |
| this.animationControlHandIndex = -1; |
| this.animationControlInitialPinchY = null; |
| this.animationScrollThreshold = 40; |
| |
| this._init().catch(function(error) { |
| console.error("Initialization failed:", error); |
| _this._showError("Initialization failed. Check console."); |
| }); |
| } |
| _create_class(Game, [ |
| { |
| key: "_init", |
| value: function _init() { |
| var _this = this; |
| return _async_to_generator(function() { |
| return _ts_generator(this, function(_state) { |
| switch(_state.label){ |
| case 0: |
| _this._setupDOM(); |
| _this._setupThree(); |
| _this._setupSpeechRecognition(); |
| return [ |
| 4, |
| _this._loadAssets() |
| ]; |
| case 1: |
| _state.sent(); |
| return [ |
| 4, |
| _this._setupHandTracking() |
| ]; |
| case 2: |
| _state.sent(); |
| |
| return [ |
| 4, |
| _this.videoElement.play() |
| ]; |
| case 3: |
| _state.sent(); |
| _this.audioManager.resumeContext(); |
| _this.speechManager.requestPermissionAndStart(); |
| _this.clock.start(); |
| window.addEventListener('resize', _this._onResize.bind(_this)); |
| _this.gameState = 'tracking'; |
| _this._animate(); |
| return [ |
| 2 |
| ]; |
| } |
| }); |
| })(); |
| } |
| }, |
| { |
| key: "_setupDOM", |
| value: function _setupDOM() { |
| var _this = this; |
| this.renderDiv.style.position = 'relative'; |
| this.renderDiv.style.width = '100vw'; |
| this.renderDiv.style.height = '100vh'; |
| this.renderDiv.style.overflow = 'hidden'; |
| this.renderDiv.style.background = '#111'; |
| |
| |
| this.videoElement = document.createElement('video'); |
| this.videoElement.style.position = 'absolute'; |
| this.videoElement.style.top = '0'; |
| this.videoElement.style.left = '0'; |
| this.videoElement.style.width = '100%'; |
| this.videoElement.style.height = '100%'; |
| this.videoElement.style.objectFit = 'cover'; |
| this.videoElement.style.transform = 'scaleX(-1)'; |
| this.videoElement.autoplay = true; |
| this.videoElement.muted = true; |
| this.videoElement.playsInline = true; |
| this.videoElement.style.zIndex = '0'; |
| this.renderDiv.appendChild(this.videoElement); |
| |
| this.gameOverContainer = document.createElement('div'); |
| this.gameOverContainer.style.position = 'absolute'; |
| this.gameOverContainer.style.top = '50%'; |
| this.gameOverContainer.style.left = '50%'; |
| this.gameOverContainer.style.transform = 'translate(-50%, -50%)'; |
| this.gameOverContainer.style.zIndex = '10'; |
| this.gameOverContainer.style.display = 'none'; |
| this.gameOverContainer.style.pointerEvents = 'none'; |
| this.gameOverContainer.style.textAlign = 'center'; |
| this.gameOverContainer.style.color = 'white'; |
| |
| this.gameOverContainer.style.fontFamily = '"Arial", "Helvetica Neue", Helvetica, sans-serif'; |
| |
| this.gameOverText = document.createElement('div'); |
| this.gameOverText.innerText = 'STATUS'; |
| this.gameOverText.style.fontSize = 'clamp(36px, 10vw, 72px)'; |
| this.gameOverText.style.fontWeight = 'bold'; |
| this.gameOverText.style.marginBottom = '10px'; |
| this.gameOverContainer.appendChild(this.gameOverText); |
| |
| this.restartHintText = document.createElement('div'); |
| this.restartHintText.innerText = '(click to restart tracking)'; |
| this.restartHintText.style.fontSize = 'clamp(16px, 3vw, 24px)'; |
| this.restartHintText.style.fontWeight = 'normal'; |
| this.restartHintText.style.opacity = '0.8'; |
| this.gameOverContainer.appendChild(this.restartHintText); |
| this.renderDiv.appendChild(this.gameOverContainer); |
| |
| this.speechBubble = document.createElement('div'); |
| this.speechBubble.id = 'speech-bubble'; |
| this.speechBubble.style.position = 'absolute'; |
| this.speechBubble.style.top = '10px'; |
| this.speechBubble.style.left = '50%'; |
| this.speechBubble.style.transform = 'translateX(-50%)'; |
| this.speechBubble.style.padding = '15px 25px'; |
| this.speechBubble.style.backgroundColor = 'rgba(255, 255, 255, 0.9)'; |
| this.speechBubble.style.border = '2px solid black'; |
| this.speechBubble.style.borderRadius = '4px'; |
| this.speechBubble.style.boxShadow = '4px 4px 0px rgba(0,0,0,1)'; |
| this.speechBubble.style.color = '#333'; |
| this.speechBubble.style.fontFamily = '"Arial", "Helvetica Neue", Helvetica, sans-serif'; |
| this.speechBubble.style.fontSize = 'clamp(16px, 3vw, 22px)'; |
| this.speechBubble.style.maxWidth = '80%'; |
| this.speechBubble.style.textAlign = 'center'; |
| this.speechBubble.style.zIndex = '25'; |
| this.speechBubble.style.opacity = '0'; |
| |
| this.speechBubble.style.transition = 'opacity 0.5s ease-in-out, transform 0.3s ease-in-out, box-shadow 0.3s ease-in-out, border 0.3s ease-in-out, padding 0.3s ease-in-out, font-size 0.3s ease-in-out, top 0.3s ease-in-out'; |
| this.speechBubble.style.pointerEvents = 'none'; |
| this.speechBubble.innerHTML = "..."; |
| this.renderDiv.appendChild(this.speechBubble); |
| |
| this.animationButtonsContainer = document.createElement('div'); |
| this.animationButtonsContainer.id = 'animation-buttons-container'; |
| this.animationButtonsContainer.style.position = 'absolute'; |
| this.animationButtonsContainer.style.bottom = 'auto'; |
| this.animationButtonsContainer.style.top = '10px'; |
| this.animationButtonsContainer.style.left = '10px'; |
| this.animationButtonsContainer.style.transform = 'none'; |
| this.animationButtonsContainer.style.zIndex = '30'; |
| this.animationButtonsContainer.style.display = 'flex'; |
| this.animationButtonsContainer.style.flexDirection = 'column'; |
| this.animationButtonsContainer.style.gap = '4px'; |
| this.animationButtonsContainer.style.opacity = '0'; |
| this.animationButtonsContainer.style.transition = 'opacity 0.3s ease-in-out'; |
| this.animationButtonsContainer.style.display = 'none'; |
| this.renderDiv.appendChild(this.animationButtonsContainer); |
| |
| this.interactionModeContainer = document.createElement('div'); |
| this.interactionModeContainer.id = 'interaction-mode-container'; |
| this.interactionModeContainer.style.position = 'absolute'; |
| this.interactionModeContainer.style.top = '10px'; |
| this.interactionModeContainer.style.right = '10px'; |
| this.interactionModeContainer.style.zIndex = '30'; |
| this.interactionModeContainer.style.display = 'flex'; |
| this.interactionModeContainer.style.flexDirection = 'column'; |
| this.interactionModeContainer.style.gap = '4px'; |
| this.renderDiv.appendChild(this.interactionModeContainer); |
| |
| [ |
| 'Drag', |
| 'Rotate', |
| 'Scale', |
| 'Animate' |
| ].forEach(function(mode) { |
| var button = document.createElement('button'); |
| button.innerText = mode; |
| button.id = "interaction-mode-".concat(mode.toLowerCase()); |
| button.style.padding = '10px 22px'; |
| button.style.fontSize = '18px'; |
| button.style.border = '2px solid black'; |
| button.style.borderRadius = '4px'; |
| button.style.cursor = 'pointer'; |
| button.style.fontWeight = 'bold'; |
| button.style.transition = 'background-color 0.2s ease, color 0.2s ease, box-shadow 0.2s ease'; |
| button.style.boxShadow = '2px 2px 0px black'; |
| button.addEventListener('click', function() { |
| return _this._setInteractionMode(mode.toLowerCase()); |
| }); |
| _this.interactionModeContainer.appendChild(button); |
| _this.interactionModeButtons[mode.toLowerCase()] = button; |
| }); |
| this._updateInteractionModeButtonStyles(); |
| this._updateInstructionText(); |
| this._setupDragAndDrop(); |
| } |
| }, |
| { |
| key: "_setupThree", |
| value: function _setupThree() { |
| var _this_interactionModeColors_this_interactionMode; |
| var width = this.renderDiv.clientWidth; |
| var height = this.renderDiv.clientHeight; |
| this.scene = new THREE.Scene(); |
| |
| this.camera = new THREE.OrthographicCamera(width / -2, width / 2, height / 2, height / -2, 1, 2000); |
| this.camera.position.z = 100; |
| this.renderer = new THREE.WebGLRenderer({ |
| alpha: true, |
| antialias: true |
| }); |
| this.renderer.setSize(width, height); |
| this.renderer.setPixelRatio(window.devicePixelRatio); |
| this.renderer.domElement.style.position = 'absolute'; |
| this.renderer.domElement.style.top = '0'; |
| this.renderer.domElement.style.left = '0'; |
| this.renderer.domElement.style.zIndex = '1'; |
| this.renderDiv.appendChild(this.renderer.domElement); |
| var ambientLight = new THREE.AmbientLight(0xffffff, 1.5); |
| this.scene.add(ambientLight); |
| var directionalLight = new THREE.DirectionalLight(0xffffff, 1.8); |
| directionalLight.position.set(0, 0, 100); |
| this.scene.add(directionalLight); |
| |
| for(var i = 0; i < 2; i++){ |
| var lineGroup = new THREE.Group(); |
| lineGroup.visible = false; |
| this.scene.add(lineGroup); |
| this.hands.push({ |
| landmarks: null, |
| anchorPos: new THREE.Vector3(), |
| lineGroup: lineGroup, |
| isPinching: false, |
| pinchPointScreen: new THREE.Vector2(), |
| isFist: false |
| }); |
| } |
| this.handLineMaterial = new THREE.LineBasicMaterial({ |
| color: 0x00ccff, |
| linewidth: 8 |
| }); |
| var initialModeHandColor = ((_this_interactionModeColors_this_interactionMode = this.interactionModeColors[this.interactionMode]) === null || _this_interactionModeColors_this_interactionMode === void 0 ? void 0 : _this_interactionModeColors_this_interactionMode.hand) || new THREE.Color(0x00ccff); |
| this.fingertipMaterialHand1 = new THREE.MeshBasicMaterial({ |
| color: initialModeHandColor.clone(), |
| side: THREE.DoubleSide, |
| transparent: true, |
| opacity: this.fingertipDefaultOpacity |
| }); |
| this.fingertipMaterialHand2 = new THREE.MeshBasicMaterial({ |
| color: initialModeHandColor.clone(), |
| side: THREE.DoubleSide, |
| transparent: true, |
| opacity: this.fingertipDefaultOpacity |
| }); |
| |
| |
| this.handConnections = [ |
| |
| [ |
| 0, |
| 1 |
| ], |
| [ |
| 1, |
| 2 |
| ], |
| [ |
| 2, |
| 3 |
| ], |
| [ |
| 3, |
| 4 |
| ], |
| |
| [ |
| 0, |
| 5 |
| ], |
| [ |
| 5, |
| 6 |
| ], |
| [ |
| 6, |
| 7 |
| ], |
| [ |
| 7, |
| 8 |
| ], |
| |
| [ |
| 0, |
| 9 |
| ], |
| [ |
| 9, |
| 10 |
| ], |
| [ |
| 10, |
| 11 |
| ], |
| [ |
| 11, |
| 12 |
| ], |
| |
| [ |
| 0, |
| 13 |
| ], |
| [ |
| 13, |
| 14 |
| ], |
| [ |
| 14, |
| 15 |
| ], |
| [ |
| 15, |
| 16 |
| ], |
| |
| [ |
| 0, |
| 17 |
| ], |
| [ |
| 17, |
| 18 |
| ], |
| [ |
| 18, |
| 19 |
| ], |
| [ |
| 19, |
| 20 |
| ], |
| |
| [ |
| 5, |
| 9 |
| ], |
| [ |
| 9, |
| 13 |
| ], |
| [ |
| 13, |
| 17 |
| ] |
| ]; |
| } |
| }, |
| { |
| key: "_loadAssets", |
| value: function _loadAssets() { |
| var _this = this; |
| return _async_to_generator(function() { |
| var gltfLoader, error; |
| return _ts_generator(this, function(_state) { |
| switch(_state.label){ |
| case 0: |
| console.log("Loading assets..."); |
| gltfLoader = new GLTFLoader(); |
| _state.label = 1; |
| case 1: |
| _state.trys.push([ |
| 1, |
| 3, |
| , |
| 4 |
| ]); |
| return [ |
| 4, |
| new Promise(function(resolve, reject) { |
| gltfLoader.load('assets/Stan.gltf', function(gltf) { |
| _this.pandaModel = gltf.scene; |
| _this.animationMixer = new THREE.AnimationMixer(_this.pandaModel); |
| _this.animationClips = gltf.animations; |
| if (_this.animationClips && _this.animationClips.length) { |
| _this.animationClips.forEach(function(clip, index) { |
| var action = _this.animationMixer.clipAction(clip); |
| var actionName = clip.name || "Animation ".concat(index + 1); |
| _this.animationActions[actionName] = action; |
| |
| var button = document.createElement('button'); |
| button.innerText = actionName; |
| button.style.padding = '5px 10px'; |
| button.style.fontSize = '13px'; |
| button.style.backgroundColor = '#f0f0f0'; |
| button.style.color = 'black'; |
| button.style.border = '2px solid black'; |
| button.style.borderRadius = '4px'; |
| button.style.cursor = 'pointer'; |
| button.style.transition = 'background-color 0.2s ease, box-shadow 0.2s ease'; |
| button.style.boxShadow = '2px 2px 0px black'; |
| button.addEventListener('click', function() { |
| return _this._playAnimation(actionName); |
| }); |
| _this.animationButtonsContainer.appendChild(button); |
| console.log("Loaded animation and created button for: ".concat(actionName)); |
| }); |
| |
| |
| var defaultActionName = Object.keys(_this.animationActions)[0]; |
| var idleActionKey = Object.keys(_this.animationActions).find(function(name) { |
| return name.toLowerCase().includes('idle'); |
| }); |
| if (idleActionKey) { |
| defaultActionName = idleActionKey; |
| console.log("Found idle animation: ".concat(defaultActionName)); |
| } else if (defaultActionName) { |
| console.log("No specific idle animation found, defaulting to first animation: ".concat(defaultActionName)); |
| } |
| if (defaultActionName && _this.animationActions[defaultActionName]) { |
| _this.currentAction = _this.animationActions[defaultActionName]; |
| _this.currentAction.play(); |
| console.log("Playing default animation: ".concat(defaultActionName)); |
| _this._updateButtonStyles(defaultActionName); |
| } else { |
| console.log("No animations found or default animation could not be played."); |
| } |
| } else { |
| console.log("Stan model has no embedded animations."); |
| } |
| |
| |
| var scale = 80; |
| _this.pandaModel.scale.set(scale, scale, scale); |
| |
| var sceneHeight = _this.renderDiv.clientHeight; |
| _this.pandaModel.position.set(0, sceneHeight * -0.45, -1000); |
| _this.scene.add(_this.pandaModel); |
| console.log("Stan GLTF model loaded and added to scene."); |
| resolve(); |
| }, undefined, function(error) { |
| console.error('An error occurred while loading the Stan GLTF model:', error); |
| reject(error); |
| }); |
| }) |
| ]; |
| case 2: |
| _state.sent(); |
| console.log("All specified assets loaded."); |
| return [ |
| 3, |
| 4 |
| ]; |
| case 3: |
| error = _state.sent(); |
| console.error("Error loading assets:", error); |
| _this._showError("Failed to load 3D model."); |
| throw error; |
| case 4: |
| return [ |
| 2 |
| ]; |
| } |
| }); |
| })(); |
| } |
| }, |
| { |
| key: "_setupHandTracking", |
| value: function _setupHandTracking() { |
| var _this = this; |
| return _async_to_generator(function() { |
| var vision, stream, error; |
| return _ts_generator(this, function(_state) { |
| switch(_state.label){ |
| case 0: |
| _state.trys.push([ |
| 0, |
| 4, |
| , |
| 5 |
| ]); |
| console.log("Setting up Hand Tracking..."); |
| return [ |
| 4, |
| FilesetResolver.forVisionTasks('https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/wasm') |
| ]; |
| case 1: |
| vision = _state.sent(); |
| return [ |
| 4, |
| HandLandmarker.createFromOptions(vision, { |
| baseOptions: { |
| modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task", |
| delegate: 'GPU' |
| }, |
| numHands: 2, |
| runningMode: 'VIDEO' |
| }) |
| ]; |
| case 2: |
| _this.handLandmarker = _state.sent(); |
| console.log("HandLandmarker created."); |
| console.log("Requesting webcam access..."); |
| return [ |
| 4, |
| navigator.mediaDevices.getUserMedia({ |
| video: { |
| facingMode: 'user', |
| width: { |
| ideal: 1920 |
| }, |
| height: { |
| ideal: 1080 |
| } |
| }, |
| audio: false |
| }) |
| ]; |
| case 3: |
| stream = _state.sent(); |
| _this.videoElement.srcObject = stream; |
| console.log("Webcam stream obtained."); |
| |
| return [ |
| 2, |
| new Promise(function(resolve) { |
| _this.videoElement.onloadedmetadata = function() { |
| console.log("Webcam metadata loaded."); |
| |
| _this.videoElement.style.width = _this.renderDiv.clientWidth + 'px'; |
| _this.videoElement.style.height = _this.renderDiv.clientHeight + 'px'; |
| resolve(); |
| }; |
| }) |
| ]; |
| case 4: |
| error = _state.sent(); |
| console.error('Error setting up Hand Tracking or Webcam:', error); |
| _this._showError("Webcam/Hand Tracking Error: ".concat(error.message, ". Please allow camera access.")); |
| throw error; |
| case 5: |
| return [ |
| 2 |
| ]; |
| } |
| }); |
| })(); |
| } |
| }, |
| { |
| key: "_updateHands", |
| value: function _updateHands() { |
| var _this = this; |
| if (!this.handLandmarker || !this.videoElement.srcObject || this.videoElement.readyState < 2 || this.videoElement.videoWidth === 0) return; |
| |
| var videoTime = this.videoElement.currentTime; |
| if (videoTime > this.lastVideoTime) { |
| this.lastVideoTime = videoTime; |
| try { |
| var _this1, _loop = function(i) { |
| var hand = _this1.hands[i]; |
| if (results.landmarks && results.landmarks[i]) { |
| var currentRawLandmarks = results.landmarks[i]; |
| if (!_this1.lastLandmarkPositions[i] || _this1.lastLandmarkPositions[i].length !== currentRawLandmarks.length) { |
| _this1.lastLandmarkPositions[i] = currentRawLandmarks.map(function(lm) { |
| return _object_spread({}, lm); |
| }); |
| } |
| var smoothedLandmarks = currentRawLandmarks.map(function(lm, lmIndex) { |
| var prevLm = _this.lastLandmarkPositions[i][lmIndex]; |
| return { |
| x: _this.smoothingFactor * lm.x + (1 - _this.smoothingFactor) * prevLm.x, |
| y: _this.smoothingFactor * lm.y + (1 - _this.smoothingFactor) * prevLm.y, |
| z: _this.smoothingFactor * lm.z + (1 - _this.smoothingFactor) * prevLm.z |
| }; |
| }); |
| _this1.lastLandmarkPositions[i] = smoothedLandmarks.map(function(lm) { |
| return _object_spread({}, lm); |
| }); |
| hand.landmarks = smoothedLandmarks; |
| var palm = smoothedLandmarks[9]; |
| var lmOriginalX = palm.x * videoParams.videoNaturalWidth; |
| var lmOriginalY = palm.y * videoParams.videoNaturalHeight; |
| var normX_visible = (lmOriginalX - videoParams.offsetX) / videoParams.visibleWidth; |
| var normY_visible = (lmOriginalY - videoParams.offsetY) / videoParams.visibleHeight; |
| var handX = (1 - normX_visible) * canvasWidth - canvasWidth / 2; |
| var handY = (1 - normY_visible) * canvasHeight - canvasHeight / 2; |
| hand.anchorPos.set(handX, handY, 1); |
| |
| var prevIsPinching = hand.isPinching; |
| |
| var thumbTipLm = smoothedLandmarks[4]; |
| var indexTipLm = smoothedLandmarks[8]; |
| if (thumbTipLm && indexTipLm) { |
| |
| var convertToScreenSpace = function(lm) { |
| var originalX = lm.x * videoParams.videoNaturalWidth; |
| var originalY = lm.y * videoParams.videoNaturalHeight; |
| var normX_visible = (originalX - videoParams.offsetX) / videoParams.visibleWidth; |
| var normY_visible = (originalY - videoParams.offsetY) / videoParams.visibleHeight; |
| return { |
| x: (1 - normX_visible) * canvasWidth - canvasWidth / 2, |
| y: (1 - normY_visible) * canvasHeight - canvasHeight / 2 |
| }; |
| }; |
| var thumbTipScreen = convertToScreenSpace(thumbTipLm); |
| var indexTipScreen = convertToScreenSpace(indexTipLm); |
| var distanceX = thumbTipScreen.x - indexTipScreen.x; |
| var distanceY = thumbTipScreen.y - indexTipScreen.y; |
| var pinchDistance = Math.sqrt(distanceX * distanceX + distanceY * distanceY); |
| var pinchThreshold = 45; |
| if (pinchDistance < pinchThreshold) { |
| hand.isPinching = true; |
| hand.pinchPointScreen.set((thumbTipScreen.x + indexTipScreen.x) / 2, (thumbTipScreen.y + indexTipScreen.y) / 2); |
| } else { |
| hand.isPinching = false; |
| } |
| } else { |
| hand.isPinching = false; |
| } |
| |
| |
| |
| var isTipNearMCP = function(tipLandmark, mcpLandmark) { |
| var threshold = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : 0.1; |
| if (!tipLandmark || !mcpLandmark) return false; |
| |
| |
| var dx = tipLandmark.x - mcpLandmark.x; |
| var dy = tipLandmark.y - mcpLandmark.y; |
| |
| var distance = Math.sqrt(dx * dx + dy * dy ); |
| return distance < threshold; |
| }; |
| var indexFingerTip = smoothedLandmarks[8]; |
| var indexFingerMcp = smoothedLandmarks[5]; |
| var middleFingerTip = smoothedLandmarks[12]; |
| var middleFingerMcp = smoothedLandmarks[9]; |
| var ringFingerTip = smoothedLandmarks[16]; |
| var ringFingerMcp = smoothedLandmarks[13]; |
| var pinkyTip = smoothedLandmarks[20]; |
| var pinkyMcp = smoothedLandmarks[17]; |
| |
| var curledFingers = 0; |
| if (isTipNearMCP(indexFingerTip, indexFingerMcp, 0.08)) curledFingers++; |
| if (isTipNearMCP(middleFingerTip, middleFingerMcp, 0.08)) curledFingers++; |
| if (isTipNearMCP(ringFingerTip, ringFingerMcp, 0.08)) curledFingers++; |
| if (isTipNearMCP(pinkyTip, pinkyMcp, 0.08)) curledFingers++; |
| var prevIsFist = hand.isFist; |
| hand.isFist = curledFingers >= 3; |
| |
| if (_this1.interactionMode === 'animate') { |
| |
| if (_this1.grabbingHandIndex !== -1 && _this1.pickedUpModel) { |
| |
| _this1.grabbingHandIndex = -1; |
| _this1.pickedUpModel = null; |
| |
| |
| _this1.rotateLastHandX = null; |
| _this1.scaleInitialPinchDistance = null; |
| _this1.scaleInitialModelScale = null; |
| } |
| if (hand.isPinching) { |
| if (!prevIsPinching && _this1.animationControlHandIndex === -1) { |
| _this1.animationControlHandIndex = i; |
| _this1.animationControlInitialPinchY = hand.pinchPointScreen.y; |
| console.log("Hand ".concat(i, " started pinch for animation control at Y: ").concat(_this1.animationControlInitialPinchY)); |
| } else if (_this1.animationControlHandIndex === i && _this1.animationControlInitialPinchY !== null) { |
| |
| var deltaY = hand.pinchPointScreen.y - _this1.animationControlInitialPinchY; |
| if (Math.abs(deltaY) > _this1.animationScrollThreshold) { |
| var animationNames = Object.keys(_this1.animationActions); |
| if (animationNames.length > 0) { |
| var currentIndex = -1; |
| |
| if (_this1.currentAction) { |
| for(var j = 0; j < animationNames.length; j++){ |
| if (_this1.animationActions[animationNames[j]] === _this1.currentAction) { |
| currentIndex = j; |
| break; |
| } |
| } |
| } |
| var nextIndex = currentIndex; |
| if (deltaY < 0) { |
| nextIndex = (currentIndex + 1) % animationNames.length; |
| console.log("Scrolling animation UP (to next)"); |
| } else { |
| nextIndex = (currentIndex - 1 + animationNames.length) % animationNames.length; |
| console.log("Scrolling animation DOWN (to previous)"); |
| } |
| if (nextIndex !== currentIndex) { |
| _this1._playAnimation(animationNames[nextIndex]); |
| } |
| } |
| |
| _this1.animationControlInitialPinchY = hand.pinchPointScreen.y; |
| } |
| } |
| } else { |
| if (prevIsPinching && _this1.animationControlHandIndex === i) { |
| console.log("Hand ".concat(i, " ended pinch for animation control.")); |
| _this1.animationControlHandIndex = -1; |
| _this1.animationControlInitialPinchY = null; |
| } |
| } |
| } else if (_this1.interactionMode === 'drag') { |
| if (hand.isPinching) { |
| if (!prevIsPinching && _this1.grabbingHandIndex === -1 && _this1.pandaModel) { |
| |
| _this1.grabbingHandIndex = i; |
| _this1.pickedUpModel = _this1.pandaModel; |
| |
| |
| _this1.modelGrabStartDepth = _this1.pickedUpModel.position.z; |
| var pinchX = hand.pinchPointScreen.x; |
| var pinchY = hand.pinchPointScreen.y; |
| |
| var ndcX = pinchX / (_this1.renderDiv.clientWidth / 2); |
| var ndcY = pinchY / (_this1.renderDiv.clientHeight / 2); |
| var pinchPoint3DWorld = new THREE.Vector3(ndcX, ndcY, 0.5); |
| pinchPoint3DWorld.unproject(_this1.camera); |
| pinchPoint3DWorld.z = _this1.modelGrabStartDepth; |
| console.log("Grab screen: (".concat(pinchX.toFixed(2), ", ").concat(pinchY.toFixed(2), "), NDC: (").concat(ndcX.toFixed(2), ", ").concat(ndcY.toFixed(2), ")")); |
| console.log("Grab 3D World (pre-offset): ".concat(pinchPoint3DWorld.x.toFixed(2), ", ").concat(pinchPoint3DWorld.y.toFixed(2), ", ").concat(pinchPoint3DWorld.z.toFixed(2))); |
| _this1.modelDragOffset.subVectors(_this1.pickedUpModel.position, pinchPoint3DWorld); |
| console.log("Hand ".concat(i, " GRABBED model for DRAG at depth ").concat(_this1.modelGrabStartDepth, ". Offset:"), _this1.modelDragOffset.x.toFixed(2), _this1.modelDragOffset.y.toFixed(2), _this1.modelDragOffset.z.toFixed(2)); |
| } else if (_this1.grabbingHandIndex === i && _this1.pickedUpModel) { |
| |
| var currentPinchX = hand.pinchPointScreen.x; |
| var currentPinchY = hand.pinchPointScreen.y; |
| var currentNdcX = currentPinchX / (_this1.renderDiv.clientWidth / 2); |
| var currentNdcY = currentPinchY / (_this1.renderDiv.clientHeight / 2); |
| var newPinchPoint3DWorld = new THREE.Vector3(currentNdcX, currentNdcY, 0.5); |
| newPinchPoint3DWorld.unproject(_this1.camera); |
| newPinchPoint3DWorld.z = _this1.modelGrabStartDepth; |
| _this1.pickedUpModel.position.addVectors(newPinchPoint3DWorld, _this1.modelDragOffset); |
| var minZ = -200; |
| var maxZ = 50; |
| _this1.pickedUpModel.position.z = Math.max(minZ, Math.min(maxZ, _this1.pickedUpModel.position.z)); |
| } |
| } else { |
| if (prevIsPinching && _this1.grabbingHandIndex === i) { |
| console.log("Hand ".concat(i, " RELEASED Stan model (Drag mode) at position:"), _this1.pickedUpModel.position); |
| _this1.grabbingHandIndex = -1; |
| _this1.pickedUpModel = null; |
| |
| } |
| } |
| } else if (_this1.interactionMode === 'rotate') { |
| if (hand.isPinching) { |
| if (!prevIsPinching && _this1.grabbingHandIndex === -1 && _this1.pandaModel) { |
| |
| _this1.grabbingHandIndex = i; |
| _this1.pickedUpModel = _this1.pandaModel; |
| _this1.rotateLastHandX = hand.pinchPointScreen.x; |
| console.log("Hand ".concat(i, " INITIATED ROTATION on model via pinch from anywhere.")); |
| } else if (_this1.grabbingHandIndex === i && _this1.pickedUpModel && _this1.rotateLastHandX !== null) { |
| var currentHandX = hand.pinchPointScreen.x; |
| var deltaX = currentHandX - _this1.rotateLastHandX; |
| if (_this1.pickedUpModel && Math.abs(deltaX) > 0.5) { |
| _this1.pickedUpModel.rotation.y -= deltaX * _this1.rotateSensitivity; |
| } |
| _this1.rotateLastHandX = currentHandX; |
| } |
| } else { |
| if (prevIsPinching && _this1.grabbingHandIndex === i) { |
| console.log("Hand ".concat(i, " RELEASED ROTATION on model (pinch ended).")); |
| _this1.grabbingHandIndex = -1; |
| _this1.pickedUpModel = null; |
| _this1.rotateLastHandX = null; |
| |
| } |
| } |
| } else if (_this1.interactionMode === 'scale') { |
| var hand0 = _this1.hands[0]; |
| var hand1 = _this1.hands[1]; |
| if (hand0 && hand1 && hand0.landmarks && hand1.landmarks && hand0.isPinching && hand1.isPinching) { |
| |
| var dist = hand0.pinchPointScreen.distanceTo(hand1.pinchPointScreen); |
| if (_this1.scaleInitialPinchDistance === null || _this1.scaleInitialModelScale === null) { |
| |
| _this1.scaleInitialPinchDistance = dist; |
| _this1.scaleInitialModelScale = _this1.pandaModel.scale.clone(); |
| _this1.grabbingHandIndex = 0; |
| _this1.pickedUpModel = _this1.pandaModel; |
| |
| console.log("Scaling initiated. Initial pinch dist: ".concat(dist.toFixed(2), ", Initial scale: ").concat(_this1.scaleInitialModelScale.x.toFixed(2))); |
| } else { |
| |
| var deltaDistance = dist - _this1.scaleInitialPinchDistance; |
| var scaleFactorChange = deltaDistance * _this1.scaleSensitivity; |
| var newScaleValue = _this1.scaleInitialModelScale.x + scaleFactorChange; |
| |
| var minScale = 10; |
| var maxScale = 300; |
| newScaleValue = Math.max(minScale, Math.min(maxScale, newScaleValue)); |
| _this1.pandaModel.scale.set(newScaleValue, newScaleValue, newScaleValue); |
| |
| } |
| } else { |
| |
| if (_this1.scaleInitialPinchDistance !== null) { |
| console.log("Scaling gesture ended."); |
| _this1.scaleInitialPinchDistance = null; |
| _this1.scaleInitialModelScale = null; |
| _this1.grabbingHandIndex = -1; |
| _this1.pickedUpModel = null; |
| |
| } |
| } |
| } |
| _this1._updateHandLines(i, smoothedLandmarks, videoParams, canvasWidth, canvasHeight); |
| } else { |
| if (hand.isPinching && _this1.grabbingHandIndex === i && _this1.interactionMode === 'drag') { |
| console.log("Hand ".concat(i, " (which was grabbing for drag) disappeared. Releasing model.")); |
| _this1.grabbingHandIndex = -1; |
| _this1.pickedUpModel = null; |
| |
| } else if (_this1.hands[i].isPinching && _this1.grabbingHandIndex === i && _this1.interactionMode === 'rotate') { |
| console.log("Hand ".concat(i, " (which was pinching for rotate) disappeared. Releasing model.")); |
| _this1.grabbingHandIndex = -1; |
| _this1.pickedUpModel = null; |
| _this1.rotateLastHandX = null; |
| |
| } else if (_this1.interactionMode === 'scale' && _this1.scaleInitialPinchDistance !== null && (i === 0 || i === 1)) { |
| var _this_hands_, _this_hands_1; |
| var hand0Exists = (_this_hands_ = _this1.hands[0]) === null || _this_hands_ === void 0 ? void 0 : _this_hands_.landmarks; |
| var hand1Exists = (_this_hands_1 = _this1.hands[1]) === null || _this_hands_1 === void 0 ? void 0 : _this_hands_1.landmarks; |
| if (!hand0Exists || !hand1Exists) { |
| console.log("Scaling gesture ended due to hand disappearance."); |
| _this1.scaleInitialPinchDistance = null; |
| _this1.scaleInitialModelScale = null; |
| _this1.grabbingHandIndex = -1; |
| _this1.pickedUpModel = null; |
| |
| } |
| } |
| hand.landmarks = null; |
| hand.isPinching = false; |
| hand.isFist = false; |
| if (hand.lineGroup) hand.lineGroup.visible = false; |
| } |
| |
| var isThisHandActivelyInteractingForSound = false; |
| if (_this1.interactionMode === 'drag' || _this1.interactionMode === 'rotate') { |
| isThisHandActivelyInteractingForSound = _this1.grabbingHandIndex === i && _this1.pickedUpModel === _this1.pandaModel; |
| } else if (_this1.interactionMode === 'animate') { |
| isThisHandActivelyInteractingForSound = _this1.animationControlHandIndex === i; |
| } |
| if (hand.isPinching && isThisHandActivelyInteractingForSound && _this1.interactionMode !== 'scale') { |
| _this1.audioManager.playInteractionClickSound(); |
| } |
| }; |
| var results = this.handLandmarker.detectForVideo(this.videoElement, performance.now()); |
| var videoParams = this._getVisibleVideoParameters(); |
| if (!videoParams) return; |
| var canvasWidth = this.renderDiv.clientWidth; |
| var canvasHeight = this.renderDiv.clientHeight; |
| for(var i = 0; i < this.hands.length; i++)_this1 = this, _loop(i); |
| |
| |
| if (this.interactionMode === 'scale' && this.scaleInitialPinchDistance !== null) { |
| var hand0 = this.hands[0]; |
| var hand1 = this.hands[1]; |
| var hand0PinchingAndVisible = hand0 && hand0.landmarks && hand0.isPinching; |
| var hand1PinchingAndVisible = hand1 && hand1.landmarks && hand1.isPinching; |
| if (hand0PinchingAndVisible && hand1PinchingAndVisible) { |
| |
| this.audioManager.playInteractionClickSound(); |
| } else { |
| |
| if (this.scaleInitialPinchDistance !== null) { |
| console.log("Scaling gesture ended (one hand stopped pinching/disappeared - post-loop check)."); |
| this.scaleInitialPinchDistance = null; |
| this.scaleInitialModelScale = null; |
| this.grabbingHandIndex = -1; |
| this.pickedUpModel = null; |
| |
| } |
| } |
| } |
| } catch (error) { |
| console.error("Error during hand detection:", error); |
| } |
| } |
| } |
| }, |
| { |
| key: "_getModelScreenBoundingBox", |
| value: function _getModelScreenBoundingBox() { |
| var _this = this; |
| if (!this.pandaModel || !this.camera || !this.renderer) { |
| return null; |
| } |
| |
| this.pandaModel.updateMatrixWorld(true); |
| var box = new THREE.Box3().setFromObject(this.pandaModel); |
| if (box.isEmpty()) { |
| return null; |
| } |
| var corners = [ |
| new THREE.Vector3(box.min.x, box.min.y, box.min.z), |
| new THREE.Vector3(box.min.x, box.min.y, box.max.z), |
| new THREE.Vector3(box.min.x, box.max.y, box.min.z), |
| new THREE.Vector3(box.min.x, box.max.y, box.max.z), |
| new THREE.Vector3(box.max.x, box.min.y, box.min.z), |
| new THREE.Vector3(box.max.x, box.min.y, box.max.z), |
| new THREE.Vector3(box.max.x, box.max.y, box.min.z), |
| new THREE.Vector3(box.max.x, box.max.y, box.max.z) |
| ]; |
| var minX = Infinity, maxX = -Infinity, minY = Infinity, maxY = -Infinity; |
| var canvasWidth = this.renderDiv.clientWidth; |
| var canvasHeight = this.renderDiv.clientHeight; |
| corners.forEach(function(corner) { |
| |
| corner.applyMatrix4(_this.pandaModel.matrixWorld); |
| |
| corner.project(_this.camera); |
| |
| |
| var screenX = corner.x * (canvasWidth / 2); |
| var screenY = corner.y * (canvasHeight / 2); |
| minX = Math.min(minX, screenX); |
| maxX = Math.max(maxX, screenX); |
| minY = Math.min(minY, screenY); |
| maxY = Math.max(maxY, screenY); |
| }); |
| if (minX === Infinity) return null; |
| return { |
| minX: minX, |
| minY: minY, |
| maxX: maxX, |
| maxY: maxY |
| }; |
| } |
| }, |
| { |
| key: "_getVisibleVideoParameters", |
| value: function _getVisibleVideoParameters() { |
| if (!this.videoElement || this.videoElement.videoWidth === 0 || this.videoElement.videoHeight === 0) { |
| return null; |
| } |
| var vNatW = this.videoElement.videoWidth; |
| var vNatH = this.videoElement.videoHeight; |
| var rW = this.renderDiv.clientWidth; |
| var rH = this.renderDiv.clientHeight; |
| if (vNatW === 0 || vNatH === 0 || rW === 0 || rH === 0) return null; |
| var videoAR = vNatW / vNatH; |
| var renderDivAR = rW / rH; |
| var finalVideoPixelX, finalVideoPixelY; |
| var visibleVideoPixelWidth, visibleVideoPixelHeight; |
| if (videoAR > renderDivAR) { |
| |
| var scale = rH / vNatH; |
| var scaledVideoWidth = vNatW * scale; |
| |
| var totalCroppedPixelsX = (scaledVideoWidth - rW) / scale; |
| finalVideoPixelX = totalCroppedPixelsX / 2; |
| finalVideoPixelY = 0; |
| visibleVideoPixelWidth = vNatW - totalCroppedPixelsX; |
| visibleVideoPixelHeight = vNatH; |
| } else { |
| |
| var scale1 = rW / vNatW; |
| var scaledVideoHeight = vNatH * scale1; |
| |
| var totalCroppedPixelsY = (scaledVideoHeight - rH) / scale1; |
| finalVideoPixelX = 0; |
| finalVideoPixelY = totalCroppedPixelsY / 2; |
| visibleVideoPixelWidth = vNatW; |
| visibleVideoPixelHeight = vNatH - totalCroppedPixelsY; |
| } |
| |
| if (visibleVideoPixelWidth <= 0 || visibleVideoPixelHeight <= 0) { |
| |
| console.warn("Calculated visible video dimension is zero or negative.", { |
| visibleVideoPixelWidth: visibleVideoPixelWidth, |
| visibleVideoPixelHeight: visibleVideoPixelHeight |
| }); |
| return { |
| offsetX: 0, |
| offsetY: 0, |
| visibleWidth: vNatW, |
| visibleHeight: vNatH, |
| videoNaturalWidth: vNatW, |
| videoNaturalHeight: vNatH |
| }; |
| } |
| return { |
| offsetX: finalVideoPixelX, |
| offsetY: finalVideoPixelY, |
| visibleWidth: visibleVideoPixelWidth, |
| visibleHeight: visibleVideoPixelHeight, |
| videoNaturalWidth: vNatW, |
| videoNaturalHeight: vNatH |
| }; |
| } |
| }, |
| { |
| |
| key: "_showStatusScreen", |
| value: function _showStatusScreen(message) { |
| var color = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : 'white', showRestartHint = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : false; |
| this.gameOverContainer.style.display = 'block'; |
| this.gameOverText.innerText = message; |
| this.gameOverText.style.color = color; |
| this.restartHintText.style.display = showRestartHint ? 'block' : 'none'; |
| |
| } |
| }, |
| { |
| key: "_showError", |
| value: function _showError(message) { |
| this.gameOverContainer.style.display = 'block'; |
| this.gameOverText.innerText = "ERROR: ".concat(message); |
| this.gameOverText.style.color = 'orange'; |
| this.restartHintText.style.display = 'true'; |
| this.gameState = 'error'; |
| |
| this.hands.forEach(function(hand) { |
| if (hand.lineGroup) hand.lineGroup.visible = false; |
| }); |
| } |
| }, |
| { |
| key: "_restartGame", |
| value: function _restartGame() { |
| console.log("Restarting tracking..."); |
| this.gameOverContainer.style.display = 'none'; |
| this.hands.forEach(function(hand) { |
| if (hand.lineGroup) { |
| hand.lineGroup.visible = false; |
| } |
| }); |
| |
| |
| |
| this.gameState = 'tracking'; |
| this.lastVideoTime = -1; |
| this.clock.start(); |
| |
| } |
| }, |
| { |
| |
| key: "_onResize", |
| value: function _onResize() { |
| var width = this.renderDiv.clientWidth; |
| var height = this.renderDiv.clientHeight; |
| |
| this.camera.left = width / -2; |
| this.camera.right = width / 2; |
| this.camera.top = height / 2; |
| this.camera.bottom = height / -2; |
| this.camera.updateProjectionMatrix(); |
| |
| this.renderer.setSize(width, height); |
| |
| this.videoElement.style.width = width + 'px'; |
| this.videoElement.style.height = height + 'px'; |
| |
| } |
| }, |
| { |
| key: "_updateHandLines", |
| value: function _updateHandLines(handIndex, landmarks, videoParams, canvasWidth, canvasHeight) { |
| var _this = this; |
| var hand = this.hands[handIndex]; |
| var lineGroup = hand.lineGroup; |
| |
| var isThisHandActivelyInteracting = false; |
| if (this.interactionMode === 'drag' || this.interactionMode === 'rotate') { |
| isThisHandActivelyInteracting = this.grabbingHandIndex === handIndex && this.pickedUpModel === this.pandaModel; |
| } else if (this.interactionMode === 'scale') { |
| |
| isThisHandActivelyInteracting = this.scaleInitialPinchDistance !== null && (handIndex === 0 || handIndex === 1); |
| } else if (this.interactionMode === 'animate') { |
| |
| isThisHandActivelyInteracting = this.animationControlHandIndex === handIndex; |
| } |
| var currentHandMaterial = handIndex === 0 ? this.fingertipMaterialHand1 : this.fingertipMaterialHand2; |
| if (currentHandMaterial) { |
| currentHandMaterial.opacity = isThisHandActivelyInteracting ? this.fingertipGrabOpacity : this.fingertipDefaultOpacity; |
| } |
| while(lineGroup.children.length){ |
| var child = lineGroup.children[0]; |
| lineGroup.remove(child); |
| if (child.geometry) child.geometry.dispose(); |
| |
| } |
| if (!landmarks || landmarks.length === 0 || !videoParams) { |
| lineGroup.visible = false; |
| return; |
| } |
| var isAnyLandmarkOffScreen = false; |
| var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined; |
| try { |
| |
| for(var _iterator = landmarks[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){ |
| var lm = _step.value; |
| var lmOriginalX = lm.x * videoParams.videoNaturalWidth; |
| var lmOriginalY = lm.y * videoParams.videoNaturalHeight; |
| var normX_visible = (lmOriginalX - videoParams.offsetX) / videoParams.visibleWidth; |
| var normY_visible = (lmOriginalY - videoParams.offsetY) / videoParams.visibleHeight; |
| if (normX_visible < 0 || normX_visible > 1 || normY_visible < 0 || normY_visible > 1) { |
| isAnyLandmarkOffScreen = true; |
| break; |
| } |
| } |
| } catch (err) { |
| _didIteratorError = true; |
| _iteratorError = err; |
| } finally{ |
| try { |
| if (!_iteratorNormalCompletion && _iterator.return != null) { |
| _iterator.return(); |
| } |
| } finally{ |
| if (_didIteratorError) { |
| throw _iteratorError; |
| } |
| } |
| } |
| if (isAnyLandmarkOffScreen) { |
| lineGroup.visible = false; |
| return; |
| } |
| |
| |
| var points3D = landmarks.map(function(lm) { |
| var lmOriginalX = lm.x * videoParams.videoNaturalWidth; |
| var lmOriginalY = lm.y * videoParams.videoNaturalHeight; |
| var normX_visible = (lmOriginalX - videoParams.offsetX) / videoParams.visibleWidth; |
| var normY_visible = (lmOriginalY - videoParams.offsetY) / videoParams.visibleHeight; |
| |
| normX_visible = Math.max(0, Math.min(1, normX_visible)); |
| normY_visible = Math.max(0, Math.min(1, normY_visible)); |
| var x = (1 - normX_visible) * canvasWidth - canvasWidth / 2; |
| var y = (1 - normY_visible) * canvasHeight - canvasHeight / 2; |
| return new THREE.Vector3(x, y, 1.1); |
| }); |
| var lineZ = 1; |
| this.handConnections.forEach(function(conn) { |
| var p1 = points3D[conn[0]]; |
| var p2 = points3D[conn[1]]; |
| if (p1 && p2) { |
| |
| var lineP1 = p1.clone().setZ(lineZ); |
| var lineP2 = p2.clone().setZ(lineZ); |
| var geometry = new THREE.BufferGeometry().setFromPoints([ |
| lineP1, |
| lineP2 |
| ]); |
| var line = new THREE.Line(geometry, _this.handLineMaterial); |
| lineGroup.add(line); |
| } |
| }); |
| |
| var fingertipRadius = 8; |
| var wristRadius = 12; |
| var circleSegments = 16; |
| this.fingertipLandmarkIndices.forEach(function(index) { |
| var landmarkPosition = points3D[index]; |
| if (landmarkPosition) { |
| var radius = index === 0 ? wristRadius : fingertipRadius; |
| var circleGeometry = new THREE.CircleGeometry(radius, circleSegments); |
| |
| var landmarkCircle = new THREE.Mesh(circleGeometry, currentHandMaterial); |
| landmarkCircle.position.copy(landmarkPosition); |
| |
| if (isThisHandActivelyInteracting) { |
| |
| |
| var currentPulseProgress = (1 + Math.sin(_this.clock.elapsedTime * _this.grabbingPulseSpeed)) / 2; |
| var scaleValue = _this.pulseBaseScale + currentPulseProgress * _this.grabbingPulseAmplitude; |
| landmarkCircle.scale.set(scaleValue, scaleValue, 1); |
| } else { |
| landmarkCircle.scale.set(_this.pulseBaseScale, _this.pulseBaseScale, 1); |
| } |
| lineGroup.add(landmarkCircle); |
| } |
| }); |
| lineGroup.visible = true; |
| } |
| }, |
| { |
| key: "_animate", |
| value: function _animate() { |
| requestAnimationFrame(this._animate.bind(this)); |
| var deltaTime = this.clock.getDelta(); |
| |
| if (this.gameState === 'tracking') { |
| this._updateHands(); |
| } |
| |
| if (this.animationMixer) { |
| this.animationMixer.update(deltaTime); |
| } |
| |
| |
| |
| this.renderer.render(this.scene, this.camera); |
| } |
| }, |
| { |
| key: "start", |
| value: function start() { |
| var _this = this; |
| |
| this.renderDiv.addEventListener('click', function() { |
| _this.audioManager.resumeContext(); |
| if (_this.gameState === 'error' || _this.gameState === 'paused') { |
| _this._restartGame(); |
| } |
| }); |
| console.log('Game setup initiated. Waiting for async operations...'); |
| |
| } |
| }, |
| { |
| key: "_updateSpeechBubbleAppearance", |
| value: function _updateSpeechBubbleAppearance() { |
| if (!this.speechBubble) return; |
| var isPlaceholder = this.speechBubble.innerHTML === "..." || this.speechBubble.innerText === "..."; |
| |
| |
| |
| var showActiveStyling = this.isSpeechActive && !isPlaceholder; |
| var translateY = isPlaceholder ? '-5px' : '0px'; |
| var scale = showActiveStyling ? '1.15' : '1.0'; |
| this.speechBubble.style.transform = "translateX(-50%) translateY(".concat(translateY, ") scale(").concat(scale, ")"); |
| if (showActiveStyling) { |
| |
| |
| this.speechBubble.style.boxShadow = '5px 5px 0px #007bff'; |
| this.speechBubble.style.border = '2px solid black'; |
| this.speechBubble.style.padding = '18px 28px'; |
| this.speechBubble.style.fontSize = 'clamp(20px, 3.5vw, 26px)'; |
| this.speechBubble.style.top = '15px'; |
| } else { |
| |
| |
| this.speechBubble.style.boxShadow = '4px 4px 0px rgba(0,0,0,1)'; |
| this.speechBubble.style.border = '2px solid black'; |
| this.speechBubble.style.padding = '15px 25px'; |
| this.speechBubble.style.fontSize = 'clamp(16px, 3vw, 22px)'; |
| this.speechBubble.style.top = '10px'; |
| } |
| } |
| }, |
| { |
| key: "_setupSpeechRecognition", |
| value: function _setupSpeechRecognition() { |
| var _this = this; |
| this.speechManager = new SpeechManager(function(finalTranscript, interimTranscript) { |
| if (_this.speechBubble) { |
| clearTimeout(_this.speechBubbleTimeout); |
| if (finalTranscript) { |
| _this.speechBubble.innerHTML = finalTranscript; |
| _this.speechBubble.style.opacity = '1'; |
| _this.speechBubbleTimeout = setTimeout(function() { |
| _this.speechBubble.innerHTML = "..."; |
| _this.speechBubble.style.opacity = '0.7'; |
| _this._updateSpeechBubbleAppearance(); |
| }, 2000); |
| } else if (interimTranscript) { |
| _this.speechBubble.innerHTML = '<i style="color: #888;">'.concat(interimTranscript, "</i>"); |
| _this.speechBubble.style.opacity = '1'; |
| } else { |
| _this.speechBubbleTimeout = setTimeout(function() { |
| if (_this.speechBubble.innerHTML !== "...") { |
| _this.speechBubble.innerHTML = "..."; |
| } |
| _this.speechBubble.style.opacity = '0.7'; |
| _this._updateSpeechBubbleAppearance(); |
| }, 500); |
| } |
| _this._updateSpeechBubbleAppearance(); |
| } |
| }, function(isActive) { |
| _this.isSpeechActive = isActive; |
| _this._updateSpeechBubbleAppearance(); |
| }, function(command) { |
| console.log("Game received command: ".concat(command)); |
| var validCommands = [ |
| 'drag', |
| 'rotate', |
| 'scale', |
| 'animate' |
| ]; |
| if (validCommands.includes(command.toLowerCase())) { |
| _this._setInteractionMode(command.toLowerCase()); |
| } else { |
| console.warn("Unrecognized command via speech: ".concat(command)); |
| } |
| }); |
| |
| if (this.speechBubble) { |
| this.speechBubble.innerHTML = "..."; |
| this.speechBubble.style.opacity = '0.7'; |
| this._updateSpeechBubbleAppearance(); |
| } |
| |
| } |
| }, |
| { |
| key: "_playAnimation", |
| value: function _playAnimation(name) { |
| if (!this.animationActions[name]) { |
| console.warn('Animation "'.concat(name, '" not found.')); |
| return; |
| } |
| var newAction = this.animationActions[name]; |
| if (this.currentAction === newAction && newAction.isRunning()) { |
| console.log('Animation "'.concat(name, '" is already playing.')); |
| return; |
| } |
| if (this.currentAction) { |
| this.currentAction.fadeOut(0.5); |
| } |
| newAction.reset().fadeIn(0.5).play(); |
| this.currentAction = newAction; |
| console.log("Playing animation: ".concat(name)); |
| this._updateButtonStyles(name); |
| } |
| }, |
| { |
| key: "_updateButtonStyles", |
| value: function _updateButtonStyles(activeAnimationName) { |
| var buttons = this.animationButtonsContainer.children; |
| for(var i = 0; i < buttons.length; i++){ |
| var button = buttons[i]; |
| var isActive = button.innerText === activeAnimationName; |
| button.style.backgroundColor = isActive ? '#007bff' : '#f0f0f0'; |
| button.style.color = isActive ? 'white' : 'black'; |
| button.style.fontWeight = isActive ? 'bold' : 'normal'; |
| |
| button.style.boxShadow = isActive ? '1px 1px 0px black' : '2px 2px 0px black'; |
| } |
| } |
| }, |
| { |
| key: "_setInteractionMode", |
| value: function _setInteractionMode(mode) { |
| var _this = this; |
| if (this.interactionMode === mode) return; |
| console.log("Setting interaction mode to: ".concat(mode)); |
| this.interactionMode = mode; |
| |
| if (this.grabbingHandIndex !== -1 && this.pickedUpModel) { |
| console.log("Interaction mode changed while grabbing. Releasing model from hand ".concat(this.grabbingHandIndex, ".")); |
| this.grabbingHandIndex = -1; |
| this.pickedUpModel = null; |
| this.rotateLastHandX = null; |
| this.scaleInitialPinchDistance = null; |
| this.scaleInitialModelScale = null; |
| |
| } |
| this._updateHandMaterialsForMode(mode); |
| this._updateInteractionModeButtonStyles(); |
| |
| if (this.animationButtonsContainer) { |
| if (mode === 'animate') { |
| this.animationButtonsContainer.style.display = 'flex'; |
| requestAnimationFrame(function() { |
| _this.animationButtonsContainer.style.opacity = '1'; |
| }); |
| } else { |
| this.animationButtonsContainer.style.opacity = '0'; |
| |
| setTimeout(function() { |
| if (_this.interactionMode !== 'animate') { |
| _this.animationButtonsContainer.style.display = 'none'; |
| } |
| }, 300); |
| } |
| } |
| this._updateInstructionText(); |
| } |
| }, |
| { |
| key: "_updateInstructionText", |
| value: function _updateInstructionText() { |
| if (this.instructionTextElement) { |
| var instruction = this.interactionModeInstructions[this.interactionMode] || "Use hand gestures to interact."; |
| this.instructionTextElement.innerText = instruction; |
| |
| |
| this.instructionTextElement.style.bottom = '10px'; |
| } |
| } |
| }, |
| { |
| key: "_updateHandMaterialsForMode", |
| value: function _updateHandMaterialsForMode(mode) { |
| var modeConfig = this.interactionModeColors[mode]; |
| var colorToSet = modeConfig ? modeConfig.hand : new THREE.Color(0x00ccff); |
| if (this.fingertipMaterialHand1) { |
| this.fingertipMaterialHand1.color.set(colorToSet); |
| } |
| if (this.fingertipMaterialHand2) { |
| this.fingertipMaterialHand2.color.set(colorToSet); |
| } |
| } |
| }, |
| { |
| key: "_updateInteractionModeButtonStyles", |
| value: function _updateInteractionModeButtonStyles() { |
| var _this = this; |
| for(var modeKey in this.interactionModeButtons){ |
| var button = this.interactionModeButtons[modeKey]; |
| var modeConfig = this.interactionModeColors[modeKey]; |
| var fallbackColor = '#6c757d'; |
| var fallbackTextColor = 'white'; |
| if (modeKey === this.interactionMode) { |
| button.style.border = '2px solid black'; |
| if (modeConfig) { |
| button.style.backgroundColor = modeConfig.base; |
| button.style.color = modeConfig.text; |
| } else { |
| button.style.backgroundColor = fallbackColor; |
| button.style.color = fallbackTextColor; |
| } |
| button.style.fontWeight = 'bold'; |
| button.style.boxShadow = '1px 1px 0px black'; |
| } else { |
| button.style.backgroundColor = 'rgba(255, 255, 255, 0.5)'; |
| button.style.border = '2px solid black'; |
| if (modeConfig) { |
| button.style.color = modeConfig.base; |
| } else { |
| button.style.color = fallbackColor; |
| } |
| button.style.fontWeight = 'bold'; |
| button.style.boxShadow = '2px 2px 0px black'; |
| } |
| } |
| |
| |
| if (this.animationButtonsContainer) { |
| if (this.interactionMode === 'animate') { |
| this.animationButtonsContainer.style.display = 'flex'; |
| requestAnimationFrame(function() { |
| _this.animationButtonsContainer.style.opacity = '1'; |
| }); |
| } else { |
| this.animationButtonsContainer.style.opacity = '0'; |
| this.animationButtonsContainer.style.display = 'none'; |
| } |
| } |
| this._updateInstructionText(); |
| } |
| }, |
| { |
| key: "_setupDragAndDrop", |
| value: function _setupDragAndDrop() { |
| var _this = this; |
| this.renderDiv.addEventListener('dragover', function(event) { |
| event.preventDefault(); |
| event.dataTransfer.dropEffect = 'copy'; |
| _this.renderDiv.style.border = '2px dashed #007bff'; |
| }); |
| this.renderDiv.addEventListener('dragleave', function(event) { |
| _this.renderDiv.style.border = 'none'; |
| }); |
| this.renderDiv.addEventListener('drop', function(event) { |
| event.preventDefault(); |
| _this.renderDiv.style.border = 'none'; |
| if (event.dataTransfer.files && event.dataTransfer.files.length > 0) { |
| var file = event.dataTransfer.files[0]; |
| var fileName = file.name.toLowerCase(); |
| var fileType = file.type.toLowerCase(); |
| if (fileName.endsWith('.gltf') || fileName.endsWith('.glb') || fileType === 'model/gltf+json' || fileType === 'model/gltf-binary') { |
| console.log("GLTF file dropped: ".concat(file.name), file); |
| |
| _this._loadDroppedModel(file); |
| } else { |
| console.warn('Dropped file is not a recognized GLTF format:', file.name, file.type); |
| _this._showStatusScreen('"'.concat(file.name, '" is not a GLTF model.'), 'orange', false); |
| setTimeout(function() { |
| if (_this.gameOverContainer.style.display === 'block' && _this.gameOverText.innerText.includes(file.name)) { |
| _this.gameOverContainer.style.display = 'none'; |
| } |
| }, 3000); |
| } |
| event.dataTransfer.clearData(); |
| } |
| }); |
| } |
| }, |
| { |
| key: "_loadDroppedModel", |
| value: function _loadDroppedModel(file) { |
| var _this = this; |
| console.log("Processing dropped model:", file.name, file.type); |
| var reader = new FileReader(); |
| reader.onload = function(e) { |
| |
| _this._parseAndLoadGltf(e.target.result, file.name, file.type); |
| }; |
| reader.onerror = function(error) { |
| console.error("FileReader error for ".concat(file.name, ":"), error); |
| _this._showError("Error reading file ".concat(file.name, ".")); |
| |
| if (_this.gameOverContainer.style.display === 'block' && _this.gameOverText.innerText.startsWith('Loading "'.concat(file.name, '"'))) { |
| _this.gameOverContainer.style.display = 'none'; |
| } |
| }; |
| var fileNameLower = file.name.toLowerCase(); |
| var fileTypeLower = file.type ? file.type.toLowerCase() : ''; |
| if (fileNameLower.endsWith('.glb') || fileTypeLower === 'model/gltf-binary') { |
| console.log("Reading ".concat(file.name, " as ArrayBuffer.")); |
| reader.readAsArrayBuffer(file); |
| } else if (fileNameLower.endsWith('.gltf') || fileTypeLower === 'model/gltf+json') { |
| console.log("Reading ".concat(file.name, " as text.")); |
| reader.readAsText(file); |
| } else { |
| var message = file.type ? "Unsupported file type: ".concat(file.type) : 'Cannot determine file type.'; |
| console.warn("Unknown file format for GLTF loader: ".concat(file.name, ", Type: ").concat(file.type)); |
| this._showError("".concat(message, " for ").concat(file.name, ". Please drop a .gltf or .glb file.")); |
| |
| if (this.gameOverContainer.style.display === 'block' && this.gameOverText.innerText.startsWith('Loading "'.concat(file.name, '"'))) { |
| this.gameOverContainer.style.display = 'none'; |
| } |
| } |
| } |
| }, |
| { |
| key: "_parseAndLoadGltf", |
| value: function _parseAndLoadGltf(content, fileName, fileType) { |
| var _this = this; |
| var loader = new GLTFLoader(); |
| try { |
| |
| |
| |
| |
| loader.parse(content, '', function(gltf) { |
| console.log("Successfully parsed GLTF model: ".concat(fileName), gltf); |
| |
| if (_this.pandaModel) { |
| _this.scene.remove(_this.pandaModel); |
| |
| console.log("Removed previous model from scene."); |
| if (_this.animationMixer) { |
| _this.animationMixer.stopAllAction(); |
| _this.currentAction = null; |
| } |
| |
| while(_this.animationButtonsContainer.firstChild){ |
| _this.animationButtonsContainer.removeChild(_this.animationButtonsContainer.firstChild); |
| } |
| _this.animationActions = {}; |
| _this.animationClips = []; |
| } |
| |
| _this.pandaModel = gltf.scene; |
| |
| var scale = 80; |
| _this.pandaModel.scale.set(scale, scale, scale); |
| var sceneHeight = _this.renderDiv.clientHeight; |
| _this.pandaModel.position.set(0, sceneHeight * -0.45, -1000); |
| |
| _this.scene.add(_this.pandaModel); |
| console.log('Added new model "'.concat(fileName, '" to scene.')); |
| |
| _this.animationMixer = new THREE.AnimationMixer(_this.pandaModel); |
| _this.animationClips = gltf.animations; |
| _this.animationActions = {}; |
| if (_this.animationClips && _this.animationClips.length) { |
| _this.animationClips.forEach(function(clip, index) { |
| var action = _this.animationMixer.clipAction(clip); |
| var actionName = clip.name || "Animation ".concat(index + 1); |
| _this.animationActions[actionName] = action; |
| var button = document.createElement('button'); |
| button.innerText = actionName; |
| button.style.padding = '5px 10px'; |
| button.style.fontSize = '13px'; |
| button.style.backgroundColor = '#f0f0f0'; |
| button.style.color = 'black'; |
| button.style.border = '2px solid black'; |
| button.style.borderRadius = '4px'; |
| button.style.cursor = 'pointer'; |
| button.style.transition = 'background-color 0.2s ease, box-shadow 0.2s ease'; |
| button.style.boxShadow = '2px 2px 0px black'; |
| button.addEventListener('click', function() { |
| return _this._playAnimation(actionName); |
| }); |
| _this.animationButtonsContainer.appendChild(button); |
| }); |
| var defaultActionName = Object.keys(_this.animationActions)[0]; |
| var idleActionKey = Object.keys(_this.animationActions).find(function(name) { |
| return name.toLowerCase().includes('idle'); |
| }); |
| if (idleActionKey) { |
| defaultActionName = idleActionKey; |
| } |
| if (defaultActionName && _this.animationActions[defaultActionName]) { |
| _this.currentAction = _this.animationActions[defaultActionName]; |
| _this.currentAction.reset().play(); |
| _this._updateButtonStyles(defaultActionName); |
| } else { |
| _this.currentAction = null; |
| } |
| } else { |
| console.log('New model "'.concat(fileName, '" has no embedded animations.')); |
| _this.currentAction = null; |
| } |
| |
| _this.grabbingHandIndex = -1; |
| _this.pickedUpModel = null; |
| _this.rotateLastHandX = null; |
| _this.scaleInitialPinchDistance = null; |
| _this.scaleInitialModelScale = null; |
| _this.animationControlHandIndex = -1; |
| _this.animationControlInitialPinchY = null; |
| |
| _this._updateInteractionModeButtonStyles(); |
| _this.loadedDroppedModelData = null; |
| }, function(error) { |
| console.error("Error parsing GLTF model ".concat(fileName, ":"), error); |
| _this._showError('Failed to parse "'.concat(fileName, '". Model might be corrupt or unsupported. Check console.')); |
| }); |
| } catch (e) { |
| |
| console.error("Critical error during GLTF parsing setup for ".concat(fileName, ":"), e); |
| this._showError('Error setting up parser for "'.concat(fileName, '".')); |
| } |
| } |
| } |
| ]); |
| return Game; |
| }(); |