InsightUX — Eye-Tracking Research Browser

""" browser_session.py InsightUX research browser. Flow: 1. Window opens on real google.com — search and click through like a normal browser (no scraping, no fake results — it's just Google). 2. Land on whatever page you want to study. Press S to start tracking. 3. Gaze dot + AOI highlighting (same as run_session.py) kicks in and logs sessions//gaze_log.jsonl + dom_log.jsonl. 4. Press E to stop. The window auto-navigates to a generated analysis_report.html: ranked attention, dwell timeline, heatmap. Prerequisite: calibration.pkl must exist in the working directory (run calibrate.py first). Run: python browser_session.py """ import os import sys import json import time import math import threading from dataclasses import replace from datetime import datetime import cv2 import numpy as np import webview import pyautogui from preprocessing.preprocessing_pipeline import ( create_face_mesh, estimate_camera_matrix, estimate_head_pose, compute_iris_radius, compute_ear, step1_normalize, step2_illumination, LEFT_EYE_INDICES, LEFT_EAR_INDICES, LEFT_IRIS_INDICES, RIGHT_EYE_INDICES, RIGHT_EAR_INDICES, RIGHT_IRIS_INDICES, ) from inference_pipeline import InsightUXPipeline, GazeAngleSmoother from session_logger import GazeLogger from analysis import generate_report # ============================================================================= # CONFIG — kept identical to run_session.py so calibration.pkl stays valid # ============================================================================= ONNX_PATH = "models/gaze_cnn_v4.onnx" CALIBRATION_PATH = "calibration.pkl" SCREEN_W, SCREEN_H = pyautogui.size() PATCH_SOURCE = "blended" POSE_SMOOTH = 0.65 MAX_JUMP = 220 NO_FACE_RESET = 15 DOM_EVERY = 6 EURO_MINCUTOFF = 0.35 EURO_BETA = 0.12 # Rolling-median window applied to (pitch, yaw) BEFORE the RBF query. # See GazeAngleSmoother in inference_pipeline.py for why this is essential # and not merely a nicety. Must match validate.py to stay comparable. ANGLE_SMOOTH_WINDOW = 10 POSE_NORM_SCALE = 30.0 HEAD_PITCH_COMPENSATION = 0.0 # reverted — 0.35 made accuracy worse, not better HEAD_YAW_COMPENSATION = 0.0 SESSIONS_ROOT = "sessions" # The window opens on this local, fully InsightUX-branded page instead of # raw google.com. What you type here still hits REAL Google — no fake # results, no scraping — this page is just the visible skin around that # one redirect step. Typed URLs go straight there instead. _LANDING_HTML = r""" InsightUX — Eye-Tracking Research Browser

InsightUX

Eye-Tracking Research Browser

Search results and websites open below as normal.
Once you land on the page you want to study, press S to start eye-tracking, E to stop.

""" def _write_landing_page(): path = os.path.abspath("insightux_landing.html") with open(path, "w", encoding="utf-8") as f: f.write(_LANDING_HTML) return "file://" + path.replace(os.sep, "/") LANDING_URL = _write_landing_page() def normalize_pose(pitch_deg, yaw_deg, roll_deg): return np.array([ pitch_deg / POSE_NORM_SCALE, yaw_deg / POSE_NORM_SCALE, roll_deg / POSE_NORM_SCALE, ], dtype=np.float32) def compensate_pitch(raw_pitch, head_pitch_deg): return raw_pitch - np.radians(head_pitch_deg) * HEAD_PITCH_COMPENSATION def compensate_yaw(raw_yaw, head_yaw_deg): return raw_yaw - np.radians(head_yaw_deg) * HEAD_YAW_COMPENSATION # ============================================================================= # ONE EURO FILTER (display smoothing only) # ============================================================================= class OneEuroFilter: def __init__(self, mincutoff=0.8, beta=0.4, dcutoff=1.0): self.mincutoff = mincutoff self.beta = beta self.dcutoff = dcutoff self.x_prev = None self.dx_prev = 0.0 self.t_prev = None @staticmethod def _alpha(cutoff, dt): tau = 1.0 / (2 * math.pi * cutoff) return 1.0 / (1.0 + tau / dt) def __call__(self, x, t): if self.x_prev is None: self.x_prev, self.t_prev = x, t return x dt = t - self.t_prev if dt <= 0: dt = 1e-3 self.t_prev = t dx = (x - self.x_prev) / dt a_d = self._alpha(self.dcutoff, dt) dx_hat = a_d * dx + (1 - a_d) * self.dx_prev cutoff = self.mincutoff + self.beta * abs(dx_hat) a = self._alpha(cutoff, dt) x_hat = a * x + (1 - a) * self.x_prev self.x_prev, self.dx_prev = x_hat, dx_hat return x_hat def reset(self): self.x_prev = None self.dx_prev = 0.0 self.t_prev = None # ============================================================================= # JS: control widget (always present) — search hint + S/Esc keybindings # ============================================================================= CONTROL_JS = r""" (function(){ if (window.__insightuxControl) { return; } window.__insightuxControl = true; // S/E (start/stop tracking) only make sense on an actual website — not on // the InsightUX landing/search page and not on a generated insights // report page. X (quit the app) always works, on every page. const isLanding = !!document.querySelector('meta[name="insightux-landing"]'); const isReport = !!document.querySelector('meta[name="insightux-report"]'); const isTrackable = !isLanding && !isReport; if (isTrackable) { const banner = document.createElement('div'); banner.id = '__insightux_pill'; banner.style.cssText = ` position:fixed; top:0; left:0; right:0; z-index:2147483647; display:flex; align-items:center; justify-content:space-between; padding:7px 16px; font:12px -apple-system,Arial; background:linear-gradient(90deg, rgba(20,18,26,0.95), rgba(30,20,40,0.95)); border-bottom:2px solid; border-image:linear-gradient(90deg,#7B2FBE,#FF2DF0) 1; box-shadow:0 3px 14px rgba(0,0,0,0.35); pointer-events:none; `; banner.innerHTML = ` InsightUX Eye + Mouse Research Browser Press S to start tracking · E to stop · H heatmap · M mouse panel · X quit `; document.documentElement.appendChild(banner); setInterval(function(){ if (!document.documentElement.contains(banner)) document.documentElement.appendChild(banner); }, 1000); } const statusEl = () => document.getElementById('__insightux_status'); window.insightuxSetStatus = function(text){ const el = statusEl(); if (el) el.textContent = text; }; function isTypingTarget(el){ if (!el) return false; const tag = el.tagName ? el.tagName.toLowerCase() : ''; return tag === 'input' || tag === 'textarea' || tag === 'select' || el.isContentEditable === true; } document.addEventListener('keydown', function(e){ const typingBlocked = isTypingTarget(e.target) || isTypingTarget(document.activeElement); const apiReady = !!(window.pywebview && window.pywebview.api); if (e.key === 'x' || e.key === 'X') { if (typingBlocked || !apiReady) return; window.pywebview.api.quit_app(); return; } if (!isTrackable) return; // S/E do nothing on the landing page or a report page if (e.key === 's' || e.key === 'S' || e.key === 'e' || e.key === 'E') { console.log('[insightux] key=' + e.key, 'typingBlocked=' + typingBlocked, 'apiReady=' + apiReady, 'focusedEl=' + (document.activeElement && document.activeElement.tagName)); } if (!apiReady) return; if (typingBlocked) { window.insightuxSetStatus("Click somewhere on the page (not a text field) before pressing S/E"); return; } if (e.key === 's' || e.key === 'S') { window.pywebview.api.start_tracking(); } else if (e.key === 'e' || e.key === 'E') { window.pywebview.api.stop_tracking(); } }, true); // capture phase — fires before page scripts can intercept/stop the event })(); """ # JS: tracking overlay (gaze dot + AOI highlighting) — injected only once # tracking actually starts. Ported directly from run_session.py's JS_SETUP. TRACKING_JS = r""" (function(){ if (window.__insightux) { return; } const state = { aois: [] }; const dwell = { pendLabel: null, pendSince: 0, activeLabel: null, emptySince: 0 }; const DWELL_MS = 420; const RELEASE_MS = 1200; let activeBox = null; const target = { fx: 0.5, fy: 0.5 }; const dot = { x: null, y: null }; const DOT_LERP = 0.07; const cv = document.createElement('canvas'); cv.id = '__insightux_canvas'; cv.style.cssText = 'position:fixed;left:0;top:0;width:100vw;height:100vh;pointer-events:none;z-index:2147483646;'; (document.body || document.documentElement).appendChild(cv); const ctx = cv.getContext('2d'); function resize(){ cv.width = window.innerWidth; cv.height = window.innerHeight; } resize(); window.addEventListener('resize', resize, {passive:true}); setInterval(function(){ if (!document.body.contains(cv)) document.body.appendChild(cv); }, 1000); const MIN_W=40, MIN_H=24, MAX_AOIS=90, MAX_SCAN=2500; const PAD_PX = 90; const TALL_WRAPPER_H = 220; const ALWAYS_SELECTOR = "nav, header, footer, img, video, iframe, h1, h2, h3, button, figure, [class*='hero'], [class*='banner'], [class*='card']"; const TEXT_CONTAINER_SELECTOR = "div, span, section, article, main, aside, p, li"; function hasOwnText(el){ for (const node of el.childNodes){ if (node.nodeType === 3 && node.textContent.trim().length > 2) return true; } return false; } function labelFor(el){ if (el.dataset && el.dataset.aoi) return el.dataset.aoi.slice(0,40); const tag = el.tagName.toLowerCase(); if (tag==='nav') return 'navbar'; if (tag==='header') return 'header'; if (tag==='footer') return 'footer'; if (tag==='img'){ const alt=(el.getAttribute('alt')||'').trim(); if (alt) return 'img: '+alt.slice(0,30); const src=el.getAttribute('src')||''; const name=src.split('/').pop().split('?')[0]; return 'img: '+(name||'image').slice(0,30); } if (tag==='video') return 'video'; if (tag==='iframe') return 'embed'; if (tag==='h1'||tag==='h2'){ const t=(el.innerText||'').trim().replace(/\s+/g,' '); if (t) return tag+': '+t.slice(0,30); } const id=el.id?('#'+el.id):''; let cls=''; if (el.className && typeof el.className==='string'){ const f=el.className.trim().split(/\s+/)[0]; if (f) cls='.'+f; } const txt=(el.innerText||'').trim().replace(/\s+/g,' ').slice(0,24); const base=id||cls||tag; return txt?(base+' ('+txt+')'):base; } function refresh(){ const vh = window.innerHeight; const raw = []; const seen = new Set(); let full = false; function consider(el){ if (full) return; // Never track InsightUX's own injected UI (the "press S/E" pill, // the gaze-dot canvas, the mouse tracker panel) as if it were page content. if (el.id === '__insightux_pill' || el.id === '__insightux_canvas' || el.id === '__insightux_mouse_panel') return; if (el.closest && el.closest('#__insightux_pill, #__insightux_canvas, #__insightux_mouse_panel')) return; const tag = el.tagName.toLowerCase(); const explicit = !!(el.dataset && el.dataset.aoi); if (!explicit){ const isAlways = (tag==='nav'||tag==='header'||tag==='footer'||tag==='img'|| tag==='video'||tag==='iframe'|| tag==='h1'||tag==='h2'||tag==='h3'||tag==='button'||tag==='figure') || (el.className && typeof el.className==='string' && /hero|banner|card/i.test(el.className)); if (!isAlways && !hasOwnText(el)) return; } const r = el.getBoundingClientRect(); if (r.widthvh) return; const label = labelFor(el); if (!label) return; const key = label+'@'+Math.round(r.left)+','+Math.round(r.top); if (seen.has(key)) return; seen.add(key); const pos = getComputedStyle(el).position; raw.push({el:el, label:label, x:Math.round(r.left), y:Math.round(r.top), w:Math.round(r.width), h:Math.round(r.height), sticky:(pos==='sticky'||pos==='fixed')}); if (raw.length >= MAX_AOIS*2) full = true; } document.querySelectorAll('[data-aoi]').forEach(consider); document.querySelectorAll(ALWAYS_SELECTOR).forEach(consider); const candidates = document.querySelectorAll(TEXT_CONTAINER_SELECTOR); for (let i=0; i { const isTallWrapper = o.h > TALL_WRAPPER_H && raw.some(o2 => o2 !== o && o.el.contains(o2.el)); return !isTallWrapper; }); state.aois = filtered.slice(0, MAX_AOIS).map(o => ({ label:o.label, x:o.x, y:o.y, w:o.w, h:o.h, sticky:o.sticky })); } refresh(); window.addEventListener('scroll', refresh, {passive:true}); setInterval(refresh, 400); window.insightuxAOIs = function(){ return JSON.stringify({ url: location.href, scrollX: Math.round(window.scrollX), scrollY: Math.round(window.scrollY), viewport:{w:window.innerWidth,h:window.innerHeight}, page:{w:document.documentElement.scrollWidth,h:document.documentElement.scrollHeight}, aois: state.aois }); }; window.insightuxUpdate = function(fx, fy){ target.fx = fx; target.fy = fy; }; function renderLoop(){ const w = window.innerWidth, h = window.innerHeight; const tx = target.fx * w, ty = target.fy * h; if (dot.x === null){ dot.x = tx; dot.y = ty; } dot.x += (tx - dot.x) * DOT_LERP; dot.y += (ty - dot.y) * DOT_LERP; const now = performance.now(); const px = dot.x, py = dot.y; let cand = null; for (const a of state.aois){ if (px>=a.x-PAD_PX && px<=a.x+a.w+PAD_PX && py>=a.y-PAD_PX && py<=a.y+a.h+PAD_PX){ if (!cand || (a.w*a.h)<(cand.w*cand.h)) cand = a; } } const candLabel = cand ? cand.label : null; if (candLabel !== dwell.pendLabel){ dwell.pendLabel = candLabel; dwell.pendSince = now; } if (cand){ dwell.emptySince = 0; if (dwell.activeLabel !== cand.label && (now - dwell.pendSince) >= DWELL_MS){ dwell.activeLabel = cand.label; } } else { if (dwell.emptySince === 0) dwell.emptySince = now; if (dwell.activeLabel && (now - dwell.emptySince) >= RELEASE_MS){ dwell.activeLabel = null; } } let active = null; if (dwell.activeLabel){ for (const a of state.aois){ if (a.label === dwell.activeLabel){ active = a; break; } } if (!active) dwell.activeLabel = null; } activeBox = active; ctx.clearRect(0,0,cv.width,cv.height); if (activeBox){ ctx.fillStyle = 'rgba(123,47,190,0.16)'; ctx.fillRect(activeBox.x, activeBox.y, activeBox.w, activeBox.h); ctx.strokeStyle = '#7B2FBE'; ctx.lineWidth = 3; ctx.strokeRect(activeBox.x, activeBox.y, activeBox.w, activeBox.h); const label = activeBox.label; ctx.font = 'bold 13px Arial'; const tw = ctx.measureText(label).width + 16; const ly = Math.max(0, activeBox.y - 22); ctx.fillStyle = '#7B2FBE'; ctx.fillRect(activeBox.x, ly, tw, 20); ctx.fillStyle = '#FFFFFF'; ctx.fillText(label, activeBox.x + 8, ly + 14); } ctx.beginPath(); ctx.arc(dot.x, dot.y, 13, 0, 2*Math.PI); ctx.fillStyle = 'rgba(255,255,255,0.9)'; ctx.fill(); ctx.beginPath(); ctx.arc(dot.x, dot.y, 9, 0, 2*Math.PI); ctx.fillStyle = '#FF2DF0'; ctx.fill(); ctx.lineWidth = 2; ctx.strokeStyle = 'rgba(0,0,0,0.55)'; ctx.stroke(); requestAnimationFrame(renderLoop); } requestAnimationFrame(renderLoop); window.__insightux = true; })(); """ # JS: mouse tracking overlay — ported feature-for-feature from the # "Mouse Tracker & Heatmap" Chrome extension (Mouse/content.js + background.js # + popup.js), adapted to run as a single injected script instead of a # content-script/background/popup trio (pywebview has no extension host). # Same trail/heatmap/dwell sampling loop, same click interest labeling, same # heatmap render, plus an in-page panel that replaces the extension's popup # (Start/Stop/Clear/Toggle Heatmap/View Logs, timer, "Most Viewed Element"). # ============================================================================= MOUSE_JS = r""" (function(){ if (window.__insightuxMouse) { return; } window.__insightuxMouse = true; let isTracking = true; let canvas = null; let panelOpen = false; let logsOpen = false; // Per-sync buffers (flushed to Python every 2s for on-disk persistence) let trailBuffer = []; let heatmapBuffer = []; let clickBuffer = []; let dwellBuffer = []; // Cumulative in-page state (mirrors the extension's background.js state) let allTrail = []; let allHeatmap = []; let allClicks = []; let dwellTotals = {}; let sessionStart = Date.now(); let sessionStop = null; let lastClientX = 0, lastClientY = 0; let lastPageX = 0, lastPageY = 0; let lastSampleTime = 0; const sampleRate = 50; const DWELL_THRESHOLD = 5000; let stationaryStart = 0; let isStationary = false; window.insightuxMouseSetTracking = function(on){ isTracking = !!on; if (isTracking) { sessionStart = Date.now(); sessionStop = null; } else { sessionStop = Date.now(); flushDwell(); } updatePanel(); }; // ---- sampling loop (trail while moving, heatmap dwell points while still) ---- setInterval(function(){ if (!isTracking) return; const currentScrollX = window.scrollX, currentScrollY = window.scrollY; if (lastPageX === 0 && lastPageY === 0 && lastClientX !== 0) { lastPageX = lastClientX + currentScrollX; lastPageY = lastClientY + currentScrollY; } const currentPageX = (lastClientX !== 0) ? (lastClientX + currentScrollX) : lastPageX; const currentPageY = (lastClientY !== 0) ? (lastClientY + currentScrollY) : lastPageY; if (currentPageX === 0 && currentPageY === 0) return; const now = Date.now(); const dt = now - lastSampleTime; if (dt > 1000) { lastSampleTime = now; return; } const dx = currentPageX - lastPageX, dy = currentPageY - lastPageY; const distance = Math.sqrt(dx * dx + dy * dy); if (distance > 2) { const pt = { x: currentPageX, y: currentPageY }; trailBuffer.push(pt); allTrail.push(pt); lastPageX = currentPageX; lastPageY = currentPageY; isStationary = false; stationaryStart = 0; } else { if (!isStationary) { isStationary = true; stationaryStart = now; } else { const dwellDuration = now - stationaryStart; if (dwellDuration > DWELL_THRESHOLD) { const pt = { x: currentPageX, y: currentPageY }; heatmapBuffer.push(pt); allHeatmap.push(pt); } } } lastSampleTime = now; }, sampleRate); // ---- dwell / element-interest tracking ---- let currentHoverLabel = null, currentHoverStartTime = 0; function handleHoverChange(newLabel){ if (!isTracking) return; if (newLabel !== currentHoverLabel) { const now = Date.now(); if (currentHoverLabel && currentHoverStartTime > 0) { const duration = now - currentHoverStartTime; if (duration > 10) recordDwell(currentHoverLabel, duration); } currentHoverLabel = newLabel; currentHoverStartTime = newLabel ? now : 0; } } function recordDwell(element, duration){ dwellBuffer.push({ element: element, duration: duration }); dwellTotals[element] = (dwellTotals[element] || 0) + duration; } function flushDwell(){ if (currentHoverLabel && currentHoverStartTime > 0) { const now = Date.now(); const duration = now - currentHoverStartTime; if (duration > 50) recordDwell(currentHoverLabel, duration); currentHoverStartTime = now; } } // ---- sync to Python every 2s -> mouse_log.jsonl in the session folder ---- setInterval(function(){ flushDwell(); if (trailBuffer.length || heatmapBuffer.length || clickBuffer.length || dwellBuffer.length) { const payload = { trail: trailBuffer.length ? trailBuffer : null, heatmap: heatmapBuffer.length ? heatmapBuffer : null, click: clickBuffer.length ? clickBuffer : null, dwell: dwellBuffer.length ? dwellBuffer : null, }; trailBuffer = []; heatmapBuffer = []; clickBuffer = []; dwellBuffer = []; if (window.pywebview && window.pywebview.api && window.pywebview.api.log_mouse_data) { try { window.pywebview.api.log_mouse_data(payload).catch(function(){}); } catch (e) {} } } updatePanel(); }, 2000); // ---- exclude InsightUX's own injected UI from every measurement ---- // (the status pill, the mouse-tracker panel, the heatmap canvas) — only // real website elements should ever show up in trail/heatmap/dwell/clicks. function isOwnUI(el){ if (!el) return false; if (el.id === '__insightux_pill' || el.id === '__insightux_canvas' || el.id === '__insightux_mouse_panel') return true; return !!(el.closest && el.closest('#__insightux_pill, #__insightux_canvas, #__insightux_mouse_panel')); } // ---- mouse position + hover tracking ---- function updateMousePos(e){ if (isOwnUI(e.target)) return; lastClientX = e.clientX; lastClientY = e.clientY; if (lastPageX === 0) lastPageX = e.pageX; if (lastPageY === 0) lastPageY = e.pageY; if (isTracking) handleHoverChange(getSmartLabel(e.target)); } document.addEventListener('mousemove', updateMousePos, true); document.addEventListener('mouseenter', updateMousePos, true); document.addEventListener('mouseover', updateMousePos, true); document.addEventListener('click', updateMousePos, true); document.addEventListener('mouseleave', function(){ lastClientX = 0; lastClientY = 0; handleHoverChange(null); }, true); document.addEventListener('click', function(e){ if (!isTracking) return; if (e.target === canvas) return; if (isOwnUI(e.target)) return; if (!isInteractable(e.target)) return; const x = e.pageX, y = e.pageY; const label = getSmartLabel(e.target) || e.target.tagName; if (['BODY', 'HTML', 'DIV', 'SPAN'].includes(label) && !e.target.innerText.trim()) return; const logEntry = { timestamp: new Date().toLocaleTimeString(), x: x, y: y, element: label, text: e.target.innerText ? e.target.innerText.substring(0, 30).replace(/(\r\n|\n|\r)/gm, ' ').trim() : '', url: window.location.href }; clickBuffer.push(logEntry); allClicks.push(logEntry); if (canvas) drawClick(x, y); updatePanel(); }, true); function isInteractable(el){ if (!el) return false; const tag = el.tagName.toLowerCase(); if (['a', 'button', 'input', 'select', 'textarea', 'details', 'summary', 'label'].includes(tag)) return true; const role = el.getAttribute('role'); if (role === 'button' || role === 'link' || role === 'menuitem' || role === 'tab') return true; try { if (window.getComputedStyle(el).cursor === 'pointer') return true; } catch (e) {} let parent = el.parentElement, depth = 0; while (parent && depth < 3) { const pTag = parent.tagName.toLowerCase(); if (['a', 'button'].includes(pTag)) return true; if (parent.getAttribute('role') === 'button') return true; parent = parent.parentElement; depth++; } if (tag === 'code' || tag === 'pre' || el.classList.contains('code') || el.closest('pre')) return true; return false; } function getSmartLabel(el){ if (!el) return null; const tag = el.tagName.toLowerCase(); if (['body', 'html', 'main', 'div', 'span', 'section', 'article'].includes(tag)) { if (el.id && (el.id.includes('logo') || el.id.includes('wrapper') || el.id.includes('container'))) return null; if (el.className && typeof el.className === 'string' && (el.className.toLowerCase().includes('logo') || el.className.toLowerCase().includes('brand'))) return null; const aria = el.getAttribute('aria-label'); if (aria) return 'Element: ' + aria; if (el.children.length === 0) { const txt = el.innerText.trim(); if (txt.length > 2 && txt.length < 50 && /[a-zA-Z0-9]/.test(txt)) return 'Element: ' + txt; } return null; } if (tag === 'a') return 'Link: ' + (el.innerText.trim().substring(0, 30) || 'Link'); if (tag === 'button') return 'Button: ' + (el.innerText.trim().substring(0, 30) || 'Button'); if (tag === 'input') return 'Input: ' + (el.placeholder || el.name || el.id || 'Input'); if (tag === 'textarea') return 'Input: Text Area'; if (tag === 'img') return 'Image: ' + (el.alt || 'Image'); if (['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(tag)) return tag.toUpperCase() + ': ' + el.innerText.trim().substring(0, 40); if (tag === 'code' || tag === 'pre') return 'Code: ' + el.innerText.trim().substring(0, 30); const text = el.innerText.trim(); if (text && text.length > 2) { if (el.classList.contains('code') || el.closest('pre')) return 'Code: ' + text.substring(0, 30); if (text.toLowerCase().includes('no message found')) return null; const isHex = /^#[0-9A-F]{6}$/i.test(text) || /^#[0-9A-F]{3}$/i.test(text); const isColorName = ['red', 'blue', 'green', 'yellow', 'black', 'white', 'orange', 'purple', 'gray', 'grey', 'pink', 'brown', 'cyan', 'magenta'].includes(text.toLowerCase()); if (isHex || isColorName) return null; if (text.length > 50) return 'Text: ' + text.substring(0, 47) + '...'; return 'Text: ' + text; } return null; } // ---- heatmap overlay canvas (ported 1:1 from the extension) ---- window.insightuxMouseToggleHeatmap = function(){ if (canvas) { document.body.removeChild(canvas); canvas = null; return; } canvas = document.createElement('canvas'); canvas.style.position = 'absolute'; canvas.style.top = '0'; canvas.style.left = '0'; canvas.style.zIndex = '2147483645'; canvas.style.pointerEvents = 'none'; canvas.width = Math.max(document.documentElement.scrollWidth, document.body.scrollWidth, document.documentElement.offsetWidth); canvas.height = Math.max(document.documentElement.scrollHeight, document.body.scrollHeight, document.documentElement.offsetHeight); document.body.appendChild(canvas); drawHeatmapHighQuality(allHeatmap, allTrail, allClicks); }; function drawClick(x, y){ if (!canvas) return; const ctx = canvas.getContext('2d'); ctx.save(); ctx.beginPath(); ctx.strokeStyle = '#00FF00'; ctx.lineWidth = 3; ctx.shadowColor = 'black'; ctx.shadowBlur = 2; const size = 10; ctx.moveTo(x - size, y - size); ctx.lineTo(x + size, y + size); ctx.moveTo(x + size, y - size); ctx.lineTo(x - size, y + size); ctx.stroke(); ctx.beginPath(); ctx.arc(x, y, size + 5, 0, Math.PI * 2); ctx.stroke(); ctx.restore(); } function drawHeatmapHighQuality(heatmapData, trailData, clickData){ if (!canvas) return; const ctx = canvas.getContext('2d'); ctx.clearRect(0, 0, canvas.width, canvas.height); let allHeatPoints = []; if (heatmapData && heatmapData.length) allHeatPoints = allHeatPoints.concat(heatmapData); if (trailData && trailData.length) allHeatPoints = allHeatPoints.concat(trailData); if (clickData && clickData.length) { clickData.forEach(function(c){ for (let i = 0; i < 5; i++) allHeatPoints.push({ x: c.x, y: c.y }); }); } if (!allHeatPoints.length) return; const radius = 60; const brushCanvas = document.createElement('canvas'); brushCanvas.width = radius * 2; brushCanvas.height = radius * 2; const brushCtx = brushCanvas.getContext('2d'); const g = brushCtx.createRadialGradient(radius, radius, 0, radius, radius, radius); g.addColorStop(0, 'rgba(0, 0, 0, 0.05)'); g.addColorStop(1, 'rgba(0, 0, 0, 0)'); brushCtx.fillStyle = g; brushCtx.fillRect(0, 0, radius * 2, radius * 2); allHeatPoints.forEach(function(point){ ctx.drawImage(brushCanvas, point.x - radius, point.y - radius); }); const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); const data = imageData.data; const gradientMap = createGradientMap(); for (let i = 0; i < data.length; i += 4) { const alpha = data[i + 3]; if (alpha > 0) { let mapIndex = Math.floor(alpha * 1.5); if (mapIndex > 255) mapIndex = 255; const cIndex = mapIndex * 4; data[i] = gradientMap[cIndex]; data[i + 1] = gradientMap[cIndex + 1]; data[i + 2] = gradientMap[cIndex + 2]; data[i + 3] = Math.min(255, 150 + alpha); } } ctx.putImageData(imageData, 0, 0); } function createGradientMap(){ const c = document.createElement('canvas'); c.width = 256; c.height = 1; const ctx = c.getContext('2d'); const g = ctx.createLinearGradient(0, 0, 256, 0); g.addColorStop(0.0, 'rgba(0, 0, 255, 0)'); g.addColorStop(0.1, 'rgba(0, 0, 255, 1)'); g.addColorStop(0.4, 'rgba(0, 255, 255, 1)'); g.addColorStop(0.6, 'rgba(0, 255, 0, 1)'); g.addColorStop(0.8, 'rgba(255, 255, 0, 1)'); g.addColorStop(1.0, 'rgba(255, 0, 0, 1)'); ctx.fillStyle = g; ctx.fillRect(0, 0, 256, 1); return ctx.getImageData(0, 0, 256, 1).data; } window.addEventListener('resize', function(){ if (canvas) { canvas.width = Math.max(document.documentElement.scrollWidth, document.body.scrollWidth, document.documentElement.offsetWidth); canvas.height = Math.max(document.documentElement.scrollHeight, document.body.scrollHeight, document.documentElement.offsetHeight); } }); // ---- insights panel: in-page port of the extension's popup.html/popup.js ---- const style = document.createElement('style'); style.textContent = ` #__insightux_mouse_panel { position: fixed; top: 44px; right: 16px; width: 260px; z-index: 2147483647; background: linear-gradient(180deg, rgba(28,24,38,0.97), rgba(20,18,26,0.97)); border: 1px solid #3a3348; border-radius: 12px; box-shadow: 0 8px 28px rgba(0,0,0,0.45); font: 12px -apple-system, 'Segoe UI', Arial; color: #f0ecf7; padding: 14px; pointer-events: auto; display: none; } #__insightux_mouse_panel.open { display: block; } #__insightux_mouse_panel h3 { margin: 0 0 8px 0; font-size: 13px; color: #f0ecf7; display:flex; align-items:center; justify-content:space-between; } #__insightux_mouse_panel .muted { color: #9a92ad; font-size: 11px; } #__insightux_mouse_panel .row { display: flex; gap: 6px; margin-bottom: 8px; flex-wrap: wrap; } #__insightux_mouse_panel button { flex: 1 1 auto; padding: 6px 8px; font-size: 11px; border-radius: 6px; border: 1px solid #3a3348; background: #241f2e; color: #f0ecf7; cursor: pointer; } #__insightux_mouse_panel button:hover { background: #302a3d; } #__insightux_mouse_panel .timer { font-weight: 600; margin-bottom: 6px; } #__insightux_mouse_panel .hero { background: rgba(123,47,190,0.15); border: 1px solid #7B2FBE; border-radius: 8px; padding: 8px; text-align: center; margin-bottom: 8px; } #__insightux_mouse_panel .hero .lbl { font-size: 9px; text-transform: uppercase; color: #c9a6f5; letter-spacing: 0.05em; } #__insightux_mouse_panel .hero .el { font-weight: 600; margin: 3px 0; color: #FF2DF0; } #__insightux_mouse_panel .interest-item, #__insightux_mouse_panel .log-item { background: #241f2e; border-radius: 6px; padding: 5px 7px; margin-bottom: 4px; font-size: 11px; } #__insightux_mouse_panel .list { max-height: 160px; overflow-y: auto; } `; document.head.appendChild(style); const panel = document.createElement('div'); panel.id = '__insightux_mouse_panel'; panel.innerHTML = `

Mouse Tracker idle

Session Time: 00:00

`; document.documentElement.appendChild(panel); document.getElementById('__mt_clear').addEventListener('click', function(){ trailBuffer = []; heatmapBuffer = []; clickBuffer = []; dwellBuffer = []; allTrail = []; allHeatmap = []; allClicks = []; dwellTotals = {}; sessionStart = Date.now(); sessionStop = null; if (canvas) { document.body.removeChild(canvas); canvas = null; } updatePanel(); }); document.getElementById('__mt_heatmap').addEventListener('click', function(){ window.insightuxMouseToggleHeatmap(); }); document.getElementById('__mt_logs').addEventListener('click', function(){ logsOpen = !logsOpen; document.getElementById('__mt_logs_list').style.display = logsOpen ? 'block' : 'none'; document.getElementById('__mt_interests').style.display = logsOpen ? 'none' : 'block'; updatePanel(); }); window.insightuxMouseTogglePanel = function(){ panelOpen = !panelOpen; panel.classList.toggle('open', panelOpen); if (panelOpen) updatePanel(); }; function formatTime(ms){ return Math.round(ms / 1000) + 's'; } function updatePanel(){ if (!panelOpen) return; document.getElementById('__mt_status').textContent = isTracking ? 'tracking' : 'stopped'; const startTs = sessionStart; const diff = isTracking ? Math.floor((Date.now() - startTs) / 1000) : (sessionStop ? Math.floor((sessionStop - startTs) / 1000) : 0); const mins = Math.floor(diff / 60).toString().padStart(2, '0'); const secs = (diff % 60).toString().padStart(2, '0'); document.getElementById('__mt_timer').textContent = 'Session Time: ' + mins + ':' + secs; const interests = Object.entries(dwellTotals).map(function(e){ return { element: e[0], duration: e[1] }; }) .sort(function(a, b){ return b.duration - a.duration; }).slice(0, 6); const hero = document.getElementById('__mt_hero'); if (interests.length) { hero.style.display = 'block'; document.getElementById('__mt_hero_el').textContent = interests[0].element; document.getElementById('__mt_hero_time').textContent = formatTime(interests[0].duration); } else { hero.style.display = 'none'; } const interestsEl = document.getElementById('__mt_interests'); if (!logsOpen) { if (interests.length > 1) { interestsEl.innerHTML = interests.slice(1).map(function(it, i){ return '

' + (i + 2) + '. ' + it.element + ' — ' + formatTime(it.duration) + '

'; }).join(''); } else { interestsEl.innerHTML = '

No other interests yet.

'; } } const logsEl = document.getElementById('__mt_logs_list'); if (logsOpen) { if (allClicks.length) { logsEl.innerHTML = allClicks.slice().reverse().slice(0, 20).map(function(l){ return '

' + l.timestamp + '
' + l.element + '
"' + l.text + '"

'; }).join(''); } else { logsEl.innerHTML = '

No clicks recorded.

'; } } } setInterval(updatePanel, 1000); // ---- hotkeys: H = toggle heatmap, M = toggle insights panel ---- function isTypingTarget(el){ if (!el) return false; const tag = el.tagName ? el.tagName.toLowerCase() : ''; return tag === 'input' || tag === 'textarea' || tag === 'select' || el.isContentEditable === true; } document.addEventListener('keydown', function(e){ const typingBlocked = isTypingTarget(e.target) || isTypingTarget(document.activeElement); if (typingBlocked) return; if (e.key === 'h' || e.key === 'H') window.insightuxMouseToggleHeatmap(); else if (e.key === 'm' || e.key === 'M') window.insightuxMouseTogglePanel(); }, true); updatePanel(); })(); """ # ============================================================================= # API exposed to JS — start/stop/quit are the entry points # ============================================================================= # Pages tracking must never start on, checked against window.get_current_url() # as a Python-side backstop to CONTROL_JS's isTrackable check (belt and # suspenders — the JS guard can't run before the JS bridge is up). _NON_TRACKABLE_URL_MARKERS = ("insightux_landing.html", "analysis_report.html") class Api: def __init__(self): self.window = None self.tracking = False self.stop_event = threading.Event() self.thread = None self.mouse_log_f = None self.mouse_close_timer = None def start_tracking(self): print("[browser_session] start_tracking() called from JS") if self.tracking: print("[browser_session] already tracking, ignoring") return False try: current_url = self.window.get_current_url() or "" except Exception: current_url = "" if any(marker in current_url for marker in _NON_TRACKABLE_URL_MARKERS): print(f"[browser_session] refusing to start on non-website page: {current_url}") self._set_status("Start tracking only works on a website — search or open a page first.") return False if not os.path.exists(CALIBRATION_PATH): print(f"[browser_session] no {CALIBRATION_PATH} found — run calibrate.py first") self._set_status("No calibration.pkl found — run calibrate.py first.") return False self.tracking = True self.stop_event.clear() session_dir = os.path.join(SESSIONS_ROOT, datetime.now().strftime("%Y%m%d_%H%M%S")) os.makedirs(session_dir, exist_ok=True) print(f"[browser_session] starting tracking thread -> {session_dir}") self._inject_tracking_overlay() self._inject_mouse_overlay() self._open_mouse_log(session_dir) self._set_mouse_tracking(True) self._set_status("Recording gaze + mouse... Press E to stop, H for heatmap, M for mouse panel") self.thread = threading.Thread( target=gaze_worker, args=(self.window, self.stop_event, session_dir, self), daemon=True ) self.thread.start() return True def stop_tracking(self): print("[browser_session] stop_tracking() called from JS") if not self.tracking: print("[browser_session] not currently tracking, ignoring") return False self._set_status("Wrapping up your session...") self._set_mouse_tracking(False) self.stop_event.set() # Give the page a moment to flush its last batch of mouse data over # log_mouse_data() before the file handle is closed. if self.mouse_close_timer: self.mouse_close_timer.cancel() self.mouse_close_timer = threading.Timer(2.5, self._close_mouse_log) self.mouse_close_timer.daemon = True self.mouse_close_timer.start() return True def quit_app(self): # X is an emergency kill switch, not a graceful shutdown — the user # wants the terminal process gone immediately, camera/threads and all. print("[browser_session] quit_app() called from JS -- terminating process") os._exit(0) def _inject_tracking_overlay(self): try: self.window.evaluate_js(TRACKING_JS) except Exception as e: print(f"[browser_session] overlay inject failed: {e}") def _inject_mouse_overlay(self): try: self.window.evaluate_js(MOUSE_JS) except Exception as e: print(f"[browser_session] mouse overlay inject failed: {e}") def _set_mouse_tracking(self, on): try: flag = "true" if on else "false" self.window.evaluate_js( f"window.insightuxMouseSetTracking && window.insightuxMouseSetTracking({flag})" ) except Exception: pass def _open_mouse_log(self, session_dir): self._close_mouse_log() path = os.path.join(session_dir, "mouse_log.jsonl") try: self.mouse_log_f = open(path, "w", buffering=1) self.mouse_log_f.write(json.dumps({"type": "meta", "t": round(time.time(), 4)}) + "\n") print(f"[browser_session] writing mouse stream to {path}") except Exception as e: print(f"[browser_session] could not open mouse log: {e}") self.mouse_log_f = None def _close_mouse_log(self): if self.mouse_log_f: try: self.mouse_log_f.close() print("[browser_session] mouse log closed") except Exception: pass self.mouse_log_f = None def log_mouse_data(self, payload): """Called from the injected Mouse Tracker overlay (MOUSE_JS) roughly every 2s while tracking, with a batch of trail / heatmap / click / dwell records — same cadence as the extension's background.js sync loop, just persisted to mouse_log.jsonl instead of chrome.storage.""" if not self.mouse_log_f: return False try: rec = dict(payload) if isinstance(payload, dict) else {} rec["type"] = "mouse_batch" rec["t"] = round(time.time(), 4) self.mouse_log_f.write(json.dumps(rec) + "\n") except Exception as e: print(f"[browser_session] mouse log write failed: {e}") return True def _set_status(self, text): try: self.window.evaluate_js(f"window.insightuxSetStatus && window.insightuxSetStatus({json.dumps(text)})") except Exception: pass api = Api() def inject_control(window): try: window.evaluate_js(CONTROL_JS) except Exception as e: print(f"[browser_session] control inject failed (will retry): {e}") # ============================================================================= # GAZE WORKER — runs in a background thread, started/stopped via Api # ============================================================================= def gaze_worker(window, stop_event, session_dir, api_ref): pipeline = InsightUXPipeline(ONNX_PATH, CALIBRATION_PATH) face_mesh = create_face_mesh(static_image_mode=False) cap = cv2.VideoCapture(0) logger = GazeLogger(SCREEN_W, SCREEN_H, session_dir=session_dir) dom_path = os.path.join(session_dir, "dom_log.jsonl") dom_f = open(dom_path, "w", buffering=1) screens_dir = os.path.join(session_dir, "screenshots") os.makedirs(screens_dir, exist_ok=True) shot_count = 0 last_shot_scroll = None last_shot_time = 0.0 SCROLL_SHOT_THRESHOLD = 60 # px scrolled before a new snapshot is worth taking MAX_SHOT_INTERVAL = 2.5 # seconds — take one anyway if you just sit still def maybe_capture_screenshot(scroll_y): """Grab a full-screen shot when the page has scrolled enough, or periodically even if it hasn't — this is what lets the report show the heatmap ON TOP of the actual page instead of floating in a void.""" nonlocal shot_count, last_shot_scroll, last_shot_time now = time.time() scrolled_enough = last_shot_scroll is None or abs(scroll_y - last_shot_scroll) >= SCROLL_SHOT_THRESHOLD time_elapsed = (now - last_shot_time) >= MAX_SHOT_INTERVAL if not (scrolled_enough or time_elapsed): return None try: img = pyautogui.screenshot() # Force to the same pixel space as gaze coordinates (SCREEN_W x # SCREEN_H) — Windows DPI scaling can otherwise return a # screenshot at a different resolution than pyautogui.size(), # which would silently misalign every heatmap point. if img.size != (SCREEN_W, SCREEN_H): img = img.resize((SCREEN_W, SCREEN_H)) shot_name = f"shot_{shot_count:04d}.png" img.save(os.path.join(screens_dir, shot_name)) shot_count += 1 last_shot_scroll = scroll_y last_shot_time = now return f"screenshots/{shot_name}" except Exception as e: print(f"[browser_session] screenshot capture failed: {e}") return None print(f"[browser_session] tracking started -> {session_dir}") cam_matrix = None last_x = SCREEN_W / 2 last_y = SCREEN_H / 2 fil_x = OneEuroFilter(mincutoff=EURO_MINCUTOFF, beta=EURO_BETA) fil_y = OneEuroFilter(mincutoff=EURO_MINCUTOFF, beta=EURO_BETA) angle_smoother = GazeAngleSmoother(window=ANGLE_SMOOTH_WINDOW) sm_pitch = sm_yaw = sm_roll = None no_face_count = 0 frame_i = 0 while not stop_event.is_set(): ret, frame = cap.read() if not ret: break if cam_matrix is None: cam_matrix = estimate_camera_matrix(frame.shape) rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) results = face_mesh.process(rgb) if not results.multi_face_landmarks: no_face_count += 1 if no_face_count >= NO_FACE_RESET: last_x, last_y = SCREEN_W / 2, SCREEN_H / 2 fil_x.reset(); fil_y.reset() angle_smoother.reset() sm_pitch = sm_yaw = sm_roll = None no_face_count = 0 continue no_face_count = 0 lms = results.multi_face_landmarks[0].landmark head_pose = estimate_head_pose(lms, frame.shape, cam_matrix) if head_pose is None: continue if sm_pitch is None: sm_pitch, sm_yaw, sm_roll = head_pose.pitch, head_pose.yaw, head_pose.roll else: b = POSE_SMOOTH sm_pitch = b*sm_pitch + (1-b)*head_pose.pitch sm_yaw = b*sm_yaw + (1-b)*head_pose.yaw sm_roll = b*sm_roll + (1-b)*head_pose.roll head_pose = replace(head_pose, pitch=sm_pitch, yaw=sm_yaw, roll=sm_roll) pose_vec = normalize_pose(sm_pitch, sm_yaw, sm_roll) # Eye aperture — the vertical cue. Calibration decides whether this or # the CNN's pitch actually tracks screen-Y, and uses the better one. ear_now = 0.5 * (compute_ear(lms, LEFT_EAR_INDICES, frame.shape) + compute_ear(lms, RIGHT_EAR_INDICES, frame.shape)) def get_patch(eye_idx, ear_idx, iris_idx): s1 = step1_normalize(frame, lms, head_pose, eye_idx, ear_idx, iris_idx) if not s1.is_open: return None if PATCH_SOURCE == "norm": return s1.norm_crop ir = compute_iris_radius(lms, iris_idx, frame.shape) s2 = step2_illumination(s1, ir) return s2.blended if s2.is_usable else None left_patch = get_patch(LEFT_EYE_INDICES, LEFT_EAR_INDICES, LEFT_IRIS_INDICES) right_patch = get_patch(RIGHT_EYE_INDICES, RIGHT_EAR_INDICES, RIGHT_IRIS_INDICES) if left_patch is None and right_patch is None: continue if left_patch is None: left_patch = right_patch if right_patch is None: right_patch = left_patch _, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(left_patch, pose_vec, right_patch) pitch = compensate_pitch(raw_pitch, sm_pitch) yaw = compensate_yaw(raw_yaw, sm_yaw) # Smooth the ANGLES before the RBF sees them. The RBF amplifies input # noise (steep gradient along the low-variance pitch axis), so noise # must be removed here — smoothing the screen coords afterwards cannot # undo amplification that already happened through a nonlinear map. pitch, yaw, ear_s = angle_smoother(pitch, yaw, ear_now) sx, sy = pipeline.calibration.predict(pitch, yaw, ear_s) sx = max(0.0, min(sx, SCREEN_W)) sy = max(0.0, min(sy, SCREEN_H)) jump = np.hypot(sx - last_x, sy - last_y) if jump > MAX_JUMP: sx = last_x + (sx - last_x) * 0.3 sy = last_y + (sy - last_y) * 0.3 last_x, last_y = sx, sy logger.log(sx, sy) now = time.time() fx = fil_x(sx / SCREEN_W, now) fy = fil_y(sy / SCREEN_H, now) try: window.evaluate_js(f"window.insightuxUpdate({fx:.5f},{fy:.5f})") except Exception: pass frame_i += 1 if frame_i % DOM_EVERY == 0: try: raw = window.evaluate_js("window.insightuxAOIs()") if raw: rec = json.loads(raw) rec["type"] = "dom" rec["t"] = round(time.time(), 4) shot_ref = maybe_capture_screenshot(rec.get("scrollY", 0)) if shot_ref: rec["screenshot"] = shot_ref dom_f.write(json.dumps(rec) + "\n") except Exception: pass time.sleep(0.005) cap.release() logger.close() dom_f.close() print(f"[browser_session] session saved in {session_dir}") # generate report and hand control back try: report_path = generate_report(session_dir) api_ref.tracking = False window.load_url("file://" + report_path) print(f"[browser_session] report ready -> {report_path}") except Exception as e: print(f"[browser_session] report generation failed: {e}") api_ref.tracking = False # ============================================================================= # MAIN # ============================================================================= def _go_fullscreen(window): # Small delay so WebView2 fully initializes and grabs keyboard focus # BEFORE going fullscreen. Passing fullscreen=True straight to # create_window is what causes the "can't type" issue and the # AccessibilityObject.Bounds recursion crash on Windows — both trace # back to the WinForms host not finishing its focus/accessibility # setup before the fullscreen transition happens. time.sleep(0.4) try: window.toggle_fullscreen() except Exception as e: print(f"[browser_session] fullscreen toggle failed: {e}") if __name__ == "__main__": os.makedirs(SESSIONS_ROOT, exist_ok=True) window = webview.create_window( "InsightUX — Eye-Tracking Research Browser", LANDING_URL, width=SCREEN_W, height=SCREEN_H, js_api=api ) api.window = window window.events.loaded += lambda: inject_control(window) window.events.shown += lambda: threading.Thread( target=_go_fullscreen, args=(window,), daemon=True ).start() # debug=True enables right-click > Inspect (DevTools) so you can see # the [insightux] console.log diagnostics from CONTROL_JS directly — # keyboard handling runs entirely in the browser, so JS-side issues # never show up in this terminal, only in DevTools' Console tab. webview.start(debug=True)