| """
|
| browser_session.py
|
| InsightUX research browser.
|
|
|
| Flow:
|
| 1. Window opens on real google.com — search and click through like a
|
| normal browser (no scraping, no fake results — it's just Google).
|
| 2. Land on whatever page you want to study. Press S to start tracking.
|
| 3. Gaze dot + AOI highlighting (same as run_session.py) kicks in and
|
| logs sessions/<timestamp>/gaze_log.jsonl + dom_log.jsonl.
|
| 4. Press E to stop. The window auto-navigates to a generated
|
| analysis_report.html: ranked attention, dwell timeline, heatmap.
|
|
|
| Prerequisite:
|
| calibration.pkl must exist in the working directory (run calibrate.py first).
|
|
|
| Run:
|
| python browser_session.py
|
| """
|
|
|
| import os
|
| import sys
|
| import json
|
| import time
|
| import math
|
| import threading
|
| from dataclasses import replace
|
| from datetime import datetime
|
|
|
| import cv2
|
| import numpy as np
|
| import webview
|
| import pyautogui
|
|
|
| from preprocessing.preprocessing_pipeline import (
|
| create_face_mesh,
|
| estimate_camera_matrix,
|
| estimate_head_pose,
|
| compute_iris_radius,
|
| compute_ear,
|
| step1_normalize,
|
| step2_illumination,
|
| LEFT_EYE_INDICES,
|
| LEFT_EAR_INDICES,
|
| LEFT_IRIS_INDICES,
|
| RIGHT_EYE_INDICES,
|
| RIGHT_EAR_INDICES,
|
| RIGHT_IRIS_INDICES,
|
| )
|
| from inference_pipeline import InsightUXPipeline, GazeAngleSmoother
|
| from session_logger import GazeLogger
|
| from analysis import generate_report
|
|
|
|
|
|
|
|
|
|
|
|
|
| ONNX_PATH = "models/gaze_cnn_v4.onnx"
|
| CALIBRATION_PATH = "calibration.pkl"
|
| SCREEN_W, SCREEN_H = pyautogui.size()
|
|
|
| PATCH_SOURCE = "blended"
|
| POSE_SMOOTH = 0.65
|
| MAX_JUMP = 220
|
| NO_FACE_RESET = 15
|
| DOM_EVERY = 6
|
|
|
| EURO_MINCUTOFF = 0.35
|
| EURO_BETA = 0.12
|
|
|
|
|
|
|
|
|
| ANGLE_SMOOTH_WINDOW = 10
|
|
|
| POSE_NORM_SCALE = 30.0
|
| HEAD_PITCH_COMPENSATION = 0.0
|
| HEAD_YAW_COMPENSATION = 0.0
|
|
|
| SESSIONS_ROOT = "sessions"
|
|
|
|
|
|
|
|
|
|
|
| _LANDING_HTML = r"""<!DOCTYPE html>
|
| <html>
|
| <head>
|
| <meta charset="utf-8">
|
| <meta name="insightux-landing" content="true">
|
| <title>InsightUX — Eye-Tracking Research Browser</title>
|
| <style>
|
| * { box-sizing: border-box; }
|
| body {
|
| margin: 0; height: 100vh; display: flex; flex-direction: column;
|
| align-items: center; justify-content: center;
|
| background: radial-gradient(circle at 50% 35%, #221a2e, #100e16 70%);
|
| font-family: -apple-system, 'Segoe UI', Arial, sans-serif; color: #f0ecf7;
|
| }
|
| .logo {
|
| font-size: 42px; font-weight: 700; letter-spacing: -0.01em;
|
| background: linear-gradient(90deg, #9B59FF, #FF2DF0);
|
| -webkit-background-clip: text; -webkit-text-fill-color: transparent;
|
| margin-bottom: 6px;
|
| }
|
| .tagline { color: #8a8098; font-size: 14px; margin-bottom: 36px; }
|
| form { width: 560px; max-width: 88vw; }
|
| input {
|
| width: 100%; padding: 15px 20px; font-size: 15px; border-radius: 26px;
|
| border: 1.5px solid #3a3348; background: #1c1826; color: #f0ecf7;
|
| outline: none; transition: border-color 0.15s;
|
| }
|
| input:focus { border-color: #9B59FF; }
|
| input::placeholder { color: #635a72; }
|
| .hint {
|
| margin-top: 18px; font-size: 12px; color: #6b6278; text-align: center;
|
| line-height: 1.6;
|
| }
|
| .hint b { color: #9B59FF; }
|
| </style>
|
| </head>
|
| <body>
|
| <div class="logo">InsightUX</div>
|
| <div class="tagline">Eye-Tracking Research Browser</div>
|
| <form id="searchForm">
|
| <input id="searchInput" type="text" autofocus
|
| placeholder="Search Google, or enter a website address">
|
| </form>
|
| <div class="hint">
|
| Search results and websites open below as normal.<br>
|
| Once you land on the page you want to study, press <b>S</b> to start eye-tracking,
|
| <b>E</b> to stop.
|
| </div>
|
| <script>
|
| document.getElementById('searchForm').addEventListener('submit', function(e){
|
| e.preventDefault();
|
| const raw = document.getElementById('searchInput').value.trim();
|
| if (!raw) return;
|
|
|
| const looksLikeUrl = /^https?:\/\//i.test(raw) ||
|
| (/^[\w-]+(\.[\w-]+)+([/?#].*)?$/i.test(raw) && !raw.includes(' '));
|
|
|
| if (looksLikeUrl) {
|
| window.location.href = raw.startsWith('http') ? raw : ('https://' + raw);
|
| } else {
|
| window.location.href = 'https://www.google.com/search?q=' + encodeURIComponent(raw);
|
| }
|
| });
|
| </script>
|
| </body>
|
| </html>
|
| """
|
|
|
|
|
| def _write_landing_page():
|
| path = os.path.abspath("insightux_landing.html")
|
| with open(path, "w", encoding="utf-8") as f:
|
| f.write(_LANDING_HTML)
|
| return "file://" + path.replace(os.sep, "/")
|
|
|
|
|
| LANDING_URL = _write_landing_page()
|
|
|
|
|
| def normalize_pose(pitch_deg, yaw_deg, roll_deg):
|
| return np.array([
|
| pitch_deg / POSE_NORM_SCALE,
|
| yaw_deg / POSE_NORM_SCALE,
|
| roll_deg / POSE_NORM_SCALE,
|
| ], dtype=np.float32)
|
|
|
| def compensate_pitch(raw_pitch, head_pitch_deg):
|
| return raw_pitch - np.radians(head_pitch_deg) * HEAD_PITCH_COMPENSATION
|
|
|
| def compensate_yaw(raw_yaw, head_yaw_deg):
|
| return raw_yaw - np.radians(head_yaw_deg) * HEAD_YAW_COMPENSATION
|
|
|
|
|
|
|
|
|
|
|
|
|
| class OneEuroFilter:
|
| def __init__(self, mincutoff=0.8, beta=0.4, dcutoff=1.0):
|
| self.mincutoff = mincutoff
|
| self.beta = beta
|
| self.dcutoff = dcutoff
|
| self.x_prev = None
|
| self.dx_prev = 0.0
|
| self.t_prev = None
|
|
|
| @staticmethod
|
| def _alpha(cutoff, dt):
|
| tau = 1.0 / (2 * math.pi * cutoff)
|
| return 1.0 / (1.0 + tau / dt)
|
|
|
| def __call__(self, x, t):
|
| if self.x_prev is None:
|
| self.x_prev, self.t_prev = x, t
|
| return x
|
| dt = t - self.t_prev
|
| if dt <= 0:
|
| dt = 1e-3
|
| self.t_prev = t
|
| dx = (x - self.x_prev) / dt
|
| a_d = self._alpha(self.dcutoff, dt)
|
| dx_hat = a_d * dx + (1 - a_d) * self.dx_prev
|
| cutoff = self.mincutoff + self.beta * abs(dx_hat)
|
| a = self._alpha(cutoff, dt)
|
| x_hat = a * x + (1 - a) * self.x_prev
|
| self.x_prev, self.dx_prev = x_hat, dx_hat
|
| return x_hat
|
|
|
| def reset(self):
|
| self.x_prev = None
|
| self.dx_prev = 0.0
|
| self.t_prev = None
|
|
|
|
|
|
|
|
|
|
|
|
|
| CONTROL_JS = r"""
|
| (function(){
|
| if (window.__insightuxControl) { return; }
|
| window.__insightuxControl = true;
|
|
|
| // S/E (start/stop tracking) only make sense on an actual website — not on
|
| // the InsightUX landing/search page and not on a generated insights
|
| // report page. X (quit the app) always works, on every page.
|
| const isLanding = !!document.querySelector('meta[name="insightux-landing"]');
|
| const isReport = !!document.querySelector('meta[name="insightux-report"]');
|
| const isTrackable = !isLanding && !isReport;
|
|
|
| if (isTrackable) {
|
| const banner = document.createElement('div');
|
| banner.id = '__insightux_pill';
|
| banner.style.cssText = `
|
| position:fixed; top:0; left:0; right:0; z-index:2147483647;
|
| display:flex; align-items:center; justify-content:space-between;
|
| padding:7px 16px; font:12px -apple-system,Arial;
|
| background:linear-gradient(90deg, rgba(20,18,26,0.95), rgba(30,20,40,0.95));
|
| border-bottom:2px solid; border-image:linear-gradient(90deg,#7B2FBE,#FF2DF0) 1;
|
| box-shadow:0 3px 14px rgba(0,0,0,0.35); pointer-events:none;
|
| `;
|
| banner.innerHTML = `
|
| <span style="display:flex;align-items:center;gap:7px;">
|
| <span style="width:8px;height:8px;border-radius:50%;background:linear-gradient(135deg,#7B2FBE,#FF2DF0);display:inline-block;"></span>
|
| <b style="color:#f0ecf7;letter-spacing:0.02em;">InsightUX</b>
|
| <span style="color:#7a7288;">Eye + Mouse Research Browser</span>
|
| </span>
|
| <span id="__insightux_status" style="color:#c9a6f5;">Press S to start tracking · E to stop · H heatmap · M mouse panel · X quit</span>
|
| `;
|
| document.documentElement.appendChild(banner);
|
|
|
| setInterval(function(){
|
| if (!document.documentElement.contains(banner)) document.documentElement.appendChild(banner);
|
| }, 1000);
|
| }
|
|
|
| const statusEl = () => document.getElementById('__insightux_status');
|
| window.insightuxSetStatus = function(text){
|
| const el = statusEl();
|
| if (el) el.textContent = text;
|
| };
|
|
|
| function isTypingTarget(el){
|
| if (!el) return false;
|
| const tag = el.tagName ? el.tagName.toLowerCase() : '';
|
| return tag === 'input' || tag === 'textarea' || tag === 'select' ||
|
| el.isContentEditable === true;
|
| }
|
|
|
| document.addEventListener('keydown', function(e){
|
| const typingBlocked = isTypingTarget(e.target) || isTypingTarget(document.activeElement);
|
| const apiReady = !!(window.pywebview && window.pywebview.api);
|
|
|
| if (e.key === 'x' || e.key === 'X') {
|
| if (typingBlocked || !apiReady) return;
|
| window.pywebview.api.quit_app();
|
| return;
|
| }
|
|
|
| if (!isTrackable) return; // S/E do nothing on the landing page or a report page
|
|
|
| if (e.key === 's' || e.key === 'S' || e.key === 'e' || e.key === 'E') {
|
| console.log('[insightux] key=' + e.key, 'typingBlocked=' + typingBlocked, 'apiReady=' + apiReady, 'focusedEl=' + (document.activeElement && document.activeElement.tagName));
|
| }
|
| if (!apiReady) return;
|
| if (typingBlocked) {
|
| window.insightuxSetStatus("Click somewhere on the page (not a text field) before pressing S/E");
|
| return;
|
| }
|
|
|
| if (e.key === 's' || e.key === 'S') {
|
| window.pywebview.api.start_tracking();
|
| } else if (e.key === 'e' || e.key === 'E') {
|
| window.pywebview.api.stop_tracking();
|
| }
|
| }, true); // capture phase — fires before page scripts can intercept/stop the event
|
| })();
|
| """
|
|
|
|
|
|
|
| TRACKING_JS = r"""
|
| (function(){
|
| if (window.__insightux) { return; }
|
| const state = { aois: [] };
|
| const dwell = { pendLabel: null, pendSince: 0, activeLabel: null, emptySince: 0 };
|
| const DWELL_MS = 420;
|
| const RELEASE_MS = 1200;
|
|
|
| let activeBox = null;
|
| const target = { fx: 0.5, fy: 0.5 };
|
| const dot = { x: null, y: null };
|
| const DOT_LERP = 0.07;
|
|
|
| const cv = document.createElement('canvas');
|
| cv.id = '__insightux_canvas';
|
| cv.style.cssText = 'position:fixed;left:0;top:0;width:100vw;height:100vh;pointer-events:none;z-index:2147483646;';
|
| (document.body || document.documentElement).appendChild(cv);
|
| const ctx = cv.getContext('2d');
|
| function resize(){ cv.width = window.innerWidth; cv.height = window.innerHeight; }
|
| resize();
|
| window.addEventListener('resize', resize, {passive:true});
|
| setInterval(function(){
|
| if (!document.body.contains(cv)) document.body.appendChild(cv);
|
| }, 1000);
|
|
|
| const MIN_W=40, MIN_H=24, MAX_AOIS=90, MAX_SCAN=2500;
|
| const PAD_PX = 90;
|
| const TALL_WRAPPER_H = 220;
|
|
|
| const ALWAYS_SELECTOR = "nav, header, footer, img, video, iframe, h1, h2, h3, button, figure, [class*='hero'], [class*='banner'], [class*='card']";
|
| const TEXT_CONTAINER_SELECTOR = "div, span, section, article, main, aside, p, li";
|
|
|
| function hasOwnText(el){
|
| for (const node of el.childNodes){
|
| if (node.nodeType === 3 && node.textContent.trim().length > 2) return true;
|
| }
|
| return false;
|
| }
|
|
|
| function labelFor(el){
|
| if (el.dataset && el.dataset.aoi) return el.dataset.aoi.slice(0,40);
|
| const tag = el.tagName.toLowerCase();
|
| if (tag==='nav') return 'navbar';
|
| if (tag==='header') return 'header';
|
| if (tag==='footer') return 'footer';
|
| if (tag==='img'){
|
| const alt=(el.getAttribute('alt')||'').trim();
|
| if (alt) return 'img: '+alt.slice(0,30);
|
| const src=el.getAttribute('src')||'';
|
| const name=src.split('/').pop().split('?')[0];
|
| return 'img: '+(name||'image').slice(0,30);
|
| }
|
| if (tag==='video') return 'video';
|
| if (tag==='iframe') return 'embed';
|
| if (tag==='h1'||tag==='h2'){
|
| const t=(el.innerText||'').trim().replace(/\s+/g,' ');
|
| if (t) return tag+': '+t.slice(0,30);
|
| }
|
| const id=el.id?('#'+el.id):'';
|
| let cls='';
|
| if (el.className && typeof el.className==='string'){
|
| const f=el.className.trim().split(/\s+/)[0];
|
| if (f) cls='.'+f;
|
| }
|
| const txt=(el.innerText||'').trim().replace(/\s+/g,' ').slice(0,24);
|
| const base=id||cls||tag;
|
| return txt?(base+' ('+txt+')'):base;
|
| }
|
|
|
| function refresh(){
|
| const vh = window.innerHeight;
|
| const raw = [];
|
| const seen = new Set();
|
| let full = false;
|
|
|
| function consider(el){
|
| if (full) return;
|
| // Never track InsightUX's own injected UI (the "press S/E" pill,
|
| // the gaze-dot canvas, the mouse tracker panel) as if it were page content.
|
| if (el.id === '__insightux_pill' || el.id === '__insightux_canvas' || el.id === '__insightux_mouse_panel') return;
|
| if (el.closest && el.closest('#__insightux_pill, #__insightux_canvas, #__insightux_mouse_panel')) return;
|
| const tag = el.tagName.toLowerCase();
|
| const explicit = !!(el.dataset && el.dataset.aoi);
|
| if (!explicit){
|
| const isAlways = (tag==='nav'||tag==='header'||tag==='footer'||tag==='img'||
|
| tag==='video'||tag==='iframe'||
|
| tag==='h1'||tag==='h2'||tag==='h3'||tag==='button'||tag==='figure') ||
|
| (el.className && typeof el.className==='string' &&
|
| /hero|banner|card/i.test(el.className));
|
| if (!isAlways && !hasOwnText(el)) return;
|
| }
|
| const r = el.getBoundingClientRect();
|
| if (r.width<MIN_W || r.height<MIN_H) return;
|
| if (r.bottom<0 || r.top>vh) return;
|
| const label = labelFor(el);
|
| if (!label) return;
|
| const key = label+'@'+Math.round(r.left)+','+Math.round(r.top);
|
| if (seen.has(key)) return;
|
| seen.add(key);
|
| const pos = getComputedStyle(el).position;
|
| raw.push({el:el, label:label, x:Math.round(r.left), y:Math.round(r.top),
|
| w:Math.round(r.width), h:Math.round(r.height),
|
| sticky:(pos==='sticky'||pos==='fixed')});
|
| if (raw.length >= MAX_AOIS*2) full = true;
|
| }
|
|
|
| document.querySelectorAll('[data-aoi]').forEach(consider);
|
| document.querySelectorAll(ALWAYS_SELECTOR).forEach(consider);
|
|
|
| const candidates = document.querySelectorAll(TEXT_CONTAINER_SELECTOR);
|
| for (let i=0; i<candidates.length && i<MAX_SCAN && !full; i++){
|
| consider(candidates[i]);
|
| }
|
|
|
| const filtered = raw.filter(o => {
|
| const isTallWrapper = o.h > TALL_WRAPPER_H &&
|
| raw.some(o2 => o2 !== o && o.el.contains(o2.el));
|
| return !isTallWrapper;
|
| });
|
|
|
| state.aois = filtered.slice(0, MAX_AOIS).map(o => ({
|
| label:o.label, x:o.x, y:o.y, w:o.w, h:o.h, sticky:o.sticky
|
| }));
|
| }
|
| refresh();
|
| window.addEventListener('scroll', refresh, {passive:true});
|
| setInterval(refresh, 400);
|
|
|
| window.insightuxAOIs = function(){
|
| return JSON.stringify({
|
| url: location.href,
|
| scrollX: Math.round(window.scrollX),
|
| scrollY: Math.round(window.scrollY),
|
| viewport:{w:window.innerWidth,h:window.innerHeight},
|
| page:{w:document.documentElement.scrollWidth,h:document.documentElement.scrollHeight},
|
| aois: state.aois
|
| });
|
| };
|
|
|
| window.insightuxUpdate = function(fx, fy){
|
| target.fx = fx;
|
| target.fy = fy;
|
| };
|
|
|
| function renderLoop(){
|
| const w = window.innerWidth, h = window.innerHeight;
|
| const tx = target.fx * w, ty = target.fy * h;
|
| if (dot.x === null){ dot.x = tx; dot.y = ty; }
|
| dot.x += (tx - dot.x) * DOT_LERP;
|
| dot.y += (ty - dot.y) * DOT_LERP;
|
|
|
| const now = performance.now();
|
| const px = dot.x, py = dot.y;
|
|
|
| let cand = null;
|
| for (const a of state.aois){
|
| if (px>=a.x-PAD_PX && px<=a.x+a.w+PAD_PX && py>=a.y-PAD_PX && py<=a.y+a.h+PAD_PX){
|
| if (!cand || (a.w*a.h)<(cand.w*cand.h)) cand = a;
|
| }
|
| }
|
|
|
| const candLabel = cand ? cand.label : null;
|
| if (candLabel !== dwell.pendLabel){ dwell.pendLabel = candLabel; dwell.pendSince = now; }
|
| if (cand){
|
| dwell.emptySince = 0;
|
| if (dwell.activeLabel !== cand.label && (now - dwell.pendSince) >= DWELL_MS){
|
| dwell.activeLabel = cand.label;
|
| }
|
| } else {
|
| if (dwell.emptySince === 0) dwell.emptySince = now;
|
| if (dwell.activeLabel && (now - dwell.emptySince) >= RELEASE_MS){
|
| dwell.activeLabel = null;
|
| }
|
| }
|
|
|
| let active = null;
|
| if (dwell.activeLabel){
|
| for (const a of state.aois){ if (a.label === dwell.activeLabel){ active = a; break; } }
|
| if (!active) dwell.activeLabel = null;
|
| }
|
| activeBox = active;
|
|
|
| ctx.clearRect(0,0,cv.width,cv.height);
|
|
|
| if (activeBox){
|
| ctx.fillStyle = 'rgba(123,47,190,0.16)';
|
| ctx.fillRect(activeBox.x, activeBox.y, activeBox.w, activeBox.h);
|
| ctx.strokeStyle = '#7B2FBE'; ctx.lineWidth = 3;
|
| ctx.strokeRect(activeBox.x, activeBox.y, activeBox.w, activeBox.h);
|
| const label = activeBox.label;
|
| ctx.font = 'bold 13px Arial';
|
| const tw = ctx.measureText(label).width + 16;
|
| const ly = Math.max(0, activeBox.y - 22);
|
| ctx.fillStyle = '#7B2FBE';
|
| ctx.fillRect(activeBox.x, ly, tw, 20);
|
| ctx.fillStyle = '#FFFFFF';
|
| ctx.fillText(label, activeBox.x + 8, ly + 14);
|
| }
|
|
|
| ctx.beginPath();
|
| ctx.arc(dot.x, dot.y, 13, 0, 2*Math.PI);
|
| ctx.fillStyle = 'rgba(255,255,255,0.9)';
|
| ctx.fill();
|
| ctx.beginPath();
|
| ctx.arc(dot.x, dot.y, 9, 0, 2*Math.PI);
|
| ctx.fillStyle = '#FF2DF0';
|
| ctx.fill();
|
| ctx.lineWidth = 2;
|
| ctx.strokeStyle = 'rgba(0,0,0,0.55)';
|
| ctx.stroke();
|
|
|
| requestAnimationFrame(renderLoop);
|
| }
|
| requestAnimationFrame(renderLoop);
|
|
|
| window.__insightux = true;
|
| })();
|
| """
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| MOUSE_JS = r"""
|
| (function(){
|
| if (window.__insightuxMouse) { return; }
|
| window.__insightuxMouse = true;
|
|
|
| let isTracking = true;
|
| let canvas = null;
|
| let panelOpen = false;
|
| let logsOpen = false;
|
|
|
| // Per-sync buffers (flushed to Python every 2s for on-disk persistence)
|
| let trailBuffer = [];
|
| let heatmapBuffer = [];
|
| let clickBuffer = [];
|
| let dwellBuffer = [];
|
|
|
| // Cumulative in-page state (mirrors the extension's background.js state)
|
| let allTrail = [];
|
| let allHeatmap = [];
|
| let allClicks = [];
|
| let dwellTotals = {};
|
|
|
| let sessionStart = Date.now();
|
| let sessionStop = null;
|
|
|
| let lastClientX = 0, lastClientY = 0;
|
| let lastPageX = 0, lastPageY = 0;
|
| let lastSampleTime = 0;
|
| const sampleRate = 50;
|
| const DWELL_THRESHOLD = 5000;
|
| let stationaryStart = 0;
|
| let isStationary = false;
|
|
|
| window.insightuxMouseSetTracking = function(on){
|
| isTracking = !!on;
|
| if (isTracking) {
|
| sessionStart = Date.now();
|
| sessionStop = null;
|
| } else {
|
| sessionStop = Date.now();
|
| flushDwell();
|
| }
|
| updatePanel();
|
| };
|
|
|
| // ---- sampling loop (trail while moving, heatmap dwell points while still) ----
|
| setInterval(function(){
|
| if (!isTracking) return;
|
| const currentScrollX = window.scrollX, currentScrollY = window.scrollY;
|
| if (lastPageX === 0 && lastPageY === 0 && lastClientX !== 0) {
|
| lastPageX = lastClientX + currentScrollX;
|
| lastPageY = lastClientY + currentScrollY;
|
| }
|
| const currentPageX = (lastClientX !== 0) ? (lastClientX + currentScrollX) : lastPageX;
|
| const currentPageY = (lastClientY !== 0) ? (lastClientY + currentScrollY) : lastPageY;
|
| if (currentPageX === 0 && currentPageY === 0) return;
|
|
|
| const now = Date.now();
|
| const dt = now - lastSampleTime;
|
| if (dt > 1000) { lastSampleTime = now; return; }
|
|
|
| const dx = currentPageX - lastPageX, dy = currentPageY - lastPageY;
|
| const distance = Math.sqrt(dx * dx + dy * dy);
|
|
|
| if (distance > 2) {
|
| const pt = { x: currentPageX, y: currentPageY };
|
| trailBuffer.push(pt); allTrail.push(pt);
|
| lastPageX = currentPageX; lastPageY = currentPageY;
|
| isStationary = false; stationaryStart = 0;
|
| } else {
|
| if (!isStationary) { isStationary = true; stationaryStart = now; }
|
| else {
|
| const dwellDuration = now - stationaryStart;
|
| if (dwellDuration > DWELL_THRESHOLD) {
|
| const pt = { x: currentPageX, y: currentPageY };
|
| heatmapBuffer.push(pt); allHeatmap.push(pt);
|
| }
|
| }
|
| }
|
| lastSampleTime = now;
|
| }, sampleRate);
|
|
|
| // ---- dwell / element-interest tracking ----
|
| let currentHoverLabel = null, currentHoverStartTime = 0;
|
| function handleHoverChange(newLabel){
|
| if (!isTracking) return;
|
| if (newLabel !== currentHoverLabel) {
|
| const now = Date.now();
|
| if (currentHoverLabel && currentHoverStartTime > 0) {
|
| const duration = now - currentHoverStartTime;
|
| if (duration > 10) recordDwell(currentHoverLabel, duration);
|
| }
|
| currentHoverLabel = newLabel;
|
| currentHoverStartTime = newLabel ? now : 0;
|
| }
|
| }
|
| function recordDwell(element, duration){
|
| dwellBuffer.push({ element: element, duration: duration });
|
| dwellTotals[element] = (dwellTotals[element] || 0) + duration;
|
| }
|
| function flushDwell(){
|
| if (currentHoverLabel && currentHoverStartTime > 0) {
|
| const now = Date.now();
|
| const duration = now - currentHoverStartTime;
|
| if (duration > 50) recordDwell(currentHoverLabel, duration);
|
| currentHoverStartTime = now;
|
| }
|
| }
|
|
|
| // ---- sync to Python every 2s -> mouse_log.jsonl in the session folder ----
|
| setInterval(function(){
|
| flushDwell();
|
| if (trailBuffer.length || heatmapBuffer.length || clickBuffer.length || dwellBuffer.length) {
|
| const payload = {
|
| trail: trailBuffer.length ? trailBuffer : null,
|
| heatmap: heatmapBuffer.length ? heatmapBuffer : null,
|
| click: clickBuffer.length ? clickBuffer : null,
|
| dwell: dwellBuffer.length ? dwellBuffer : null,
|
| };
|
| trailBuffer = []; heatmapBuffer = []; clickBuffer = []; dwellBuffer = [];
|
| if (window.pywebview && window.pywebview.api && window.pywebview.api.log_mouse_data) {
|
| try { window.pywebview.api.log_mouse_data(payload).catch(function(){}); } catch (e) {}
|
| }
|
| }
|
| updatePanel();
|
| }, 2000);
|
|
|
| // ---- exclude InsightUX's own injected UI from every measurement ----
|
| // (the status pill, the mouse-tracker panel, the heatmap canvas) — only
|
| // real website elements should ever show up in trail/heatmap/dwell/clicks.
|
| function isOwnUI(el){
|
| if (!el) return false;
|
| if (el.id === '__insightux_pill' || el.id === '__insightux_canvas' || el.id === '__insightux_mouse_panel') return true;
|
| return !!(el.closest && el.closest('#__insightux_pill, #__insightux_canvas, #__insightux_mouse_panel'));
|
| }
|
|
|
| // ---- mouse position + hover tracking ----
|
| function updateMousePos(e){
|
| if (isOwnUI(e.target)) return;
|
| lastClientX = e.clientX; lastClientY = e.clientY;
|
| if (lastPageX === 0) lastPageX = e.pageX;
|
| if (lastPageY === 0) lastPageY = e.pageY;
|
| if (isTracking) handleHoverChange(getSmartLabel(e.target));
|
| }
|
| document.addEventListener('mousemove', updateMousePos, true);
|
| document.addEventListener('mouseenter', updateMousePos, true);
|
| document.addEventListener('mouseover', updateMousePos, true);
|
| document.addEventListener('click', updateMousePos, true);
|
| document.addEventListener('mouseleave', function(){
|
| lastClientX = 0; lastClientY = 0;
|
| handleHoverChange(null);
|
| }, true);
|
|
|
| document.addEventListener('click', function(e){
|
| if (!isTracking) return;
|
| if (e.target === canvas) return;
|
| if (isOwnUI(e.target)) return;
|
| if (!isInteractable(e.target)) return;
|
|
|
| const x = e.pageX, y = e.pageY;
|
| const label = getSmartLabel(e.target) || e.target.tagName;
|
| if (['BODY', 'HTML', 'DIV', 'SPAN'].includes(label) && !e.target.innerText.trim()) return;
|
|
|
| const logEntry = {
|
| timestamp: new Date().toLocaleTimeString(),
|
| x: x, y: y, element: label,
|
| text: e.target.innerText ? e.target.innerText.substring(0, 30).replace(/(\r\n|\n|\r)/gm, ' ').trim() : '',
|
| url: window.location.href
|
| };
|
| clickBuffer.push(logEntry); allClicks.push(logEntry);
|
| if (canvas) drawClick(x, y);
|
| updatePanel();
|
| }, true);
|
|
|
| function isInteractable(el){
|
| if (!el) return false;
|
| const tag = el.tagName.toLowerCase();
|
| if (['a', 'button', 'input', 'select', 'textarea', 'details', 'summary', 'label'].includes(tag)) return true;
|
| const role = el.getAttribute('role');
|
| if (role === 'button' || role === 'link' || role === 'menuitem' || role === 'tab') return true;
|
| try { if (window.getComputedStyle(el).cursor === 'pointer') return true; } catch (e) {}
|
| let parent = el.parentElement, depth = 0;
|
| while (parent && depth < 3) {
|
| const pTag = parent.tagName.toLowerCase();
|
| if (['a', 'button'].includes(pTag)) return true;
|
| if (parent.getAttribute('role') === 'button') return true;
|
| parent = parent.parentElement; depth++;
|
| }
|
| if (tag === 'code' || tag === 'pre' || el.classList.contains('code') || el.closest('pre')) return true;
|
| return false;
|
| }
|
|
|
| function getSmartLabel(el){
|
| if (!el) return null;
|
| const tag = el.tagName.toLowerCase();
|
| if (['body', 'html', 'main', 'div', 'span', 'section', 'article'].includes(tag)) {
|
| if (el.id && (el.id.includes('logo') || el.id.includes('wrapper') || el.id.includes('container'))) return null;
|
| if (el.className && typeof el.className === 'string' &&
|
| (el.className.toLowerCase().includes('logo') || el.className.toLowerCase().includes('brand'))) return null;
|
| const aria = el.getAttribute('aria-label');
|
| if (aria) return 'Element: ' + aria;
|
| if (el.children.length === 0) {
|
| const txt = el.innerText.trim();
|
| if (txt.length > 2 && txt.length < 50 && /[a-zA-Z0-9]/.test(txt)) return 'Element: ' + txt;
|
| }
|
| return null;
|
| }
|
| if (tag === 'a') return 'Link: ' + (el.innerText.trim().substring(0, 30) || 'Link');
|
| if (tag === 'button') return 'Button: ' + (el.innerText.trim().substring(0, 30) || 'Button');
|
| if (tag === 'input') return 'Input: ' + (el.placeholder || el.name || el.id || 'Input');
|
| if (tag === 'textarea') return 'Input: Text Area';
|
| if (tag === 'img') return 'Image: ' + (el.alt || 'Image');
|
| if (['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(tag)) return tag.toUpperCase() + ': ' + el.innerText.trim().substring(0, 40);
|
| if (tag === 'code' || tag === 'pre') return 'Code: ' + el.innerText.trim().substring(0, 30);
|
| const text = el.innerText.trim();
|
| if (text && text.length > 2) {
|
| if (el.classList.contains('code') || el.closest('pre')) return 'Code: ' + text.substring(0, 30);
|
| if (text.toLowerCase().includes('no message found')) return null;
|
| const isHex = /^#[0-9A-F]{6}$/i.test(text) || /^#[0-9A-F]{3}$/i.test(text);
|
| const isColorName = ['red', 'blue', 'green', 'yellow', 'black', 'white', 'orange', 'purple', 'gray', 'grey', 'pink', 'brown', 'cyan', 'magenta'].includes(text.toLowerCase());
|
| if (isHex || isColorName) return null;
|
| if (text.length > 50) return 'Text: ' + text.substring(0, 47) + '...';
|
| return 'Text: ' + text;
|
| }
|
| return null;
|
| }
|
|
|
| // ---- heatmap overlay canvas (ported 1:1 from the extension) ----
|
| window.insightuxMouseToggleHeatmap = function(){
|
| if (canvas) { document.body.removeChild(canvas); canvas = null; return; }
|
| canvas = document.createElement('canvas');
|
| canvas.style.position = 'absolute';
|
| canvas.style.top = '0'; canvas.style.left = '0';
|
| canvas.style.zIndex = '2147483645';
|
| canvas.style.pointerEvents = 'none';
|
| canvas.width = Math.max(document.documentElement.scrollWidth, document.body.scrollWidth, document.documentElement.offsetWidth);
|
| canvas.height = Math.max(document.documentElement.scrollHeight, document.body.scrollHeight, document.documentElement.offsetHeight);
|
| document.body.appendChild(canvas);
|
| drawHeatmapHighQuality(allHeatmap, allTrail, allClicks);
|
| };
|
|
|
| function drawClick(x, y){
|
| if (!canvas) return;
|
| const ctx = canvas.getContext('2d');
|
| ctx.save();
|
| ctx.beginPath();
|
| ctx.strokeStyle = '#00FF00'; ctx.lineWidth = 3;
|
| ctx.shadowColor = 'black'; ctx.shadowBlur = 2;
|
| const size = 10;
|
| ctx.moveTo(x - size, y - size); ctx.lineTo(x + size, y + size);
|
| ctx.moveTo(x + size, y - size); ctx.lineTo(x - size, y + size);
|
| ctx.stroke();
|
| ctx.beginPath(); ctx.arc(x, y, size + 5, 0, Math.PI * 2); ctx.stroke();
|
| ctx.restore();
|
| }
|
|
|
| function drawHeatmapHighQuality(heatmapData, trailData, clickData){
|
| if (!canvas) return;
|
| const ctx = canvas.getContext('2d');
|
| ctx.clearRect(0, 0, canvas.width, canvas.height);
|
| let allHeatPoints = [];
|
| if (heatmapData && heatmapData.length) allHeatPoints = allHeatPoints.concat(heatmapData);
|
| if (trailData && trailData.length) allHeatPoints = allHeatPoints.concat(trailData);
|
| if (clickData && clickData.length) {
|
| clickData.forEach(function(c){ for (let i = 0; i < 5; i++) allHeatPoints.push({ x: c.x, y: c.y }); });
|
| }
|
| if (!allHeatPoints.length) return;
|
| const radius = 60;
|
| const brushCanvas = document.createElement('canvas');
|
| brushCanvas.width = radius * 2; brushCanvas.height = radius * 2;
|
| const brushCtx = brushCanvas.getContext('2d');
|
| const g = brushCtx.createRadialGradient(radius, radius, 0, radius, radius, radius);
|
| g.addColorStop(0, 'rgba(0, 0, 0, 0.05)'); g.addColorStop(1, 'rgba(0, 0, 0, 0)');
|
| brushCtx.fillStyle = g; brushCtx.fillRect(0, 0, radius * 2, radius * 2);
|
| allHeatPoints.forEach(function(point){ ctx.drawImage(brushCanvas, point.x - radius, point.y - radius); });
|
| const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
| const data = imageData.data;
|
| const gradientMap = createGradientMap();
|
| for (let i = 0; i < data.length; i += 4) {
|
| const alpha = data[i + 3];
|
| if (alpha > 0) {
|
| let mapIndex = Math.floor(alpha * 1.5);
|
| if (mapIndex > 255) mapIndex = 255;
|
| const cIndex = mapIndex * 4;
|
| data[i] = gradientMap[cIndex]; data[i + 1] = gradientMap[cIndex + 1]; data[i + 2] = gradientMap[cIndex + 2];
|
| data[i + 3] = Math.min(255, 150 + alpha);
|
| }
|
| }
|
| ctx.putImageData(imageData, 0, 0);
|
| }
|
|
|
| function createGradientMap(){
|
| const c = document.createElement('canvas'); c.width = 256; c.height = 1;
|
| const ctx = c.getContext('2d');
|
| const g = ctx.createLinearGradient(0, 0, 256, 0);
|
| g.addColorStop(0.0, 'rgba(0, 0, 255, 0)');
|
| g.addColorStop(0.1, 'rgba(0, 0, 255, 1)');
|
| g.addColorStop(0.4, 'rgba(0, 255, 255, 1)');
|
| g.addColorStop(0.6, 'rgba(0, 255, 0, 1)');
|
| g.addColorStop(0.8, 'rgba(255, 255, 0, 1)');
|
| g.addColorStop(1.0, 'rgba(255, 0, 0, 1)');
|
| ctx.fillStyle = g; ctx.fillRect(0, 0, 256, 1);
|
| return ctx.getImageData(0, 0, 256, 1).data;
|
| }
|
|
|
| window.addEventListener('resize', function(){
|
| if (canvas) {
|
| canvas.width = Math.max(document.documentElement.scrollWidth, document.body.scrollWidth, document.documentElement.offsetWidth);
|
| canvas.height = Math.max(document.documentElement.scrollHeight, document.body.scrollHeight, document.documentElement.offsetHeight);
|
| }
|
| });
|
|
|
| // ---- insights panel: in-page port of the extension's popup.html/popup.js ----
|
| const style = document.createElement('style');
|
| style.textContent = `
|
| #__insightux_mouse_panel {
|
| position: fixed; top: 44px; right: 16px; width: 260px; z-index: 2147483647;
|
| background: linear-gradient(180deg, rgba(28,24,38,0.97), rgba(20,18,26,0.97));
|
| border: 1px solid #3a3348; border-radius: 12px; box-shadow: 0 8px 28px rgba(0,0,0,0.45);
|
| font: 12px -apple-system, 'Segoe UI', Arial; color: #f0ecf7; padding: 14px;
|
| pointer-events: auto; display: none;
|
| }
|
| #__insightux_mouse_panel.open { display: block; }
|
| #__insightux_mouse_panel h3 { margin: 0 0 8px 0; font-size: 13px; color: #f0ecf7;
|
| display:flex; align-items:center; justify-content:space-between; }
|
| #__insightux_mouse_panel .muted { color: #9a92ad; font-size: 11px; }
|
| #__insightux_mouse_panel .row { display: flex; gap: 6px; margin-bottom: 8px; flex-wrap: wrap; }
|
| #__insightux_mouse_panel button {
|
| flex: 1 1 auto; padding: 6px 8px; font-size: 11px; border-radius: 6px; border: 1px solid #3a3348;
|
| background: #241f2e; color: #f0ecf7; cursor: pointer;
|
| }
|
| #__insightux_mouse_panel button:hover { background: #302a3d; }
|
| #__insightux_mouse_panel .timer { font-weight: 600; margin-bottom: 6px; }
|
| #__insightux_mouse_panel .hero {
|
| background: rgba(123,47,190,0.15); border: 1px solid #7B2FBE; border-radius: 8px;
|
| padding: 8px; text-align: center; margin-bottom: 8px;
|
| }
|
| #__insightux_mouse_panel .hero .lbl { font-size: 9px; text-transform: uppercase; color: #c9a6f5; letter-spacing: 0.05em; }
|
| #__insightux_mouse_panel .hero .el { font-weight: 600; margin: 3px 0; color: #FF2DF0; }
|
| #__insightux_mouse_panel .interest-item, #__insightux_mouse_panel .log-item {
|
| background: #241f2e; border-radius: 6px; padding: 5px 7px; margin-bottom: 4px; font-size: 11px;
|
| }
|
| #__insightux_mouse_panel .list { max-height: 160px; overflow-y: auto; }
|
| `;
|
| document.head.appendChild(style);
|
|
|
| const panel = document.createElement('div');
|
| panel.id = '__insightux_mouse_panel';
|
| panel.innerHTML = `
|
| <h3>Mouse Tracker <span class="muted" id="__mt_status">idle</span></h3>
|
| <div class="timer" id="__mt_timer">Session Time: 00:00</div>
|
| <div class="row">
|
| <button id="__mt_clear">Clear</button>
|
| <button id="__mt_heatmap">Heatmap</button>
|
| <button id="__mt_logs">Logs</button>
|
| </div>
|
| <div id="__mt_hero" class="hero" style="display:none;">
|
| <div class="lbl">Most Viewed Element</div>
|
| <div class="el" id="__mt_hero_el">-</div>
|
| <div id="__mt_hero_time">0s</div>
|
| </div>
|
| <div id="__mt_interests" class="list"></div>
|
| <div id="__mt_logs_list" class="list" style="display:none;"></div>
|
| `;
|
| document.documentElement.appendChild(panel);
|
|
|
| document.getElementById('__mt_clear').addEventListener('click', function(){
|
| trailBuffer = []; heatmapBuffer = []; clickBuffer = []; dwellBuffer = [];
|
| allTrail = []; allHeatmap = []; allClicks = []; dwellTotals = {};
|
| sessionStart = Date.now(); sessionStop = null;
|
| if (canvas) { document.body.removeChild(canvas); canvas = null; }
|
| updatePanel();
|
| });
|
| document.getElementById('__mt_heatmap').addEventListener('click', function(){ window.insightuxMouseToggleHeatmap(); });
|
| document.getElementById('__mt_logs').addEventListener('click', function(){
|
| logsOpen = !logsOpen;
|
| document.getElementById('__mt_logs_list').style.display = logsOpen ? 'block' : 'none';
|
| document.getElementById('__mt_interests').style.display = logsOpen ? 'none' : 'block';
|
| updatePanel();
|
| });
|
|
|
| window.insightuxMouseTogglePanel = function(){
|
| panelOpen = !panelOpen;
|
| panel.classList.toggle('open', panelOpen);
|
| if (panelOpen) updatePanel();
|
| };
|
|
|
| function formatTime(ms){ return Math.round(ms / 1000) + 's'; }
|
|
|
| function updatePanel(){
|
| if (!panelOpen) return;
|
| document.getElementById('__mt_status').textContent = isTracking ? 'tracking' : 'stopped';
|
| const startTs = sessionStart;
|
| const diff = isTracking ? Math.floor((Date.now() - startTs) / 1000)
|
| : (sessionStop ? Math.floor((sessionStop - startTs) / 1000) : 0);
|
| const mins = Math.floor(diff / 60).toString().padStart(2, '0');
|
| const secs = (diff % 60).toString().padStart(2, '0');
|
| document.getElementById('__mt_timer').textContent = 'Session Time: ' + mins + ':' + secs;
|
|
|
| const interests = Object.entries(dwellTotals).map(function(e){ return { element: e[0], duration: e[1] }; })
|
| .sort(function(a, b){ return b.duration - a.duration; }).slice(0, 6);
|
|
|
| const hero = document.getElementById('__mt_hero');
|
| if (interests.length) {
|
| hero.style.display = 'block';
|
| document.getElementById('__mt_hero_el').textContent = interests[0].element;
|
| document.getElementById('__mt_hero_time').textContent = formatTime(interests[0].duration);
|
| } else {
|
| hero.style.display = 'none';
|
| }
|
|
|
| const interestsEl = document.getElementById('__mt_interests');
|
| if (!logsOpen) {
|
| if (interests.length > 1) {
|
| interestsEl.innerHTML = interests.slice(1).map(function(it, i){
|
| return '<div class="interest-item">' + (i + 2) + '. <b>' + it.element + '</b> — ' + formatTime(it.duration) + '</div>';
|
| }).join('');
|
| } else {
|
| interestsEl.innerHTML = '<div class="muted">No other interests yet.</div>';
|
| }
|
| }
|
|
|
| const logsEl = document.getElementById('__mt_logs_list');
|
| if (logsOpen) {
|
| if (allClicks.length) {
|
| logsEl.innerHTML = allClicks.slice().reverse().slice(0, 20).map(function(l){
|
| return '<div class="log-item"><span class="muted">' + l.timestamp + '</span><br><b>' + l.element + '</b><br>"' + l.text + '"</div>';
|
| }).join('');
|
| } else {
|
| logsEl.innerHTML = '<div class="muted">No clicks recorded.</div>';
|
| }
|
| }
|
| }
|
| setInterval(updatePanel, 1000);
|
|
|
| // ---- hotkeys: H = toggle heatmap, M = toggle insights panel ----
|
| function isTypingTarget(el){
|
| if (!el) return false;
|
| const tag = el.tagName ? el.tagName.toLowerCase() : '';
|
| return tag === 'input' || tag === 'textarea' || tag === 'select' || el.isContentEditable === true;
|
| }
|
| document.addEventListener('keydown', function(e){
|
| const typingBlocked = isTypingTarget(e.target) || isTypingTarget(document.activeElement);
|
| if (typingBlocked) return;
|
| if (e.key === 'h' || e.key === 'H') window.insightuxMouseToggleHeatmap();
|
| else if (e.key === 'm' || e.key === 'M') window.insightuxMouseTogglePanel();
|
| }, true);
|
|
|
| updatePanel();
|
| })();
|
| """
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| _NON_TRACKABLE_URL_MARKERS = ("insightux_landing.html", "analysis_report.html")
|
|
|
|
|
| class Api:
|
| def __init__(self):
|
| self.window = None
|
| self.tracking = False
|
| self.stop_event = threading.Event()
|
| self.thread = None
|
| self.mouse_log_f = None
|
| self.mouse_close_timer = None
|
|
|
| def start_tracking(self):
|
| print("[browser_session] start_tracking() called from JS")
|
| if self.tracking:
|
| print("[browser_session] already tracking, ignoring")
|
| return False
|
| try:
|
| current_url = self.window.get_current_url() or ""
|
| except Exception:
|
| current_url = ""
|
| if any(marker in current_url for marker in _NON_TRACKABLE_URL_MARKERS):
|
| print(f"[browser_session] refusing to start on non-website page: {current_url}")
|
| self._set_status("Start tracking only works on a website — search or open a page first.")
|
| return False
|
| if not os.path.exists(CALIBRATION_PATH):
|
| print(f"[browser_session] no {CALIBRATION_PATH} found — run calibrate.py first")
|
| self._set_status("No calibration.pkl found — run calibrate.py first.")
|
| return False
|
|
|
| self.tracking = True
|
| self.stop_event.clear()
|
| session_dir = os.path.join(SESSIONS_ROOT, datetime.now().strftime("%Y%m%d_%H%M%S"))
|
| os.makedirs(session_dir, exist_ok=True)
|
| print(f"[browser_session] starting tracking thread -> {session_dir}")
|
|
|
| self._inject_tracking_overlay()
|
| self._inject_mouse_overlay()
|
| self._open_mouse_log(session_dir)
|
| self._set_mouse_tracking(True)
|
| self._set_status("Recording gaze + mouse... Press E to stop, H for heatmap, M for mouse panel")
|
|
|
| self.thread = threading.Thread(
|
| target=gaze_worker, args=(self.window, self.stop_event, session_dir, self),
|
| daemon=True
|
| )
|
| self.thread.start()
|
| return True
|
|
|
| def stop_tracking(self):
|
| print("[browser_session] stop_tracking() called from JS")
|
| if not self.tracking:
|
| print("[browser_session] not currently tracking, ignoring")
|
| return False
|
| self._set_status("Wrapping up your session...")
|
| self._set_mouse_tracking(False)
|
| self.stop_event.set()
|
|
|
|
|
|
|
| if self.mouse_close_timer:
|
| self.mouse_close_timer.cancel()
|
| self.mouse_close_timer = threading.Timer(2.5, self._close_mouse_log)
|
| self.mouse_close_timer.daemon = True
|
| self.mouse_close_timer.start()
|
| return True
|
|
|
| def quit_app(self):
|
|
|
|
|
| print("[browser_session] quit_app() called from JS -- terminating process")
|
| os._exit(0)
|
|
|
| def _inject_tracking_overlay(self):
|
| try:
|
| self.window.evaluate_js(TRACKING_JS)
|
| except Exception as e:
|
| print(f"[browser_session] overlay inject failed: {e}")
|
|
|
| def _inject_mouse_overlay(self):
|
| try:
|
| self.window.evaluate_js(MOUSE_JS)
|
| except Exception as e:
|
| print(f"[browser_session] mouse overlay inject failed: {e}")
|
|
|
| def _set_mouse_tracking(self, on):
|
| try:
|
| flag = "true" if on else "false"
|
| self.window.evaluate_js(
|
| f"window.insightuxMouseSetTracking && window.insightuxMouseSetTracking({flag})"
|
| )
|
| except Exception:
|
| pass
|
|
|
| def _open_mouse_log(self, session_dir):
|
| self._close_mouse_log()
|
| path = os.path.join(session_dir, "mouse_log.jsonl")
|
| try:
|
| self.mouse_log_f = open(path, "w", buffering=1)
|
| self.mouse_log_f.write(json.dumps({"type": "meta", "t": round(time.time(), 4)}) + "\n")
|
| print(f"[browser_session] writing mouse stream to {path}")
|
| except Exception as e:
|
| print(f"[browser_session] could not open mouse log: {e}")
|
| self.mouse_log_f = None
|
|
|
| def _close_mouse_log(self):
|
| if self.mouse_log_f:
|
| try:
|
| self.mouse_log_f.close()
|
| print("[browser_session] mouse log closed")
|
| except Exception:
|
| pass
|
| self.mouse_log_f = None
|
|
|
| def log_mouse_data(self, payload):
|
| """Called from the injected Mouse Tracker overlay (MOUSE_JS) roughly
|
| every 2s while tracking, with a batch of trail / heatmap / click /
|
| dwell records — same cadence as the extension's background.js sync
|
| loop, just persisted to mouse_log.jsonl instead of chrome.storage."""
|
| if not self.mouse_log_f:
|
| return False
|
| try:
|
| rec = dict(payload) if isinstance(payload, dict) else {}
|
| rec["type"] = "mouse_batch"
|
| rec["t"] = round(time.time(), 4)
|
| self.mouse_log_f.write(json.dumps(rec) + "\n")
|
| except Exception as e:
|
| print(f"[browser_session] mouse log write failed: {e}")
|
| return True
|
|
|
| def _set_status(self, text):
|
| try:
|
| self.window.evaluate_js(f"window.insightuxSetStatus && window.insightuxSetStatus({json.dumps(text)})")
|
| except Exception:
|
| pass
|
|
|
|
|
| api = Api()
|
|
|
|
|
| def inject_control(window):
|
| try:
|
| window.evaluate_js(CONTROL_JS)
|
| except Exception as e:
|
| print(f"[browser_session] control inject failed (will retry): {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| def gaze_worker(window, stop_event, session_dir, api_ref):
|
| pipeline = InsightUXPipeline(ONNX_PATH, CALIBRATION_PATH)
|
| face_mesh = create_face_mesh(static_image_mode=False)
|
| cap = cv2.VideoCapture(0)
|
| logger = GazeLogger(SCREEN_W, SCREEN_H, session_dir=session_dir)
|
| dom_path = os.path.join(session_dir, "dom_log.jsonl")
|
| dom_f = open(dom_path, "w", buffering=1)
|
|
|
| screens_dir = os.path.join(session_dir, "screenshots")
|
| os.makedirs(screens_dir, exist_ok=True)
|
| shot_count = 0
|
| last_shot_scroll = None
|
| last_shot_time = 0.0
|
| SCROLL_SHOT_THRESHOLD = 60
|
| MAX_SHOT_INTERVAL = 2.5
|
|
|
| def maybe_capture_screenshot(scroll_y):
|
| """Grab a full-screen shot when the page has scrolled enough, or
|
| periodically even if it hasn't — this is what lets the report show
|
| the heatmap ON TOP of the actual page instead of floating in a void."""
|
| nonlocal shot_count, last_shot_scroll, last_shot_time
|
| now = time.time()
|
| scrolled_enough = last_shot_scroll is None or abs(scroll_y - last_shot_scroll) >= SCROLL_SHOT_THRESHOLD
|
| time_elapsed = (now - last_shot_time) >= MAX_SHOT_INTERVAL
|
| if not (scrolled_enough or time_elapsed):
|
| return None
|
| try:
|
| img = pyautogui.screenshot()
|
|
|
|
|
|
|
|
|
| if img.size != (SCREEN_W, SCREEN_H):
|
| img = img.resize((SCREEN_W, SCREEN_H))
|
| shot_name = f"shot_{shot_count:04d}.png"
|
| img.save(os.path.join(screens_dir, shot_name))
|
| shot_count += 1
|
| last_shot_scroll = scroll_y
|
| last_shot_time = now
|
| return f"screenshots/{shot_name}"
|
| except Exception as e:
|
| print(f"[browser_session] screenshot capture failed: {e}")
|
| return None
|
|
|
| print(f"[browser_session] tracking started -> {session_dir}")
|
|
|
| cam_matrix = None
|
| last_x = SCREEN_W / 2
|
| last_y = SCREEN_H / 2
|
| fil_x = OneEuroFilter(mincutoff=EURO_MINCUTOFF, beta=EURO_BETA)
|
| fil_y = OneEuroFilter(mincutoff=EURO_MINCUTOFF, beta=EURO_BETA)
|
| angle_smoother = GazeAngleSmoother(window=ANGLE_SMOOTH_WINDOW)
|
| sm_pitch = sm_yaw = sm_roll = None
|
| no_face_count = 0
|
| frame_i = 0
|
|
|
| while not stop_event.is_set():
|
| ret, frame = cap.read()
|
| if not ret:
|
| break
|
|
|
| if cam_matrix is None:
|
| cam_matrix = estimate_camera_matrix(frame.shape)
|
|
|
| rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| results = face_mesh.process(rgb)
|
|
|
| if not results.multi_face_landmarks:
|
| no_face_count += 1
|
| if no_face_count >= NO_FACE_RESET:
|
| last_x, last_y = SCREEN_W / 2, SCREEN_H / 2
|
| fil_x.reset(); fil_y.reset()
|
| angle_smoother.reset()
|
| sm_pitch = sm_yaw = sm_roll = None
|
| no_face_count = 0
|
| continue
|
| no_face_count = 0
|
|
|
| lms = results.multi_face_landmarks[0].landmark
|
| head_pose = estimate_head_pose(lms, frame.shape, cam_matrix)
|
| if head_pose is None:
|
| continue
|
|
|
| if sm_pitch is None:
|
| sm_pitch, sm_yaw, sm_roll = head_pose.pitch, head_pose.yaw, head_pose.roll
|
| else:
|
| b = POSE_SMOOTH
|
| sm_pitch = b*sm_pitch + (1-b)*head_pose.pitch
|
| sm_yaw = b*sm_yaw + (1-b)*head_pose.yaw
|
| sm_roll = b*sm_roll + (1-b)*head_pose.roll
|
| head_pose = replace(head_pose, pitch=sm_pitch, yaw=sm_yaw, roll=sm_roll)
|
|
|
| pose_vec = normalize_pose(sm_pitch, sm_yaw, sm_roll)
|
|
|
|
|
|
|
| ear_now = 0.5 * (compute_ear(lms, LEFT_EAR_INDICES, frame.shape) +
|
| compute_ear(lms, RIGHT_EAR_INDICES, frame.shape))
|
|
|
| def get_patch(eye_idx, ear_idx, iris_idx):
|
| s1 = step1_normalize(frame, lms, head_pose, eye_idx, ear_idx, iris_idx)
|
| if not s1.is_open:
|
| return None
|
| if PATCH_SOURCE == "norm":
|
| return s1.norm_crop
|
| ir = compute_iris_radius(lms, iris_idx, frame.shape)
|
| s2 = step2_illumination(s1, ir)
|
| return s2.blended if s2.is_usable else None
|
|
|
| left_patch = get_patch(LEFT_EYE_INDICES, LEFT_EAR_INDICES, LEFT_IRIS_INDICES)
|
| right_patch = get_patch(RIGHT_EYE_INDICES, RIGHT_EAR_INDICES, RIGHT_IRIS_INDICES)
|
| if left_patch is None and right_patch is None:
|
| continue
|
| if left_patch is None: left_patch = right_patch
|
| if right_patch is None: right_patch = left_patch
|
|
|
| _, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(left_patch, pose_vec, right_patch)
|
| pitch = compensate_pitch(raw_pitch, sm_pitch)
|
| yaw = compensate_yaw(raw_yaw, sm_yaw)
|
|
|
|
|
|
|
|
|
|
|
| pitch, yaw, ear_s = angle_smoother(pitch, yaw, ear_now)
|
|
|
| sx, sy = pipeline.calibration.predict(pitch, yaw, ear_s)
|
| sx = max(0.0, min(sx, SCREEN_W))
|
| sy = max(0.0, min(sy, SCREEN_H))
|
|
|
| jump = np.hypot(sx - last_x, sy - last_y)
|
| if jump > MAX_JUMP:
|
| sx = last_x + (sx - last_x) * 0.3
|
| sy = last_y + (sy - last_y) * 0.3
|
| last_x, last_y = sx, sy
|
|
|
| logger.log(sx, sy)
|
|
|
| now = time.time()
|
| fx = fil_x(sx / SCREEN_W, now)
|
| fy = fil_y(sy / SCREEN_H, now)
|
| try:
|
| window.evaluate_js(f"window.insightuxUpdate({fx:.5f},{fy:.5f})")
|
| except Exception:
|
| pass
|
|
|
| frame_i += 1
|
| if frame_i % DOM_EVERY == 0:
|
| try:
|
| raw = window.evaluate_js("window.insightuxAOIs()")
|
| if raw:
|
| rec = json.loads(raw)
|
| rec["type"] = "dom"
|
| rec["t"] = round(time.time(), 4)
|
| shot_ref = maybe_capture_screenshot(rec.get("scrollY", 0))
|
| if shot_ref:
|
| rec["screenshot"] = shot_ref
|
| dom_f.write(json.dumps(rec) + "\n")
|
| except Exception:
|
| pass
|
|
|
| time.sleep(0.005)
|
|
|
| cap.release()
|
| logger.close()
|
| dom_f.close()
|
| print(f"[browser_session] session saved in {session_dir}")
|
|
|
|
|
| try:
|
| report_path = generate_report(session_dir)
|
| api_ref.tracking = False
|
| window.load_url("file://" + report_path)
|
| print(f"[browser_session] report ready -> {report_path}")
|
| except Exception as e:
|
| print(f"[browser_session] report generation failed: {e}")
|
| api_ref.tracking = False
|
|
|
|
|
|
|
|
|
|
|
|
|
| def _go_fullscreen(window):
|
|
|
|
|
|
|
|
|
|
|
|
|
| time.sleep(0.4)
|
| try:
|
| window.toggle_fullscreen()
|
| except Exception as e:
|
| print(f"[browser_session] fullscreen toggle failed: {e}")
|
|
|
|
|
| if __name__ == "__main__":
|
| os.makedirs(SESSIONS_ROOT, exist_ok=True)
|
|
|
| window = webview.create_window(
|
| "InsightUX — Eye-Tracking Research Browser", LANDING_URL,
|
| width=SCREEN_W, height=SCREEN_H, js_api=api
|
| )
|
| api.window = window
|
| window.events.loaded += lambda: inject_control(window)
|
| window.events.shown += lambda: threading.Thread(
|
| target=_go_fullscreen, args=(window,), daemon=True
|
| ).start()
|
|
|
|
|
|
|
|
|
|
|
| webview.start(debug=True) |
