| """ |
| browser_session.py |
| InsightUX research browser. |
| |
| Flow: |
| 1. Window opens on real google.com — search and click through like a |
| normal browser (no scraping, no fake results — it's just Google). |
| 2. Land on whatever page you want to study. Press S to start tracking. |
| 3. Gaze dot + AOI highlighting (same as run_session.py) kicks in and |
| logs sessions/<timestamp>/gaze_log.jsonl + dom_log.jsonl. |
| 4. Press E to stop. The window auto-navigates to a generated |
| analysis_report.html: ranked attention, dwell timeline, heatmap. |
| |
| Prerequisite: |
| calibration.pkl must exist in the working directory (run calibrate.py first). |
| |
| Run: |
| python browser_session.py |
| """ |
|
|
| import os |
| import sys |
| import json |
| import time |
| import math |
| import threading |
| from dataclasses import replace |
| from datetime import datetime |
|
|
| import cv2 |
| import numpy as np |
| import webview |
| import pyautogui |
|
|
| from preprocessing.preprocessing_pipeline import ( |
| create_face_mesh, |
| estimate_camera_matrix, |
| estimate_head_pose, |
| compute_iris_radius, |
| compute_ear, |
| step1_normalize, |
| step2_illumination, |
| LEFT_EYE_INDICES, |
| LEFT_EAR_INDICES, |
| LEFT_IRIS_INDICES, |
| RIGHT_EYE_INDICES, |
| RIGHT_EAR_INDICES, |
| RIGHT_IRIS_INDICES, |
| ) |
| from inference_pipeline import InsightUXPipeline, GazeAngleSmoother |
| from session_logger import GazeLogger |
| from analysis import generate_report |
|
|
|
|
| |
| |
| |
|
|
| ONNX_PATH = "models/gaze_cnn_v4.onnx" |
| CALIBRATION_PATH = "calibration.pkl" |
| SCREEN_W, SCREEN_H = pyautogui.size() |
|
|
| PATCH_SOURCE = "blended" |
| POSE_SMOOTH = 0.65 |
| MAX_JUMP = 220 |
| NO_FACE_RESET = 15 |
| DOM_EVERY = 6 |
|
|
| EURO_MINCUTOFF = 0.35 |
| EURO_BETA = 0.12 |
|
|
| |
| |
| |
| ANGLE_SMOOTH_WINDOW = 10 |
|
|
| POSE_NORM_SCALE = 30.0 |
| HEAD_PITCH_COMPENSATION = 0.0 |
| HEAD_YAW_COMPENSATION = 0.0 |
|
|
| SESSIONS_ROOT = "sessions" |
|
|
| |
| |
| |
| |
| _LANDING_HTML = r"""<!DOCTYPE html> |
| <html> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="insightux-landing" content="true"> |
| <title>InsightUX — Eye-Tracking Research Browser</title> |
| <style> |
| * { box-sizing: border-box; } |
| body { |
| margin: 0; height: 100vh; display: flex; flex-direction: column; |
| align-items: center; justify-content: center; |
| background: radial-gradient(circle at 50% 35%, #221a2e, #100e16 70%); |
| font-family: -apple-system, 'Segoe UI', Arial, sans-serif; color: #f0ecf7; |
| } |
| .logo { |
| font-size: 42px; font-weight: 700; letter-spacing: -0.01em; |
| background: linear-gradient(90deg, #9B59FF, #FF2DF0); |
| -webkit-background-clip: text; -webkit-text-fill-color: transparent; |
| margin-bottom: 6px; |
| } |
| .tagline { color: #8a8098; font-size: 14px; margin-bottom: 36px; } |
| form { width: 560px; max-width: 88vw; } |
| input { |
| width: 100%; padding: 15px 20px; font-size: 15px; border-radius: 26px; |
| border: 1.5px solid #3a3348; background: #1c1826; color: #f0ecf7; |
| outline: none; transition: border-color 0.15s; |
| } |
| input:focus { border-color: #9B59FF; } |
| input::placeholder { color: #635a72; } |
| .hint { |
| margin-top: 18px; font-size: 12px; color: #6b6278; text-align: center; |
| line-height: 1.6; |
| } |
| .hint b { color: #9B59FF; } |
| </style> |
| </head> |
| <body> |
| <div class="logo">InsightUX</div> |
| <div class="tagline">Eye-Tracking Research Browser</div> |
| <form id="searchForm"> |
| <input id="searchInput" type="text" autofocus |
| placeholder="Search Google, or enter a website address"> |
| </form> |
| <div class="hint"> |
| Search results and websites open below as normal.<br> |
| Once you land on the page you want to study, press <b>S</b> to start eye-tracking, |
| <b>E</b> to stop. |
| </div> |
| <script> |
| document.getElementById('searchForm').addEventListener('submit', function(e){ |
| e.preventDefault(); |
| const raw = document.getElementById('searchInput').value.trim(); |
| if (!raw) return; |
| |
| const looksLikeUrl = /^https?:\/\//i.test(raw) || |
| (/^[\w-]+(\.[\w-]+)+([/?#].*)?$/i.test(raw) && !raw.includes(' ')); |
| |
| if (looksLikeUrl) { |
| window.location.href = raw.startsWith('http') ? raw : ('https://' + raw); |
| } else { |
| window.location.href = 'https://www.google.com/search?q=' + encodeURIComponent(raw); |
| } |
| }); |
| </script> |
| </body> |
| </html> |
| """ |
|
|
|
|
| def _write_landing_page(): |
| path = os.path.abspath("insightux_landing.html") |
| with open(path, "w", encoding="utf-8") as f: |
| f.write(_LANDING_HTML) |
| return "file://" + path.replace(os.sep, "/") |
|
|
|
|
| LANDING_URL = _write_landing_page() |
|
|
|
|
| def normalize_pose(pitch_deg, yaw_deg, roll_deg): |
| return np.array([ |
| pitch_deg / POSE_NORM_SCALE, |
| yaw_deg / POSE_NORM_SCALE, |
| roll_deg / POSE_NORM_SCALE, |
| ], dtype=np.float32) |
|
|
| def compensate_pitch(raw_pitch, head_pitch_deg): |
| return raw_pitch - np.radians(head_pitch_deg) * HEAD_PITCH_COMPENSATION |
|
|
| def compensate_yaw(raw_yaw, head_yaw_deg): |
| return raw_yaw - np.radians(head_yaw_deg) * HEAD_YAW_COMPENSATION |
|
|
|
|
| |
| |
| |
|
|
| class OneEuroFilter: |
| def __init__(self, mincutoff=0.8, beta=0.4, dcutoff=1.0): |
| self.mincutoff = mincutoff |
| self.beta = beta |
| self.dcutoff = dcutoff |
| self.x_prev = None |
| self.dx_prev = 0.0 |
| self.t_prev = None |
|
|
| @staticmethod |
| def _alpha(cutoff, dt): |
| tau = 1.0 / (2 * math.pi * cutoff) |
| return 1.0 / (1.0 + tau / dt) |
|
|
| def __call__(self, x, t): |
| if self.x_prev is None: |
| self.x_prev, self.t_prev = x, t |
| return x |
| dt = t - self.t_prev |
| if dt <= 0: |
| dt = 1e-3 |
| self.t_prev = t |
| dx = (x - self.x_prev) / dt |
| a_d = self._alpha(self.dcutoff, dt) |
| dx_hat = a_d * dx + (1 - a_d) * self.dx_prev |
| cutoff = self.mincutoff + self.beta * abs(dx_hat) |
| a = self._alpha(cutoff, dt) |
| x_hat = a * x + (1 - a) * self.x_prev |
| self.x_prev, self.dx_prev = x_hat, dx_hat |
| return x_hat |
|
|
| def reset(self): |
| self.x_prev = None |
| self.dx_prev = 0.0 |
| self.t_prev = None |
|
|
|
|
| |
| |
| |
|
|
| CONTROL_JS = r""" |
| (function(){ |
| if (window.__insightuxControl) { return; } |
| // The local landing page is already fully InsightUX-branded — don't |
| // stack a second banner on top of it. |
| if (document.querySelector('meta[name="insightux-landing"]')) { return; } |
| window.__insightuxControl = true; |
| |
| const banner = document.createElement('div'); |
| banner.id = '__insightux_pill'; |
| banner.style.cssText = ` |
| position:fixed; top:0; left:0; right:0; z-index:2147483647; |
| display:flex; align-items:center; justify-content:space-between; |
| padding:7px 16px; font:12px -apple-system,Arial; |
| background:linear-gradient(90deg, rgba(20,18,26,0.95), rgba(30,20,40,0.95)); |
| border-bottom:2px solid; border-image:linear-gradient(90deg,#7B2FBE,#FF2DF0) 1; |
| box-shadow:0 3px 14px rgba(0,0,0,0.35); pointer-events:none; |
| `; |
| banner.innerHTML = ` |
| <span style="display:flex;align-items:center;gap:7px;"> |
| <span style="width:8px;height:8px;border-radius:50%;background:linear-gradient(135deg,#7B2FBE,#FF2DF0);display:inline-block;"></span> |
| <b style="color:#f0ecf7;letter-spacing:0.02em;">InsightUX</b> |
| <span style="color:#7a7288;">Eye-Tracking Research Browser</span> |
| </span> |
| <span id="__insightux_status" style="color:#c9a6f5;">Press S to start eye-tracking on this page</span> |
| `; |
| document.documentElement.appendChild(banner); |
| |
| setInterval(function(){ |
| if (!document.documentElement.contains(banner)) document.documentElement.appendChild(banner); |
| }, 1000); |
| |
| const statusEl = () => document.getElementById('__insightux_status'); |
| window.insightuxSetStatus = function(text){ |
| const el = statusEl(); |
| if (el) el.textContent = text; |
| }; |
| |
| function isTypingTarget(el){ |
| if (!el) return false; |
| const tag = el.tagName ? el.tagName.toLowerCase() : ''; |
| return tag === 'input' || tag === 'textarea' || tag === 'select' || |
| el.isContentEditable === true; |
| } |
| |
| document.addEventListener('keydown', function(e){ |
| const typingBlocked = isTypingTarget(e.target) || isTypingTarget(document.activeElement); |
| const apiReady = !!(window.pywebview && window.pywebview.api); |
| if (e.key === 's' || e.key === 'S' || e.key === 'e' || e.key === 'E') { |
| console.log('[insightux] key=' + e.key, 'typingBlocked=' + typingBlocked, 'apiReady=' + apiReady, 'focusedEl=' + (document.activeElement && document.activeElement.tagName)); |
| } |
| if (!apiReady) return; |
| if (typingBlocked) { |
| window.insightuxSetStatus("Click somewhere on the page (not a text field) before pressing S/E"); |
| return; |
| } |
| |
| if (e.key === 's' || e.key === 'S') { |
| window.pywebview.api.start_tracking(); |
| } else if (e.key === 'e' || e.key === 'E') { |
| window.pywebview.api.stop_tracking(); |
| } |
| }, true); // capture phase — fires before page scripts can intercept/stop the event |
| })(); |
| """ |
|
|
| |
| |
| TRACKING_JS = r""" |
| (function(){ |
| if (window.__insightux) { return; } |
| const state = { aois: [] }; |
| const dwell = { pendLabel: null, pendSince: 0, activeLabel: null, emptySince: 0 }; |
| const DWELL_MS = 420; |
| const RELEASE_MS = 1200; |
| |
| let activeBox = null; |
| const target = { fx: 0.5, fy: 0.5 }; |
| const dot = { x: null, y: null }; |
| const DOT_LERP = 0.07; |
| |
| const cv = document.createElement('canvas'); |
| cv.id = '__insightux_canvas'; |
| cv.style.cssText = 'position:fixed;left:0;top:0;width:100vw;height:100vh;pointer-events:none;z-index:2147483646;'; |
| (document.body || document.documentElement).appendChild(cv); |
| const ctx = cv.getContext('2d'); |
| function resize(){ cv.width = window.innerWidth; cv.height = window.innerHeight; } |
| resize(); |
| window.addEventListener('resize', resize, {passive:true}); |
| setInterval(function(){ |
| if (!document.body.contains(cv)) document.body.appendChild(cv); |
| }, 1000); |
| |
| const MIN_W=40, MIN_H=24, MAX_AOIS=90, MAX_SCAN=2500; |
| const PAD_PX = 90; |
| const TALL_WRAPPER_H = 220; |
| |
| const ALWAYS_SELECTOR = "nav, header, footer, img, video, iframe, h1, h2, h3, button, figure, [class*='hero'], [class*='banner'], [class*='card']"; |
| const TEXT_CONTAINER_SELECTOR = "div, span, section, article, main, aside, p, li"; |
| |
| function hasOwnText(el){ |
| for (const node of el.childNodes){ |
| if (node.nodeType === 3 && node.textContent.trim().length > 2) return true; |
| } |
| return false; |
| } |
| |
| function labelFor(el){ |
| if (el.dataset && el.dataset.aoi) return el.dataset.aoi.slice(0,40); |
| const tag = el.tagName.toLowerCase(); |
| if (tag==='nav') return 'navbar'; |
| if (tag==='header') return 'header'; |
| if (tag==='footer') return 'footer'; |
| if (tag==='img'){ |
| const alt=(el.getAttribute('alt')||'').trim(); |
| if (alt) return 'img: '+alt.slice(0,30); |
| const src=el.getAttribute('src')||''; |
| const name=src.split('/').pop().split('?')[0]; |
| return 'img: '+(name||'image').slice(0,30); |
| } |
| if (tag==='video') return 'video'; |
| if (tag==='iframe') return 'embed'; |
| if (tag==='h1'||tag==='h2'){ |
| const t=(el.innerText||'').trim().replace(/\s+/g,' '); |
| if (t) return tag+': '+t.slice(0,30); |
| } |
| const id=el.id?('#'+el.id):''; |
| let cls=''; |
| if (el.className && typeof el.className==='string'){ |
| const f=el.className.trim().split(/\s+/)[0]; |
| if (f) cls='.'+f; |
| } |
| const txt=(el.innerText||'').trim().replace(/\s+/g,' ').slice(0,24); |
| const base=id||cls||tag; |
| return txt?(base+' ('+txt+')'):base; |
| } |
| |
| function refresh(){ |
| const vh = window.innerHeight; |
| const raw = []; |
| const seen = new Set(); |
| let full = false; |
| |
| function consider(el){ |
| if (full) return; |
| // Never track InsightUX's own injected UI (the "press S/E" pill, |
| // the gaze-dot canvas) as if it were page content. |
| if (el.id === '__insightux_pill' || el.id === '__insightux_canvas') return; |
| if (el.closest && el.closest('#__insightux_pill, #__insightux_canvas')) return; |
| const tag = el.tagName.toLowerCase(); |
| const explicit = !!(el.dataset && el.dataset.aoi); |
| if (!explicit){ |
| const isAlways = (tag==='nav'||tag==='header'||tag==='footer'||tag==='img'|| |
| tag==='video'||tag==='iframe'|| |
| tag==='h1'||tag==='h2'||tag==='h3'||tag==='button'||tag==='figure') || |
| (el.className && typeof el.className==='string' && |
| /hero|banner|card/i.test(el.className)); |
| if (!isAlways && !hasOwnText(el)) return; |
| } |
| const r = el.getBoundingClientRect(); |
| if (r.width<MIN_W || r.height<MIN_H) return; |
| if (r.bottom<0 || r.top>vh) return; |
| const label = labelFor(el); |
| if (!label) return; |
| const key = label+'@'+Math.round(r.left)+','+Math.round(r.top); |
| if (seen.has(key)) return; |
| seen.add(key); |
| const pos = getComputedStyle(el).position; |
| raw.push({el:el, label:label, x:Math.round(r.left), y:Math.round(r.top), |
| w:Math.round(r.width), h:Math.round(r.height), |
| sticky:(pos==='sticky'||pos==='fixed')}); |
| if (raw.length >= MAX_AOIS*2) full = true; |
| } |
| |
| document.querySelectorAll('[data-aoi]').forEach(consider); |
| document.querySelectorAll(ALWAYS_SELECTOR).forEach(consider); |
| |
| const candidates = document.querySelectorAll(TEXT_CONTAINER_SELECTOR); |
| for (let i=0; i<candidates.length && i<MAX_SCAN && !full; i++){ |
| consider(candidates[i]); |
| } |
| |
| const filtered = raw.filter(o => { |
| const isTallWrapper = o.h > TALL_WRAPPER_H && |
| raw.some(o2 => o2 !== o && o.el.contains(o2.el)); |
| return !isTallWrapper; |
| }); |
| |
| state.aois = filtered.slice(0, MAX_AOIS).map(o => ({ |
| label:o.label, x:o.x, y:o.y, w:o.w, h:o.h, sticky:o.sticky |
| })); |
| } |
| refresh(); |
| window.addEventListener('scroll', refresh, {passive:true}); |
| setInterval(refresh, 400); |
| |
| window.insightuxAOIs = function(){ |
| return JSON.stringify({ |
| url: location.href, |
| scrollX: Math.round(window.scrollX), |
| scrollY: Math.round(window.scrollY), |
| viewport:{w:window.innerWidth,h:window.innerHeight}, |
| page:{w:document.documentElement.scrollWidth,h:document.documentElement.scrollHeight}, |
| aois: state.aois |
| }); |
| }; |
| |
| window.insightuxUpdate = function(fx, fy){ |
| target.fx = fx; |
| target.fy = fy; |
| }; |
| |
| function renderLoop(){ |
| const w = window.innerWidth, h = window.innerHeight; |
| const tx = target.fx * w, ty = target.fy * h; |
| if (dot.x === null){ dot.x = tx; dot.y = ty; } |
| dot.x += (tx - dot.x) * DOT_LERP; |
| dot.y += (ty - dot.y) * DOT_LERP; |
| |
| const now = performance.now(); |
| const px = dot.x, py = dot.y; |
| |
| let cand = null; |
| for (const a of state.aois){ |
| if (px>=a.x-PAD_PX && px<=a.x+a.w+PAD_PX && py>=a.y-PAD_PX && py<=a.y+a.h+PAD_PX){ |
| if (!cand || (a.w*a.h)<(cand.w*cand.h)) cand = a; |
| } |
| } |
| |
| const candLabel = cand ? cand.label : null; |
| if (candLabel !== dwell.pendLabel){ dwell.pendLabel = candLabel; dwell.pendSince = now; } |
| if (cand){ |
| dwell.emptySince = 0; |
| if (dwell.activeLabel !== cand.label && (now - dwell.pendSince) >= DWELL_MS){ |
| dwell.activeLabel = cand.label; |
| } |
| } else { |
| if (dwell.emptySince === 0) dwell.emptySince = now; |
| if (dwell.activeLabel && (now - dwell.emptySince) >= RELEASE_MS){ |
| dwell.activeLabel = null; |
| } |
| } |
| |
| let active = null; |
| if (dwell.activeLabel){ |
| for (const a of state.aois){ if (a.label === dwell.activeLabel){ active = a; break; } } |
| if (!active) dwell.activeLabel = null; |
| } |
| activeBox = active; |
| |
| ctx.clearRect(0,0,cv.width,cv.height); |
| |
| if (activeBox){ |
| ctx.fillStyle = 'rgba(123,47,190,0.16)'; |
| ctx.fillRect(activeBox.x, activeBox.y, activeBox.w, activeBox.h); |
| ctx.strokeStyle = '#7B2FBE'; ctx.lineWidth = 3; |
| ctx.strokeRect(activeBox.x, activeBox.y, activeBox.w, activeBox.h); |
| const label = activeBox.label; |
| ctx.font = 'bold 13px Arial'; |
| const tw = ctx.measureText(label).width + 16; |
| const ly = Math.max(0, activeBox.y - 22); |
| ctx.fillStyle = '#7B2FBE'; |
| ctx.fillRect(activeBox.x, ly, tw, 20); |
| ctx.fillStyle = '#FFFFFF'; |
| ctx.fillText(label, activeBox.x + 8, ly + 14); |
| } |
| |
| ctx.beginPath(); |
| ctx.arc(dot.x, dot.y, 13, 0, 2*Math.PI); |
| ctx.fillStyle = 'rgba(255,255,255,0.9)'; |
| ctx.fill(); |
| ctx.beginPath(); |
| ctx.arc(dot.x, dot.y, 9, 0, 2*Math.PI); |
| ctx.fillStyle = '#FF2DF0'; |
| ctx.fill(); |
| ctx.lineWidth = 2; |
| ctx.strokeStyle = 'rgba(0,0,0,0.55)'; |
| ctx.stroke(); |
| |
| requestAnimationFrame(renderLoop); |
| } |
| requestAnimationFrame(renderLoop); |
| |
| window.__insightux = true; |
| })(); |
| """ |
|
|
|
|
| |
| |
| |
|
|
| class Api: |
| def __init__(self): |
| self.window = None |
| self.tracking = False |
| self.stop_event = threading.Event() |
| self.thread = None |
|
|
| def start_tracking(self): |
| print("[browser_session] start_tracking() called from JS") |
| if self.tracking: |
| print("[browser_session] already tracking, ignoring") |
| return False |
| if not os.path.exists(CALIBRATION_PATH): |
| print(f"[browser_session] no {CALIBRATION_PATH} found — run calibrate.py first") |
| self._set_status("No calibration.pkl found — run calibrate.py first.") |
| return False |
|
|
| self.tracking = True |
| self.stop_event.clear() |
| session_dir = os.path.join(SESSIONS_ROOT, datetime.now().strftime("%Y%m%d_%H%M%S")) |
| os.makedirs(session_dir, exist_ok=True) |
| print(f"[browser_session] starting tracking thread -> {session_dir}") |
|
|
| self._inject_tracking_overlay() |
| self._set_status("Recording... Press E to stop") |
|
|
| self.thread = threading.Thread( |
| target=gaze_worker, args=(self.window, self.stop_event, session_dir, self), |
| daemon=True |
| ) |
| self.thread.start() |
| return True |
|
|
| def stop_tracking(self): |
| print("[browser_session] stop_tracking() called from JS") |
| if not self.tracking: |
| print("[browser_session] not currently tracking, ignoring") |
| return False |
| self._set_status("Wrapping up your session...") |
| self.stop_event.set() |
| return True |
|
|
| def _inject_tracking_overlay(self): |
| try: |
| self.window.evaluate_js(TRACKING_JS) |
| except Exception as e: |
| print(f"[browser_session] overlay inject failed: {e}") |
|
|
| def _set_status(self, text): |
| try: |
| self.window.evaluate_js(f"window.insightuxSetStatus && window.insightuxSetStatus({json.dumps(text)})") |
| except Exception: |
| pass |
|
|
|
|
| api = Api() |
|
|
|
|
| def inject_control(window): |
| try: |
| window.evaluate_js(CONTROL_JS) |
| except Exception as e: |
| print(f"[browser_session] control inject failed (will retry): {e}") |
|
|
|
|
| |
| |
| |
|
|
| def gaze_worker(window, stop_event, session_dir, api_ref): |
| pipeline = InsightUXPipeline(ONNX_PATH, CALIBRATION_PATH) |
| face_mesh = create_face_mesh(static_image_mode=False) |
| cap = cv2.VideoCapture(0) |
| logger = GazeLogger(SCREEN_W, SCREEN_H, session_dir=session_dir) |
| dom_path = os.path.join(session_dir, "dom_log.jsonl") |
| dom_f = open(dom_path, "w", buffering=1) |
|
|
| screens_dir = os.path.join(session_dir, "screenshots") |
| os.makedirs(screens_dir, exist_ok=True) |
| shot_count = 0 |
| last_shot_scroll = None |
| last_shot_time = 0.0 |
| SCROLL_SHOT_THRESHOLD = 60 |
| MAX_SHOT_INTERVAL = 2.5 |
|
|
| def maybe_capture_screenshot(scroll_y): |
| """Grab a full-screen shot when the page has scrolled enough, or |
| periodically even if it hasn't — this is what lets the report show |
| the heatmap ON TOP of the actual page instead of floating in a void.""" |
| nonlocal shot_count, last_shot_scroll, last_shot_time |
| now = time.time() |
| scrolled_enough = last_shot_scroll is None or abs(scroll_y - last_shot_scroll) >= SCROLL_SHOT_THRESHOLD |
| time_elapsed = (now - last_shot_time) >= MAX_SHOT_INTERVAL |
| if not (scrolled_enough or time_elapsed): |
| return None |
| try: |
| img = pyautogui.screenshot() |
| |
| |
| |
| |
| if img.size != (SCREEN_W, SCREEN_H): |
| img = img.resize((SCREEN_W, SCREEN_H)) |
| shot_name = f"shot_{shot_count:04d}.png" |
| img.save(os.path.join(screens_dir, shot_name)) |
| shot_count += 1 |
| last_shot_scroll = scroll_y |
| last_shot_time = now |
| return f"screenshots/{shot_name}" |
| except Exception as e: |
| print(f"[browser_session] screenshot capture failed: {e}") |
| return None |
|
|
| print(f"[browser_session] tracking started -> {session_dir}") |
|
|
| cam_matrix = None |
| last_x = SCREEN_W / 2 |
| last_y = SCREEN_H / 2 |
| fil_x = OneEuroFilter(mincutoff=EURO_MINCUTOFF, beta=EURO_BETA) |
| fil_y = OneEuroFilter(mincutoff=EURO_MINCUTOFF, beta=EURO_BETA) |
| angle_smoother = GazeAngleSmoother(window=ANGLE_SMOOTH_WINDOW) |
| sm_pitch = sm_yaw = sm_roll = None |
| no_face_count = 0 |
| frame_i = 0 |
|
|
| while not stop_event.is_set(): |
| ret, frame = cap.read() |
| if not ret: |
| break |
|
|
| if cam_matrix is None: |
| cam_matrix = estimate_camera_matrix(frame.shape) |
|
|
| rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) |
| results = face_mesh.process(rgb) |
|
|
| if not results.multi_face_landmarks: |
| no_face_count += 1 |
| if no_face_count >= NO_FACE_RESET: |
| last_x, last_y = SCREEN_W / 2, SCREEN_H / 2 |
| fil_x.reset(); fil_y.reset() |
| angle_smoother.reset() |
| sm_pitch = sm_yaw = sm_roll = None |
| no_face_count = 0 |
| continue |
| no_face_count = 0 |
|
|
| lms = results.multi_face_landmarks[0].landmark |
| head_pose = estimate_head_pose(lms, frame.shape, cam_matrix) |
| if head_pose is None: |
| continue |
|
|
| if sm_pitch is None: |
| sm_pitch, sm_yaw, sm_roll = head_pose.pitch, head_pose.yaw, head_pose.roll |
| else: |
| b = POSE_SMOOTH |
| sm_pitch = b*sm_pitch + (1-b)*head_pose.pitch |
| sm_yaw = b*sm_yaw + (1-b)*head_pose.yaw |
| sm_roll = b*sm_roll + (1-b)*head_pose.roll |
| head_pose = replace(head_pose, pitch=sm_pitch, yaw=sm_yaw, roll=sm_roll) |
|
|
| pose_vec = normalize_pose(sm_pitch, sm_yaw, sm_roll) |
|
|
| |
| |
| ear_now = 0.5 * (compute_ear(lms, LEFT_EAR_INDICES, frame.shape) + |
| compute_ear(lms, RIGHT_EAR_INDICES, frame.shape)) |
|
|
| def get_patch(eye_idx, ear_idx, iris_idx): |
| s1 = step1_normalize(frame, lms, head_pose, eye_idx, ear_idx, iris_idx) |
| if not s1.is_open: |
| return None |
| if PATCH_SOURCE == "norm": |
| return s1.norm_crop |
| ir = compute_iris_radius(lms, iris_idx, frame.shape) |
| s2 = step2_illumination(s1, ir) |
| return s2.blended if s2.is_usable else None |
|
|
| left_patch = get_patch(LEFT_EYE_INDICES, LEFT_EAR_INDICES, LEFT_IRIS_INDICES) |
| right_patch = get_patch(RIGHT_EYE_INDICES, RIGHT_EAR_INDICES, RIGHT_IRIS_INDICES) |
| if left_patch is None and right_patch is None: |
| continue |
| if left_patch is None: left_patch = right_patch |
| if right_patch is None: right_patch = left_patch |
|
|
| _, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(left_patch, pose_vec, right_patch) |
| pitch = compensate_pitch(raw_pitch, sm_pitch) |
| yaw = compensate_yaw(raw_yaw, sm_yaw) |
|
|
| |
| |
| |
| |
| pitch, yaw, ear_s = angle_smoother(pitch, yaw, ear_now) |
|
|
| sx, sy = pipeline.calibration.predict(pitch, yaw, ear_s) |
| sx = max(0.0, min(sx, SCREEN_W)) |
| sy = max(0.0, min(sy, SCREEN_H)) |
|
|
| jump = np.hypot(sx - last_x, sy - last_y) |
| if jump > MAX_JUMP: |
| sx = last_x + (sx - last_x) * 0.3 |
| sy = last_y + (sy - last_y) * 0.3 |
| last_x, last_y = sx, sy |
|
|
| logger.log(sx, sy) |
|
|
| now = time.time() |
| fx = fil_x(sx / SCREEN_W, now) |
| fy = fil_y(sy / SCREEN_H, now) |
| try: |
| window.evaluate_js(f"window.insightuxUpdate({fx:.5f},{fy:.5f})") |
| except Exception: |
| pass |
|
|
| frame_i += 1 |
| if frame_i % DOM_EVERY == 0: |
| try: |
| raw = window.evaluate_js("window.insightuxAOIs()") |
| if raw: |
| rec = json.loads(raw) |
| rec["type"] = "dom" |
| rec["t"] = round(time.time(), 4) |
| shot_ref = maybe_capture_screenshot(rec.get("scrollY", 0)) |
| if shot_ref: |
| rec["screenshot"] = shot_ref |
| dom_f.write(json.dumps(rec) + "\n") |
| except Exception: |
| pass |
|
|
| time.sleep(0.005) |
|
|
| cap.release() |
| logger.close() |
| dom_f.close() |
| print(f"[browser_session] session saved in {session_dir}") |
|
|
| |
| try: |
| report_path = generate_report(session_dir) |
| api_ref.tracking = False |
| window.load_url("file://" + report_path) |
| print(f"[browser_session] report ready -> {report_path}") |
| except Exception as e: |
| print(f"[browser_session] report generation failed: {e}") |
| api_ref.tracking = False |
|
|
|
|
| |
| |
| |
|
|
| def _go_fullscreen(window): |
| |
| |
| |
| |
| |
| |
| time.sleep(0.4) |
| try: |
| window.toggle_fullscreen() |
| except Exception as e: |
| print(f"[browser_session] fullscreen toggle failed: {e}") |
|
|
|
|
| if __name__ == "__main__": |
| os.makedirs(SESSIONS_ROOT, exist_ok=True) |
|
|
| window = webview.create_window( |
| "InsightUX — Eye-Tracking Research Browser", LANDING_URL, |
| width=SCREEN_W, height=SCREEN_H, js_api=api |
| ) |
| api.window = window |
| window.events.loaded += lambda: inject_control(window) |
| window.events.shown += lambda: threading.Thread( |
| target=_go_fullscreen, args=(window,), daemon=True |
| ).start() |
|
|
| |
| |
| |
| |
| webview.start(debug=True) |
