Add browser chrome, packaging pipeline, and fix UTF-8 crash in calibration
Browse filesBrowser chrome: top toolbar (back/forward/reload/home/address bar) and left
sidebar (Calibrate/Validate/Start Session) replace the old status pill, so
the window reads as a normal browser. Page content is pushed below/right of
the chrome via forced body padding instead of being hidden underneath it.
Session overlays now re-inject on navigation so a tracking session survives
back/forward. Added an in-app "update available" notice backed by
version.json.
Packaging: calibrate.py refactored into main() so it can run as
`--mode calibrate` inside a frozen build instead of shelling out to a
missing .py file; RESOURCE_DIR/DATA_DIR split (bundled assets vs. per-user
writable data) added to all three entry points. New packaging/ with a
PyInstaller onedir spec and an Inno Setup installer script (per-user install,
fixed AppId for in-place upgrades).
Fix: Windows consoles default to cp1252, so a single unicode character in a
print() (calibrate.py's low-sample "⚠" warning) crashed the whole
calibration run with UnicodeEncodeError — hit for real mid-session. All
three entry points now force UTF-8 stdout/stderr with errors="replace".
- .gitattributes +1 -0
- .gitignore +5 -0
- README.md +3 -0
- browser_session.py +395 -59
- calibrate.py +538 -511
- packaging/InsightUX.spec +131 -0
- packaging/README.md +87 -0
- packaging/installer.iss +87 -0
- requirements.txt +1 -0
- validate.py +20 -2
- version.json +5 -0
|
@@ -1,3 +1,4 @@
|
|
| 1 |
*.onnx.data filter=lfs diff=lfs merge=lfs -text
|
| 2 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 3 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 1 |
*.onnx.data filter=lfs diff=lfs merge=lfs -text
|
| 2 |
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 3 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.exe filter=lfs diff=lfs merge=lfs -text
|
|
@@ -16,6 +16,7 @@
|
|
| 16 |
!.gitattributes
|
| 17 |
!README.md
|
| 18 |
!requirements.txt
|
|
|
|
| 19 |
|
| 20 |
!calibrate.py
|
| 21 |
!validate.py
|
|
@@ -24,6 +25,10 @@
|
|
| 24 |
!inference_pipeline.py
|
| 25 |
!session_logger.py
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
!preprocessing/__init__.py
|
| 28 |
!preprocessing/preprocessing_pipeline.py
|
| 29 |
|
|
|
|
| 16 |
!.gitattributes
|
| 17 |
!README.md
|
| 18 |
!requirements.txt
|
| 19 |
+
!version.json
|
| 20 |
|
| 21 |
!calibrate.py
|
| 22 |
!validate.py
|
|
|
|
| 25 |
!inference_pipeline.py
|
| 26 |
!session_logger.py
|
| 27 |
|
| 28 |
+
!packaging/InsightUX.spec
|
| 29 |
+
!packaging/installer.iss
|
| 30 |
+
!packaging/README.md
|
| 31 |
+
|
| 32 |
!preprocessing/__init__.py
|
| 33 |
!preprocessing/preprocessing_pipeline.py
|
| 34 |
|
|
@@ -52,6 +52,9 @@ venv\Scripts\activate
|
|
| 52 |
pip install -r requirements.txt
|
| 53 |
```
|
| 54 |
|
|
|
|
|
|
|
|
|
|
| 55 |
**Do not `pip install --upgrade` pywebview or pythonnet.** `requirements.txt`
|
| 56 |
pins `pywebview==4.4.1` and `pythonnet==3.0.3` deliberately — newer versions
|
| 57 |
have a bug in their Windows backend that freezes the app window and floods
|
|
|
|
| 52 |
pip install -r requirements.txt
|
| 53 |
```
|
| 54 |
|
| 55 |
+
This includes `onnx`, which `calibrate.py` needs when it exports the
|
| 56 |
+
fine-tuned model after training.
|
| 57 |
+
|
| 58 |
**Do not `pip install --upgrade` pywebview or pythonnet.** `requirements.txt`
|
| 59 |
pins `pywebview==4.4.1` and `pythonnet==3.0.3` deliberately — newer versions
|
| 60 |
have a bug in their Windows backend that freezes the app window and floods
|
|
@@ -24,9 +24,20 @@ import json
|
|
| 24 |
import time
|
| 25 |
import math
|
| 26 |
import threading
|
|
|
|
| 27 |
from dataclasses import replace
|
| 28 |
from datetime import datetime
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
import cv2
|
| 31 |
import numpy as np
|
| 32 |
import webview
|
|
@@ -56,8 +67,25 @@ from analysis import generate_report
|
|
| 56 |
# CONFIG — kept identical to run_session.py so calibration.pkl stays valid
|
| 57 |
# =============================================================================
|
| 58 |
|
| 59 |
-
|
| 60 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
SCREEN_W, SCREEN_H = pyautogui.size()
|
| 62 |
|
| 63 |
PATCH_SOURCE = "blended"
|
|
@@ -78,7 +106,7 @@ POSE_NORM_SCALE = 30.0
|
|
| 78 |
HEAD_PITCH_COMPENSATION = 0.0 # reverted — 0.35 made accuracy worse, not better
|
| 79 |
HEAD_YAW_COMPENSATION = 0.0
|
| 80 |
|
| 81 |
-
SESSIONS_ROOT = "sessions"
|
| 82 |
|
| 83 |
# The window opens on this local, fully InsightUX-branded page instead of
|
| 84 |
# raw google.com. What you type here still hits REAL Google — no fake
|
|
@@ -154,7 +182,7 @@ _LANDING_HTML = r"""<!DOCTYPE html>
|
|
| 154 |
|
| 155 |
|
| 156 |
def _write_landing_page():
|
| 157 |
-
path = os.path.
|
| 158 |
with open(path, "w", encoding="utf-8") as f:
|
| 159 |
f.write(_LANDING_HTML)
|
| 160 |
return "file://" + path.replace(os.sep, "/")
|
|
@@ -219,52 +247,184 @@ class OneEuroFilter:
|
|
| 219 |
|
| 220 |
|
| 221 |
# =============================================================================
|
| 222 |
-
# JS:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
# =============================================================================
|
| 224 |
|
| 225 |
-
|
| 226 |
(function(){
|
| 227 |
if (window.__insightuxControl) { return; }
|
| 228 |
window.__insightuxControl = true;
|
| 229 |
|
| 230 |
// S/E (start/stop tracking) only make sense on an actual website — not on
|
| 231 |
// the InsightUX landing/search page and not on a generated insights
|
| 232 |
-
// report page. X (quit
|
| 233 |
const isLanding = !!document.querySelector('meta[name="insightux-landing"]');
|
| 234 |
const isReport = !!document.querySelector('meta[name="insightux-report"]');
|
| 235 |
const isTrackable = !isLanding && !isReport;
|
| 236 |
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
border-bottom:2px solid; border-image:linear-gradient(90deg,#7B2FBE,#FF2DF0) 1;
|
| 246 |
-
box-shadow:0 3px 14px rgba(0,0,0,0.35);
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
}
|
| 261 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 262 |
|
| 263 |
const statusEl = () => document.getElementById('__insightux_status');
|
| 264 |
window.insightuxSetStatus = function(text){
|
| 265 |
const el = statusEl();
|
| 266 |
if (el) el.textContent = text;
|
| 267 |
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
|
| 269 |
function isTypingTarget(el){
|
| 270 |
if (!el) return false;
|
|
@@ -302,12 +462,15 @@ CONTROL_JS = r"""
|
|
| 302 |
}, true); // capture phase — fires before page scripts can intercept/stop the event
|
| 303 |
})();
|
| 304 |
"""
|
|
|
|
| 305 |
|
| 306 |
# JS: tracking overlay (gaze dot + AOI highlighting) — injected only once
|
| 307 |
# tracking actually starts. Ported directly from run_session.py's JS_SETUP.
|
| 308 |
TRACKING_JS = r"""
|
| 309 |
(function(){
|
| 310 |
if (window.__insightux) { return; }
|
|
|
|
|
|
|
| 311 |
const state = { aois: [] };
|
| 312 |
const dwell = { pendLabel: null, pendSince: 0, activeLabel: null, emptySince: 0 };
|
| 313 |
const DWELL_MS = 420;
|
|
@@ -382,10 +545,10 @@ TRACKING_JS = r"""
|
|
| 382 |
|
| 383 |
function consider(el){
|
| 384 |
if (full) return;
|
| 385 |
-
// Never track InsightUX's own injected UI (
|
| 386 |
-
//
|
| 387 |
-
if (
|
| 388 |
-
if (el.closest && el.closest(
|
| 389 |
const tag = el.tagName.toLowerCase();
|
| 390 |
const explicit = !!(el.dataset && el.dataset.aoi);
|
| 391 |
if (!explicit){
|
|
@@ -662,12 +825,14 @@ MOUSE_JS = r"""
|
|
| 662 |
}, 2000);
|
| 663 |
|
| 664 |
// ---- exclude InsightUX's own injected UI from every measurement ----
|
| 665 |
-
// (the
|
| 666 |
// real website elements should ever show up in trail/heatmap/dwell/clicks.
|
|
|
|
|
|
|
| 667 |
function isOwnUI(el){
|
| 668 |
if (!el) return false;
|
| 669 |
-
if (
|
| 670 |
-
return !!(el.closest && el.closest(
|
| 671 |
}
|
| 672 |
|
| 673 |
// ---- mouse position + hover tracking ----
|
|
@@ -992,11 +1157,22 @@ MOUSE_JS = r"""
|
|
| 992 |
# =============================================================================
|
| 993 |
|
| 994 |
# Pages tracking must never start on, checked against window.get_current_url()
|
| 995 |
-
# as a Python-side backstop to
|
| 996 |
# suspenders — the JS guard can't run before the JS bridge is up).
|
| 997 |
_NON_TRACKABLE_URL_MARKERS = ("insightux_landing.html", "analysis_report.html")
|
| 998 |
|
| 999 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1000 |
class Api:
|
| 1001 |
def __init__(self):
|
| 1002 |
self.window = None
|
|
@@ -1005,6 +1181,33 @@ class Api:
|
|
| 1005 |
self.thread = None
|
| 1006 |
self.mouse_log_f = None
|
| 1007 |
self.mouse_close_timer = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1008 |
|
| 1009 |
def start_tracking(self):
|
| 1010 |
print("[browser_session] start_tracking() called from JS")
|
|
@@ -1019,6 +1222,10 @@ class Api:
|
|
| 1019 |
print(f"[browser_session] refusing to start on non-website page: {current_url}")
|
| 1020 |
self._set_status("Start tracking only works on a website — search or open a page first.")
|
| 1021 |
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1022 |
if not os.path.exists(CALIBRATION_PATH):
|
| 1023 |
print(f"[browser_session] no {CALIBRATION_PATH} found — run calibrate.py first")
|
| 1024 |
self._set_status("No calibration.pkl found — run calibrate.py first.")
|
|
@@ -1034,6 +1241,7 @@ class Api:
|
|
| 1034 |
self._inject_mouse_overlay()
|
| 1035 |
self._open_mouse_log(session_dir)
|
| 1036 |
self._set_mouse_tracking(True)
|
|
|
|
| 1037 |
self._set_status("Recording gaze + mouse... Press E to stop, H for heatmap, M for mouse panel")
|
| 1038 |
|
| 1039 |
self.thread = threading.Thread(
|
|
@@ -1050,6 +1258,7 @@ class Api:
|
|
| 1050 |
return False
|
| 1051 |
self._set_status("Wrapping up your session...")
|
| 1052 |
self._set_mouse_tracking(False)
|
|
|
|
| 1053 |
self.stop_event.set()
|
| 1054 |
|
| 1055 |
# Give the page a moment to flush its last batch of mouse data over
|
|
@@ -1067,6 +1276,77 @@ class Api:
|
|
| 1067 |
print("[browser_session] quit_app() called from JS -- terminating process")
|
| 1068 |
os._exit(0)
|
| 1069 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1070 |
def _inject_tracking_overlay(self):
|
| 1071 |
try:
|
| 1072 |
self.window.evaluate_js(TRACKING_JS)
|
|
@@ -1134,11 +1414,51 @@ class Api:
|
|
| 1134 |
api = Api()
|
| 1135 |
|
| 1136 |
|
| 1137 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1138 |
try:
|
| 1139 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1140 |
except Exception as e:
|
| 1141 |
-
print(f"[browser_session]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1142 |
|
| 1143 |
|
| 1144 |
# =============================================================================
|
|
@@ -1343,20 +1663,36 @@ def _go_fullscreen(window):
|
|
| 1343 |
|
| 1344 |
|
| 1345 |
if __name__ == "__main__":
|
| 1346 |
-
|
| 1347 |
-
|
| 1348 |
-
|
| 1349 |
-
|
| 1350 |
-
|
| 1351 |
-
)
|
| 1352 |
-
|
| 1353 |
-
|
| 1354 |
-
|
| 1355 |
-
|
| 1356 |
-
|
| 1357 |
-
|
| 1358 |
-
|
| 1359 |
-
|
| 1360 |
-
|
| 1361 |
-
|
| 1362 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
import time
|
| 25 |
import math
|
| 26 |
import threading
|
| 27 |
+
import subprocess
|
| 28 |
from dataclasses import replace
|
| 29 |
from datetime import datetime
|
| 30 |
|
| 31 |
+
# Windows consoles default to a non-UTF-8 codepage (cp1252) — a stray
|
| 32 |
+
# unicode character in any print() (ours, or one of calibrate.py/validate.py's
|
| 33 |
+
# once relaunched as a subprocess sharing this process's stdout handle) would
|
| 34 |
+
# otherwise crash the whole process with UnicodeEncodeError. Confirmed this
|
| 35 |
+
# happened for real during a calibration run mid-session.
|
| 36 |
+
if hasattr(sys.stdout, "reconfigure"):
|
| 37 |
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
| 38 |
+
if hasattr(sys.stderr, "reconfigure"):
|
| 39 |
+
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
| 40 |
+
|
| 41 |
import cv2
|
| 42 |
import numpy as np
|
| 43 |
import webview
|
|
|
|
| 67 |
# CONFIG — kept identical to run_session.py so calibration.pkl stays valid
|
| 68 |
# =============================================================================
|
| 69 |
|
| 70 |
+
# Bundled read-only assets (models/checkpoints) vs. per-user writable data
|
| 71 |
+
# (calibration.pkl, sessions/, the generated landing page) need different
|
| 72 |
+
# roots once frozen by PyInstaller — onedir nests bundled data under
|
| 73 |
+
# _internal/, alongside a persistent folder holding InsightUX.exe itself.
|
| 74 |
+
if getattr(sys, "frozen", False):
|
| 75 |
+
RESOURCE_DIR = sys._MEIPASS
|
| 76 |
+
DATA_DIR = os.path.dirname(sys.executable)
|
| 77 |
+
else:
|
| 78 |
+
RESOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 79 |
+
DATA_DIR = RESOURCE_DIR
|
| 80 |
+
BASE_DIR = DATA_DIR # kept as an alias: subprocess cwd for calibrate/validate relaunches
|
| 81 |
+
|
| 82 |
+
# Bump alongside packaging/installer.iss's AppVersion on every release, and
|
| 83 |
+
# update version.json in the same commit — see packaging/README.md.
|
| 84 |
+
VERSION = "1.0.0"
|
| 85 |
+
VERSION_CHECK_URL = "https://huggingface.co/arpitasethiii/insightux/resolve/main/version.json"
|
| 86 |
+
|
| 87 |
+
ONNX_PATH = os.path.join(RESOURCE_DIR, "models", "gaze_cnn_v4.onnx")
|
| 88 |
+
CALIBRATION_PATH = os.path.join(DATA_DIR, "calibration.pkl")
|
| 89 |
SCREEN_W, SCREEN_H = pyautogui.size()
|
| 90 |
|
| 91 |
PATCH_SOURCE = "blended"
|
|
|
|
| 106 |
HEAD_PITCH_COMPENSATION = 0.0 # reverted — 0.35 made accuracy worse, not better
|
| 107 |
HEAD_YAW_COMPENSATION = 0.0
|
| 108 |
|
| 109 |
+
SESSIONS_ROOT = os.path.join(DATA_DIR, "sessions")
|
| 110 |
|
| 111 |
# The window opens on this local, fully InsightUX-branded page instead of
|
| 112 |
# raw google.com. What you type here still hits REAL Google — no fake
|
|
|
|
| 182 |
|
| 183 |
|
| 184 |
def _write_landing_page():
|
| 185 |
+
path = os.path.join(DATA_DIR, "insightux_landing.html")
|
| 186 |
with open(path, "w", encoding="utf-8") as f:
|
| 187 |
f.write(_LANDING_HTML)
|
| 188 |
return "file://" + path.replace(os.sep, "/")
|
|
|
|
| 247 |
|
| 248 |
|
| 249 |
# =============================================================================
|
| 250 |
+
# JS: browser chrome (always present) — top toolbar (back/forward/reload/
|
| 251 |
+
# home/address bar) + left sidebar (Calibrate/Validate/Start Session), so the
|
| 252 |
+
# window reads as a normal browser instead of a bare content pane. Purely an
|
| 253 |
+
# HTML/CSS/JS overlay drawn on top of whatever page is loaded — pywebview
|
| 254 |
+
# 4.4.1 has no native toolbar API, so this is the only way to add persistent
|
| 255 |
+
# chrome without switching GUI frameworks.
|
| 256 |
+
#
|
| 257 |
+
# The page itself is pushed down/right by exactly the toolbar height and
|
| 258 |
+
# sidebar width (via forced <body> padding, not just floating the chrome on
|
| 259 |
+
# top of it) so real page content is never hidden underneath — the chrome
|
| 260 |
+
# and the website occupy clearly separate regions instead of overlapping.
|
| 261 |
+
# overflow-x is force-hidden as a safety net against the small width
|
| 262 |
+
# reduction ever introducing a horizontal scrollbar. The one edge case this
|
| 263 |
+
# can't fully solve: a small minority of sites use position:fixed elements
|
| 264 |
+
# of their own pinned to the true viewport edges (independent of body
|
| 265 |
+
# padding) — those can still end up visually behind our chrome.
|
| 266 |
# =============================================================================
|
| 267 |
|
| 268 |
+
CHROME_JS = r"""
|
| 269 |
(function(){
|
| 270 |
if (window.__insightuxControl) { return; }
|
| 271 |
window.__insightuxControl = true;
|
| 272 |
|
| 273 |
// S/E (start/stop tracking) only make sense on an actual website — not on
|
| 274 |
// the InsightUX landing/search page and not on a generated insights
|
| 275 |
+
// report page. The toolbar/sidebar, and X (quit), always work everywhere.
|
| 276 |
const isLanding = !!document.querySelector('meta[name="insightux-landing"]');
|
| 277 |
const isReport = !!document.querySelector('meta[name="insightux-report"]');
|
| 278 |
const isTrackable = !isLanding && !isReport;
|
| 279 |
|
| 280 |
+
const style = document.createElement('style');
|
| 281 |
+
style.textContent = `
|
| 282 |
+
html {
|
| 283 |
+
overflow-x: hidden !important;
|
| 284 |
+
}
|
| 285 |
+
body {
|
| 286 |
+
box-sizing: border-box !important;
|
| 287 |
+
width: 100vw !important;
|
| 288 |
+
margin: 0 !important;
|
| 289 |
+
padding-top: 46px !important;
|
| 290 |
+
padding-left: 84px !important;
|
| 291 |
+
overflow-x: hidden !important;
|
| 292 |
+
}
|
| 293 |
+
#__insightux_toolbar {
|
| 294 |
+
position:fixed; top:0; left:0; right:0; height:46px; z-index:2147483647;
|
| 295 |
+
display:flex; align-items:center; gap:10px; padding:0 12px; box-sizing:border-box;
|
| 296 |
+
background:linear-gradient(90deg, rgba(20,18,26,0.97), rgba(30,20,40,0.97));
|
| 297 |
border-bottom:2px solid; border-image:linear-gradient(90deg,#7B2FBE,#FF2DF0) 1;
|
| 298 |
+
box-shadow:0 3px 14px rgba(0,0,0,0.35);
|
| 299 |
+
font:12px -apple-system,'Segoe UI',Arial; color:#f0ecf7;
|
| 300 |
+
}
|
| 301 |
+
#__insightux_toolbar button {
|
| 302 |
+
flex:0 0 auto; width:30px; height:30px; border-radius:8px; cursor:pointer;
|
| 303 |
+
background:#241f2e; border:1px solid #3a3348; color:#f0ecf7; font-size:15px;
|
| 304 |
+
display:flex; align-items:center; justify-content:center;
|
| 305 |
+
}
|
| 306 |
+
#__insightux_toolbar button:hover { background:#302a3d; }
|
| 307 |
+
#__iux_addr_form { flex:1 1 auto; min-width:0; }
|
| 308 |
+
#__iux_addr {
|
| 309 |
+
width:100%; box-sizing:border-box; padding:7px 14px; border-radius:16px;
|
| 310 |
+
border:1.5px solid #3a3348; background:#1c1826; color:#f0ecf7; outline:none;
|
| 311 |
+
font-size:12.5px;
|
| 312 |
+
}
|
| 313 |
+
#__iux_addr:focus { border-color:#9B59FF; }
|
| 314 |
+
#__insightux_status {
|
| 315 |
+
flex:0 0 auto; color:#c9a6f5; white-space:nowrap; max-width:30vw;
|
| 316 |
+
overflow:hidden; text-overflow:ellipsis;
|
| 317 |
+
}
|
| 318 |
+
#__insightux_sidebar {
|
| 319 |
+
position:fixed; top:46px; left:0; bottom:0; width:84px; z-index:2147483647;
|
| 320 |
+
display:flex; flex-direction:column; align-items:stretch; gap:8px; padding:10px 8px;
|
| 321 |
+
box-sizing:border-box; font:11px -apple-system,'Segoe UI',Arial;
|
| 322 |
+
background:linear-gradient(180deg, rgba(24,20,32,0.97), rgba(16,14,22,0.97));
|
| 323 |
+
border-right:2px solid; border-image:linear-gradient(180deg,#7B2FBE,#FF2DF0) 1;
|
| 324 |
+
box-shadow:3px 0 14px rgba(0,0,0,0.35);
|
| 325 |
+
}
|
| 326 |
+
#__insightux_sidebar button {
|
| 327 |
+
background:#241f2e; border:1px solid #3a3348; color:#f0ecf7; border-radius:10px;
|
| 328 |
+
padding:10px 4px; cursor:pointer; line-height:1.5; font-size:11px; text-align:center;
|
| 329 |
+
}
|
| 330 |
+
#__insightux_sidebar button:hover { background:#302a3d; }
|
| 331 |
+
#__iux_session.active { background:linear-gradient(135deg,#7B2FBE,#FF2DF0); border-color:transparent; }
|
| 332 |
+
#__insightux_sidebar .__iux_hints {
|
| 333 |
+
margin-top:auto; color:#7a7288; font-size:9.5px; line-height:1.6; text-align:center;
|
| 334 |
+
}
|
| 335 |
+
#__insightux_update {
|
| 336 |
+
display:none; cursor:pointer; flex:0 0 auto; white-space:nowrap;
|
| 337 |
+
padding:5px 12px; border-radius:12px; font-weight:600; font-size:11px;
|
| 338 |
+
background:linear-gradient(135deg,#7B2FBE,#FF2DF0); color:#fff;
|
| 339 |
+
}
|
| 340 |
+
#__insightux_update:hover { filter:brightness(1.12); }
|
| 341 |
+
`;
|
| 342 |
+
document.head.appendChild(style);
|
| 343 |
+
|
| 344 |
+
const toolbar = document.createElement('div');
|
| 345 |
+
toolbar.id = '__insightux_toolbar';
|
| 346 |
+
toolbar.innerHTML = `
|
| 347 |
+
<button type="button" id="__iux_back" title="Back">←</button>
|
| 348 |
+
<button type="button" id="__iux_fwd" title="Forward">→</button>
|
| 349 |
+
<button type="button" id="__iux_reload" title="Reload">↻</button>
|
| 350 |
+
<button type="button" id="__iux_home" title="Home">⌂</button>
|
| 351 |
+
<form id="__iux_addr_form"><input id="__iux_addr" type="text" autocomplete="off"
|
| 352 |
+
placeholder="Search or enter address"></form>
|
| 353 |
+
<span id="__insightux_update" title="Click to open the download page"></span>
|
| 354 |
+
<span id="__insightux_status">Press S to start tracking · E to stop · H heatmap · M mouse panel · X quit</span>
|
| 355 |
+
`;
|
| 356 |
+
document.documentElement.appendChild(toolbar);
|
| 357 |
+
|
| 358 |
+
const sidebar = document.createElement('div');
|
| 359 |
+
sidebar.id = '__insightux_sidebar';
|
| 360 |
+
sidebar.innerHTML = `
|
| 361 |
+
<button type="button" id="__iux_calibrate">👁<br>Calibrate</button>
|
| 362 |
+
<button type="button" id="__iux_validate">🎯<br>Validate</button>
|
| 363 |
+
<button type="button" id="__iux_session">▶<br>Start<br>Session</button>
|
| 364 |
+
<div class="__iux_hints">S/E start/stop<br>H heatmap<br>M panel<br>X quit</div>
|
| 365 |
+
`;
|
| 366 |
+
document.documentElement.appendChild(sidebar);
|
| 367 |
+
|
| 368 |
+
setInterval(function(){
|
| 369 |
+
if (!document.documentElement.contains(toolbar)) document.documentElement.appendChild(toolbar);
|
| 370 |
+
if (!document.documentElement.contains(sidebar)) document.documentElement.appendChild(sidebar);
|
| 371 |
+
}, 1000);
|
| 372 |
|
| 373 |
const statusEl = () => document.getElementById('__insightux_status');
|
| 374 |
window.insightuxSetStatus = function(text){
|
| 375 |
const el = statusEl();
|
| 376 |
if (el) el.textContent = text;
|
| 377 |
};
|
| 378 |
+
window.insightuxSetSessionUI = function(isTracking){
|
| 379 |
+
const btn = document.getElementById('__iux_session');
|
| 380 |
+
if (!btn) return;
|
| 381 |
+
btn.innerHTML = isTracking ? '■<br>Stop<br>Session' : '▶<br>Start<br>Session';
|
| 382 |
+
btn.classList.toggle('active', isTracking);
|
| 383 |
+
};
|
| 384 |
+
window.insightuxShowUpdate = function(version, url){
|
| 385 |
+
const el = document.getElementById('__insightux_update');
|
| 386 |
+
if (!el) return;
|
| 387 |
+
el.textContent = 'Update available (v' + version + ')';
|
| 388 |
+
el.style.display = 'inline-block';
|
| 389 |
+
el.onclick = function(){
|
| 390 |
+
if (window.pywebview && window.pywebview.api && window.pywebview.api.open_update_page) {
|
| 391 |
+
window.pywebview.api.open_update_page(url);
|
| 392 |
+
}
|
| 393 |
+
};
|
| 394 |
+
};
|
| 395 |
+
|
| 396 |
+
// -- address bar / navigation (pure DOM history/location, no Python round-trip) --
|
| 397 |
+
const addrInput = document.getElementById('__iux_addr');
|
| 398 |
+
addrInput.value = isLanding ? '' : location.href;
|
| 399 |
+
document.getElementById('__iux_addr_form').addEventListener('submit', function(e){
|
| 400 |
+
e.preventDefault();
|
| 401 |
+
const raw = addrInput.value.trim();
|
| 402 |
+
if (!raw) return;
|
| 403 |
+
const looksLikeUrl = /^https?:\/\//i.test(raw) ||
|
| 404 |
+
(/^[\w-]+(\.[\w-]+)+([/?#].*)?$/i.test(raw) && !raw.includes(' '));
|
| 405 |
+
if (looksLikeUrl) {
|
| 406 |
+
location.href = raw.startsWith('http') ? raw : ('https://' + raw);
|
| 407 |
+
} else {
|
| 408 |
+
location.href = 'https://www.google.com/search?q=' + encodeURIComponent(raw);
|
| 409 |
+
}
|
| 410 |
+
});
|
| 411 |
+
document.getElementById('__iux_back').addEventListener('click', function(){ history.back(); });
|
| 412 |
+
document.getElementById('__iux_fwd').addEventListener('click', function(){ history.forward(); });
|
| 413 |
+
document.getElementById('__iux_reload').addEventListener('click', function(){ location.reload(); });
|
| 414 |
+
document.getElementById('__iux_home').addEventListener('click', function(){
|
| 415 |
+
location.href = __INSIGHTUX_LANDING_URL__;
|
| 416 |
+
});
|
| 417 |
+
|
| 418 |
+
// -- sidebar actions (Python decides validity — camera/session conflicts,
|
| 419 |
+
// wrong page — and reports back through the status text) --
|
| 420 |
+
function callApi(method){
|
| 421 |
+
if (window.pywebview && window.pywebview.api && window.pywebview.api[method]) {
|
| 422 |
+
window.pywebview.api[method]();
|
| 423 |
+
}
|
| 424 |
+
}
|
| 425 |
+
document.getElementById('__iux_calibrate').addEventListener('click', function(){ callApi('start_calibration'); });
|
| 426 |
+
document.getElementById('__iux_validate').addEventListener('click', function(){ callApi('run_validation'); });
|
| 427 |
+
document.getElementById('__iux_session').addEventListener('click', function(){ callApi('toggle_session'); });
|
| 428 |
|
| 429 |
function isTypingTarget(el){
|
| 430 |
if (!el) return false;
|
|
|
|
| 462 |
}, true); // capture phase — fires before page scripts can intercept/stop the event
|
| 463 |
})();
|
| 464 |
"""
|
| 465 |
+
CHROME_JS = CHROME_JS.replace("__INSIGHTUX_LANDING_URL__", json.dumps(LANDING_URL))
|
| 466 |
|
| 467 |
# JS: tracking overlay (gaze dot + AOI highlighting) — injected only once
|
| 468 |
# tracking actually starts. Ported directly from run_session.py's JS_SETUP.
|
| 469 |
TRACKING_JS = r"""
|
| 470 |
(function(){
|
| 471 |
if (window.__insightux) { return; }
|
| 472 |
+
const OWN_UI_IDS = new Set(['__insightux_toolbar', '__insightux_sidebar', '__insightux_pill', '__insightux_canvas', '__insightux_mouse_panel']);
|
| 473 |
+
const OWN_UI_SELECTOR = '#__insightux_toolbar, #__insightux_sidebar, #__insightux_pill, #__insightux_canvas, #__insightux_mouse_panel';
|
| 474 |
const state = { aois: [] };
|
| 475 |
const dwell = { pendLabel: null, pendSince: 0, activeLabel: null, emptySince: 0 };
|
| 476 |
const DWELL_MS = 420;
|
|
|
|
| 545 |
|
| 546 |
function consider(el){
|
| 547 |
if (full) return;
|
| 548 |
+
// Never track InsightUX's own injected UI (toolbar, sidebar, the
|
| 549 |
+
// gaze-dot canvas, the mouse tracker panel) as if it were page content.
|
| 550 |
+
if (OWN_UI_IDS.has(el.id)) return;
|
| 551 |
+
if (el.closest && el.closest(OWN_UI_SELECTOR)) return;
|
| 552 |
const tag = el.tagName.toLowerCase();
|
| 553 |
const explicit = !!(el.dataset && el.dataset.aoi);
|
| 554 |
if (!explicit){
|
|
|
|
| 825 |
}, 2000);
|
| 826 |
|
| 827 |
// ---- exclude InsightUX's own injected UI from every measurement ----
|
| 828 |
+
// (the toolbar, sidebar, mouse-tracker panel, heatmap canvas) — only
|
| 829 |
// real website elements should ever show up in trail/heatmap/dwell/clicks.
|
| 830 |
+
const OWN_UI_IDS = new Set(['__insightux_toolbar', '__insightux_sidebar', '__insightux_pill', '__insightux_canvas', '__insightux_mouse_panel']);
|
| 831 |
+
const OWN_UI_SELECTOR = '#__insightux_toolbar, #__insightux_sidebar, #__insightux_pill, #__insightux_canvas, #__insightux_mouse_panel';
|
| 832 |
function isOwnUI(el){
|
| 833 |
if (!el) return false;
|
| 834 |
+
if (OWN_UI_IDS.has(el.id)) return true;
|
| 835 |
+
return !!(el.closest && el.closest(OWN_UI_SELECTOR));
|
| 836 |
}
|
| 837 |
|
| 838 |
// ---- mouse position + hover tracking ----
|
|
|
|
| 1157 |
# =============================================================================
|
| 1158 |
|
| 1159 |
# Pages tracking must never start on, checked against window.get_current_url()
|
| 1160 |
+
# as a Python-side backstop to CHROME_JS's isTrackable check (belt and
|
| 1161 |
# suspenders — the JS guard can't run before the JS bridge is up).
|
| 1162 |
_NON_TRACKABLE_URL_MARKERS = ("insightux_landing.html", "analysis_report.html")
|
| 1163 |
|
| 1164 |
|
| 1165 |
+
def _relaunch_args(mode):
|
| 1166 |
+
"""Command to spawn calibrate.py/validate.py's logic in a fresh process.
|
| 1167 |
+
Frozen (PyInstaller) builds have no bundled interpreter to hand a .py
|
| 1168 |
+
file to, and sys.executable IS the frozen exe itself — so a frozen build
|
| 1169 |
+
relaunches itself with --mode instead. Unpackaged/dev mode still runs
|
| 1170 |
+
from the venv exactly as before."""
|
| 1171 |
+
if getattr(sys, "frozen", False):
|
| 1172 |
+
return [sys.executable, "--mode", mode]
|
| 1173 |
+
return [sys.executable, os.path.abspath(__file__), "--mode", mode]
|
| 1174 |
+
|
| 1175 |
+
|
| 1176 |
class Api:
|
| 1177 |
def __init__(self):
|
| 1178 |
self.window = None
|
|
|
|
| 1181 |
self.thread = None
|
| 1182 |
self.mouse_log_f = None
|
| 1183 |
self.mouse_close_timer = None
|
| 1184 |
+
self.calib_proc = None
|
| 1185 |
+
self.validate_proc = None
|
| 1186 |
+
self.update_info = None
|
| 1187 |
+
|
| 1188 |
+
def _notify_update(self, version, url):
|
| 1189 |
+
try:
|
| 1190 |
+
self.window.evaluate_js(
|
| 1191 |
+
f"window.insightuxShowUpdate && window.insightuxShowUpdate("
|
| 1192 |
+
f"{json.dumps(version)}, {json.dumps(url)})"
|
| 1193 |
+
)
|
| 1194 |
+
except Exception:
|
| 1195 |
+
pass
|
| 1196 |
+
|
| 1197 |
+
def open_update_page(self, url):
|
| 1198 |
+
print(f"[browser_session] open_update_page() called from JS: {url}")
|
| 1199 |
+
import webbrowser
|
| 1200 |
+
try:
|
| 1201 |
+
webbrowser.open(url)
|
| 1202 |
+
except Exception as e:
|
| 1203 |
+
print(f"[browser_session] failed to open update url: {e}")
|
| 1204 |
+
return True
|
| 1205 |
+
|
| 1206 |
+
def toggle_session(self):
|
| 1207 |
+
"""Single entry point for the sidebar's Start/Stop Session button —
|
| 1208 |
+
lets the JS button stay dumb (always call the same thing) while
|
| 1209 |
+
Python decides which action makes sense from current state."""
|
| 1210 |
+
return self.stop_tracking() if self.tracking else self.start_tracking()
|
| 1211 |
|
| 1212 |
def start_tracking(self):
|
| 1213 |
print("[browser_session] start_tracking() called from JS")
|
|
|
|
| 1222 |
print(f"[browser_session] refusing to start on non-website page: {current_url}")
|
| 1223 |
self._set_status("Start tracking only works on a website — search or open a page first.")
|
| 1224 |
return False
|
| 1225 |
+
if self._external_proc_running():
|
| 1226 |
+
print("[browser_session] refusing to start, calibration/validation still running")
|
| 1227 |
+
self._set_status("Wait for calibration/validation to finish before starting a session.")
|
| 1228 |
+
return False
|
| 1229 |
if not os.path.exists(CALIBRATION_PATH):
|
| 1230 |
print(f"[browser_session] no {CALIBRATION_PATH} found — run calibrate.py first")
|
| 1231 |
self._set_status("No calibration.pkl found — run calibrate.py first.")
|
|
|
|
| 1241 |
self._inject_mouse_overlay()
|
| 1242 |
self._open_mouse_log(session_dir)
|
| 1243 |
self._set_mouse_tracking(True)
|
| 1244 |
+
self._set_session_ui(True)
|
| 1245 |
self._set_status("Recording gaze + mouse... Press E to stop, H for heatmap, M for mouse panel")
|
| 1246 |
|
| 1247 |
self.thread = threading.Thread(
|
|
|
|
| 1258 |
return False
|
| 1259 |
self._set_status("Wrapping up your session...")
|
| 1260 |
self._set_mouse_tracking(False)
|
| 1261 |
+
self._set_session_ui(False)
|
| 1262 |
self.stop_event.set()
|
| 1263 |
|
| 1264 |
# Give the page a moment to flush its last batch of mouse data over
|
|
|
|
| 1276 |
print("[browser_session] quit_app() called from JS -- terminating process")
|
| 1277 |
os._exit(0)
|
| 1278 |
|
| 1279 |
+
# -- calibration / validation ------------------------------------------
|
| 1280 |
+
# calibrate.py and validate.py are unmodified, standalone OpenCV/pyautogui
|
| 1281 |
+
# scripts (their own fullscreen window, their own event loop) — they were
|
| 1282 |
+
# never meant to share a process with pywebview's GUI loop. Launching them
|
| 1283 |
+
# as a subprocess reuses them exactly as-is instead of rewriting their
|
| 1284 |
+
# display logic into the browser.
|
| 1285 |
+
|
| 1286 |
+
def _external_proc_running(self):
|
| 1287 |
+
running = lambda p: p is not None and p.poll() is None
|
| 1288 |
+
return running(self.calib_proc) or running(self.validate_proc)
|
| 1289 |
+
|
| 1290 |
+
def start_calibration(self):
|
| 1291 |
+
print("[browser_session] start_calibration() called from JS")
|
| 1292 |
+
if self.tracking:
|
| 1293 |
+
self._set_status("Stop the current session (E) before calibrating.")
|
| 1294 |
+
return False
|
| 1295 |
+
if self._external_proc_running():
|
| 1296 |
+
self._set_status("Calibration/validation is already running in another window.")
|
| 1297 |
+
return False
|
| 1298 |
+
try:
|
| 1299 |
+
self.calib_proc = subprocess.Popen(_relaunch_args("calibrate"), cwd=BASE_DIR)
|
| 1300 |
+
except Exception as e:
|
| 1301 |
+
print(f"[browser_session] failed to launch calibrate.py: {e}")
|
| 1302 |
+
self._set_status("Could not launch calibration — see terminal for details.")
|
| 1303 |
+
return False
|
| 1304 |
+
self._set_status("Calibration launched in a separate window — follow the dots there.")
|
| 1305 |
+
threading.Thread(
|
| 1306 |
+
target=self._watch_external_proc, args=(self.calib_proc, "Calibration"), daemon=True
|
| 1307 |
+
).start()
|
| 1308 |
+
return True
|
| 1309 |
+
|
| 1310 |
+
def run_validation(self):
|
| 1311 |
+
print("[browser_session] run_validation() called from JS")
|
| 1312 |
+
if self.tracking:
|
| 1313 |
+
self._set_status("Stop the current session (E) before validating.")
|
| 1314 |
+
return False
|
| 1315 |
+
if self._external_proc_running():
|
| 1316 |
+
self._set_status("Calibration/validation is already running in another window.")
|
| 1317 |
+
return False
|
| 1318 |
+
if not os.path.exists(CALIBRATION_PATH):
|
| 1319 |
+
self._set_status("No calibration.pkl found — run Calibrate first.")
|
| 1320 |
+
return False
|
| 1321 |
+
try:
|
| 1322 |
+
self.validate_proc = subprocess.Popen(_relaunch_args("validate"), cwd=BASE_DIR)
|
| 1323 |
+
except Exception as e:
|
| 1324 |
+
print(f"[browser_session] failed to launch validate.py: {e}")
|
| 1325 |
+
self._set_status("Could not launch validation — see terminal for details.")
|
| 1326 |
+
return False
|
| 1327 |
+
self._set_status("Validation launched in a separate window — look at each dot.")
|
| 1328 |
+
threading.Thread(
|
| 1329 |
+
target=self._watch_external_proc, args=(self.validate_proc, "Validation"), daemon=True
|
| 1330 |
+
).start()
|
| 1331 |
+
return True
|
| 1332 |
+
|
| 1333 |
+
def _watch_external_proc(self, proc, label):
|
| 1334 |
+
proc.wait()
|
| 1335 |
+
print(f"[browser_session] {label} process exited (code {proc.returncode})")
|
| 1336 |
+
if label == "Calibration":
|
| 1337 |
+
self._set_status("Calibration finished — check the terminal for the quality readout, then press S or Start Session.")
|
| 1338 |
+
else:
|
| 1339 |
+
self._set_status("Validation finished — check the terminal for the accuracy numbers.")
|
| 1340 |
+
|
| 1341 |
+
def _set_session_ui(self, is_tracking):
|
| 1342 |
+
try:
|
| 1343 |
+
flag = "true" if is_tracking else "false"
|
| 1344 |
+
self.window.evaluate_js(
|
| 1345 |
+
f"window.insightuxSetSessionUI && window.insightuxSetSessionUI({flag})"
|
| 1346 |
+
)
|
| 1347 |
+
except Exception:
|
| 1348 |
+
pass
|
| 1349 |
+
|
| 1350 |
def _inject_tracking_overlay(self):
|
| 1351 |
try:
|
| 1352 |
self.window.evaluate_js(TRACKING_JS)
|
|
|
|
| 1414 |
api = Api()
|
| 1415 |
|
| 1416 |
|
| 1417 |
+
def _version_tuple(v):
|
| 1418 |
+
return tuple(int(p) for p in v.split(".") if p.isdigit())
|
| 1419 |
+
|
| 1420 |
+
|
| 1421 |
+
def check_for_update():
|
| 1422 |
+
"""Runs once in a background thread at startup. Publishing a new release
|
| 1423 |
+
is still fully manual (bump VERSION here + AppVersion in installer.iss +
|
| 1424 |
+
version.json, rebuild, push) — this only automates the *checking* side,
|
| 1425 |
+
per the confirmed scope. No internet / any failure here is silent:
|
| 1426 |
+
the user just doesn't see an update banner, never an error."""
|
| 1427 |
+
import urllib.request
|
| 1428 |
try:
|
| 1429 |
+
with urllib.request.urlopen(VERSION_CHECK_URL, timeout=4) as resp:
|
| 1430 |
+
data = json.loads(resp.read().decode("utf-8"))
|
| 1431 |
+
latest = data.get("latest", "")
|
| 1432 |
+
url = data.get("url", "")
|
| 1433 |
+
if latest and url and _version_tuple(latest) > _version_tuple(VERSION):
|
| 1434 |
+
api.update_info = {"version": latest, "url": url}
|
| 1435 |
+
api._notify_update(latest, url)
|
| 1436 |
+
print(f"[browser_session] update available: {VERSION} -> {latest}")
|
| 1437 |
except Exception as e:
|
| 1438 |
+
print(f"[browser_session] update check skipped: {e}")
|
| 1439 |
+
|
| 1440 |
+
|
| 1441 |
+
def on_page_loaded(window):
|
| 1442 |
+
"""Fires on every page load (navigation, back/forward, reload — not just
|
| 1443 |
+
the first page). The chrome must be re-injected every time since each
|
| 1444 |
+
navigation is a fresh document. If a session is already running, the
|
| 1445 |
+
gaze/mouse overlays need the same treatment or navigating mid-session
|
| 1446 |
+
would silently kill the dot and the mouse tracker on the new page."""
|
| 1447 |
+
try:
|
| 1448 |
+
window.evaluate_js(CHROME_JS)
|
| 1449 |
+
except Exception as e:
|
| 1450 |
+
print(f"[browser_session] chrome inject failed (will retry): {e}")
|
| 1451 |
+
|
| 1452 |
+
if api.tracking:
|
| 1453 |
+
api._inject_tracking_overlay()
|
| 1454 |
+
api._inject_mouse_overlay()
|
| 1455 |
+
api._set_mouse_tracking(True)
|
| 1456 |
+
api._set_session_ui(True)
|
| 1457 |
+
|
| 1458 |
+
# CHROME_JS was just re-injected fresh on this page and has no memory of
|
| 1459 |
+
# an update found on a previous page — re-tell it if one was found.
|
| 1460 |
+
if api.update_info:
|
| 1461 |
+
api._notify_update(api.update_info["version"], api.update_info["url"])
|
| 1462 |
|
| 1463 |
|
| 1464 |
# =============================================================================
|
|
|
|
| 1663 |
|
| 1664 |
|
| 1665 |
if __name__ == "__main__":
|
| 1666 |
+
# A packaged build is one exe with no separate calibrate.py/validate.py
|
| 1667 |
+
# files to shell out to — --mode makes it a single self-dispatching
|
| 1668 |
+
# entry point instead. See _relaunch_args() above for the launch side.
|
| 1669 |
+
import argparse
|
| 1670 |
+
parser = argparse.ArgumentParser()
|
| 1671 |
+
parser.add_argument("--mode", choices=["browser", "calibrate", "validate"], default="browser")
|
| 1672 |
+
args = parser.parse_args()
|
| 1673 |
+
|
| 1674 |
+
if args.mode == "calibrate":
|
| 1675 |
+
import calibrate
|
| 1676 |
+
calibrate.main()
|
| 1677 |
+
elif args.mode == "validate":
|
| 1678 |
+
import validate
|
| 1679 |
+
validate.main()
|
| 1680 |
+
else:
|
| 1681 |
+
os.makedirs(SESSIONS_ROOT, exist_ok=True)
|
| 1682 |
+
|
| 1683 |
+
window = webview.create_window(
|
| 1684 |
+
"InsightUX — Eye-Tracking Research Browser", LANDING_URL,
|
| 1685 |
+
width=SCREEN_W, height=SCREEN_H, js_api=api
|
| 1686 |
+
)
|
| 1687 |
+
api.window = window
|
| 1688 |
+
window.events.loaded += lambda: on_page_loaded(window)
|
| 1689 |
+
window.events.shown += lambda: threading.Thread(
|
| 1690 |
+
target=_go_fullscreen, args=(window,), daemon=True
|
| 1691 |
+
).start()
|
| 1692 |
+
threading.Thread(target=check_for_update, daemon=True).start()
|
| 1693 |
+
|
| 1694 |
+
# debug=True enables right-click > Inspect (DevTools) so you can see
|
| 1695 |
+
# the [insightux] console.log diagnostics from CHROME_JS directly —
|
| 1696 |
+
# keyboard handling runs entirely in the browser, so JS-side issues
|
| 1697 |
+
# never show up in this terminal, only in DevTools' Console tab.
|
| 1698 |
+
webview.start(debug=True)
|
|
@@ -4,10 +4,32 @@ import time
|
|
| 4 |
import os
|
| 5 |
import random
|
| 6 |
import pyautogui
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
import warnings
|
| 9 |
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
| 10 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
from preprocessing.preprocessing_pipeline import (
|
| 12 |
create_face_mesh,
|
| 13 |
estimate_camera_matrix,
|
|
@@ -420,8 +442,8 @@ def filter_unreliable_points(gaze_vectors, screen_points, head_pitches_deg,
|
|
| 420 |
# CONFIG
|
| 421 |
# =============================================================================
|
| 422 |
|
| 423 |
-
ONNX_PATH = "models
|
| 424 |
-
CKPT_PATH = "checkpoints
|
| 425 |
|
| 426 |
# --- Fine-tune target geometry -------------------------------------------
|
| 427 |
# USE_MEASURED_GEOMETRY=False keeps the original hand-picked constants
|
|
@@ -481,530 +503,535 @@ FRAMES_KEPT_PER_POINT = 12
|
|
| 481 |
FINETUNE_ENABLED = True
|
| 482 |
|
| 483 |
|
| 484 |
-
|
| 485 |
-
#
|
| 486 |
-
#
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
#
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
cv2.
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
cv2.waitKey(1)
|
| 537 |
-
|
| 538 |
-
samples = []
|
| 539 |
-
ear_list = []
|
| 540 |
-
l_list = []
|
| 541 |
-
r_list = []
|
| 542 |
-
p_list = []
|
| 543 |
-
hp_list = []
|
| 544 |
-
|
| 545 |
-
point_blink_skips = 0
|
| 546 |
-
point_light_skips = 0
|
| 547 |
-
|
| 548 |
-
start = time.time()
|
| 549 |
-
|
| 550 |
-
while time.time() - start < duration:
|
| 551 |
-
ret, frame = cap.read()
|
| 552 |
-
if not ret:
|
| 553 |
-
continue
|
| 554 |
-
|
| 555 |
-
if cam_matrix is None:
|
| 556 |
-
cam_matrix = estimate_camera_matrix(frame.shape)
|
| 557 |
-
|
| 558 |
-
# LIVE LIGHTING CHECK — skip this frame and tell the person, instead
|
| 559 |
-
# of silently feeding a too-dark/too-bright frame into the model.
|
| 560 |
-
light_ok, light_msg = check_lighting(frame)
|
| 561 |
-
gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
| 562 |
-
session_brightness.append(float(np.mean(gray_frame)))
|
| 563 |
-
if not light_ok:
|
| 564 |
-
point_light_skips += 1
|
| 565 |
-
screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
|
| 566 |
-
cv2.circle(screen, (sx, sy), 18, (0, 140, 255), 2)
|
| 567 |
-
cv2.putText(screen, f"Point {idx+1}/{total} - {light_msg}",
|
| 568 |
-
(50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 140, 255), 2)
|
| 569 |
cv2.imshow("Calibration", screen)
|
| 570 |
cv2.waitKey(1)
|
| 571 |
-
continue
|
| 572 |
|
| 573 |
-
|
| 574 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 575 |
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 592 |
screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
|
| 593 |
-
|
| 594 |
-
cv2.
|
| 595 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 596 |
cv2.imshow("Calibration", screen)
|
| 597 |
-
cv2.waitKey(1)
|
| 598 |
-
continue
|
| 599 |
-
|
| 600 |
-
# FIX A: normalize pose to [-1, +1] before CNN
|
| 601 |
-
pose_vec = normalize_pose(head_pose)
|
| 602 |
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
if not s1.is_open:
|
| 606 |
-
return None
|
| 607 |
-
ir = compute_iris_radius(lms, iris_idx, frame.shape)
|
| 608 |
-
s2 = step2_illumination(s1, ir)
|
| 609 |
-
return s2.blended if s2.is_usable else None
|
| 610 |
|
| 611 |
-
|
| 612 |
-
|
| 613 |
|
| 614 |
-
if
|
|
|
|
|
|
|
| 615 |
continue
|
| 616 |
-
if l is None: l = r
|
| 617 |
-
if r is None: r = l
|
| 618 |
-
|
| 619 |
-
_, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(l, pose_vec, r)
|
| 620 |
-
|
| 621 |
-
pitch = compensate_pitch(raw_pitch, head_pose.pitch)
|
| 622 |
-
yaw = compensate_yaw(raw_yaw, head_pose.yaw)
|
| 623 |
-
|
| 624 |
-
samples.append([pitch, yaw])
|
| 625 |
-
ear_list.append(avg_ear)
|
| 626 |
-
# solvePnP's tvec is in the same units as FACE_3D_MODEL (mm), so
|
| 627 |
-
# tvec[2] is a genuine measurement of how far your face is from the
|
| 628 |
-
# camera. We were computing this every frame and throwing it away.
|
| 629 |
-
try:
|
| 630 |
-
session_distance_mm.append(float(head_pose.tvec[2]))
|
| 631 |
-
except Exception:
|
| 632 |
-
pass
|
| 633 |
-
l_list.append(l)
|
| 634 |
-
r_list.append(r)
|
| 635 |
-
p_list.append(pose_vec)
|
| 636 |
-
hp_list.append(head_pose.pitch)
|
| 637 |
-
|
| 638 |
-
elapsed = time.time() - start
|
| 639 |
-
screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
|
| 640 |
-
angle = int(360 * elapsed / duration)
|
| 641 |
-
cv2.ellipse(screen, (sx, sy), (28, 28), -90, 0, angle, (0, 180, 0), 3)
|
| 642 |
-
cv2.circle(screen, (sx, sy), 18, (0, 255, 0), -1)
|
| 643 |
-
cv2.putText(screen, f"Point {idx+1}/{total} - Look at the dot",
|
| 644 |
-
(50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
|
| 645 |
-
if point_blink_skips or point_light_skips:
|
| 646 |
-
cv2.putText(screen, f"skipped: {point_blink_skips} blink, {point_light_skips} lighting",
|
| 647 |
-
(50, 85), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (140, 140, 140), 1)
|
| 648 |
-
cv2.imshow("Calibration", screen)
|
| 649 |
-
|
| 650 |
-
if cv2.waitKey(1) & 0xFF == 27:
|
| 651 |
-
break
|
| 652 |
-
|
| 653 |
-
session_blink_skips += point_blink_skips
|
| 654 |
-
session_light_skips += point_light_skips
|
| 655 |
-
|
| 656 |
-
if not samples:
|
| 657 |
-
print(f"Point {idx+1}: no samples, skipping.")
|
| 658 |
-
points_with_low_yield.append(idx + 1)
|
| 659 |
-
continue
|
| 660 |
-
|
| 661 |
-
if len(samples) < MIN_SAMPLES_OK:
|
| 662 |
-
print(f" ⚠ Point {idx+1}: only {len(samples)} valid samples "
|
| 663 |
-
f"({point_blink_skips} blink-skipped, {point_light_skips} light-skipped) — may be unreliable")
|
| 664 |
-
points_with_low_yield.append(idx + 1)
|
| 665 |
-
|
| 666 |
-
samples_arr = np.array(samples)
|
| 667 |
-
avg = np.median(samples_arr, axis=0)
|
| 668 |
-
avg_hp_deg = float(np.median(hp_list))
|
| 669 |
-
|
| 670 |
-
# WITHIN-POINT DISPERSION = the noise floor. While you stared at ONE
|
| 671 |
-
# fixed dot, how much did the model's predicted pitch/yaw wobble
|
| 672 |
-
# frame-to-frame? This is the single most important number in the whole
|
| 673 |
-
# calibration and it was never being measured. Compare it against the
|
| 674 |
-
# BETWEEN-point spread (the actual signal) at the end of the run: if
|
| 675 |
-
# noise >= signal on an axis, that axis is unrecoverable by any
|
| 676 |
-
# calibration math, and no amount of RBF tuning will fix it.
|
| 677 |
-
# MAD (median absolute deviation) rather than std, so one blink-tail
|
| 678 |
-
# frame can't inflate it.
|
| 679 |
-
mad_pitch = float(np.median(np.abs(samples_arr[:, 0] - avg[0])))
|
| 680 |
-
mad_yaw = float(np.median(np.abs(samples_arr[:, 1] - avg[1])))
|
| 681 |
-
|
| 682 |
-
gaze_vectors.append(list(avg))
|
| 683 |
-
screen_points.append([sx, sy])
|
| 684 |
-
head_pitches_deg.append(avg_hp_deg)
|
| 685 |
-
point_dispersion.append([mad_pitch, mad_yaw])
|
| 686 |
-
# Eye aperture for this point. Looking DOWN lowers the eyelid, so this is
|
| 687 |
-
# a physically independent vertical cue — and the CNN's pitch output has
|
| 688 |
-
# proven nearly blind vertically (r~0.41). calibrate() will measure both
|
| 689 |
-
# and pick whichever actually tracks screen-Y.
|
| 690 |
-
point_ear.append(float(np.median(ear_list)))
|
| 691 |
-
|
| 692 |
-
# Keep a BUNDLE of genuinely different frames for this point — not one
|
| 693 |
-
# "representative" frame. Both the fine-tune and the post-fine-tune RBF
|
| 694 |
-
# refit read from this bundle, so both get real frame diversity and real
|
| 695 |
-
# noise averaging instead of anchoring on a single arbitrary frame.
|
| 696 |
-
n_avail = len(l_list)
|
| 697 |
-
n_take = min(FRAMES_KEPT_PER_POINT, n_avail)
|
| 698 |
-
indices = np.linspace(0, n_avail - 1, n_take, dtype=int)
|
| 699 |
-
point_frames.append({
|
| 700 |
-
"lefts": [l_list[i] for i in indices],
|
| 701 |
-
"rights": [r_list[i] for i in indices],
|
| 702 |
-
"poses": [p_list[i] for i in indices],
|
| 703 |
-
"hps": [hp_list[i] for i in indices],
|
| 704 |
-
})
|
| 705 |
-
|
| 706 |
-
print(f"Point {idx+1:2d}/{total} | pitch={avg[0]:+.4f} yaw={avg[1]:+.4f} "
|
| 707 |
-
f"| jitter(pitch)={mad_pitch:.4f} jitter(yaw)={mad_yaw:.4f} "
|
| 708 |
-
f"| {len(indices)} frames kept "
|
| 709 |
-
f"| skipped: {point_blink_skips} blink, {point_light_skips} lighting")
|
| 710 |
-
|
| 711 |
-
cap.release()
|
| 712 |
-
cv2.destroyAllWindows()
|
| 713 |
-
|
| 714 |
-
if len(gaze_vectors) < 4:
|
| 715 |
-
print(f"ERROR: only {len(gaze_vectors)} points collected, need at least 4.")
|
| 716 |
-
exit(1)
|
| 717 |
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
(
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 726 |
|
| 727 |
-
n_pts = len(screen_points)
|
| 728 |
-
if n_pts < 4:
|
| 729 |
-
|
| 730 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 731 |
|
| 732 |
-
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 741 |
for i in range(n_pts):
|
| 742 |
pf = point_frames[i]
|
|
|
|
| 743 |
for k in range(len(pf["lefts"])):
|
| 744 |
-
|
| 745 |
-
|
| 746 |
-
|
| 747 |
-
|
| 748 |
-
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
|
| 754 |
-
|
| 755 |
-
|
| 756 |
-
|
| 757 |
-
|
| 758 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 759 |
else:
|
| 760 |
-
|
| 761 |
-
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
|
| 765 |
-
|
| 766 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 767 |
else:
|
| 768 |
-
|
| 769 |
-
|
| 770 |
-
|
| 771 |
-
|
| 772 |
-
|
| 773 |
-
|
| 774 |
-
|
| 775 |
-
|
| 776 |
-
|
| 777 |
-
|
| 778 |
-
|
| 779 |
-
|
| 780 |
-
|
| 781 |
-
|
| 782 |
-
|
| 783 |
-
|
| 784 |
-
|
| 785 |
-
|
| 786 |
-
|
| 787 |
-
|
| 788 |
-
|
| 789 |
-
|
| 790 |
-
|
| 791 |
-
|
| 792 |
-
|
| 793 |
-
|
| 794 |
-
|
| 795 |
-
|
| 796 |
-
|
| 797 |
-
|
| 798 |
-
|
| 799 |
-
|
| 800 |
-
#
|
| 801 |
-
|
| 802 |
-
|
| 803 |
-
|
| 804 |
-
|
| 805 |
-
|
| 806 |
-
|
| 807 |
-
|
| 808 |
-
|
| 809 |
-
|
| 810 |
-
|
| 811 |
-
print("
|
| 812 |
-
|
| 813 |
-
|
| 814 |
-
|
| 815 |
-
|
| 816 |
-
|
| 817 |
-
|
| 818 |
-
|
| 819 |
-
|
| 820 |
-
|
| 821 |
-
|
| 822 |
-
|
| 823 |
-
|
| 824 |
-
|
| 825 |
-
|
| 826 |
-
_, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(l, pose, r)
|
| 827 |
-
pitch = compensate_pitch(raw_pitch, hp)
|
| 828 |
-
yaw = compensate_yaw(raw_yaw, hy)
|
| 829 |
-
per_frame.append([pitch, yaw])
|
| 830 |
-
|
| 831 |
-
per_frame = np.array(per_frame)
|
| 832 |
-
med = np.median(per_frame, axis=0)
|
| 833 |
-
adapted_gaze_vectors.append(list(med))
|
| 834 |
-
adapted_dispersion.append([
|
| 835 |
-
float(np.median(np.abs(per_frame[:, 0] - med[0]))),
|
| 836 |
-
float(np.median(np.abs(per_frame[:, 1] - med[1]))),
|
| 837 |
-
])
|
| 838 |
-
|
| 839 |
-
assert len(adapted_gaze_vectors) == len(screen_points), \
|
| 840 |
-
f"Length mismatch: {len(adapted_gaze_vectors)} gaze vs {len(screen_points)} screen"
|
| 841 |
-
|
| 842 |
-
pipeline.calibration.calibrate(
|
| 843 |
-
np.array(adapted_gaze_vectors),
|
| 844 |
-
np.array(screen_points),
|
| 845 |
-
ear=np.array(point_ear),
|
| 846 |
-
screen_size=(SCREEN_W, SCREEN_H) # <-- fixes the 1493x933 clamp bug
|
| 847 |
-
)
|
| 848 |
-
pipeline.calibration.save("calibration.pkl")
|
| 849 |
-
|
| 850 |
-
import pickle
|
| 851 |
-
_all_hp = [hp for pf in point_frames for hp in pf["hps"]]
|
| 852 |
-
_all_pose = [p for pf in point_frames for p in pf["poses"]]
|
| 853 |
-
baseline_pitch = float(np.median(_all_hp)) if _all_hp else 0.0
|
| 854 |
-
baseline_yaw = float(np.median([float(p[1]) * POSE_NORM_SCALE for p in _all_pose])) if _all_pose else 0.0
|
| 855 |
-
baseline_roll = float(np.median([float(p[2]) * POSE_NORM_SCALE for p in _all_pose])) if _all_pose else 0.0
|
| 856 |
-
with open("baseline_pose.pkl", "wb") as f:
|
| 857 |
-
pickle.dump({"pitch": baseline_pitch, "yaw": baseline_yaw, "roll": baseline_roll}, f)
|
| 858 |
-
print(f"Saved baseline_pose.pkl: pitch={baseline_pitch:+.2f}, yaw={baseline_yaw:+.2f}, roll={baseline_roll:+.2f}")
|
| 859 |
-
|
| 860 |
-
print(f"\nCalibration complete! {n_pts}/{total} points used.")
|
| 861 |
-
if finetuned:
|
| 862 |
-
print("CNN fine-tuned on your eyes + RBF calibration fitted.")
|
| 863 |
-
else:
|
| 864 |
-
print("RBF calibration fitted (CNN fine-tuning skipped).")
|
| 865 |
-
|
| 866 |
-
# =============================================================================
|
| 867 |
-
# SESSION DIAGNOSTIC SUMMARY — the "tell the person if something's off" ask.
|
| 868 |
-
# Printed once, in plain language, instead of them having to interpret
|
| 869 |
-
# per-point numbers themselves.
|
| 870 |
-
# =============================================================================
|
| 871 |
-
# =============================================================================
|
| 872 |
-
# SIGNAL-TO-NOISE — THE VERDICT
|
| 873 |
-
#
|
| 874 |
-
# This is the number that decides whether an axis is fixable at all.
|
| 875 |
-
#
|
| 876 |
-
# SIGNAL = how much the model's output changes between DIFFERENT screen
|
| 877 |
-
# positions (spread of the per-point medians). This is the real
|
| 878 |
-
# information the RBF has to work with.
|
| 879 |
-
# NOISE = how much the model's output wobbles frame-to-frame while you
|
| 880 |
-
# stare at ONE fixed dot (median within-point jitter).
|
| 881 |
-
#
|
| 882 |
-
# If NOISE >= SIGNAL on an axis, the model cannot tell "you looked lower"
|
| 883 |
-
# apart from "the same look, one frame later". No RBF, no gain correction,
|
| 884 |
-
# no clamp tuning can recover information that was never there. That axis
|
| 885 |
-
# needs a better MODEL, not better calibration math.
|
| 886 |
-
# =============================================================================
|
| 887 |
-
gv_arr = np.array(adapted_gaze_vectors)
|
| 888 |
-
disp_arr = np.array(adapted_dispersion)
|
| 889 |
-
sp_arr = np.array(screen_points)
|
| 890 |
-
|
| 891 |
-
signal_pitch = float(np.std(gv_arr[:, 0]))
|
| 892 |
-
signal_yaw = float(np.std(gv_arr[:, 1]))
|
| 893 |
-
noise_pitch = float(np.median(disp_arr[:, 0]))
|
| 894 |
-
noise_yaw = float(np.median(disp_arr[:, 1]))
|
| 895 |
-
|
| 896 |
-
snr_pitch = signal_pitch / noise_pitch if noise_pitch > 1e-9 else float("inf")
|
| 897 |
-
snr_yaw = signal_yaw / noise_yaw if noise_yaw > 1e-9 else float("inf")
|
| 898 |
-
|
| 899 |
-
# CORRELATION — the metric that actually decides usability.
|
| 900 |
-
#
|
| 901 |
-
# An earlier version of this block reported only SNR (spread vs jitter). That
|
| 902 |
-
# was MISLEADING and nearly sent us chasing the wrong fix: a signal can have
|
| 903 |
-
# plenty of spread and still be useless if that spread is not ORDERED by the
|
| 904 |
-
# thing you're predicting. Pitch once reported SNR 1.76 ("marginal") while its
|
| 905 |
-
# per-row averages went down, up, up — i.e. no monotonic relationship with
|
| 906 |
-
# screen row at all. Spread was real; ordering was not.
|
| 907 |
-
#
|
| 908 |
-
# Pearson r between model output and true screen coordinate is the honest
|
| 909 |
-
# test. r near +/-1 means the model tracks that axis. r near 0 means it does
|
| 910 |
-
# not, no matter how much spread there is.
|
| 911 |
-
def _pearson(a, b):
|
| 912 |
-
a = np.asarray(a, dtype=float); b = np.asarray(b, dtype=float)
|
| 913 |
-
if a.std() < 1e-12 or b.std() < 1e-12:
|
| 914 |
-
return 0.0
|
| 915 |
-
return float(np.corrcoef(a, b)[0, 1])
|
| 916 |
-
|
| 917 |
-
r_yaw = _pearson(gv_arr[:, 1], sp_arr[:, 0]) # yaw vs screen X
|
| 918 |
-
r_pitch = _pearson(gv_arr[:, 0], sp_arr[:, 1]) # pitch vs screen Y
|
| 919 |
-
r_ear = _pearson(np.array(point_ear), sp_arr[:, 1]) # eye aperture vs screen Y
|
| 920 |
-
|
| 921 |
-
def _corr_verdict(r):
|
| 922 |
-
ar = abs(r)
|
| 923 |
-
if ar >= 0.90:
|
| 924 |
-
return "EXCELLENT — model tracks this axis cleanly"
|
| 925 |
-
if ar >= 0.70:
|
| 926 |
-
return "GOOD — usable, some slop"
|
| 927 |
-
if ar >= 0.50:
|
| 928 |
-
return "WEAK — expect significant error on this axis"
|
| 929 |
-
return "BROKEN — model output barely relates to this axis at all"
|
| 930 |
-
|
| 931 |
-
print("\n========== DOES THE MODEL TRACK THE SCREEN? (the real test) ==========")
|
| 932 |
-
print(f"HORIZONTAL yaw vs screen-X : r = {r_yaw:+.3f} (SNR {snr_yaw:.2f})")
|
| 933 |
-
print(f" -> {_corr_verdict(r_yaw)}")
|
| 934 |
-
print(f"VERTICAL pitch vs screen-Y: r = {r_pitch:+.3f} (SNR {snr_pitch:.2f})")
|
| 935 |
-
print(f" -> {_corr_verdict(r_pitch)}")
|
| 936 |
-
print(f"VERTICAL EYE APERTURE vs screen-Y: r = {r_ear:+.3f}")
|
| 937 |
-
print(f" -> {_corr_verdict(r_ear)}")
|
| 938 |
-
print()
|
| 939 |
-
_best_vert = max(abs(r_pitch), abs(r_ear))
|
| 940 |
-
print(f"Best available vertical cue: "
|
| 941 |
-
f"{'EYE APERTURE' if abs(r_ear) > abs(r_pitch) else 'CNN PITCH'} "
|
| 942 |
-
f"(r={_best_vert:+.3f})")
|
| 943 |
-
print()
|
| 944 |
-
if _best_vert < 0.5:
|
| 945 |
-
print("VERDICT: the model's PITCH output does not meaningfully track where you")
|
| 946 |
-
print("look vertically. This is NOT a calibration problem — the RBF cannot map")
|
| 947 |
-
print("an input that carries no ordered information about screen height. No")
|
| 948 |
-
print("amount of clamp/smoothing/gain tuning will fix it. The fix is the MODEL:")
|
| 949 |
-
print("its pitch head needs retraining, or vertical gaze needs a different")
|
| 950 |
-
print("feature (e.g. eyelid aperture / iris-centre offset within the socket),")
|
| 951 |
-
print("which the current eye-patch CNN is evidently not learning.")
|
| 952 |
-
elif _best_vert < abs(r_yaw) - 0.15:
|
| 953 |
-
print("VERDICT: vertical tracks the screen, but noticeably worse than")
|
| 954 |
-
print("horizontal. Calibration is doing its job; expect up/down to stay the")
|
| 955 |
-
print("looser axis until the model improves.")
|
| 956 |
-
else:
|
| 957 |
-
print("VERDICT: both axes track the screen. Any remaining error is in the")
|
| 958 |
-
print("calibration mapping or the noise floor, not in the model's ability")
|
| 959 |
-
print("to see where you're looking.")
|
| 960 |
-
print("=====================================================================")
|
| 961 |
-
|
| 962 |
-
print("\n================ CALIBRATION QUALITY SUMMARY ================")
|
| 963 |
-
if session_brightness:
|
| 964 |
-
avg_bright = float(np.mean(session_brightness))
|
| 965 |
-
pct_dark = 100 * sum(1 for b in session_brightness if b < LIGHT_MIN_BRIGHTNESS) / len(session_brightness)
|
| 966 |
-
pct_bright = 100 * sum(1 for b in session_brightness if b > LIGHT_MAX_BRIGHTNESS) / len(session_brightness)
|
| 967 |
-
print(f"Average brightness: {avg_bright:.0f} (comfortable range: {LIGHT_MIN_BRIGHTNESS}-{LIGHT_MAX_BRIGHTNESS})")
|
| 968 |
-
if pct_dark > 10:
|
| 969 |
-
print(f"⚠ Lighting was too DARK for {pct_dark:.0f}% of frames. "
|
| 970 |
-
f"Add a light source facing your face, or face a window, before recalibrating.")
|
| 971 |
-
if pct_bright > 10:
|
| 972 |
-
print(f"⚠ Lighting was too BRIGHT for {pct_bright:.0f}% of frames "
|
| 973 |
-
f"(backlight or a light directly behind you?). Try facing away from strong light sources.")
|
| 974 |
-
if pct_dark <= 10 and pct_bright <= 10:
|
| 975 |
-
print("Lighting was consistently good throughout.")
|
| 976 |
-
|
| 977 |
-
if session_blink_skips > 0:
|
| 978 |
-
print(f"Blinking accounted for {session_blink_skips} skipped frames across the session "
|
| 979 |
-
f"— normal, this is expected and was handled automatically.")
|
| 980 |
-
|
| 981 |
-
if points_with_low_yield:
|
| 982 |
-
print(f"⚠ These points had low sample counts and may be less accurate: "
|
| 983 |
-
f"{', '.join(str(p) for p in points_with_low_yield)}. "
|
| 984 |
-
f"If tracking feels off in that part of the screen, consider recalibrating.")
|
| 985 |
-
else:
|
| 986 |
-
print("All points collected a healthy number of samples.")
|
| 987 |
-
print("===============================================================")
|
| 988 |
-
|
| 989 |
-
# --- Diagnostic: verify vertical pitch separation ---
|
| 990 |
-
print("\n--- Pitch by screen row (should increase top -> bottom) ---")
|
| 991 |
-
gv_arr = np.array(adapted_gaze_vectors)
|
| 992 |
-
sp_arr = np.array(screen_points)
|
| 993 |
-
thresholds = [(0, SCREEN_H*0.25, "Top (y<25%) "),
|
| 994 |
-
(SCREEN_H*0.25, SCREEN_H*0.5, "Mid-hi (25-50%) "),
|
| 995 |
-
(SCREEN_H*0.5, SCREEN_H*0.75, "Mid-lo (50-75%) "),
|
| 996 |
-
(SCREEN_H*0.75, SCREEN_H+1, "Bottom (y>75%) ")]
|
| 997 |
-
for lo, hi, label in thresholds:
|
| 998 |
-
mask = (sp_arr[:, 1] >= lo) & (sp_arr[:, 1] < hi)
|
| 999 |
-
if mask.any():
|
| 1000 |
-
print(f" {label}: avg pitch = {gv_arr[mask, 0].mean():.4f}")
|
| 1001 |
-
|
| 1002 |
-
print("\n--- Yaw by screen column (should increase left -> right) ---")
|
| 1003 |
-
col_thresholds = [(0, SCREEN_W*0.25, "Left (x<25%) "),
|
| 1004 |
-
(SCREEN_W*0.25, SCREEN_W*0.5, "Mid-lf (25-50%) "),
|
| 1005 |
-
(SCREEN_W*0.5, SCREEN_W*0.75, "Mid-rt (50-75%) "),
|
| 1006 |
-
(SCREEN_W*0.75, SCREEN_W+1, "Right (x>75%) ")]
|
| 1007 |
-
for lo, hi, label in col_thresholds:
|
| 1008 |
-
mask = (sp_arr[:, 0] >= lo) & (sp_arr[:, 0] < hi)
|
| 1009 |
-
if mask.any():
|
| 1010 |
-
print(f" {label}: avg yaw = {gv_arr[mask, 1].mean():.4f}")
|
|
|
|
| 4 |
import os
|
| 5 |
import random
|
| 6 |
import pyautogui
|
| 7 |
+
import sys
|
| 8 |
+
|
| 9 |
+
# Windows consoles default to a non-UTF-8 codepage (cp1252). A single
|
| 10 |
+
# unicode character in a print() (the "⚠" warning glyph below, for example)
|
| 11 |
+
# then crashes the whole calibration run with UnicodeEncodeError — this is
|
| 12 |
+
# not hypothetical, it happened mid-session. errors="replace" means an
|
| 13 |
+
# unencodable glyph degrades to "?" instead of killing the process.
|
| 14 |
+
if hasattr(sys.stdout, "reconfigure"):
|
| 15 |
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
| 16 |
+
if hasattr(sys.stderr, "reconfigure"):
|
| 17 |
+
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
| 18 |
|
| 19 |
import warnings
|
| 20 |
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
| 21 |
|
| 22 |
+
# Bundled read-only assets (models/checkpoints) vs. per-user writable data
|
| 23 |
+
# (calibration.pkl, baseline_pose.pkl) need different roots once frozen by
|
| 24 |
+
# PyInstaller (onedir nests bundled data under _internal/, alongside a
|
| 25 |
+
# persistent folder holding the actual InsightUX.exe).
|
| 26 |
+
if getattr(sys, "frozen", False):
|
| 27 |
+
RESOURCE_DIR = sys._MEIPASS
|
| 28 |
+
DATA_DIR = os.path.dirname(sys.executable)
|
| 29 |
+
else:
|
| 30 |
+
RESOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 31 |
+
DATA_DIR = RESOURCE_DIR
|
| 32 |
+
|
| 33 |
from preprocessing.preprocessing_pipeline import (
|
| 34 |
create_face_mesh,
|
| 35 |
estimate_camera_matrix,
|
|
|
|
| 442 |
# CONFIG
|
| 443 |
# =============================================================================
|
| 444 |
|
| 445 |
+
ONNX_PATH = os.path.join(RESOURCE_DIR, "models", "gaze_cnn_v4.onnx")
|
| 446 |
+
CKPT_PATH = os.path.join(RESOURCE_DIR, "checkpoints", "best_model_v4.pt")
|
| 447 |
|
| 448 |
# --- Fine-tune target geometry -------------------------------------------
|
| 449 |
# USE_MEASURED_GEOMETRY=False keeps the original hand-picked constants
|
|
|
|
| 503 |
FINETUNE_ENABLED = True
|
| 504 |
|
| 505 |
|
| 506 |
+
def main():
|
| 507 |
+
# =============================================================================
|
| 508 |
+
# MAIN CALIBRATION
|
| 509 |
+
# =============================================================================
|
| 510 |
+
|
| 511 |
+
pipeline = InsightUXPipeline(ONNX_PATH)
|
| 512 |
+
face_mesh = create_face_mesh(static_image_mode=False)
|
| 513 |
+
cap = cv2.VideoCapture(0)
|
| 514 |
+
|
| 515 |
+
cam_matrix = None
|
| 516 |
+
|
| 517 |
+
gaze_vectors = [] # per-point median [pitch, yaw] (pre-fine-tune)
|
| 518 |
+
screen_points = [] # per-point [sx, sy]
|
| 519 |
+
head_pitches_deg = [] # per-point median head pitch (deg)
|
| 520 |
+
point_dispersion = [] # per-point [mad_pitch, mad_yaw] <- the noise floor
|
| 521 |
+
point_frames = [] # per-point dict of REAL diverse frames (see below)
|
| 522 |
+
point_ear = [] # per-point median eye-aperture (candidate vertical cue)
|
| 523 |
+
|
| 524 |
+
# Session-wide diagnostics, so we can warn about systemic issues at the end
|
| 525 |
+
# rather than just per-point.
|
| 526 |
+
session_brightness = []
|
| 527 |
+
session_distance_mm = [] # viewing distance from solvePnP — real geometry, not a guess
|
| 528 |
+
session_blink_skips = 0
|
| 529 |
+
session_light_skips = 0
|
| 530 |
+
points_with_low_yield = []
|
| 531 |
+
|
| 532 |
+
cam_matrix_ref = [None]
|
| 533 |
+
face_orientation_gate(face_mesh, cap, cam_matrix_ref)
|
| 534 |
+
if cam_matrix_ref[0] is not None:
|
| 535 |
+
cam_matrix = cam_matrix_ref[0]
|
| 536 |
+
|
| 537 |
+
cv2.namedWindow("Calibration", cv2.WINDOW_NORMAL)
|
| 538 |
+
cv2.setWindowProperty("Calibration", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
|
| 539 |
+
|
| 540 |
+
total = len(points)
|
| 541 |
+
print(f"Calibration started - {total} points")
|
| 542 |
+
|
| 543 |
+
for idx, (px, py) in enumerate(points):
|
| 544 |
+
sx, sy = int(px * SCREEN_W), int(py * SCREEN_H)
|
| 545 |
+
duration = get_duration(py)
|
| 546 |
+
|
| 547 |
+
settle_s = 1.5 if idx == 0 else 0.4
|
| 548 |
+
settle_start = time.time()
|
| 549 |
+
while time.time() - settle_start < settle_s:
|
| 550 |
+
ret, frame = cap.read()
|
| 551 |
+
if not ret:
|
| 552 |
+
continue
|
| 553 |
+
screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
|
| 554 |
+
cv2.circle(screen, (sx, sy), 18, (80, 80, 80), -1)
|
| 555 |
+
msg = "Get ready..." if idx == 0 else "Settling..."
|
| 556 |
+
cv2.putText(screen, msg, (50, 50),
|
| 557 |
+
cv2.FONT_HERSHEY_SIMPLEX, 1, (180, 180, 180), 2)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 558 |
cv2.imshow("Calibration", screen)
|
| 559 |
cv2.waitKey(1)
|
|
|
|
| 560 |
|
| 561 |
+
samples = []
|
| 562 |
+
ear_list = []
|
| 563 |
+
l_list = []
|
| 564 |
+
r_list = []
|
| 565 |
+
p_list = []
|
| 566 |
+
hp_list = []
|
| 567 |
+
|
| 568 |
+
point_blink_skips = 0
|
| 569 |
+
point_light_skips = 0
|
| 570 |
+
|
| 571 |
+
start = time.time()
|
| 572 |
+
|
| 573 |
+
while time.time() - start < duration:
|
| 574 |
+
ret, frame = cap.read()
|
| 575 |
+
if not ret:
|
| 576 |
+
continue
|
| 577 |
+
|
| 578 |
+
if cam_matrix is None:
|
| 579 |
+
cam_matrix = estimate_camera_matrix(frame.shape)
|
| 580 |
+
|
| 581 |
+
# LIVE LIGHTING CHECK — skip this frame and tell the person, instead
|
| 582 |
+
# of silently feeding a too-dark/too-bright frame into the model.
|
| 583 |
+
light_ok, light_msg = check_lighting(frame)
|
| 584 |
+
gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
| 585 |
+
session_brightness.append(float(np.mean(gray_frame)))
|
| 586 |
+
if not light_ok:
|
| 587 |
+
point_light_skips += 1
|
| 588 |
+
screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
|
| 589 |
+
cv2.circle(screen, (sx, sy), 18, (0, 140, 255), 2)
|
| 590 |
+
cv2.putText(screen, f"Point {idx+1}/{total} - {light_msg}",
|
| 591 |
+
(50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 140, 255), 2)
|
| 592 |
+
cv2.imshow("Calibration", screen)
|
| 593 |
+
cv2.waitKey(1)
|
| 594 |
+
continue
|
| 595 |
+
|
| 596 |
+
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 597 |
+
results = face_mesh.process(rgb)
|
| 598 |
+
|
| 599 |
+
if not results.multi_face_landmarks:
|
| 600 |
+
continue
|
| 601 |
|
| 602 |
+
lms = results.multi_face_landmarks[0].landmark
|
| 603 |
+
head_pose = estimate_head_pose(lms, frame.shape, cam_matrix)
|
| 604 |
+
if head_pose is None:
|
| 605 |
+
continue
|
| 606 |
+
|
| 607 |
+
# LIVE BLINK CHECK — average EAR across both eyes. Below threshold
|
| 608 |
+
# means eyes are closed/closing; that frame's eye patches carry no
|
| 609 |
+
# real gaze signal and would just add noise to this point's median.
|
| 610 |
+
ear_l = compute_ear(lms, LEFT_EAR_INDICES, frame.shape)
|
| 611 |
+
ear_r = compute_ear(lms, RIGHT_EAR_INDICES, frame.shape)
|
| 612 |
+
avg_ear = (ear_l + ear_r) / 2.0
|
| 613 |
+
if avg_ear < BLINK_EAR_THRESHOLD:
|
| 614 |
+
point_blink_skips += 1
|
| 615 |
+
screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
|
| 616 |
+
cv2.circle(screen, (sx, sy), 18, (180, 180, 0), 2)
|
| 617 |
+
cv2.putText(screen, f"Point {idx+1}/{total} - Blink detected, skipping frame",
|
| 618 |
+
(50, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (180, 180, 0), 2)
|
| 619 |
+
cv2.imshow("Calibration", screen)
|
| 620 |
+
cv2.waitKey(1)
|
| 621 |
+
continue
|
| 622 |
+
|
| 623 |
+
# FIX A: normalize pose to [-1, +1] before CNN
|
| 624 |
+
pose_vec = normalize_pose(head_pose)
|
| 625 |
+
|
| 626 |
+
def process_eye(eye_idx, ear_idx, iris_idx):
|
| 627 |
+
s1 = step1_normalize(frame, lms, head_pose, eye_idx, ear_idx, iris_idx)
|
| 628 |
+
if not s1.is_open:
|
| 629 |
+
return None
|
| 630 |
+
ir = compute_iris_radius(lms, iris_idx, frame.shape)
|
| 631 |
+
s2 = step2_illumination(s1, ir)
|
| 632 |
+
return s2.blended if s2.is_usable else None
|
| 633 |
+
|
| 634 |
+
l = process_eye(LEFT_EYE_INDICES, LEFT_EAR_INDICES, LEFT_IRIS_INDICES)
|
| 635 |
+
r = process_eye(RIGHT_EYE_INDICES, RIGHT_EAR_INDICES, RIGHT_IRIS_INDICES)
|
| 636 |
+
|
| 637 |
+
if l is None and r is None:
|
| 638 |
+
continue
|
| 639 |
+
if l is None: l = r
|
| 640 |
+
if r is None: r = l
|
| 641 |
+
|
| 642 |
+
_, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(l, pose_vec, r)
|
| 643 |
+
|
| 644 |
+
pitch = compensate_pitch(raw_pitch, head_pose.pitch)
|
| 645 |
+
yaw = compensate_yaw(raw_yaw, head_pose.yaw)
|
| 646 |
+
|
| 647 |
+
samples.append([pitch, yaw])
|
| 648 |
+
ear_list.append(avg_ear)
|
| 649 |
+
# solvePnP's tvec is in the same units as FACE_3D_MODEL (mm), so
|
| 650 |
+
# tvec[2] is a genuine measurement of how far your face is from the
|
| 651 |
+
# camera. We were computing this every frame and throwing it away.
|
| 652 |
+
try:
|
| 653 |
+
session_distance_mm.append(float(head_pose.tvec[2]))
|
| 654 |
+
except Exception:
|
| 655 |
+
pass
|
| 656 |
+
l_list.append(l)
|
| 657 |
+
r_list.append(r)
|
| 658 |
+
p_list.append(pose_vec)
|
| 659 |
+
hp_list.append(head_pose.pitch)
|
| 660 |
+
|
| 661 |
+
elapsed = time.time() - start
|
| 662 |
screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
|
| 663 |
+
angle = int(360 * elapsed / duration)
|
| 664 |
+
cv2.ellipse(screen, (sx, sy), (28, 28), -90, 0, angle, (0, 180, 0), 3)
|
| 665 |
+
cv2.circle(screen, (sx, sy), 18, (0, 255, 0), -1)
|
| 666 |
+
cv2.putText(screen, f"Point {idx+1}/{total} - Look at the dot",
|
| 667 |
+
(50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
|
| 668 |
+
if point_blink_skips or point_light_skips:
|
| 669 |
+
cv2.putText(screen, f"skipped: {point_blink_skips} blink, {point_light_skips} lighting",
|
| 670 |
+
(50, 85), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (140, 140, 140), 1)
|
| 671 |
cv2.imshow("Calibration", screen)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 672 |
|
| 673 |
+
if cv2.waitKey(1) & 0xFF == 27:
|
| 674 |
+
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 675 |
|
| 676 |
+
session_blink_skips += point_blink_skips
|
| 677 |
+
session_light_skips += point_light_skips
|
| 678 |
|
| 679 |
+
if not samples:
|
| 680 |
+
print(f"Point {idx+1}: no samples, skipping.")
|
| 681 |
+
points_with_low_yield.append(idx + 1)
|
| 682 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 683 |
|
| 684 |
+
if len(samples) < MIN_SAMPLES_OK:
|
| 685 |
+
print(f" ⚠ Point {idx+1}: only {len(samples)} valid samples "
|
| 686 |
+
f"({point_blink_skips} blink-skipped, {point_light_skips} light-skipped) — may be unreliable")
|
| 687 |
+
points_with_low_yield.append(idx + 1)
|
| 688 |
+
|
| 689 |
+
samples_arr = np.array(samples)
|
| 690 |
+
avg = np.median(samples_arr, axis=0)
|
| 691 |
+
avg_hp_deg = float(np.median(hp_list))
|
| 692 |
+
|
| 693 |
+
# WITHIN-POINT DISPERSION = the noise floor. While you stared at ONE
|
| 694 |
+
# fixed dot, how much did the model's predicted pitch/yaw wobble
|
| 695 |
+
# frame-to-frame? This is the single most important number in the whole
|
| 696 |
+
# calibration and it was never being measured. Compare it against the
|
| 697 |
+
# BETWEEN-point spread (the actual signal) at the end of the run: if
|
| 698 |
+
# noise >= signal on an axis, that axis is unrecoverable by any
|
| 699 |
+
# calibration math, and no amount of RBF tuning will fix it.
|
| 700 |
+
# MAD (median absolute deviation) rather than std, so one blink-tail
|
| 701 |
+
# frame can't inflate it.
|
| 702 |
+
mad_pitch = float(np.median(np.abs(samples_arr[:, 0] - avg[0])))
|
| 703 |
+
mad_yaw = float(np.median(np.abs(samples_arr[:, 1] - avg[1])))
|
| 704 |
+
|
| 705 |
+
gaze_vectors.append(list(avg))
|
| 706 |
+
screen_points.append([sx, sy])
|
| 707 |
+
head_pitches_deg.append(avg_hp_deg)
|
| 708 |
+
point_dispersion.append([mad_pitch, mad_yaw])
|
| 709 |
+
# Eye aperture for this point. Looking DOWN lowers the eyelid, so this is
|
| 710 |
+
# a physically independent vertical cue — and the CNN's pitch output has
|
| 711 |
+
# proven nearly blind vertically (r~0.41). calibrate() will measure both
|
| 712 |
+
# and pick whichever actually tracks screen-Y.
|
| 713 |
+
point_ear.append(float(np.median(ear_list)))
|
| 714 |
+
|
| 715 |
+
# Keep a BUNDLE of genuinely different frames for this point — not one
|
| 716 |
+
# "representative" frame. Both the fine-tune and the post-fine-tune RBF
|
| 717 |
+
# refit read from this bundle, so both get real frame diversity and real
|
| 718 |
+
# noise averaging instead of anchoring on a single arbitrary frame.
|
| 719 |
+
n_avail = len(l_list)
|
| 720 |
+
n_take = min(FRAMES_KEPT_PER_POINT, n_avail)
|
| 721 |
+
indices = np.linspace(0, n_avail - 1, n_take, dtype=int)
|
| 722 |
+
point_frames.append({
|
| 723 |
+
"lefts": [l_list[i] for i in indices],
|
| 724 |
+
"rights": [r_list[i] for i in indices],
|
| 725 |
+
"poses": [p_list[i] for i in indices],
|
| 726 |
+
"hps": [hp_list[i] for i in indices],
|
| 727 |
+
})
|
| 728 |
+
|
| 729 |
+
print(f"Point {idx+1:2d}/{total} | pitch={avg[0]:+.4f} yaw={avg[1]:+.4f} "
|
| 730 |
+
f"| jitter(pitch)={mad_pitch:.4f} jitter(yaw)={mad_yaw:.4f} "
|
| 731 |
+
f"| {len(indices)} frames kept "
|
| 732 |
+
f"| skipped: {point_blink_skips} blink, {point_light_skips} lighting")
|
| 733 |
+
|
| 734 |
+
cap.release()
|
| 735 |
+
cv2.destroyAllWindows()
|
| 736 |
+
|
| 737 |
+
if len(gaze_vectors) < 4:
|
| 738 |
+
print(f"ERROR: only {len(gaze_vectors)} points collected, need at least 4.")
|
| 739 |
+
exit(1)
|
| 740 |
+
|
| 741 |
+
# =============================================================================
|
| 742 |
+
# OUTLIER FILTER
|
| 743 |
+
# =============================================================================
|
| 744 |
+
(gaze_vectors, screen_points, head_pitches_deg,
|
| 745 |
+
point_dispersion, point_frames, point_ear) = filter_unreliable_points(
|
| 746 |
+
gaze_vectors, screen_points, head_pitches_deg,
|
| 747 |
+
point_dispersion, point_frames, point_ear, factor=3.0
|
| 748 |
+
)
|
| 749 |
|
| 750 |
+
n_pts = len(screen_points)
|
| 751 |
+
if n_pts < 4:
|
| 752 |
+
print("ERROR: too many points removed, need at least 4 reliable points.")
|
| 753 |
+
exit(1)
|
| 754 |
+
|
| 755 |
+
# =============================================================================
|
| 756 |
+
# FINE-TUNE (optional — see FINETUNE_ENABLED comment for why it defaults OFF)
|
| 757 |
+
# Now fed the REAL diverse frames from each point's bundle. Previously this
|
| 758 |
+
# received the same single frame duplicated 5x per point, so it trained on
|
| 759 |
+
# 16 unique images while reporting 80.
|
| 760 |
+
# =============================================================================
|
| 761 |
+
finetuned = False
|
| 762 |
+
if FINETUNE_ENABLED:
|
| 763 |
+
ft_lefts, ft_rights, ft_poses, ft_sp = [], [], [], []
|
| 764 |
+
for i in range(n_pts):
|
| 765 |
+
pf = point_frames[i]
|
| 766 |
+
for k in range(len(pf["lefts"])):
|
| 767 |
+
ft_lefts.append(pf["lefts"][k])
|
| 768 |
+
ft_rights.append(pf["rights"][k])
|
| 769 |
+
ft_poses.append(pf["poses"][k])
|
| 770 |
+
ft_sp.append(screen_points[i])
|
| 771 |
+
|
| 772 |
+
n_unique = len(ft_lefts)
|
| 773 |
+
|
| 774 |
+
# Derive the fine-tune target geometry from REAL numbers instead of the
|
| 775 |
+
# fabricated 0.6/0.4 constants.
|
| 776 |
+
_diag_mm = SCREEN_DIAGONAL_INCHES * 25.4
|
| 777 |
+
_aspect = SCREEN_W / float(SCREEN_H)
|
| 778 |
+
_scr_h_mm = _diag_mm / np.sqrt(_aspect ** 2 + 1.0)
|
| 779 |
+
_scr_w_mm = _aspect * _scr_h_mm
|
| 780 |
+
if session_distance_mm:
|
| 781 |
+
_dist_mm = float(np.median(session_distance_mm))
|
| 782 |
+
else:
|
| 783 |
+
_dist_mm = 500.0
|
| 784 |
+
# Guard against a wild solvePnP outlier producing nonsense geometry.
|
| 785 |
+
_dist_mm = float(np.clip(_dist_mm, 300.0, 900.0))
|
| 786 |
|
| 787 |
+
if USE_MEASURED_GEOMETRY:
|
| 788 |
+
K_H = (_scr_w_mm / 2.0) / _dist_mm
|
| 789 |
+
K_V = (_scr_h_mm / 2.0) / _dist_mm
|
| 790 |
+
else:
|
| 791 |
+
K_H, K_V = 0.6, 0.4 # original constants — currently working
|
| 792 |
+
|
| 793 |
+
print(f"\n--- Fine-tuning CNN on your eyes ({n_unique} REAL frames, "
|
| 794 |
+
f"{n_pts} points) ---")
|
| 795 |
+
print(f"[Geometry] screen {SCREEN_DIAGONAL_INCHES}\" -> "
|
| 796 |
+
f"{_scr_w_mm:.0f}x{_scr_h_mm:.0f}mm | measured viewing distance "
|
| 797 |
+
f"{_dist_mm:.0f}mm")
|
| 798 |
+
print(f"[Geometry] fine-tune targets: k_h={K_H:.3f} k_v={K_V:.3f} "
|
| 799 |
+
f"(ratio {K_H/max(K_V,1e-6):.2f}) "
|
| 800 |
+
f"[{'measured' if USE_MEASURED_GEOMETRY else 'original constants'}]")
|
| 801 |
+
finetuned = finetune_on_calibration_v4(
|
| 802 |
+
onnx_path = ONNX_PATH,
|
| 803 |
+
left_patches = ft_lefts,
|
| 804 |
+
right_patches = ft_rights,
|
| 805 |
+
head_poses = ft_poses,
|
| 806 |
+
screen_points = ft_sp,
|
| 807 |
+
screen_w = SCREEN_W,
|
| 808 |
+
screen_h = SCREEN_H,
|
| 809 |
+
ckpt_path = CKPT_PATH,
|
| 810 |
+
out_onnx_path = ONNX_PATH,
|
| 811 |
+
steps = 200,
|
| 812 |
+
lr = 1e-4,
|
| 813 |
+
k_h = K_H,
|
| 814 |
+
k_v = K_V,
|
| 815 |
+
)
|
| 816 |
+
else:
|
| 817 |
+
print("\n--- Fine-tuning SKIPPED (FINETUNE_ENABLED = False) ---")
|
| 818 |
+
print(" Using the base model as-is. The RBF maps its output to screen")
|
| 819 |
+
print(" coordinates, so the model's absolute scale does not need to be")
|
| 820 |
+
print(" 'correct' — only monotonic. See the FINETUNE_ENABLED comment.")
|
| 821 |
+
|
| 822 |
+
# =============================================================================
|
| 823 |
+
# FIT RBF CALIBRATION
|
| 824 |
+
#
|
| 825 |
+
# CRITICAL FIX: re-predict across EVERY kept frame for each point and take the
|
| 826 |
+
# MEDIAN, instead of predicting from one arbitrary "representative" frame.
|
| 827 |
+
#
|
| 828 |
+
# The old code computed a careful median over 50-150 frames during collection,
|
| 829 |
+
# then threw it away and refit the RBF from a single frame per point. That
|
| 830 |
+
# discarded all noise averaging and anchored the entire calibration on 16
|
| 831 |
+
# single, possibly-noisy frames — which is exactly the kind of thing that
|
| 832 |
+
# makes edges and corners unstable.
|
| 833 |
+
# =============================================================================
|
| 834 |
+
print("\n--- Fitting RBF calibration ---")
|
| 835 |
+
pipeline = InsightUXPipeline(ONNX_PATH)
|
| 836 |
+
|
| 837 |
+
adapted_gaze_vectors = []
|
| 838 |
+
adapted_dispersion = []
|
| 839 |
for i in range(n_pts):
|
| 840 |
pf = point_frames[i]
|
| 841 |
+
per_frame = []
|
| 842 |
for k in range(len(pf["lefts"])):
|
| 843 |
+
l = pf["lefts"][k]
|
| 844 |
+
r = pf["rights"][k]
|
| 845 |
+
pose = pf["poses"][k]
|
| 846 |
+
hp = pf["hps"][k]
|
| 847 |
+
hy = float(pose[1]) * POSE_NORM_SCALE # recover raw head yaw (deg)
|
| 848 |
+
|
| 849 |
+
_, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(l, pose, r)
|
| 850 |
+
pitch = compensate_pitch(raw_pitch, hp)
|
| 851 |
+
yaw = compensate_yaw(raw_yaw, hy)
|
| 852 |
+
per_frame.append([pitch, yaw])
|
| 853 |
+
|
| 854 |
+
per_frame = np.array(per_frame)
|
| 855 |
+
med = np.median(per_frame, axis=0)
|
| 856 |
+
adapted_gaze_vectors.append(list(med))
|
| 857 |
+
adapted_dispersion.append([
|
| 858 |
+
float(np.median(np.abs(per_frame[:, 0] - med[0]))),
|
| 859 |
+
float(np.median(np.abs(per_frame[:, 1] - med[1]))),
|
| 860 |
+
])
|
| 861 |
+
|
| 862 |
+
assert len(adapted_gaze_vectors) == len(screen_points), \
|
| 863 |
+
f"Length mismatch: {len(adapted_gaze_vectors)} gaze vs {len(screen_points)} screen"
|
| 864 |
+
|
| 865 |
+
pipeline.calibration.calibrate(
|
| 866 |
+
np.array(adapted_gaze_vectors),
|
| 867 |
+
np.array(screen_points),
|
| 868 |
+
ear=np.array(point_ear),
|
| 869 |
+
screen_size=(SCREEN_W, SCREEN_H) # <-- fixes the 1493x933 clamp bug
|
| 870 |
+
)
|
| 871 |
+
pipeline.calibration.save(os.path.join(DATA_DIR, "calibration.pkl"))
|
| 872 |
+
|
| 873 |
+
import pickle
|
| 874 |
+
_all_hp = [hp for pf in point_frames for hp in pf["hps"]]
|
| 875 |
+
_all_pose = [p for pf in point_frames for p in pf["poses"]]
|
| 876 |
+
baseline_pitch = float(np.median(_all_hp)) if _all_hp else 0.0
|
| 877 |
+
baseline_yaw = float(np.median([float(p[1]) * POSE_NORM_SCALE for p in _all_pose])) if _all_pose else 0.0
|
| 878 |
+
baseline_roll = float(np.median([float(p[2]) * POSE_NORM_SCALE for p in _all_pose])) if _all_pose else 0.0
|
| 879 |
+
with open(os.path.join(DATA_DIR, "baseline_pose.pkl"), "wb") as f:
|
| 880 |
+
pickle.dump({"pitch": baseline_pitch, "yaw": baseline_yaw, "roll": baseline_roll}, f)
|
| 881 |
+
print(f"Saved baseline_pose.pkl: pitch={baseline_pitch:+.2f}, yaw={baseline_yaw:+.2f}, roll={baseline_roll:+.2f}")
|
| 882 |
+
|
| 883 |
+
print(f"\nCalibration complete! {n_pts}/{total} points used.")
|
| 884 |
+
if finetuned:
|
| 885 |
+
print("CNN fine-tuned on your eyes + RBF calibration fitted.")
|
| 886 |
else:
|
| 887 |
+
print("RBF calibration fitted (CNN fine-tuning skipped).")
|
| 888 |
+
|
| 889 |
+
# =============================================================================
|
| 890 |
+
# SESSION DIAGNOSTIC SUMMARY — the "tell the person if something's off" ask.
|
| 891 |
+
# Printed once, in plain language, instead of them having to interpret
|
| 892 |
+
# per-point numbers themselves.
|
| 893 |
+
# =============================================================================
|
| 894 |
+
# =============================================================================
|
| 895 |
+
# SIGNAL-TO-NOISE — THE VERDICT
|
| 896 |
+
#
|
| 897 |
+
# This is the number that decides whether an axis is fixable at all.
|
| 898 |
+
#
|
| 899 |
+
# SIGNAL = how much the model's output changes between DIFFERENT screen
|
| 900 |
+
# positions (spread of the per-point medians). This is the real
|
| 901 |
+
# information the RBF has to work with.
|
| 902 |
+
# NOISE = how much the model's output wobbles frame-to-frame while you
|
| 903 |
+
# stare at ONE fixed dot (median within-point jitter).
|
| 904 |
+
#
|
| 905 |
+
# If NOISE >= SIGNAL on an axis, the model cannot tell "you looked lower"
|
| 906 |
+
# apart from "the same look, one frame later". No RBF, no gain correction,
|
| 907 |
+
# no clamp tuning can recover information that was never there. That axis
|
| 908 |
+
# needs a better MODEL, not better calibration math.
|
| 909 |
+
# =============================================================================
|
| 910 |
+
gv_arr = np.array(adapted_gaze_vectors)
|
| 911 |
+
disp_arr = np.array(adapted_dispersion)
|
| 912 |
+
sp_arr = np.array(screen_points)
|
| 913 |
+
|
| 914 |
+
signal_pitch = float(np.std(gv_arr[:, 0]))
|
| 915 |
+
signal_yaw = float(np.std(gv_arr[:, 1]))
|
| 916 |
+
noise_pitch = float(np.median(disp_arr[:, 0]))
|
| 917 |
+
noise_yaw = float(np.median(disp_arr[:, 1]))
|
| 918 |
+
|
| 919 |
+
snr_pitch = signal_pitch / noise_pitch if noise_pitch > 1e-9 else float("inf")
|
| 920 |
+
snr_yaw = signal_yaw / noise_yaw if noise_yaw > 1e-9 else float("inf")
|
| 921 |
+
|
| 922 |
+
# CORRELATION — the metric that actually decides usability.
|
| 923 |
+
#
|
| 924 |
+
# An earlier version of this block reported only SNR (spread vs jitter). That
|
| 925 |
+
# was MISLEADING and nearly sent us chasing the wrong fix: a signal can have
|
| 926 |
+
# plenty of spread and still be useless if that spread is not ORDERED by the
|
| 927 |
+
# thing you're predicting. Pitch once reported SNR 1.76 ("marginal") while its
|
| 928 |
+
# per-row averages went down, up, up — i.e. no monotonic relationship with
|
| 929 |
+
# screen row at all. Spread was real; ordering was not.
|
| 930 |
+
#
|
| 931 |
+
# Pearson r between model output and true screen coordinate is the honest
|
| 932 |
+
# test. r near +/-1 means the model tracks that axis. r near 0 means it does
|
| 933 |
+
# not, no matter how much spread there is.
|
| 934 |
+
def _pearson(a, b):
|
| 935 |
+
a = np.asarray(a, dtype=float); b = np.asarray(b, dtype=float)
|
| 936 |
+
if a.std() < 1e-12 or b.std() < 1e-12:
|
| 937 |
+
return 0.0
|
| 938 |
+
return float(np.corrcoef(a, b)[0, 1])
|
| 939 |
+
|
| 940 |
+
r_yaw = _pearson(gv_arr[:, 1], sp_arr[:, 0]) # yaw vs screen X
|
| 941 |
+
r_pitch = _pearson(gv_arr[:, 0], sp_arr[:, 1]) # pitch vs screen Y
|
| 942 |
+
r_ear = _pearson(np.array(point_ear), sp_arr[:, 1]) # eye aperture vs screen Y
|
| 943 |
+
|
| 944 |
+
def _corr_verdict(r):
|
| 945 |
+
ar = abs(r)
|
| 946 |
+
if ar >= 0.90:
|
| 947 |
+
return "EXCELLENT — model tracks this axis cleanly"
|
| 948 |
+
if ar >= 0.70:
|
| 949 |
+
return "GOOD — usable, some slop"
|
| 950 |
+
if ar >= 0.50:
|
| 951 |
+
return "WEAK — expect significant error on this axis"
|
| 952 |
+
return "BROKEN — model output barely relates to this axis at all"
|
| 953 |
+
|
| 954 |
+
print("\n========== DOES THE MODEL TRACK THE SCREEN? (the real test) ==========")
|
| 955 |
+
print(f"HORIZONTAL yaw vs screen-X : r = {r_yaw:+.3f} (SNR {snr_yaw:.2f})")
|
| 956 |
+
print(f" -> {_corr_verdict(r_yaw)}")
|
| 957 |
+
print(f"VERTICAL pitch vs screen-Y: r = {r_pitch:+.3f} (SNR {snr_pitch:.2f})")
|
| 958 |
+
print(f" -> {_corr_verdict(r_pitch)}")
|
| 959 |
+
print(f"VERTICAL EYE APERTURE vs screen-Y: r = {r_ear:+.3f}")
|
| 960 |
+
print(f" -> {_corr_verdict(r_ear)}")
|
| 961 |
+
print()
|
| 962 |
+
_best_vert = max(abs(r_pitch), abs(r_ear))
|
| 963 |
+
print(f"Best available vertical cue: "
|
| 964 |
+
f"{'EYE APERTURE' if abs(r_ear) > abs(r_pitch) else 'CNN PITCH'} "
|
| 965 |
+
f"(r={_best_vert:+.3f})")
|
| 966 |
+
print()
|
| 967 |
+
if _best_vert < 0.5:
|
| 968 |
+
print("VERDICT: the model's PITCH output does not meaningfully track where you")
|
| 969 |
+
print("look vertically. This is NOT a calibration problem — the RBF cannot map")
|
| 970 |
+
print("an input that carries no ordered information about screen height. No")
|
| 971 |
+
print("amount of clamp/smoothing/gain tuning will fix it. The fix is the MODEL:")
|
| 972 |
+
print("its pitch head needs retraining, or vertical gaze needs a different")
|
| 973 |
+
print("feature (e.g. eyelid aperture / iris-centre offset within the socket),")
|
| 974 |
+
print("which the current eye-patch CNN is evidently not learning.")
|
| 975 |
+
elif _best_vert < abs(r_yaw) - 0.15:
|
| 976 |
+
print("VERDICT: vertical tracks the screen, but noticeably worse than")
|
| 977 |
+
print("horizontal. Calibration is doing its job; expect up/down to stay the")
|
| 978 |
+
print("looser axis until the model improves.")
|
| 979 |
else:
|
| 980 |
+
print("VERDICT: both axes track the screen. Any remaining error is in the")
|
| 981 |
+
print("calibration mapping or the noise floor, not in the model's ability")
|
| 982 |
+
print("to see where you're looking.")
|
| 983 |
+
print("=====================================================================")
|
| 984 |
+
|
| 985 |
+
print("\n================ CALIBRATION QUALITY SUMMARY ================")
|
| 986 |
+
if session_brightness:
|
| 987 |
+
avg_bright = float(np.mean(session_brightness))
|
| 988 |
+
pct_dark = 100 * sum(1 for b in session_brightness if b < LIGHT_MIN_BRIGHTNESS) / len(session_brightness)
|
| 989 |
+
pct_bright = 100 * sum(1 for b in session_brightness if b > LIGHT_MAX_BRIGHTNESS) / len(session_brightness)
|
| 990 |
+
print(f"Average brightness: {avg_bright:.0f} (comfortable range: {LIGHT_MIN_BRIGHTNESS}-{LIGHT_MAX_BRIGHTNESS})")
|
| 991 |
+
if pct_dark > 10:
|
| 992 |
+
print(f"⚠ Lighting was too DARK for {pct_dark:.0f}% of frames. "
|
| 993 |
+
f"Add a light source facing your face, or face a window, before recalibrating.")
|
| 994 |
+
if pct_bright > 10:
|
| 995 |
+
print(f"⚠ Lighting was too BRIGHT for {pct_bright:.0f}% of frames "
|
| 996 |
+
f"(backlight or a light directly behind you?). Try facing away from strong light sources.")
|
| 997 |
+
if pct_dark <= 10 and pct_bright <= 10:
|
| 998 |
+
print("Lighting was consistently good throughout.")
|
| 999 |
+
|
| 1000 |
+
if session_blink_skips > 0:
|
| 1001 |
+
print(f"Blinking accounted for {session_blink_skips} skipped frames across the session "
|
| 1002 |
+
f"— normal, this is expected and was handled automatically.")
|
| 1003 |
+
|
| 1004 |
+
if points_with_low_yield:
|
| 1005 |
+
print(f"⚠ These points had low sample counts and may be less accurate: "
|
| 1006 |
+
f"{', '.join(str(p) for p in points_with_low_yield)}. "
|
| 1007 |
+
f"If tracking feels off in that part of the screen, consider recalibrating.")
|
| 1008 |
+
else:
|
| 1009 |
+
print("All points collected a healthy number of samples.")
|
| 1010 |
+
print("===============================================================")
|
| 1011 |
+
|
| 1012 |
+
# --- Diagnostic: verify vertical pitch separation ---
|
| 1013 |
+
print("\n--- Pitch by screen row (should increase top -> bottom) ---")
|
| 1014 |
+
gv_arr = np.array(adapted_gaze_vectors)
|
| 1015 |
+
sp_arr = np.array(screen_points)
|
| 1016 |
+
thresholds = [(0, SCREEN_H*0.25, "Top (y<25%) "),
|
| 1017 |
+
(SCREEN_H*0.25, SCREEN_H*0.5, "Mid-hi (25-50%) "),
|
| 1018 |
+
(SCREEN_H*0.5, SCREEN_H*0.75, "Mid-lo (50-75%) "),
|
| 1019 |
+
(SCREEN_H*0.75, SCREEN_H+1, "Bottom (y>75%) ")]
|
| 1020 |
+
for lo, hi, label in thresholds:
|
| 1021 |
+
mask = (sp_arr[:, 1] >= lo) & (sp_arr[:, 1] < hi)
|
| 1022 |
+
if mask.any():
|
| 1023 |
+
print(f" {label}: avg pitch = {gv_arr[mask, 0].mean():.4f}")
|
| 1024 |
+
|
| 1025 |
+
print("\n--- Yaw by screen column (should increase left -> right) ---")
|
| 1026 |
+
col_thresholds = [(0, SCREEN_W*0.25, "Left (x<25%) "),
|
| 1027 |
+
(SCREEN_W*0.25, SCREEN_W*0.5, "Mid-lf (25-50%) "),
|
| 1028 |
+
(SCREEN_W*0.5, SCREEN_W*0.75, "Mid-rt (50-75%) "),
|
| 1029 |
+
(SCREEN_W*0.75, SCREEN_W+1, "Right (x>75%) ")]
|
| 1030 |
+
for lo, hi, label in col_thresholds:
|
| 1031 |
+
mask = (sp_arr[:, 0] >= lo) & (sp_arr[:, 0] < hi)
|
| 1032 |
+
if mask.any():
|
| 1033 |
+
print(f" {label}: avg yaw = {gv_arr[mask, 1].mean():.4f}")
|
| 1034 |
+
|
| 1035 |
+
|
| 1036 |
+
if __name__ == "__main__":
|
| 1037 |
+
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- mode: python ; coding: utf-8 -*-
|
| 2 |
+
#
|
| 3 |
+
# InsightUX PyInstaller build spec — onedir, not onefile.
|
| 4 |
+
#
|
| 5 |
+
# Onefile re-extracts the whole bundle (torch alone is ~1.2GB) to a temp
|
| 6 |
+
# directory on EVERY launch, which is a bad startup-time tradeoff on top of
|
| 7 |
+
# an already-heavy ML stack. Onedir installs once as a persistent folder
|
| 8 |
+
# (InsightUX.exe + _internal/), which is also just what installer.iss wants
|
| 9 |
+
# to package anyway.
|
| 10 |
+
#
|
| 11 |
+
# Build (from the repo root, inside the project venv — this project's venv
|
| 12 |
+
# uses Python 3.10, which is the proven-working interpreter; PyInstaller
|
| 13 |
+
# must be installed into that same venv first: `pip install pyinstaller`):
|
| 14 |
+
# pyinstaller packaging/InsightUX.spec --noconfirm
|
| 15 |
+
#
|
| 16 |
+
# Output lands in dist/InsightUX/ — that whole folder is what
|
| 17 |
+
# packaging/installer.iss packages into the installer.
|
| 18 |
+
#
|
| 19 |
+
# Known risk areas likely to need iteration on the FIRST real build (this is
|
| 20 |
+
# normal for this dependency stack, not a sign anything here is wrong):
|
| 21 |
+
# - mediapipe ships its own .tflite/.binarypb data files as package data;
|
| 22 |
+
# collect_all() below should catch them, but if face_mesh fails to load
|
| 23 |
+
# at runtime with a "file not found", that's the first place to check.
|
| 24 |
+
# - onnxruntime/torch/torchvision ship native DLLs; "DLL load failed" at
|
| 25 |
+
# startup means collect_all() missed one — check dist/InsightUX/_internal
|
| 26 |
+
# for what's actually present vs. what the traceback wants.
|
| 27 |
+
# - pywebview + pythonnet's `clr` bridge needs the Microsoft Edge WebView2
|
| 28 |
+
# Runtime on the TARGET machine. Usually preinstalled on modern Windows
|
| 29 |
+
# 10/11, but installer.iss bundles + silently runs Microsoft's WebView2
|
| 30 |
+
# bootstrapper as a safety net for older/locked-down machines.
|
| 31 |
+
|
| 32 |
+
import os
|
| 33 |
+
import dis
|
| 34 |
+
from PyInstaller.utils.hooks import collect_all
|
| 35 |
+
import PyInstaller.lib.modulegraph.util as _mg_util
|
| 36 |
+
|
| 37 |
+
# A handful of very large single-file modules in this dependency tree
|
| 38 |
+
# (bottle.py, matplotlib/pyplot.py, and possibly others not yet hit) trip a
|
| 39 |
+
# real dis.get_instructions() bug on this Python 3.10.0 build — an IndexError
|
| 40 |
+
# walking co_consts for certain EXTENDED_ARG-heavy bytecode in big modules.
|
| 41 |
+
# Confirmed this is purely a build-time INTROSPECTION bug, not a real problem:
|
| 42 |
+
# `import bottle` / `import matplotlib.pyplot` both work completely normally
|
| 43 |
+
# at actual runtime; only PyInstaller's static bytecode scan of them (used to
|
| 44 |
+
# discover further hidden imports) crashes.
|
| 45 |
+
#
|
| 46 |
+
# Patched once here instead of excluding modules one at a time as each is
|
| 47 |
+
# discovered (there's no way to know in advance how many more of these exist
|
| 48 |
+
# inside torch's ~1.2GB of dependencies). Worst case if this ever masks a
|
| 49 |
+
# real import PyInstaller would otherwise have auto-discovered: the
|
| 50 |
+
# hiddenimports/collect_all() list below already covers every package this
|
| 51 |
+
# app actually needs bundled, so a skipped scan on one giant module isn't
|
| 52 |
+
# depended on for correctness.
|
| 53 |
+
_orig_iterate_instructions = _mg_util.iterate_instructions
|
| 54 |
+
|
| 55 |
+
def _safe_iterate_instructions(code_object):
|
| 56 |
+
try:
|
| 57 |
+
yield from _orig_iterate_instructions(code_object)
|
| 58 |
+
except Exception as e:
|
| 59 |
+
print(f"[InsightUX.spec] bytecode scan skipped on a module "
|
| 60 |
+
f"(dis bug workaround): {type(e).__name__}: {e}")
|
| 61 |
+
return
|
| 62 |
+
|
| 63 |
+
_mg_util.iterate_instructions = _safe_iterate_instructions
|
| 64 |
+
|
| 65 |
+
PACKAGING_DIR = os.path.abspath(os.path.dirname(os.path.abspath(SPEC)))
|
| 66 |
+
REPO_ROOT = os.path.dirname(PACKAGING_DIR)
|
| 67 |
+
|
| 68 |
+
datas = [
|
| 69 |
+
(os.path.join(REPO_ROOT, "models", "gaze_cnn_v4.onnx"), "models"),
|
| 70 |
+
(os.path.join(REPO_ROOT, "models", "gaze_cnn_v4.onnx.data"), "models"),
|
| 71 |
+
(os.path.join(REPO_ROOT, "checkpoints", "best_model_v4.pt"), "checkpoints"),
|
| 72 |
+
]
|
| 73 |
+
binaries = []
|
| 74 |
+
# models.model_v4 is only reached via a try/except-guarded dynamic import
|
| 75 |
+
# inside calibrate.py's fine-tune function — spell it out explicitly rather
|
| 76 |
+
# than trust static analysis to follow it through the try/except.
|
| 77 |
+
hiddenimports = ["models.model_v4"]
|
| 78 |
+
|
| 79 |
+
# The blunt-but-reliable safety net for packages that ship native binaries
|
| 80 |
+
# and/or non-.py package data PyInstaller's built-in hooks don't fully catch.
|
| 81 |
+
for pkg in ("mediapipe", "onnxruntime", "torch", "torchvision", "cv2"):
|
| 82 |
+
pkg_datas, pkg_binaries, pkg_hiddenimports = collect_all(pkg)
|
| 83 |
+
datas += pkg_datas
|
| 84 |
+
binaries += pkg_binaries
|
| 85 |
+
hiddenimports += pkg_hiddenimports
|
| 86 |
+
|
| 87 |
+
a = Analysis(
|
| 88 |
+
[os.path.join(REPO_ROOT, "browser_session.py")],
|
| 89 |
+
pathex=[REPO_ROOT],
|
| 90 |
+
binaries=binaries,
|
| 91 |
+
datas=datas,
|
| 92 |
+
hiddenimports=hiddenimports,
|
| 93 |
+
hookspath=[],
|
| 94 |
+
hooksconfig={},
|
| 95 |
+
runtime_hooks=[],
|
| 96 |
+
excludes=[],
|
| 97 |
+
noarchive=False,
|
| 98 |
+
)
|
| 99 |
+
pyz = PYZ(a.pure)
|
| 100 |
+
|
| 101 |
+
exe = EXE(
|
| 102 |
+
pyz,
|
| 103 |
+
a.scripts,
|
| 104 |
+
[],
|
| 105 |
+
exclude_binaries=True,
|
| 106 |
+
name="InsightUX",
|
| 107 |
+
debug=False,
|
| 108 |
+
bootloader_ignore_signals=False,
|
| 109 |
+
strip=False,
|
| 110 |
+
upx=False,
|
| 111 |
+
# Keep the console. calibrate.py's live lighting/blink/quality readouts
|
| 112 |
+
# and the correlation "VERDICT" summary are the primary calibration
|
| 113 |
+
# feedback today — hiding the console would silently drop the one thing
|
| 114 |
+
# a user needs to read after every calibration run.
|
| 115 |
+
console=True,
|
| 116 |
+
disable_windowed_traceback=False,
|
| 117 |
+
target_arch=None,
|
| 118 |
+
codesign_identity=None,
|
| 119 |
+
entitlements_file=None,
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
coll = COLLECT(
|
| 123 |
+
exe,
|
| 124 |
+
a.binaries,
|
| 125 |
+
a.zipfiles,
|
| 126 |
+
a.datas,
|
| 127 |
+
strip=False,
|
| 128 |
+
upx=False,
|
| 129 |
+
upx_exclude=[],
|
| 130 |
+
name="InsightUX",
|
| 131 |
+
)
|
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Packaging InsightUX as a downloadable installer
|
| 2 |
+
|
| 3 |
+
Turns the cloned-repo-plus-venv workflow into a single Windows installer
|
| 4 |
+
(`InsightUX-Setup-<version>.exe`) that bundles Python, torch, mediapipe,
|
| 5 |
+
onnxruntime, opencv, and pywebview — end users need nothing pre-installed.
|
| 6 |
+
|
| 7 |
+
## One-time setup (per machine you build from)
|
| 8 |
+
|
| 9 |
+
1. In the project venv: `pip install pyinstaller`
|
| 10 |
+
2. Install [Inno Setup 6](https://jrsoftware.org/isdl.php) (free) — provides
|
| 11 |
+
`ISCC.exe`, the command-line compiler `installer.iss` needs. Not required
|
| 12 |
+
just to build the app itself, only to build the final installer.
|
| 13 |
+
3. (Optional but recommended) Download Microsoft's WebView2 bootstrapper
|
| 14 |
+
(`MicrosoftEdgeWebview2Setup.exe`) from
|
| 15 |
+
[Microsoft's evergreen bootstrapper page](https://developer.microsoft.com/microsoft-edge/webview2/)
|
| 16 |
+
and place it in this `packaging/` folder before compiling the installer —
|
| 17 |
+
`installer.iss` bundles and silently runs it if present, as a safety net
|
| 18 |
+
for machines missing the WebView2 Runtime (usually already present on
|
| 19 |
+
modern Windows 10/11, but not guaranteed on older/locked-down machines).
|
| 20 |
+
If you skip this, the installer still builds fine, just without that
|
| 21 |
+
safety net.
|
| 22 |
+
|
| 23 |
+
## Build
|
| 24 |
+
|
| 25 |
+
From the repo root:
|
| 26 |
+
|
| 27 |
+
```powershell
|
| 28 |
+
pyinstaller packaging\InsightUX.spec --noconfirm
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
Output: `dist\InsightUX\` (a folder — `InsightUX.exe` + `_internal\`). Run
|
| 32 |
+
`dist\InsightUX\InsightUX.exe` directly first to confirm it actually starts
|
| 33 |
+
before building the installer — much faster to debug a missing-DLL/missing-
|
| 34 |
+
data-file error at this stage than after wrapping it in an installer.
|
| 35 |
+
|
| 36 |
+
**Expect to iterate on the first build.** mediapipe, torch, and onnxruntime
|
| 37 |
+
all ship native binaries and non-`.py` data files that PyInstaller's default
|
| 38 |
+
hooks don't always fully catch — see the comments at the top of
|
| 39 |
+
`InsightUX.spec` for what's already collected and where to look if the
|
| 40 |
+
frozen exe fails to start.
|
| 41 |
+
|
| 42 |
+
Then build the installer:
|
| 43 |
+
|
| 44 |
+
```powershell
|
| 45 |
+
iscc packaging\installer.iss
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
Output: `packaging\dist_installer\InsightUX-Setup-<version>.exe`.
|
| 49 |
+
|
| 50 |
+
## Release checklist
|
| 51 |
+
|
| 52 |
+
Every time you ship a code change as an update:
|
| 53 |
+
|
| 54 |
+
1. Bump `VERSION` in [browser_session.py](../browser_session.py).
|
| 55 |
+
2. Bump `AppVersion`/`MyAppVersion` in [installer.iss](installer.iss) to match.
|
| 56 |
+
3. Rebuild (`pyinstaller ...` then `iscc ...`).
|
| 57 |
+
4. Update [version.json](../version.json) at the repo root — `"latest"` to
|
| 58 |
+
the new version, `"url"` to the new installer's HF download URL.
|
| 59 |
+
5. `git lfs` tracks `*.exe` already (see `.gitattributes`) — add and push the
|
| 60 |
+
new installer + `version.json` to the `origin` (Hugging Face) remote.
|
| 61 |
+
|
| 62 |
+
That's it — nothing about *publishing* a release is automated. What's
|
| 63 |
+
automated is the *checking*: every running copy of InsightUX pings
|
| 64 |
+
`version.json` once at startup (`check_for_update()` in browser_session.py)
|
| 65 |
+
and shows a clickable "Update available" pill in the toolbar if `"latest"`
|
| 66 |
+
is newer than its own `VERSION`. Clicking it opens `"url"` in the user's
|
| 67 |
+
default browser — still a manual download+run, just automated the "is there
|
| 68 |
+
something new" question. No internet, or any failure fetching the file, is
|
| 69 |
+
silent (no banner, never an error).
|
| 70 |
+
|
| 71 |
+
## Why these choices
|
| 72 |
+
|
| 73 |
+
- **Onedir, not onefile** — onefile re-extracts the whole ~1.5-2.5GB bundle
|
| 74 |
+
to a temp directory on every launch. With torch/mediapipe already this
|
| 75 |
+
heavy, that's a genuinely bad startup-time hit. Onedir just runs directly
|
| 76 |
+
from the installed folder.
|
| 77 |
+
- **Per-user install (`{localappdata}`), not Program Files** — no
|
| 78 |
+
admin/UAC prompt, and avoids Program Files' write-permission restrictions
|
| 79 |
+
for `calibration.pkl`/`sessions/`, which the running app writes directly
|
| 80 |
+
into its own folder (see `RESOURCE_DIR`/`DATA_DIR` in browser_session.py).
|
| 81 |
+
- **Fixed `AppId` GUID in installer.iss** — this is what makes a new
|
| 82 |
+
installer upgrade the existing install in place instead of creating a
|
| 83 |
+
second copy. Never regenerate it.
|
| 84 |
+
- **torch is bundled** (not excluded) — its fine-tuning step measurably
|
| 85 |
+
improves calibration accuracy (169-204px mean error vs. 275px without it,
|
| 86 |
+
per calibrate.py's own numbers). Confirmed with the project owner as
|
| 87 |
+
worth the larger download.
|
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
; InsightUX Windows installer — Inno Setup script.
|
| 2 |
+
;
|
| 3 |
+
; Build (requires Inno Setup / ISCC installed locally, and dist/InsightUX/
|
| 4 |
+
; already built via `pyinstaller packaging/InsightUX.spec` first):
|
| 5 |
+
; iscc packaging/installer.iss
|
| 6 |
+
; Output lands in packaging/dist_installer/InsightUX-Setup-<version>.exe.
|
| 7 |
+
;
|
| 8 |
+
; AppId is a FIXED GUID, generated once and never changed. It's the whole
|
| 9 |
+
; mechanism behind "push updates through the installer": every future build
|
| 10 |
+
; keeps this same AppId + a higher AppVersion, so re-running a new installer
|
| 11 |
+
; upgrades the existing install in place (same folder, same shortcuts)
|
| 12 |
+
; instead of creating a second copy. Do not regenerate this GUID.
|
| 13 |
+
;
|
| 14 |
+
; Installs per-user (PrivilegesRequired=lowest) to %LocalAppData%\InsightUX —
|
| 15 |
+
; no admin/UAC prompt, and critically, no Program Files write-permission
|
| 16 |
+
; problems for calibration.pkl/sessions/, which the running app writes
|
| 17 |
+
; directly into its own install folder (see browser_session.py's DATA_DIR).
|
| 18 |
+
;
|
| 19 |
+
; calibration.pkl, baseline_pose.pkl, and sessions/ are deliberately NOT
|
| 20 |
+
; listed under [Files] below. Inno Setup only ever touches files it tracked
|
| 21 |
+
; as having installed — so upgrades overwrite app files without touching
|
| 22 |
+
; those, and a normal uninstall leaves them behind too (the [Code] section
|
| 23 |
+
; below tells the user this explicitly on uninstall, rather than silently).
|
| 24 |
+
|
| 25 |
+
#define MyAppName "InsightUX"
|
| 26 |
+
#define MyAppVersion "1.0.0"
|
| 27 |
+
#define MyAppPublisher "InsightUX"
|
| 28 |
+
#define MyAppExeName "InsightUX.exe"
|
| 29 |
+
#define MyBuildDir "..\dist\InsightUX"
|
| 30 |
+
|
| 31 |
+
[Setup]
|
| 32 |
+
AppId={{6DE35CE2-05CB-4375-BFFE-D63E9181F1B9}
|
| 33 |
+
AppName={#MyAppName}
|
| 34 |
+
AppVersion={#MyAppVersion}
|
| 35 |
+
AppPublisher={#MyAppPublisher}
|
| 36 |
+
DefaultDirName={localappdata}\InsightUX
|
| 37 |
+
DefaultGroupName=InsightUX
|
| 38 |
+
DisableProgramGroupPage=yes
|
| 39 |
+
PrivilegesRequired=lowest
|
| 40 |
+
ArchitecturesInstallIn64BitMode=x64compatible
|
| 41 |
+
OutputDir=dist_installer
|
| 42 |
+
OutputBaseFilename=InsightUX-Setup-{#MyAppVersion}
|
| 43 |
+
Compression=lzma2
|
| 44 |
+
SolidCompression=yes
|
| 45 |
+
WizardStyle=modern
|
| 46 |
+
; A ~1.5-2.5GB installer (torch is bundled deliberately, see README) needs
|
| 47 |
+
; the larger internal chunk size or ISCC can fail to build at all.
|
| 48 |
+
DiskSpanning=no
|
| 49 |
+
|
| 50 |
+
[Tasks]
|
| 51 |
+
Name: "desktopicon"; Description: "Create a &desktop icon"; GroupDescription: "Additional icons:"
|
| 52 |
+
|
| 53 |
+
[Files]
|
| 54 |
+
Source: "{#MyBuildDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
|
| 55 |
+
; Microsoft's WebView2 bootstrapper is NOT part of this repo (an external
|
| 56 |
+
; download — see packaging/README.md for the official link). If you've
|
| 57 |
+
; placed it next to this .iss before compiling, it gets bundled and
|
| 58 |
+
; silently run below as a safety net for machines missing the runtime.
|
| 59 |
+
; Harmless to (re-)run even if WebView2 is already installed.
|
| 60 |
+
#ifexist "MicrosoftEdgeWebview2Setup.exe"
|
| 61 |
+
Source: "MicrosoftEdgeWebview2Setup.exe"; DestDir: "{tmp}"; Flags: deleteafterinstall
|
| 62 |
+
#endif
|
| 63 |
+
|
| 64 |
+
[Icons]
|
| 65 |
+
Name: "{group}\InsightUX"; Filename: "{app}\{#MyAppExeName}"; WorkingDir: "{app}"
|
| 66 |
+
Name: "{group}\Uninstall InsightUX"; Filename: "{uninstallexe}"
|
| 67 |
+
Name: "{commondesktop}\InsightUX"; Filename: "{app}\{#MyAppExeName}"; WorkingDir: "{app}"; Tasks: desktopicon
|
| 68 |
+
|
| 69 |
+
[Run]
|
| 70 |
+
#ifexist "MicrosoftEdgeWebview2Setup.exe"
|
| 71 |
+
Filename: "{tmp}\MicrosoftEdgeWebview2Setup.exe"; Parameters: "/silent /install"; StatusMsg: "Checking Microsoft Edge WebView2 Runtime..."; Flags: waituntilterminated
|
| 72 |
+
#endif
|
| 73 |
+
Filename: "{app}\{#MyAppExeName}"; Description: "Launch InsightUX"; Flags: nowait postinstall skipifsilent
|
| 74 |
+
|
| 75 |
+
[Code]
|
| 76 |
+
procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep);
|
| 77 |
+
begin
|
| 78 |
+
if CurUninstallStep = usPostUninstall then
|
| 79 |
+
begin
|
| 80 |
+
if FileExists(ExpandConstant('{app}\calibration.pkl')) or
|
| 81 |
+
DirExists(ExpandConstant('{app}\sessions')) then
|
| 82 |
+
MsgBox('Your calibration profile and saved sessions were kept in:' + #13#10 +
|
| 83 |
+
ExpandConstant('{app}') + #13#10 +
|
| 84 |
+
'Delete that folder yourself if you want a completely clean removal.',
|
| 85 |
+
mbInformation, MB_OK);
|
| 86 |
+
end;
|
| 87 |
+
end;
|
|
@@ -12,6 +12,7 @@
|
|
| 12 |
# --- Core CV / inference ---
|
| 13 |
opencv-python==4.10.0.84
|
| 14 |
mediapipe==0.10.18
|
|
|
|
| 15 |
onnxruntime==1.19.2
|
| 16 |
numpy==1.26.4
|
| 17 |
scipy==1.13.1
|
|
|
|
| 12 |
# --- Core CV / inference ---
|
| 13 |
opencv-python==4.10.0.84
|
| 14 |
mediapipe==0.10.18
|
| 15 |
+
onnx==1.16.2
|
| 16 |
onnxruntime==1.19.2
|
| 17 |
numpy==1.26.4
|
| 18 |
scipy==1.13.1
|
|
@@ -19,11 +19,29 @@ Prints:
|
|
| 19 |
PATCH_SOURCE must match calibrate.py and run_session.py.
|
| 20 |
"""
|
| 21 |
|
|
|
|
|
|
|
| 22 |
import cv2
|
| 23 |
import numpy as np
|
| 24 |
import time
|
| 25 |
import pyautogui
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
from preprocessing.preprocessing_pipeline import (
|
| 29 |
create_face_mesh,
|
|
@@ -43,8 +61,8 @@ from preprocessing.preprocessing_pipeline import (
|
|
| 43 |
from inference_pipeline import InsightUXPipeline, GazeAngleSmoother
|
| 44 |
|
| 45 |
|
| 46 |
-
ONNX_PATH = "models
|
| 47 |
-
CALIBRATION_PATH = "calibration.pkl"
|
| 48 |
|
| 49 |
SCREEN_W, SCREEN_H = pyautogui.size()
|
| 50 |
PATCH_SOURCE = "blended" # MUST match calibrate.py and main_webcam_pipeline.py
|
|
|
|
| 19 |
PATCH_SOURCE must match calibrate.py and run_session.py.
|
| 20 |
"""
|
| 21 |
|
| 22 |
+
import os
|
| 23 |
+
import sys
|
| 24 |
import cv2
|
| 25 |
import numpy as np
|
| 26 |
import time
|
| 27 |
import pyautogui
|
| 28 |
|
| 29 |
+
# Windows consoles default to a non-UTF-8 codepage (cp1252) — a stray
|
| 30 |
+
# unicode character in any print() would otherwise crash the whole process
|
| 31 |
+
# with UnicodeEncodeError. See calibrate.py for where this was hit for real.
|
| 32 |
+
if hasattr(sys.stdout, "reconfigure"):
|
| 33 |
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
| 34 |
+
if hasattr(sys.stderr, "reconfigure"):
|
| 35 |
+
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
| 36 |
+
|
| 37 |
+
# Same RESOURCE_DIR/DATA_DIR split as calibrate.py and browser_session.py —
|
| 38 |
+
# bundled read-only assets vs. per-user writable data, once frozen.
|
| 39 |
+
if getattr(sys, "frozen", False):
|
| 40 |
+
RESOURCE_DIR = sys._MEIPASS
|
| 41 |
+
DATA_DIR = os.path.dirname(sys.executable)
|
| 42 |
+
else:
|
| 43 |
+
RESOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 44 |
+
DATA_DIR = RESOURCE_DIR
|
| 45 |
|
| 46 |
from preprocessing.preprocessing_pipeline import (
|
| 47 |
create_face_mesh,
|
|
|
|
| 61 |
from inference_pipeline import InsightUXPipeline, GazeAngleSmoother
|
| 62 |
|
| 63 |
|
| 64 |
+
ONNX_PATH = os.path.join(RESOURCE_DIR, "models", "gaze_cnn_v4.onnx")
|
| 65 |
+
CALIBRATION_PATH = os.path.join(DATA_DIR, "calibration.pkl")
|
| 66 |
|
| 67 |
SCREEN_W, SCREEN_H = pyautogui.size()
|
| 68 |
PATCH_SOURCE = "blended" # MUST match calibrate.py and main_webcam_pipeline.py
|
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"latest": "1.0.0",
|
| 3 |
+
"url": "https://huggingface.co/arpitasethiii/insightux/resolve/main/InsightUX-Setup-1.0.0.exe",
|
| 4 |
+
"notes": "Initial downloadable installer release."
|
| 5 |
+
}
|