Sanyam0385 commited on
Commit
1b83c9c
·
1 Parent(s): 49d46a6

Add browser chrome, packaging pipeline, and fix UTF-8 crash in calibration

Browse files

Browser chrome: top toolbar (back/forward/reload/home/address bar) and left
sidebar (Calibrate/Validate/Start Session) replace the old status pill, so
the window reads as a normal browser. Page content is pushed below/right of
the chrome via forced body padding instead of being hidden underneath it.
Session overlays now re-inject on navigation so a tracking session survives
back/forward. Added an in-app "update available" notice backed by
version.json.

Packaging: calibrate.py refactored into main() so it can run as
`--mode calibrate` inside a frozen build instead of shelling out to a
missing .py file; RESOURCE_DIR/DATA_DIR split (bundled assets vs. per-user
writable data) added to all three entry points. New packaging/ with a
PyInstaller onedir spec and an Inno Setup installer script (per-user install,
fixed AppId for in-place upgrades).

Fix: Windows consoles default to cp1252, so a single unicode character in a
print() (calibrate.py's low-sample "⚠" warning) crashed the whole
calibration run with UnicodeEncodeError — hit for real mid-session. All
three entry points now force UTF-8 stdout/stderr with errors="replace".

.gitattributes CHANGED
@@ -1,3 +1,4 @@
1
  *.onnx.data filter=lfs diff=lfs merge=lfs -text
2
  *.pt filter=lfs diff=lfs merge=lfs -text
3
  *.onnx filter=lfs diff=lfs merge=lfs -text
 
 
1
  *.onnx.data filter=lfs diff=lfs merge=lfs -text
2
  *.pt filter=lfs diff=lfs merge=lfs -text
3
  *.onnx filter=lfs diff=lfs merge=lfs -text
4
+ *.exe filter=lfs diff=lfs merge=lfs -text
.gitignore CHANGED
@@ -16,6 +16,7 @@
16
  !.gitattributes
17
  !README.md
18
  !requirements.txt
 
19
 
20
  !calibrate.py
21
  !validate.py
@@ -24,6 +25,10 @@
24
  !inference_pipeline.py
25
  !session_logger.py
26
 
 
 
 
 
27
  !preprocessing/__init__.py
28
  !preprocessing/preprocessing_pipeline.py
29
 
 
16
  !.gitattributes
17
  !README.md
18
  !requirements.txt
19
+ !version.json
20
 
21
  !calibrate.py
22
  !validate.py
 
25
  !inference_pipeline.py
26
  !session_logger.py
27
 
28
+ !packaging/InsightUX.spec
29
+ !packaging/installer.iss
30
+ !packaging/README.md
31
+
32
  !preprocessing/__init__.py
33
  !preprocessing/preprocessing_pipeline.py
34
 
README.md CHANGED
@@ -52,6 +52,9 @@ venv\Scripts\activate
52
  pip install -r requirements.txt
53
  ```
54
 
 
 
 
55
  **Do not `pip install --upgrade` pywebview or pythonnet.** `requirements.txt`
56
  pins `pywebview==4.4.1` and `pythonnet==3.0.3` deliberately — newer versions
57
  have a bug in their Windows backend that freezes the app window and floods
 
52
  pip install -r requirements.txt
53
  ```
54
 
55
+ This includes `onnx`, which `calibrate.py` needs when it exports the
56
+ fine-tuned model after training.
57
+
58
  **Do not `pip install --upgrade` pywebview or pythonnet.** `requirements.txt`
59
  pins `pywebview==4.4.1` and `pythonnet==3.0.3` deliberately — newer versions
60
  have a bug in their Windows backend that freezes the app window and floods
browser_session.py CHANGED
@@ -24,9 +24,20 @@ import json
24
  import time
25
  import math
26
  import threading
 
27
  from dataclasses import replace
28
  from datetime import datetime
29
 
 
 
 
 
 
 
 
 
 
 
30
  import cv2
31
  import numpy as np
32
  import webview
@@ -56,8 +67,25 @@ from analysis import generate_report
56
  # CONFIG — kept identical to run_session.py so calibration.pkl stays valid
57
  # =============================================================================
58
 
59
- ONNX_PATH = "models/gaze_cnn_v4.onnx"
60
- CALIBRATION_PATH = "calibration.pkl"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
  SCREEN_W, SCREEN_H = pyautogui.size()
62
 
63
  PATCH_SOURCE = "blended"
@@ -78,7 +106,7 @@ POSE_NORM_SCALE = 30.0
78
  HEAD_PITCH_COMPENSATION = 0.0 # reverted — 0.35 made accuracy worse, not better
79
  HEAD_YAW_COMPENSATION = 0.0
80
 
81
- SESSIONS_ROOT = "sessions"
82
 
83
  # The window opens on this local, fully InsightUX-branded page instead of
84
  # raw google.com. What you type here still hits REAL Google — no fake
@@ -154,7 +182,7 @@ _LANDING_HTML = r"""<!DOCTYPE html>
154
 
155
 
156
  def _write_landing_page():
157
- path = os.path.abspath("insightux_landing.html")
158
  with open(path, "w", encoding="utf-8") as f:
159
  f.write(_LANDING_HTML)
160
  return "file://" + path.replace(os.sep, "/")
@@ -219,52 +247,184 @@ class OneEuroFilter:
219
 
220
 
221
  # =============================================================================
222
- # JS: control widget (always present) — search hint + S/Esc keybindings
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  # =============================================================================
224
 
225
- CONTROL_JS = r"""
226
  (function(){
227
  if (window.__insightuxControl) { return; }
228
  window.__insightuxControl = true;
229
 
230
  // S/E (start/stop tracking) only make sense on an actual website — not on
231
  // the InsightUX landing/search page and not on a generated insights
232
- // report page. X (quit the app) always works, on every page.
233
  const isLanding = !!document.querySelector('meta[name="insightux-landing"]');
234
  const isReport = !!document.querySelector('meta[name="insightux-report"]');
235
  const isTrackable = !isLanding && !isReport;
236
 
237
- if (isTrackable) {
238
- const banner = document.createElement('div');
239
- banner.id = '__insightux_pill';
240
- banner.style.cssText = `
241
- position:fixed; top:0; left:0; right:0; z-index:2147483647;
242
- display:flex; align-items:center; justify-content:space-between;
243
- padding:7px 16px; font:12px -apple-system,Arial;
244
- background:linear-gradient(90deg, rgba(20,18,26,0.95), rgba(30,20,40,0.95));
 
 
 
 
 
 
 
 
 
245
  border-bottom:2px solid; border-image:linear-gradient(90deg,#7B2FBE,#FF2DF0) 1;
246
- box-shadow:0 3px 14px rgba(0,0,0,0.35); pointer-events:none;
247
- `;
248
- banner.innerHTML = `
249
- <span style="display:flex;align-items:center;gap:7px;">
250
- <span style="width:8px;height:8px;border-radius:50%;background:linear-gradient(135deg,#7B2FBE,#FF2DF0);display:inline-block;"></span>
251
- <b style="color:#f0ecf7;letter-spacing:0.02em;">InsightUX</b>
252
- <span style="color:#7a7288;">Eye + Mouse Research Browser</span>
253
- </span>
254
- <span id="__insightux_status" style="color:#c9a6f5;">Press S to start tracking &middot; E to stop &middot; H heatmap &middot; M mouse panel &middot; X quit</span>
255
- `;
256
- document.documentElement.appendChild(banner);
257
-
258
- setInterval(function(){
259
- if (!document.documentElement.contains(banner)) document.documentElement.appendChild(banner);
260
- }, 1000);
261
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
262
 
263
  const statusEl = () => document.getElementById('__insightux_status');
264
  window.insightuxSetStatus = function(text){
265
  const el = statusEl();
266
  if (el) el.textContent = text;
267
  };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
268
 
269
  function isTypingTarget(el){
270
  if (!el) return false;
@@ -302,12 +462,15 @@ CONTROL_JS = r"""
302
  }, true); // capture phase — fires before page scripts can intercept/stop the event
303
  })();
304
  """
 
305
 
306
  # JS: tracking overlay (gaze dot + AOI highlighting) — injected only once
307
  # tracking actually starts. Ported directly from run_session.py's JS_SETUP.
308
  TRACKING_JS = r"""
309
  (function(){
310
  if (window.__insightux) { return; }
 
 
311
  const state = { aois: [] };
312
  const dwell = { pendLabel: null, pendSince: 0, activeLabel: null, emptySince: 0 };
313
  const DWELL_MS = 420;
@@ -382,10 +545,10 @@ TRACKING_JS = r"""
382
 
383
  function consider(el){
384
  if (full) return;
385
- // Never track InsightUX's own injected UI (the "press S/E" pill,
386
- // the gaze-dot canvas, the mouse tracker panel) as if it were page content.
387
- if (el.id === '__insightux_pill' || el.id === '__insightux_canvas' || el.id === '__insightux_mouse_panel') return;
388
- if (el.closest && el.closest('#__insightux_pill, #__insightux_canvas, #__insightux_mouse_panel')) return;
389
  const tag = el.tagName.toLowerCase();
390
  const explicit = !!(el.dataset && el.dataset.aoi);
391
  if (!explicit){
@@ -662,12 +825,14 @@ MOUSE_JS = r"""
662
  }, 2000);
663
 
664
  // ---- exclude InsightUX's own injected UI from every measurement ----
665
- // (the status pill, the mouse-tracker panel, the heatmap canvas) — only
666
  // real website elements should ever show up in trail/heatmap/dwell/clicks.
 
 
667
  function isOwnUI(el){
668
  if (!el) return false;
669
- if (el.id === '__insightux_pill' || el.id === '__insightux_canvas' || el.id === '__insightux_mouse_panel') return true;
670
- return !!(el.closest && el.closest('#__insightux_pill, #__insightux_canvas, #__insightux_mouse_panel'));
671
  }
672
 
673
  // ---- mouse position + hover tracking ----
@@ -992,11 +1157,22 @@ MOUSE_JS = r"""
992
  # =============================================================================
993
 
994
  # Pages tracking must never start on, checked against window.get_current_url()
995
- # as a Python-side backstop to CONTROL_JS's isTrackable check (belt and
996
  # suspenders — the JS guard can't run before the JS bridge is up).
997
  _NON_TRACKABLE_URL_MARKERS = ("insightux_landing.html", "analysis_report.html")
998
 
999
 
 
 
 
 
 
 
 
 
 
 
 
1000
  class Api:
1001
  def __init__(self):
1002
  self.window = None
@@ -1005,6 +1181,33 @@ class Api:
1005
  self.thread = None
1006
  self.mouse_log_f = None
1007
  self.mouse_close_timer = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1008
 
1009
  def start_tracking(self):
1010
  print("[browser_session] start_tracking() called from JS")
@@ -1019,6 +1222,10 @@ class Api:
1019
  print(f"[browser_session] refusing to start on non-website page: {current_url}")
1020
  self._set_status("Start tracking only works on a website — search or open a page first.")
1021
  return False
 
 
 
 
1022
  if not os.path.exists(CALIBRATION_PATH):
1023
  print(f"[browser_session] no {CALIBRATION_PATH} found — run calibrate.py first")
1024
  self._set_status("No calibration.pkl found — run calibrate.py first.")
@@ -1034,6 +1241,7 @@ class Api:
1034
  self._inject_mouse_overlay()
1035
  self._open_mouse_log(session_dir)
1036
  self._set_mouse_tracking(True)
 
1037
  self._set_status("Recording gaze + mouse... Press E to stop, H for heatmap, M for mouse panel")
1038
 
1039
  self.thread = threading.Thread(
@@ -1050,6 +1258,7 @@ class Api:
1050
  return False
1051
  self._set_status("Wrapping up your session...")
1052
  self._set_mouse_tracking(False)
 
1053
  self.stop_event.set()
1054
 
1055
  # Give the page a moment to flush its last batch of mouse data over
@@ -1067,6 +1276,77 @@ class Api:
1067
  print("[browser_session] quit_app() called from JS -- terminating process")
1068
  os._exit(0)
1069
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1070
  def _inject_tracking_overlay(self):
1071
  try:
1072
  self.window.evaluate_js(TRACKING_JS)
@@ -1134,11 +1414,51 @@ class Api:
1134
  api = Api()
1135
 
1136
 
1137
- def inject_control(window):
 
 
 
 
 
 
 
 
 
 
1138
  try:
1139
- window.evaluate_js(CONTROL_JS)
 
 
 
 
 
 
 
1140
  except Exception as e:
1141
- print(f"[browser_session] control inject failed (will retry): {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1142
 
1143
 
1144
  # =============================================================================
@@ -1343,20 +1663,36 @@ def _go_fullscreen(window):
1343
 
1344
 
1345
  if __name__ == "__main__":
1346
- os.makedirs(SESSIONS_ROOT, exist_ok=True)
1347
-
1348
- window = webview.create_window(
1349
- "InsightUX — Eye-Tracking Research Browser", LANDING_URL,
1350
- width=SCREEN_W, height=SCREEN_H, js_api=api
1351
- )
1352
- api.window = window
1353
- window.events.loaded += lambda: inject_control(window)
1354
- window.events.shown += lambda: threading.Thread(
1355
- target=_go_fullscreen, args=(window,), daemon=True
1356
- ).start()
1357
-
1358
- # debug=True enables right-click > Inspect (DevTools) so you can see
1359
- # the [insightux] console.log diagnostics from CONTROL_JS directly —
1360
- # keyboard handling runs entirely in the browser, so JS-side issues
1361
- # never show up in this terminal, only in DevTools' Console tab.
1362
- webview.start(debug=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  import time
25
  import math
26
  import threading
27
+ import subprocess
28
  from dataclasses import replace
29
  from datetime import datetime
30
 
31
+ # Windows consoles default to a non-UTF-8 codepage (cp1252) — a stray
32
+ # unicode character in any print() (ours, or one of calibrate.py/validate.py's
33
+ # once relaunched as a subprocess sharing this process's stdout handle) would
34
+ # otherwise crash the whole process with UnicodeEncodeError. Confirmed this
35
+ # happened for real during a calibration run mid-session.
36
+ if hasattr(sys.stdout, "reconfigure"):
37
+ sys.stdout.reconfigure(encoding="utf-8", errors="replace")
38
+ if hasattr(sys.stderr, "reconfigure"):
39
+ sys.stderr.reconfigure(encoding="utf-8", errors="replace")
40
+
41
  import cv2
42
  import numpy as np
43
  import webview
 
67
  # CONFIG — kept identical to run_session.py so calibration.pkl stays valid
68
  # =============================================================================
69
 
70
+ # Bundled read-only assets (models/checkpoints) vs. per-user writable data
71
+ # (calibration.pkl, sessions/, the generated landing page) need different
72
+ # roots once frozen by PyInstaller — onedir nests bundled data under
73
+ # _internal/, alongside a persistent folder holding InsightUX.exe itself.
74
+ if getattr(sys, "frozen", False):
75
+ RESOURCE_DIR = sys._MEIPASS
76
+ DATA_DIR = os.path.dirname(sys.executable)
77
+ else:
78
+ RESOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
79
+ DATA_DIR = RESOURCE_DIR
80
+ BASE_DIR = DATA_DIR # kept as an alias: subprocess cwd for calibrate/validate relaunches
81
+
82
+ # Bump alongside packaging/installer.iss's AppVersion on every release, and
83
+ # update version.json in the same commit — see packaging/README.md.
84
+ VERSION = "1.0.0"
85
+ VERSION_CHECK_URL = "https://huggingface.co/arpitasethiii/insightux/resolve/main/version.json"
86
+
87
+ ONNX_PATH = os.path.join(RESOURCE_DIR, "models", "gaze_cnn_v4.onnx")
88
+ CALIBRATION_PATH = os.path.join(DATA_DIR, "calibration.pkl")
89
  SCREEN_W, SCREEN_H = pyautogui.size()
90
 
91
  PATCH_SOURCE = "blended"
 
106
  HEAD_PITCH_COMPENSATION = 0.0 # reverted — 0.35 made accuracy worse, not better
107
  HEAD_YAW_COMPENSATION = 0.0
108
 
109
+ SESSIONS_ROOT = os.path.join(DATA_DIR, "sessions")
110
 
111
  # The window opens on this local, fully InsightUX-branded page instead of
112
  # raw google.com. What you type here still hits REAL Google — no fake
 
182
 
183
 
184
  def _write_landing_page():
185
+ path = os.path.join(DATA_DIR, "insightux_landing.html")
186
  with open(path, "w", encoding="utf-8") as f:
187
  f.write(_LANDING_HTML)
188
  return "file://" + path.replace(os.sep, "/")
 
247
 
248
 
249
  # =============================================================================
250
+ # JS: browser chrome (always present) — top toolbar (back/forward/reload/
251
+ # home/address bar) + left sidebar (Calibrate/Validate/Start Session), so the
252
+ # window reads as a normal browser instead of a bare content pane. Purely an
253
+ # HTML/CSS/JS overlay drawn on top of whatever page is loaded — pywebview
254
+ # 4.4.1 has no native toolbar API, so this is the only way to add persistent
255
+ # chrome without switching GUI frameworks.
256
+ #
257
+ # The page itself is pushed down/right by exactly the toolbar height and
258
+ # sidebar width (via forced <body> padding, not just floating the chrome on
259
+ # top of it) so real page content is never hidden underneath — the chrome
260
+ # and the website occupy clearly separate regions instead of overlapping.
261
+ # overflow-x is force-hidden as a safety net against the small width
262
+ # reduction ever introducing a horizontal scrollbar. The one edge case this
263
+ # can't fully solve: a small minority of sites use position:fixed elements
264
+ # of their own pinned to the true viewport edges (independent of body
265
+ # padding) — those can still end up visually behind our chrome.
266
  # =============================================================================
267
 
268
+ CHROME_JS = r"""
269
  (function(){
270
  if (window.__insightuxControl) { return; }
271
  window.__insightuxControl = true;
272
 
273
  // S/E (start/stop tracking) only make sense on an actual website — not on
274
  // the InsightUX landing/search page and not on a generated insights
275
+ // report page. The toolbar/sidebar, and X (quit), always work everywhere.
276
  const isLanding = !!document.querySelector('meta[name="insightux-landing"]');
277
  const isReport = !!document.querySelector('meta[name="insightux-report"]');
278
  const isTrackable = !isLanding && !isReport;
279
 
280
+ const style = document.createElement('style');
281
+ style.textContent = `
282
+ html {
283
+ overflow-x: hidden !important;
284
+ }
285
+ body {
286
+ box-sizing: border-box !important;
287
+ width: 100vw !important;
288
+ margin: 0 !important;
289
+ padding-top: 46px !important;
290
+ padding-left: 84px !important;
291
+ overflow-x: hidden !important;
292
+ }
293
+ #__insightux_toolbar {
294
+ position:fixed; top:0; left:0; right:0; height:46px; z-index:2147483647;
295
+ display:flex; align-items:center; gap:10px; padding:0 12px; box-sizing:border-box;
296
+ background:linear-gradient(90deg, rgba(20,18,26,0.97), rgba(30,20,40,0.97));
297
  border-bottom:2px solid; border-image:linear-gradient(90deg,#7B2FBE,#FF2DF0) 1;
298
+ box-shadow:0 3px 14px rgba(0,0,0,0.35);
299
+ font:12px -apple-system,'Segoe UI',Arial; color:#f0ecf7;
300
+ }
301
+ #__insightux_toolbar button {
302
+ flex:0 0 auto; width:30px; height:30px; border-radius:8px; cursor:pointer;
303
+ background:#241f2e; border:1px solid #3a3348; color:#f0ecf7; font-size:15px;
304
+ display:flex; align-items:center; justify-content:center;
305
+ }
306
+ #__insightux_toolbar button:hover { background:#302a3d; }
307
+ #__iux_addr_form { flex:1 1 auto; min-width:0; }
308
+ #__iux_addr {
309
+ width:100%; box-sizing:border-box; padding:7px 14px; border-radius:16px;
310
+ border:1.5px solid #3a3348; background:#1c1826; color:#f0ecf7; outline:none;
311
+ font-size:12.5px;
312
+ }
313
+ #__iux_addr:focus { border-color:#9B59FF; }
314
+ #__insightux_status {
315
+ flex:0 0 auto; color:#c9a6f5; white-space:nowrap; max-width:30vw;
316
+ overflow:hidden; text-overflow:ellipsis;
317
+ }
318
+ #__insightux_sidebar {
319
+ position:fixed; top:46px; left:0; bottom:0; width:84px; z-index:2147483647;
320
+ display:flex; flex-direction:column; align-items:stretch; gap:8px; padding:10px 8px;
321
+ box-sizing:border-box; font:11px -apple-system,'Segoe UI',Arial;
322
+ background:linear-gradient(180deg, rgba(24,20,32,0.97), rgba(16,14,22,0.97));
323
+ border-right:2px solid; border-image:linear-gradient(180deg,#7B2FBE,#FF2DF0) 1;
324
+ box-shadow:3px 0 14px rgba(0,0,0,0.35);
325
+ }
326
+ #__insightux_sidebar button {
327
+ background:#241f2e; border:1px solid #3a3348; color:#f0ecf7; border-radius:10px;
328
+ padding:10px 4px; cursor:pointer; line-height:1.5; font-size:11px; text-align:center;
329
+ }
330
+ #__insightux_sidebar button:hover { background:#302a3d; }
331
+ #__iux_session.active { background:linear-gradient(135deg,#7B2FBE,#FF2DF0); border-color:transparent; }
332
+ #__insightux_sidebar .__iux_hints {
333
+ margin-top:auto; color:#7a7288; font-size:9.5px; line-height:1.6; text-align:center;
334
+ }
335
+ #__insightux_update {
336
+ display:none; cursor:pointer; flex:0 0 auto; white-space:nowrap;
337
+ padding:5px 12px; border-radius:12px; font-weight:600; font-size:11px;
338
+ background:linear-gradient(135deg,#7B2FBE,#FF2DF0); color:#fff;
339
+ }
340
+ #__insightux_update:hover { filter:brightness(1.12); }
341
+ `;
342
+ document.head.appendChild(style);
343
+
344
+ const toolbar = document.createElement('div');
345
+ toolbar.id = '__insightux_toolbar';
346
+ toolbar.innerHTML = `
347
+ <button type="button" id="__iux_back" title="Back">&#8592;</button>
348
+ <button type="button" id="__iux_fwd" title="Forward">&#8594;</button>
349
+ <button type="button" id="__iux_reload" title="Reload">&#8635;</button>
350
+ <button type="button" id="__iux_home" title="Home">&#8962;</button>
351
+ <form id="__iux_addr_form"><input id="__iux_addr" type="text" autocomplete="off"
352
+ placeholder="Search or enter address"></form>
353
+ <span id="__insightux_update" title="Click to open the download page"></span>
354
+ <span id="__insightux_status">Press S to start tracking &middot; E to stop &middot; H heatmap &middot; M mouse panel &middot; X quit</span>
355
+ `;
356
+ document.documentElement.appendChild(toolbar);
357
+
358
+ const sidebar = document.createElement('div');
359
+ sidebar.id = '__insightux_sidebar';
360
+ sidebar.innerHTML = `
361
+ <button type="button" id="__iux_calibrate">&#128065;<br>Calibrate</button>
362
+ <button type="button" id="__iux_validate">&#127919;<br>Validate</button>
363
+ <button type="button" id="__iux_session">&#9654;<br>Start<br>Session</button>
364
+ <div class="__iux_hints">S/E start&#47;stop<br>H heatmap<br>M panel<br>X quit</div>
365
+ `;
366
+ document.documentElement.appendChild(sidebar);
367
+
368
+ setInterval(function(){
369
+ if (!document.documentElement.contains(toolbar)) document.documentElement.appendChild(toolbar);
370
+ if (!document.documentElement.contains(sidebar)) document.documentElement.appendChild(sidebar);
371
+ }, 1000);
372
 
373
  const statusEl = () => document.getElementById('__insightux_status');
374
  window.insightuxSetStatus = function(text){
375
  const el = statusEl();
376
  if (el) el.textContent = text;
377
  };
378
+ window.insightuxSetSessionUI = function(isTracking){
379
+ const btn = document.getElementById('__iux_session');
380
+ if (!btn) return;
381
+ btn.innerHTML = isTracking ? '&#9632;<br>Stop<br>Session' : '&#9654;<br>Start<br>Session';
382
+ btn.classList.toggle('active', isTracking);
383
+ };
384
+ window.insightuxShowUpdate = function(version, url){
385
+ const el = document.getElementById('__insightux_update');
386
+ if (!el) return;
387
+ el.textContent = 'Update available (v' + version + ')';
388
+ el.style.display = 'inline-block';
389
+ el.onclick = function(){
390
+ if (window.pywebview && window.pywebview.api && window.pywebview.api.open_update_page) {
391
+ window.pywebview.api.open_update_page(url);
392
+ }
393
+ };
394
+ };
395
+
396
+ // -- address bar / navigation (pure DOM history/location, no Python round-trip) --
397
+ const addrInput = document.getElementById('__iux_addr');
398
+ addrInput.value = isLanding ? '' : location.href;
399
+ document.getElementById('__iux_addr_form').addEventListener('submit', function(e){
400
+ e.preventDefault();
401
+ const raw = addrInput.value.trim();
402
+ if (!raw) return;
403
+ const looksLikeUrl = /^https?:\/\//i.test(raw) ||
404
+ (/^[\w-]+(\.[\w-]+)+([/?#].*)?$/i.test(raw) && !raw.includes(' '));
405
+ if (looksLikeUrl) {
406
+ location.href = raw.startsWith('http') ? raw : ('https://' + raw);
407
+ } else {
408
+ location.href = 'https://www.google.com/search?q=' + encodeURIComponent(raw);
409
+ }
410
+ });
411
+ document.getElementById('__iux_back').addEventListener('click', function(){ history.back(); });
412
+ document.getElementById('__iux_fwd').addEventListener('click', function(){ history.forward(); });
413
+ document.getElementById('__iux_reload').addEventListener('click', function(){ location.reload(); });
414
+ document.getElementById('__iux_home').addEventListener('click', function(){
415
+ location.href = __INSIGHTUX_LANDING_URL__;
416
+ });
417
+
418
+ // -- sidebar actions (Python decides validity — camera/session conflicts,
419
+ // wrong page — and reports back through the status text) --
420
+ function callApi(method){
421
+ if (window.pywebview && window.pywebview.api && window.pywebview.api[method]) {
422
+ window.pywebview.api[method]();
423
+ }
424
+ }
425
+ document.getElementById('__iux_calibrate').addEventListener('click', function(){ callApi('start_calibration'); });
426
+ document.getElementById('__iux_validate').addEventListener('click', function(){ callApi('run_validation'); });
427
+ document.getElementById('__iux_session').addEventListener('click', function(){ callApi('toggle_session'); });
428
 
429
  function isTypingTarget(el){
430
  if (!el) return false;
 
462
  }, true); // capture phase — fires before page scripts can intercept/stop the event
463
  })();
464
  """
465
+ CHROME_JS = CHROME_JS.replace("__INSIGHTUX_LANDING_URL__", json.dumps(LANDING_URL))
466
 
467
  # JS: tracking overlay (gaze dot + AOI highlighting) — injected only once
468
  # tracking actually starts. Ported directly from run_session.py's JS_SETUP.
469
  TRACKING_JS = r"""
470
  (function(){
471
  if (window.__insightux) { return; }
472
+ const OWN_UI_IDS = new Set(['__insightux_toolbar', '__insightux_sidebar', '__insightux_pill', '__insightux_canvas', '__insightux_mouse_panel']);
473
+ const OWN_UI_SELECTOR = '#__insightux_toolbar, #__insightux_sidebar, #__insightux_pill, #__insightux_canvas, #__insightux_mouse_panel';
474
  const state = { aois: [] };
475
  const dwell = { pendLabel: null, pendSince: 0, activeLabel: null, emptySince: 0 };
476
  const DWELL_MS = 420;
 
545
 
546
  function consider(el){
547
  if (full) return;
548
+ // Never track InsightUX's own injected UI (toolbar, sidebar, the
549
+ // gaze-dot canvas, the mouse tracker panel) as if it were page content.
550
+ if (OWN_UI_IDS.has(el.id)) return;
551
+ if (el.closest && el.closest(OWN_UI_SELECTOR)) return;
552
  const tag = el.tagName.toLowerCase();
553
  const explicit = !!(el.dataset && el.dataset.aoi);
554
  if (!explicit){
 
825
  }, 2000);
826
 
827
  // ---- exclude InsightUX's own injected UI from every measurement ----
828
+ // (the toolbar, sidebar, mouse-tracker panel, heatmap canvas) — only
829
  // real website elements should ever show up in trail/heatmap/dwell/clicks.
830
+ const OWN_UI_IDS = new Set(['__insightux_toolbar', '__insightux_sidebar', '__insightux_pill', '__insightux_canvas', '__insightux_mouse_panel']);
831
+ const OWN_UI_SELECTOR = '#__insightux_toolbar, #__insightux_sidebar, #__insightux_pill, #__insightux_canvas, #__insightux_mouse_panel';
832
  function isOwnUI(el){
833
  if (!el) return false;
834
+ if (OWN_UI_IDS.has(el.id)) return true;
835
+ return !!(el.closest && el.closest(OWN_UI_SELECTOR));
836
  }
837
 
838
  // ---- mouse position + hover tracking ----
 
1157
  # =============================================================================
1158
 
1159
  # Pages tracking must never start on, checked against window.get_current_url()
1160
+ # as a Python-side backstop to CHROME_JS's isTrackable check (belt and
1161
  # suspenders — the JS guard can't run before the JS bridge is up).
1162
  _NON_TRACKABLE_URL_MARKERS = ("insightux_landing.html", "analysis_report.html")
1163
 
1164
 
1165
+ def _relaunch_args(mode):
1166
+ """Command to spawn calibrate.py/validate.py's logic in a fresh process.
1167
+ Frozen (PyInstaller) builds have no bundled interpreter to hand a .py
1168
+ file to, and sys.executable IS the frozen exe itself — so a frozen build
1169
+ relaunches itself with --mode instead. Unpackaged/dev mode still runs
1170
+ from the venv exactly as before."""
1171
+ if getattr(sys, "frozen", False):
1172
+ return [sys.executable, "--mode", mode]
1173
+ return [sys.executable, os.path.abspath(__file__), "--mode", mode]
1174
+
1175
+
1176
  class Api:
1177
  def __init__(self):
1178
  self.window = None
 
1181
  self.thread = None
1182
  self.mouse_log_f = None
1183
  self.mouse_close_timer = None
1184
+ self.calib_proc = None
1185
+ self.validate_proc = None
1186
+ self.update_info = None
1187
+
1188
+ def _notify_update(self, version, url):
1189
+ try:
1190
+ self.window.evaluate_js(
1191
+ f"window.insightuxShowUpdate && window.insightuxShowUpdate("
1192
+ f"{json.dumps(version)}, {json.dumps(url)})"
1193
+ )
1194
+ except Exception:
1195
+ pass
1196
+
1197
+ def open_update_page(self, url):
1198
+ print(f"[browser_session] open_update_page() called from JS: {url}")
1199
+ import webbrowser
1200
+ try:
1201
+ webbrowser.open(url)
1202
+ except Exception as e:
1203
+ print(f"[browser_session] failed to open update url: {e}")
1204
+ return True
1205
+
1206
+ def toggle_session(self):
1207
+ """Single entry point for the sidebar's Start/Stop Session button —
1208
+ lets the JS button stay dumb (always call the same thing) while
1209
+ Python decides which action makes sense from current state."""
1210
+ return self.stop_tracking() if self.tracking else self.start_tracking()
1211
 
1212
  def start_tracking(self):
1213
  print("[browser_session] start_tracking() called from JS")
 
1222
  print(f"[browser_session] refusing to start on non-website page: {current_url}")
1223
  self._set_status("Start tracking only works on a website — search or open a page first.")
1224
  return False
1225
+ if self._external_proc_running():
1226
+ print("[browser_session] refusing to start, calibration/validation still running")
1227
+ self._set_status("Wait for calibration/validation to finish before starting a session.")
1228
+ return False
1229
  if not os.path.exists(CALIBRATION_PATH):
1230
  print(f"[browser_session] no {CALIBRATION_PATH} found — run calibrate.py first")
1231
  self._set_status("No calibration.pkl found — run calibrate.py first.")
 
1241
  self._inject_mouse_overlay()
1242
  self._open_mouse_log(session_dir)
1243
  self._set_mouse_tracking(True)
1244
+ self._set_session_ui(True)
1245
  self._set_status("Recording gaze + mouse... Press E to stop, H for heatmap, M for mouse panel")
1246
 
1247
  self.thread = threading.Thread(
 
1258
  return False
1259
  self._set_status("Wrapping up your session...")
1260
  self._set_mouse_tracking(False)
1261
+ self._set_session_ui(False)
1262
  self.stop_event.set()
1263
 
1264
  # Give the page a moment to flush its last batch of mouse data over
 
1276
  print("[browser_session] quit_app() called from JS -- terminating process")
1277
  os._exit(0)
1278
 
1279
+ # -- calibration / validation ------------------------------------------
1280
+ # calibrate.py and validate.py are unmodified, standalone OpenCV/pyautogui
1281
+ # scripts (their own fullscreen window, their own event loop) — they were
1282
+ # never meant to share a process with pywebview's GUI loop. Launching them
1283
+ # as a subprocess reuses them exactly as-is instead of rewriting their
1284
+ # display logic into the browser.
1285
+
1286
+ def _external_proc_running(self):
1287
+ running = lambda p: p is not None and p.poll() is None
1288
+ return running(self.calib_proc) or running(self.validate_proc)
1289
+
1290
+ def start_calibration(self):
1291
+ print("[browser_session] start_calibration() called from JS")
1292
+ if self.tracking:
1293
+ self._set_status("Stop the current session (E) before calibrating.")
1294
+ return False
1295
+ if self._external_proc_running():
1296
+ self._set_status("Calibration/validation is already running in another window.")
1297
+ return False
1298
+ try:
1299
+ self.calib_proc = subprocess.Popen(_relaunch_args("calibrate"), cwd=BASE_DIR)
1300
+ except Exception as e:
1301
+ print(f"[browser_session] failed to launch calibrate.py: {e}")
1302
+ self._set_status("Could not launch calibration — see terminal for details.")
1303
+ return False
1304
+ self._set_status("Calibration launched in a separate window — follow the dots there.")
1305
+ threading.Thread(
1306
+ target=self._watch_external_proc, args=(self.calib_proc, "Calibration"), daemon=True
1307
+ ).start()
1308
+ return True
1309
+
1310
+ def run_validation(self):
1311
+ print("[browser_session] run_validation() called from JS")
1312
+ if self.tracking:
1313
+ self._set_status("Stop the current session (E) before validating.")
1314
+ return False
1315
+ if self._external_proc_running():
1316
+ self._set_status("Calibration/validation is already running in another window.")
1317
+ return False
1318
+ if not os.path.exists(CALIBRATION_PATH):
1319
+ self._set_status("No calibration.pkl found — run Calibrate first.")
1320
+ return False
1321
+ try:
1322
+ self.validate_proc = subprocess.Popen(_relaunch_args("validate"), cwd=BASE_DIR)
1323
+ except Exception as e:
1324
+ print(f"[browser_session] failed to launch validate.py: {e}")
1325
+ self._set_status("Could not launch validation — see terminal for details.")
1326
+ return False
1327
+ self._set_status("Validation launched in a separate window — look at each dot.")
1328
+ threading.Thread(
1329
+ target=self._watch_external_proc, args=(self.validate_proc, "Validation"), daemon=True
1330
+ ).start()
1331
+ return True
1332
+
1333
+ def _watch_external_proc(self, proc, label):
1334
+ proc.wait()
1335
+ print(f"[browser_session] {label} process exited (code {proc.returncode})")
1336
+ if label == "Calibration":
1337
+ self._set_status("Calibration finished — check the terminal for the quality readout, then press S or Start Session.")
1338
+ else:
1339
+ self._set_status("Validation finished — check the terminal for the accuracy numbers.")
1340
+
1341
+ def _set_session_ui(self, is_tracking):
1342
+ try:
1343
+ flag = "true" if is_tracking else "false"
1344
+ self.window.evaluate_js(
1345
+ f"window.insightuxSetSessionUI && window.insightuxSetSessionUI({flag})"
1346
+ )
1347
+ except Exception:
1348
+ pass
1349
+
1350
  def _inject_tracking_overlay(self):
1351
  try:
1352
  self.window.evaluate_js(TRACKING_JS)
 
1414
  api = Api()
1415
 
1416
 
1417
+ def _version_tuple(v):
1418
+ return tuple(int(p) for p in v.split(".") if p.isdigit())
1419
+
1420
+
1421
+ def check_for_update():
1422
+ """Runs once in a background thread at startup. Publishing a new release
1423
+ is still fully manual (bump VERSION here + AppVersion in installer.iss +
1424
+ version.json, rebuild, push) — this only automates the *checking* side,
1425
+ per the confirmed scope. No internet / any failure here is silent:
1426
+ the user just doesn't see an update banner, never an error."""
1427
+ import urllib.request
1428
  try:
1429
+ with urllib.request.urlopen(VERSION_CHECK_URL, timeout=4) as resp:
1430
+ data = json.loads(resp.read().decode("utf-8"))
1431
+ latest = data.get("latest", "")
1432
+ url = data.get("url", "")
1433
+ if latest and url and _version_tuple(latest) > _version_tuple(VERSION):
1434
+ api.update_info = {"version": latest, "url": url}
1435
+ api._notify_update(latest, url)
1436
+ print(f"[browser_session] update available: {VERSION} -> {latest}")
1437
  except Exception as e:
1438
+ print(f"[browser_session] update check skipped: {e}")
1439
+
1440
+
1441
+ def on_page_loaded(window):
1442
+ """Fires on every page load (navigation, back/forward, reload — not just
1443
+ the first page). The chrome must be re-injected every time since each
1444
+ navigation is a fresh document. If a session is already running, the
1445
+ gaze/mouse overlays need the same treatment or navigating mid-session
1446
+ would silently kill the dot and the mouse tracker on the new page."""
1447
+ try:
1448
+ window.evaluate_js(CHROME_JS)
1449
+ except Exception as e:
1450
+ print(f"[browser_session] chrome inject failed (will retry): {e}")
1451
+
1452
+ if api.tracking:
1453
+ api._inject_tracking_overlay()
1454
+ api._inject_mouse_overlay()
1455
+ api._set_mouse_tracking(True)
1456
+ api._set_session_ui(True)
1457
+
1458
+ # CHROME_JS was just re-injected fresh on this page and has no memory of
1459
+ # an update found on a previous page — re-tell it if one was found.
1460
+ if api.update_info:
1461
+ api._notify_update(api.update_info["version"], api.update_info["url"])
1462
 
1463
 
1464
  # =============================================================================
 
1663
 
1664
 
1665
  if __name__ == "__main__":
1666
+ # A packaged build is one exe with no separate calibrate.py/validate.py
1667
+ # files to shell out to — --mode makes it a single self-dispatching
1668
+ # entry point instead. See _relaunch_args() above for the launch side.
1669
+ import argparse
1670
+ parser = argparse.ArgumentParser()
1671
+ parser.add_argument("--mode", choices=["browser", "calibrate", "validate"], default="browser")
1672
+ args = parser.parse_args()
1673
+
1674
+ if args.mode == "calibrate":
1675
+ import calibrate
1676
+ calibrate.main()
1677
+ elif args.mode == "validate":
1678
+ import validate
1679
+ validate.main()
1680
+ else:
1681
+ os.makedirs(SESSIONS_ROOT, exist_ok=True)
1682
+
1683
+ window = webview.create_window(
1684
+ "InsightUX — Eye-Tracking Research Browser", LANDING_URL,
1685
+ width=SCREEN_W, height=SCREEN_H, js_api=api
1686
+ )
1687
+ api.window = window
1688
+ window.events.loaded += lambda: on_page_loaded(window)
1689
+ window.events.shown += lambda: threading.Thread(
1690
+ target=_go_fullscreen, args=(window,), daemon=True
1691
+ ).start()
1692
+ threading.Thread(target=check_for_update, daemon=True).start()
1693
+
1694
+ # debug=True enables right-click > Inspect (DevTools) so you can see
1695
+ # the [insightux] console.log diagnostics from CHROME_JS directly —
1696
+ # keyboard handling runs entirely in the browser, so JS-side issues
1697
+ # never show up in this terminal, only in DevTools' Console tab.
1698
+ webview.start(debug=True)
calibrate.py CHANGED
@@ -4,10 +4,32 @@ import time
4
  import os
5
  import random
6
  import pyautogui
 
 
 
 
 
 
 
 
 
 
 
7
 
8
  import warnings
9
  warnings.filterwarnings("ignore", category=DeprecationWarning)
10
 
 
 
 
 
 
 
 
 
 
 
 
11
  from preprocessing.preprocessing_pipeline import (
12
  create_face_mesh,
13
  estimate_camera_matrix,
@@ -420,8 +442,8 @@ def filter_unreliable_points(gaze_vectors, screen_points, head_pitches_deg,
420
  # CONFIG
421
  # =============================================================================
422
 
423
- ONNX_PATH = "models/gaze_cnn_v4.onnx"
424
- CKPT_PATH = "checkpoints/best_model_v4.pt"
425
 
426
  # --- Fine-tune target geometry -------------------------------------------
427
  # USE_MEASURED_GEOMETRY=False keeps the original hand-picked constants
@@ -481,530 +503,535 @@ FRAMES_KEPT_PER_POINT = 12
481
  FINETUNE_ENABLED = True
482
 
483
 
484
- # =============================================================================
485
- # MAIN CALIBRATION
486
- # =============================================================================
487
-
488
- pipeline = InsightUXPipeline(ONNX_PATH)
489
- face_mesh = create_face_mesh(static_image_mode=False)
490
- cap = cv2.VideoCapture(0)
491
-
492
- cam_matrix = None
493
-
494
- gaze_vectors = [] # per-point median [pitch, yaw] (pre-fine-tune)
495
- screen_points = [] # per-point [sx, sy]
496
- head_pitches_deg = [] # per-point median head pitch (deg)
497
- point_dispersion = [] # per-point [mad_pitch, mad_yaw] <- the noise floor
498
- point_frames = [] # per-point dict of REAL diverse frames (see below)
499
- point_ear = [] # per-point median eye-aperture (candidate vertical cue)
500
-
501
- # Session-wide diagnostics, so we can warn about systemic issues at the end
502
- # rather than just per-point.
503
- session_brightness = []
504
- session_distance_mm = [] # viewing distance from solvePnP — real geometry, not a guess
505
- session_blink_skips = 0
506
- session_light_skips = 0
507
- points_with_low_yield = []
508
-
509
- cam_matrix_ref = [None]
510
- face_orientation_gate(face_mesh, cap, cam_matrix_ref)
511
- if cam_matrix_ref[0] is not None:
512
- cam_matrix = cam_matrix_ref[0]
513
-
514
- cv2.namedWindow("Calibration", cv2.WINDOW_NORMAL)
515
- cv2.setWindowProperty("Calibration", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
516
-
517
- total = len(points)
518
- print(f"Calibration started - {total} points")
519
-
520
- for idx, (px, py) in enumerate(points):
521
- sx, sy = int(px * SCREEN_W), int(py * SCREEN_H)
522
- duration = get_duration(py)
523
-
524
- settle_s = 1.5 if idx == 0 else 0.4
525
- settle_start = time.time()
526
- while time.time() - settle_start < settle_s:
527
- ret, frame = cap.read()
528
- if not ret:
529
- continue
530
- screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
531
- cv2.circle(screen, (sx, sy), 18, (80, 80, 80), -1)
532
- msg = "Get ready..." if idx == 0 else "Settling..."
533
- cv2.putText(screen, msg, (50, 50),
534
- cv2.FONT_HERSHEY_SIMPLEX, 1, (180, 180, 180), 2)
535
- cv2.imshow("Calibration", screen)
536
- cv2.waitKey(1)
537
-
538
- samples = []
539
- ear_list = []
540
- l_list = []
541
- r_list = []
542
- p_list = []
543
- hp_list = []
544
-
545
- point_blink_skips = 0
546
- point_light_skips = 0
547
-
548
- start = time.time()
549
-
550
- while time.time() - start < duration:
551
- ret, frame = cap.read()
552
- if not ret:
553
- continue
554
-
555
- if cam_matrix is None:
556
- cam_matrix = estimate_camera_matrix(frame.shape)
557
-
558
- # LIVE LIGHTING CHECK — skip this frame and tell the person, instead
559
- # of silently feeding a too-dark/too-bright frame into the model.
560
- light_ok, light_msg = check_lighting(frame)
561
- gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
562
- session_brightness.append(float(np.mean(gray_frame)))
563
- if not light_ok:
564
- point_light_skips += 1
565
- screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
566
- cv2.circle(screen, (sx, sy), 18, (0, 140, 255), 2)
567
- cv2.putText(screen, f"Point {idx+1}/{total} - {light_msg}",
568
- (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 140, 255), 2)
569
  cv2.imshow("Calibration", screen)
570
  cv2.waitKey(1)
571
- continue
572
 
573
- rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
574
- results = face_mesh.process(rgb)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
575
 
576
- if not results.multi_face_landmarks:
577
- continue
578
-
579
- lms = results.multi_face_landmarks[0].landmark
580
- head_pose = estimate_head_pose(lms, frame.shape, cam_matrix)
581
- if head_pose is None:
582
- continue
583
-
584
- # LIVE BLINK CHECK — average EAR across both eyes. Below threshold
585
- # means eyes are closed/closing; that frame's eye patches carry no
586
- # real gaze signal and would just add noise to this point's median.
587
- ear_l = compute_ear(lms, LEFT_EAR_INDICES, frame.shape)
588
- ear_r = compute_ear(lms, RIGHT_EAR_INDICES, frame.shape)
589
- avg_ear = (ear_l + ear_r) / 2.0
590
- if avg_ear < BLINK_EAR_THRESHOLD:
591
- point_blink_skips += 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
592
  screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
593
- cv2.circle(screen, (sx, sy), 18, (180, 180, 0), 2)
594
- cv2.putText(screen, f"Point {idx+1}/{total} - Blink detected, skipping frame",
595
- (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (180, 180, 0), 2)
 
 
 
 
 
596
  cv2.imshow("Calibration", screen)
597
- cv2.waitKey(1)
598
- continue
599
-
600
- # FIX A: normalize pose to [-1, +1] before CNN
601
- pose_vec = normalize_pose(head_pose)
602
 
603
- def process_eye(eye_idx, ear_idx, iris_idx):
604
- s1 = step1_normalize(frame, lms, head_pose, eye_idx, ear_idx, iris_idx)
605
- if not s1.is_open:
606
- return None
607
- ir = compute_iris_radius(lms, iris_idx, frame.shape)
608
- s2 = step2_illumination(s1, ir)
609
- return s2.blended if s2.is_usable else None
610
 
611
- l = process_eye(LEFT_EYE_INDICES, LEFT_EAR_INDICES, LEFT_IRIS_INDICES)
612
- r = process_eye(RIGHT_EYE_INDICES, RIGHT_EAR_INDICES, RIGHT_IRIS_INDICES)
613
 
614
- if l is None and r is None:
 
 
615
  continue
616
- if l is None: l = r
617
- if r is None: r = l
618
-
619
- _, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(l, pose_vec, r)
620
-
621
- pitch = compensate_pitch(raw_pitch, head_pose.pitch)
622
- yaw = compensate_yaw(raw_yaw, head_pose.yaw)
623
-
624
- samples.append([pitch, yaw])
625
- ear_list.append(avg_ear)
626
- # solvePnP's tvec is in the same units as FACE_3D_MODEL (mm), so
627
- # tvec[2] is a genuine measurement of how far your face is from the
628
- # camera. We were computing this every frame and throwing it away.
629
- try:
630
- session_distance_mm.append(float(head_pose.tvec[2]))
631
- except Exception:
632
- pass
633
- l_list.append(l)
634
- r_list.append(r)
635
- p_list.append(pose_vec)
636
- hp_list.append(head_pose.pitch)
637
-
638
- elapsed = time.time() - start
639
- screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
640
- angle = int(360 * elapsed / duration)
641
- cv2.ellipse(screen, (sx, sy), (28, 28), -90, 0, angle, (0, 180, 0), 3)
642
- cv2.circle(screen, (sx, sy), 18, (0, 255, 0), -1)
643
- cv2.putText(screen, f"Point {idx+1}/{total} - Look at the dot",
644
- (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
645
- if point_blink_skips or point_light_skips:
646
- cv2.putText(screen, f"skipped: {point_blink_skips} blink, {point_light_skips} lighting",
647
- (50, 85), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (140, 140, 140), 1)
648
- cv2.imshow("Calibration", screen)
649
-
650
- if cv2.waitKey(1) & 0xFF == 27:
651
- break
652
-
653
- session_blink_skips += point_blink_skips
654
- session_light_skips += point_light_skips
655
-
656
- if not samples:
657
- print(f"Point {idx+1}: no samples, skipping.")
658
- points_with_low_yield.append(idx + 1)
659
- continue
660
-
661
- if len(samples) < MIN_SAMPLES_OK:
662
- print(f" ⚠ Point {idx+1}: only {len(samples)} valid samples "
663
- f"({point_blink_skips} blink-skipped, {point_light_skips} light-skipped) — may be unreliable")
664
- points_with_low_yield.append(idx + 1)
665
-
666
- samples_arr = np.array(samples)
667
- avg = np.median(samples_arr, axis=0)
668
- avg_hp_deg = float(np.median(hp_list))
669
-
670
- # WITHIN-POINT DISPERSION = the noise floor. While you stared at ONE
671
- # fixed dot, how much did the model's predicted pitch/yaw wobble
672
- # frame-to-frame? This is the single most important number in the whole
673
- # calibration and it was never being measured. Compare it against the
674
- # BETWEEN-point spread (the actual signal) at the end of the run: if
675
- # noise >= signal on an axis, that axis is unrecoverable by any
676
- # calibration math, and no amount of RBF tuning will fix it.
677
- # MAD (median absolute deviation) rather than std, so one blink-tail
678
- # frame can't inflate it.
679
- mad_pitch = float(np.median(np.abs(samples_arr[:, 0] - avg[0])))
680
- mad_yaw = float(np.median(np.abs(samples_arr[:, 1] - avg[1])))
681
-
682
- gaze_vectors.append(list(avg))
683
- screen_points.append([sx, sy])
684
- head_pitches_deg.append(avg_hp_deg)
685
- point_dispersion.append([mad_pitch, mad_yaw])
686
- # Eye aperture for this point. Looking DOWN lowers the eyelid, so this is
687
- # a physically independent vertical cue — and the CNN's pitch output has
688
- # proven nearly blind vertically (r~0.41). calibrate() will measure both
689
- # and pick whichever actually tracks screen-Y.
690
- point_ear.append(float(np.median(ear_list)))
691
-
692
- # Keep a BUNDLE of genuinely different frames for this point — not one
693
- # "representative" frame. Both the fine-tune and the post-fine-tune RBF
694
- # refit read from this bundle, so both get real frame diversity and real
695
- # noise averaging instead of anchoring on a single arbitrary frame.
696
- n_avail = len(l_list)
697
- n_take = min(FRAMES_KEPT_PER_POINT, n_avail)
698
- indices = np.linspace(0, n_avail - 1, n_take, dtype=int)
699
- point_frames.append({
700
- "lefts": [l_list[i] for i in indices],
701
- "rights": [r_list[i] for i in indices],
702
- "poses": [p_list[i] for i in indices],
703
- "hps": [hp_list[i] for i in indices],
704
- })
705
-
706
- print(f"Point {idx+1:2d}/{total} | pitch={avg[0]:+.4f} yaw={avg[1]:+.4f} "
707
- f"| jitter(pitch)={mad_pitch:.4f} jitter(yaw)={mad_yaw:.4f} "
708
- f"| {len(indices)} frames kept "
709
- f"| skipped: {point_blink_skips} blink, {point_light_skips} lighting")
710
-
711
- cap.release()
712
- cv2.destroyAllWindows()
713
-
714
- if len(gaze_vectors) < 4:
715
- print(f"ERROR: only {len(gaze_vectors)} points collected, need at least 4.")
716
- exit(1)
717
 
718
- # =============================================================================
719
- # OUTLIER FILTER
720
- # =============================================================================
721
- (gaze_vectors, screen_points, head_pitches_deg,
722
- point_dispersion, point_frames, point_ear) = filter_unreliable_points(
723
- gaze_vectors, screen_points, head_pitches_deg,
724
- point_dispersion, point_frames, point_ear, factor=3.0
725
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
726
 
727
- n_pts = len(screen_points)
728
- if n_pts < 4:
729
- print("ERROR: too many points removed, need at least 4 reliable points.")
730
- exit(1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
731
 
732
- # =============================================================================
733
- # FINE-TUNE (optional see FINETUNE_ENABLED comment for why it defaults OFF)
734
- # Now fed the REAL diverse frames from each point's bundle. Previously this
735
- # received the same single frame duplicated 5x per point, so it trained on
736
- # 16 unique images while reporting 80.
737
- # =============================================================================
738
- finetuned = False
739
- if FINETUNE_ENABLED:
740
- ft_lefts, ft_rights, ft_poses, ft_sp = [], [], [], []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
741
  for i in range(n_pts):
742
  pf = point_frames[i]
 
743
  for k in range(len(pf["lefts"])):
744
- ft_lefts.append(pf["lefts"][k])
745
- ft_rights.append(pf["rights"][k])
746
- ft_poses.append(pf["poses"][k])
747
- ft_sp.append(screen_points[i])
748
-
749
- n_unique = len(ft_lefts)
750
-
751
- # Derive the fine-tune target geometry from REAL numbers instead of the
752
- # fabricated 0.6/0.4 constants.
753
- _diag_mm = SCREEN_DIAGONAL_INCHES * 25.4
754
- _aspect = SCREEN_W / float(SCREEN_H)
755
- _scr_h_mm = _diag_mm / np.sqrt(_aspect ** 2 + 1.0)
756
- _scr_w_mm = _aspect * _scr_h_mm
757
- if session_distance_mm:
758
- _dist_mm = float(np.median(session_distance_mm))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
759
  else:
760
- _dist_mm = 500.0
761
- # Guard against a wild solvePnP outlier producing nonsense geometry.
762
- _dist_mm = float(np.clip(_dist_mm, 300.0, 900.0))
763
-
764
- if USE_MEASURED_GEOMETRY:
765
- K_H = (_scr_w_mm / 2.0) / _dist_mm
766
- K_V = (_scr_h_mm / 2.0) / _dist_mm
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
767
  else:
768
- K_H, K_V = 0.6, 0.4 # original constants currently working
769
-
770
- print(f"\n--- Fine-tuning CNN on your eyes ({n_unique} REAL frames, "
771
- f"{n_pts} points) ---")
772
- print(f"[Geometry] screen {SCREEN_DIAGONAL_INCHES}\" -> "
773
- f"{_scr_w_mm:.0f}x{_scr_h_mm:.0f}mm | measured viewing distance "
774
- f"{_dist_mm:.0f}mm")
775
- print(f"[Geometry] fine-tune targets: k_h={K_H:.3f} k_v={K_V:.3f} "
776
- f"(ratio {K_H/max(K_V,1e-6):.2f}) "
777
- f"[{'measured' if USE_MEASURED_GEOMETRY else 'original constants'}]")
778
- finetuned = finetune_on_calibration_v4(
779
- onnx_path = ONNX_PATH,
780
- left_patches = ft_lefts,
781
- right_patches = ft_rights,
782
- head_poses = ft_poses,
783
- screen_points = ft_sp,
784
- screen_w = SCREEN_W,
785
- screen_h = SCREEN_H,
786
- ckpt_path = CKPT_PATH,
787
- out_onnx_path = ONNX_PATH,
788
- steps = 200,
789
- lr = 1e-4,
790
- k_h = K_H,
791
- k_v = K_V,
792
- )
793
- else:
794
- print("\n--- Fine-tuning SKIPPED (FINETUNE_ENABLED = False) ---")
795
- print(" Using the base model as-is. The RBF maps its output to screen")
796
- print(" coordinates, so the model's absolute scale does not need to be")
797
- print(" 'correct' only monotonic. See the FINETUNE_ENABLED comment.")
798
-
799
- # =============================================================================
800
- # FIT RBF CALIBRATION
801
- #
802
- # CRITICAL FIX: re-predict across EVERY kept frame for each point and take the
803
- # MEDIAN, instead of predicting from one arbitrary "representative" frame.
804
- #
805
- # The old code computed a careful median over 50-150 frames during collection,
806
- # then threw it away and refit the RBF from a single frame per point. That
807
- # discarded all noise averaging and anchored the entire calibration on 16
808
- # single, possibly-noisy frames which is exactly the kind of thing that
809
- # makes edges and corners unstable.
810
- # =============================================================================
811
- print("\n--- Fitting RBF calibration ---")
812
- pipeline = InsightUXPipeline(ONNX_PATH)
813
-
814
- adapted_gaze_vectors = []
815
- adapted_dispersion = []
816
- for i in range(n_pts):
817
- pf = point_frames[i]
818
- per_frame = []
819
- for k in range(len(pf["lefts"])):
820
- l = pf["lefts"][k]
821
- r = pf["rights"][k]
822
- pose = pf["poses"][k]
823
- hp = pf["hps"][k]
824
- hy = float(pose[1]) * POSE_NORM_SCALE # recover raw head yaw (deg)
825
-
826
- _, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(l, pose, r)
827
- pitch = compensate_pitch(raw_pitch, hp)
828
- yaw = compensate_yaw(raw_yaw, hy)
829
- per_frame.append([pitch, yaw])
830
-
831
- per_frame = np.array(per_frame)
832
- med = np.median(per_frame, axis=0)
833
- adapted_gaze_vectors.append(list(med))
834
- adapted_dispersion.append([
835
- float(np.median(np.abs(per_frame[:, 0] - med[0]))),
836
- float(np.median(np.abs(per_frame[:, 1] - med[1]))),
837
- ])
838
-
839
- assert len(adapted_gaze_vectors) == len(screen_points), \
840
- f"Length mismatch: {len(adapted_gaze_vectors)} gaze vs {len(screen_points)} screen"
841
-
842
- pipeline.calibration.calibrate(
843
- np.array(adapted_gaze_vectors),
844
- np.array(screen_points),
845
- ear=np.array(point_ear),
846
- screen_size=(SCREEN_W, SCREEN_H) # <-- fixes the 1493x933 clamp bug
847
- )
848
- pipeline.calibration.save("calibration.pkl")
849
-
850
- import pickle
851
- _all_hp = [hp for pf in point_frames for hp in pf["hps"]]
852
- _all_pose = [p for pf in point_frames for p in pf["poses"]]
853
- baseline_pitch = float(np.median(_all_hp)) if _all_hp else 0.0
854
- baseline_yaw = float(np.median([float(p[1]) * POSE_NORM_SCALE for p in _all_pose])) if _all_pose else 0.0
855
- baseline_roll = float(np.median([float(p[2]) * POSE_NORM_SCALE for p in _all_pose])) if _all_pose else 0.0
856
- with open("baseline_pose.pkl", "wb") as f:
857
- pickle.dump({"pitch": baseline_pitch, "yaw": baseline_yaw, "roll": baseline_roll}, f)
858
- print(f"Saved baseline_pose.pkl: pitch={baseline_pitch:+.2f}, yaw={baseline_yaw:+.2f}, roll={baseline_roll:+.2f}")
859
-
860
- print(f"\nCalibration complete! {n_pts}/{total} points used.")
861
- if finetuned:
862
- print("CNN fine-tuned on your eyes + RBF calibration fitted.")
863
- else:
864
- print("RBF calibration fitted (CNN fine-tuning skipped).")
865
-
866
- # =============================================================================
867
- # SESSION DIAGNOSTIC SUMMARY — the "tell the person if something's off" ask.
868
- # Printed once, in plain language, instead of them having to interpret
869
- # per-point numbers themselves.
870
- # =============================================================================
871
- # =============================================================================
872
- # SIGNAL-TO-NOISE — THE VERDICT
873
- #
874
- # This is the number that decides whether an axis is fixable at all.
875
- #
876
- # SIGNAL = how much the model's output changes between DIFFERENT screen
877
- # positions (spread of the per-point medians). This is the real
878
- # information the RBF has to work with.
879
- # NOISE = how much the model's output wobbles frame-to-frame while you
880
- # stare at ONE fixed dot (median within-point jitter).
881
- #
882
- # If NOISE >= SIGNAL on an axis, the model cannot tell "you looked lower"
883
- # apart from "the same look, one frame later". No RBF, no gain correction,
884
- # no clamp tuning can recover information that was never there. That axis
885
- # needs a better MODEL, not better calibration math.
886
- # =============================================================================
887
- gv_arr = np.array(adapted_gaze_vectors)
888
- disp_arr = np.array(adapted_dispersion)
889
- sp_arr = np.array(screen_points)
890
-
891
- signal_pitch = float(np.std(gv_arr[:, 0]))
892
- signal_yaw = float(np.std(gv_arr[:, 1]))
893
- noise_pitch = float(np.median(disp_arr[:, 0]))
894
- noise_yaw = float(np.median(disp_arr[:, 1]))
895
-
896
- snr_pitch = signal_pitch / noise_pitch if noise_pitch > 1e-9 else float("inf")
897
- snr_yaw = signal_yaw / noise_yaw if noise_yaw > 1e-9 else float("inf")
898
-
899
- # CORRELATION — the metric that actually decides usability.
900
- #
901
- # An earlier version of this block reported only SNR (spread vs jitter). That
902
- # was MISLEADING and nearly sent us chasing the wrong fix: a signal can have
903
- # plenty of spread and still be useless if that spread is not ORDERED by the
904
- # thing you're predicting. Pitch once reported SNR 1.76 ("marginal") while its
905
- # per-row averages went down, up, up — i.e. no monotonic relationship with
906
- # screen row at all. Spread was real; ordering was not.
907
- #
908
- # Pearson r between model output and true screen coordinate is the honest
909
- # test. r near +/-1 means the model tracks that axis. r near 0 means it does
910
- # not, no matter how much spread there is.
911
- def _pearson(a, b):
912
- a = np.asarray(a, dtype=float); b = np.asarray(b, dtype=float)
913
- if a.std() < 1e-12 or b.std() < 1e-12:
914
- return 0.0
915
- return float(np.corrcoef(a, b)[0, 1])
916
-
917
- r_yaw = _pearson(gv_arr[:, 1], sp_arr[:, 0]) # yaw vs screen X
918
- r_pitch = _pearson(gv_arr[:, 0], sp_arr[:, 1]) # pitch vs screen Y
919
- r_ear = _pearson(np.array(point_ear), sp_arr[:, 1]) # eye aperture vs screen Y
920
-
921
- def _corr_verdict(r):
922
- ar = abs(r)
923
- if ar >= 0.90:
924
- return "EXCELLENT — model tracks this axis cleanly"
925
- if ar >= 0.70:
926
- return "GOOD — usable, some slop"
927
- if ar >= 0.50:
928
- return "WEAK — expect significant error on this axis"
929
- return "BROKEN — model output barely relates to this axis at all"
930
-
931
- print("\n========== DOES THE MODEL TRACK THE SCREEN? (the real test) ==========")
932
- print(f"HORIZONTAL yaw vs screen-X : r = {r_yaw:+.3f} (SNR {snr_yaw:.2f})")
933
- print(f" -> {_corr_verdict(r_yaw)}")
934
- print(f"VERTICAL pitch vs screen-Y: r = {r_pitch:+.3f} (SNR {snr_pitch:.2f})")
935
- print(f" -> {_corr_verdict(r_pitch)}")
936
- print(f"VERTICAL EYE APERTURE vs screen-Y: r = {r_ear:+.3f}")
937
- print(f" -> {_corr_verdict(r_ear)}")
938
- print()
939
- _best_vert = max(abs(r_pitch), abs(r_ear))
940
- print(f"Best available vertical cue: "
941
- f"{'EYE APERTURE' if abs(r_ear) > abs(r_pitch) else 'CNN PITCH'} "
942
- f"(r={_best_vert:+.3f})")
943
- print()
944
- if _best_vert < 0.5:
945
- print("VERDICT: the model's PITCH output does not meaningfully track where you")
946
- print("look vertically. This is NOT a calibration problem — the RBF cannot map")
947
- print("an input that carries no ordered information about screen height. No")
948
- print("amount of clamp/smoothing/gain tuning will fix it. The fix is the MODEL:")
949
- print("its pitch head needs retraining, or vertical gaze needs a different")
950
- print("feature (e.g. eyelid aperture / iris-centre offset within the socket),")
951
- print("which the current eye-patch CNN is evidently not learning.")
952
- elif _best_vert < abs(r_yaw) - 0.15:
953
- print("VERDICT: vertical tracks the screen, but noticeably worse than")
954
- print("horizontal. Calibration is doing its job; expect up/down to stay the")
955
- print("looser axis until the model improves.")
956
- else:
957
- print("VERDICT: both axes track the screen. Any remaining error is in the")
958
- print("calibration mapping or the noise floor, not in the model's ability")
959
- print("to see where you're looking.")
960
- print("=====================================================================")
961
-
962
- print("\n================ CALIBRATION QUALITY SUMMARY ================")
963
- if session_brightness:
964
- avg_bright = float(np.mean(session_brightness))
965
- pct_dark = 100 * sum(1 for b in session_brightness if b < LIGHT_MIN_BRIGHTNESS) / len(session_brightness)
966
- pct_bright = 100 * sum(1 for b in session_brightness if b > LIGHT_MAX_BRIGHTNESS) / len(session_brightness)
967
- print(f"Average brightness: {avg_bright:.0f} (comfortable range: {LIGHT_MIN_BRIGHTNESS}-{LIGHT_MAX_BRIGHTNESS})")
968
- if pct_dark > 10:
969
- print(f"⚠ Lighting was too DARK for {pct_dark:.0f}% of frames. "
970
- f"Add a light source facing your face, or face a window, before recalibrating.")
971
- if pct_bright > 10:
972
- print(f"⚠ Lighting was too BRIGHT for {pct_bright:.0f}% of frames "
973
- f"(backlight or a light directly behind you?). Try facing away from strong light sources.")
974
- if pct_dark <= 10 and pct_bright <= 10:
975
- print("Lighting was consistently good throughout.")
976
-
977
- if session_blink_skips > 0:
978
- print(f"Blinking accounted for {session_blink_skips} skipped frames across the session "
979
- f"— normal, this is expected and was handled automatically.")
980
-
981
- if points_with_low_yield:
982
- print(f"⚠ These points had low sample counts and may be less accurate: "
983
- f"{', '.join(str(p) for p in points_with_low_yield)}. "
984
- f"If tracking feels off in that part of the screen, consider recalibrating.")
985
- else:
986
- print("All points collected a healthy number of samples.")
987
- print("===============================================================")
988
-
989
- # --- Diagnostic: verify vertical pitch separation ---
990
- print("\n--- Pitch by screen row (should increase top -> bottom) ---")
991
- gv_arr = np.array(adapted_gaze_vectors)
992
- sp_arr = np.array(screen_points)
993
- thresholds = [(0, SCREEN_H*0.25, "Top (y<25%) "),
994
- (SCREEN_H*0.25, SCREEN_H*0.5, "Mid-hi (25-50%) "),
995
- (SCREEN_H*0.5, SCREEN_H*0.75, "Mid-lo (50-75%) "),
996
- (SCREEN_H*0.75, SCREEN_H+1, "Bottom (y>75%) ")]
997
- for lo, hi, label in thresholds:
998
- mask = (sp_arr[:, 1] >= lo) & (sp_arr[:, 1] < hi)
999
- if mask.any():
1000
- print(f" {label}: avg pitch = {gv_arr[mask, 0].mean():.4f}")
1001
-
1002
- print("\n--- Yaw by screen column (should increase left -> right) ---")
1003
- col_thresholds = [(0, SCREEN_W*0.25, "Left (x<25%) "),
1004
- (SCREEN_W*0.25, SCREEN_W*0.5, "Mid-lf (25-50%) "),
1005
- (SCREEN_W*0.5, SCREEN_W*0.75, "Mid-rt (50-75%) "),
1006
- (SCREEN_W*0.75, SCREEN_W+1, "Right (x>75%) ")]
1007
- for lo, hi, label in col_thresholds:
1008
- mask = (sp_arr[:, 0] >= lo) & (sp_arr[:, 0] < hi)
1009
- if mask.any():
1010
- print(f" {label}: avg yaw = {gv_arr[mask, 1].mean():.4f}")
 
4
  import os
5
  import random
6
  import pyautogui
7
+ import sys
8
+
9
+ # Windows consoles default to a non-UTF-8 codepage (cp1252). A single
10
+ # unicode character in a print() (the "⚠" warning glyph below, for example)
11
+ # then crashes the whole calibration run with UnicodeEncodeError — this is
12
+ # not hypothetical, it happened mid-session. errors="replace" means an
13
+ # unencodable glyph degrades to "?" instead of killing the process.
14
+ if hasattr(sys.stdout, "reconfigure"):
15
+ sys.stdout.reconfigure(encoding="utf-8", errors="replace")
16
+ if hasattr(sys.stderr, "reconfigure"):
17
+ sys.stderr.reconfigure(encoding="utf-8", errors="replace")
18
 
19
  import warnings
20
  warnings.filterwarnings("ignore", category=DeprecationWarning)
21
 
22
+ # Bundled read-only assets (models/checkpoints) vs. per-user writable data
23
+ # (calibration.pkl, baseline_pose.pkl) need different roots once frozen by
24
+ # PyInstaller (onedir nests bundled data under _internal/, alongside a
25
+ # persistent folder holding the actual InsightUX.exe).
26
+ if getattr(sys, "frozen", False):
27
+ RESOURCE_DIR = sys._MEIPASS
28
+ DATA_DIR = os.path.dirname(sys.executable)
29
+ else:
30
+ RESOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
31
+ DATA_DIR = RESOURCE_DIR
32
+
33
  from preprocessing.preprocessing_pipeline import (
34
  create_face_mesh,
35
  estimate_camera_matrix,
 
442
  # CONFIG
443
  # =============================================================================
444
 
445
+ ONNX_PATH = os.path.join(RESOURCE_DIR, "models", "gaze_cnn_v4.onnx")
446
+ CKPT_PATH = os.path.join(RESOURCE_DIR, "checkpoints", "best_model_v4.pt")
447
 
448
  # --- Fine-tune target geometry -------------------------------------------
449
  # USE_MEASURED_GEOMETRY=False keeps the original hand-picked constants
 
503
  FINETUNE_ENABLED = True
504
 
505
 
506
+ def main():
507
+ # =============================================================================
508
+ # MAIN CALIBRATION
509
+ # =============================================================================
510
+
511
+ pipeline = InsightUXPipeline(ONNX_PATH)
512
+ face_mesh = create_face_mesh(static_image_mode=False)
513
+ cap = cv2.VideoCapture(0)
514
+
515
+ cam_matrix = None
516
+
517
+ gaze_vectors = [] # per-point median [pitch, yaw] (pre-fine-tune)
518
+ screen_points = [] # per-point [sx, sy]
519
+ head_pitches_deg = [] # per-point median head pitch (deg)
520
+ point_dispersion = [] # per-point [mad_pitch, mad_yaw] <- the noise floor
521
+ point_frames = [] # per-point dict of REAL diverse frames (see below)
522
+ point_ear = [] # per-point median eye-aperture (candidate vertical cue)
523
+
524
+ # Session-wide diagnostics, so we can warn about systemic issues at the end
525
+ # rather than just per-point.
526
+ session_brightness = []
527
+ session_distance_mm = [] # viewing distance from solvePnP — real geometry, not a guess
528
+ session_blink_skips = 0
529
+ session_light_skips = 0
530
+ points_with_low_yield = []
531
+
532
+ cam_matrix_ref = [None]
533
+ face_orientation_gate(face_mesh, cap, cam_matrix_ref)
534
+ if cam_matrix_ref[0] is not None:
535
+ cam_matrix = cam_matrix_ref[0]
536
+
537
+ cv2.namedWindow("Calibration", cv2.WINDOW_NORMAL)
538
+ cv2.setWindowProperty("Calibration", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
539
+
540
+ total = len(points)
541
+ print(f"Calibration started - {total} points")
542
+
543
+ for idx, (px, py) in enumerate(points):
544
+ sx, sy = int(px * SCREEN_W), int(py * SCREEN_H)
545
+ duration = get_duration(py)
546
+
547
+ settle_s = 1.5 if idx == 0 else 0.4
548
+ settle_start = time.time()
549
+ while time.time() - settle_start < settle_s:
550
+ ret, frame = cap.read()
551
+ if not ret:
552
+ continue
553
+ screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
554
+ cv2.circle(screen, (sx, sy), 18, (80, 80, 80), -1)
555
+ msg = "Get ready..." if idx == 0 else "Settling..."
556
+ cv2.putText(screen, msg, (50, 50),
557
+ cv2.FONT_HERSHEY_SIMPLEX, 1, (180, 180, 180), 2)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
558
  cv2.imshow("Calibration", screen)
559
  cv2.waitKey(1)
 
560
 
561
+ samples = []
562
+ ear_list = []
563
+ l_list = []
564
+ r_list = []
565
+ p_list = []
566
+ hp_list = []
567
+
568
+ point_blink_skips = 0
569
+ point_light_skips = 0
570
+
571
+ start = time.time()
572
+
573
+ while time.time() - start < duration:
574
+ ret, frame = cap.read()
575
+ if not ret:
576
+ continue
577
+
578
+ if cam_matrix is None:
579
+ cam_matrix = estimate_camera_matrix(frame.shape)
580
+
581
+ # LIVE LIGHTING CHECK — skip this frame and tell the person, instead
582
+ # of silently feeding a too-dark/too-bright frame into the model.
583
+ light_ok, light_msg = check_lighting(frame)
584
+ gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
585
+ session_brightness.append(float(np.mean(gray_frame)))
586
+ if not light_ok:
587
+ point_light_skips += 1
588
+ screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
589
+ cv2.circle(screen, (sx, sy), 18, (0, 140, 255), 2)
590
+ cv2.putText(screen, f"Point {idx+1}/{total} - {light_msg}",
591
+ (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 140, 255), 2)
592
+ cv2.imshow("Calibration", screen)
593
+ cv2.waitKey(1)
594
+ continue
595
+
596
+ rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
597
+ results = face_mesh.process(rgb)
598
+
599
+ if not results.multi_face_landmarks:
600
+ continue
601
 
602
+ lms = results.multi_face_landmarks[0].landmark
603
+ head_pose = estimate_head_pose(lms, frame.shape, cam_matrix)
604
+ if head_pose is None:
605
+ continue
606
+
607
+ # LIVE BLINK CHECK — average EAR across both eyes. Below threshold
608
+ # means eyes are closed/closing; that frame's eye patches carry no
609
+ # real gaze signal and would just add noise to this point's median.
610
+ ear_l = compute_ear(lms, LEFT_EAR_INDICES, frame.shape)
611
+ ear_r = compute_ear(lms, RIGHT_EAR_INDICES, frame.shape)
612
+ avg_ear = (ear_l + ear_r) / 2.0
613
+ if avg_ear < BLINK_EAR_THRESHOLD:
614
+ point_blink_skips += 1
615
+ screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
616
+ cv2.circle(screen, (sx, sy), 18, (180, 180, 0), 2)
617
+ cv2.putText(screen, f"Point {idx+1}/{total} - Blink detected, skipping frame",
618
+ (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (180, 180, 0), 2)
619
+ cv2.imshow("Calibration", screen)
620
+ cv2.waitKey(1)
621
+ continue
622
+
623
+ # FIX A: normalize pose to [-1, +1] before CNN
624
+ pose_vec = normalize_pose(head_pose)
625
+
626
+ def process_eye(eye_idx, ear_idx, iris_idx):
627
+ s1 = step1_normalize(frame, lms, head_pose, eye_idx, ear_idx, iris_idx)
628
+ if not s1.is_open:
629
+ return None
630
+ ir = compute_iris_radius(lms, iris_idx, frame.shape)
631
+ s2 = step2_illumination(s1, ir)
632
+ return s2.blended if s2.is_usable else None
633
+
634
+ l = process_eye(LEFT_EYE_INDICES, LEFT_EAR_INDICES, LEFT_IRIS_INDICES)
635
+ r = process_eye(RIGHT_EYE_INDICES, RIGHT_EAR_INDICES, RIGHT_IRIS_INDICES)
636
+
637
+ if l is None and r is None:
638
+ continue
639
+ if l is None: l = r
640
+ if r is None: r = l
641
+
642
+ _, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(l, pose_vec, r)
643
+
644
+ pitch = compensate_pitch(raw_pitch, head_pose.pitch)
645
+ yaw = compensate_yaw(raw_yaw, head_pose.yaw)
646
+
647
+ samples.append([pitch, yaw])
648
+ ear_list.append(avg_ear)
649
+ # solvePnP's tvec is in the same units as FACE_3D_MODEL (mm), so
650
+ # tvec[2] is a genuine measurement of how far your face is from the
651
+ # camera. We were computing this every frame and throwing it away.
652
+ try:
653
+ session_distance_mm.append(float(head_pose.tvec[2]))
654
+ except Exception:
655
+ pass
656
+ l_list.append(l)
657
+ r_list.append(r)
658
+ p_list.append(pose_vec)
659
+ hp_list.append(head_pose.pitch)
660
+
661
+ elapsed = time.time() - start
662
  screen = np.zeros((SCREEN_H, SCREEN_W, 3), dtype=np.uint8)
663
+ angle = int(360 * elapsed / duration)
664
+ cv2.ellipse(screen, (sx, sy), (28, 28), -90, 0, angle, (0, 180, 0), 3)
665
+ cv2.circle(screen, (sx, sy), 18, (0, 255, 0), -1)
666
+ cv2.putText(screen, f"Point {idx+1}/{total} - Look at the dot",
667
+ (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
668
+ if point_blink_skips or point_light_skips:
669
+ cv2.putText(screen, f"skipped: {point_blink_skips} blink, {point_light_skips} lighting",
670
+ (50, 85), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (140, 140, 140), 1)
671
  cv2.imshow("Calibration", screen)
 
 
 
 
 
672
 
673
+ if cv2.waitKey(1) & 0xFF == 27:
674
+ break
 
 
 
 
 
675
 
676
+ session_blink_skips += point_blink_skips
677
+ session_light_skips += point_light_skips
678
 
679
+ if not samples:
680
+ print(f"Point {idx+1}: no samples, skipping.")
681
+ points_with_low_yield.append(idx + 1)
682
  continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
683
 
684
+ if len(samples) < MIN_SAMPLES_OK:
685
+ print(f" ⚠ Point {idx+1}: only {len(samples)} valid samples "
686
+ f"({point_blink_skips} blink-skipped, {point_light_skips} light-skipped) — may be unreliable")
687
+ points_with_low_yield.append(idx + 1)
688
+
689
+ samples_arr = np.array(samples)
690
+ avg = np.median(samples_arr, axis=0)
691
+ avg_hp_deg = float(np.median(hp_list))
692
+
693
+ # WITHIN-POINT DISPERSION = the noise floor. While you stared at ONE
694
+ # fixed dot, how much did the model's predicted pitch/yaw wobble
695
+ # frame-to-frame? This is the single most important number in the whole
696
+ # calibration and it was never being measured. Compare it against the
697
+ # BETWEEN-point spread (the actual signal) at the end of the run: if
698
+ # noise >= signal on an axis, that axis is unrecoverable by any
699
+ # calibration math, and no amount of RBF tuning will fix it.
700
+ # MAD (median absolute deviation) rather than std, so one blink-tail
701
+ # frame can't inflate it.
702
+ mad_pitch = float(np.median(np.abs(samples_arr[:, 0] - avg[0])))
703
+ mad_yaw = float(np.median(np.abs(samples_arr[:, 1] - avg[1])))
704
+
705
+ gaze_vectors.append(list(avg))
706
+ screen_points.append([sx, sy])
707
+ head_pitches_deg.append(avg_hp_deg)
708
+ point_dispersion.append([mad_pitch, mad_yaw])
709
+ # Eye aperture for this point. Looking DOWN lowers the eyelid, so this is
710
+ # a physically independent vertical cue — and the CNN's pitch output has
711
+ # proven nearly blind vertically (r~0.41). calibrate() will measure both
712
+ # and pick whichever actually tracks screen-Y.
713
+ point_ear.append(float(np.median(ear_list)))
714
+
715
+ # Keep a BUNDLE of genuinely different frames for this point — not one
716
+ # "representative" frame. Both the fine-tune and the post-fine-tune RBF
717
+ # refit read from this bundle, so both get real frame diversity and real
718
+ # noise averaging instead of anchoring on a single arbitrary frame.
719
+ n_avail = len(l_list)
720
+ n_take = min(FRAMES_KEPT_PER_POINT, n_avail)
721
+ indices = np.linspace(0, n_avail - 1, n_take, dtype=int)
722
+ point_frames.append({
723
+ "lefts": [l_list[i] for i in indices],
724
+ "rights": [r_list[i] for i in indices],
725
+ "poses": [p_list[i] for i in indices],
726
+ "hps": [hp_list[i] for i in indices],
727
+ })
728
+
729
+ print(f"Point {idx+1:2d}/{total} | pitch={avg[0]:+.4f} yaw={avg[1]:+.4f} "
730
+ f"| jitter(pitch)={mad_pitch:.4f} jitter(yaw)={mad_yaw:.4f} "
731
+ f"| {len(indices)} frames kept "
732
+ f"| skipped: {point_blink_skips} blink, {point_light_skips} lighting")
733
+
734
+ cap.release()
735
+ cv2.destroyAllWindows()
736
+
737
+ if len(gaze_vectors) < 4:
738
+ print(f"ERROR: only {len(gaze_vectors)} points collected, need at least 4.")
739
+ exit(1)
740
+
741
+ # =============================================================================
742
+ # OUTLIER FILTER
743
+ # =============================================================================
744
+ (gaze_vectors, screen_points, head_pitches_deg,
745
+ point_dispersion, point_frames, point_ear) = filter_unreliable_points(
746
+ gaze_vectors, screen_points, head_pitches_deg,
747
+ point_dispersion, point_frames, point_ear, factor=3.0
748
+ )
749
 
750
+ n_pts = len(screen_points)
751
+ if n_pts < 4:
752
+ print("ERROR: too many points removed, need at least 4 reliable points.")
753
+ exit(1)
754
+
755
+ # =============================================================================
756
+ # FINE-TUNE (optional — see FINETUNE_ENABLED comment for why it defaults OFF)
757
+ # Now fed the REAL diverse frames from each point's bundle. Previously this
758
+ # received the same single frame duplicated 5x per point, so it trained on
759
+ # 16 unique images while reporting 80.
760
+ # =============================================================================
761
+ finetuned = False
762
+ if FINETUNE_ENABLED:
763
+ ft_lefts, ft_rights, ft_poses, ft_sp = [], [], [], []
764
+ for i in range(n_pts):
765
+ pf = point_frames[i]
766
+ for k in range(len(pf["lefts"])):
767
+ ft_lefts.append(pf["lefts"][k])
768
+ ft_rights.append(pf["rights"][k])
769
+ ft_poses.append(pf["poses"][k])
770
+ ft_sp.append(screen_points[i])
771
+
772
+ n_unique = len(ft_lefts)
773
+
774
+ # Derive the fine-tune target geometry from REAL numbers instead of the
775
+ # fabricated 0.6/0.4 constants.
776
+ _diag_mm = SCREEN_DIAGONAL_INCHES * 25.4
777
+ _aspect = SCREEN_W / float(SCREEN_H)
778
+ _scr_h_mm = _diag_mm / np.sqrt(_aspect ** 2 + 1.0)
779
+ _scr_w_mm = _aspect * _scr_h_mm
780
+ if session_distance_mm:
781
+ _dist_mm = float(np.median(session_distance_mm))
782
+ else:
783
+ _dist_mm = 500.0
784
+ # Guard against a wild solvePnP outlier producing nonsense geometry.
785
+ _dist_mm = float(np.clip(_dist_mm, 300.0, 900.0))
786
 
787
+ if USE_MEASURED_GEOMETRY:
788
+ K_H = (_scr_w_mm / 2.0) / _dist_mm
789
+ K_V = (_scr_h_mm / 2.0) / _dist_mm
790
+ else:
791
+ K_H, K_V = 0.6, 0.4 # original constants — currently working
792
+
793
+ print(f"\n--- Fine-tuning CNN on your eyes ({n_unique} REAL frames, "
794
+ f"{n_pts} points) ---")
795
+ print(f"[Geometry] screen {SCREEN_DIAGONAL_INCHES}\" -> "
796
+ f"{_scr_w_mm:.0f}x{_scr_h_mm:.0f}mm | measured viewing distance "
797
+ f"{_dist_mm:.0f}mm")
798
+ print(f"[Geometry] fine-tune targets: k_h={K_H:.3f} k_v={K_V:.3f} "
799
+ f"(ratio {K_H/max(K_V,1e-6):.2f}) "
800
+ f"[{'measured' if USE_MEASURED_GEOMETRY else 'original constants'}]")
801
+ finetuned = finetune_on_calibration_v4(
802
+ onnx_path = ONNX_PATH,
803
+ left_patches = ft_lefts,
804
+ right_patches = ft_rights,
805
+ head_poses = ft_poses,
806
+ screen_points = ft_sp,
807
+ screen_w = SCREEN_W,
808
+ screen_h = SCREEN_H,
809
+ ckpt_path = CKPT_PATH,
810
+ out_onnx_path = ONNX_PATH,
811
+ steps = 200,
812
+ lr = 1e-4,
813
+ k_h = K_H,
814
+ k_v = K_V,
815
+ )
816
+ else:
817
+ print("\n--- Fine-tuning SKIPPED (FINETUNE_ENABLED = False) ---")
818
+ print(" Using the base model as-is. The RBF maps its output to screen")
819
+ print(" coordinates, so the model's absolute scale does not need to be")
820
+ print(" 'correct' — only monotonic. See the FINETUNE_ENABLED comment.")
821
+
822
+ # =============================================================================
823
+ # FIT RBF CALIBRATION
824
+ #
825
+ # CRITICAL FIX: re-predict across EVERY kept frame for each point and take the
826
+ # MEDIAN, instead of predicting from one arbitrary "representative" frame.
827
+ #
828
+ # The old code computed a careful median over 50-150 frames during collection,
829
+ # then threw it away and refit the RBF from a single frame per point. That
830
+ # discarded all noise averaging and anchored the entire calibration on 16
831
+ # single, possibly-noisy frames — which is exactly the kind of thing that
832
+ # makes edges and corners unstable.
833
+ # =============================================================================
834
+ print("\n--- Fitting RBF calibration ---")
835
+ pipeline = InsightUXPipeline(ONNX_PATH)
836
+
837
+ adapted_gaze_vectors = []
838
+ adapted_dispersion = []
839
  for i in range(n_pts):
840
  pf = point_frames[i]
841
+ per_frame = []
842
  for k in range(len(pf["lefts"])):
843
+ l = pf["lefts"][k]
844
+ r = pf["rights"][k]
845
+ pose = pf["poses"][k]
846
+ hp = pf["hps"][k]
847
+ hy = float(pose[1]) * POSE_NORM_SCALE # recover raw head yaw (deg)
848
+
849
+ _, _, raw_pitch, raw_yaw = pipeline.predict_gaze_vector(l, pose, r)
850
+ pitch = compensate_pitch(raw_pitch, hp)
851
+ yaw = compensate_yaw(raw_yaw, hy)
852
+ per_frame.append([pitch, yaw])
853
+
854
+ per_frame = np.array(per_frame)
855
+ med = np.median(per_frame, axis=0)
856
+ adapted_gaze_vectors.append(list(med))
857
+ adapted_dispersion.append([
858
+ float(np.median(np.abs(per_frame[:, 0] - med[0]))),
859
+ float(np.median(np.abs(per_frame[:, 1] - med[1]))),
860
+ ])
861
+
862
+ assert len(adapted_gaze_vectors) == len(screen_points), \
863
+ f"Length mismatch: {len(adapted_gaze_vectors)} gaze vs {len(screen_points)} screen"
864
+
865
+ pipeline.calibration.calibrate(
866
+ np.array(adapted_gaze_vectors),
867
+ np.array(screen_points),
868
+ ear=np.array(point_ear),
869
+ screen_size=(SCREEN_W, SCREEN_H) # <-- fixes the 1493x933 clamp bug
870
+ )
871
+ pipeline.calibration.save(os.path.join(DATA_DIR, "calibration.pkl"))
872
+
873
+ import pickle
874
+ _all_hp = [hp for pf in point_frames for hp in pf["hps"]]
875
+ _all_pose = [p for pf in point_frames for p in pf["poses"]]
876
+ baseline_pitch = float(np.median(_all_hp)) if _all_hp else 0.0
877
+ baseline_yaw = float(np.median([float(p[1]) * POSE_NORM_SCALE for p in _all_pose])) if _all_pose else 0.0
878
+ baseline_roll = float(np.median([float(p[2]) * POSE_NORM_SCALE for p in _all_pose])) if _all_pose else 0.0
879
+ with open(os.path.join(DATA_DIR, "baseline_pose.pkl"), "wb") as f:
880
+ pickle.dump({"pitch": baseline_pitch, "yaw": baseline_yaw, "roll": baseline_roll}, f)
881
+ print(f"Saved baseline_pose.pkl: pitch={baseline_pitch:+.2f}, yaw={baseline_yaw:+.2f}, roll={baseline_roll:+.2f}")
882
+
883
+ print(f"\nCalibration complete! {n_pts}/{total} points used.")
884
+ if finetuned:
885
+ print("CNN fine-tuned on your eyes + RBF calibration fitted.")
886
  else:
887
+ print("RBF calibration fitted (CNN fine-tuning skipped).")
888
+
889
+ # =============================================================================
890
+ # SESSION DIAGNOSTIC SUMMARY — the "tell the person if something's off" ask.
891
+ # Printed once, in plain language, instead of them having to interpret
892
+ # per-point numbers themselves.
893
+ # =============================================================================
894
+ # =============================================================================
895
+ # SIGNAL-TO-NOISE — THE VERDICT
896
+ #
897
+ # This is the number that decides whether an axis is fixable at all.
898
+ #
899
+ # SIGNAL = how much the model's output changes between DIFFERENT screen
900
+ # positions (spread of the per-point medians). This is the real
901
+ # information the RBF has to work with.
902
+ # NOISE = how much the model's output wobbles frame-to-frame while you
903
+ # stare at ONE fixed dot (median within-point jitter).
904
+ #
905
+ # If NOISE >= SIGNAL on an axis, the model cannot tell "you looked lower"
906
+ # apart from "the same look, one frame later". No RBF, no gain correction,
907
+ # no clamp tuning can recover information that was never there. That axis
908
+ # needs a better MODEL, not better calibration math.
909
+ # =============================================================================
910
+ gv_arr = np.array(adapted_gaze_vectors)
911
+ disp_arr = np.array(adapted_dispersion)
912
+ sp_arr = np.array(screen_points)
913
+
914
+ signal_pitch = float(np.std(gv_arr[:, 0]))
915
+ signal_yaw = float(np.std(gv_arr[:, 1]))
916
+ noise_pitch = float(np.median(disp_arr[:, 0]))
917
+ noise_yaw = float(np.median(disp_arr[:, 1]))
918
+
919
+ snr_pitch = signal_pitch / noise_pitch if noise_pitch > 1e-9 else float("inf")
920
+ snr_yaw = signal_yaw / noise_yaw if noise_yaw > 1e-9 else float("inf")
921
+
922
+ # CORRELATION — the metric that actually decides usability.
923
+ #
924
+ # An earlier version of this block reported only SNR (spread vs jitter). That
925
+ # was MISLEADING and nearly sent us chasing the wrong fix: a signal can have
926
+ # plenty of spread and still be useless if that spread is not ORDERED by the
927
+ # thing you're predicting. Pitch once reported SNR 1.76 ("marginal") while its
928
+ # per-row averages went down, up, up — i.e. no monotonic relationship with
929
+ # screen row at all. Spread was real; ordering was not.
930
+ #
931
+ # Pearson r between model output and true screen coordinate is the honest
932
+ # test. r near +/-1 means the model tracks that axis. r near 0 means it does
933
+ # not, no matter how much spread there is.
934
+ def _pearson(a, b):
935
+ a = np.asarray(a, dtype=float); b = np.asarray(b, dtype=float)
936
+ if a.std() < 1e-12 or b.std() < 1e-12:
937
+ return 0.0
938
+ return float(np.corrcoef(a, b)[0, 1])
939
+
940
+ r_yaw = _pearson(gv_arr[:, 1], sp_arr[:, 0]) # yaw vs screen X
941
+ r_pitch = _pearson(gv_arr[:, 0], sp_arr[:, 1]) # pitch vs screen Y
942
+ r_ear = _pearson(np.array(point_ear), sp_arr[:, 1]) # eye aperture vs screen Y
943
+
944
+ def _corr_verdict(r):
945
+ ar = abs(r)
946
+ if ar >= 0.90:
947
+ return "EXCELLENT — model tracks this axis cleanly"
948
+ if ar >= 0.70:
949
+ return "GOOD — usable, some slop"
950
+ if ar >= 0.50:
951
+ return "WEAK — expect significant error on this axis"
952
+ return "BROKEN — model output barely relates to this axis at all"
953
+
954
+ print("\n========== DOES THE MODEL TRACK THE SCREEN? (the real test) ==========")
955
+ print(f"HORIZONTAL yaw vs screen-X : r = {r_yaw:+.3f} (SNR {snr_yaw:.2f})")
956
+ print(f" -> {_corr_verdict(r_yaw)}")
957
+ print(f"VERTICAL pitch vs screen-Y: r = {r_pitch:+.3f} (SNR {snr_pitch:.2f})")
958
+ print(f" -> {_corr_verdict(r_pitch)}")
959
+ print(f"VERTICAL EYE APERTURE vs screen-Y: r = {r_ear:+.3f}")
960
+ print(f" -> {_corr_verdict(r_ear)}")
961
+ print()
962
+ _best_vert = max(abs(r_pitch), abs(r_ear))
963
+ print(f"Best available vertical cue: "
964
+ f"{'EYE APERTURE' if abs(r_ear) > abs(r_pitch) else 'CNN PITCH'} "
965
+ f"(r={_best_vert:+.3f})")
966
+ print()
967
+ if _best_vert < 0.5:
968
+ print("VERDICT: the model's PITCH output does not meaningfully track where you")
969
+ print("look vertically. This is NOT a calibration problem — the RBF cannot map")
970
+ print("an input that carries no ordered information about screen height. No")
971
+ print("amount of clamp/smoothing/gain tuning will fix it. The fix is the MODEL:")
972
+ print("its pitch head needs retraining, or vertical gaze needs a different")
973
+ print("feature (e.g. eyelid aperture / iris-centre offset within the socket),")
974
+ print("which the current eye-patch CNN is evidently not learning.")
975
+ elif _best_vert < abs(r_yaw) - 0.15:
976
+ print("VERDICT: vertical tracks the screen, but noticeably worse than")
977
+ print("horizontal. Calibration is doing its job; expect up/down to stay the")
978
+ print("looser axis until the model improves.")
979
  else:
980
+ print("VERDICT: both axes track the screen. Any remaining error is in the")
981
+ print("calibration mapping or the noise floor, not in the model's ability")
982
+ print("to see where you're looking.")
983
+ print("=====================================================================")
984
+
985
+ print("\n================ CALIBRATION QUALITY SUMMARY ================")
986
+ if session_brightness:
987
+ avg_bright = float(np.mean(session_brightness))
988
+ pct_dark = 100 * sum(1 for b in session_brightness if b < LIGHT_MIN_BRIGHTNESS) / len(session_brightness)
989
+ pct_bright = 100 * sum(1 for b in session_brightness if b > LIGHT_MAX_BRIGHTNESS) / len(session_brightness)
990
+ print(f"Average brightness: {avg_bright:.0f} (comfortable range: {LIGHT_MIN_BRIGHTNESS}-{LIGHT_MAX_BRIGHTNESS})")
991
+ if pct_dark > 10:
992
+ print(f"⚠ Lighting was too DARK for {pct_dark:.0f}% of frames. "
993
+ f"Add a light source facing your face, or face a window, before recalibrating.")
994
+ if pct_bright > 10:
995
+ print(f"⚠ Lighting was too BRIGHT for {pct_bright:.0f}% of frames "
996
+ f"(backlight or a light directly behind you?). Try facing away from strong light sources.")
997
+ if pct_dark <= 10 and pct_bright <= 10:
998
+ print("Lighting was consistently good throughout.")
999
+
1000
+ if session_blink_skips > 0:
1001
+ print(f"Blinking accounted for {session_blink_skips} skipped frames across the session "
1002
+ f"— normal, this is expected and was handled automatically.")
1003
+
1004
+ if points_with_low_yield:
1005
+ print(f"⚠ These points had low sample counts and may be less accurate: "
1006
+ f"{', '.join(str(p) for p in points_with_low_yield)}. "
1007
+ f"If tracking feels off in that part of the screen, consider recalibrating.")
1008
+ else:
1009
+ print("All points collected a healthy number of samples.")
1010
+ print("===============================================================")
1011
+
1012
+ # --- Diagnostic: verify vertical pitch separation ---
1013
+ print("\n--- Pitch by screen row (should increase top -> bottom) ---")
1014
+ gv_arr = np.array(adapted_gaze_vectors)
1015
+ sp_arr = np.array(screen_points)
1016
+ thresholds = [(0, SCREEN_H*0.25, "Top (y<25%) "),
1017
+ (SCREEN_H*0.25, SCREEN_H*0.5, "Mid-hi (25-50%) "),
1018
+ (SCREEN_H*0.5, SCREEN_H*0.75, "Mid-lo (50-75%) "),
1019
+ (SCREEN_H*0.75, SCREEN_H+1, "Bottom (y>75%) ")]
1020
+ for lo, hi, label in thresholds:
1021
+ mask = (sp_arr[:, 1] >= lo) & (sp_arr[:, 1] < hi)
1022
+ if mask.any():
1023
+ print(f" {label}: avg pitch = {gv_arr[mask, 0].mean():.4f}")
1024
+
1025
+ print("\n--- Yaw by screen column (should increase left -> right) ---")
1026
+ col_thresholds = [(0, SCREEN_W*0.25, "Left (x<25%) "),
1027
+ (SCREEN_W*0.25, SCREEN_W*0.5, "Mid-lf (25-50%) "),
1028
+ (SCREEN_W*0.5, SCREEN_W*0.75, "Mid-rt (50-75%) "),
1029
+ (SCREEN_W*0.75, SCREEN_W+1, "Right (x>75%) ")]
1030
+ for lo, hi, label in col_thresholds:
1031
+ mask = (sp_arr[:, 0] >= lo) & (sp_arr[:, 0] < hi)
1032
+ if mask.any():
1033
+ print(f" {label}: avg yaw = {gv_arr[mask, 1].mean():.4f}")
1034
+
1035
+
1036
+ if __name__ == "__main__":
1037
+ main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
packaging/InsightUX.spec ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- mode: python ; coding: utf-8 -*-
2
+ #
3
+ # InsightUX PyInstaller build spec — onedir, not onefile.
4
+ #
5
+ # Onefile re-extracts the whole bundle (torch alone is ~1.2GB) to a temp
6
+ # directory on EVERY launch, which is a bad startup-time tradeoff on top of
7
+ # an already-heavy ML stack. Onedir installs once as a persistent folder
8
+ # (InsightUX.exe + _internal/), which is also just what installer.iss wants
9
+ # to package anyway.
10
+ #
11
+ # Build (from the repo root, inside the project venv — this project's venv
12
+ # uses Python 3.10, which is the proven-working interpreter; PyInstaller
13
+ # must be installed into that same venv first: `pip install pyinstaller`):
14
+ # pyinstaller packaging/InsightUX.spec --noconfirm
15
+ #
16
+ # Output lands in dist/InsightUX/ — that whole folder is what
17
+ # packaging/installer.iss packages into the installer.
18
+ #
19
+ # Known risk areas likely to need iteration on the FIRST real build (this is
20
+ # normal for this dependency stack, not a sign anything here is wrong):
21
+ # - mediapipe ships its own .tflite/.binarypb data files as package data;
22
+ # collect_all() below should catch them, but if face_mesh fails to load
23
+ # at runtime with a "file not found", that's the first place to check.
24
+ # - onnxruntime/torch/torchvision ship native DLLs; "DLL load failed" at
25
+ # startup means collect_all() missed one — check dist/InsightUX/_internal
26
+ # for what's actually present vs. what the traceback wants.
27
+ # - pywebview + pythonnet's `clr` bridge needs the Microsoft Edge WebView2
28
+ # Runtime on the TARGET machine. Usually preinstalled on modern Windows
29
+ # 10/11, but installer.iss bundles + silently runs Microsoft's WebView2
30
+ # bootstrapper as a safety net for older/locked-down machines.
31
+
32
+ import os
33
+ import dis
34
+ from PyInstaller.utils.hooks import collect_all
35
+ import PyInstaller.lib.modulegraph.util as _mg_util
36
+
37
+ # A handful of very large single-file modules in this dependency tree
38
+ # (bottle.py, matplotlib/pyplot.py, and possibly others not yet hit) trip a
39
+ # real dis.get_instructions() bug on this Python 3.10.0 build — an IndexError
40
+ # walking co_consts for certain EXTENDED_ARG-heavy bytecode in big modules.
41
+ # Confirmed this is purely a build-time INTROSPECTION bug, not a real problem:
42
+ # `import bottle` / `import matplotlib.pyplot` both work completely normally
43
+ # at actual runtime; only PyInstaller's static bytecode scan of them (used to
44
+ # discover further hidden imports) crashes.
45
+ #
46
+ # Patched once here instead of excluding modules one at a time as each is
47
+ # discovered (there's no way to know in advance how many more of these exist
48
+ # inside torch's ~1.2GB of dependencies). Worst case if this ever masks a
49
+ # real import PyInstaller would otherwise have auto-discovered: the
50
+ # hiddenimports/collect_all() list below already covers every package this
51
+ # app actually needs bundled, so a skipped scan on one giant module isn't
52
+ # depended on for correctness.
53
+ _orig_iterate_instructions = _mg_util.iterate_instructions
54
+
55
+ def _safe_iterate_instructions(code_object):
56
+ try:
57
+ yield from _orig_iterate_instructions(code_object)
58
+ except Exception as e:
59
+ print(f"[InsightUX.spec] bytecode scan skipped on a module "
60
+ f"(dis bug workaround): {type(e).__name__}: {e}")
61
+ return
62
+
63
+ _mg_util.iterate_instructions = _safe_iterate_instructions
64
+
65
+ PACKAGING_DIR = os.path.abspath(os.path.dirname(os.path.abspath(SPEC)))
66
+ REPO_ROOT = os.path.dirname(PACKAGING_DIR)
67
+
68
+ datas = [
69
+ (os.path.join(REPO_ROOT, "models", "gaze_cnn_v4.onnx"), "models"),
70
+ (os.path.join(REPO_ROOT, "models", "gaze_cnn_v4.onnx.data"), "models"),
71
+ (os.path.join(REPO_ROOT, "checkpoints", "best_model_v4.pt"), "checkpoints"),
72
+ ]
73
+ binaries = []
74
+ # models.model_v4 is only reached via a try/except-guarded dynamic import
75
+ # inside calibrate.py's fine-tune function — spell it out explicitly rather
76
+ # than trust static analysis to follow it through the try/except.
77
+ hiddenimports = ["models.model_v4"]
78
+
79
+ # The blunt-but-reliable safety net for packages that ship native binaries
80
+ # and/or non-.py package data PyInstaller's built-in hooks don't fully catch.
81
+ for pkg in ("mediapipe", "onnxruntime", "torch", "torchvision", "cv2"):
82
+ pkg_datas, pkg_binaries, pkg_hiddenimports = collect_all(pkg)
83
+ datas += pkg_datas
84
+ binaries += pkg_binaries
85
+ hiddenimports += pkg_hiddenimports
86
+
87
+ a = Analysis(
88
+ [os.path.join(REPO_ROOT, "browser_session.py")],
89
+ pathex=[REPO_ROOT],
90
+ binaries=binaries,
91
+ datas=datas,
92
+ hiddenimports=hiddenimports,
93
+ hookspath=[],
94
+ hooksconfig={},
95
+ runtime_hooks=[],
96
+ excludes=[],
97
+ noarchive=False,
98
+ )
99
+ pyz = PYZ(a.pure)
100
+
101
+ exe = EXE(
102
+ pyz,
103
+ a.scripts,
104
+ [],
105
+ exclude_binaries=True,
106
+ name="InsightUX",
107
+ debug=False,
108
+ bootloader_ignore_signals=False,
109
+ strip=False,
110
+ upx=False,
111
+ # Keep the console. calibrate.py's live lighting/blink/quality readouts
112
+ # and the correlation "VERDICT" summary are the primary calibration
113
+ # feedback today — hiding the console would silently drop the one thing
114
+ # a user needs to read after every calibration run.
115
+ console=True,
116
+ disable_windowed_traceback=False,
117
+ target_arch=None,
118
+ codesign_identity=None,
119
+ entitlements_file=None,
120
+ )
121
+
122
+ coll = COLLECT(
123
+ exe,
124
+ a.binaries,
125
+ a.zipfiles,
126
+ a.datas,
127
+ strip=False,
128
+ upx=False,
129
+ upx_exclude=[],
130
+ name="InsightUX",
131
+ )
packaging/README.md ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Packaging InsightUX as a downloadable installer
2
+
3
+ Turns the cloned-repo-plus-venv workflow into a single Windows installer
4
+ (`InsightUX-Setup-<version>.exe`) that bundles Python, torch, mediapipe,
5
+ onnxruntime, opencv, and pywebview — end users need nothing pre-installed.
6
+
7
+ ## One-time setup (per machine you build from)
8
+
9
+ 1. In the project venv: `pip install pyinstaller`
10
+ 2. Install [Inno Setup 6](https://jrsoftware.org/isdl.php) (free) — provides
11
+ `ISCC.exe`, the command-line compiler `installer.iss` needs. Not required
12
+ just to build the app itself, only to build the final installer.
13
+ 3. (Optional but recommended) Download Microsoft's WebView2 bootstrapper
14
+ (`MicrosoftEdgeWebview2Setup.exe`) from
15
+ [Microsoft's evergreen bootstrapper page](https://developer.microsoft.com/microsoft-edge/webview2/)
16
+ and place it in this `packaging/` folder before compiling the installer —
17
+ `installer.iss` bundles and silently runs it if present, as a safety net
18
+ for machines missing the WebView2 Runtime (usually already present on
19
+ modern Windows 10/11, but not guaranteed on older/locked-down machines).
20
+ If you skip this, the installer still builds fine, just without that
21
+ safety net.
22
+
23
+ ## Build
24
+
25
+ From the repo root:
26
+
27
+ ```powershell
28
+ pyinstaller packaging\InsightUX.spec --noconfirm
29
+ ```
30
+
31
+ Output: `dist\InsightUX\` (a folder — `InsightUX.exe` + `_internal\`). Run
32
+ `dist\InsightUX\InsightUX.exe` directly first to confirm it actually starts
33
+ before building the installer — much faster to debug a missing-DLL/missing-
34
+ data-file error at this stage than after wrapping it in an installer.
35
+
36
+ **Expect to iterate on the first build.** mediapipe, torch, and onnxruntime
37
+ all ship native binaries and non-`.py` data files that PyInstaller's default
38
+ hooks don't always fully catch — see the comments at the top of
39
+ `InsightUX.spec` for what's already collected and where to look if the
40
+ frozen exe fails to start.
41
+
42
+ Then build the installer:
43
+
44
+ ```powershell
45
+ iscc packaging\installer.iss
46
+ ```
47
+
48
+ Output: `packaging\dist_installer\InsightUX-Setup-<version>.exe`.
49
+
50
+ ## Release checklist
51
+
52
+ Every time you ship a code change as an update:
53
+
54
+ 1. Bump `VERSION` in [browser_session.py](../browser_session.py).
55
+ 2. Bump `AppVersion`/`MyAppVersion` in [installer.iss](installer.iss) to match.
56
+ 3. Rebuild (`pyinstaller ...` then `iscc ...`).
57
+ 4. Update [version.json](../version.json) at the repo root — `"latest"` to
58
+ the new version, `"url"` to the new installer's HF download URL.
59
+ 5. `git lfs` tracks `*.exe` already (see `.gitattributes`) — add and push the
60
+ new installer + `version.json` to the `origin` (Hugging Face) remote.
61
+
62
+ That's it — nothing about *publishing* a release is automated. What's
63
+ automated is the *checking*: every running copy of InsightUX pings
64
+ `version.json` once at startup (`check_for_update()` in browser_session.py)
65
+ and shows a clickable "Update available" pill in the toolbar if `"latest"`
66
+ is newer than its own `VERSION`. Clicking it opens `"url"` in the user's
67
+ default browser — still a manual download+run, just automated the "is there
68
+ something new" question. No internet, or any failure fetching the file, is
69
+ silent (no banner, never an error).
70
+
71
+ ## Why these choices
72
+
73
+ - **Onedir, not onefile** — onefile re-extracts the whole ~1.5-2.5GB bundle
74
+ to a temp directory on every launch. With torch/mediapipe already this
75
+ heavy, that's a genuinely bad startup-time hit. Onedir just runs directly
76
+ from the installed folder.
77
+ - **Per-user install (`{localappdata}`), not Program Files** — no
78
+ admin/UAC prompt, and avoids Program Files' write-permission restrictions
79
+ for `calibration.pkl`/`sessions/`, which the running app writes directly
80
+ into its own folder (see `RESOURCE_DIR`/`DATA_DIR` in browser_session.py).
81
+ - **Fixed `AppId` GUID in installer.iss** — this is what makes a new
82
+ installer upgrade the existing install in place instead of creating a
83
+ second copy. Never regenerate it.
84
+ - **torch is bundled** (not excluded) — its fine-tuning step measurably
85
+ improves calibration accuracy (169-204px mean error vs. 275px without it,
86
+ per calibrate.py's own numbers). Confirmed with the project owner as
87
+ worth the larger download.
packaging/installer.iss ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ; InsightUX Windows installer — Inno Setup script.
2
+ ;
3
+ ; Build (requires Inno Setup / ISCC installed locally, and dist/InsightUX/
4
+ ; already built via `pyinstaller packaging/InsightUX.spec` first):
5
+ ; iscc packaging/installer.iss
6
+ ; Output lands in packaging/dist_installer/InsightUX-Setup-<version>.exe.
7
+ ;
8
+ ; AppId is a FIXED GUID, generated once and never changed. It's the whole
9
+ ; mechanism behind "push updates through the installer": every future build
10
+ ; keeps this same AppId + a higher AppVersion, so re-running a new installer
11
+ ; upgrades the existing install in place (same folder, same shortcuts)
12
+ ; instead of creating a second copy. Do not regenerate this GUID.
13
+ ;
14
+ ; Installs per-user (PrivilegesRequired=lowest) to %LocalAppData%\InsightUX —
15
+ ; no admin/UAC prompt, and critically, no Program Files write-permission
16
+ ; problems for calibration.pkl/sessions/, which the running app writes
17
+ ; directly into its own install folder (see browser_session.py's DATA_DIR).
18
+ ;
19
+ ; calibration.pkl, baseline_pose.pkl, and sessions/ are deliberately NOT
20
+ ; listed under [Files] below. Inno Setup only ever touches files it tracked
21
+ ; as having installed — so upgrades overwrite app files without touching
22
+ ; those, and a normal uninstall leaves them behind too (the [Code] section
23
+ ; below tells the user this explicitly on uninstall, rather than silently).
24
+
25
+ #define MyAppName "InsightUX"
26
+ #define MyAppVersion "1.0.0"
27
+ #define MyAppPublisher "InsightUX"
28
+ #define MyAppExeName "InsightUX.exe"
29
+ #define MyBuildDir "..\dist\InsightUX"
30
+
31
+ [Setup]
32
+ AppId={{6DE35CE2-05CB-4375-BFFE-D63E9181F1B9}
33
+ AppName={#MyAppName}
34
+ AppVersion={#MyAppVersion}
35
+ AppPublisher={#MyAppPublisher}
36
+ DefaultDirName={localappdata}\InsightUX
37
+ DefaultGroupName=InsightUX
38
+ DisableProgramGroupPage=yes
39
+ PrivilegesRequired=lowest
40
+ ArchitecturesInstallIn64BitMode=x64compatible
41
+ OutputDir=dist_installer
42
+ OutputBaseFilename=InsightUX-Setup-{#MyAppVersion}
43
+ Compression=lzma2
44
+ SolidCompression=yes
45
+ WizardStyle=modern
46
+ ; A ~1.5-2.5GB installer (torch is bundled deliberately, see README) needs
47
+ ; the larger internal chunk size or ISCC can fail to build at all.
48
+ DiskSpanning=no
49
+
50
+ [Tasks]
51
+ Name: "desktopicon"; Description: "Create a &desktop icon"; GroupDescription: "Additional icons:"
52
+
53
+ [Files]
54
+ Source: "{#MyBuildDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
55
+ ; Microsoft's WebView2 bootstrapper is NOT part of this repo (an external
56
+ ; download — see packaging/README.md for the official link). If you've
57
+ ; placed it next to this .iss before compiling, it gets bundled and
58
+ ; silently run below as a safety net for machines missing the runtime.
59
+ ; Harmless to (re-)run even if WebView2 is already installed.
60
+ #ifexist "MicrosoftEdgeWebview2Setup.exe"
61
+ Source: "MicrosoftEdgeWebview2Setup.exe"; DestDir: "{tmp}"; Flags: deleteafterinstall
62
+ #endif
63
+
64
+ [Icons]
65
+ Name: "{group}\InsightUX"; Filename: "{app}\{#MyAppExeName}"; WorkingDir: "{app}"
66
+ Name: "{group}\Uninstall InsightUX"; Filename: "{uninstallexe}"
67
+ Name: "{commondesktop}\InsightUX"; Filename: "{app}\{#MyAppExeName}"; WorkingDir: "{app}"; Tasks: desktopicon
68
+
69
+ [Run]
70
+ #ifexist "MicrosoftEdgeWebview2Setup.exe"
71
+ Filename: "{tmp}\MicrosoftEdgeWebview2Setup.exe"; Parameters: "/silent /install"; StatusMsg: "Checking Microsoft Edge WebView2 Runtime..."; Flags: waituntilterminated
72
+ #endif
73
+ Filename: "{app}\{#MyAppExeName}"; Description: "Launch InsightUX"; Flags: nowait postinstall skipifsilent
74
+
75
+ [Code]
76
+ procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep);
77
+ begin
78
+ if CurUninstallStep = usPostUninstall then
79
+ begin
80
+ if FileExists(ExpandConstant('{app}\calibration.pkl')) or
81
+ DirExists(ExpandConstant('{app}\sessions')) then
82
+ MsgBox('Your calibration profile and saved sessions were kept in:' + #13#10 +
83
+ ExpandConstant('{app}') + #13#10 +
84
+ 'Delete that folder yourself if you want a completely clean removal.',
85
+ mbInformation, MB_OK);
86
+ end;
87
+ end;
requirements.txt CHANGED
@@ -12,6 +12,7 @@
12
  # --- Core CV / inference ---
13
  opencv-python==4.10.0.84
14
  mediapipe==0.10.18
 
15
  onnxruntime==1.19.2
16
  numpy==1.26.4
17
  scipy==1.13.1
 
12
  # --- Core CV / inference ---
13
  opencv-python==4.10.0.84
14
  mediapipe==0.10.18
15
+ onnx==1.16.2
16
  onnxruntime==1.19.2
17
  numpy==1.26.4
18
  scipy==1.13.1
validate.py CHANGED
@@ -19,11 +19,29 @@ Prints:
19
  PATCH_SOURCE must match calibrate.py and run_session.py.
20
  """
21
 
 
 
22
  import cv2
23
  import numpy as np
24
  import time
25
  import pyautogui
26
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
 
28
  from preprocessing.preprocessing_pipeline import (
29
  create_face_mesh,
@@ -43,8 +61,8 @@ from preprocessing.preprocessing_pipeline import (
43
  from inference_pipeline import InsightUXPipeline, GazeAngleSmoother
44
 
45
 
46
- ONNX_PATH = "models/gaze_cnn_v4.onnx"
47
- CALIBRATION_PATH = "calibration.pkl"
48
 
49
  SCREEN_W, SCREEN_H = pyautogui.size()
50
  PATCH_SOURCE = "blended" # MUST match calibrate.py and main_webcam_pipeline.py
 
19
  PATCH_SOURCE must match calibrate.py and run_session.py.
20
  """
21
 
22
+ import os
23
+ import sys
24
  import cv2
25
  import numpy as np
26
  import time
27
  import pyautogui
28
 
29
+ # Windows consoles default to a non-UTF-8 codepage (cp1252) — a stray
30
+ # unicode character in any print() would otherwise crash the whole process
31
+ # with UnicodeEncodeError. See calibrate.py for where this was hit for real.
32
+ if hasattr(sys.stdout, "reconfigure"):
33
+ sys.stdout.reconfigure(encoding="utf-8", errors="replace")
34
+ if hasattr(sys.stderr, "reconfigure"):
35
+ sys.stderr.reconfigure(encoding="utf-8", errors="replace")
36
+
37
+ # Same RESOURCE_DIR/DATA_DIR split as calibrate.py and browser_session.py —
38
+ # bundled read-only assets vs. per-user writable data, once frozen.
39
+ if getattr(sys, "frozen", False):
40
+ RESOURCE_DIR = sys._MEIPASS
41
+ DATA_DIR = os.path.dirname(sys.executable)
42
+ else:
43
+ RESOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
44
+ DATA_DIR = RESOURCE_DIR
45
 
46
  from preprocessing.preprocessing_pipeline import (
47
  create_face_mesh,
 
61
  from inference_pipeline import InsightUXPipeline, GazeAngleSmoother
62
 
63
 
64
+ ONNX_PATH = os.path.join(RESOURCE_DIR, "models", "gaze_cnn_v4.onnx")
65
+ CALIBRATION_PATH = os.path.join(DATA_DIR, "calibration.pkl")
66
 
67
  SCREEN_W, SCREEN_H = pyautogui.size()
68
  PATCH_SOURCE = "blended" # MUST match calibrate.py and main_webcam_pipeline.py
version.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "latest": "1.0.0",
3
+ "url": "https://huggingface.co/arpitasethiii/insightux/resolve/main/InsightUX-Setup-1.0.0.exe",
4
+ "notes": "Initial downloadable installer release."
5
+ }