analysis.py

"""
analysis.py
Turns a raw InsightUX session (gaze_log.jsonl + dom_log.jsonl + screenshots/)
into a self-contained HTML report: ranked attention (in plain language),
a dwell timeline, and per-scroll-position heatmaps drawn directly on top
of what the page actually looked like.

Usage:
    from analysis import generate_report
    report_path = generate_report(session_dir)   # returns abs path to html

PAD_PX must match the value used in browser_session.py's injected
TRACKING_JS, or the post-hoc AOI attribution here will disagree with what
the user visually saw highlighted during the session.
"""

import os
import re
import json
import bisect

PAD_PX = 90  # keep identical to TRACKING_JS's PAD_PX in browser_session.py


# =============================================================================
# LOADING
# =============================================================================

def _load_jsonl(path):
    records = []
    if not os.path.exists(path):
        return records
    with open(path, "r") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                records.append(json.loads(line))
            except json.JSONDecodeError:
                continue
    return records


def load_session(session_dir):
    gaze = [r for r in _load_jsonl(os.path.join(session_dir, "gaze_log.jsonl"))
            if r.get("type") == "gaze"]
    dom = [r for r in _load_jsonl(os.path.join(session_dir, "dom_log.jsonl"))
           if r.get("type") == "dom"]
    gaze.sort(key=lambda r: r["t"])
    dom.sort(key=lambda r: r["t"])
    return gaze, dom


# =============================================================================
# MOUSE ACTIVITY (from mouse_log.jsonl — batches pushed by the in-page Mouse
# Tracker overlay, same trail/heatmap/click/dwell shape as the standalone
# "Mouse Tracker & Heatmap" Chrome extension this was ported from)
# =============================================================================

def load_mouse_batches(session_dir):
    return [r for r in _load_jsonl(os.path.join(session_dir, "mouse_log.jsonl"))
            if r.get("type") == "mouse_batch"]


def summarize_mouse(session_dir):
    batches = load_mouse_batches(session_dir)

    dwell_totals = {}
    clicks = []
    trail_points = 0
    heatmap_points = 0

    for b in batches:
        for item in (b.get("dwell") or []):
            element = item.get("element")
            duration = item.get("duration", 0)
            if not element:
                continue
            dwell_totals[element] = dwell_totals.get(element, 0) + duration
        for c in (b.get("click") or []):
            clicks.append(c)
        trail_points += len(b.get("trail") or [])
        heatmap_points += len(b.get("heatmap") or [])

    interests = sorted(
        ({"element": k, "seconds": round(v / 1000.0, 1)} for k, v in dwell_totals.items()),
        key=lambda r: -r["seconds"]
    )[:10]

    clicks.sort(key=lambda c: c.get("timestamp", ""))

    return {
        "interests": interests,
        "clicks": clicks[-50:],
        "click_count": len(clicks),
        "trail_points": trail_points,
        "heatmap_points": heatmap_points,
    }


# =============================================================================
# HUMAN-READABLE LABELS
# Raw AOI labels come straight out of the DOM (tag names, CSS classes,
# truncated text). Fine for matching, unreadable for a report. This maps
# them to plain language without losing which element they refer to.
# =============================================================================

def friendly_label(raw):
    if not raw:
        return "Unlabeled area"
    if raw == "navbar":
        return "Navigation bar"
    if raw == "header":
        return "Page header"
    if raw == "footer":
        return "Page footer"
    if raw == "video":
        return "Video"
    if raw == "embed":
        return "Embedded content (PDF, player, or widget)"
    if raw.startswith("img: "):
        return "Image — " + raw[5:]
    m = re.match(r"^(h[123]):\s*(.*)$", raw)
    if m:
        level = {"h1": "Main heading", "h2": "Heading", "h3": "Sub-heading"}[m.group(1)]
        return f"{level} — \u201c{m.group(2)}\u201d"
    m = re.match(r"^p \((.*)\)$", raw)
    if m:
        return f"Text — \u201c{m.group(1)}\u2026\u201d"
    if raw.startswith("#"):
        return "Section: " + raw[1:]
    if raw.startswith("."):
        return "Block: " + raw[1:]
    return raw[0].upper() + raw[1:] if raw else raw


# =============================================================================
# GAZE -> AOI ATTRIBUTION
# =============================================================================

def _find_aoi(px, py, aois):
    """Smallest padded AOI containing (px, py), or None. Mirrors the JS hit-test."""
    best = None
    best_area = None
    for a in aois:
        x, y, w, h = a["x"], a["y"], a["w"], a["h"]
        if (x - PAD_PX) <= px <= (x + w + PAD_PX) and (y - PAD_PX) <= py <= (y + h + PAD_PX):
            area = w * h
            if best is None or area < best_area:
                best = a
                best_area = area
    return best["label"] if best else None


def attribute_gaze(gaze, dom):
    """For each gaze sample, find the most recent dom snapshot at/before it
    and test which AOI the point falls in. Returns [(t, raw_label_or_None)]."""
    if not dom:
        return [(g["t"], None) for g in gaze]

    dom_times = [d["t"] for d in dom]
    out = []
    for g in gaze:
        idx = bisect.bisect_right(dom_times, g["t"]) - 1
        if idx < 0:
            out.append((g["t"], None))
            continue
        snap = dom[idx]
        label = _find_aoi(g["sx"], g["sy"], snap.get("aois", []))
        out.append((g["t"], label))
    return out


# =============================================================================
# METRICS
# =============================================================================

def compute_dwell_ranking(attributed):
    if len(attributed) < 2:
        return []
    totals, hits = {}, {}
    for i in range(len(attributed) - 1):
        t0, label = attributed[i]
        t1, _ = attributed[i + 1]
        dt = max(0.0, t1 - t0)
        if label:
            totals[label] = totals.get(label, 0.0) + dt
            hits[label] = hits.get(label, 0) + 1
    total_time = sum(totals.values()) or 1.0
    ranking = sorted(
        ({"label": friendly_label(k), "seconds": round(v, 2),
          "pct": round(100 * v / total_time, 1), "hits": hits[k]}
         for k, v in totals.items()),
        key=lambda r: -r["seconds"]
    )
    return ranking


def compute_timeline(attributed):
    segments = []
    if not attributed:
        return segments
    cur_raw = attributed[0][1]
    seg_start = attributed[0][0]
    last_t = attributed[0][0]
    for t, label in attributed[1:]:
        if label != cur_raw:
            segments.append({"start": round(seg_start, 2), "end": round(last_t, 2),
                              "label": friendly_label(cur_raw) if cur_raw else None})
            cur_raw = label
            seg_start = t
        last_t = t
    segments.append({"start": round(seg_start, 2), "end": round(last_t, 2),
                      "label": friendly_label(cur_raw) if cur_raw else None})
    return segments


def session_summary(gaze, dom):
    if not gaze:
        return {"duration": 0.0, "url": None, "samples": 0}
    duration = gaze[-1]["t"] - gaze[0]["t"]
    url = dom[-1]["url"] if dom else None
    return {"duration": round(duration, 1), "url": url, "samples": len(gaze)}


# =============================================================================
# SCREENSHOT-BACKED HEATMAP SEGMENTS
# Groups gaze points against whichever screenshot was on screen at the time,
# so the heatmap draws on top of the real page instead of a blank canvas.
# =============================================================================

def build_screenshot_segments(gaze, dom):
    shot_events = [d for d in dom if d.get("screenshot")]
    if not shot_events:
        return []

    shot_times = [d["t"] for d in shot_events]
    buckets = [[] for _ in shot_events]

    for g in gaze:
        idx = bisect.bisect_right(shot_times, g["t"]) - 1
        if idx < 0:
            idx = 0
        buckets[idx].append({"sx": round(g["sx"], 1), "sy": round(g["sy"], 1)})

    segments = []
    for ev, pts in zip(shot_events, buckets):
        if not pts:
            continue
        segments.append({
            "screenshot": ev["screenshot"],
            "scrollY": ev.get("scrollY", 0),
            "points": pts,
            "duration": 0.0,  # filled below
        })

    if segments:
        total_pts = sum(len(s["points"]) for s in segments)
        total_time = (gaze[-1]["t"] - gaze[0]["t"]) if len(gaze) > 1 else 0.0
        for s in segments:
            share = (len(s["points"]) / total_pts) if total_pts else 0
            s["duration"] = round(share * total_time, 1)

    return segments


# =============================================================================
# HTML REPORT
# =============================================================================

_TEMPLATE = r"""<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="insightux-report" content="true">
<title>InsightUX Session Report</title>
<style>
  :root {
    --purple: #7B2FBE; --purple-light: #9B59FF; --pink: #FF2DF0;
    --bg: #14121a; --panel: #1e1b26; --text: #f0ecf7; --dim: #9a92ad;
  }
  * { box-sizing: border-box; }
  body {
    margin: 0; padding: 32px; background: var(--bg); color: var(--text);
    font-family: -apple-system, 'Segoe UI', Arial, sans-serif;
  }
  h1 { margin: 0 0 4px 0; font-size: 22px; }
  .sub { color: var(--dim); font-size: 13px; margin-bottom: 24px; }
  .grid { display: grid; grid-template-columns: 1fr 1fr; gap: 24px; }
  .panel {
    background: var(--panel); border: 1px solid #302a3d; border-radius: 12px;
    padding: 20px;
  }
  .panel h2 { margin: 0 0 14px 0; font-size: 14px; color: var(--purple-light);
              text-transform: uppercase; letter-spacing: 0.06em; }
  table { width: 100%; border-collapse: collapse; font-size: 13px; }
  th { text-align: left; color: var(--dim); font-weight: 500; padding: 6px 8px;
       border-bottom: 1px solid #302a3d; }
  td { padding: 7px 8px; border-bottom: 1px solid #241f2e; }
  .bar-bg { background: #2a2436; border-radius: 4px; height: 6px; margin-top: 4px; overflow: hidden; }
  .bar-fill { height: 100%; background: linear-gradient(90deg, var(--purple), var(--pink)); }
  canvas { display: block; }
  .full { grid-column: 1 / -1; }
  .stat-row { display: flex; gap: 24px; margin-bottom: 20px; }
  .stat { background: var(--panel); border: 1px solid #302a3d; border-radius: 12px;
          padding: 14px 20px; }
  .stat .v { font-size: 20px; font-weight: 600; color: var(--purple-light); }
  .stat .l { font-size: 11px; color: var(--dim); text-transform: uppercase; }
  .empty { color: var(--dim); font-size: 13px; padding: 20px 0; text-align: center; }
  .legend { font-size: 12px; color: var(--dim); line-height: 1.7; }
  .legend b { color: var(--text); }
  .legend-scale { display: flex; align-items: center; gap: 8px; margin: 10px 0; }
  .legend-scale .bar { flex: 1; height: 10px; border-radius: 5px;
    background: linear-gradient(90deg, #0000ff, #00e5ff, #00ff5a, #e8ff00, #ff2d00); }
  .shot-wrap { position: relative; border-radius: 8px; overflow: hidden; background: #100e16; }
  .shot-wrap img, .shot-wrap canvas { display: block; width: 100%; height: auto; }
  .shot-wrap canvas { position: absolute; top: 0; left: 0; }
  .shot-nav { display: flex; align-items: center; justify-content: space-between;
              margin-top: 10px; font-size: 12px; color: var(--dim); }
  .shot-nav button {
    background: #2a2436; border: 1px solid #3a3348; color: var(--text);
    padding: 6px 14px; border-radius: 6px; cursor: pointer; font-size: 12px;
  }
  .shot-nav button:hover { background: #352d44; }
  .shot-nav button:disabled { opacity: 0.35; cursor: default; }
</style>
</head>
<body>

<h1>InsightUX Session Report</h1>
<div class="sub">__URL__ &nbsp;·&nbsp; generated from __SAMPLES__ gaze samples</div>

<div class="stat-row">
  <div class="stat"><div class="v">__DURATION__s</div><div class="l">Session Length</div></div>
  <div class="stat"><div class="v">__NUM_ELEMENTS__</div><div class="l">Elements Fixated</div></div>
  <div class="stat"><div class="v">__TOP_LABEL__</div><div class="l">Most Attended</div></div>
</div>

<div class="grid">
  <div class="panel">
    <h2>Ranked Attention</h2>
    <div id="rankingTable"></div>
  </div>
  <div class="panel">
    <h2>How to read this report</h2>
    <div class="legend">
      <p><b>Ranked Attention</b> — every part of the page you looked at for a
      meaningful stretch of time, ordered by how long you spent there.</p>
      <p><b>Heatmap</b> — drawn directly on a screenshot of the page as it
      appeared during the session. Color shows how much attention that spot
      received:</p>
      <div class="legend-scale">
        <span>Little</span><div class="bar"></div><span>A lot</span>
      </div>
      <p>Each card below is a different scroll position — the report splits
      the page automatically whenever you scrolled far enough that the view
      changed meaningfully.</p>
      <p><b>Timeline</b> — the same attention data laid out across time,
      so you can see the order things were looked at, not just the totals.</p>
    </div>
  </div>
  <div class="panel full">
    <h2>Heatmap by Scroll Position</h2>
    <div id="heatmapArea"></div>
  </div>
  <div class="panel full">
    <h2>Attention Timeline</h2>
    <canvas id="timeline" width="1240" height="320"></canvas>
  </div>
  <div class="panel">
    <h2>Mouse — Most Interacted Elements</h2>
    <div id="mouseInterests"></div>
  </div>
  <div class="panel">
    <h2>Mouse — Click Log</h2>
    <div id="mouseClicks" style="max-height:260px;overflow-y:auto;"></div>
  </div>
</div>

<script>
const RANKING  = __RANKING_JSON__;
const TIMELINE = __TIMELINE_JSON__;
const SEGMENTS = __SEGMENTS_JSON__;
const DURATION = __DURATION_JSON__;
const MOUSE_INTERESTS = __MOUSE_INTERESTS_JSON__;
const MOUSE_CLICKS = __MOUSE_CLICKS_JSON__;

// ---------- Mouse interests / clicks (from the in-page Mouse Tracker) ----------
(function(){
  const el = document.getElementById('mouseInterests');
  if (!MOUSE_INTERESTS.length) {
    el.innerHTML = '<div class="empty">No mouse dwell data recorded for this session.</div>';
  } else {
    let html = '<table><tr><th>#</th><th>Element</th><th>Time</th></tr>';
    MOUSE_INTERESTS.forEach((r, i) => {
      html += `<tr><td>${i+1}</td><td>${r.element}</td><td>${r.seconds}s</td></tr>`;
    });
    html += '</table>';
    el.innerHTML = html;
  }

  const clickEl = document.getElementById('mouseClicks');
  if (!MOUSE_CLICKS.length) {
    clickEl.innerHTML = '<div class="empty">No clicks recorded for this session.</div>';
  } else {
    let html = '<ul style="padding-left:0;margin:0;list-style:none;">';
    MOUSE_CLICKS.slice().reverse().forEach(c => {
      html += `<li style="margin-bottom:8px;border-bottom:1px solid #302a3d;padding-bottom:6px;">
                 <span style="color:var(--dim);font-size:11px;">${c.timestamp || ''}</span><br>
                 <b>${c.element || ''}</b><br>
                 <span style="color:var(--text);font-size:12px;">"${(c.text || '').replace(/</g,'&lt;')}"</span>
               </li>`;
    });
    html += '</ul>';
    clickEl.innerHTML = html;
  }
})();

// ---------- Ranked table ----------
(function(){
  const el = document.getElementById('rankingTable');
  if (!RANKING.length) { el.innerHTML = '<div class="empty">No elements were fixated long enough to register.</div>'; return; }
  let html = '<table><tr><th>#</th><th>Element</th><th>Dwell</th><th>Share</th><th>Hits</th></tr>';
  RANKING.slice(0, 15).forEach((r, i) => {
    html += `<tr>
      <td>${i+1}</td>
      <td>${r.label}<div class="bar-bg"><div class="bar-fill" style="width:${r.pct}%"></div></div></td>
      <td>${r.seconds}s</td>
      <td>${r.pct}%</td>
      <td>${r.hits}</td>
    </tr>`;
  });
  html += '</table>';
  el.innerHTML = html;
})();

// ---------- Classic red/green/blue heat colormap ----------
function jetColor(t){
  t = Math.max(0, Math.min(1, t));
  const stops = [
    [0.00, 0,   0,   255],
    [0.30, 0,   229, 255],
    [0.55, 0,   255, 90],
    [0.75, 232, 255, 0],
    [1.00, 255, 45,  0],
  ];
  for (let i = 0; i < stops.length - 1; i++){
    const [t0,r0,g0,b0] = stops[i], [t1,r1,g1,b1] = stops[i+1];
    if (t >= t0 && t <= t1){
      const k = (t - t0) / (t1 - t0 || 1);
      return [r0+(r1-r0)*k, g0+(g1-g0)*k, b0+(b1-b0)*k];
    }
  }
  return [255,45,0];
}

// ---------- Screenshot-backed heatmaps, paginated ----------
(function(){
  const area = document.getElementById('heatmapArea');
  if (!SEGMENTS.length){
    area.innerHTML = '<div class="empty">No page screenshots were captured for this session ' +
      '(older session, or screen-capture failed) — nothing to overlay a heatmap on.</div>';
    return;
  }

  let cur = 0;

  function renderSegment(i){
    const seg = SEGMENTS[i];
    area.innerHTML = `
      <div class="shot-wrap" id="shotWrap">
        <img id="shotImg" src="${seg.screenshot}">
        <canvas id="shotCanvas"></canvas>
      </div>
      <div class="shot-nav">
        <button id="prevBtn" ${i===0?'disabled':''}>&larr; Earlier</button>
        <span>Scroll segment ${i+1} of ${SEGMENTS.length} &middot; ~${seg.duration}s of attention here</span>
        <button id="nextBtn" ${i===SEGMENTS.length-1?'disabled':''}>Later &rarr;</button>
      </div>
    `;

    const img = document.getElementById('shotImg');
    const cv  = document.getElementById('shotCanvas');
    const ctx = cv.getContext('2d');

    function paint(){
      cv.width = img.naturalWidth;
      cv.height = img.naturalHeight;

      const off = document.createElement('canvas');
      off.width = cv.width; off.height = cv.height;
      const octx = off.getContext('2d');

      seg.points.forEach(p => {
        const grad = octx.createRadialGradient(p.sx, p.sy, 0, p.sx, p.sy, 55);
        grad.addColorStop(0, 'rgba(255,255,255,0.09)');
        grad.addColorStop(1, 'rgba(255,255,255,0)');
        octx.fillStyle = grad;
        octx.beginPath();
        octx.arc(p.sx, p.sy, 55, 0, 2*Math.PI);
        octx.fill();
      });

      const idata = octx.getImageData(0, 0, off.width, off.height);
      const d = idata.data;
      for (let k = 0; k < d.length; k += 4){
        const a = d[k+3] / 255;
        if (a <= 0) continue;
        const t = Math.min(1, a * 3.0);
        const [r,g,b] = jetColor(t);
        d[k] = r; d[k+1] = g; d[k+2] = b; d[k+3] = Math.min(235, a*520);
      }
      octx.putImageData(idata, 0, 0);
      ctx.clearRect(0,0,cv.width,cv.height);
      ctx.drawImage(off, 0, 0);
    }

    if (img.complete) paint(); else img.onload = paint;

    document.getElementById('prevBtn').onclick = () => { if (cur>0){ cur--; renderSegment(cur); } };
    document.getElementById('nextBtn').onclick = () => { if (cur<SEGMENTS.length-1){ cur++; renderSegment(cur); } };
  }

  renderSegment(cur);
})();

// ---------- Timeline ----------
(function(){
  const cv = document.getElementById('timeline');
  const ctx = cv.getContext('2d');
  ctx.fillStyle = '#100e16';
  ctx.fillRect(0, 0, cv.width, cv.height);
  if (!TIMELINE.length || DURATION <= 0) { return; }

  const labels = [...new Set(TIMELINE.filter(s => s.label).map(s => s.label))];
  const rowH = Math.min(28, (cv.height - 40) / Math.max(labels.length, 1));
  const leftPad = 220, topPad = 10, plotW = cv.width - leftPad - 20;

  ctx.font = '11px -apple-system, Arial';
  ctx.fillStyle = '#9a92ad';
  labels.forEach((lab, i) => {
    const y = topPad + i * rowH;
    ctx.fillText(lab.length > 28 ? lab.slice(0,28)+'…' : lab, 4, y + rowH*0.65);
  });

  const colors = ['#7B2FBE', '#9B59FF', '#FF2DF0', '#5EC8D8', '#F0A868', '#7BE0A0'];
  TIMELINE.forEach(seg => {
    if (!seg.label) return;
    const rowIdx = labels.indexOf(seg.label);
    const x = leftPad + (seg.start / DURATION) * plotW;
    const w = Math.max(2, ((seg.end - seg.start) / DURATION) * plotW);
    const y = topPad + rowIdx * rowH + 3;
    ctx.fillStyle = colors[rowIdx % colors.length];
    ctx.fillRect(x, y, w, rowH - 6);
  });

  ctx.strokeStyle = '#302a3d';
  ctx.beginPath();
  ctx.moveTo(leftPad, cv.height - 18);
  ctx.lineTo(cv.width - 10, cv.height - 18);
  ctx.stroke();
  ctx.fillStyle = '#9a92ad';
  [0, 0.25, 0.5, 0.75, 1].forEach(f => {
    const x = leftPad + f * plotW;
    ctx.fillText((f * DURATION).toFixed(1) + 's', x, cv.height - 4);
  });
})();
</script>
</body>
</html>
"""


def generate_report(session_dir):
    gaze, dom = load_session(session_dir)
    attributed = attribute_gaze(gaze, dom)

    ranking = compute_dwell_ranking(attributed)
    timeline = compute_timeline(attributed)
    segments = build_screenshot_segments(gaze, dom)
    summary = session_summary(gaze, dom)
    mouse = summarize_mouse(session_dir)

    top_label = ranking[0]["label"] if ranking else "—"

    html = _TEMPLATE
    html = html.replace("__URL__", summary["url"] or "Unknown page")
    html = html.replace("__SAMPLES__", str(summary["samples"]))
    html = html.replace("__DURATION__", str(summary["duration"]))
    html = html.replace("__NUM_ELEMENTS__", str(len(ranking)))
    html = html.replace("__TOP_LABEL__", top_label)
    html = html.replace("__RANKING_JSON__", json.dumps(ranking))
    html = html.replace("__TIMELINE_JSON__", json.dumps(timeline))
    html = html.replace("__SEGMENTS_JSON__", json.dumps(segments))
    html = html.replace("__DURATION_JSON__", json.dumps(summary["duration"]))
    html = html.replace("__MOUSE_INTERESTS_JSON__", json.dumps(mouse["interests"]))
    html = html.replace("__MOUSE_CLICKS_JSON__", json.dumps(mouse["clicks"]))

    out_path = os.path.join(session_dir, "analysis_report.html")
    with open(out_path, "w", encoding="utf-8") as f:
        f.write(html)

    return os.path.abspath(out_path)


if __name__ == "__main__":
    import sys
    session_dir = sys.argv[1] if len(sys.argv) > 1 else os.path.join("sessions", "live")
    path = generate_report(session_dir)
    print(f"Report written to: {path}")