feat(audit): Tier 4 functionality (API smoke + Playwright E2E) + selftest fixtures

audit/e2e/insurancebot_e2e.js

@@ -0,0 +1,151 @@
+/**
+ * Tier-4 E2E — deterministic UI-presence journeys for the Insurance Advisor
+ * frontend. Run via the playwright-skill runner:
+ *   node /Users/rohitsar/.claude/skills/playwright-skill/run.js \
+ *     audit/e2e/insurancebot_e2e.js
+ *
+ * The runner injects Playwright (resolved from the skill's node_modules) and
+ * passes this file through verbatim because it already has `require(` + an
+ * async IIFE. Every journey resolves to a boolean; everything is wrapped so a
+ * single `RJSON {...}` line is ALWAYS printed to stdout — even on a hard
+ * failure — so the Python check can parse it (no RJSON => WARN/infra, never a
+ * silent FAIL). These are presence/no-error checks only: NO multi-turn LLM
+ * chat (that is flaky/slow and not a deterministic UI signal).
+ */
+const { chromium } = require('playwright');
+
+const TARGET = process.env.TARGET_URL || 'http://localhost:3000';
+
+(async () => {
+  const R = {
+    loads: false,
+    marketplaceRenders: false,
+    logosOk: false,
+    composerWorks: false,
+    noConsoleError: false,
+  };
+
+  let browser;
+  // Console / page errors collected across the whole session.
+  const errors = [];
+
+  try {
+    browser = await chromium.launch({ headless: true });
+    const context = await browser.newContext({
+      viewport: { width: 1440, height: 900 },
+    });
+    const page = await context.newPage();
+
+    page.on('pageerror', (e) => errors.push('pageerror: ' + (e && e.message)));
+    page.on('console', (msg) => {
+      if (msg.type() === 'error') errors.push('console.error: ' + msg.text());
+    });
+
+    // ---- journey: loads -------------------------------------------------
+    try {
+      await page.goto(TARGET, { waitUntil: 'domcontentloaded', timeout: 60000 });
+      await page.waitForSelector('h1', { state: 'visible', timeout: 30000 });
+      R.loads = (await page.locator('h1').count()) > 0;
+    } catch (e) {
+      errors.push('loads: ' + (e && e.message));
+    }
+
+    // ---- journey: marketplaceRenders -----------------------------------
+    // Open the Policy-Library / marketplace panel, assert >=1 policy card is
+    // visible, and assert no horizontal overflow at 1440px.
+    try {
+      const trigger = page
+        .locator(
+          'button:has-text("Policy Library"), button:has-text("पॉलिसी लाइब्रेरी")'
+        )
+        .first();
+      await trigger.click({ timeout: 20000 });
+      await page.waitForSelector('.policy-card', {
+        state: 'visible',
+        timeout: 30000,
+      });
+      const cards = await page.locator('.policy-card:visible').count();
+      const overflow = await page.evaluate(
+        () =>
+          document.documentElement.scrollWidth >
+          window.innerWidth + 1
+      );
+      R.marketplaceRenders = cards >= 1 && !overflow;
+    } catch (e) {
+      errors.push('marketplaceRenders: ' + (e && e.message));
+    }
+
+    // ---- journey: logosOk ----------------------------------------------
+    // At least one insurer logo <img src*="insurer-logos"> with naturalWidth>0.
+    try {
+      await page
+        .waitForSelector('img[src*="insurer-logos"]', { timeout: 20000 })
+        .catch(() => {});
+      const logoLoaded = await page.evaluate(() => {
+        const imgs = Array.from(
+          document.querySelectorAll('img[src*="insurer-logos"]')
+        );
+        if (imgs.length === 0) return false;
+        return imgs.some((im) => im.naturalWidth && im.naturalWidth > 0);
+      });
+      R.logosOk = logoLoaded;
+    } catch (e) {
+      errors.push('logosOk: ' + (e && e.message));
+    }
+
+    // ---- journey: composerWorks ----------------------------------------
+    // The chat composer textarea is fillable and the Send button is clickable
+    // once non-empty. (We do NOT actually submit a turn — no LLM dependency.)
+    try {
+      // The marketplace panel is a single-active <aside> that REPLACES the
+      // chat view, so the composer textarea stays in the DOM but is hidden
+      // while the panel is open. Toggle the Policy-Library trigger again to
+      // close the panel (its onClick is togglePanel(...)), then wait for the
+      // composer to become visible again.
+      const ml = page
+        .locator(
+          'button:has-text("Policy Library"), button:has-text("पॉलिसी लाइब्रेरी")'
+        )
+        .first();
+      if (await ml.isVisible().catch(() => false)) {
+        await ml.click({ timeout: 10000 }).catch(() => {});
+      }
+      const ta = page.locator('textarea[aria-label="Message"]').first();
+      await ta.waitFor({ state: 'visible', timeout: 20000 });
+      await ta.fill('audit e2e presence check');
+      const val = await ta.inputValue();
+      const send = page
+        .locator('button:has-text("Send")')
+        .first();
+      // After typing, Send must no longer be disabled (it gates on input).
+      const sendEnabled = await send.isEnabled().catch(() => false);
+      R.composerWorks = val === 'audit e2e presence check' && sendEnabled;
+    } catch (e) {
+      errors.push('composerWorks: ' + (e && e.message));
+    }
+
+    // ---- journey: noConsoleError ---------------------------------------
+    // No pageerror / console.error captured during the whole session.
+    R.noConsoleError = errors.every(
+      (m) =>
+        !m.startsWith('pageerror:') && !m.startsWith('console.error:')
+    );
+  } catch (fatal) {
+    // Hard failure (browser launch, etc.) — journeys stay false; still emit
+    // RJSON so the Python side classifies (FAIL/WARN), never a silent hang.
+    errors.push('fatal: ' + (fatal && fatal.message));
+  } finally {
+    try {
+      if (browser) await browser.close();
+    } catch (_) {
+      /* ignore */
+    }
+  }
+
+  if (errors.length) {
+    console.log('E2E_DIAG ' + JSON.stringify(errors.slice(0, 20)));
+  }
+  console.log('RJSON ' + JSON.stringify(R));
+  // Exit 0 regardless: the boolean payload (not the exit code) is the signal.
+  process.exit(0);
+})();

audit/selftest_fixtures.py

@@ -311,6 +311,156 @@ def _f_t3_3():
             helper.unlink()
 
 
+# ---------------------------------------------------------------------------
+# Tier 4 fixtures.
+#
+# T4 checks hit a LIVE local backend / frontend / playwright runner, so a
+# file-on-disk fixture (the T1–T3 pattern) cannot force a deterministic FAIL.
+# Instead each fixture monkeypatches the *module seams* of
+# `audit.tier4_functional` (`_backend_up`, `_get`, `_post`, the module's
+# `urllib.request.urlopen`, `sh`) so the check exercises its real logic against
+# a controlled response and returns FAIL. Every patch is captured BEFORE the
+# yield and restored UNCONDITIONALLY in `finally` — even if the check raises —
+# so no module attr (and no file) is left mutated. No repo files are touched.
+# ---------------------------------------------------------------------------
+
+
+@contextlib.contextmanager
+def _f_t4_1():
+    """Backend 'up' but /api/health reports a non-ok status -> T4.1 FAIL."""
+    import audit.tier4_functional as m
+    orig_up, orig_get = m._backend_up, m._get
+    m._backend_up = lambda: True
+    m._get = lambda p, timeout=20: (200, '{"status":"bad"}')
+    try:
+        yield
+    finally:
+        m._backend_up, m._get = orig_up, orig_get
+
+
+@contextlib.contextmanager
+def _f_t4_2():
+    """Coverage counts far outside the sane window -> T4.2 FAIL."""
+    import audit.tier4_functional as m
+    orig_up, orig_get = m._backend_up, m._get
+    m._backend_up = lambda: True
+    m._get = lambda p, timeout=20: (
+        200,
+        '{"total_policies":1,"total_insurers":1,"total_chunks":1}',
+    )
+    try:
+        yield
+    finally:
+        m._backend_up, m._get = orig_up, orig_get
+
+
+@contextlib.contextmanager
+def _f_t4_3():
+    """Chat returns an empty reply_text -> T4.3 FAIL (empty reply)."""
+    import audit.tier4_functional as m
+    orig_up, orig_post = m._backend_up, m._post
+    m._backend_up = lambda: True
+    m._post = lambda p, payload, timeout=90: (
+        200,
+        '{"reply_text":"","brain_used":"x"}',
+    )
+    try:
+        yield
+    finally:
+        m._backend_up, m._post = orig_up, orig_post
+
+
+@contextlib.contextmanager
+def _f_t4_4():
+    """Make the junk-PDF upload POST 'succeed' (fake 200) -> T4.4 FAIL.
+
+    T4.4 calls `urllib.request.urlopen` directly (not via `_post`). A real junk
+    upload raises HTTPError(4xx) -> PASS. To force the FAIL branch we patch the
+    `urllib.request.urlopen` *as referenced inside tier4_functional* so the
+    POST returns a fake 200 response instead of raising — i.e. the app
+    'accepted junk', which T4.4 must report as FAIL.
+    """
+    import audit.tier4_functional as m
+    orig_up = m._backend_up
+    orig_urlopen = m.urllib.request.urlopen
+
+    class _FakeResp:
+        status = 200
+
+        def read(self):
+            return b"{}"
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *a):
+            return False
+
+    def _fake_urlopen(req, *a, **k):
+        return _FakeResp()
+
+    m._backend_up = lambda: True
+    m.urllib.request.urlopen = _fake_urlopen
+    try:
+        yield
+    finally:
+        m._backend_up = orig_up
+        m.urllib.request.urlopen = orig_urlopen
+
+
+@contextlib.contextmanager
+def _f_t4_5():
+    """Profile/session endpoints return HTTP 500 -> T4.5 FAIL."""
+    import audit.tier4_functional as m
+    orig_up, orig_get, orig_post = m._backend_up, m._get, m._post
+    m._backend_up = lambda: True
+    m._get = lambda p, timeout=20: (500, "err")
+    m._post = lambda p, payload, timeout=90: (500, "err")
+    try:
+        yield
+    finally:
+        m._backend_up, m._get, m._post = orig_up, orig_get, orig_post
+
+
+@contextlib.contextmanager
+def _f_t4_e2e():
+    """Frontend 'reachable' + e2e run reports a failed journey -> T4.E2E FAIL.
+
+    PW_RUN exists on this machine, so `os.path.exists(PW_RUN)` is already True.
+    We patch the module's `urllib.request.urlopen` so the FE :3000 reachability
+    probe 'passes' (fake 200) regardless of whether a dev server is up, and
+    patch the module's `sh` to return a fake CompletedProcess whose stdout is
+    `RJSON {"loads":false}` — a failing journey, which T4.E2E must report as
+    FAIL. No real browser/runner is spawned.
+    """
+    import subprocess
+    import audit.tier4_functional as m
+    orig_urlopen = m.urllib.request.urlopen
+    orig_sh = m.sh
+
+    class _FakeResp:
+        status = 200
+
+        def read(self):
+            return b"<html><h1>x</h1></html>"
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *a):
+            return False
+
+    m.urllib.request.urlopen = lambda req, *a, **k: _FakeResp()
+    m.sh = lambda cmd, timeout=120: subprocess.CompletedProcess(
+        cmd, 0, stdout='RJSON {"loads":false}\n', stderr=""
+    )
+    try:
+        yield
+    finally:
+        m.urllib.request.urlopen = orig_urlopen
+        m.sh = orig_sh
+
+
 FIXTURES.update({
     "T1.1": _f_t1_1,
     "T1.2": _f_t1_2,
@@ -326,4 +476,10 @@ FIXTURES.update({
     "T3.1": _f_t3_1,
     "T3.2": _f_t3_2,
     "T3.3": _f_t3_3,
+    "T4.1": _f_t4_1,
+    "T4.2": _f_t4_2,
+    "T4.3": _f_t4_3,
+    "T4.4": _f_t4_4,
+    "T4.5": _f_t4_5,
+    "T4.E2E": _f_t4_e2e,
 })

audit/tier4_functional.py

@@ -1 +1,147 @@
-# filled in a later task
+"""Tier 4 — functionality: API smoke (deterministic) + Playwright E2E.
+Infra-dependent: SKIPs (never false-PASS, never crash) when the local
+backend / frontend / playwright runner is unavailable."""
+from __future__ import annotations
+import json, os, time, urllib.request, urllib.error
+from audit.core import register, Result, Status, REPO, sh
+
+BK = "http://127.0.0.1:8000"
+FE = "http://localhost:3000"
+PW_RUN = "/Users/rohitsar/.claude/skills/playwright-skill/run.js"
+
+
+def _get(path, timeout=20):
+    with urllib.request.urlopen(BK + path, timeout=timeout) as f:
+        return f.status, f.read().decode("utf-8", "replace")
+
+
+def _post(path, payload, timeout=90):
+    req = urllib.request.Request(
+        BK + path, data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json"}, method="POST")
+    with urllib.request.urlopen(req, timeout=timeout) as f:
+        return f.status, f.read().decode("utf-8", "replace")
+
+
+def _backend_up():
+    try:
+        s, b = _get("/api/health", timeout=6)
+        return s == 200 and '"status":"ok"' in b
+    except Exception:
+        return False
+
+
+@register("T4.1", "functional", "API: health + version")
+def t4_1():
+    if not _backend_up():
+        return Result("T4.1", Status.SKIP, "no local backend on :8000", "start uvicorn backend.main:app --port 8000")
+    try:
+        _, hb = _get("/api/health")
+        _, vb = _get("/api/version")
+        ok = '"status":"ok"' in hb and json.loads(vb) is not None
+        return Result("T4.1", Status.PASS if ok else Status.FAIL,
+                      "health ok + version json" if ok else f"health={hb[:80]} version={vb[:80]}",
+                      "" if ok else "investigate /api/health or /api/version")
+    except Exception as e:
+        return Result("T4.1", Status.FAIL, f"{type(e).__name__}: {e}", "endpoint error")
+
+
+@register("T4.2", "functional", "API: coverage counts sane")
+def t4_2():
+    if not _backend_up():
+        return Result("T4.2", Status.SKIP, "no local backend", "start backend")
+    try:
+        _, b = _get("/api/coverage", timeout=40)
+        d = json.loads(b)
+        p, i, c = d.get("total_policies"), d.get("total_insurers"), d.get("total_chunks")
+        ok = isinstance(p, int) and 130 <= p <= 170 and i == 20 and isinstance(c, int) and c > 5000
+        return Result("T4.2", Status.PASS if ok else Status.FAIL,
+                      f"policies={p} insurers={i} chunks={c}",
+                      "" if ok else "coverage outside expected (~148 policies / 20 insurers / >5000 chunks)")
+    except Exception as e:
+        return Result("T4.2", Status.FAIL, f"{type(e).__name__}: {e}", "coverage endpoint error")
+
+
+@register("T4.3", "functional", "API: chat returns a grounded reply")
+def t4_3():
+    if not _backend_up():
+        return Result("T4.3", Status.SKIP, "no local backend", "start backend")
+    try:
+        _, b = _post("/api/chat",
+                     {"user_text": "What is a waiting period in health insurance?",
+                      "session_id": f"audit-smoke-{int(time.time())}"}, timeout=90)
+        d = json.loads(b)
+        reply = (d.get("reply_text") or "").strip()
+        brain = d.get("brain_used", "")
+        if not reply:
+            return Result("T4.3", Status.FAIL, f"empty reply (brain={brain})", "chat returned no text")
+        if "error_fallback" in brain:
+            return Result("T4.3", Status.WARN, f"reply via {brain}", "LLM chain degraded (env/keys) — verify live")
+        return Result("T4.3", Status.PASS, f"reply ok ({len(reply)} chars, brain={brain})")
+    except Exception as e:
+        return Result("T4.3", Status.WARN, f"{type(e).__name__}: {e}",
+                      "chat slow/unavailable (LLM dependency); not a code FAIL on its own")
+
+
+@register("T4.4", "functional", "API: upload-policy rejects junk")
+def t4_4():
+    if not _backend_up():
+        return Result("T4.4", Status.SKIP, "no local backend", "start backend")
+    boundary = "----auditST"
+    body = (f"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; "
+            f"filename=\"junk.pdf\"\r\nContent-Type: application/pdf\r\n\r\n"
+            + "not a real pdf " * 10 + f"\r\n--{boundary}--\r\n").encode()
+    req = urllib.request.Request(BK + "/api/upload-policy", data=body, method="POST",
+                                 headers={"Content-Type": f"multipart/form-data; boundary={boundary}"})
+    try:
+        urllib.request.urlopen(req, timeout=30)
+        return Result("T4.4", Status.FAIL, "junk PDF was accepted", "upload security gates not rejecting")
+    except urllib.error.HTTPError as e:
+        return (Result("T4.4", Status.PASS, f"junk rejected (HTTP {e.code})")
+                if e.code in (400, 413, 415, 422)
+                else Result("T4.4", Status.FAIL, f"unexpected HTTP {e.code}", "check upload gate"))
+    except Exception as e:
+        return Result("T4.4", Status.WARN, f"{type(e).__name__}: {e}", "upload endpoint unreachable")
+
+
+@register("T4.5", "functional", "API: profile + session endpoints")
+def t4_5():
+    if not _backend_up():
+        return Result("T4.5", Status.SKIP, "no local backend", "start backend")
+    sid = f"audit-smoke-{int(time.time())}"
+    try:
+        s1, _ = _get(f"/api/profile/completeness?session_id={sid}", timeout=20)
+        s2, _ = _post("/api/session/clear", {"session_id": sid}, timeout=20)
+        ok = s1 == 200 and s2 == 200
+        return Result("T4.5", Status.PASS if ok else Status.FAIL,
+                      f"profile/completeness={s1} session/clear={s2}",
+                      "" if ok else "profile/session endpoint non-200")
+    except Exception as e:
+        return Result("T4.5", Status.FAIL, f"{type(e).__name__}: {e}", "profile/session error")
+
+
+@register("T4.E2E", "functional", "Frontend E2E journeys (Playwright)")
+def t4_e2e():
+    if not os.path.exists(PW_RUN):
+        return Result("T4.E2E", Status.SKIP, "playwright-skill runner not found",
+                      "install/locate the playwright-skill to enable E2E")
+    try:
+        with urllib.request.urlopen(FE, timeout=6) as f:
+            if f.status != 200:
+                raise RuntimeError("frontend not 200")
+    except Exception:
+        return Result("T4.E2E", Status.SKIP, "no frontend on :3000",
+                      "run `cd frontend && npm run dev` to enable E2E")
+    r = sh(["node", PW_RUN, str(REPO / "audit/e2e/insurancebot_e2e.js")], timeout=300)
+    out = r.stdout + r.stderr
+    import re
+    m = re.search(r"RJSON (\{.*\})", out)
+    if not m:
+        return Result("T4.E2E", Status.WARN, "no RJSON from e2e run (infra/flake)",
+                      "inspect audit/e2e/insurancebot_e2e.js output; not a code FAIL alone")
+    d = json.loads(m.group(1))
+    fails = [k for k, v in d.items() if v is False]
+    if fails:
+        return Result("T4.E2E", Status.FAIL, f"E2E journey failures: {fails}",
+                      "investigate the failing UI journey")
+    return Result("T4.E2E", Status.PASS, f"E2E ok: {sorted(d)}")