fsanyoto commited on
Commit
e78e601
·
verified ·
1 Parent(s): eb27769

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. RELEASES.json +7 -1
  2. VERSION +1 -1
  3. api/automation_engine.py +411 -19
  4. api/connectors_ig.py +169 -18
  5. api/odoo_relational.py +19 -0
  6. api/providers.py +12 -0
  7. api/rollup_sql.py +164 -0
  8. api/routes_alerts.py +254 -254
  9. api/routes_auth.py +217 -217
  10. api/routes_customers.py +346 -346
  11. api/routes_grid.py +832 -832
  12. api/routes_keychain.py +278 -278
  13. api/routes_platform_admin.py +618 -618
  14. api/routes_products.py +294 -294
  15. platform/aios_grid.py +0 -0
  16. platform/core/grid_events.py +0 -0
  17. platform/core/links.py +59 -59
  18. platform/core/registry.py +199 -199
  19. platform/core/store_pg.py +325 -325
  20. platform/core/table_store.py +595 -595
  21. platform/core/user_tables.py +52 -0
  22. platform/evals/analyst_golden.yml +104 -104
  23. platform/harness/filter_sql.py +673 -673
  24. platform/harness/semantic.py +34 -3
  25. platform/harness/tables.py +129 -129
  26. platform/harness/tools.py +510 -510
  27. platform/harness/windows.py +390 -390
  28. platform/model/metrics/returns.yml +55 -55
  29. platform/model/skills/sales.skill.yml +184 -184
  30. platform/model/topics/credit_notes.yml +48 -48
  31. platform/modules/customer_data.py +684 -684
  32. platform/modules/customers.py +0 -0
  33. platform/modules/digest.py +227 -227
  34. platform/modules/pricing.py +395 -395
  35. platform/procurement_suppliers.json +0 -0
  36. requirements.txt +47 -47
  37. web/public/sample_customers.json +529 -529
  38. web/src/alerts/AlertsPane.tsx +329 -329
  39. web/src/apiContract.ts +237 -237
  40. web/src/automation/AutomationBuilder.tsx +0 -0
  41. web/src/automation/AutomationDetail.tsx +0 -0
  42. web/src/automation/AutomationFind.tsx +893 -893
  43. web/src/automation/AutomationSurface.tsx +663 -663
  44. web/src/automation/automationApi.ts +0 -0
  45. web/src/automation/steps.ts +422 -422
  46. web/src/customer-grid/ColumnMenu.tsx +0 -0
  47. web/src/customer-grid/CustomerGrid.tsx +0 -0
  48. web/src/customer-grid/MapView.tsx +946 -946
  49. web/src/customer-grid/OverlaySurface.tsx +355 -355
  50. web/src/customer-grid/RecordDetail.tsx +0 -0
RELEASES.json CHANGED
@@ -1,6 +1,12 @@
1
  {
2
- "current": "v20 (442e4a1)",
3
  "releases": [
 
 
 
 
 
 
4
  {
5
  "version": "v20",
6
  "sha": "442e4a1",
 
1
  {
2
+ "current": "v21 (5fdffce)",
3
  "releases": [
4
+ {
5
+ "version": "v21",
6
+ "sha": "5fdffce",
7
+ "date": "2026-08-09",
8
+ "subject": "release v21"
9
+ },
10
  {
11
  "version": "v20",
12
  "sha": "442e4a1",
VERSION CHANGED
@@ -1 +1 @@
1
- v20 (442e4a1)
 
1
+ v21 (5fdffce)
api/automation_engine.py CHANGED
@@ -2252,18 +2252,32 @@ def _release(tenant, auto_id):
2252
  _RUNNING.pop((tenant, str(auto_id)), None)
2253
 
2254
 
2255
- def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None):
2256
  """THE ONE store write a run performs on the `automations` bucket.
2257
 
2258
  `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here
2259
  because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to
2260
  invent a node status when this map is absent (every run stored before W19-C has no `steps`,
2261
  and painting those nodes green from the run's overall state would be a fabrication).
 
 
 
 
 
 
 
 
 
 
 
2262
  """
2263
  entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400),
2264
  "counts": {k: int(v) for k, v in (counts or {}).items()
2265
  if isinstance(v, (int, float))},
2266
  "steps": {str(k): str(v) for k, v in (steps or {}).items()},
 
 
 
2267
  "affected": list(affected or [])[:200]}
2268
 
2269
  def _up(cur):
@@ -2572,11 +2586,13 @@ POST_SNAPSHOT_FIELDS = [
2572
  # and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a
2573
  # module-level import, the names it wants already exist and the cycle still resolves.
2574
  from connectors_ig import ( # noqa: E402
2575
- BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_REELS, BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT,
2576
  BD_RECORD_PRICE_SPEC,
2577
- _bd_comment, _bd_deferral, _bd_first_url, _bd_post_metrics, _bd_rows, _bd_tagged_location,
 
2578
  _first, _ig_int,
2579
- bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready, depth_refusal,
 
2580
  ig_handle, pull_profile,
2581
  )
2582
 
@@ -6249,6 +6265,26 @@ DEFAULT_ENRICH_COOLDOWN_DAYS = 30
6249
  #: discovery just found rather than re-walking the oldest leads in the book.
6250
  DEFAULT_ENRICH_SORT = "first_found"
6251
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6252
 
6253
  def _order_key(value):
6254
  """Order one CELL. Numbers numerically, everything else as text.
@@ -6286,7 +6322,41 @@ def _days_since(day, today=None):
6286
  return ((today or _dt.date.today()) - d).days
6287
 
6288
 
6289
- def enrich_selection(rt, table_key, cfg, profile_key, today=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6290
  """⭐ 2026-08-07 (owner ruling) — WHICH records this enrich step spends on, in order.
6291
 
6292
  Returns `(ordered_row_ids, note)`. The note is the honest account of what the selection did
@@ -6403,12 +6473,32 @@ def enrich_selection(rt, table_key, cfg, profile_key, today=None):
6403
  days = max(1, days)
6404
 
6405
  picked, cooled, blank_handle = [], 0, 0
 
 
 
 
 
 
 
 
 
 
 
 
 
6406
  for rid, r in ordered:
6407
  if len(picked) >= quota:
6408
  break
6409
- if not str(r.get(profile_key) or "").strip():
 
6410
  blank_handle += 1
6411
  continue
 
 
 
 
 
 
6412
  if cooling:
6413
  since = _days_since(r.get("enriched_at"), today=today)
6414
  if since is not None and since < days:
@@ -6421,9 +6511,14 @@ def enrich_selection(rt, table_key, cfg, profile_key, today=None):
6421
  # `run_plain` follow. Silence here would make a shrinking selection invisible.
6422
  if cooled:
6423
  notes.append(f"{cooled} skipped as enriched in the last {days} days")
 
 
 
 
 
6424
  if blank_handle:
6425
  notes.append(f"{blank_handle} skipped with no handle")
6426
- if len(picked) < quota and (cooled or blank_handle or ordered):
6427
  notes.append(f"{len(picked)} of the {quota} asked for — the list ran out")
6428
  return picked, "; ".join(notes)
6429
 
@@ -6494,7 +6589,14 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
6494
  # entry, so calling it per record is O(existing) per record — survivable at 5,000 rows and an
6495
  # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts.
6496
  enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
 
 
 
 
6497
  "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False}
 
 
 
6498
  # ⭐ 2026-08-07 — THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD.
6499
  # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk
6500
  # would be O(rows²) and, worse, would re-answer "which 25 records" every time a record walked
@@ -6630,7 +6732,7 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
6630
  # not this run's work, and the selection's own note already accounts for it.
6631
  aid = str(act.get("id") or "")
6632
  if aid not in enrich_plan:
6633
- chosen, sel_note = enrich_selection(rt, table, cfg, pkey)
6634
  enrich_plan[aid] = set(chosen)
6635
  if sel_note:
6636
  enrich["notes"].append(sel_note)
@@ -6642,6 +6744,10 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
6642
  if enrich["profiles"]:
6643
  time.sleep(PACE_SECONDS) # >=2 s between profiles (R7), as the runner does
6644
  enrich["profiles"] += 1
 
 
 
 
6645
  res = pull_profile(handle_raw,
6646
  max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
6647
  log=log,
@@ -6650,7 +6756,12 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
6650
  post_metrics=bool(cfg.get("postMetrics")),
6651
  comment_metrics=bool(cfg.get("commentMetrics")),
6652
  pending_metrics=(enrich["pending"]
 
 
6653
  if not cfg.get("dryRun") else None))
 
 
 
6654
  pulled = _iso()
6655
  if res["state"] in ("ok", "partial"):
6656
  enrich["ok"] += 1
@@ -6671,7 +6782,19 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
6671
  _act_row_patch(patches, table, rid, cells)
6672
  else:
6673
  enrich["blocked"] += 1
6674
- enrich["notes"].append(_s(res.get("note"), 90) or res["state"])
 
 
 
 
 
 
 
 
 
 
 
 
6675
  elif kind == "find_records":
6676
  found = find_records(rt, cfg.get("table"), cfg.get("cond"),
6677
  int(cfg.get("limit") or 25))
@@ -6782,10 +6905,32 @@ def _enrich_flush(rt, defn, username, acc, log):
6782
  # ⚠ IT IS A COUNT, not a flag, so `run_now`'s existing non-zero merge carries it without a
6783
  # special case — and so the run entry itself records that this happened.
6784
  return {"enrichUnbound": 1}
 
 
 
 
 
 
6785
  if not acc.get("profiles"):
6786
- return {}
 
6787
  out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]}
 
 
 
 
6788
  if not acc.get("dry"):
 
 
 
 
 
 
 
 
 
 
 
6789
  queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""),
6790
  acc.get("pending") or [])
6791
  if queued:
@@ -6924,6 +7069,189 @@ def queue_pending_metric_snapshots(rt, auto_id, pending):
6924
  return added[0]
6925
 
6926
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6927
  def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log):
6928
  """Write a completed metric snapshot through the canonical Post/Comment graph once."""
6929
  if not (idents or snapshots or comments):
@@ -6974,11 +7302,23 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
6974
  return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_metrics": "idle"})
6975
  step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}")
6976
  remaining, idents, snapshots, comments = [], [], [], []
6977
- ready, waiting = 0, 0
6978
  for task in tasks:
 
 
 
 
 
 
 
 
 
 
 
 
6979
  payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"})
6980
  rows = _bd_rows(payload) if not note else []
6981
- building = (not rows or _bd_deferral(payload) or
6982
  (len(rows) == 1 and str(rows[0].get("status") or "") in
6983
  ("running", "building", "collecting")))
6984
  if building:
@@ -7014,14 +7354,25 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print,
7014
  {"pendingMetricSnapshots": remaining or None})
7015
  written = _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log)
7016
  counts = {"metricBatchesCollected": ready, "metricBatchesPending": waiting,
 
7017
  "postEngagementSnapshots": written["snapshots"],
7018
  "commentsCollected": written["comments"]}
 
 
 
 
 
 
 
 
 
 
7019
  if waiting:
7020
  return ("partial",
7021
  f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; "
7022
- f"{waiting} still building and will be collected automatically",
7023
  counts, [], {"capture_metrics": "partial", "write": "ok"})
7024
- return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected",
7025
  counts, [], {"capture_metrics": "ok", "write": "ok"})
7026
 
7027
 
@@ -8740,12 +9091,27 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None):
8740
  # first and do not start another profile scrape while it is outstanding: re-running the
8741
  # action would buy duplicate engagement reads and reintroduce the timeout this handoff
8742
  # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`.
 
 
 
 
 
 
 
 
 
 
 
 
 
8743
  if _pending_metric_tasks(defn):
8744
  state, summary, counts, affected, steps = collect_pending_metric_snapshots(
8745
  rt, defn, username=username, log=log,
8746
  step=lambda text: _step(tenant, auto_id, text))
8747
- return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected,
8748
- steps)
 
 
8749
  runner = RUNNERS.get(defn.get("kind"))
8750
  if runner is None:
8751
  return _commit_run(rt, auto_id, "error",
@@ -8779,9 +9145,17 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None):
8779
  # A failing action must not fail the RUN: the machine steps already wrote their rows and
8780
  # reporting that as an error would misdescribe what happened. It degrades to `partial`
8781
  # with the reason in the summary — the cap_note discipline.
 
 
 
 
8782
  try:
8783
  a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [],
8784
  username=username, log=log)
 
 
 
 
8785
  counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}}
8786
  if a_counts.get("enrichMetricBatchesPending"):
8787
  batches = int(a_counts["enrichMetricBatchesPending"])
@@ -8798,9 +9172,22 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None):
8798
  summary += (" — the Instagram step did not run: this database has no profile "
8799
  "column. Name one on the step, or mark a text column as the "
8800
  "Instagram profile")
 
 
 
 
 
 
 
8801
  if a_counts.get("enrichBlocked"):
8802
  state = "partial" if state != "error" else state
8803
- summary += f"{int(a_counts['enrichBlocked'])} profile read(s) were blocked"
 
 
 
 
 
 
8804
  # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the
8805
  # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it
8806
  # logged the cap and rolled up `ok`, so a flow that had silently stopped writing
@@ -8814,7 +9201,7 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None):
8814
  state = "partial" if state != "error" else state
8815
  summary = f"{summary} — the actions did not finish ({type(e).__name__})"
8816
  return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected,
8817
- steps)
8818
  finally:
8819
  _release(tenant, auto_id)
8820
 
@@ -8857,7 +9244,12 @@ def pending_collect_ids(rt):
8857
  continue
8858
  pending_discovery = (d.get("kind") == "discover_instagram"
8859
  and str((d.get("state") or {}).get("pendingSnapshot") or "").strip())
8860
- if pending_discovery or _pending_metric_tasks(d):
 
 
 
 
 
8861
  out.append(aid)
8862
  return sorted(out)
8863
 
 
2252
  _RUNNING.pop((tenant, str(auto_id)), None)
2253
 
2254
 
2255
+ def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None, notes=None):
2256
  """THE ONE store write a run performs on the `automations` bucket.
2257
 
2258
  `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here
2259
  because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to
2260
  invent a node status when this map is absent (every run stored before W19-C has no `steps`,
2261
  and painting those nodes green from the run's overall state would be a fabrication).
2262
+
2263
+ ⭐⭐ `notes` CLOSES DEBT D-103 (2026-08-09). A run's `counts` are integers and the
2264
+ comprehension below drops everything else — so the per-record sentence a vendor gave us had
2265
+ literally nowhere to live, and *"1 profile read(s) were blocked"* was the whole of what the
2266
+ product could say. MEASURED on nurilab: the same record blocked on three separate runs and
2267
+ the reason was unrecoverable from the store afterwards, so the diagnosis had to be rebuilt by
2268
+ calling the vendors by hand. The note is the most valuable thing a run produces — a dead
2269
+ handle, a vendor hiccup and an exhausted key are three different actions — and it was the one
2270
+ thing thrown away.
2271
+ ⚠ ON THE RUN, NOT ON THE RECORD. D-103's own prescription: no tenant table gains a column it
2272
+ did not ask for, and W25/R4 already retired writing status STRINGS into people's grids.
2273
  """
2274
  entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400),
2275
  "counts": {k: int(v) for k, v in (counts or {}).items()
2276
  if isinstance(v, (int, float))},
2277
  "steps": {str(k): str(v) for k, v in (steps or {}).items()},
2278
+ # Bounded on both axes: a 100-profile run must not put 100 sentences into a store
2279
+ # entry that is kept 20 deep per automation.
2280
+ "notes": [_s(n, 300) for n in (notes or []) if str(n or "").strip()][:25],
2281
  "affected": list(affected or [])[:200]}
2282
 
2283
  def _up(cur):
 
2586
  # and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a
2587
  # module-level import, the names it wants already exist and the cycle still resolves.
2588
  from connectors_ig import ( # noqa: E402
2589
+ BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_PROFILES, BD_DS_REELS, BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT,
2590
  BD_RECORD_PRICE_SPEC,
2591
+ _bd_comment, _bd_deferral, _bd_first_url, _bd_post_metrics, _bd_profile, _bd_rows,
2592
+ _bd_tagged_location,
2593
  _first, _ig_int,
2594
+ bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready,
2595
+ bd_snapshot_progress, depth_refusal,
2596
  ig_handle, pull_profile,
2597
  )
2598
 
 
6265
  #: discovery just found rather than re-walking the oldest leads in the book.
6266
  DEFAULT_ENRICH_SORT = "first_found"
6267
 
6268
+ #: ⭐⭐ 2026-08-09 (DEBT D-103) — WHERE A RUN'S PER-RECORD REASONS TRAVEL.
6269
+ #: A run's `counts` are integers and `_commit_run` drops everything non-numeric, so the sentence
6270
+ #: a vendor gave us had nowhere to ride and was thrown away at three separate layers. This key is
6271
+ #: a deliberate passenger IN the counts dict, popped by `run_now` before the counts are stored.
6272
+ #: ⛔ A RESERVED KEY, NOT A COUNT: it must never be rendered as one, which is why it starts with
6273
+ #: an underscore — `COUNT_LABELS` on the client is an allow-list and cannot pick it up by
6274
+ #: accident, and `_commit_run`'s numeric filter is the second net under it.
6275
+ RUN_NOTES_KEY = "_notes"
6276
+ #: How long a handle a VENDOR SAID DOES NOT EXIST is left alone before it is tried again.
6277
+ #: ⚠ NOT permanent, and not the success cooldown either. Permanent would be wrong — a handle can
6278
+ #: be renamed back or a suspension lifted — but re-buying a dead account every morning is what
6279
+ #: the owner reported ("blocked AGAIN"), so the floor is one paid attempt per handle per month
6280
+ #: instead of one per day. Editing the handle changes the key and retries immediately, which is
6281
+ #: the behaviour someone correcting a typo expects.
6282
+ NOT_FOUND_RETRY_DAYS = 30
6283
+ #: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An
6284
+ #: unbounded pending list is the forever-loop this whole change exists to remove, wearing the
6285
+ #: opposite mask ([[gate-answers-the-wrong-question]]).
6286
+ PENDING_PROFILE_MAX_HOURS = 24
6287
+
6288
 
6289
  def _order_key(value):
6290
  """Order one CELL. Numbers numerically, everything else as text.
 
6322
  return ((today or _dt.date.today()) - d).days
6323
 
6324
 
6325
+ def _hours_since(stamp):
6326
+ """Whole hours since a full ISO stamp, or None when it cannot be read.
6327
+
6328
+ ⚠ ISO, NOT `YYYY-MM-DD` — `_days_since` above answers a question in DAYS about a `date`
6329
+ column and truncates to 10 characters. A pending snapshot's age is measured in hours and its
6330
+ stamp carries a time, so reusing that function would read every entry as "queued at
6331
+ midnight". Same reason its None means UNKNOWN: an unreadable stamp must not age a paid,
6332
+ outstanding snapshot out of the queue.
6333
+ """
6334
+ import datetime as _dt
6335
+ s = str(stamp or "").strip()
6336
+ if not s:
6337
+ return None
6338
+ try:
6339
+ t = _dt.datetime.fromisoformat(s.replace("Z", "+00:00"))
6340
+ except ValueError:
6341
+ return None
6342
+ now = _dt.datetime.now(_dt.timezone.utc)
6343
+ if t.tzinfo is None:
6344
+ t = t.replace(tzinfo=_dt.timezone.utc)
6345
+ return max(0, int((now - t).total_seconds() // 3600))
6346
+
6347
+
6348
+ def _gone_key(row, handle):
6349
+ """The identity a "this account does not exist" verdict is remembered under.
6350
+
6351
+ `(platform, handle)` — wave 26 R4's dedup key, not the handle alone, because `@x` on
6352
+ Instagram and `@x` on TikTok are two accounts. Blank platform means Instagram, which is what
6353
+ every row on a preset profile table is and what `preset_cells` stamps.
6354
+ """
6355
+ plat = str((row or {}).get("platform") or PLATFORM_INSTAGRAM).strip().lower()
6356
+ return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}"
6357
+
6358
+
6359
+ def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
6360
  """⭐ 2026-08-07 (owner ruling) — WHICH records this enrich step spends on, in order.
6361
 
6362
  Returns `(ordered_row_ids, note)`. The note is the honest account of what the selection did
 
6473
  days = max(1, days)
6474
 
6475
  picked, cooled, blank_handle = [], 0, 0
6476
+ # ⭐⭐ 2026-08-09 — HANDLES A VENDOR HAS ALREADY SAID DO NOT EXIST.
6477
+ #
6478
+ # ⛔ THE DEFECT THIS CLOSES IS STRUCTURAL, AND IT IS THE WORD "AGAIN" IN THE OWNER'S REPORT.
6479
+ # A blocked read writes NO cells, so `enriched_at` stays unset, so `_days_since(None)` is
6480
+ # None, so the cooldown above can never exclude the row — while `Followers is empty` keeps it
6481
+ # in the Pending cohort by construction. MEASURED on nurilab: one dead handle, re-bought at
6482
+ # 06:00 on three consecutive days, reported each time as an opaque "1 blocked". No vendor fix
6483
+ # removes that loop; only a memory of the verdict does.
6484
+ #
6485
+ # ⚠ THEY ARE **NAMED**, NOT SILENTLY DROPPED. The whole point is that the owner can act — the
6486
+ # note goes into the run every single time, not only on the run that discovered it, because a
6487
+ # run that quietly reports "0 records walked" tomorrow puts them straight back at "wtf".
6488
+ skipped_gone = []
6489
  for rid, r in ordered:
6490
  if len(picked) >= quota:
6491
  break
6492
+ raw_handle = str(r.get(profile_key) or "").strip()
6493
+ if not raw_handle:
6494
  blank_handle += 1
6495
  continue
6496
+ verdict = (gone or {}).get(_gone_key(r, raw_handle))
6497
+ if isinstance(verdict, dict):
6498
+ since_gone = _days_since(verdict.get("at"), today=today)
6499
+ if since_gone is None or since_gone < NOT_FOUND_RETRY_DAYS:
6500
+ skipped_gone.append(raw_handle)
6501
+ continue
6502
  if cooling:
6503
  since = _days_since(r.get("enriched_at"), today=today)
6504
  if since is not None and since < days:
 
6511
  # `run_plain` follow. Silence here would make a shrinking selection invisible.
6512
  if cooled:
6513
  notes.append(f"{cooled} skipped as enriched in the last {days} days")
6514
+ if skipped_gone:
6515
+ shown = ", ".join(f"@{h}" for h in skipped_gone[:5])
6516
+ notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account "
6517
+ f"({shown}{', …' if len(skipped_gone) > 5 else ''}) — delete the row or "
6518
+ f"correct the handle; it is retried after {NOT_FOUND_RETRY_DAYS} days")
6519
  if blank_handle:
6520
  notes.append(f"{blank_handle} skipped with no handle")
6521
+ if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered):
6522
  notes.append(f"{len(picked)} of the {quota} asked for — the list ran out")
6523
  return picked, "; ".join(notes)
6524
 
 
6589
  # entry, so calling it per record is O(existing) per record — survivable at 5,000 rows and an
6590
  # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts.
6591
  enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
6592
+ # ⭐ 2026-08-09 — deferred PROFILE snapshots (paid, still building at the vendor)
6593
+ # and handles a vendor has said do not exist. Both are collected across the whole
6594
+ # walk and written once, for the same reason every other list here is.
6595
+ "pendingProfiles": [], "gone": {},
6596
  "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False}
6597
+ # The verdicts this automation already holds, read ONCE — `enrich_selection` consults them
6598
+ # per record and re-reading the definition per row would be a store read per record.
6599
+ known_gone = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})
6600
  # ⭐ 2026-08-07 — THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD.
6601
  # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk
6602
  # would be O(rows²) and, worse, would re-answer "which 25 records" every time a record walked
 
6732
  # not this run's work, and the selection's own note already accounts for it.
6733
  aid = str(act.get("id") or "")
6734
  if aid not in enrich_plan:
6735
+ chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone)
6736
  enrich_plan[aid] = set(chosen)
6737
  if sel_note:
6738
  enrich["notes"].append(sel_note)
 
6744
  if enrich["profiles"]:
6745
  time.sleep(PACE_SECONDS) # >=2 s between profiles (R7), as the runner does
6746
  enrich["profiles"] += 1
6747
+ # ⭐ THE DEFERRED-PROFILE SINK IS PER RECORD so the snapshot can be stamped with
6748
+ # the row it belongs to: the collector writes preset cells back onto THAT record,
6749
+ # and a run-wide list would have no way to say which handle each snapshot was for.
6750
+ pend_prof = []
6751
  res = pull_profile(handle_raw,
6752
  max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
6753
  log=log,
 
6756
  post_metrics=bool(cfg.get("postMetrics")),
6757
  comment_metrics=bool(cfg.get("commentMetrics")),
6758
  pending_metrics=(enrich["pending"]
6759
+ if not cfg.get("dryRun") else None),
6760
+ pending_profile=(pend_prof
6761
  if not cfg.get("dryRun") else None))
6762
+ for _p in pend_prof:
6763
+ _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso()
6764
+ enrich["pendingProfiles"].extend(pend_prof)
6765
  pulled = _iso()
6766
  if res["state"] in ("ok", "partial"):
6767
  enrich["ok"] += 1
 
6782
  _act_row_patch(patches, table, rid, cells)
6783
  else:
6784
  enrich["blocked"] += 1
6785
+ # ⭐⭐ D-103 — THE NOTE IS NAMED AND KEPT AT FULL LENGTH.
6786
+ # It used to be `_s(note, 90)` with no handle attached: three runs blocked
6787
+ # the same record and the store could not say which record, let alone why.
6788
+ # 90 characters also truncated the one measured sentence mid-clause. This is
6789
+ # the most valuable thing the run produces and it now reaches the run entry.
6790
+ enrich["notes"].append(
6791
+ f"@{handle_raw}: {_s(res.get('note'), 300) or res['state']}")
6792
+ # A vendor STATING that the account does not exist is remembered, so the next
6793
+ # run stops paying to be told the same thing.
6794
+ if res.get("gone"):
6795
+ enrich["gone"][_gone_key(row, handle_raw)] = {
6796
+ "at": _iso(), "handle": handle_raw,
6797
+ "note": _s(res.get("note"), 300)}
6798
  elif kind == "find_records":
6799
  found = find_records(rt, cfg.get("table"), cfg.get("cond"),
6800
  int(cfg.get("limit") or 25))
 
6905
  # ⚠ IT IS A COUNT, not a flag, so `run_now`'s existing non-zero merge carries it without a
6906
  # special case — and so the run entry itself records that this happened.
6907
  return {"enrichUnbound": 1}
6908
+ # ⭐⭐ 2026-08-09 — THE NOTES SURVIVE A RUN THAT READ NOTHING, and that is not a detail.
6909
+ # `if not acc["profiles"]: return {}` is exactly the branch a run takes when EVERY candidate
6910
+ # was skipped as a known-dead handle — so the sentence explaining why the automation appears
6911
+ # to do nothing would have been dropped on precisely the runs that most need it, and the
6912
+ # owner would be back at "0 records walked, wtf". The selection note is produced before any
6913
+ # profile is read and must outlive that early return.
6914
  if not acc.get("profiles"):
6915
+ notes = list(acc.get("notes") or [])
6916
+ return {RUN_NOTES_KEY: notes} if notes else {}
6917
  out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]}
6918
+ if acc.get("notes"):
6919
+ out[RUN_NOTES_KEY] = list(acc["notes"])
6920
+ # ⭐ D-103's own prescription: "the block note survives on the RUN … not a status column, so
6921
+ # no table gains a column it did not ask for". `RUN_NOTES_KEY` is that channel.
6922
  if not acc.get("dry"):
6923
+ # The vendor's not-found verdicts, merged into engine state (never a tenant column and
6924
+ # never a status string — W25/R4 retired those). Merged rather than replaced: a run that
6925
+ # walked one record must not forget what earlier runs learned about the others.
6926
+ if acc.get("gone"):
6927
+ merged = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})
6928
+ merged.update(acc["gone"])
6929
+ set_state(rt, str(defn.get("id") or ""), {"enrichNotFound": merged})
6930
+ pending_profiles = queue_pending_profile_snapshots(
6931
+ rt, str(defn.get("id") or ""), acc.get("pendingProfiles") or [])
6932
+ if pending_profiles:
6933
+ out["enrichProfileBatchesPending"] = pending_profiles
6934
  queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""),
6935
  acc.get("pending") or [])
6936
  if queued:
 
7069
  return added[0]
7070
 
7071
 
7072
+ def _pending_profile_tasks(defn):
7073
+ """Validated, deduplicated deferred PROFILE snapshots from automation state.
7074
+
7075
+ ⭐⭐ ITS OWN LIST, NOT `pendingMetricSnapshots`, and the separation is load-bearing rather
7076
+ than tidy. `_pending_metric_tasks` filters on `datasetId in {posts, reels, comments}` and
7077
+ `kind in {posts, comments}` — so a profile entry appended to that list is silently dropped to
7078
+ zero by its own validator, and even if it survived, the collector would hand it to
7079
+ `_write_collected_metric_rows`, a posts/comments writer that has nothing to do with a profile
7080
+ row. Reusing the name would have shipped a green no-op of exactly the class this change
7081
+ exists to remove.
7082
+
7083
+ ⚠ A PROFILE TASK CARRIES ITS DESTINATION (`table` + `rowId`). A collected profile is written
7084
+ back as PRESET CELLS onto the record that asked for it, so unlike a metric batch it cannot be
7085
+ resolved from the handle alone: two databases may both hold `@x`.
7086
+ """
7087
+ raw = ((defn or {}).get("state") or {}).get("pendingProfileSnapshots") or []
7088
+ out, seen = [], set()
7089
+ for item in raw if isinstance(raw, list) else []:
7090
+ if not isinstance(item, dict):
7091
+ continue
7092
+ sid = str(item.get("snapshotId") or "").strip()
7093
+ dataset = str(item.get("datasetId") or "").strip()
7094
+ handle = str(item.get("influencer") or "").strip().lstrip("@").lower()
7095
+ table = str(item.get("table") or "").strip()
7096
+ row_id = str(item.get("rowId") or "").strip()
7097
+ # ⛔ `sd_` ONLY. A `snap_…` corpus id sent to `/datasets/v3/…` is a flat 404 about a
7098
+ # snapshot that is alive (§2c), and a malformed id must never be handed back to a vendor
7099
+ # endpoint at all.
7100
+ if (not sid.startswith("sd_") or dataset != BD_DS_PROFILES or not handle
7101
+ or not table or not row_id):
7102
+ continue
7103
+ if sid in seen:
7104
+ continue
7105
+ seen.add(sid)
7106
+ out.append({"snapshotId": sid, "datasetId": dataset, "kind": "profile",
7107
+ "influencer": handle, "table": table, "rowId": row_id,
7108
+ "requestedAt": str(item.get("requestedAt") or ""),
7109
+ "lastChecked": str(item.get("lastChecked") or ""),
7110
+ "lastNote": _s(item.get("lastNote"), 200)})
7111
+ return out
7112
+
7113
+
7114
+ def queue_pending_profile_snapshots(rt, auto_id, pending):
7115
+ """Durably retain deferred profile snapshots. Starts no new paid request. Returns how many
7116
+ were newly added."""
7117
+ aid = str(auto_id or "").strip()
7118
+ if not aid:
7119
+ return 0
7120
+ incoming = _pending_profile_tasks({"state": {"pendingProfileSnapshots": pending}})
7121
+ if not incoming:
7122
+ return 0
7123
+ added = [0]
7124
+
7125
+ def _up(cur):
7126
+ cur = cur if isinstance(cur, dict) else {}
7127
+ definition = cur.get(aid)
7128
+ if not isinstance(definition, dict):
7129
+ return cur
7130
+ state = definition.setdefault("state", {})
7131
+ existing = _pending_profile_tasks(definition)
7132
+ known = {x["snapshotId"] for x in existing}
7133
+ for task in incoming:
7134
+ if task["snapshotId"] not in known:
7135
+ existing.append(task)
7136
+ known.add(task["snapshotId"])
7137
+ added[0] += 1
7138
+ state["pendingProfileSnapshots"] = existing
7139
+ return cur
7140
+
7141
+ rt.update(STORE_KEY, _up, flush="sync")
7142
+ return added[0]
7143
+
7144
+
7145
+ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print, step=_no_step):
7146
+ """Finish deferred PROFILE reads the tenant has already paid for. Starts no new scrape.
7147
+
7148
+ ⭐⭐ 2026-08-09 — THE HALF THAT DID NOT EXIST. `bd_scrape` has always accepted a `deferred`
7149
+ list, and every post/reel/comment call passed one; the PROFILE call did not, so a profile the
7150
+ vendor took longer than `BD_SCRAPE_WAIT` to collect was billed and its snapshot id thrown
7151
+ away — on every run, forever. MEASURED on nurilab: `collection_duration` 320 s against a
7152
+ 180 s budget, two abandoned `sd_…` snapshots in two runs.
7153
+
7154
+ ⛔ IT CLOSES A TASK THAT FINISHED EMPTY. A pending entry that can never resolve is the same
7155
+ forever-loop wearing a different mask, so `bd_snapshot_progress` deciding `done` with zero
7156
+ records ends the task with the vendor's reason attached — and, when the vendor blames the
7157
+ target rather than itself, records the not-found verdict so the handle stops being re-bought.
7158
+ """
7159
+ tasks = _pending_profile_tasks(defn)
7160
+ if not tasks:
7161
+ return ("ok", "No pending profile reads.", {}, [], {"capture_paid": "idle"})
7162
+ step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}")
7163
+ remaining, patches, affected, notes = [], {}, [], []
7164
+ acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
7165
+ "pendingProfiles": [], "gone": {}, "profiles": 0, "ok": 0, "blocked": 0,
7166
+ "notes": [], "dry": False}
7167
+ ready = waiting = closed = 0
7168
+ for task in tasks:
7169
+ stale_h = _hours_since(task.get("requestedAt"))
7170
+ state, records, empty_note = bd_snapshot_progress(task["snapshotId"])
7171
+ if state in ("done", "failed") and not records:
7172
+ closed += 1
7173
+ acc["profiles"] += 1
7174
+ acc["blocked"] += 1
7175
+ note = f"@{task['influencer']}: {_s(empty_note, 240)}"
7176
+ notes.append(note)
7177
+ acc["notes"].append(note)
7178
+ # ⭐ `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a
7179
+ # profile request that means the account could not be reached at all, so the verdict
7180
+ # is remembered and the selection stops re-buying it (`NOT_FOUND_RETRY_DAYS`).
7181
+ # ⚠ `done`-with-zero is NOT remembered: "we found no matches" is a statement about
7182
+ # the query, and turning it into "this account does not exist" would silently retire
7183
+ # live handles.
7184
+ if state == "failed":
7185
+ acc["gone"][_gone_key({}, task["influencer"])] = {
7186
+ "at": _iso(), "handle": task["influencer"], "note": _s(empty_note, 300)}
7187
+ continue
7188
+ rows = []
7189
+ if state != "running":
7190
+ payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"})
7191
+ rows = _bd_rows(payload) if not err else []
7192
+ if rows and (_bd_deferral(payload) or
7193
+ (len(rows) == 1 and str(rows[0].get("status") or "") in
7194
+ ("running", "building", "collecting"))):
7195
+ rows = []
7196
+ if not rows:
7197
+ # ⚠ BOUNDED. A snapshot the vendor never finishes must not be polled until the end of
7198
+ # time; after `PENDING_PROFILE_MAX_HOURS` it is dropped WITH a sentence, never
7199
+ # silently. An unbounded queue is the forever-loop this change removes, inverted.
7200
+ if stale_h is not None and stale_h >= PENDING_PROFILE_MAX_HOURS:
7201
+ closed += 1
7202
+ note = (f"@{task['influencer']}: the source never finished the profile read "
7203
+ f"queued {int(stale_h)}h ago ({task['snapshotId']}) — it was dropped; "
7204
+ f"the next run will ask again")
7205
+ notes.append(note)
7206
+ acc["notes"].append(note)
7207
+ continue
7208
+ waiting += 1
7209
+ remaining.append({**task, "lastChecked": _iso(),
7210
+ "lastNote": _s("still building", 200)})
7211
+ continue
7212
+ ready += 1
7213
+ acc["profiles"] += 1
7214
+ profile = _bd_profile(rows[0], task["influencer"])
7215
+ if profile.get("followers") is None and profile.get("following") is None:
7216
+ acc["blocked"] += 1
7217
+ note = (f"@{task['influencer']}: the source delivered the profile but no "
7218
+ f"follower/following counts were readable in it")
7219
+ notes.append(note)
7220
+ acc["notes"].append(note)
7221
+ continue
7222
+ acc["ok"] += 1
7223
+ res = {"state": "ok", "profile": profile, "posts": [], "comments": [],
7224
+ "via": "brightdata:deferred", "note": ""}
7225
+ pulled = _iso()
7226
+ snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
7227
+ acc["snaps"].append(snap_row)
7228
+ acc["posts"].extend(idents)
7229
+ acc["psnaps"].extend(metric_rows)
7230
+ acc["comments"].extend(comment_rows)
7231
+ _act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled))
7232
+ affected.append(task["rowId"])
7233
+ notes.append(f"@{task['influencer']}: collected the profile the source had already been "
7234
+ f"paid for ({profile.get('followers')} followers)")
7235
+
7236
+ set_state(rt, str(defn.get("id") or ""),
7237
+ {"pendingProfileSnapshots": remaining or None})
7238
+ counts = _enrich_flush(rt, defn, username, acc, log)
7239
+ counts.pop(RUN_NOTES_KEY, None) # this function owns the note list below
7240
+ counts.update(_commit_action_writes(rt, str(_flow_table(defn) or ""), patches, {},
7241
+ username, log))
7242
+ counts.update({"profileBatchesCollected": ready, "profileBatchesPending": waiting,
7243
+ "profileBatchesEmpty": closed})
7244
+ if notes:
7245
+ counts[RUN_NOTES_KEY] = notes
7246
+ head = (f"{ready} deferred profile read{'' if ready == 1 else 's'} collected"
7247
+ if ready else "no deferred profile read was ready")
7248
+ tail = "".join([f"; {closed} finished with nothing to collect" if closed else "",
7249
+ f"; {waiting} still building" if waiting else ""])
7250
+ state = "ok" if ready and not closed else "partial"
7251
+ return (state, head + tail, counts, affected,
7252
+ {"capture_paid": "ok" if ready else "partial", "write": "ok" if ready else "idle"})
7253
+
7254
+
7255
  def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log):
7256
  """Write a completed metric snapshot through the canonical Post/Comment graph once."""
7257
  if not (idents or snapshots or comments):
 
7302
  return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_metrics": "idle"})
7303
  step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}")
7304
  remaining, idents, snapshots, comments = [], [], [], []
7305
+ ready, waiting, closed, closed_notes = 0, 0, 0, []
7306
  for task in tasks:
7307
+ # ⭐⭐ 2026-08-09 — ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be
7308
+ # `not rows or …`, so a snapshot the vendor had FINISHED with zero records was re-queued
7309
+ # as "still building" on every tick — forever, because nothing about it would ever
7310
+ # change. That is the same forever-loop the profile path was measured in, one dataset
7311
+ # over, and it was latent here the whole time.
7312
+ state, records, empty_note = bd_snapshot_progress(task["snapshotId"])
7313
+ if state in ("done", "failed") and not records:
7314
+ # ⛔ CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect;
7315
+ # keeping the entry would be a pending task that can never resolve.
7316
+ closed += 1
7317
+ closed_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}")
7318
+ continue
7319
  payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"})
7320
  rows = _bd_rows(payload) if not note else []
7321
+ building = (state == "running" or not rows or _bd_deferral(payload) or
7322
  (len(rows) == 1 and str(rows[0].get("status") or "") in
7323
  ("running", "building", "collecting")))
7324
  if building:
 
7354
  {"pendingMetricSnapshots": remaining or None})
7355
  written = _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log)
7356
  counts = {"metricBatchesCollected": ready, "metricBatchesPending": waiting,
7357
+ "metricBatchesEmpty": closed,
7358
  "postEngagementSnapshots": written["snapshots"],
7359
  "commentsCollected": written["comments"]}
7360
+ # ⚠ THE RUNNER CONTRACT STAYS A 5-TUPLE and the notes ride in `counts` under the reserved
7361
+ # `RUN_NOTES_KEY`, which `run_now` pops. Widening the tuple for one runner would make four
7362
+ # other call sites disagree about the shape of a run — and `_commit_run` already drops
7363
+ # non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather
7364
+ # than to a crash.
7365
+ if closed_notes:
7366
+ counts[RUN_NOTES_KEY] = closed_notes
7367
+ # ⚠ THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is
7368
+ # a real outcome the tenant paid for and it must read as an answer, not as a disappearance.
7369
+ tail = (f"; {closed} finished with nothing to collect" if closed else "")
7370
  if waiting:
7371
  return ("partial",
7372
  f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; "
7373
+ f"{waiting} still building and will be collected automatically{tail}",
7374
  counts, [], {"capture_metrics": "partial", "write": "ok"})
7375
+ return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}",
7376
  counts, [], {"capture_metrics": "ok", "write": "ok"})
7377
 
7378
 
 
9091
  # first and do not start another profile scrape while it is outstanding: re-running the
9092
  # action would buy duplicate engagement reads and reintroduce the timeout this handoff
9093
  # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`.
9094
+ # ⭐ 2026-08-09 — THE PROFILE HANDOFF IS COLLECTED FIRST, for the same reason and one
9095
+ # rung earlier: a profile snapshot the vendor is still building is paid work, and
9096
+ # starting a fresh scrape for the same handle would buy the identical row a second time.
9097
+ # Ahead of the metric collector because the profile IS the thing the run was asked for;
9098
+ # the engagement batches hang off it.
9099
+ if _pending_profile_tasks(defn):
9100
+ state, summary, counts, affected, steps = collect_pending_profile_snapshots(
9101
+ rt, defn, username=username, log=log,
9102
+ step=lambda text: _step(tenant, auto_id, text))
9103
+ return _commit_run(rt, auto_id, state, summary,
9104
+ {k: v for k, v in counts.items() if k != RUN_NOTES_KEY},
9105
+ state != "error", affected, steps,
9106
+ notes=counts.get(RUN_NOTES_KEY))
9107
  if _pending_metric_tasks(defn):
9108
  state, summary, counts, affected, steps = collect_pending_metric_snapshots(
9109
  rt, defn, username=username, log=log,
9110
  step=lambda text: _step(tenant, auto_id, text))
9111
+ return _commit_run(rt, auto_id, state, summary,
9112
+ {k: v for k, v in counts.items() if k != RUN_NOTES_KEY},
9113
+ state != "error", affected, steps,
9114
+ notes=counts.get(RUN_NOTES_KEY))
9115
  runner = RUNNERS.get(defn.get("kind"))
9116
  if runner is None:
9117
  return _commit_run(rt, auto_id, "error",
 
9145
  # A failing action must not fail the RUN: the machine steps already wrote their rows and
9146
  # reporting that as an error would misdescribe what happened. It degrades to `partial`
9147
  # with the reason in the summary — the cap_note discipline.
9148
+ # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`,
9149
+ # which now reads this name — an assignment only on the success path would turn any
9150
+ # action failure into a NameError inside the handler that exists to prevent exactly that.
9151
+ run_notes = []
9152
  try:
9153
  a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [],
9154
  username=username, log=log)
9155
+ # ⭐⭐ D-103 — POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so
9156
+ # the runner contract keeps its shape, and they must leave before the merge or they
9157
+ # would be a "count" everywhere downstream.
9158
+ run_notes = list(a_counts.pop(RUN_NOTES_KEY, None) or [])
9159
  counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}}
9160
  if a_counts.get("enrichMetricBatchesPending"):
9161
  batches = int(a_counts["enrichMetricBatchesPending"])
 
9172
  summary += (" — the Instagram step did not run: this database has no profile "
9173
  "column. Name one on the step, or mark a text column as the "
9174
  "Instagram profile")
9175
+ if a_counts.get("enrichProfileBatchesPending"):
9176
+ # ⭐ The paid profile the vendor is still building. Said out loud so a run that
9177
+ # looks like a failure is read as the handoff it is — the tick finishes it.
9178
+ n = int(a_counts["enrichProfileBatchesPending"])
9179
+ state = "partial" if state != "error" else state
9180
+ summary += (f" — {n} profile read{'' if n == 1 else 's'} took longer than the "
9181
+ "wait allows and will be collected automatically, at no extra cost")
9182
  if a_counts.get("enrichBlocked"):
9183
  state = "partial" if state != "error" else state
9184
+ # ⭐⭐ D-103THE REASON IS IN THE SENTENCE, not only behind a click. "1 profile
9185
+ # read(s) were blocked" is the exact string the owner read three mornings running
9186
+ # before asking "wtf is going on"; it names a quantity and withholds the one
9187
+ # thing that would let anybody act. The first note is the vendor's own words.
9188
+ blocked_note = next((n for n in run_notes if ": " in n), "")
9189
+ summary += (f" — {int(a_counts['enrichBlocked'])} profile read(s) were blocked"
9190
+ + (f". {_s(blocked_note, 220)}" if blocked_note else ""))
9191
  # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the
9192
  # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it
9193
  # logged the cap and rolled up `ok`, so a flow that had silently stopped writing
 
9201
  state = "partial" if state != "error" else state
9202
  summary = f"{summary} — the actions did not finish ({type(e).__name__})"
9203
  return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected,
9204
+ steps, notes=run_notes)
9205
  finally:
9206
  _release(tenant, auto_id)
9207
 
 
9244
  continue
9245
  pending_discovery = (d.get("kind") == "discover_instagram"
9246
  and str((d.get("state") or {}).get("pendingSnapshot") or "").strip())
9247
+ # 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile
9248
+ # deferral would be stored and never collected, which is the same defect it fixes wearing
9249
+ # a queue: `due_ids` only returns SCHEDULED automations, and the automation this was
9250
+ # measured on is `trigger: manual`. A capability written down but never walked is what
9251
+ # this whole change is about, so it must not be reintroduced one function later.
9252
+ if pending_discovery or _pending_metric_tasks(d) or _pending_profile_tasks(d):
9253
  out.append(aid)
9254
  return sorted(out)
9255
 
api/connectors_ig.py CHANGED
@@ -237,6 +237,16 @@ BD_DS_COMMENTS = "gd_ltppn085pokosxh13" # Instagram Comments: opt-in full
237
  BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id
238
  BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_…"}
239
  BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /<sd_id>?format=json -> the rows
 
 
 
 
 
 
 
 
 
 
240
  BD_PATH_FILTER = "/datasets/filter" # ⚠ NO /v3/ — the CORPUS query (discovery)
241
  BD_PATH_FILTER_SNAPSHOT = "/datasets/snapshot" # /<snap_id> -> status
242
  #: A sync scrape MEASURED at 15–23 s for one URL and ~40 s for two, so the timeout is generous;
@@ -860,6 +870,64 @@ def _bd_deferral(payload):
860
  return ""
861
 
862
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
863
  def bd_scrape(dataset_id, urls, wait=None, deferred=None):
864
  """Scrape a batch. `(rows, note)`; a note means it did not answer.
865
 
@@ -881,15 +949,22 @@ def bd_scrape(dataset_id, urls, wait=None, deferred=None):
881
  budget = BD_SCRAPE_WAIT if wait is None else float(wait)
882
  waited = 0.0
883
  while True:
884
- got, gnote = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"})
885
- if not gnote:
886
- rows = _bd_rows(got)
887
- # The status document is itself JSON, so "did it deliver?" is decided by whether
888
- # what came back looks like ROWS — never by the HTTP code.
889
- if rows and not _bd_deferral(got) and not (
890
- len(rows) == 1 and str(rows[0].get("status") or "") in
891
- ("running", "building", "collecting")):
892
- return rows, ""
 
 
 
 
 
 
 
893
  if waited >= budget:
894
  break
895
  time.sleep(BD_SCRAPE_POLL)
@@ -927,6 +1002,17 @@ APIFY_ACTOR_POSTS = os.environ.get("AIOS_APIFY_ACTOR") or "apify~instagram-scrap
927
  APIFY_WAIT = float(os.environ.get("AIOS_APIFY_WAIT") or 240)
928
 
929
 
 
 
 
 
 
 
 
 
 
 
 
930
  def apify_key():
931
  """The key, or ''. Read fresh every call, exactly as `bd_key` is."""
932
  return (os.environ.get("AIOS_APIFY_KEY") or "").strip()
@@ -1022,6 +1108,25 @@ def apify_profile(handle):
1022
  return None, "Apify's answer was not readable JSON"
1023
  if not isinstance(items, list) or not items:
1024
  return None, "Apify answered with no profile for that handle"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1025
  prof = providers.normalize_profile_apify(items[0])
1026
  if not prof:
1027
  return None, "Apify's profile row was not readable"
@@ -1109,7 +1214,8 @@ def _tag_metric_deferrals(items, start, kind, influencer_key):
1109
 
1110
 
1111
  def pull_profile_bd(url, max_posts=None, post_metrics=False,
1112
- comment_metrics=False, log=print, pending_metrics=None):
 
1113
  """The paid rung: one profile, EXACT counts, plus the top posts the anonymous surface hides.
1114
 
1115
  Same return contract as `pull_profile` (`{state, profile, posts, via, note}`) so the runner
@@ -1134,7 +1240,17 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False,
1134
  if not handle:
1135
  return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
1136
  "profile": {}, "posts": [], "comments": [], "via": ""}
1137
- rows, note = bd_scrape(BD_DS_PROFILES, [f"https://www.instagram.com/{handle}/"])
 
 
 
 
 
 
 
 
 
 
1138
  node = rows[0] if rows else {}
1139
  profile = _bd_profile(node, handle) if node else {}
1140
  unreadable = profile.get("followers") is None and profile.get("following") is None
@@ -1168,7 +1284,16 @@ def pull_profile_bd(url, max_posts=None, post_metrics=False,
1168
  f"the vendor's pre-collected profile corpus — the values are its last "
1169
  f"capture, not a read taken just now")
1170
  else:
 
 
 
 
 
 
 
 
1171
  return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
 
1172
  "note": f"{scrape_note}; the profile corpus did not answer either ({cnote})"}
1173
  if profile.get("followers") is None and profile.get("following") is None:
1174
  # It answered 200 with something we could not read. Say THAT — not "0 followers".
@@ -1621,7 +1746,8 @@ def bd_filter_rows(snapshot_id):
1621
 
1622
  def pull_profile(url, max_posts=None, log=print, tier="anonymous",
1623
  fallback=True,
1624
- post_metrics=False, comment_metrics=False, pending_metrics=None):
 
1625
  """Everything readable about one public profile, cheapest-honest rung last.
1626
 
1627
  Returns `{state, profile, posts, via, note}` with state ∈ ok | partial | blocked | error.
@@ -1650,14 +1776,23 @@ def pull_profile(url, max_posts=None, log=print, tier="anonymous",
1650
  return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
1651
  "profile": {}, "posts": [], "comments": [], "via": ""}
1652
  attempts = []
 
 
 
 
1653
 
1654
  if clean_tier(tier) == "brightdata":
1655
  paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics,
1656
  comment_metrics=comment_metrics, log=log,
1657
- pending_metrics=pending_metrics)
 
1658
  if paid["state"] in ("ok", "partial"):
1659
  return paid
1660
- attempts.append(f"brightdata:{_s(paid.get('note'), 90)}")
 
 
 
 
1661
  # ⭐ THE APIFY RUNG (owner report 2026-08-09: "both Bright Data and Apify working in
1662
  # tandem"). `providers.py` has declared `ig_profile -> ("brightdata", "apify")` since
1663
  # 2026-08-08 and nothing ever walked it: Bright Data failing went straight to the
@@ -1678,13 +1813,20 @@ def pull_profile(url, max_posts=None, log=print, tier="anonymous",
1678
  "via": "apify",
1679
  "note": ("Bright Data could not read this profile "
1680
  f"({_s(paid.get('note'), 90)}), so it came from Apify")}
1681
- attempts.append(f"apify:{_s(a_note or 'answered without follower counts', 90)}")
 
 
 
 
 
1682
  time.sleep(PACE_SECONDS)
1683
  if not fallback:
1684
  return {"state": "blocked", "profile": {}, "posts": [], "comments": [],
1685
- "via": "brightdata",
1686
- "note": "the paid rungs did not answer and the anonymous fallback is turned "
1687
- f"off for this automation ({'; '.join(attempts) or 'blocked'})"}
 
 
1688
  time.sleep(PACE_SECONDS)
1689
 
1690
  # RUNG 1 — the web client's own profile endpoint. Public, no cookie; the one that used to
@@ -1758,6 +1900,15 @@ def pull_profile(url, max_posts=None, log=print, tier="anonymous",
1758
  return {"state": "partial", "profile": profile, "posts": [], "via": "og:description",
1759
  "note": "counts are the page's own abbreviations; media is not anonymously "
1760
  "readable (" + ", ".join(attempts) + ")"}
 
 
 
 
 
 
 
 
 
1761
  if status in (401, 403, 429) or status == 0:
1762
  return {"state": "blocked", "profile": {}, "posts": [], "via": "",
1763
  "note": f"Instagram refused the anonymous read ({', '.join(attempts)})"}
 
237
  BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id
238
  BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_…"}
239
  BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /<sd_id>?format=json -> the rows
240
+ #: ⭐⭐ 2026-08-09 — THE SNAPSHOT'S OWN STATUS DOCUMENT, and asking it is the difference between
241
+ #: "the vendor is still working" and "the vendor finished and collected nothing". MEASURED on
242
+ #: `sd_msl80v7l1ti14rhcd6` (nurilab's `roxyfoxypinky`): `{"status": "ready", "records": 0,
243
+ #: "errors": 1, "error_codes": {"crawl_error": 1}, "collection_duration": 320513}`. The rows
244
+ #: endpoint answers `[]` for that snapshot forever, which `bd_scrape`'s old `if rows:` poll could
245
+ #: not distinguish from a snapshot mid-build — so it burned its whole budget and then reported
246
+ #: *"the records are collected, not lost"* about a batch that had collected nothing at all.
247
+ #: ⛔ SCRAPER NAMESPACE ONLY (`sd_…`). The corpus has no twin here; its status is
248
+ #: `BD_PATH_FILTER_SNAPSHOT/<snap_id>`, and crossing the two 404s (§2c).
249
+ BD_PATH_PROGRESS = "/datasets/v3/progress" # /<sd_id> -> {status, records, errors, error_codes}
250
  BD_PATH_FILTER = "/datasets/filter" # ⚠ NO /v3/ — the CORPUS query (discovery)
251
  BD_PATH_FILTER_SNAPSHOT = "/datasets/snapshot" # /<snap_id> -> status
252
  #: A sync scrape MEASURED at 15–23 s for one URL and ~40 s for two, so the timeout is generous;
 
870
  return ""
871
 
872
 
873
+ def bd_snapshot_progress(sid):
874
+ """One scraper snapshot's status. `(state, records, note)`.
875
+
876
+ state ∈ `running` | `done` | `failed` | `unknown`:
877
+ · `running` — the vendor is still collecting; poll again.
878
+ · `done` — finished. `records` says whether it produced anything.
879
+ · `failed` — finished, produced NOTHING, and the vendor blamed the TARGET (an
880
+ `error_codes` entry such as `crawl_error`). On a profile request that is the
881
+ vendor saying it could not reach that account at all, which is a different
882
+ action from "we found no matches" — so it is a distinct state rather than a
883
+ phrase inside the note. A caller that has to grep an English sentence to
884
+ decide what happened is a caller whose behaviour changes when someone
885
+ improves the wording.
886
+ · `unknown` — the status call itself did not answer; fall back to probing the rows.
887
+
888
+ ⭐⭐ 2026-08-09 — THE QUESTION `bd_scrape` COULD NOT ASK. Its poll loop advanced only on
889
+ `if rows:`, so a snapshot the vendor had FINISHED with zero records looked byte-for-byte like
890
+ one still building: the rows endpoint answers `[]` in both cases. MEASURED on nurilab's one
891
+ pending handle — `status: ready, records: 0, errors: 1, crawl_error: 1` after 320 s — while
892
+ the enrich run reported *"deferred … the records are collected, not lost"*, which was false
893
+ about that batch and had been re-reported on three separate runs.
894
+
895
+ ⛔ `unknown` IS NOT `done`, and the distinction is the whole reason this returns three states
896
+ rather than a boolean. When the status call itself fails we must fall back to the old
897
+ row-probing behaviour, not conclude the snapshot is empty — an unreachable status endpoint
898
+ would otherwise turn every deferral into a confident "the vendor found nothing".
899
+
900
+ ⚠ THE NOTE IS DELIBERATELY VENDOR-NEUTRAL about WHAT was being read. This serves profile,
901
+ post, reel and comment snapshots alike; a sentence naming "the account" would be wrong on
902
+ three of the four, and the caller knows which it asked for.
903
+ """
904
+ payload, err = bd_call(f"{BD_PATH_PROGRESS}/{sid}", None, None)
905
+ if err or not isinstance(payload, dict):
906
+ return "unknown", 0, ""
907
+ status = str(payload.get("status") or "").strip().lower()
908
+
909
+ def _n(key):
910
+ try:
911
+ return int(payload.get(key) or 0)
912
+ except (TypeError, ValueError):
913
+ return 0
914
+
915
+ records, errors = _n("records"), _n("errors")
916
+ if status not in ("ready", "done", "failed", "error"):
917
+ return "running", records, ""
918
+ if records > 0:
919
+ return "done", records, ""
920
+ codes = payload.get("error_codes")
921
+ named = (", ".join(f"{str(k).replace('_', ' ')} x{v}"
922
+ for k, v in sorted(codes.items()))
923
+ if isinstance(codes, dict) and codes else "")
924
+ if errors or named or status in ("failed", "error"):
925
+ return "failed", 0, ("the source finished this request and collected nothing"
926
+ + (f" — it reported {named}" if named else "")
927
+ + "; the target is unreachable, private, or no longer exists")
928
+ return "done", 0, "the source finished this request and found no records for it"
929
+
930
+
931
  def bd_scrape(dataset_id, urls, wait=None, deferred=None):
932
  """Scrape a batch. `(rows, note)`; a note means it did not answer.
933
 
 
949
  budget = BD_SCRAPE_WAIT if wait is None else float(wait)
950
  waited = 0.0
951
  while True:
952
+ # THE STATUS DOCUMENT IS ASKED FIRST, because it is the only surface that can say
953
+ # "finished, and there was nothing". Fetching rows while the vendor is still collecting
954
+ # is also a wasted call on every single poll.
955
+ state, records, empty_note = bd_snapshot_progress(sid)
956
+ if state in ("done", "failed") and not records:
957
+ return [], empty_note # a definitive EMPTY, never our timeout sentence
958
+ if state != "running":
959
+ got, gnote = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"})
960
+ if not gnote:
961
+ rows = _bd_rows(got)
962
+ # The status document is itself JSON, so "did it deliver?" is decided by whether
963
+ # what came back looks like ROWS — never by the HTTP code.
964
+ if rows and not _bd_deferral(got) and not (
965
+ len(rows) == 1 and str(rows[0].get("status") or "") in
966
+ ("running", "building", "collecting")):
967
+ return rows, ""
968
  if waited >= budget:
969
  break
970
  time.sleep(BD_SCRAPE_POLL)
 
1002
  APIFY_WAIT = float(os.environ.get("AIOS_APIFY_WAIT") or 240)
1003
 
1004
 
1005
+ #: ⭐⭐ THE ONE SENTENCE FOR "THIS HANDLE HAS NO ACCOUNT BEHIND IT", named rather than inlined so
1006
+ #: that the engine can recognise it and act on it. A vendor saying *not found* is categorically
1007
+ #: different from a vendor being slow, rate-limited or out of credit: the first will never
1008
+ #: succeed however many times it is retried, and every retry is billed. `pull_profile` promotes
1009
+ #: a rung that returns this into `res["gone"]`, which is what lets a run stop re-buying a dead
1010
+ #: handle every morning ([[flag-shipped-without-its-writer]] — the flag needs a writer AND a
1011
+ #: reader, so both are in this file's call graph).
1012
+ ACCOUNT_GONE_NOTE = ("Instagram has no account with that handle — the source reports it does "
1013
+ "not exist (it may have been deleted, renamed, or suspended)")
1014
+
1015
+
1016
  def apify_key():
1017
  """The key, or ''. Read fresh every call, exactly as `bd_key` is."""
1018
  return (os.environ.get("AIOS_APIFY_KEY") or "").strip()
 
1108
  return None, "Apify's answer was not readable JSON"
1109
  if not isinstance(items, list) or not items:
1110
  return None, "Apify answered with no profile for that handle"
1111
+ # ⭐⭐ 2026-08-09 — THE VENDOR'S ERROR ENVELOPE IS AN ANSWER, AND IT WAS BEING THROWN AWAY.
1112
+ # MEASURED on nurilab's one pending handle: Apify returned
1113
+ # `{"url": …, "username": "roxyfoxypinky", "error": "not_found",
1114
+ # "errorDescription": "Post does not exist"}` — a 200, a well-formed item, and the single
1115
+ # most useful sentence any provider produced about that account. `normalize_profile_apify`
1116
+ # mapped it into a "profile" carrying username + url, this function returned `(prof, "")`
1117
+ # meaning SUCCESS, and the caller — finding no follower count on it — recorded the useless
1118
+ # `apify: answered without follower counts`. Three runs asked, three runs were told, and the
1119
+ # owner still could not find out why the row was blank.
1120
+ # ⚠ `error` RIDES EVERY ITEM AS `null` on the success path (measured on `sriyynntt`), so the
1121
+ # test must be truthiness, never key presence.
1122
+ # ⚠ AND THE VENDOR'S OWN WORDING IS NOT REPEATED VERBATIM: it says "Post does not exist" for
1123
+ # a PROFILE url, which describes the wrong kind of object to anyone reading a profile row.
1124
+ err = str((items[0] or {}).get("error") or "").strip()
1125
+ if err:
1126
+ desc = str((items[0] or {}).get("errorDescription") or "").strip()
1127
+ if err in ("not_found", "not-found", "notfound"):
1128
+ return None, ACCOUNT_GONE_NOTE
1129
+ return None, f"the source could not read that account ({desc or err})"
1130
  prof = providers.normalize_profile_apify(items[0])
1131
  if not prof:
1132
  return None, "Apify's profile row was not readable"
 
1214
 
1215
 
1216
  def pull_profile_bd(url, max_posts=None, post_metrics=False,
1217
+ comment_metrics=False, log=print, pending_metrics=None,
1218
+ pending_profile=None):
1219
  """The paid rung: one profile, EXACT counts, plus the top posts the anonymous surface hides.
1220
 
1221
  Same return contract as `pull_profile` (`{state, profile, posts, via, note}`) so the runner
 
1240
  if not handle:
1241
  return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
1242
  "profile": {}, "posts": [], "comments": [], "via": ""}
1243
+ # ⭐⭐ 2026-08-09 — THE PROFILE SCRAPE NOW HANDS ITS DEFERRAL OVER, and until today it was the
1244
+ # ONLY `bd_scrape` caller that did not. Every post, reel and comment call below passes
1245
+ # `deferred=pending_metrics`; this one passed nothing, so a profile the vendor took longer
1246
+ # than `BD_SCRAPE_WAIT` to collect was BILLED and its snapshot id discarded — on every run,
1247
+ # forever. MEASURED on nurilab: two runs, two fresh `sd_…` snapshots, both abandoned, the
1248
+ # second at a vendor-reported `collection_duration` of 320 s against our 180 s budget.
1249
+ # That is [[artifact-with-no-importer]] in a function signature: the parameter existed, the
1250
+ # machinery to collect it existed, and nothing handed the list in.
1251
+ _deferred = []
1252
+ rows, note = bd_scrape(BD_DS_PROFILES, [f"https://www.instagram.com/{handle}/"],
1253
+ deferred=_deferred)
1254
  node = rows[0] if rows else {}
1255
  profile = _bd_profile(node, handle) if node else {}
1256
  unreadable = profile.get("followers") is None and profile.get("following") is None
 
1284
  f"the vendor's pre-collected profile corpus — the values are its last "
1285
  f"capture, not a read taken just now")
1286
  else:
1287
+ # ⭐ THE DEFERRAL IS REGISTERED ONLY WHEN THE PAID RUNG GIVES UP. If the corpus
1288
+ # answered we already have the profile, and queueing a second write of the same
1289
+ # identity would add churn for a row we did not need. Here we have nothing — and the
1290
+ # snapshot is both paid for and the freshest answer that will ever exist for this
1291
+ # handle, so it is the one thing worth carrying forward.
1292
+ if isinstance(pending_profile, list):
1293
+ for d in _deferred:
1294
+ pending_profile.append({**d, "kind": "profile", "influencer": handle})
1295
  return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
1296
+ "deferredProfile": [d.get("snapshotId") for d in _deferred],
1297
  "note": f"{scrape_note}; the profile corpus did not answer either ({cnote})"}
1298
  if profile.get("followers") is None and profile.get("following") is None:
1299
  # It answered 200 with something we could not read. Say THAT — not "0 followers".
 
1746
 
1747
  def pull_profile(url, max_posts=None, log=print, tier="anonymous",
1748
  fallback=True,
1749
+ post_metrics=False, comment_metrics=False, pending_metrics=None,
1750
+ pending_profile=None):
1751
  """Everything readable about one public profile, cheapest-honest rung last.
1752
 
1753
  Returns `{state, profile, posts, via, note}` with state ∈ ok | partial | blocked | error.
 
1776
  return {"state": "error", "note": f"{url!r} is not an Instagram profile URL",
1777
  "profile": {}, "posts": [], "comments": [], "via": ""}
1778
  attempts = []
1779
+ # ⭐ 2026-08-09 — SET BY A RUNG THAT SAW A VENDOR SAY *this account does not exist*. It rides
1780
+ # out on the result so the engine can stop re-buying a dead handle every morning; it is never
1781
+ # inferred from a mere failure, only from a vendor stating it.
1782
+ gone = ""
1783
 
1784
  if clean_tier(tier) == "brightdata":
1785
  paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics,
1786
  comment_metrics=comment_metrics, log=log,
1787
+ pending_metrics=pending_metrics,
1788
+ pending_profile=pending_profile)
1789
  if paid["state"] in ("ok", "partial"):
1790
  return paid
1791
+ # ⚠ 200, NOT 90. This string is the ONLY account of what the vendor said, and at 90 the
1792
+ # measured sentence truncated to *"…was not ready within"* — mid-clause, with the budget
1793
+ # and the snapshot id cut off. A reason nobody can read is the reason being discarded
1794
+ # with extra steps (D-103).
1795
+ attempts.append(f"brightdata:{_s(paid.get('note'), 200)}")
1796
  # ⭐ THE APIFY RUNG (owner report 2026-08-09: "both Bright Data and Apify working in
1797
  # tandem"). `providers.py` has declared `ig_profile -> ("brightdata", "apify")` since
1798
  # 2026-08-08 and nothing ever walked it: Bright Data failing went straight to the
 
1813
  "via": "apify",
1814
  "note": ("Bright Data could not read this profile "
1815
  f"({_s(paid.get('note'), 90)}), so it came from Apify")}
1816
+ # A VENDOR SAYING *not found* IS THE ANSWER, NOT A FAILED ATTEMPT. Recorded here
1817
+ # and carried to whichever return this walk reaches, so the run can say the one thing
1818
+ # that ends the loop instead of the fourth variation on "we could not read it".
1819
+ if a_note == ACCOUNT_GONE_NOTE:
1820
+ gone = a_note
1821
+ attempts.append(f"apify:{_s(a_note or 'answered without follower counts', 200)}")
1822
  time.sleep(PACE_SECONDS)
1823
  if not fallback:
1824
  return {"state": "blocked", "profile": {}, "posts": [], "comments": [],
1825
+ "via": "brightdata", "gone": bool(gone),
1826
+ "note": (gone + " (the anonymous fallback is turned off for this automation)"
1827
+ if gone else
1828
+ "the paid rungs did not answer and the anonymous fallback is turned "
1829
+ f"off for this automation ({'; '.join(attempts) or 'blocked'})")}
1830
  time.sleep(PACE_SECONDS)
1831
 
1832
  # RUNG 1 — the web client's own profile endpoint. Public, no cookie; the one that used to
 
1900
  return {"state": "partial", "profile": profile, "posts": [], "via": "og:description",
1901
  "note": "counts are the page's own abbreviations; media is not anonymously "
1902
  "readable (" + ", ".join(attempts) + ")"}
1903
+ # ⭐⭐ THE DEFINITIVE ANSWER WINS OVER THE LAST RUNG'S SHRUG. If a vendor said the account does
1904
+ # not exist and the free rungs then also found nothing, the honest report is *"this account
1905
+ # does not exist"* — not *"nothing anonymously readable"*, which is a statement about US.
1906
+ # ⚠ It is checked HERE, after the free rungs have run, and not as an early return: an
1907
+ # anonymous read that DOES succeed above proves the vendor wrong and returns `ok`/`partial`
1908
+ # on its own. A vendor's not-found is strong evidence, never a reason to stop looking.
1909
+ if gone:
1910
+ return {"state": "blocked", "profile": {}, "posts": [], "via": "", "gone": True,
1911
+ "note": f"{gone}. Nothing else could read it either ({', '.join(attempts)})"}
1912
  if status in (401, 403, 429) or status == 0:
1913
  return {"state": "blocked", "profile": {}, "posts": [], "via": "",
1914
  "note": f"Instagram refused the anonymous read ({', '.join(attempts)})"}
api/odoo_relational.py CHANGED
@@ -139,6 +139,25 @@ def customer_fields():
139
  "default": True,
140
  "rollup": {"link": "invoices", "field": "due_date", "fn": "latest",
141
  "sortBy": "due_date", "sortDir": "asc"}},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
142
  {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay",
143
  "default": False},
144
  )]
 
139
  "default": True,
140
  "rollup": {"link": "invoices", "field": "due_date", "fn": "latest",
141
  "sortBy": "due_date", "sortDir": "asc"}},
142
+ # ⭐ THE READ-THROUGH ROLLUP — the column C8 said this substrate "cannot host", now hosted
143
+ # WITHOUT hosting the rows (owner 2026-08-09). It names a governed TOPIC and a METRIC KEY;
144
+ # `rollup_sql` answers every customer in ONE grouped query against DuckDB.
145
+ # MEASURED: 1,748 customers over 256,810 order lines in 232 ms, and PARITY-PROVEN at
146
+ # 1,716/1,716 customers TO THE CENT against the topic's own scope.
147
+ #
148
+ # ⛔ IT DOES NOT AND CANNOT COME FROM `ut_odoo_invoices`. That table is OPEN AR only
149
+ # (`_AR_WHERE` filters `payment_state IN ('not_paid','partial')`), so a rollup over the
150
+ # `invoices` link would sum UNPAID invoices and call it sales — a wrong number that looks
151
+ # right. Different question, different source; stated so nobody "simplifies" it later.
152
+ #
153
+ # ⚠ `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced flag —
154
+ # NOT posted-invoice billing (that is `customer_invoices`, a different topic which says so
155
+ # itself). The metric KEY carries that distinction; this field only names the key, which is
156
+ # the whole reason it may not carry SQL of its own.
157
+ {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup",
158
+ "source": "overlay", "default": True, "agg": "sum",
159
+ "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced",
160
+ "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}},
161
  {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay",
162
  "default": False},
163
  )]
api/providers.py CHANGED
@@ -348,6 +348,18 @@ def normalize_profile_apify(row):
348
  """
349
  if not isinstance(row, dict):
350
  return None
 
 
 
 
 
 
 
 
 
 
 
 
351
  handle = str(row.get("username") or "").strip()
352
  if not handle:
353
  return None
 
348
  """
349
  if not isinstance(row, dict):
350
  return None
351
+ # ⛔ AN ERROR ENVELOPE IS NOT A PROFILE (measured 2026-08-09). Apify answers a dead handle
352
+ # with a 200 and a well-formed item — `{"username": …, "error": "not_found",
353
+ # "errorDescription": "Post does not exist"}` — and this function used to build a "profile"
354
+ # out of it: a username, a url and a source_payload, with every measured field absent. It
355
+ # then read to the caller as a vendor that answered, so the reason was replaced by a shrug.
356
+ # Refused HERE as well as in `connectors_ig.apify_profile` on purpose: the boundary rule this
357
+ # module exists for is that nothing downstream ever sees a vendor-shaped key, and a vendor's
358
+ # ERROR shape is the one that must never become a row.
359
+ # ⚠ TRUTHINESS, NOT KEY PRESENCE — a successful item carries `error: null` (measured on
360
+ # `sriyynntt`), so `"error" in row` would refuse every good profile.
361
+ if row.get("error"):
362
+ return None
363
  handle = str(row.get("username") or "").strip()
364
  if not handle:
365
  return None
api/rollup_sql.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """rollup_sql.py — the READ-THROUGH rollup: one grouped SQL query answers every parent row.
2
+
3
+ ⛔ WHY THIS EXISTS. A linked rollup folds rows that live in the store, and `MAX_ROWS = 5000`
4
+ bounds them for a measured reason: every `ut_*` table lives inside ONE `user_tables.json` that is
5
+ parsed and deep-copied on essentially every request (MEASURED 2026-08-09 on nurilab — 5.1 MB /
6
+ 1,567 rows = 67 ms per copy, and rendering a 91-row table still pays for all of it). Royal's
7
+ 254k+ order lines cannot live there and never will: at that density the blob is ~829 MB and one
8
+ deep copy is ~9.8 s.
9
+
10
+ ⭐ SO THIS KIND NEVER COPIES THE ROWS AT ALL. It names a governed semantic TOPIC and a METRIC KEY,
11
+ and `store_query(..., group_by=[dim])` answers EVERY parent in one pass against DuckDB.
12
+ MEASURED: 1,748 customers over 256,810 order lines in **232 ms**. The row count stops being the
13
+ product's problem and becomes the database's, which is the whole of D-87's "read-through binding".
14
+
15
+ ⚠ A METRIC KEY, NEVER A FILTER FRAGMENT — the decision this module is built around.
16
+ `model/metrics/*.yml` already carries each metric's scope, its `store_filter_sql` AND the matching
17
+ `live_domain`, whose own comment reads *"BOTH or store_parity compares two different questions"*.
18
+ Binding a rollup to the KEY inherits the scope and the live-parity oracle for free. Letting a
19
+ rollup carry SQL would mint a second definition of a number the semantic layer exists to define
20
+ once, and the two would drift in silence.
21
+
22
+ ⛔ TRUNCATION IS A WRONG NUMBER HERE, NOT A SHORT LIST. A row window is honest because counts and
23
+ totals are computed over the full scope beside it; a GROUP window has no such companion — the
24
+ groups ARE the answer. `store_query` now reports `truncated`, and this module REFUSES to write a
25
+ single cell when it is set. Half a rollup is worse than none: it looks finished.
26
+ """
27
+ import sys
28
+ from pathlib import Path
29
+
30
+ _HERE = Path(__file__).resolve().parent
31
+ _PLATFORM = _HERE.parents[1] / "platform"
32
+ for _p in (str(_HERE), str(_PLATFORM)):
33
+ if _p not in sys.path:
34
+ sys.path.insert(0, _p)
35
+
36
+
37
+ class RollupSourceError(Exception):
38
+ """The rollup could not be computed HONESTLY — no cells are written when this is raised."""
39
+
40
+
41
+ def _ut():
42
+ import core.user_tables as user_tables
43
+ return user_tables
44
+
45
+
46
+ def source_fields(table_def):
47
+ """Every source-backed rollup field on a table definition, in declaration order."""
48
+ out = []
49
+ for f in ((table_def or {}).get("fields") or []):
50
+ if not isinstance(f, dict) or f.get("type") != "rollup":
51
+ continue
52
+ bag = (f.get("rollup") or {}).get("source")
53
+ if isinstance(bag, dict) and bag.get("topic") and bag.get("measure"):
54
+ out.append(f)
55
+ return out
56
+
57
+
58
+ def group_values(bag, today=None):
59
+ """`{group_id: value}` for one source bag — ONE grouped query over the whole scope.
60
+
61
+ ⚠ Keyed by the dim's `_id` column, never its label. Two customers can share a display name;
62
+ `partner_id` is what the parent row actually joins on.
63
+ """
64
+ from harness import semantic as sem
65
+ from harness import windows as W
66
+
67
+ topic = str(bag.get("topic") or "")
68
+ measure = str(bag.get("measure") or "")
69
+ dim = str(bag.get("groupBy") or "")
70
+ window = str(bag.get("window") or "").strip().lower()
71
+
72
+ date_from = date_to = None
73
+ if window and window != "all_time":
74
+ # ⚠ RESOLVED HERE, AGAINST `today`, not stored as a date pair. `harness.windows` owns what
75
+ # "ytd" means so there is one implementation; a literal year-start baked into the field
76
+ # would be right until 1 January and wrong afterwards with nothing to notice.
77
+ rng = W.resolve({"kind": window}, today or _today())
78
+ if rng is None:
79
+ raise RollupSourceError(
80
+ f"window {window!r} could not be resolved — refusing rather than widening this "
81
+ f"rollup to all time")
82
+ date_from, date_to = rng
83
+
84
+ try:
85
+ res = sem.store_query(topic, [measure], group_by=[dim],
86
+ date_from=date_from, date_to=date_to,
87
+ limit=sem.MAX_GROUPS, today=today)
88
+ except Exception as e: # noqa: BLE001
89
+ raise RollupSourceError(f"{type(e).__name__}: {e}") from e
90
+
91
+ if res.get("truncated"):
92
+ # ⛔ THE REFUSAL THAT MATTERS. See the module header: a truncated GROUP set is a wrong
93
+ # answer per parent, not a shortened one, and it would look completely normal on screen.
94
+ raise RollupSourceError(
95
+ f"{topic}/{measure} grouped by {dim} exceeded {sem.MAX_GROUPS} groups — refusing to "
96
+ f"write cells from a truncated result")
97
+
98
+ id_col = f"{dim}_id"
99
+ out = {}
100
+ for row in res.get("rows") or []:
101
+ gid = row.get(id_col)
102
+ if gid is None:
103
+ continue
104
+ out[str(gid)] = row.get(measure)
105
+ return out
106
+
107
+
108
+ def _today():
109
+ import datetime as _dt
110
+ return _dt.date.today().strftime("%Y-%m-%d")
111
+
112
+
113
+ def compute(rt, table_key, today=None, tables=None):
114
+ """Write every source-backed rollup cell on `table_key`. Returns `{field_key: cells_written}`.
115
+
116
+ `tables` (a live `user_tables` dict) is the gate's injection point — the same shape
117
+ `automation_engine.compute_relation_cells` takes, so this can be proven against a fixture
118
+ without a store.
119
+ """
120
+ ut = _ut()
121
+ owned = tables is not None
122
+ blob = tables if owned else (rt.get(ut.STORE_KEY) or {})
123
+ tdef = (blob or {}).get(str(table_key)) or {}
124
+ fields = source_fields(tdef)
125
+ if not fields:
126
+ return {}
127
+
128
+ stamp = today or _today()
129
+ plans = []
130
+ for f in fields:
131
+ bag = f["rollup"]["source"]
132
+ # ⚠ Resolved BEFORE anything is written. A rollup that refuses must leave every cell as it
133
+ # was — a partially applied pass would mix two vintages of the same column.
134
+ plans.append((f["key"], str(bag.get("on") or ""), group_values(bag, today=stamp)))
135
+
136
+ written = {}
137
+
138
+ def _apply(cur):
139
+ t = cur.get(str(table_key))
140
+ if t is None:
141
+ return cur
142
+ rows = t.setdefault("rows", {})
143
+ for fkey, on, values in plans:
144
+ n = 0
145
+ for row in rows.values():
146
+ if not isinstance(row, dict):
147
+ continue
148
+ join = str(row.get(on) or "").strip()
149
+ if not join:
150
+ continue
151
+ # ⚠ int-ish join keys arrive as "5280" or 5280 depending on who wrote them.
152
+ hit = values.get(join)
153
+ if hit is None and join.endswith(".0"):
154
+ hit = values.get(join[:-2])
155
+ row[fkey] = "" if hit is None else str(hit)
156
+ n += 1
157
+ written[fkey] = n
158
+ return cur
159
+
160
+ if owned:
161
+ _apply(blob)
162
+ else:
163
+ rt.update(ut.STORE_KEY, _apply, flush="async")
164
+ return written
api/routes_alerts.py CHANGED
@@ -1,254 +1,254 @@
1
- """routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT).
2
-
3
- GET /api/v1/alerts -> {alerts:[...]}
4
- POST /api/v1/alerts <- {viewId, topic, label?}
5
- DELETE /api/v1/alerts/{alert_id}
6
- POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
7
- GET /api/v1/notifications -> {unread, items:[...]}
8
- POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
9
-
10
- The semantics — an alert is a view plus a remembered matched set, a notification is a NEW
11
- ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning.
12
- This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
13
-
14
- ⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
15
- for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
16
- full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
17
- book, and the notification would name customers that user may not see — a permission leak wearing
18
- a notification's clothes. The owner's own scope is the only correct basis for their alert.
19
-
20
- ⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
21
- `routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
22
- alert can see is by construction a row its owner could open. Re-implementing the filter here
23
- would be a second definition of "matches", and those two would drift.
24
- """
25
- from fastapi import APIRouter, Body, Depends
26
-
27
- import core.alerts as alerts
28
- from deps import Session, err, require_session
29
-
30
- router = APIRouter(prefix="/api/v1")
31
-
32
- #: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
33
- _TOPICS = ("customer", "product")
34
-
35
-
36
- def _topic_or_400(raw):
37
- topic = str(raw or "").strip().lower()
38
- if topic.startswith("ut_") or topic in _TOPICS:
39
- return topic
40
- raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
41
-
42
-
43
- def _owner_session(session: Session, owner: str):
44
- """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
45
-
46
- ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an
47
- owner session is built by swapping the `user` RECORD and letting both derive themselves. An
48
- earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
49
- first write hook of the wave; the properties are the single definition of who a session is,
50
- and going around them is how a session with an admin flag and a non-admin record exists.
51
-
52
- Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather
53
- than evaluating as somebody else, which is the fail-closed direction.
54
- """
55
- import core.users as users
56
-
57
- if str(owner) == str(session.uname):
58
- return session
59
- rec = (users.registry() or {}).get(str(owner))
60
- if not isinstance(rec, dict) or not rec.get("active", True):
61
- return None
62
- # `_public` is THE definition of what a session may know about its own account (never a hash
63
- # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a
64
- # second definition, and the one that leaks is always the copy.
65
- return Session(tenant=session.tenant, user=users._public(str(owner), rec),
66
- claims=session.claims, runtime=session.runtime)
67
-
68
-
69
- def _evaluate(session: Session, rec: dict):
70
- """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in."""
71
- import aios_grid
72
- from harness import filter_eval
73
-
74
- owner_sess = _owner_session(session, rec.get("owner"))
75
- if owner_sess is None:
76
- return {"skipped": "owner_unavailable"}
77
- topic = str(rec.get("topic") or "")
78
- try:
79
- if topic.startswith("ut_"):
80
- from routes_tables import ut_assembly
81
- g = ut_assembly(owner_sess, topic,
82
- storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}")
83
- else:
84
- from routes_customers import grid_assembly
85
- g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
86
- except Exception as e: # noqa: BLE001
87
- return {"skipped": "unavailable", "detail": type(e).__name__}
88
-
89
- view = (g.get("views") or {}).get(str(rec.get("viewId")))
90
- if not isinstance(view, dict):
91
- # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
92
- # deleting the alert: an alert that silently vanishes is indistinguishable from one that
93
- # never fires, and the user cannot debug what is not there.
94
- return {"skipped": "view_missing"}
95
-
96
- # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived
97
- # and overlay values on a row. Evaluating a filter against raw pool dicts would silently
98
- # never match any condition on a user-created or measure column.
99
- rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], g["ws"].get("overlays"),
100
- derived=g.get("derived"))
101
- config = view.get("config") or view
102
- ctx = filter_eval.EvalCtx(
103
- cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
104
- for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
105
- measure_sets=g.get("measure_sets") or {},
106
- today=g.get("today"))
107
- pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
108
- member_pids=config.get("memberPids"))
109
- labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
110
- return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
111
- labels=labels, partial=False, st=session.runtime)
112
-
113
-
114
- @router.get("/alerts")
115
- def list_alerts(session: Session = Depends(require_session)):
116
- return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
117
- st=session.runtime)}
118
-
119
-
120
- @router.post("/alerts")
121
- def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
122
- body = body or {}
123
- view_id = str(body.get("viewId") or "").strip()
124
- if not view_id:
125
- raise err(400, "bad_view", "an alert needs the id of the view it watches")
126
- topic = _topic_or_400(body.get("topic"))
127
- _require_filtered_view(session, topic, view_id)
128
- import uuid
129
- aid = f"al_{uuid.uuid4().hex[:12]}"
130
- rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
131
- label=body.get("label") or "", st=session.runtime)
132
- # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
133
- # Deferring this to the first write hook would mean the next edit announces the whole view.
134
- outcome = _evaluate(session, rec)
135
- return {"alert": {**rec, "seeded": True}, "first": outcome}
136
-
137
-
138
- def _require_filtered_view(session: Session, topic: str, view_id: str):
139
- """400 unless `view_id` exists on `topic` AND actually narrows something.
140
-
141
- ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
142
- that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
143
- (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
144
- entrant again — there is nothing left to enter. The owner's words are *"when a Record gets
145
- into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
146
- discovered by never being notified.
147
-
148
- `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a
149
- half-typed rule is not a filter, and this must agree with what actually narrows or it would
150
- accept a view whose one rule the engine then ignores.
151
- """
152
- from harness import filter_eval
153
-
154
- try:
155
- if topic.startswith("ut_"):
156
- from routes_tables import ut_assembly
157
- g = ut_assembly(session, topic,
158
- storage_key=f"{session.tenant}:{topic}:{session.uname}")
159
- else:
160
- from routes_customers import grid_assembly
161
- g = grid_assembly(session, scope=topic, consume_corrections=False)
162
- except Exception: # noqa: BLE001
163
- raise err(503, "unavailable", "the table is unavailable — try again in a moment")
164
- view = (g.get("views") or {}).get(str(view_id))
165
- if not isinstance(view, dict):
166
- raise err(404, "no_view", "that view does not exist on this table")
167
- nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
168
-
169
- def _any_active(ns):
170
- for n in ns or ():
171
- if isinstance(n, dict) and isinstance(n.get("children"), list):
172
- if _any_active(n["children"]):
173
- return True
174
- elif filter_eval.is_rule_active(n):
175
- return True
176
- return False
177
-
178
- if not _any_active(nodes):
179
- raise err(400, "no_filter",
180
- "this view has no active filter, so no record can ever ENTER it — add a "
181
- "condition to the view first, then create the alert")
182
-
183
-
184
- @router.delete("/alerts/{alert_id}")
185
- def delete_alert(alert_id: str, session: Session = Depends(require_session)):
186
- rec = next((r for r in alerts.list_alerts(st=session.runtime)
187
- if str(r.get("id")) == str(alert_id)), None)
188
- if rec is None:
189
- raise err(404, "no_alert", "that alert does not exist")
190
- if str(rec.get("owner")) != str(session.uname) and not session.admin:
191
- raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
192
- alerts.delete(alert_id, st=session.runtime)
193
- return {"ok": True}
194
-
195
-
196
- @router.post("/alerts/{alert_id}/run")
197
- def run_alert(alert_id: str, session: Session = Depends(require_session)):
198
- rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
199
- st=session.runtime)
200
- if str(r.get("id")) == str(alert_id)), None)
201
- if rec is None:
202
- raise err(404, "no_alert", "that alert does not exist")
203
- return _evaluate(session, rec)
204
-
205
-
206
- @router.get("/notifications")
207
- def notifications(session: Session = Depends(require_session)):
208
- """The inbox — RE-EVALUATED on read, which is a deliberate design choice.
209
-
210
- ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
211
- plan was a push hook: the automation engine calls `after_write` when it lands rows. But
212
- `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
213
- evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside
214
- a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
215
- construction that leaks scope.
216
-
217
- Pulling on read has none of that: the caller IS a session, the assemblies are already
218
- scope-cached, and the user cannot observe the difference — an inbox is only ever read by
219
- someone opening it. The cost is that a notification is minted when you LOOK rather than when
220
- the row landed, so the `at` stamp is detection time, not arrival time.
221
-
222
- `after_write` stays exported for the day the engine can hand over a real identity.
223
- """
224
- for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
225
- try:
226
- _evaluate(session, rec)
227
- except Exception: # noqa: BLE001
228
- continue # one bad alert must not empty the pane
229
- return alerts.inbox(session.uname, st=session.runtime)
230
-
231
-
232
- @router.post("/notifications/read")
233
- def read_notifications(body: dict = Body(default=None),
234
- session: Session = Depends(require_session)):
235
- body = body or {}
236
- ids = body.get("ids")
237
- if ids is not None and not isinstance(ids, list):
238
- raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
239
- return alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
240
- st=session.runtime)
241
-
242
-
243
- def after_write(session: Session, topic_key: str):
244
- """THE WRITE HOOK — call after a write that could change what a view matches.
245
-
246
- Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
247
- upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
248
- edit that triggered it.
249
- """
250
- try:
251
- return alerts.after_write(topic_key, st=session.runtime,
252
- runner=lambda rec: _evaluate(session, rec))
253
- except Exception: # noqa: BLE001
254
- return {"evaluated": 0}
 
1
+ """routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT).
2
+
3
+ GET /api/v1/alerts -> {alerts:[...]}
4
+ POST /api/v1/alerts <- {viewId, topic, label?}
5
+ DELETE /api/v1/alerts/{alert_id}
6
+ POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
7
+ GET /api/v1/notifications -> {unread, items:[...]}
8
+ POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
9
+
10
+ The semantics — an alert is a view plus a remembered matched set, a notification is a NEW
11
+ ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning.
12
+ This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
13
+
14
+ ⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
15
+ for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
16
+ full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
17
+ book, and the notification would name customers that user may not see — a permission leak wearing
18
+ a notification's clothes. The owner's own scope is the only correct basis for their alert.
19
+
20
+ ⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
21
+ `routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
22
+ alert can see is by construction a row its owner could open. Re-implementing the filter here
23
+ would be a second definition of "matches", and those two would drift.
24
+ """
25
+ from fastapi import APIRouter, Body, Depends
26
+
27
+ import core.alerts as alerts
28
+ from deps import Session, err, require_session
29
+
30
+ router = APIRouter(prefix="/api/v1")
31
+
32
+ #: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
33
+ _TOPICS = ("customer", "product")
34
+
35
+
36
+ def _topic_or_400(raw):
37
+ topic = str(raw or "").strip().lower()
38
+ if topic.startswith("ut_") or topic in _TOPICS:
39
+ return topic
40
+ raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
41
+
42
+
43
+ def _owner_session(session: Session, owner: str):
44
+ """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
45
+
46
+ ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an
47
+ owner session is built by swapping the `user` RECORD and letting both derive themselves. An
48
+ earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
49
+ first write hook of the wave; the properties are the single definition of who a session is,
50
+ and going around them is how a session with an admin flag and a non-admin record exists.
51
+
52
+ Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather
53
+ than evaluating as somebody else, which is the fail-closed direction.
54
+ """
55
+ import core.users as users
56
+
57
+ if str(owner) == str(session.uname):
58
+ return session
59
+ rec = (users.registry() or {}).get(str(owner))
60
+ if not isinstance(rec, dict) or not rec.get("active", True):
61
+ return None
62
+ # `_public` is THE definition of what a session may know about its own account (never a hash
63
+ # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a
64
+ # second definition, and the one that leaks is always the copy.
65
+ return Session(tenant=session.tenant, user=users._public(str(owner), rec),
66
+ claims=session.claims, runtime=session.runtime)
67
+
68
+
69
+ def _evaluate(session: Session, rec: dict):
70
+ """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in."""
71
+ import aios_grid
72
+ from harness import filter_eval
73
+
74
+ owner_sess = _owner_session(session, rec.get("owner"))
75
+ if owner_sess is None:
76
+ return {"skipped": "owner_unavailable"}
77
+ topic = str(rec.get("topic") or "")
78
+ try:
79
+ if topic.startswith("ut_"):
80
+ from routes_tables import ut_assembly
81
+ g = ut_assembly(owner_sess, topic,
82
+ storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}")
83
+ else:
84
+ from routes_customers import grid_assembly
85
+ g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
86
+ except Exception as e: # noqa: BLE001
87
+ return {"skipped": "unavailable", "detail": type(e).__name__}
88
+
89
+ view = (g.get("views") or {}).get(str(rec.get("viewId")))
90
+ if not isinstance(view, dict):
91
+ # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
92
+ # deleting the alert: an alert that silently vanishes is indistinguishable from one that
93
+ # never fires, and the user cannot debug what is not there.
94
+ return {"skipped": "view_missing"}
95
+
96
+ # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived
97
+ # and overlay values on a row. Evaluating a filter against raw pool dicts would silently
98
+ # never match any condition on a user-created or measure column.
99
+ rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], g["ws"].get("overlays"),
100
+ derived=g.get("derived"))
101
+ config = view.get("config") or view
102
+ ctx = filter_eval.EvalCtx(
103
+ cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
104
+ for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
105
+ measure_sets=g.get("measure_sets") or {},
106
+ today=g.get("today"))
107
+ pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
108
+ member_pids=config.get("memberPids"))
109
+ labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
110
+ return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
111
+ labels=labels, partial=False, st=session.runtime)
112
+
113
+
114
+ @router.get("/alerts")
115
+ def list_alerts(session: Session = Depends(require_session)):
116
+ return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
117
+ st=session.runtime)}
118
+
119
+
120
+ @router.post("/alerts")
121
+ def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
122
+ body = body or {}
123
+ view_id = str(body.get("viewId") or "").strip()
124
+ if not view_id:
125
+ raise err(400, "bad_view", "an alert needs the id of the view it watches")
126
+ topic = _topic_or_400(body.get("topic"))
127
+ _require_filtered_view(session, topic, view_id)
128
+ import uuid
129
+ aid = f"al_{uuid.uuid4().hex[:12]}"
130
+ rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
131
+ label=body.get("label") or "", st=session.runtime)
132
+ # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
133
+ # Deferring this to the first write hook would mean the next edit announces the whole view.
134
+ outcome = _evaluate(session, rec)
135
+ return {"alert": {**rec, "seeded": True}, "first": outcome}
136
+
137
+
138
+ def _require_filtered_view(session: Session, topic: str, view_id: str):
139
+ """400 unless `view_id` exists on `topic` AND actually narrows something.
140
+
141
+ ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
142
+ that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
143
+ (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
144
+ entrant again — there is nothing left to enter. The owner's words are *"when a Record gets
145
+ into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
146
+ discovered by never being notified.
147
+
148
+ `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a
149
+ half-typed rule is not a filter, and this must agree with what actually narrows or it would
150
+ accept a view whose one rule the engine then ignores.
151
+ """
152
+ from harness import filter_eval
153
+
154
+ try:
155
+ if topic.startswith("ut_"):
156
+ from routes_tables import ut_assembly
157
+ g = ut_assembly(session, topic,
158
+ storage_key=f"{session.tenant}:{topic}:{session.uname}")
159
+ else:
160
+ from routes_customers import grid_assembly
161
+ g = grid_assembly(session, scope=topic, consume_corrections=False)
162
+ except Exception: # noqa: BLE001
163
+ raise err(503, "unavailable", "the table is unavailable — try again in a moment")
164
+ view = (g.get("views") or {}).get(str(view_id))
165
+ if not isinstance(view, dict):
166
+ raise err(404, "no_view", "that view does not exist on this table")
167
+ nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
168
+
169
+ def _any_active(ns):
170
+ for n in ns or ():
171
+ if isinstance(n, dict) and isinstance(n.get("children"), list):
172
+ if _any_active(n["children"]):
173
+ return True
174
+ elif filter_eval.is_rule_active(n):
175
+ return True
176
+ return False
177
+
178
+ if not _any_active(nodes):
179
+ raise err(400, "no_filter",
180
+ "this view has no active filter, so no record can ever ENTER it — add a "
181
+ "condition to the view first, then create the alert")
182
+
183
+
184
+ @router.delete("/alerts/{alert_id}")
185
+ def delete_alert(alert_id: str, session: Session = Depends(require_session)):
186
+ rec = next((r for r in alerts.list_alerts(st=session.runtime)
187
+ if str(r.get("id")) == str(alert_id)), None)
188
+ if rec is None:
189
+ raise err(404, "no_alert", "that alert does not exist")
190
+ if str(rec.get("owner")) != str(session.uname) and not session.admin:
191
+ raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
192
+ alerts.delete(alert_id, st=session.runtime)
193
+ return {"ok": True}
194
+
195
+
196
+ @router.post("/alerts/{alert_id}/run")
197
+ def run_alert(alert_id: str, session: Session = Depends(require_session)):
198
+ rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
199
+ st=session.runtime)
200
+ if str(r.get("id")) == str(alert_id)), None)
201
+ if rec is None:
202
+ raise err(404, "no_alert", "that alert does not exist")
203
+ return _evaluate(session, rec)
204
+
205
+
206
+ @router.get("/notifications")
207
+ def notifications(session: Session = Depends(require_session)):
208
+ """The inbox — RE-EVALUATED on read, which is a deliberate design choice.
209
+
210
+ ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
211
+ plan was a push hook: the automation engine calls `after_write` when it lands rows. But
212
+ `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
213
+ evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside
214
+ a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
215
+ construction that leaks scope.
216
+
217
+ Pulling on read has none of that: the caller IS a session, the assemblies are already
218
+ scope-cached, and the user cannot observe the difference — an inbox is only ever read by
219
+ someone opening it. The cost is that a notification is minted when you LOOK rather than when
220
+ the row landed, so the `at` stamp is detection time, not arrival time.
221
+
222
+ `after_write` stays exported for the day the engine can hand over a real identity.
223
+ """
224
+ for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
225
+ try:
226
+ _evaluate(session, rec)
227
+ except Exception: # noqa: BLE001
228
+ continue # one bad alert must not empty the pane
229
+ return alerts.inbox(session.uname, st=session.runtime)
230
+
231
+
232
+ @router.post("/notifications/read")
233
+ def read_notifications(body: dict = Body(default=None),
234
+ session: Session = Depends(require_session)):
235
+ body = body or {}
236
+ ids = body.get("ids")
237
+ if ids is not None and not isinstance(ids, list):
238
+ raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
239
+ return alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
240
+ st=session.runtime)
241
+
242
+
243
+ def after_write(session: Session, topic_key: str):
244
+ """THE WRITE HOOK — call after a write that could change what a view matches.
245
+
246
+ Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
247
+ upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
248
+ edit that triggered it.
249
+ """
250
+ try:
251
+ return alerts.after_write(topic_key, st=session.runtime,
252
+ runner=lambda rec: _evaluate(session, rec))
253
+ except Exception: # noqa: BLE001
254
+ return {"evaluated": 0}
api/routes_auth.py CHANGED
@@ -1,217 +1,217 @@
1
- """routes_auth.py — X2's auth leg: login / logout / me (EXIT-3a).
2
-
3
- Password auth against the SAME `core/users` accounts the Streamlit `gate()` uses, so both
4
- front-ends share one identity today. The amended D-3 (our own OIDC client against Authentik, with
5
- a branded login page) lands when D-1/D-2 unblock (owner blocker B-4); this is the interim leg it
6
- replaces, and it is deliberately the same account store so the migration is a swap of the
7
- CREDENTIAL check, not of the user model.
8
- """
9
- import base64 as _b64
10
- import hmac
11
- import os
12
- import re as _re
13
- import time
14
- from collections import OrderedDict
15
-
16
- from fastapi import APIRouter, Body, Depends, Request, Response
17
-
18
- import aios_session
19
- from deps import Session, err, require_session, users, perms
20
-
21
- router = APIRouter(prefix="/api/v1/auth")
22
-
23
- #: THE SERVER OWNS BACKOFF (X5): the login page shows an inline error and never paces itself, so
24
- #: the pacing has to be here or it does not exist. Bounded so it cannot grow into a memory leak.
25
- #: ⚠ HONEST LIMIT: per-PROCESS. Behind several workers an attacker gets this budget per worker,
26
- #: and it resets on deploy. It raises the cost of online guessing against PBKDF2-200k; it is not
27
- #: a substitute for the real thing, which arrives with Authentik (D-1) owning rate limits centrally.
28
- #: ⚠ KEYED BY (TENANT, USERNAME), not by username. Two tenants can each have an 'admin' or a
29
- #: 'boss', and a shared bucket would mean failed logins against tenant B lock the same name out of
30
- #: tenant A — a cross-tenant coupling in the one wave whose point is that tenants cannot reach
31
- #: each other. It is also a (weak) existence oracle: a 429 for a name you never tried here would
32
- #: tell you somebody else did.
33
- _FAILS = OrderedDict()
34
- _MAX_TRACKED = 2048
35
- _LOCKOUT_AFTER = 8
36
- _LOCKOUT_SECONDS = 60
37
-
38
-
39
- def _throttled(key, now=None):
40
- now = now or time.time()
41
- hit = _FAILS.get(key)
42
- if not hit:
43
- return 0
44
- count, last = hit
45
- if count < _LOCKOUT_AFTER:
46
- return 0
47
- remaining = int(_LOCKOUT_SECONDS - (now - last))
48
- return remaining if remaining > 0 else 0
49
-
50
-
51
- def _note_failure(key, now=None):
52
- now = now or time.time()
53
- count, last = _FAILS.get(key, (0, 0.0))
54
- # A window that has fully elapsed starts the count again — a user who mistypes twice today
55
- # and twice next week is not an attacker.
56
- if now - last > _LOCKOUT_SECONDS:
57
- count = 0
58
- _FAILS[key] = (count + 1, now)
59
- _FAILS.move_to_end(key)
60
- while len(_FAILS) > _MAX_TRACKED:
61
- _FAILS.popitem(last=False)
62
-
63
-
64
- def _clear_failures(key):
65
- _FAILS.pop(key, None)
66
-
67
-
68
- def _public_user(user):
69
- """What the client is allowed to know about itself. Never a hash, a salt, or the epoch —
70
- the epoch is a server-side revocation handle and a client has no use for it."""
71
- return {"username": user.get("username", ""), "name": user.get("name", ""),
72
- "role": user.get("role", "user"),
73
- # Wave 18 (C1-TENANT, R1): the company this session is bound to — the client shows
74
- # it and tests assert the binding; the SERVER-side truth stays in the signed cookie.
75
- "tenant": str(user.get("tenant") or "royal-imports").strip().lower(),
76
- "bus": perms.allowed_bu_labels(user),
77
- "team_id": perms.scope_team_id(user),
78
- "agent": perms.scope_agent(user),
79
- "modules": sorted(perms.allowed_modules(user) or []) or "all",
80
- "landing": perms.landing_page(user),
81
- # Wave 14 C-AVATAR — the user's OWN photo (Settings preview + the shell chip).
82
- # Everyone else's rides workspace.userAvatars, keyed by display name.
83
- "avatar": user.get("avatar") or None}
84
-
85
-
86
- #: Wave 14 C-AVATAR — the write-side wall. PNG/JPEG data URLs only; the client downscales to
87
- #: <=128px before posting, and the server re-validates because a stored data URL is served to
88
- #: every grid session verbatim.
89
- _AVATAR_RE = _re.compile(r"^data:image/(png|jpeg);base64,([A-Za-z0-9+/=]+)$")
90
- _AVATAR_MAX_BYTES = 64 * 1024
91
-
92
-
93
- @router.post("/me/avatar")
94
- def set_avatar(body: dict = Body(default=None),
95
- session: Session = Depends(require_session)):
96
- import core.store as _store
97
- raw = str((body or {}).get("dataUrl") or "")
98
- m = _AVATAR_RE.match(raw)
99
- if not m:
100
- raise err(400, "bad_avatar",
101
- "expected a data:image/png;base64,... or image/jpeg data URL")
102
- try:
103
- blob = _b64.b64decode(m.group(2), validate=True)
104
- except Exception:
105
- raise err(400, "bad_avatar", "that data URL is not valid base64")
106
- if len(blob) > _AVATAR_MAX_BYTES:
107
- raise err(400, "bad_avatar",
108
- f"avatar too large - {_AVATAR_MAX_BYTES // 1024}KB decoded max "
109
- f"(downscale to 128px before posting)")
110
- if not _store.available():
111
- raise err(503, "store_unavailable", "the tenant store is unavailable")
112
- # session.uname is the canonical account handle; the session's user RECORD does not
113
- # carry a "username" key (it is the registry's dict key), so keying the write off the
114
- # record would silently no-op — caught by X7's fresh /me read.
115
- users.set_avatar(session.uname, raw)
116
- u2 = dict(session.user)
117
- u2["avatar"] = raw
118
- return {"user": _public_user(u2)}
119
-
120
-
121
- @router.delete("/me/avatar")
122
- def clear_avatar(session: Session = Depends(require_session)):
123
- import core.store as _store
124
- if not _store.available():
125
- raise err(503, "store_unavailable", "the tenant store is unavailable")
126
- users.set_avatar(session.uname, None)
127
- u2 = dict(session.user)
128
- u2.pop("avatar", None)
129
- return {"user": _public_user(u2)}
130
-
131
-
132
- @router.post("/login")
133
- def login(request: Request, response: Response, body: dict = Body(default=None)):
134
- """Wave 18 (C1-TENANT, R1): ONE login box — the ACCOUNT decides the tenant.
135
-
136
- The pre-wave flow validated a caller-posted tenant slug and minted the session for it, so
137
- the same credential could sign in to any registered tenant. Now the credential resolves
138
- FIRST (username or email — `users.verify` takes both) and the session binds to the tenant
139
- ON THE RECORD; the posted `tenant` field is accepted for wire-compat and ignored. An
140
- account whose tenant no longer resolves (deleted / suspended record) gets the same 401 as
141
- a bad password — "which tenants exist" is not the login form's question to answer.
142
- """
143
- body = body or {}
144
- uname = str(body.get("username") or "").strip().lower()
145
- pw = str(body.get("password") or "")
146
-
147
- # Throttle on the IDENTIFIER alone: the tenant is not known until the credential resolves,
148
- # and a per-(tenant, uname) key would let an attacker reset their budget by rotating slugs.
149
- throttle_key = ("*", uname)
150
- wait = _throttled(throttle_key)
151
- if wait:
152
- raise err(429, "too_many_attempts",
153
- f"too many failed attempts — try again in {wait} seconds")
154
-
155
- user = users.verify(uname, pw) if (uname and pw) else None
156
- if not user:
157
- # CONSTANT-TIME-ISH 401. `users.verify` returns immediately for an unknown username (no
158
- # record, no PBKDF2) and burns ~200k iterations for a known one, so the response time
159
- # answers "does this account exist?" to anyone with a stopwatch. Burning one equivalent
160
- # hash on the failure path removes that oracle. It costs nothing on the happy path.
161
- try:
162
- users._hash(pw or "x", "00" * 16)
163
- except Exception:
164
- pass
165
- _note_failure(throttle_key)
166
- raise err(401, "invalid_credentials", "that username and password do not match")
167
-
168
- tenant = str(user.get("tenant") or "royal-imports").strip().lower()
169
- from harness import runtime
170
- try:
171
- runtime.get_runtime(tenant)
172
- except KeyError:
173
- _note_failure(throttle_key)
174
- raise err(401, "invalid_credentials", "that username and password do not match")
175
-
176
- _clear_failures(throttle_key)
177
- value, _claims = aios_session.mint(tenant, user["username"], int(user.get("epoch") or 0))
178
- aios_session.set_cookie(response, request, value)
179
- # Wave 19 (R4): the login stamp. THE RESOLVED username, never the posted identifier — the
180
- # person may have typed an email address and `users.verify` resolved it to the registry key;
181
- # stamping what was typed would write onto a key that does not exist. Fail-silent inside
182
- # `touch_login` and a no-op for the emergency-master identity (which has no record to stamp),
183
- # so the one thing this cannot do is turn a good credential into a failed sign-in.
184
- users.touch_login(user["username"])
185
- # `touch_login` also sets `last_active`, so tell the session resolver it has been seen —
186
- # otherwise the very next request stamps it again and the hourly throttle is off by one write
187
- # per sign-in.
188
- import deps as _deps
189
- _deps.note_active(user["username"])
190
- return {"user": _public_user(user)}
191
-
192
-
193
- @router.post("/logout", status_code=204)
194
- def logout(request: Request):
195
- """204 and the cookie is cleared. Deliberately NOT session-gated: logging out must work from
196
- an already-invalid session, or a user holding a broken cookie has no way to get rid of it.
197
-
198
- ⚠ THE COOKIE IS CLEARED ON THE RETURNED RESPONSE, not on an injected `response` param. When a
199
- handler RETURNS a Response object, FastAPI ships that object — anything written to the
200
- injected `response` is silently dropped. The first version of this route took `response:
201
- Response`, called `clear_cookie` on it, then returned a fresh `Response(204)`: the 204 was
202
- correct, no `Set-Cookie` was ever sent, and the session stayed alive after a "successful"
203
- logout. Caught by asserting `/auth/me` is 401 AFTER the logout rather than trusting the 204.
204
-
205
- ⚠ This clears the BROWSER's copy only — the signed value stays cryptographically valid until
206
- it expires, which is the honest cost of a stateless session. "Sign me out everywhere" is
207
- `core.users.bump_epoch` (a password change already does it); D2's Postgres session mirror
208
- makes per-device revocation possible and arrives with C-2.
209
- """
210
- out = Response(status_code=204)
211
- aios_session.clear_cookie(out, request)
212
- return out
213
-
214
-
215
- @router.get("/me")
216
- def me(session: Session = Depends(require_session)):
217
- return {"user": _public_user(session.user), "tenant": session.tenant}
 
1
+ """routes_auth.py — X2's auth leg: login / logout / me (EXIT-3a).
2
+
3
+ Password auth against the SAME `core/users` accounts the Streamlit `gate()` uses, so both
4
+ front-ends share one identity today. The amended D-3 (our own OIDC client against Authentik, with
5
+ a branded login page) lands when D-1/D-2 unblock (owner blocker B-4); this is the interim leg it
6
+ replaces, and it is deliberately the same account store so the migration is a swap of the
7
+ CREDENTIAL check, not of the user model.
8
+ """
9
+ import base64 as _b64
10
+ import hmac
11
+ import os
12
+ import re as _re
13
+ import time
14
+ from collections import OrderedDict
15
+
16
+ from fastapi import APIRouter, Body, Depends, Request, Response
17
+
18
+ import aios_session
19
+ from deps import Session, err, require_session, users, perms
20
+
21
+ router = APIRouter(prefix="/api/v1/auth")
22
+
23
+ #: THE SERVER OWNS BACKOFF (X5): the login page shows an inline error and never paces itself, so
24
+ #: the pacing has to be here or it does not exist. Bounded so it cannot grow into a memory leak.
25
+ #: ⚠ HONEST LIMIT: per-PROCESS. Behind several workers an attacker gets this budget per worker,
26
+ #: and it resets on deploy. It raises the cost of online guessing against PBKDF2-200k; it is not
27
+ #: a substitute for the real thing, which arrives with Authentik (D-1) owning rate limits centrally.
28
+ #: ⚠ KEYED BY (TENANT, USERNAME), not by username. Two tenants can each have an 'admin' or a
29
+ #: 'boss', and a shared bucket would mean failed logins against tenant B lock the same name out of
30
+ #: tenant A — a cross-tenant coupling in the one wave whose point is that tenants cannot reach
31
+ #: each other. It is also a (weak) existence oracle: a 429 for a name you never tried here would
32
+ #: tell you somebody else did.
33
+ _FAILS = OrderedDict()
34
+ _MAX_TRACKED = 2048
35
+ _LOCKOUT_AFTER = 8
36
+ _LOCKOUT_SECONDS = 60
37
+
38
+
39
+ def _throttled(key, now=None):
40
+ now = now or time.time()
41
+ hit = _FAILS.get(key)
42
+ if not hit:
43
+ return 0
44
+ count, last = hit
45
+ if count < _LOCKOUT_AFTER:
46
+ return 0
47
+ remaining = int(_LOCKOUT_SECONDS - (now - last))
48
+ return remaining if remaining > 0 else 0
49
+
50
+
51
+ def _note_failure(key, now=None):
52
+ now = now or time.time()
53
+ count, last = _FAILS.get(key, (0, 0.0))
54
+ # A window that has fully elapsed starts the count again — a user who mistypes twice today
55
+ # and twice next week is not an attacker.
56
+ if now - last > _LOCKOUT_SECONDS:
57
+ count = 0
58
+ _FAILS[key] = (count + 1, now)
59
+ _FAILS.move_to_end(key)
60
+ while len(_FAILS) > _MAX_TRACKED:
61
+ _FAILS.popitem(last=False)
62
+
63
+
64
+ def _clear_failures(key):
65
+ _FAILS.pop(key, None)
66
+
67
+
68
+ def _public_user(user):
69
+ """What the client is allowed to know about itself. Never a hash, a salt, or the epoch —
70
+ the epoch is a server-side revocation handle and a client has no use for it."""
71
+ return {"username": user.get("username", ""), "name": user.get("name", ""),
72
+ "role": user.get("role", "user"),
73
+ # Wave 18 (C1-TENANT, R1): the company this session is bound to — the client shows
74
+ # it and tests assert the binding; the SERVER-side truth stays in the signed cookie.
75
+ "tenant": str(user.get("tenant") or "royal-imports").strip().lower(),
76
+ "bus": perms.allowed_bu_labels(user),
77
+ "team_id": perms.scope_team_id(user),
78
+ "agent": perms.scope_agent(user),
79
+ "modules": sorted(perms.allowed_modules(user) or []) or "all",
80
+ "landing": perms.landing_page(user),
81
+ # Wave 14 C-AVATAR — the user's OWN photo (Settings preview + the shell chip).
82
+ # Everyone else's rides workspace.userAvatars, keyed by display name.
83
+ "avatar": user.get("avatar") or None}
84
+
85
+
86
+ #: Wave 14 C-AVATAR — the write-side wall. PNG/JPEG data URLs only; the client downscales to
87
+ #: <=128px before posting, and the server re-validates because a stored data URL is served to
88
+ #: every grid session verbatim.
89
+ _AVATAR_RE = _re.compile(r"^data:image/(png|jpeg);base64,([A-Za-z0-9+/=]+)$")
90
+ _AVATAR_MAX_BYTES = 64 * 1024
91
+
92
+
93
+ @router.post("/me/avatar")
94
+ def set_avatar(body: dict = Body(default=None),
95
+ session: Session = Depends(require_session)):
96
+ import core.store as _store
97
+ raw = str((body or {}).get("dataUrl") or "")
98
+ m = _AVATAR_RE.match(raw)
99
+ if not m:
100
+ raise err(400, "bad_avatar",
101
+ "expected a data:image/png;base64,... or image/jpeg data URL")
102
+ try:
103
+ blob = _b64.b64decode(m.group(2), validate=True)
104
+ except Exception:
105
+ raise err(400, "bad_avatar", "that data URL is not valid base64")
106
+ if len(blob) > _AVATAR_MAX_BYTES:
107
+ raise err(400, "bad_avatar",
108
+ f"avatar too large - {_AVATAR_MAX_BYTES // 1024}KB decoded max "
109
+ f"(downscale to 128px before posting)")
110
+ if not _store.available():
111
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
112
+ # session.uname is the canonical account handle; the session's user RECORD does not
113
+ # carry a "username" key (it is the registry's dict key), so keying the write off the
114
+ # record would silently no-op — caught by X7's fresh /me read.
115
+ users.set_avatar(session.uname, raw)
116
+ u2 = dict(session.user)
117
+ u2["avatar"] = raw
118
+ return {"user": _public_user(u2)}
119
+
120
+
121
+ @router.delete("/me/avatar")
122
+ def clear_avatar(session: Session = Depends(require_session)):
123
+ import core.store as _store
124
+ if not _store.available():
125
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
126
+ users.set_avatar(session.uname, None)
127
+ u2 = dict(session.user)
128
+ u2.pop("avatar", None)
129
+ return {"user": _public_user(u2)}
130
+
131
+
132
+ @router.post("/login")
133
+ def login(request: Request, response: Response, body: dict = Body(default=None)):
134
+ """Wave 18 (C1-TENANT, R1): ONE login box — the ACCOUNT decides the tenant.
135
+
136
+ The pre-wave flow validated a caller-posted tenant slug and minted the session for it, so
137
+ the same credential could sign in to any registered tenant. Now the credential resolves
138
+ FIRST (username or email — `users.verify` takes both) and the session binds to the tenant
139
+ ON THE RECORD; the posted `tenant` field is accepted for wire-compat and ignored. An
140
+ account whose tenant no longer resolves (deleted / suspended record) gets the same 401 as
141
+ a bad password — "which tenants exist" is not the login form's question to answer.
142
+ """
143
+ body = body or {}
144
+ uname = str(body.get("username") or "").strip().lower()
145
+ pw = str(body.get("password") or "")
146
+
147
+ # Throttle on the IDENTIFIER alone: the tenant is not known until the credential resolves,
148
+ # and a per-(tenant, uname) key would let an attacker reset their budget by rotating slugs.
149
+ throttle_key = ("*", uname)
150
+ wait = _throttled(throttle_key)
151
+ if wait:
152
+ raise err(429, "too_many_attempts",
153
+ f"too many failed attempts — try again in {wait} seconds")
154
+
155
+ user = users.verify(uname, pw) if (uname and pw) else None
156
+ if not user:
157
+ # CONSTANT-TIME-ISH 401. `users.verify` returns immediately for an unknown username (no
158
+ # record, no PBKDF2) and burns ~200k iterations for a known one, so the response time
159
+ # answers "does this account exist?" to anyone with a stopwatch. Burning one equivalent
160
+ # hash on the failure path removes that oracle. It costs nothing on the happy path.
161
+ try:
162
+ users._hash(pw or "x", "00" * 16)
163
+ except Exception:
164
+ pass
165
+ _note_failure(throttle_key)
166
+ raise err(401, "invalid_credentials", "that username and password do not match")
167
+
168
+ tenant = str(user.get("tenant") or "royal-imports").strip().lower()
169
+ from harness import runtime
170
+ try:
171
+ runtime.get_runtime(tenant)
172
+ except KeyError:
173
+ _note_failure(throttle_key)
174
+ raise err(401, "invalid_credentials", "that username and password do not match")
175
+
176
+ _clear_failures(throttle_key)
177
+ value, _claims = aios_session.mint(tenant, user["username"], int(user.get("epoch") or 0))
178
+ aios_session.set_cookie(response, request, value)
179
+ # Wave 19 (R4): the login stamp. THE RESOLVED username, never the posted identifier — the
180
+ # person may have typed an email address and `users.verify` resolved it to the registry key;
181
+ # stamping what was typed would write onto a key that does not exist. Fail-silent inside
182
+ # `touch_login` and a no-op for the emergency-master identity (which has no record to stamp),
183
+ # so the one thing this cannot do is turn a good credential into a failed sign-in.
184
+ users.touch_login(user["username"])
185
+ # `touch_login` also sets `last_active`, so tell the session resolver it has been seen —
186
+ # otherwise the very next request stamps it again and the hourly throttle is off by one write
187
+ # per sign-in.
188
+ import deps as _deps
189
+ _deps.note_active(user["username"])
190
+ return {"user": _public_user(user)}
191
+
192
+
193
+ @router.post("/logout", status_code=204)
194
+ def logout(request: Request):
195
+ """204 and the cookie is cleared. Deliberately NOT session-gated: logging out must work from
196
+ an already-invalid session, or a user holding a broken cookie has no way to get rid of it.
197
+
198
+ ⚠ THE COOKIE IS CLEARED ON THE RETURNED RESPONSE, not on an injected `response` param. When a
199
+ handler RETURNS a Response object, FastAPI ships that object — anything written to the
200
+ injected `response` is silently dropped. The first version of this route took `response:
201
+ Response`, called `clear_cookie` on it, then returned a fresh `Response(204)`: the 204 was
202
+ correct, no `Set-Cookie` was ever sent, and the session stayed alive after a "successful"
203
+ logout. Caught by asserting `/auth/me` is 401 AFTER the logout rather than trusting the 204.
204
+
205
+ ⚠ This clears the BROWSER's copy only — the signed value stays cryptographically valid until
206
+ it expires, which is the honest cost of a stateless session. "Sign me out everywhere" is
207
+ `core.users.bump_epoch` (a password change already does it); D2's Postgres session mirror
208
+ makes per-device revocation possible and arrives with C-2.
209
+ """
210
+ out = Response(status_code=204)
211
+ aios_session.clear_cookie(out, request)
212
+ return out
213
+
214
+
215
+ @router.get("/me")
216
+ def me(session: Session = Depends(require_session)):
217
+ return {"user": _public_user(session.user), "tenant": session.tenant}
api/routes_customers.py CHANGED
@@ -1,346 +1,346 @@
1
- """routes_customers.py — X2's read + write of the customer table, BU-SCOPED (EXIT-3b / EXIT-2a).
2
-
3
- Two things happen here that did not happen in the pre-wave `main.py`:
4
-
5
- 1. **ROWS ARE SCOPED TO THE SESSION.** The old `/api/customers` served `cl.pool()` — the whole
6
- book, to anyone holding the shared APP_PASSWORD. Now the pool is built with
7
- `(team_id, agent_name)` derived from the USER RECORD, so a Royal-only user never receives a
8
- Fisch row and an agent-linked login never receives another rep's book. The scope is applied
9
- at the QUERY, not as a post-filter, so there is no moment at which the other BU's rows exist
10
- in this response.
11
-
12
- 2. **THE OVERLAY FORK IS GONE.** `aios-web/api/data/overlay.json` was a SECOND writable home
13
- for the same user-owned fields the Streamlit app keeps in the tenant store — two truths, and
14
- whichever process you asked last was right. Reads and writes now both go through
15
- `modules.customer_data`'s table-workspace functions: ONE store (C1c, ARCHITECTURE §1a rule 3).
16
-
17
- ⚠ ONE STORE IS NOT YET ONE CACHE (strangler-period, booked honestly). `core.store.get()` is
18
- cache-first per PROCESS, so a write from the Streamlit container is invisible to a running API
19
- container until its cache is refreshed, and vice versa. Deleting the fork removes the second
20
- SOURCE OF TRUTH; it does not make the two runtimes coherent. The fix is X4/Postgres (task C-4,
21
- owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtimes — a TestClient
22
- proof is single-process and would report green on exactly the thing that is still broken.
23
- """
24
- import time
25
-
26
- from fastapi import APIRouter, Body, Depends
27
-
28
- import scope_cache
29
- from deps import Session, err, module_gate, perms
30
-
31
- router = APIRouter(prefix="/api/v1")
32
-
33
- #: The surface these routes serve. Both legs are gated on it, so a user without the grant gets a
34
- #: 403 rather than an empty table that looks like "you have no customers".
35
- MODULE = "customer_data"
36
-
37
- _CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
38
-
39
-
40
- def _pool_rows(session: Session):
41
- """The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that
42
- may be shared between users.
43
-
44
- ⛔ WHAT MAY BE CACHED HERE, AND WHY THE LINE IS EXACTLY HERE. The cache key is
45
- `(team_id, agent)` and the cached value is the raw pool: Odoo-source columns only. That is
46
- genuinely scope-shaped — two users with the same BU and the same book are asking the same
47
- question, and `cl.pool()` is expensive (an Odoo pull plus reconciliation).
48
-
49
- The FULL PAYLOAD is NOT cacheable on this key, and caching it here was a real defect I shipped
50
- and then removed. `fields` comes from `fields_from_workspace(ws)` — that user's own `custom_`
51
- and `measure_` columns — and every overlay cell comes from `ws['overlays']`; the table
52
- workspace is read as `data[username]` and written as `patch_table_overlay(uname, …)`, i.e.
53
- PER USER. So two Royal-only users with no agent link share `(6, None)` and the second one
54
- would have been served the FIRST one's private notes and private columns. The scope key was
55
- right for the pool and wrong for everything wrapped around it.
56
-
57
- It survived a green 129-check battery because every fixture user had a DISTINCT
58
- `(team_id, agent)` pair, so no two of them ever collided on the key — the test set could not
59
- express the bug. `verify_api.py` now carries a same-scope second user for exactly this.
60
- """
61
- rt = session.runtime
62
- team_id, agent = _team_agent(session)
63
- return _pool_for(rt, team_id, agent)
64
-
65
-
66
- def _pool_for(rt, team_id, agent):
67
- """The cached pool for an explicit scope — session-free so the prewarm thread and the
68
- stale-refresh path can call it. STALE-WHILE-REFRESH (scope_cache): once a copy exists no
69
- request blocks on the 10–30s Odoo rebuild again; only a scope's FIRST-ever build does.
70
-
71
- DEBT-2 (2026-08-04): while the tenant's RESOLVED Odoo source is PAUSED this path never
72
- goes live — it serves the in-process copy at any age, else the persisted pause-time
73
- snapshot (restart-safe), else answers 503. Never a WIDER scope's snapshot: handing a
74
- scoped user the consolidated rows would widen their book, which is worse than an error."""
75
- import modules.customer_data as cl
76
- import routes_keychain
77
-
78
- key = ("pool", team_id, agent)
79
-
80
- if routes_keychain.odoo_paused(rt):
81
- hit = rt.pool_cache.get(key)
82
- if hit:
83
- return hit[1]
84
- snap = routes_keychain.load_pool_snapshot(rt, team_id, agent)
85
- if snap is not None:
86
- rt.pool_cache[key] = snap # seed, so the memo stamps stay coherent
87
- return snap[1]
88
- raise err(503, "connector_paused",
89
- "this data source is paused and no snapshot exists for your scope — "
90
- "an admin can resume it under Settings → Connectors")
91
-
92
- def _build():
93
- # ⚠ THE SCOPE GOES INTO THE BUILDER. `pool(agent_name, team_id)` is the same reconciled
94
- # builder the Streamlit page uses — passing the scope here is what makes the isolation a
95
- # property of the QUERY instead of a filter someone can forget to apply downstream.
96
- return cl.pool(agent, team_id)
97
-
98
- def _evict():
99
- # Bounded: a scope cache that only ever grows is a memory leak in a shared process.
100
- if len(rt.pool_cache) > 16:
101
- for stale in sorted(rt.pool_cache, key=lambda k: rt.pool_cache[k][0])[:8]:
102
- rt.pool_cache.pop(stale, None)
103
-
104
- return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, _build, _evict)
105
-
106
-
107
- def warm_default(rt):
108
- """Boot prewarm: the consolidated pool `(None, None)` — the scope every admin and every
109
- all-BU account lands on. Called from main.py's prewarm thread only."""
110
- _pool_for(rt, None, None)
111
-
112
-
113
- def _team_agent(session: Session):
114
- """The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by
115
- `_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree.
116
-
117
- ⛔ WAVE 15 R1: THIS NOW COMES FROM THE PERMANENT FILTER (C-PERM amendment 3). `team_id` is
118
- not a row filter — `customer_data._pool_build` passes it into `cust._cust_rev` three times
119
- and into `_cadence_bulk`, so it decides what `rev`/`ly`/`ltm`/`aov`/`status` MEAN. Enforcing
120
- a BU purely as a post-filter would keep the row list right and silently consolidate every
121
- number. So the pushdown survives as a DERIVATION OF the declared filter rather than a second
122
- wall beside it, and `perm_scope.derive_pool_scope` falls back to the legacy `bus`/`agent`
123
- derivation for any record the migration has not reached yet.
124
- """
125
- import core.perm_scope as perm_scope
126
- return perm_scope.derive_pool_scope(session.user, MODULE)
127
-
128
-
129
- def _pool_stamp(rt, team_id, agent):
130
- """The cached pool's build timestamp — the DATA STAMP in every measure-memo key, so a pool
131
- refresh invalidates the memoised answers exactly when the underlying rows changed."""
132
- entry = rt.pool_cache.get(("pool", team_id, agent))
133
- return entry[0] if isinstance(entry, tuple) and entry else 0
134
-
135
-
136
- def _measure_err(tag, e):
137
- try:
138
- import harness.telemetry as _tel
139
- _tel.error(f"api:{tag}", e)
140
- except Exception:
141
- pass
142
-
143
-
144
- def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
145
- consume_corrections: bool = True):
146
- """ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
147
- events route (2026-07-31 — the standalone measure gap, owner item 1).
148
-
149
- What it adds over the pre-wave hand-rolled `_payload` loop, and why it replaced it:
150
-
151
- * rows go through `aios_grid.rows_from_pool` — the SAME builder the embedded host uses,
152
- so `lat`/`lon` (the Map view's data), `_created` and every derived column ride each row
153
- by construction instead of by a second loop that drifts. The hand loop was written to
154
- mirror the pre-Map contract and silently dropped the coordinates: the shell's Map view
155
- had nothing to plot ("the Map no longer works").
156
- * `derived` carries the cohort column's cells AND the measure columns' values, resolved
157
- through `core.measure_resolve` (EXIT-5's extraction of the `_cl_measure_*` family).
158
- Without them every measure column the owner built rendered BLANK in the shell.
159
- * `measures` (the offer) and `measure_sets` (condition answers) are computed here so the
160
- events route can finally validate measure fields/conditions instead of refusing them
161
- (an empty `measure_offer` made `clean_measure_field` reject every create over HTTP).
162
-
163
- Memos live on the TENANT RUNTIME (`rt.measure_memo` / `rt.mset_memo`) — bounded by the
164
- module's own clear-past-cap rule, keyed on (stamp, scope, pool identity, question), nothing
165
- user-shaped in them.
166
- """
167
- import aios_grid
168
- from core import grid_events, measure_resolve
169
-
170
- import core.perm_scope as perm_scope
171
-
172
- rt = session.runtime
173
- team_id, agent = _team_agent(session)
174
- rows_src = _pool_for(rt, team_id, agent)
175
- # ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that
176
- # frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so
177
- # scoping here means a row this account may not see never enters ANY of them, rather than
178
- # being filtered out of one payload and surviving in another.
179
- #
180
- # Evaluated against the CANONICAL field list, not the per-user assembled one, for two
181
- # reasons: the assembled list is not built yet (it needs `pids`), and a permanent filter may
182
- # only ever name a canonical field anyway — `routes_admin._clean_perms` validates it against
183
- # exactly this schema and 400s otherwise. `permits()` denies on anything it cannot answer.
184
- rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS)
185
- pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
186
- ws = grid_events.table_workspace(
187
- _ctx_for(session, pids), allowed_pids=pids,
188
- consume_corrections=consume_corrections)
189
- workspace, fields, views, lists = aios_grid.workspace_wire(
190
- ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key)
191
- # THE FIELD WALL — a TRANSITIVE closure (C-PERM amendment 5), so hiding a field also hides
192
- # every formula computed FROM it. Formulas evaluate in the browser from `{ref}`s, so
193
- # shipping a dependent formula while withholding its input either leaks the input through
194
- # the formula's value or silently computes a wrong one; only removing both is coherent.
195
- # Applied AFTER workspace_wire because custom + measure columns are what it must cover.
196
- hidden = perm_scope.hidden_keys(session.user, MODULE, fields)
197
- if hidden:
198
- fields = [f for f in fields if f.get("key") not in hidden]
199
- # The field LIST and the ROW payload are two different wires. Narrowing only the first
200
- # would leave the value sitting in the second, where anything can read it.
201
- rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src]
202
-
203
- today = time.strftime("%Y-%m-%d")
204
- stamp = _pool_stamp(rt, team_id, agent)
205
- measures = measure_resolve.offer(team_id, on_error=_measure_err)
206
- measure_sets = measure_resolve.condition_sets(
207
- [v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp,
208
- rt.mset_memo, on_error=_measure_err)
209
- # The derived channel: cohort membership cells + measure column values, ONE dict — the
210
- # same read-only channel the embed host hands to rows_from_pool.
211
- derived = aios_grid.cohort_cells(lists)
212
- for pid, cells in measure_resolve.column_values(
213
- fields, team_id, pids, today, stamp, rt.measure_memo,
214
- on_error=_measure_err).items():
215
- derived.setdefault(pid, {}).update(cells)
216
-
217
- return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
218
- "fields": fields, "views": views, "lists": lists, "derived": derived,
219
- "measures": measures, "measure_sets": measure_sets, "today": today,
220
- "team_id": team_id}
221
-
222
-
223
- def _payload(session: Session):
224
- """`{fields, rows, today, pulled_at}` — X2's shape, which `verify_fields_contract.py`
225
- referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by
226
- construction); see `grid_assembly` for what that fixed.
227
-
228
- ⚠ `rows_src` is the SHARED cached list — `rows_from_pool` reads it and builds NEW dicts,
229
- never mutating a cached row (the same-scope-second-user leak rule).
230
- """
231
- import aios_grid
232
-
233
- g = grid_assembly(session)
234
- rows = aios_grid.rows_from_pool(
235
- g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"])
236
- return {"fields": g["fields"], "rows": rows,
237
- # `today` rides the payload because every relative date condition must resolve against
238
- # the TENANT's day, never the browser's — a client that falls back to its own clock
239
- # disagrees with the server for everyone west of it.
240
- "today": g["today"],
241
- "pulled_at": time.strftime("%Y-%m-%d %H:%M")}
242
-
243
-
244
- def _ctx_for(session: Session, pids):
245
- """An EventCtx for the READ path — no fallback workspace, so a store outage is a 503 rather
246
- than a phantom in-memory workspace an API request cannot persist."""
247
- from core import grid_events
248
- return grid_events.EventCtx(
249
- uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[],
250
- # C-PERM: the write wall's field half. Computed from the CANONICAL contract because
251
- # `fields=[]` here — the closure only needs the schema, not this user's column list.
252
- hidden_keys=_hidden_for(session),
253
- admin=session.admin, fallback_ws=None, seen_ids={})
254
-
255
-
256
-
257
- def _hidden_for(session: Session):
258
- """The fields this session's permissions hide — the write wall's half of C-PERM.
259
-
260
- Read paths strip these from both wires so they cannot be SEEN; this is what stops them
261
- being WRITTEN by a caller who knows the key. Evaluated against the canonical contract, the
262
- same schema `routes_admin._clean_perms` validates a hiddenFields entry against.
263
- """
264
- import aios_grid
265
- import core.perm_scope as perm_scope
266
-
267
- return perm_scope.hidden_keys(session.user, MODULE, aios_grid.FIELDS)
268
-
269
- def allowed_pids(session: Session):
270
- """The pids this session may touch — the POOL's own ids, so the write wall and the read scope
271
- can never disagree.
272
-
273
- Reads `_pool_rows` rather than `_payload`: the wall only needs identities, and going through
274
- the full payload would pay for a workspace read and a row assembly on every write.
275
-
276
- ⛔ THE PERMANENT FILTER APPLIES HERE TOO, AND FORGETTING IT IS A WRITE-WITHOUT-READ HOLE.
277
- `_pool_rows` is built with the DERIVED pushdown, which expresses only what a `(team_id,
278
- agent)` pair can express. Any part of the wall the pushdown cannot carry — `revenue > 1000`,
279
- a nested group, a condition on any other column — leaves the pool WIDER than the filter. Read
280
- paths close that gap with `apply_row_scope`; without the same call here the write wall would
281
- be the wider set, and a restricted user could PATCH a row this API will not show them.
282
- Same function, same order as `grid_assembly`, so the two walls cannot drift.
283
- """
284
- import core.perm_scope as perm_scope
285
- import aios_grid
286
-
287
- rows = perm_scope.apply_row_scope(_pool_rows(session), session.user, MODULE,
288
- aios_grid.FIELDS)
289
- return frozenset(r["pid"] for r in rows if r.get("pid") is not None)
290
-
291
-
292
- @router.get("/customers")
293
- def customers(session: Session = Depends(module_gate(MODULE))):
294
- return _payload(session)
295
-
296
-
297
- @router.patch("/customers/{pid}")
298
- def patch_customer(pid: int, body: dict = Body(default=None),
299
- session: Session = Depends(module_gate(MODULE))):
300
- """Write the EDITABLE overlay stratum only — Odoo stays read-only, forever.
301
-
302
- Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing
303
- the store directly: that handler is where the per-key `permissions.edit` wall, the pid wall
304
- and the truncation rules live, and a second implementation of those would be a second set of
305
- them to keep in step. The response reports what was ACCEPTED, which is not always what was
306
- asked for.
307
- """
308
- from core import grid_events
309
-
310
- updates = dict(body or {})
311
- if not updates:
312
- raise err(400, "empty_patch", "no fields to update")
313
- pool = allowed_pids(session)
314
- if pid not in pool:
315
- # 403, not 404: the pid may well exist — it is simply not in this session's book, and
316
- # saying "no such customer" would confirm the opposite to anyone who guessed right.
317
- raise err(403, "out_of_scope", "that customer is not in your book")
318
- payload = _payload(session)
319
- ctx = grid_events.EventCtx(
320
- uname=session.uname, allowed_pids=pool, fields=payload["fields"],
321
- admin=session.admin, fallback_ws=None, seen_ids={},
322
- hidden_keys=_hidden_for(session))
323
- try:
324
- grid_events.handle_one(
325
- {"id": f"patch:{pid}:{time.time_ns()}", "type": "overlay_patch",
326
- "pid": pid, "updates": updates}, ctx)
327
- except grid_events.StoreUnavailable:
328
- raise err(503, "store_unavailable",
329
- "the tenant store is unavailable — your change was not saved")
330
-
331
- # What actually landed, read back from the store rather than echoed from the request: a
332
- # refused key or a truncated value must not be reported as accepted.
333
- stored = (grid_events.table_workspace(_ctx_for(session, pool), allowed_pids=None)
334
- .get("overlays") or {}).get(str(pid)) or {}
335
- accepted = {k: stored.get(k) for k in updates if k in stored}
336
- refused = sorted(k for k in updates if k not in accepted or stored.get(k) != str(updates[k]))
337
- # No cache to patch: the overlay stratum is re-read from the store on every `_payload`, so
338
- # read-your-writes within this runtime is a property of the design rather than of a
339
- # write-through step somebody has to remember. (It was a write-through step while the whole
340
- # payload was cached on a scope key — the arrangement that leaked one user's notes to
341
- # another. See `_pool_rows`.) Cross-RUNTIME coherence is still not claimed: the module
342
- # docstring says why, and Postgres is the fix.
343
- out = {"ok": True, "pid": pid, "updates": accepted}
344
- if refused:
345
- out["refused"] = refused
346
- return out
 
1
+ """routes_customers.py — X2's read + write of the customer table, BU-SCOPED (EXIT-3b / EXIT-2a).
2
+
3
+ Two things happen here that did not happen in the pre-wave `main.py`:
4
+
5
+ 1. **ROWS ARE SCOPED TO THE SESSION.** The old `/api/customers` served `cl.pool()` — the whole
6
+ book, to anyone holding the shared APP_PASSWORD. Now the pool is built with
7
+ `(team_id, agent_name)` derived from the USER RECORD, so a Royal-only user never receives a
8
+ Fisch row and an agent-linked login never receives another rep's book. The scope is applied
9
+ at the QUERY, not as a post-filter, so there is no moment at which the other BU's rows exist
10
+ in this response.
11
+
12
+ 2. **THE OVERLAY FORK IS GONE.** `aios-web/api/data/overlay.json` was a SECOND writable home
13
+ for the same user-owned fields the Streamlit app keeps in the tenant store — two truths, and
14
+ whichever process you asked last was right. Reads and writes now both go through
15
+ `modules.customer_data`'s table-workspace functions: ONE store (C1c, ARCHITECTURE §1a rule 3).
16
+
17
+ ⚠ ONE STORE IS NOT YET ONE CACHE (strangler-period, booked honestly). `core.store.get()` is
18
+ cache-first per PROCESS, so a write from the Streamlit container is invisible to a running API
19
+ container until its cache is refreshed, and vice versa. Deleting the fork removes the second
20
+ SOURCE OF TRUTH; it does not make the two runtimes coherent. The fix is X4/Postgres (task C-4,
21
+ owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtimes — a TestClient
22
+ proof is single-process and would report green on exactly the thing that is still broken.
23
+ """
24
+ import time
25
+
26
+ from fastapi import APIRouter, Body, Depends
27
+
28
+ import scope_cache
29
+ from deps import Session, err, module_gate, perms
30
+
31
+ router = APIRouter(prefix="/api/v1")
32
+
33
+ #: The surface these routes serve. Both legs are gated on it, so a user without the grant gets a
34
+ #: 403 rather than an empty table that looks like "you have no customers".
35
+ MODULE = "customer_data"
36
+
37
+ _CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
38
+
39
+
40
+ def _pool_rows(session: Session):
41
+ """The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that
42
+ may be shared between users.
43
+
44
+ ⛔ WHAT MAY BE CACHED HERE, AND WHY THE LINE IS EXACTLY HERE. The cache key is
45
+ `(team_id, agent)` and the cached value is the raw pool: Odoo-source columns only. That is
46
+ genuinely scope-shaped — two users with the same BU and the same book are asking the same
47
+ question, and `cl.pool()` is expensive (an Odoo pull plus reconciliation).
48
+
49
+ The FULL PAYLOAD is NOT cacheable on this key, and caching it here was a real defect I shipped
50
+ and then removed. `fields` comes from `fields_from_workspace(ws)` — that user's own `custom_`
51
+ and `measure_` columns — and every overlay cell comes from `ws['overlays']`; the table
52
+ workspace is read as `data[username]` and written as `patch_table_overlay(uname, …)`, i.e.
53
+ PER USER. So two Royal-only users with no agent link share `(6, None)` and the second one
54
+ would have been served the FIRST one's private notes and private columns. The scope key was
55
+ right for the pool and wrong for everything wrapped around it.
56
+
57
+ It survived a green 129-check battery because every fixture user had a DISTINCT
58
+ `(team_id, agent)` pair, so no two of them ever collided on the key — the test set could not
59
+ express the bug. `verify_api.py` now carries a same-scope second user for exactly this.
60
+ """
61
+ rt = session.runtime
62
+ team_id, agent = _team_agent(session)
63
+ return _pool_for(rt, team_id, agent)
64
+
65
+
66
+ def _pool_for(rt, team_id, agent):
67
+ """The cached pool for an explicit scope — session-free so the prewarm thread and the
68
+ stale-refresh path can call it. STALE-WHILE-REFRESH (scope_cache): once a copy exists no
69
+ request blocks on the 10–30s Odoo rebuild again; only a scope's FIRST-ever build does.
70
+
71
+ DEBT-2 (2026-08-04): while the tenant's RESOLVED Odoo source is PAUSED this path never
72
+ goes live — it serves the in-process copy at any age, else the persisted pause-time
73
+ snapshot (restart-safe), else answers 503. Never a WIDER scope's snapshot: handing a
74
+ scoped user the consolidated rows would widen their book, which is worse than an error."""
75
+ import modules.customer_data as cl
76
+ import routes_keychain
77
+
78
+ key = ("pool", team_id, agent)
79
+
80
+ if routes_keychain.odoo_paused(rt):
81
+ hit = rt.pool_cache.get(key)
82
+ if hit:
83
+ return hit[1]
84
+ snap = routes_keychain.load_pool_snapshot(rt, team_id, agent)
85
+ if snap is not None:
86
+ rt.pool_cache[key] = snap # seed, so the memo stamps stay coherent
87
+ return snap[1]
88
+ raise err(503, "connector_paused",
89
+ "this data source is paused and no snapshot exists for your scope — "
90
+ "an admin can resume it under Settings → Connectors")
91
+
92
+ def _build():
93
+ # ⚠ THE SCOPE GOES INTO THE BUILDER. `pool(agent_name, team_id)` is the same reconciled
94
+ # builder the Streamlit page uses — passing the scope here is what makes the isolation a
95
+ # property of the QUERY instead of a filter someone can forget to apply downstream.
96
+ return cl.pool(agent, team_id)
97
+
98
+ def _evict():
99
+ # Bounded: a scope cache that only ever grows is a memory leak in a shared process.
100
+ if len(rt.pool_cache) > 16:
101
+ for stale in sorted(rt.pool_cache, key=lambda k: rt.pool_cache[k][0])[:8]:
102
+ rt.pool_cache.pop(stale, None)
103
+
104
+ return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, _build, _evict)
105
+
106
+
107
+ def warm_default(rt):
108
+ """Boot prewarm: the consolidated pool `(None, None)` — the scope every admin and every
109
+ all-BU account lands on. Called from main.py's prewarm thread only."""
110
+ _pool_for(rt, None, None)
111
+
112
+
113
+ def _team_agent(session: Session):
114
+ """The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by
115
+ `_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree.
116
+
117
+ ⛔ WAVE 15 R1: THIS NOW COMES FROM THE PERMANENT FILTER (C-PERM amendment 3). `team_id` is
118
+ not a row filter — `customer_data._pool_build` passes it into `cust._cust_rev` three times
119
+ and into `_cadence_bulk`, so it decides what `rev`/`ly`/`ltm`/`aov`/`status` MEAN. Enforcing
120
+ a BU purely as a post-filter would keep the row list right and silently consolidate every
121
+ number. So the pushdown survives as a DERIVATION OF the declared filter rather than a second
122
+ wall beside it, and `perm_scope.derive_pool_scope` falls back to the legacy `bus`/`agent`
123
+ derivation for any record the migration has not reached yet.
124
+ """
125
+ import core.perm_scope as perm_scope
126
+ return perm_scope.derive_pool_scope(session.user, MODULE)
127
+
128
+
129
+ def _pool_stamp(rt, team_id, agent):
130
+ """The cached pool's build timestamp — the DATA STAMP in every measure-memo key, so a pool
131
+ refresh invalidates the memoised answers exactly when the underlying rows changed."""
132
+ entry = rt.pool_cache.get(("pool", team_id, agent))
133
+ return entry[0] if isinstance(entry, tuple) and entry else 0
134
+
135
+
136
+ def _measure_err(tag, e):
137
+ try:
138
+ import harness.telemetry as _tel
139
+ _tel.error(f"api:{tag}", e)
140
+ except Exception:
141
+ pass
142
+
143
+
144
+ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
145
+ consume_corrections: bool = True):
146
+ """ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
147
+ events route (2026-07-31 — the standalone measure gap, owner item 1).
148
+
149
+ What it adds over the pre-wave hand-rolled `_payload` loop, and why it replaced it:
150
+
151
+ * rows go through `aios_grid.rows_from_pool` — the SAME builder the embedded host uses,
152
+ so `lat`/`lon` (the Map view's data), `_created` and every derived column ride each row
153
+ by construction instead of by a second loop that drifts. The hand loop was written to
154
+ mirror the pre-Map contract and silently dropped the coordinates: the shell's Map view
155
+ had nothing to plot ("the Map no longer works").
156
+ * `derived` carries the cohort column's cells AND the measure columns' values, resolved
157
+ through `core.measure_resolve` (EXIT-5's extraction of the `_cl_measure_*` family).
158
+ Without them every measure column the owner built rendered BLANK in the shell.
159
+ * `measures` (the offer) and `measure_sets` (condition answers) are computed here so the
160
+ events route can finally validate measure fields/conditions instead of refusing them
161
+ (an empty `measure_offer` made `clean_measure_field` reject every create over HTTP).
162
+
163
+ Memos live on the TENANT RUNTIME (`rt.measure_memo` / `rt.mset_memo`) — bounded by the
164
+ module's own clear-past-cap rule, keyed on (stamp, scope, pool identity, question), nothing
165
+ user-shaped in them.
166
+ """
167
+ import aios_grid
168
+ from core import grid_events, measure_resolve
169
+
170
+ import core.perm_scope as perm_scope
171
+
172
+ rt = session.runtime
173
+ team_id, agent = _team_agent(session)
174
+ rows_src = _pool_for(rt, team_id, agent)
175
+ # ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that
176
+ # frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so
177
+ # scoping here means a row this account may not see never enters ANY of them, rather than
178
+ # being filtered out of one payload and surviving in another.
179
+ #
180
+ # Evaluated against the CANONICAL field list, not the per-user assembled one, for two
181
+ # reasons: the assembled list is not built yet (it needs `pids`), and a permanent filter may
182
+ # only ever name a canonical field anyway — `routes_admin._clean_perms` validates it against
183
+ # exactly this schema and 400s otherwise. `permits()` denies on anything it cannot answer.
184
+ rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS)
185
+ pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
186
+ ws = grid_events.table_workspace(
187
+ _ctx_for(session, pids), allowed_pids=pids,
188
+ consume_corrections=consume_corrections)
189
+ workspace, fields, views, lists = aios_grid.workspace_wire(
190
+ ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key)
191
+ # THE FIELD WALL — a TRANSITIVE closure (C-PERM amendment 5), so hiding a field also hides
192
+ # every formula computed FROM it. Formulas evaluate in the browser from `{ref}`s, so
193
+ # shipping a dependent formula while withholding its input either leaks the input through
194
+ # the formula's value or silently computes a wrong one; only removing both is coherent.
195
+ # Applied AFTER workspace_wire because custom + measure columns are what it must cover.
196
+ hidden = perm_scope.hidden_keys(session.user, MODULE, fields)
197
+ if hidden:
198
+ fields = [f for f in fields if f.get("key") not in hidden]
199
+ # The field LIST and the ROW payload are two different wires. Narrowing only the first
200
+ # would leave the value sitting in the second, where anything can read it.
201
+ rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src]
202
+
203
+ today = time.strftime("%Y-%m-%d")
204
+ stamp = _pool_stamp(rt, team_id, agent)
205
+ measures = measure_resolve.offer(team_id, on_error=_measure_err)
206
+ measure_sets = measure_resolve.condition_sets(
207
+ [v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp,
208
+ rt.mset_memo, on_error=_measure_err)
209
+ # The derived channel: cohort membership cells + measure column values, ONE dict — the
210
+ # same read-only channel the embed host hands to rows_from_pool.
211
+ derived = aios_grid.cohort_cells(lists)
212
+ for pid, cells in measure_resolve.column_values(
213
+ fields, team_id, pids, today, stamp, rt.measure_memo,
214
+ on_error=_measure_err).items():
215
+ derived.setdefault(pid, {}).update(cells)
216
+
217
+ return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
218
+ "fields": fields, "views": views, "lists": lists, "derived": derived,
219
+ "measures": measures, "measure_sets": measure_sets, "today": today,
220
+ "team_id": team_id}
221
+
222
+
223
+ def _payload(session: Session):
224
+ """`{fields, rows, today, pulled_at}` — X2's shape, which `verify_fields_contract.py`
225
+ referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by
226
+ construction); see `grid_assembly` for what that fixed.
227
+
228
+ ⚠ `rows_src` is the SHARED cached list — `rows_from_pool` reads it and builds NEW dicts,
229
+ never mutating a cached row (the same-scope-second-user leak rule).
230
+ """
231
+ import aios_grid
232
+
233
+ g = grid_assembly(session)
234
+ rows = aios_grid.rows_from_pool(
235
+ g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"])
236
+ return {"fields": g["fields"], "rows": rows,
237
+ # `today` rides the payload because every relative date condition must resolve against
238
+ # the TENANT's day, never the browser's — a client that falls back to its own clock
239
+ # disagrees with the server for everyone west of it.
240
+ "today": g["today"],
241
+ "pulled_at": time.strftime("%Y-%m-%d %H:%M")}
242
+
243
+
244
+ def _ctx_for(session: Session, pids):
245
+ """An EventCtx for the READ path — no fallback workspace, so a store outage is a 503 rather
246
+ than a phantom in-memory workspace an API request cannot persist."""
247
+ from core import grid_events
248
+ return grid_events.EventCtx(
249
+ uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[],
250
+ # C-PERM: the write wall's field half. Computed from the CANONICAL contract because
251
+ # `fields=[]` here — the closure only needs the schema, not this user's column list.
252
+ hidden_keys=_hidden_for(session),
253
+ admin=session.admin, fallback_ws=None, seen_ids={})
254
+
255
+
256
+
257
+ def _hidden_for(session: Session):
258
+ """The fields this session's permissions hide — the write wall's half of C-PERM.
259
+
260
+ Read paths strip these from both wires so they cannot be SEEN; this is what stops them
261
+ being WRITTEN by a caller who knows the key. Evaluated against the canonical contract, the
262
+ same schema `routes_admin._clean_perms` validates a hiddenFields entry against.
263
+ """
264
+ import aios_grid
265
+ import core.perm_scope as perm_scope
266
+
267
+ return perm_scope.hidden_keys(session.user, MODULE, aios_grid.FIELDS)
268
+
269
+ def allowed_pids(session: Session):
270
+ """The pids this session may touch — the POOL's own ids, so the write wall and the read scope
271
+ can never disagree.
272
+
273
+ Reads `_pool_rows` rather than `_payload`: the wall only needs identities, and going through
274
+ the full payload would pay for a workspace read and a row assembly on every write.
275
+
276
+ ⛔ THE PERMANENT FILTER APPLIES HERE TOO, AND FORGETTING IT IS A WRITE-WITHOUT-READ HOLE.
277
+ `_pool_rows` is built with the DERIVED pushdown, which expresses only what a `(team_id,
278
+ agent)` pair can express. Any part of the wall the pushdown cannot carry — `revenue > 1000`,
279
+ a nested group, a condition on any other column — leaves the pool WIDER than the filter. Read
280
+ paths close that gap with `apply_row_scope`; without the same call here the write wall would
281
+ be the wider set, and a restricted user could PATCH a row this API will not show them.
282
+ Same function, same order as `grid_assembly`, so the two walls cannot drift.
283
+ """
284
+ import core.perm_scope as perm_scope
285
+ import aios_grid
286
+
287
+ rows = perm_scope.apply_row_scope(_pool_rows(session), session.user, MODULE,
288
+ aios_grid.FIELDS)
289
+ return frozenset(r["pid"] for r in rows if r.get("pid") is not None)
290
+
291
+
292
+ @router.get("/customers")
293
+ def customers(session: Session = Depends(module_gate(MODULE))):
294
+ return _payload(session)
295
+
296
+
297
+ @router.patch("/customers/{pid}")
298
+ def patch_customer(pid: int, body: dict = Body(default=None),
299
+ session: Session = Depends(module_gate(MODULE))):
300
+ """Write the EDITABLE overlay stratum only — Odoo stays read-only, forever.
301
+
302
+ Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing
303
+ the store directly: that handler is where the per-key `permissions.edit` wall, the pid wall
304
+ and the truncation rules live, and a second implementation of those would be a second set of
305
+ them to keep in step. The response reports what was ACCEPTED, which is not always what was
306
+ asked for.
307
+ """
308
+ from core import grid_events
309
+
310
+ updates = dict(body or {})
311
+ if not updates:
312
+ raise err(400, "empty_patch", "no fields to update")
313
+ pool = allowed_pids(session)
314
+ if pid not in pool:
315
+ # 403, not 404: the pid may well exist — it is simply not in this session's book, and
316
+ # saying "no such customer" would confirm the opposite to anyone who guessed right.
317
+ raise err(403, "out_of_scope", "that customer is not in your book")
318
+ payload = _payload(session)
319
+ ctx = grid_events.EventCtx(
320
+ uname=session.uname, allowed_pids=pool, fields=payload["fields"],
321
+ admin=session.admin, fallback_ws=None, seen_ids={},
322
+ hidden_keys=_hidden_for(session))
323
+ try:
324
+ grid_events.handle_one(
325
+ {"id": f"patch:{pid}:{time.time_ns()}", "type": "overlay_patch",
326
+ "pid": pid, "updates": updates}, ctx)
327
+ except grid_events.StoreUnavailable:
328
+ raise err(503, "store_unavailable",
329
+ "the tenant store is unavailable — your change was not saved")
330
+
331
+ # What actually landed, read back from the store rather than echoed from the request: a
332
+ # refused key or a truncated value must not be reported as accepted.
333
+ stored = (grid_events.table_workspace(_ctx_for(session, pool), allowed_pids=None)
334
+ .get("overlays") or {}).get(str(pid)) or {}
335
+ accepted = {k: stored.get(k) for k in updates if k in stored}
336
+ refused = sorted(k for k in updates if k not in accepted or stored.get(k) != str(updates[k]))
337
+ # No cache to patch: the overlay stratum is re-read from the store on every `_payload`, so
338
+ # read-your-writes within this runtime is a property of the design rather than of a
339
+ # write-through step somebody has to remember. (It was a write-through step while the whole
340
+ # payload was cached on a scope key — the arrangement that leaked one user's notes to
341
+ # another. See `_pool_rows`.) Cross-RUNTIME coherence is still not claimed: the module
342
+ # docstring says why, and Postgres is the fix.
343
+ out = {"ok": True, "pid": pid, "updates": accepted}
344
+ if refused:
345
+ out["refused"] = refused
346
+ return out
api/routes_grid.py CHANGED
@@ -1,832 +1,832 @@
1
- """routes_grid.py — X2's write seam over HTTP: the SECOND adapter on `core.grid_events` (EXIT-1c).
2
-
3
- `POST /api/v1/grid/events` takes the component's event objects VERBATIM — the same objects the
4
- Streamlit host receives through the component value slot, unchanged — and runs them through the
5
- same `core.grid_events.handle_events` the Streamlit adapter runs. That is the whole point of
6
- EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated
7
- anything here would be a second wall to keep in step, and the two would drift on the first change.
8
-
9
- DEDUP IS PER REQUEST, and that is a deliberate limit, not an oversight. The Streamlit adapter's
10
- `seen_ids` lives in `st.session_state` — genuinely per user session — because the client resends
11
- its recent 24-event window on every emit inside one page session. A stateless API has no such
12
- dict, and inventing a per-user server-side one would be exactly the resident per-tenant state
13
- EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole
14
- purpose — a batch that repeats an id processes it once), and a genuinely replayed request is
15
- handled by the operations being idempotent. The one event where a replay is observable is
16
- `add_to_list` (it would add the same pids to the same cohort twice — a set union, so the
17
- membership is unchanged, but the toast count repeats). Noted rather than papered over; a
18
- server-side idempotency key belongs with D2's session mirror (C-2).
19
-
20
- STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises
21
- `StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read
22
- back. A 200 over a write that evaporated is the failure this rule exists to prevent.
23
- """
24
- import datetime as dt
25
- import time
26
-
27
- from fastapi import APIRouter, Body, Depends
28
-
29
- from deps import Session, err, module_gate, perms, require_session
30
-
31
- router = APIRouter(prefix="/api/v1")
32
-
33
- MODULE = "customer_data"
34
-
35
- #: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a
36
- #: legitimate client — refuse it rather than doing 500 store writes on one request.
37
- _MAX_EVENTS = 24
38
-
39
-
40
- def _ctx(session: Session, fields, pids, **kw):
41
- from core import grid_events
42
- import aios_grid
43
- import core.perm_scope as perm_scope
44
-
45
- # Wave 16 C-TOPIC: the ctx is TOPIC-SHAPED. The product scope swaps all three of the
46
- # things a write is validated against — the module the permission wall reads, the canonical
47
- # contract the hidden-field closure runs over, and the TABLE OPS the write lands in.
48
- # Getting any one of them from the other topic is the "validated against the wrong field
49
- # contract" near-miss the wave-15 routes_products header warned about.
50
- scope_key = str(kw.get("scope_key") or "")
51
- if scope_key == "product":
52
- import modules.product_data as pd
53
- from routes_products import MODULE as _PMOD, pd_fields
54
-
55
- module, canonical = _PMOD, pd_fields(consolidated=True)
56
- kw.setdefault("table", pd.TABLE_OPS)
57
- hidden = perm_scope.hidden_keys(session.user, module, canonical)
58
- elif scope_key.startswith("ut_"):
59
- # Wave 18 C3-UT: a user table has no module in the permission wall (its wall is
60
- # `user_tables.may_open`, already applied by the assembly this route ran first), so
61
- # the hidden-field closure is EMPTY rather than borrowed from another topic's contract.
62
- import core.table_store as table_store
63
-
64
- kw.setdefault("table",
65
- table_store.make(f"{scope_key}_table_workspace", st=session.runtime))
66
- hidden = frozenset()
67
- else:
68
- module, canonical = MODULE, aios_grid.FIELDS
69
- hidden = perm_scope.hidden_keys(session.user, module, canonical)
70
- return grid_events.EventCtx(
71
- uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin,
72
- # C-PERM: the write wall's field half, on the EVENTS transport too. This route takes the
73
- # component's event objects verbatim, so a hidden key would otherwise arrive here with
74
- # nothing between it and the store.
75
- hidden_keys=hidden,
76
- # ⭐ WAVE 25 (R6b, closes D-16) — THE TENANT HANDLE, on EVERY scope. This is the line
77
- # D-16's exit condition names, and without it the rest of the fix is inert: the handler
78
- # falls back to the module-global `core.store`, which is tenant #0's repo, so a Nurilab
79
- # user's documents, cohorts and — through `_tops` — their whole customer/product table
80
- # workspace were written into Royal Imports' dataset. Note it is set for the CUSTOMER
81
- # and PRODUCT branches too, not only `ut_`: those two never passed a scoped `table`, so
82
- # they were the ones actually resolving to tenant #0 on every request.
83
- st=session.runtime,
84
- fallback_ws=None, seen_ids={}, **kw)
85
-
86
-
87
- #: The surfaces the ONE grid serves. `cohort` is the same table over hand-curated SETS rather
88
- #: than over the whole scoped pool — `app.py:page_cohort` calls the same `_table_grid` with
89
- #: `cohort_mode=True, scope_key='cohort'`. `product` (wave 16 C-TOPIC) is the SKU table:
90
- #: same engine, its own pool, field contract and workspace BUCKET (routes_products).
91
- _SCOPES = ("customer", "cohort", "product")
92
-
93
-
94
- def _scope_or_400(raw):
95
- """⛔ REFUSE AN UNKNOWN SCOPE, never default it. A typo silently served as `customer` would
96
- hand a user the whole book on a page they opened to see one cohort — the widening direction,
97
- which is the one that must fail closed.
98
-
99
- Wave 18 (C3-UT): a `ut_`-prefixed scope names a USER TABLE and passes through here —
100
- existence and the per-table wall are enforced by `routes_tables.ut_assembly` (404/403),
101
- which every consumer of such a scope goes through. Passing an unknown ut key therefore
102
- still fails closed, just one layer down where the store can actually be consulted."""
103
- scope = (raw or "customer").strip().lower()
104
- if scope.startswith("ut_"):
105
- return scope
106
- if scope not in _SCOPES:
107
- raise err(400, "bad_scope",
108
- f"scope must be one of {', '.join(_SCOPES)} — refusing to guess")
109
- return scope
110
-
111
-
112
- @router.get("/workspace")
113
- def workspace(scope: str = "customer",
114
- session: Session = Depends(require_session)):
115
- """The durable table workspace for this session: views, fields, overlays, folders.
116
-
117
- ⛔ WAVE 21 (C4): the wall is TOPIC-SHAPED, so the DEPENDENCY is session-only and each scope
118
- asserts its own gate below. The old `module_gate("customer_data")` dependency 403'd a
119
- `ut_*` workspace for any tenant whose catalogue omits the customer module (loopable/
120
- nurilab/gtmlab ship modules w/o it) — and the client then fell back to the shared
121
- localStorage bucket + demo data, which is exactly the "new database shows RI's customer
122
- fields" defect. A user table's real wall is `user_tables.may_open`, enforced inside
123
- `ut_assembly` (404/403, one layer down where the store can be consulted).
124
-
125
- ⚠ WHY THIS EXISTS (S1↔S2, 2026-07-30 — an X2 AMENDMENT, see the split doc). X2 fixes
126
- `/customers` at the exact shape `verify_fields_contract.py` referees, so the workspace cannot
127
- ride along in it without forking the thing that gate keeps single-sourced. Without a
128
- workspace route the standalone shell's write path would be WRITE-ONLY: events persist
129
- server-side and nothing reads them back on reload, so a saved view looks lost to the user
130
- even though the store has it. This route closes that read-back gap at its own URL.
131
-
132
- `allowed_pids` is passed so a SHARED view's `memberPids` are re-scoped to THIS reader — the
133
- wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a
134
- full-access user.
135
- """
136
- from core import grid_events
137
- from routes_customers import grid_assembly
138
-
139
- scope = _scope_or_400(scope)
140
- # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need
141
- # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a
142
- # session (its wall is the table's own, inside the assembly).
143
- if not (scope == "product" or scope.startswith("ut_")):
144
- session.require(MODULE)
145
- # ── Wave 18 C3-UT: a USER TABLE'S workspace — the third topic through the one wire. The
146
- # storage key has no (bu, agent) because a user table has no Odoo scope; per-user by the
147
- # table's own wall (creator or admin, `user_tables.may_open`).
148
- if scope.startswith("ut_"):
149
- from routes_tables import ut_assembly
150
-
151
- storage_key = f"{session.tenant}:{scope}:{session.uname}"
152
- try:
153
- g = ut_assembly(session, scope, storage_key=storage_key)
154
- except grid_events.StoreUnavailable:
155
- raise err(503, "store_unavailable", "the tenant store is unavailable")
156
- workspace = g["workspace"]
157
- workspace["overlays"] = g["ws"].get("overlays") or {}
158
- # The field contract rides the cheap re-read on EVERY topic — see the customer branch
159
- # below for why (a user table's first cohort changes its contract too: `_cohorts(ctx)`
160
- # is scope-parameterized, so `ut_` surfaces have their own sets).
161
- workspace["fields"] = g["fields"]
162
- workspace["measures"] = g["measures"]
163
- workspace["measureSets"] = g["measure_sets"]
164
- # ⚠ `g["derived"]`, NOT `{}` (2026-08-04). R9 gave every topic its own cohort sets, and
165
- # `ut_assembly` has been building this topic's cohort CELLS since — but this route threw
166
- # them away, so the Locked-views column arrived on the cheap re-read with permanently
167
- # empty values. A column that exists and can never have one is worse than an absent
168
- # column: it reads as "this record is in no locked view", which is a claim.
169
- workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()}
170
- workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
171
- try:
172
- from core import users as _users
173
- workspace["userOptions"] = _users.assignable_people(tenant=session.tenant)
174
- workspace["userAvatars"] = {k: v for k, v in
175
- _users.avatar_map(tenant=session.tenant).items()
176
- if str(v).startswith("data:image/")}
177
- except Exception:
178
- workspace["userOptions"] = []
179
- workspace["userAvatars"] = {}
180
- workspace["scopeKey"] = scope
181
- return {"workspace": workspace}
182
-
183
- # ── Wave 16 C-TOPIC: the PRODUCT surface has its own assembly (own pool, own field
184
- # contract, own workspace BUCKET) and the contract's own storage key. It branches before
185
- # the customer derivation because the customer storage key embeds (bu, agent) while the
186
- # product one is deliberately `all:all` — one product workspace per user (the wave doc's
187
- # C-TOPIC line), since BU narrows the product VALUES, not which workspace you own.
188
- if scope == "product":
189
- from routes_products import MODULE as PRODUCT_MODULE, product_assembly
190
-
191
- # ⛔ THE GATE CHANGES WITH THE SCOPE (wave 21 C4: the dependency is session-only now,
192
- # so this line IS the product wall — not a second one on top of customer_data).
193
- session.require(PRODUCT_MODULE)
194
- storage_key = f"{session.tenant}:product-list:{session.uname}:all:all"
195
- try:
196
- g = product_assembly(session, scope=scope, storage_key=storage_key)
197
- except grid_events.StoreUnavailable:
198
- raise err(503, "store_unavailable", "the tenant store is unavailable")
199
- workspace = g["workspace"]
200
- workspace["overlays"] = g["ws"].get("overlays") or {}
201
- # R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own
202
- # first-cohort contract change. Same reason as the customer branch below.
203
- workspace["fields"] = g["fields"]
204
- # The measure channel is EMPTY on this topic (customer-grain descope) — stated
205
- # explicitly so the client's pickers grey rather than guess.
206
- workspace["measures"] = g["measures"]
207
- workspace["measureSets"] = g["measure_sets"]
208
- # `g["derived"]` — the same correction as the user-table branch above, for the same
209
- # reason: `product_assembly` builds this topic's cohort cells and this route discarded
210
- # them. The measure half stays empty on this topic by descope, which is a different
211
- # statement and one the offer already makes.
212
- workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()}
213
- workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
214
- try:
215
- from core import users as _users
216
- workspace["userOptions"] = _users.assignable_people(tenant=session.tenant)
217
- workspace["userAvatars"] = {k: v for k, v in
218
- _users.avatar_map(tenant=session.tenant).items()
219
- if str(v).startswith("data:image/")}
220
- except Exception:
221
- workspace["userOptions"] = []
222
- workspace["userAvatars"] = {}
223
- workspace["scopeKey"] = scope
224
- return {"workspace": workspace}
225
-
226
- # The storage key mirrors the host's own convention byte-for-byte (app.py's `_table_grid`
227
- # call sites) with the session's TENANT in the host's hardcoded slot, so localStorage state
228
- # carries across embed ⇄ standalone on the same browser.
229
- # Wave 15 R1 — the SAME derivation the pool uses (`routes_customers._team_agent`), so the
230
- # storage key cannot drift from the scope it names. Value-identical for every migrated
231
- # record (verify_perm_scope section D proves the derivation reproduces the legacy scope
232
- # exactly), so nobody's saved localStorage state moves when the migration runs.
233
- import core.perm_scope as perm_scope
234
- team_id, agent = perm_scope.derive_pool_scope(session.user, MODULE)
235
- bu = team_id if team_id is not None else "all"
236
- if scope == "cohort":
237
- storage_key = f"{session.tenant}:cohort:{session.uname}:{bu}"
238
- else:
239
- storage_key = f"{session.tenant}:customer-list:{session.uname}:{bu}:{agent or 'all'}"
240
-
241
- # ⛔ THE WIRE SHAPE IS THE SHARED PROJECTION (`aios_grid.workspace_wire`), not the store
242
- # shape. The first version returned the store dict with no `storageKey` — and the client
243
- # validator requires one, so the standalone shell DISCARDED the whole workspace: saved views
244
- # never rendered, `cohortMode` never arrived, and the Cohort route silently drew the Customer
245
- # surface. One projection for both servers is the fix that cannot drift.
246
- try:
247
- g = grid_assembly(session, scope=scope, storage_key=storage_key)
248
- except grid_events.StoreUnavailable:
249
- raise err(503, "store_unavailable", "the tenant store is unavailable")
250
- workspace = g["workspace"]
251
- # ADDITIVE to the wire: the caller's own overlay cells. The grid reads overlays from the
252
- # /customers rows, but this route is the cheap read-back a write can be verified against
253
- # (verify_api's overlay probe) without paying the pool call. Per-user by construction —
254
- # `table_workspace` is this session's workspace.
255
- workspace["overlays"] = g["ws"].get("overlays") or {}
256
- # ⭐ THE FIELD CONTRACT, 2026-08-04 — and it is NOT decoration.
257
- #
258
- # `fields_from_workspace(ws, cohorts=bool(cohort_lists))` appends the derived "Locked
259
- # views" column ONLY when the caller owns at least one cohort. So a user's FIRST cohort
260
- # CHANGES THE FIELD CONTRACT — and the rows call that used to be the only carrier of
261
- # `fields` is deliberately never re-fetched on a write (it is a 15-minute-cached Odoo
262
- # pull; this route is the cheap re-read). The client therefore had no way to learn about
263
- # that column short of a remount, which is the second half of the owner's "it only shows
264
- # up when I switch modules and come back".
265
- #
266
- # ⚠ `g["fields"]` is the SAME list `_payload` serves on `/customers` — same assembly,
267
- # same permission wall (`hidden_keys` already applied) — so the two wires cannot disagree
268
- # about what a column is. Taking it from anywhere else would fork the contract that
269
- # verify_fields_contract.py exists to keep single-sourced.
270
- workspace["fields"] = g["fields"]
271
- # ── the STANDALONE measure channel (owner item 1, 2026-07-31) ────────────────────────────
272
- # The embed receives these as top-level render args; standalone lifts them off THIS route
273
- # (useCustomerData merges them into the payload slots the grid already reads). They ride
274
- # the workspace rather than /customers because a durable write re-reads exactly this route
275
- # (WORKSPACE_STALE), so a new measure column populates without refetching the heavy pool.
276
- workspace["measures"] = g["measures"]
277
- workspace["measureSets"] = g["measure_sets"]
278
- # Derived cells (cohort column + measure columns), keyed by pid. JSON object keys are
279
- # strings; the client indexes with String(pid).
280
- workspace["derived"] = {str(pid): cells for pid, cells in g["derived"].items()}
281
- # Who is looking (permissions verdicts) + the assignee choices for `user`-typed columns —
282
- # the two other host-only render args the shell was missing (fail-closed without them:
283
- # restricted fields uneditable, assignee picker empty).
284
- workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
285
- try:
286
- from core import users as _users
287
- workspace["userOptions"] = _users.assignable_people(tenant=session.tenant)
288
- # Wave 14 C-AVATAR — the options vocabulary's companion: display name -> data URL.
289
- # Absent entries fall back to the client's initials disc; a non-data: value is
290
- # dropped (defence in depth beside the write-side wall in routes_auth).
291
- workspace["userAvatars"] = {k: v for k, v in
292
- _users.avatar_map(tenant=session.tenant).items()
293
- if str(v).startswith("data:image/")}
294
- except Exception:
295
- workspace["userOptions"] = []
296
- workspace["userAvatars"] = {}
297
-
298
- # THE SURFACE STAMP — a MIRROR of the host's own (`_table_grid`'s hide/cohort stamps).
299
- # ⚠ Deliberately NOT `hideViews`: `cohortMode` is what swaps the Views panel for the cohort
300
- # list, and two switches for one behaviour would drift.
301
- workspace["scopeKey"] = scope
302
- if scope == "cohort":
303
- workspace["cohortMode"] = True
304
- # The create pane offers "this cohort table only" (default) vs "all customer tables".
305
- workspace["scopeChoice"] = True
306
- return {"workspace": workspace}
307
-
308
-
309
- # ─────────────────────────────────────────────── the TIME-SERIES channel (C-TS, 2026-08-02)
310
- _TS_BUCKETS = ("week", "month", "quarter", "year")
311
- _TS_MAX_BUCKETS = 120
312
- _TS_MAX_FIELDS = 12
313
- _TS_MAX_PIDS = 5000
314
- _TS_MAX_LAST_N = 120
315
- #: C-TSWIN (wave 14): sliding windows cost ONE aggregate query per (metric, bucket) — the
316
- #: price of "bucket N == the grid cell at bucket N's end". The product cap keeps a legal
317
- #: 120-bucket × 12-metric ask from becoming 1,440 queries in one request; typical panels
318
- #: (12 × 3) sit two orders of magnitude under it. Dated amendment in the split doc.
319
- _TS_MAX_CELLS = 720
320
-
321
-
322
- def _ts_start_of(bucket, d):
323
- """The calendar START of the bucket holding `d` (week = Monday, the vocabulary rule)."""
324
- if bucket == "week":
325
- return d - dt.timedelta(days=d.weekday())
326
- if bucket == "month":
327
- return d.replace(day=1)
328
- if bucket == "quarter":
329
- return d.replace(month=((d.month - 1) // 3) * 3 + 1, day=1)
330
- return d.replace(month=1, day=1)
331
-
332
-
333
- def _ts_next(bucket, d):
334
- if bucket == "week":
335
- return d + dt.timedelta(days=7)
336
- if bucket == "year":
337
- return d.replace(year=d.year + 1)
338
- step = 1 if bucket == "month" else 3
339
- m = d.month + step
340
- return dt.date(d.year + (m - 1) // 12, (m - 1) % 12 + 1, 1)
341
-
342
-
343
- def _ts_prev(bucket, d):
344
- if bucket == "week":
345
- return d - dt.timedelta(days=7)
346
- if bucket == "year":
347
- return d.replace(year=d.year - 1)
348
- step = 1 if bucket == "month" else 3
349
- y, m = d.year, d.month - step
350
- while m < 1:
351
- m += 12
352
- y -= 1
353
- return dt.date(y, m, 1)
354
-
355
-
356
- def _ts_label(bucket, start):
357
- if bucket == "week":
358
- return f"Wk of {start.strftime('%b %d')}"
359
- if bucket == "month":
360
- return start.strftime("%b %Y")
361
- if bucket == "quarter":
362
- return f"Q{(start.month - 1) // 3 + 1} {start.year}"
363
- return str(start.year)
364
-
365
-
366
- @router.post("/grid/timeseries")
367
- def grid_timeseries(body: dict = Body(default=None), scope: str = "customer",
368
- session: Session = Depends(module_gate(MODULE))):
369
- """Pooled measure values per calendar bucket — the time-series view's data channel.
370
-
371
- C-TSWIN (wave 14, ruling R1): AS-OF semantics. Each metric's OWN stored window is
372
- re-resolved per bucket with `today := min(bucket end, real today)` — `ytd` is cumulative
373
- from Jan 1, `last_90_days` trailing, `all_time` cumulative ever; bucket N equals what the
374
- grid's measure column would show if today were bucket N's end. Fixed-range (`custom`)
375
- windows cannot slide and are dropped per field as `window_fixed`. Rows carry
376
- `window: {kind, label}` so a cumulative row cannot be misread as periodic, and there is
377
- NO `total` column — sliding windows overlap, so a sum of columns would double-count.
378
-
379
- The client sends the pids its view currently matches (the filter IS the scope); the server
380
- intersects them with the session's book, so the request can only ever NARROW. Values are
381
- POOLED aggregates computed by the semantic layer's own expression (the additivity law: an
382
- average is computed at pool grain, never averaged over per-customer answers). A bucket
383
- with no rows is 0 for a sum/count and null for anything else; a bucket that has not
384
- STARTED yet is null for every kind — an unstarted period's YTD is unanswered, not 0.
385
- """
386
- from core import measure_resolve
387
- from harness import windows as _wn
388
- from routes_customers import _pool_stamp, _team_agent, allowed_pids
389
-
390
- # Wave 16 C-TOPIC: the channel is CUSTOMER-GRAIN (measure_resolve's own grain), so the
391
- # product surface is refused in words rather than answered with an id-space accident —
392
- # product pids are CRC32 hashes and would mostly fall outside the customer book anyway,
393
- # but "mostly" is not a wall. The client hides the mode for this topic; this is the
394
- # server's half of the same refusal.
395
- _sc = _scope_or_400(scope)
396
- if _sc == "product" or _sc.startswith("ut_"):
397
- raise err(400, "bad_timeseries",
398
- "this surface has no time-series channel — measures are customer-grain")
399
-
400
- body = body or {}
401
- bucket = body.get("bucket")
402
- if bucket not in _TS_BUCKETS:
403
- raise err(400, "bad_timeseries", "bucket must be one of week, month, quarter, year")
404
- raw_fields = body.get("fields")
405
- if not isinstance(raw_fields, list) or not raw_fields:
406
- raise err(400, "bad_timeseries", "fields must be a non-empty list of field keys")
407
- if len(raw_fields) > _TS_MAX_FIELDS:
408
- raise err(400, "bad_timeseries", f"at most {_TS_MAX_FIELDS} fields per request")
409
- raw_pids = body.get("pids")
410
- if not isinstance(raw_pids, list) or not raw_pids:
411
- raise err(400, "bad_timeseries", "pids must be a non-empty list")
412
- if len(raw_pids) > _TS_MAX_PIDS:
413
- raise err(400, "bad_timeseries", f"at most {_TS_MAX_PIDS} pids per request")
414
- try:
415
- wanted = {int(p) for p in raw_pids}
416
- except (TypeError, ValueError):
417
- raise err(400, "bad_timeseries", "pids must be integers")
418
- pool = wanted & {int(p) for p in allowed_pids(session)}
419
- if not pool:
420
- # 403, not an empty 200: the caller asked about customers outside their book, and an
421
- # all-zero series over nobody would read as "no activity", not "not yours".
422
- raise err(403, "out_of_scope", "none of those customers are in your book")
423
-
424
- span = body.get("span")
425
- if not isinstance(span, dict):
426
- raise err(400, "bad_timeseries", "span must be {'lastN': n} or {'from': .., 'to': ..}")
427
- today = time.strftime("%Y-%m-%d")
428
- t = dt.date.fromisoformat(today)
429
- n = span.get("lastN")
430
- starts = []
431
- if n is not None:
432
- if not isinstance(n, int) or isinstance(n, bool) or not (1 <= n <= _TS_MAX_LAST_N):
433
- raise err(400, "bad_timeseries", f"lastN must be 1..{_TS_MAX_LAST_N}")
434
- cur = _ts_start_of(bucket, t)
435
- starts = [cur]
436
- for _ in range(n - 1):
437
- cur = _ts_prev(bucket, cur)
438
- starts.append(cur)
439
- starts.reverse()
440
- else:
441
- try:
442
- d_from = dt.date.fromisoformat(str(span.get("from")))
443
- d_to = dt.date.fromisoformat(str(span.get("to")))
444
- except (TypeError, ValueError):
445
- raise err(400, "bad_timeseries", "span.from/to must be ISO dates (YYYY-MM-DD)")
446
- if d_from > d_to:
447
- d_from, d_to = d_to, d_from
448
- cur = _ts_start_of(bucket, d_from)
449
- while cur <= d_to:
450
- starts.append(cur)
451
- if len(starts) > _TS_MAX_BUCKETS:
452
- raise err(400, "bad_timeseries",
453
- f"that span is more than {_TS_MAX_BUCKETS} {bucket} buckets - "
454
- f"narrow it")
455
- cur = _ts_next(bucket, cur)
456
- if not starts:
457
- raise err(400, "bad_timeseries", "the span holds no buckets")
458
- ends = [_ts_next(bucket, s) - dt.timedelta(days=1) for s in starts]
459
-
460
- rt = session.runtime
461
- if not rt.available():
462
- raise err(503, "store_unavailable", "the tenant store is unavailable")
463
- import modules.customer_data as cl_mod
464
- # Read-only route: it must not consume a one-shot label-correction ack the browser has
465
- # not seen yet (the same rule event validation follows).
466
- ws = cl_mod.table_workspace(session.uname, consume_corrections=False)
467
- fdefs = ws.get("fields") or {}
468
- keys, dropped, seen = [], [], set()
469
- for k in raw_fields:
470
- k = str(k or "")
471
- if not k or k in seen:
472
- continue
473
- seen.add(k)
474
- fd = fdefs.get(k)
475
- if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict):
476
- # Named, never silent: v1 eligibility is measure-backed fields only (C-TS).
477
- dropped.append({"field": k, "reason": "not_a_measure_field"})
478
- continue
479
- keys.append(k)
480
- # ⚡ WAVE 17 R9 (amendment 2026-08-03, GRID's cross-fence ask) — A SHEET WITH NO MEASURE
481
- # FIELDS IS NO LONGER A 400. It answers with the BUCKET GRID and no rows.
482
- #
483
- # Why this is the honest direction and not a loosening: the bucket starts and their labels
484
- # are SERVER math (`_ts_start_of` / `_ts_next` / `_ts_label`), and R9 has the client
485
- # synthesizing snapshot rows for preset columns that have no window to slide. Those rows
486
- # must be painted under the SAME headings as everything else, so the client needs the
487
- # columns even when the server has no series to put in them. Refusing the whole request
488
- # meant the panel could offer nothing at all on this tenant — the shipped contract has zero
489
- # measure-backed presets (`aios_grid.py`: "this branch is currently MEMBERLESS"), which is
490
- # what item 5 is actually about.
491
- #
492
- # Nothing is invented by this: `rows` is empty and `meta.dropped` NAMES every field and why.
493
- # A 400 is still returned for a request that is malformed (bad bucket, bad span, no fields
494
- # at all) — this only stops treating "I asked about columns you cannot serve" as an error.
495
- mfields, slid_keys = [], []
496
- for k in keys:
497
- m = dict(fdefs[k]["measure"])
498
- if (m.get("window") or {}).get("kind") == "custom":
499
- # C-TSWIN: a fixed date range cannot slide across buckets — named, never silent,
500
- # the same posture as not_a_measure_field.
501
- dropped.append({"field": k, "reason": "window_fixed"})
502
- continue
503
- mfields.append({"key": k, "measure": m})
504
- slid_keys.append(k)
505
- # Same amendment as above: every metric being fixed-range is a sheet with no SERIES, not a
506
- # broken request. The columns still stand, and `window_fixed` still names each refusal.
507
- if len(starts) * len(mfields) > _TS_MAX_CELLS:
508
- raise err(400, "bad_timeseries",
509
- "that ask is too wide - narrow the span or pick fewer metrics")
510
-
511
- team_id, agent = _team_agent(session)
512
- stamp = _pool_stamp(rt, team_id, agent)
513
- problems = []
514
- bucket_bounds = [(s.isoformat(), e.isoformat()) for s, e in zip(starts, ends)]
515
- # No slidable metrics = nothing to resolve. Skipping the call rather than asking the
516
- # resolver about an empty list keeps the memo free of a meaningless key.
517
- answers = measure_resolve.series_values(
518
- mfields, bucket, bucket_bounds,
519
- team_id, frozenset(pool), today, stamp, rt.series_memo,
520
- on_error=lambda tag, e: problems.append(str(e)[:200])) if mfields else {}
521
-
522
- columns = []
523
- for s, e in zip(starts, ends):
524
- col = {"key": s.isoformat(), "label": _ts_label(bucket, s),
525
- "from": s.isoformat(), "to": e.isoformat()}
526
- if e > t:
527
- col["partial"] = True
528
- columns.append(col)
529
- rows = []
530
- for k in slid_keys:
531
- ans = answers.get(k)
532
- if ans is None:
533
- dropped.append({"field": k, "reason": "unresolvable"})
534
- continue
535
- vals_by = ans.get("values") or {}
536
- agg_kind = str(ans.get("agg") or "sum")
537
- zero_fill = agg_kind in ("sum", "count")
538
- vals = []
539
- for s in starts:
540
- if s > t:
541
- # C-TSWIN: an unstarted period is unanswered for EVERY agg kind — a future
542
- # month's YTD zero-filled to 0 would read as "the year reset".
543
- vals.append(None)
544
- else:
545
- vals.append(vals_by.get(s.isoformat(), 0 if zero_fill else None))
546
- wspec = (fdefs[k].get("measure") or {}).get("window")
547
- wnorm = _wn.normalize(wspec) or {}
548
- rows.append({"field": k, "label": str(fdefs[k].get("label") or k)[:120],
549
- "agg": agg_kind, "values": vals,
550
- "window": {"kind": str(wnorm.get("kind") or ""),
551
- "label": _wn.label(wspec)}})
552
- meta = {"pool": len(pool), "today": today, "bucket": bucket}
553
- if dropped:
554
- meta["dropped"] = dropped
555
- if problems:
556
- meta["problems"] = problems[:5]
557
- return {"columns": columns, "rows": rows, "meta": meta}
558
-
559
-
560
- #: C-CAL caps (wave 17, owner item 9 / ruling R4). A month of days, a handful of metrics.
561
- #: Every one of these is a 400 WITH ITS REASON, never a silent trim: a calendar that quietly
562
- #: answered 20 of the 31 days asked about would paint eleven blank cells that look like days
563
- #: with no activity.
564
- _CAL_MAX_GROUPS = 31
565
- _CAL_MAX_FIELDS = 6
566
- _CAL_MAX_CELLS = 186
567
-
568
-
569
- @router.post("/grid/calendar_metrics")
570
- def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "customer",
571
- session: Session = Depends(module_gate(MODULE))):
572
- """Per-DAY measure values with each metric's own window slid to that day (C-CAL / R4).
573
-
574
- ⛔ WHY THIS EXISTS AT ALL. The calendar's summary cells used to aggregate the row VALUES the
575
- grid already held — which for a measure column means "every member's YTD **as of today**",
576
- summed and printed under a date in March. The number was arithmetically fine and semantically
577
- a lie: it answered a question about today while sitting in a cell labelled with another day.
578
- R4: a metric in a day cell is computed AS OF THAT DAY.
579
-
580
- The shape differs from the time-series channel in exactly one way, and it is the reason this
581
- is a separate route rather than a parameter: **every group carries its OWN pid set**. A
582
- calendar day holds the records the date field placed there, so day-to-day the subject
583
- changes. One `allowed_pids` for the whole request — the TS channel's shape — would compute
584
- each day over everybody, which is a different question again.
585
-
586
- Static (non-measure) fields are NOT served here. They have no window to slide, so the
587
- client's own per-day aggregation over row values stays correct for them; sending them would
588
- invite a second implementation of arithmetic that already works.
589
- """
590
- from core import measure_resolve
591
- from harness import windows as _wn
592
- from routes_customers import _pool_stamp, _team_agent, allowed_pids
593
-
594
- # The same refusal the TS channel makes, for the same reason: measures are customer-grain
595
- # and product pids are CRC32 hashes of SKU codes. "Mostly outside the book" is not a wall.
596
- _sc_cal = _scope_or_400(scope)
597
- if _sc_cal == "product" or _sc_cal.startswith("ut_"):
598
- raise err(400, "bad_calendar_metrics",
599
- "the product surface has no measure channel yet — measures are customer-grain")
600
-
601
- body = body or {}
602
- raw_groups = body.get("groups")
603
- if not isinstance(raw_groups, list) or not raw_groups:
604
- raise err(400, "bad_calendar_metrics",
605
- "groups must be a non-empty list of {key: 'YYYY-MM-DD', pids: [...]}")
606
- if len(raw_groups) > _CAL_MAX_GROUPS:
607
- raise err(400, "bad_calendar_metrics",
608
- f"at most {_CAL_MAX_GROUPS} days per request (one month)")
609
- raw_fields = body.get("fields")
610
- if not isinstance(raw_fields, list) or not raw_fields:
611
- raise err(400, "bad_calendar_metrics", "fields must be a non-empty list of field keys")
612
- if len(raw_fields) > _CAL_MAX_FIELDS:
613
- raise err(400, "bad_calendar_metrics", f"at most {_CAL_MAX_FIELDS} metrics per request")
614
- if len(raw_groups) * len(raw_fields) > _CAL_MAX_CELLS:
615
- raise err(400, "bad_calendar_metrics",
616
- "that ask is too wide - fewer days or fewer metrics")
617
-
618
- book = {int(p) for p in allowed_pids(session)}
619
- groups, seen_days = [], set()
620
- for g in raw_groups:
621
- if not isinstance(g, dict):
622
- raise err(400, "bad_calendar_metrics", "every group must be an object")
623
- day = str(g.get("key") or "")
624
- try:
625
- d = dt.date.fromisoformat(day)
626
- except (TypeError, ValueError):
627
- raise err(400, "bad_calendar_metrics",
628
- "every group key must be an ISO date (YYYY-MM-DD)")
629
- if day in seen_days:
630
- raise err(400, "bad_calendar_metrics", f"day {day} appears twice")
631
- seen_days.add(day)
632
- raw_pids = g.get("pids")
633
- if not isinstance(raw_pids, list):
634
- raise err(400, "bad_calendar_metrics", "every group needs a pids list")
635
- try:
636
- wanted = {int(p) for p in raw_pids}
637
- except (TypeError, ValueError):
638
- raise err(400, "bad_calendar_metrics", "pids must be integers")
639
- # NARROW-ONLY, per group. The client sends what its calendar placed; the server can
640
- # only ever remove from that, never add.
641
- groups.append((day, d, frozenset(wanted & book)))
642
- if sum(len(p) for _, _, p in groups) > _TS_MAX_PIDS:
643
- raise err(400, "bad_calendar_metrics",
644
- f"at most {_TS_MAX_PIDS} customer references per request")
645
-
646
- rt = session.runtime
647
- if not rt.available():
648
- raise err(503, "store_unavailable", "the tenant store is unavailable")
649
- import modules.customer_data as cl_mod
650
- ws = cl_mod.table_workspace(session.uname, consume_corrections=False)
651
- fdefs = ws.get("fields") or {}
652
- keys, dropped, seen = [], [], set()
653
- for k in raw_fields:
654
- k = str(k or "")
655
- if not k or k in seen:
656
- continue
657
- seen.add(k)
658
- fd = fdefs.get(k)
659
- if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict):
660
- # The TS channel's vocabulary, deliberately reused rather than re-coined: the client
661
- # already knows how to say these three words to a reader.
662
- dropped.append({"field": k, "reason": "not_a_measure_field"})
663
- continue
664
- if ((fd["measure"].get("window") or {}).get("kind") == "custom"):
665
- dropped.append({"field": k, "reason": "window_fixed"})
666
- continue
667
- keys.append(k)
668
- if not keys:
669
- raise err(400, "bad_calendar_metrics",
670
- "none of the requested fields are measure fields with a window that can "
671
- "slide to a day")
672
-
673
- today = time.strftime("%Y-%m-%d")
674
- t = dt.date.fromisoformat(today)
675
- team_id, agent = _team_agent(session)
676
- stamp = _pool_stamp(rt, team_id, agent)
677
- problems = []
678
- values = {k: {} for k in keys}
679
- for day, d, pids in groups:
680
- # A day that has not happened is unanswered for EVERY aggregate kind — the unstarted
681
- # bucket law at day grain. Answering 0 would say "we sold nothing", which is a claim
682
- # about a day nobody has lived through yet.
683
- if d > t or not pids:
684
- for k in keys:
685
- values[k][day] = None
686
- continue
687
- answers = measure_resolve.series_values(
688
- [{"key": k, "measure": dict(fdefs[k]["measure"])} for k in keys],
689
- "day", [(day, day)], team_id, pids, today, stamp, rt.series_memo,
690
- on_error=lambda tag, e: problems.append(str(e)[:200]))
691
- for k in keys:
692
- ans = answers.get(k)
693
- if ans is None:
694
- values[k][day] = None
695
- continue
696
- vals_by = ans.get("values") or {}
697
- zero_fill = str(ans.get("agg") or "sum") in ("sum", "count")
698
- values[k][day] = vals_by.get(day, 0 if zero_fill else None)
699
-
700
- for k in keys:
701
- if all(v is None for v in values[k].values()):
702
- # Every day unanswerable is a FIELD-level failure, and saying so is the difference
703
- # between "no activity that month" and "this metric could not be computed".
704
- dropped.append({"field": k, "reason": "unresolvable"})
705
- out = {"values": values, "today": today,
706
- "windows": {k: {"kind": str((_wn.normalize(
707
- (fdefs[k].get("measure") or {}).get("window")) or {}).get("kind") or ""),
708
- "label": _wn.label((fdefs[k].get("measure") or {}).get("window"))}
709
- for k in keys}}
710
- if dropped:
711
- out["dropped"] = dropped
712
- if problems:
713
- out["problems"] = problems[:5]
714
- return out
715
-
716
-
717
- @router.post("/grid/events")
718
- def grid_events_route(body: dict = Body(default=None),
719
- session: Session = Depends(require_session)):
720
- """`{events: [<component event objects, verbatim>]}` → `{results, doc?, toast?}`.
721
-
722
- Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the
723
- same sessions the read door (`/workspace`) admits, or a tenant without the customer module
724
- can SEE its own user tables and not write to them."""
725
- from core import grid_events
726
- from routes_customers import grid_assembly
727
-
728
- events = (body or {}).get("events")
729
- if events is None and isinstance(body, dict) and body.get("type"):
730
- events = [body] # a single event object, the legacy shape
731
- if not isinstance(events, list):
732
- raise err(400, "bad_events", "expected {events: [...]}")
733
- if len(events) > _MAX_EVENTS:
734
- raise err(400, "too_many_events",
735
- f"at most {_MAX_EVENTS} events per request (the client's resend window)")
736
-
737
- # Validated with the SAME predicate as the read route. An unrecognised scopeKey used to be
738
- # passed through verbatim, and `core.grid_events` only ever compares it to 'cohort' — so a
739
- # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on
740
- # what a scope is, or the surface you read is not the surface you wrote.
741
- scope = _scope_or_400((body or {}).get("scopeKey"))
742
- # Wave 21 C4 — same per-scope gate as /workspace (read and write doors must agree).
743
- if not (scope == "product" or scope.startswith("ut_")):
744
- session.require(MODULE)
745
- # This assembly validates the write; it is not a payload the browser will render. Leave
746
- # one-shot field-name correction acks queued for the subsequent /workspace refresh.
747
- # Wave 16 C-TOPIC: the PRODUCT topic gets the product assembly — product field contract,
748
- # product pids, and (below) the product TABLE OPS, so a product event is validated against
749
- # and lands in the product bucket. The measure/cohort context is honestly EMPTY there:
750
- # `clean_measure_field` refuses measure creates on this surface by construction (the
751
- # customer-grain descope), which is the fail-closed shape, not an accident.
752
- if scope == "product":
753
- from routes_products import MODULE as PRODUCT_MODULE, product_assembly
754
-
755
- session.require(PRODUCT_MODULE) # the product wall (dependency is session-only, C4)
756
- g = product_assembly(session, consume_corrections=False)
757
- elif scope.startswith("ut_"):
758
- # Wave 18 C3-UT — the user-table wall (creator/admin) is inside the assembly; the
759
- # measure/cohort context is honestly EMPTY (customer-grain machinery, no meaning here).
760
- from routes_tables import ut_assembly
761
-
762
- g = ut_assembly(session, scope, consume_corrections=False)
763
- else:
764
- g = grid_assembly(session, scope=scope, consume_corrections=False)
765
- # ⚠ THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`,
766
- # `clean_measure_field` had an empty admission list and every measure-column create over
767
- # HTTP was silently refused; without `measure_keys`, `clean_filter_tree` stripped every
768
- # measure CONDITION out of a saved view. The embed always passed these; the API adapter
769
- # simply had not been given them — the standalone shell could read measures it could
770
- # never write.
771
- ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope,
772
- measure_keys=frozenset(m["key"] for m in g["measures"]),
773
- resolved_ids=frozenset(g["measure_sets"]),
774
- cohort_ids=frozenset(c["id"] for c in g["lists"]),
775
- measure_offer=tuple(g["measures"]),
776
- visible_views=tuple(g["views"]))
777
-
778
- # Per-event results so the client can tell which of a batch landed — the component's own
779
- # bridge has no response channel at all, so this is strictly more than the embed gets.
780
- results = []
781
- try:
782
- for one in events:
783
- eid = str(one.get("id") or "") if isinstance(one, dict) else ""
784
- rerender = grid_events.handle_one(one, ctx)
785
- results.append({"id": eid, "rerender": bool(rerender)})
786
- except grid_events.StoreUnavailable:
787
- raise err(503, "store_unavailable",
788
- "the tenant store is unavailable — none of your changes were saved")
789
-
790
- out = {"results": results, "rerender": any(r["rerender"] for r in results)}
791
- if ctx.out.doc is not None:
792
- out["doc"] = ctx.out.doc
793
- if ctx.out.toast is not None:
794
- out["toast"] = ctx.out.toast
795
-
796
- # ── ⭐ owner item 2 (2026-08-03): THE NEW MEASURE COLUMN'S VALUES, ONE ROUND TRIP SOONER ──
797
- #
798
- # Creating a measure column cost the browser TWO sequential trips before a single number
799
- # appeared: this one to persist the field, then a whole `/workspace` to compute it. The
800
- # second cannot start until the first lands (the resolver reads the PERSISTED field), so the
801
- # wait was structural, not slow code — the owner's "it takes some time for the data to
802
- # populate". The values are computed here instead, immediately after the write, and ride
803
- # this response.
804
- #
805
- # ⚠ IT COSTS NOTHING EXTRA TO COMPUTE. The expensive part is one DuckDB aggregate over the
806
- # book, and `rt.measure_memo` is keyed on (pool stamp, scope, pool, measure, window) — so
807
- # the `/workspace` re-read that still follows HITS the memo instead of doing this work. The
808
- # query happens once either way; only its position moved.
809
- #
810
- # ⚠ NARROW ON PURPOSE. Gated to an actual measure-column write, so an overlay edit or a
811
- # cohort add — the overwhelming majority of events — never pays for a second assembly.
812
- #
813
- # ⚠ AND IT IS A SHORTCUT, NOT A PATH. Any failure is swallowed: `WORKSPACE_STALE` still
814
- # fires from `rerender`, and the re-read still delivers these values exactly as it does
815
- # today. Nothing depends on this having worked.
816
- if out["rerender"] and scope in ("customer", "cohort") and any(
817
- isinstance(e, dict) and e.get("type") == "field_upsert"
818
- and str(((e.get("field") or {}) if isinstance(e.get("field"), dict) else {})
819
- .get("key") or "").startswith("measure_")
820
- for e in events):
821
- try:
822
- fresh = grid_assembly(session, scope=scope, consume_corrections=False)
823
- out["derived"] = {str(pid): cells for pid, cells in fresh["derived"].items()}
824
- except Exception:
825
- pass
826
- # ⚠ NOTHING IS INVALIDATED HERE, on purpose. The runtime cache holds ONLY the scope-shaped
827
- # Odoo pool (see `routes_customers._pool_rows`), and no event on this route can change an
828
- # Odoo column — Odoo is read-only. Everything an event DOES change (overlays, fields, views,
829
- # folders, cohorts) is re-read from the store on the next request. An earlier version cleared
830
- # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh
831
- # data that was never in it.
832
- return out
 
1
+ """routes_grid.py — X2's write seam over HTTP: the SECOND adapter on `core.grid_events` (EXIT-1c).
2
+
3
+ `POST /api/v1/grid/events` takes the component's event objects VERBATIM — the same objects the
4
+ Streamlit host receives through the component value slot, unchanged — and runs them through the
5
+ same `core.grid_events.handle_events` the Streamlit adapter runs. That is the whole point of
6
+ EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated
7
+ anything here would be a second wall to keep in step, and the two would drift on the first change.
8
+
9
+ DEDUP IS PER REQUEST, and that is a deliberate limit, not an oversight. The Streamlit adapter's
10
+ `seen_ids` lives in `st.session_state` — genuinely per user session — because the client resends
11
+ its recent 24-event window on every emit inside one page session. A stateless API has no such
12
+ dict, and inventing a per-user server-side one would be exactly the resident per-tenant state
13
+ EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole
14
+ purpose — a batch that repeats an id processes it once), and a genuinely replayed request is
15
+ handled by the operations being idempotent. The one event where a replay is observable is
16
+ `add_to_list` (it would add the same pids to the same cohort twice — a set union, so the
17
+ membership is unchanged, but the toast count repeats). Noted rather than papered over; a
18
+ server-side idempotency key belongs with D2's session mirror (C-2).
19
+
20
+ STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises
21
+ `StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read
22
+ back. A 200 over a write that evaporated is the failure this rule exists to prevent.
23
+ """
24
+ import datetime as dt
25
+ import time
26
+
27
+ from fastapi import APIRouter, Body, Depends
28
+
29
+ from deps import Session, err, module_gate, perms, require_session
30
+
31
+ router = APIRouter(prefix="/api/v1")
32
+
33
+ MODULE = "customer_data"
34
+
35
+ #: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a
36
+ #: legitimate client — refuse it rather than doing 500 store writes on one request.
37
+ _MAX_EVENTS = 24
38
+
39
+
40
+ def _ctx(session: Session, fields, pids, **kw):
41
+ from core import grid_events
42
+ import aios_grid
43
+ import core.perm_scope as perm_scope
44
+
45
+ # Wave 16 C-TOPIC: the ctx is TOPIC-SHAPED. The product scope swaps all three of the
46
+ # things a write is validated against — the module the permission wall reads, the canonical
47
+ # contract the hidden-field closure runs over, and the TABLE OPS the write lands in.
48
+ # Getting any one of them from the other topic is the "validated against the wrong field
49
+ # contract" near-miss the wave-15 routes_products header warned about.
50
+ scope_key = str(kw.get("scope_key") or "")
51
+ if scope_key == "product":
52
+ import modules.product_data as pd
53
+ from routes_products import MODULE as _PMOD, pd_fields
54
+
55
+ module, canonical = _PMOD, pd_fields(consolidated=True)
56
+ kw.setdefault("table", pd.TABLE_OPS)
57
+ hidden = perm_scope.hidden_keys(session.user, module, canonical)
58
+ elif scope_key.startswith("ut_"):
59
+ # Wave 18 C3-UT: a user table has no module in the permission wall (its wall is
60
+ # `user_tables.may_open`, already applied by the assembly this route ran first), so
61
+ # the hidden-field closure is EMPTY rather than borrowed from another topic's contract.
62
+ import core.table_store as table_store
63
+
64
+ kw.setdefault("table",
65
+ table_store.make(f"{scope_key}_table_workspace", st=session.runtime))
66
+ hidden = frozenset()
67
+ else:
68
+ module, canonical = MODULE, aios_grid.FIELDS
69
+ hidden = perm_scope.hidden_keys(session.user, module, canonical)
70
+ return grid_events.EventCtx(
71
+ uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin,
72
+ # C-PERM: the write wall's field half, on the EVENTS transport too. This route takes the
73
+ # component's event objects verbatim, so a hidden key would otherwise arrive here with
74
+ # nothing between it and the store.
75
+ hidden_keys=hidden,
76
+ # ⭐ WAVE 25 (R6b, closes D-16) — THE TENANT HANDLE, on EVERY scope. This is the line
77
+ # D-16's exit condition names, and without it the rest of the fix is inert: the handler
78
+ # falls back to the module-global `core.store`, which is tenant #0's repo, so a Nurilab
79
+ # user's documents, cohorts and — through `_tops` — their whole customer/product table
80
+ # workspace were written into Royal Imports' dataset. Note it is set for the CUSTOMER
81
+ # and PRODUCT branches too, not only `ut_`: those two never passed a scoped `table`, so
82
+ # they were the ones actually resolving to tenant #0 on every request.
83
+ st=session.runtime,
84
+ fallback_ws=None, seen_ids={}, **kw)
85
+
86
+
87
+ #: The surfaces the ONE grid serves. `cohort` is the same table over hand-curated SETS rather
88
+ #: than over the whole scoped pool — `app.py:page_cohort` calls the same `_table_grid` with
89
+ #: `cohort_mode=True, scope_key='cohort'`. `product` (wave 16 C-TOPIC) is the SKU table:
90
+ #: same engine, its own pool, field contract and workspace BUCKET (routes_products).
91
+ _SCOPES = ("customer", "cohort", "product")
92
+
93
+
94
+ def _scope_or_400(raw):
95
+ """⛔ REFUSE AN UNKNOWN SCOPE, never default it. A typo silently served as `customer` would
96
+ hand a user the whole book on a page they opened to see one cohort — the widening direction,
97
+ which is the one that must fail closed.
98
+
99
+ Wave 18 (C3-UT): a `ut_`-prefixed scope names a USER TABLE and passes through here —
100
+ existence and the per-table wall are enforced by `routes_tables.ut_assembly` (404/403),
101
+ which every consumer of such a scope goes through. Passing an unknown ut key therefore
102
+ still fails closed, just one layer down where the store can actually be consulted."""
103
+ scope = (raw or "customer").strip().lower()
104
+ if scope.startswith("ut_"):
105
+ return scope
106
+ if scope not in _SCOPES:
107
+ raise err(400, "bad_scope",
108
+ f"scope must be one of {', '.join(_SCOPES)} — refusing to guess")
109
+ return scope
110
+
111
+
112
+ @router.get("/workspace")
113
+ def workspace(scope: str = "customer",
114
+ session: Session = Depends(require_session)):
115
+ """The durable table workspace for this session: views, fields, overlays, folders.
116
+
117
+ ⛔ WAVE 21 (C4): the wall is TOPIC-SHAPED, so the DEPENDENCY is session-only and each scope
118
+ asserts its own gate below. The old `module_gate("customer_data")` dependency 403'd a
119
+ `ut_*` workspace for any tenant whose catalogue omits the customer module (loopable/
120
+ nurilab/gtmlab ship modules w/o it) — and the client then fell back to the shared
121
+ localStorage bucket + demo data, which is exactly the "new database shows RI's customer
122
+ fields" defect. A user table's real wall is `user_tables.may_open`, enforced inside
123
+ `ut_assembly` (404/403, one layer down where the store can be consulted).
124
+
125
+ ⚠ WHY THIS EXISTS (S1↔S2, 2026-07-30 — an X2 AMENDMENT, see the split doc). X2 fixes
126
+ `/customers` at the exact shape `verify_fields_contract.py` referees, so the workspace cannot
127
+ ride along in it without forking the thing that gate keeps single-sourced. Without a
128
+ workspace route the standalone shell's write path would be WRITE-ONLY: events persist
129
+ server-side and nothing reads them back on reload, so a saved view looks lost to the user
130
+ even though the store has it. This route closes that read-back gap at its own URL.
131
+
132
+ `allowed_pids` is passed so a SHARED view's `memberPids` are re-scoped to THIS reader — the
133
+ wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a
134
+ full-access user.
135
+ """
136
+ from core import grid_events
137
+ from routes_customers import grid_assembly
138
+
139
+ scope = _scope_or_400(scope)
140
+ # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need
141
+ # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a
142
+ # session (its wall is the table's own, inside the assembly).
143
+ if not (scope == "product" or scope.startswith("ut_")):
144
+ session.require(MODULE)
145
+ # ── Wave 18 C3-UT: a USER TABLE'S workspace — the third topic through the one wire. The
146
+ # storage key has no (bu, agent) because a user table has no Odoo scope; per-user by the
147
+ # table's own wall (creator or admin, `user_tables.may_open`).
148
+ if scope.startswith("ut_"):
149
+ from routes_tables import ut_assembly
150
+
151
+ storage_key = f"{session.tenant}:{scope}:{session.uname}"
152
+ try:
153
+ g = ut_assembly(session, scope, storage_key=storage_key)
154
+ except grid_events.StoreUnavailable:
155
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
156
+ workspace = g["workspace"]
157
+ workspace["overlays"] = g["ws"].get("overlays") or {}
158
+ # The field contract rides the cheap re-read on EVERY topic — see the customer branch
159
+ # below for why (a user table's first cohort changes its contract too: `_cohorts(ctx)`
160
+ # is scope-parameterized, so `ut_` surfaces have their own sets).
161
+ workspace["fields"] = g["fields"]
162
+ workspace["measures"] = g["measures"]
163
+ workspace["measureSets"] = g["measure_sets"]
164
+ # ⚠ `g["derived"]`, NOT `{}` (2026-08-04). R9 gave every topic its own cohort sets, and
165
+ # `ut_assembly` has been building this topic's cohort CELLS since — but this route threw
166
+ # them away, so the Locked-views column arrived on the cheap re-read with permanently
167
+ # empty values. A column that exists and can never have one is worse than an absent
168
+ # column: it reads as "this record is in no locked view", which is a claim.
169
+ workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()}
170
+ workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
171
+ try:
172
+ from core import users as _users
173
+ workspace["userOptions"] = _users.assignable_people(tenant=session.tenant)
174
+ workspace["userAvatars"] = {k: v for k, v in
175
+ _users.avatar_map(tenant=session.tenant).items()
176
+ if str(v).startswith("data:image/")}
177
+ except Exception:
178
+ workspace["userOptions"] = []
179
+ workspace["userAvatars"] = {}
180
+ workspace["scopeKey"] = scope
181
+ return {"workspace": workspace}
182
+
183
+ # ── Wave 16 C-TOPIC: the PRODUCT surface has its own assembly (own pool, own field
184
+ # contract, own workspace BUCKET) and the contract's own storage key. It branches before
185
+ # the customer derivation because the customer storage key embeds (bu, agent) while the
186
+ # product one is deliberately `all:all` — one product workspace per user (the wave doc's
187
+ # C-TOPIC line), since BU narrows the product VALUES, not which workspace you own.
188
+ if scope == "product":
189
+ from routes_products import MODULE as PRODUCT_MODULE, product_assembly
190
+
191
+ # ⛔ THE GATE CHANGES WITH THE SCOPE (wave 21 C4: the dependency is session-only now,
192
+ # so this line IS the product wall — not a second one on top of customer_data).
193
+ session.require(PRODUCT_MODULE)
194
+ storage_key = f"{session.tenant}:product-list:{session.uname}:all:all"
195
+ try:
196
+ g = product_assembly(session, scope=scope, storage_key=storage_key)
197
+ except grid_events.StoreUnavailable:
198
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
199
+ workspace = g["workspace"]
200
+ workspace["overlays"] = g["ws"].get("overlays") or {}
201
+ # R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own
202
+ # first-cohort contract change. Same reason as the customer branch below.
203
+ workspace["fields"] = g["fields"]
204
+ # The measure channel is EMPTY on this topic (customer-grain descope) — stated
205
+ # explicitly so the client's pickers grey rather than guess.
206
+ workspace["measures"] = g["measures"]
207
+ workspace["measureSets"] = g["measure_sets"]
208
+ # `g["derived"]` — the same correction as the user-table branch above, for the same
209
+ # reason: `product_assembly` builds this topic's cohort cells and this route discarded
210
+ # them. The measure half stays empty on this topic by descope, which is a different
211
+ # statement and one the offer already makes.
212
+ workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()}
213
+ workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
214
+ try:
215
+ from core import users as _users
216
+ workspace["userOptions"] = _users.assignable_people(tenant=session.tenant)
217
+ workspace["userAvatars"] = {k: v for k, v in
218
+ _users.avatar_map(tenant=session.tenant).items()
219
+ if str(v).startswith("data:image/")}
220
+ except Exception:
221
+ workspace["userOptions"] = []
222
+ workspace["userAvatars"] = {}
223
+ workspace["scopeKey"] = scope
224
+ return {"workspace": workspace}
225
+
226
+ # The storage key mirrors the host's own convention byte-for-byte (app.py's `_table_grid`
227
+ # call sites) with the session's TENANT in the host's hardcoded slot, so localStorage state
228
+ # carries across embed ⇄ standalone on the same browser.
229
+ # Wave 15 R1 — the SAME derivation the pool uses (`routes_customers._team_agent`), so the
230
+ # storage key cannot drift from the scope it names. Value-identical for every migrated
231
+ # record (verify_perm_scope section D proves the derivation reproduces the legacy scope
232
+ # exactly), so nobody's saved localStorage state moves when the migration runs.
233
+ import core.perm_scope as perm_scope
234
+ team_id, agent = perm_scope.derive_pool_scope(session.user, MODULE)
235
+ bu = team_id if team_id is not None else "all"
236
+ if scope == "cohort":
237
+ storage_key = f"{session.tenant}:cohort:{session.uname}:{bu}"
238
+ else:
239
+ storage_key = f"{session.tenant}:customer-list:{session.uname}:{bu}:{agent or 'all'}"
240
+
241
+ # ⛔ THE WIRE SHAPE IS THE SHARED PROJECTION (`aios_grid.workspace_wire`), not the store
242
+ # shape. The first version returned the store dict with no `storageKey` — and the client
243
+ # validator requires one, so the standalone shell DISCARDED the whole workspace: saved views
244
+ # never rendered, `cohortMode` never arrived, and the Cohort route silently drew the Customer
245
+ # surface. One projection for both servers is the fix that cannot drift.
246
+ try:
247
+ g = grid_assembly(session, scope=scope, storage_key=storage_key)
248
+ except grid_events.StoreUnavailable:
249
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
250
+ workspace = g["workspace"]
251
+ # ADDITIVE to the wire: the caller's own overlay cells. The grid reads overlays from the
252
+ # /customers rows, but this route is the cheap read-back a write can be verified against
253
+ # (verify_api's overlay probe) without paying the pool call. Per-user by construction —
254
+ # `table_workspace` is this session's workspace.
255
+ workspace["overlays"] = g["ws"].get("overlays") or {}
256
+ # ⭐ THE FIELD CONTRACT, 2026-08-04 — and it is NOT decoration.
257
+ #
258
+ # `fields_from_workspace(ws, cohorts=bool(cohort_lists))` appends the derived "Locked
259
+ # views" column ONLY when the caller owns at least one cohort. So a user's FIRST cohort
260
+ # CHANGES THE FIELD CONTRACT — and the rows call that used to be the only carrier of
261
+ # `fields` is deliberately never re-fetched on a write (it is a 15-minute-cached Odoo
262
+ # pull; this route is the cheap re-read). The client therefore had no way to learn about
263
+ # that column short of a remount, which is the second half of the owner's "it only shows
264
+ # up when I switch modules and come back".
265
+ #
266
+ # ⚠ `g["fields"]` is the SAME list `_payload` serves on `/customers` — same assembly,
267
+ # same permission wall (`hidden_keys` already applied) — so the two wires cannot disagree
268
+ # about what a column is. Taking it from anywhere else would fork the contract that
269
+ # verify_fields_contract.py exists to keep single-sourced.
270
+ workspace["fields"] = g["fields"]
271
+ # ── the STANDALONE measure channel (owner item 1, 2026-07-31) ────────────────────────────
272
+ # The embed receives these as top-level render args; standalone lifts them off THIS route
273
+ # (useCustomerData merges them into the payload slots the grid already reads). They ride
274
+ # the workspace rather than /customers because a durable write re-reads exactly this route
275
+ # (WORKSPACE_STALE), so a new measure column populates without refetching the heavy pool.
276
+ workspace["measures"] = g["measures"]
277
+ workspace["measureSets"] = g["measure_sets"]
278
+ # Derived cells (cohort column + measure columns), keyed by pid. JSON object keys are
279
+ # strings; the client indexes with String(pid).
280
+ workspace["derived"] = {str(pid): cells for pid, cells in g["derived"].items()}
281
+ # Who is looking (permissions verdicts) + the assignee choices for `user`-typed columns —
282
+ # the two other host-only render args the shell was missing (fail-closed without them:
283
+ # restricted fields uneditable, assignee picker empty).
284
+ workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)}
285
+ try:
286
+ from core import users as _users
287
+ workspace["userOptions"] = _users.assignable_people(tenant=session.tenant)
288
+ # Wave 14 C-AVATAR — the options vocabulary's companion: display name -> data URL.
289
+ # Absent entries fall back to the client's initials disc; a non-data: value is
290
+ # dropped (defence in depth beside the write-side wall in routes_auth).
291
+ workspace["userAvatars"] = {k: v for k, v in
292
+ _users.avatar_map(tenant=session.tenant).items()
293
+ if str(v).startswith("data:image/")}
294
+ except Exception:
295
+ workspace["userOptions"] = []
296
+ workspace["userAvatars"] = {}
297
+
298
+ # THE SURFACE STAMP — a MIRROR of the host's own (`_table_grid`'s hide/cohort stamps).
299
+ # ⚠ Deliberately NOT `hideViews`: `cohortMode` is what swaps the Views panel for the cohort
300
+ # list, and two switches for one behaviour would drift.
301
+ workspace["scopeKey"] = scope
302
+ if scope == "cohort":
303
+ workspace["cohortMode"] = True
304
+ # The create pane offers "this cohort table only" (default) vs "all customer tables".
305
+ workspace["scopeChoice"] = True
306
+ return {"workspace": workspace}
307
+
308
+
309
+ # ─────────────────────────────────────────────── the TIME-SERIES channel (C-TS, 2026-08-02)
310
+ _TS_BUCKETS = ("week", "month", "quarter", "year")
311
+ _TS_MAX_BUCKETS = 120
312
+ _TS_MAX_FIELDS = 12
313
+ _TS_MAX_PIDS = 5000
314
+ _TS_MAX_LAST_N = 120
315
+ #: C-TSWIN (wave 14): sliding windows cost ONE aggregate query per (metric, bucket) — the
316
+ #: price of "bucket N == the grid cell at bucket N's end". The product cap keeps a legal
317
+ #: 120-bucket × 12-metric ask from becoming 1,440 queries in one request; typical panels
318
+ #: (12 × 3) sit two orders of magnitude under it. Dated amendment in the split doc.
319
+ _TS_MAX_CELLS = 720
320
+
321
+
322
+ def _ts_start_of(bucket, d):
323
+ """The calendar START of the bucket holding `d` (week = Monday, the vocabulary rule)."""
324
+ if bucket == "week":
325
+ return d - dt.timedelta(days=d.weekday())
326
+ if bucket == "month":
327
+ return d.replace(day=1)
328
+ if bucket == "quarter":
329
+ return d.replace(month=((d.month - 1) // 3) * 3 + 1, day=1)
330
+ return d.replace(month=1, day=1)
331
+
332
+
333
+ def _ts_next(bucket, d):
334
+ if bucket == "week":
335
+ return d + dt.timedelta(days=7)
336
+ if bucket == "year":
337
+ return d.replace(year=d.year + 1)
338
+ step = 1 if bucket == "month" else 3
339
+ m = d.month + step
340
+ return dt.date(d.year + (m - 1) // 12, (m - 1) % 12 + 1, 1)
341
+
342
+
343
+ def _ts_prev(bucket, d):
344
+ if bucket == "week":
345
+ return d - dt.timedelta(days=7)
346
+ if bucket == "year":
347
+ return d.replace(year=d.year - 1)
348
+ step = 1 if bucket == "month" else 3
349
+ y, m = d.year, d.month - step
350
+ while m < 1:
351
+ m += 12
352
+ y -= 1
353
+ return dt.date(y, m, 1)
354
+
355
+
356
+ def _ts_label(bucket, start):
357
+ if bucket == "week":
358
+ return f"Wk of {start.strftime('%b %d')}"
359
+ if bucket == "month":
360
+ return start.strftime("%b %Y")
361
+ if bucket == "quarter":
362
+ return f"Q{(start.month - 1) // 3 + 1} {start.year}"
363
+ return str(start.year)
364
+
365
+
366
+ @router.post("/grid/timeseries")
367
+ def grid_timeseries(body: dict = Body(default=None), scope: str = "customer",
368
+ session: Session = Depends(module_gate(MODULE))):
369
+ """Pooled measure values per calendar bucket — the time-series view's data channel.
370
+
371
+ C-TSWIN (wave 14, ruling R1): AS-OF semantics. Each metric's OWN stored window is
372
+ re-resolved per bucket with `today := min(bucket end, real today)` — `ytd` is cumulative
373
+ from Jan 1, `last_90_days` trailing, `all_time` cumulative ever; bucket N equals what the
374
+ grid's measure column would show if today were bucket N's end. Fixed-range (`custom`)
375
+ windows cannot slide and are dropped per field as `window_fixed`. Rows carry
376
+ `window: {kind, label}` so a cumulative row cannot be misread as periodic, and there is
377
+ NO `total` column — sliding windows overlap, so a sum of columns would double-count.
378
+
379
+ The client sends the pids its view currently matches (the filter IS the scope); the server
380
+ intersects them with the session's book, so the request can only ever NARROW. Values are
381
+ POOLED aggregates computed by the semantic layer's own expression (the additivity law: an
382
+ average is computed at pool grain, never averaged over per-customer answers). A bucket
383
+ with no rows is 0 for a sum/count and null for anything else; a bucket that has not
384
+ STARTED yet is null for every kind — an unstarted period's YTD is unanswered, not 0.
385
+ """
386
+ from core import measure_resolve
387
+ from harness import windows as _wn
388
+ from routes_customers import _pool_stamp, _team_agent, allowed_pids
389
+
390
+ # Wave 16 C-TOPIC: the channel is CUSTOMER-GRAIN (measure_resolve's own grain), so the
391
+ # product surface is refused in words rather than answered with an id-space accident —
392
+ # product pids are CRC32 hashes and would mostly fall outside the customer book anyway,
393
+ # but "mostly" is not a wall. The client hides the mode for this topic; this is the
394
+ # server's half of the same refusal.
395
+ _sc = _scope_or_400(scope)
396
+ if _sc == "product" or _sc.startswith("ut_"):
397
+ raise err(400, "bad_timeseries",
398
+ "this surface has no time-series channel — measures are customer-grain")
399
+
400
+ body = body or {}
401
+ bucket = body.get("bucket")
402
+ if bucket not in _TS_BUCKETS:
403
+ raise err(400, "bad_timeseries", "bucket must be one of week, month, quarter, year")
404
+ raw_fields = body.get("fields")
405
+ if not isinstance(raw_fields, list) or not raw_fields:
406
+ raise err(400, "bad_timeseries", "fields must be a non-empty list of field keys")
407
+ if len(raw_fields) > _TS_MAX_FIELDS:
408
+ raise err(400, "bad_timeseries", f"at most {_TS_MAX_FIELDS} fields per request")
409
+ raw_pids = body.get("pids")
410
+ if not isinstance(raw_pids, list) or not raw_pids:
411
+ raise err(400, "bad_timeseries", "pids must be a non-empty list")
412
+ if len(raw_pids) > _TS_MAX_PIDS:
413
+ raise err(400, "bad_timeseries", f"at most {_TS_MAX_PIDS} pids per request")
414
+ try:
415
+ wanted = {int(p) for p in raw_pids}
416
+ except (TypeError, ValueError):
417
+ raise err(400, "bad_timeseries", "pids must be integers")
418
+ pool = wanted & {int(p) for p in allowed_pids(session)}
419
+ if not pool:
420
+ # 403, not an empty 200: the caller asked about customers outside their book, and an
421
+ # all-zero series over nobody would read as "no activity", not "not yours".
422
+ raise err(403, "out_of_scope", "none of those customers are in your book")
423
+
424
+ span = body.get("span")
425
+ if not isinstance(span, dict):
426
+ raise err(400, "bad_timeseries", "span must be {'lastN': n} or {'from': .., 'to': ..}")
427
+ today = time.strftime("%Y-%m-%d")
428
+ t = dt.date.fromisoformat(today)
429
+ n = span.get("lastN")
430
+ starts = []
431
+ if n is not None:
432
+ if not isinstance(n, int) or isinstance(n, bool) or not (1 <= n <= _TS_MAX_LAST_N):
433
+ raise err(400, "bad_timeseries", f"lastN must be 1..{_TS_MAX_LAST_N}")
434
+ cur = _ts_start_of(bucket, t)
435
+ starts = [cur]
436
+ for _ in range(n - 1):
437
+ cur = _ts_prev(bucket, cur)
438
+ starts.append(cur)
439
+ starts.reverse()
440
+ else:
441
+ try:
442
+ d_from = dt.date.fromisoformat(str(span.get("from")))
443
+ d_to = dt.date.fromisoformat(str(span.get("to")))
444
+ except (TypeError, ValueError):
445
+ raise err(400, "bad_timeseries", "span.from/to must be ISO dates (YYYY-MM-DD)")
446
+ if d_from > d_to:
447
+ d_from, d_to = d_to, d_from
448
+ cur = _ts_start_of(bucket, d_from)
449
+ while cur <= d_to:
450
+ starts.append(cur)
451
+ if len(starts) > _TS_MAX_BUCKETS:
452
+ raise err(400, "bad_timeseries",
453
+ f"that span is more than {_TS_MAX_BUCKETS} {bucket} buckets - "
454
+ f"narrow it")
455
+ cur = _ts_next(bucket, cur)
456
+ if not starts:
457
+ raise err(400, "bad_timeseries", "the span holds no buckets")
458
+ ends = [_ts_next(bucket, s) - dt.timedelta(days=1) for s in starts]
459
+
460
+ rt = session.runtime
461
+ if not rt.available():
462
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
463
+ import modules.customer_data as cl_mod
464
+ # Read-only route: it must not consume a one-shot label-correction ack the browser has
465
+ # not seen yet (the same rule event validation follows).
466
+ ws = cl_mod.table_workspace(session.uname, consume_corrections=False)
467
+ fdefs = ws.get("fields") or {}
468
+ keys, dropped, seen = [], [], set()
469
+ for k in raw_fields:
470
+ k = str(k or "")
471
+ if not k or k in seen:
472
+ continue
473
+ seen.add(k)
474
+ fd = fdefs.get(k)
475
+ if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict):
476
+ # Named, never silent: v1 eligibility is measure-backed fields only (C-TS).
477
+ dropped.append({"field": k, "reason": "not_a_measure_field"})
478
+ continue
479
+ keys.append(k)
480
+ # ⚡ WAVE 17 R9 (amendment 2026-08-03, GRID's cross-fence ask) — A SHEET WITH NO MEASURE
481
+ # FIELDS IS NO LONGER A 400. It answers with the BUCKET GRID and no rows.
482
+ #
483
+ # Why this is the honest direction and not a loosening: the bucket starts and their labels
484
+ # are SERVER math (`_ts_start_of` / `_ts_next` / `_ts_label`), and R9 has the client
485
+ # synthesizing snapshot rows for preset columns that have no window to slide. Those rows
486
+ # must be painted under the SAME headings as everything else, so the client needs the
487
+ # columns even when the server has no series to put in them. Refusing the whole request
488
+ # meant the panel could offer nothing at all on this tenant — the shipped contract has zero
489
+ # measure-backed presets (`aios_grid.py`: "this branch is currently MEMBERLESS"), which is
490
+ # what item 5 is actually about.
491
+ #
492
+ # Nothing is invented by this: `rows` is empty and `meta.dropped` NAMES every field and why.
493
+ # A 400 is still returned for a request that is malformed (bad bucket, bad span, no fields
494
+ # at all) — this only stops treating "I asked about columns you cannot serve" as an error.
495
+ mfields, slid_keys = [], []
496
+ for k in keys:
497
+ m = dict(fdefs[k]["measure"])
498
+ if (m.get("window") or {}).get("kind") == "custom":
499
+ # C-TSWIN: a fixed date range cannot slide across buckets — named, never silent,
500
+ # the same posture as not_a_measure_field.
501
+ dropped.append({"field": k, "reason": "window_fixed"})
502
+ continue
503
+ mfields.append({"key": k, "measure": m})
504
+ slid_keys.append(k)
505
+ # Same amendment as above: every metric being fixed-range is a sheet with no SERIES, not a
506
+ # broken request. The columns still stand, and `window_fixed` still names each refusal.
507
+ if len(starts) * len(mfields) > _TS_MAX_CELLS:
508
+ raise err(400, "bad_timeseries",
509
+ "that ask is too wide - narrow the span or pick fewer metrics")
510
+
511
+ team_id, agent = _team_agent(session)
512
+ stamp = _pool_stamp(rt, team_id, agent)
513
+ problems = []
514
+ bucket_bounds = [(s.isoformat(), e.isoformat()) for s, e in zip(starts, ends)]
515
+ # No slidable metrics = nothing to resolve. Skipping the call rather than asking the
516
+ # resolver about an empty list keeps the memo free of a meaningless key.
517
+ answers = measure_resolve.series_values(
518
+ mfields, bucket, bucket_bounds,
519
+ team_id, frozenset(pool), today, stamp, rt.series_memo,
520
+ on_error=lambda tag, e: problems.append(str(e)[:200])) if mfields else {}
521
+
522
+ columns = []
523
+ for s, e in zip(starts, ends):
524
+ col = {"key": s.isoformat(), "label": _ts_label(bucket, s),
525
+ "from": s.isoformat(), "to": e.isoformat()}
526
+ if e > t:
527
+ col["partial"] = True
528
+ columns.append(col)
529
+ rows = []
530
+ for k in slid_keys:
531
+ ans = answers.get(k)
532
+ if ans is None:
533
+ dropped.append({"field": k, "reason": "unresolvable"})
534
+ continue
535
+ vals_by = ans.get("values") or {}
536
+ agg_kind = str(ans.get("agg") or "sum")
537
+ zero_fill = agg_kind in ("sum", "count")
538
+ vals = []
539
+ for s in starts:
540
+ if s > t:
541
+ # C-TSWIN: an unstarted period is unanswered for EVERY agg kind — a future
542
+ # month's YTD zero-filled to 0 would read as "the year reset".
543
+ vals.append(None)
544
+ else:
545
+ vals.append(vals_by.get(s.isoformat(), 0 if zero_fill else None))
546
+ wspec = (fdefs[k].get("measure") or {}).get("window")
547
+ wnorm = _wn.normalize(wspec) or {}
548
+ rows.append({"field": k, "label": str(fdefs[k].get("label") or k)[:120],
549
+ "agg": agg_kind, "values": vals,
550
+ "window": {"kind": str(wnorm.get("kind") or ""),
551
+ "label": _wn.label(wspec)}})
552
+ meta = {"pool": len(pool), "today": today, "bucket": bucket}
553
+ if dropped:
554
+ meta["dropped"] = dropped
555
+ if problems:
556
+ meta["problems"] = problems[:5]
557
+ return {"columns": columns, "rows": rows, "meta": meta}
558
+
559
+
560
+ #: C-CAL caps (wave 17, owner item 9 / ruling R4). A month of days, a handful of metrics.
561
+ #: Every one of these is a 400 WITH ITS REASON, never a silent trim: a calendar that quietly
562
+ #: answered 20 of the 31 days asked about would paint eleven blank cells that look like days
563
+ #: with no activity.
564
+ _CAL_MAX_GROUPS = 31
565
+ _CAL_MAX_FIELDS = 6
566
+ _CAL_MAX_CELLS = 186
567
+
568
+
569
+ @router.post("/grid/calendar_metrics")
570
+ def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "customer",
571
+ session: Session = Depends(module_gate(MODULE))):
572
+ """Per-DAY measure values with each metric's own window slid to that day (C-CAL / R4).
573
+
574
+ ⛔ WHY THIS EXISTS AT ALL. The calendar's summary cells used to aggregate the row VALUES the
575
+ grid already held — which for a measure column means "every member's YTD **as of today**",
576
+ summed and printed under a date in March. The number was arithmetically fine and semantically
577
+ a lie: it answered a question about today while sitting in a cell labelled with another day.
578
+ R4: a metric in a day cell is computed AS OF THAT DAY.
579
+
580
+ The shape differs from the time-series channel in exactly one way, and it is the reason this
581
+ is a separate route rather than a parameter: **every group carries its OWN pid set**. A
582
+ calendar day holds the records the date field placed there, so day-to-day the subject
583
+ changes. One `allowed_pids` for the whole request — the TS channel's shape — would compute
584
+ each day over everybody, which is a different question again.
585
+
586
+ Static (non-measure) fields are NOT served here. They have no window to slide, so the
587
+ client's own per-day aggregation over row values stays correct for them; sending them would
588
+ invite a second implementation of arithmetic that already works.
589
+ """
590
+ from core import measure_resolve
591
+ from harness import windows as _wn
592
+ from routes_customers import _pool_stamp, _team_agent, allowed_pids
593
+
594
+ # The same refusal the TS channel makes, for the same reason: measures are customer-grain
595
+ # and product pids are CRC32 hashes of SKU codes. "Mostly outside the book" is not a wall.
596
+ _sc_cal = _scope_or_400(scope)
597
+ if _sc_cal == "product" or _sc_cal.startswith("ut_"):
598
+ raise err(400, "bad_calendar_metrics",
599
+ "the product surface has no measure channel yet — measures are customer-grain")
600
+
601
+ body = body or {}
602
+ raw_groups = body.get("groups")
603
+ if not isinstance(raw_groups, list) or not raw_groups:
604
+ raise err(400, "bad_calendar_metrics",
605
+ "groups must be a non-empty list of {key: 'YYYY-MM-DD', pids: [...]}")
606
+ if len(raw_groups) > _CAL_MAX_GROUPS:
607
+ raise err(400, "bad_calendar_metrics",
608
+ f"at most {_CAL_MAX_GROUPS} days per request (one month)")
609
+ raw_fields = body.get("fields")
610
+ if not isinstance(raw_fields, list) or not raw_fields:
611
+ raise err(400, "bad_calendar_metrics", "fields must be a non-empty list of field keys")
612
+ if len(raw_fields) > _CAL_MAX_FIELDS:
613
+ raise err(400, "bad_calendar_metrics", f"at most {_CAL_MAX_FIELDS} metrics per request")
614
+ if len(raw_groups) * len(raw_fields) > _CAL_MAX_CELLS:
615
+ raise err(400, "bad_calendar_metrics",
616
+ "that ask is too wide - fewer days or fewer metrics")
617
+
618
+ book = {int(p) for p in allowed_pids(session)}
619
+ groups, seen_days = [], set()
620
+ for g in raw_groups:
621
+ if not isinstance(g, dict):
622
+ raise err(400, "bad_calendar_metrics", "every group must be an object")
623
+ day = str(g.get("key") or "")
624
+ try:
625
+ d = dt.date.fromisoformat(day)
626
+ except (TypeError, ValueError):
627
+ raise err(400, "bad_calendar_metrics",
628
+ "every group key must be an ISO date (YYYY-MM-DD)")
629
+ if day in seen_days:
630
+ raise err(400, "bad_calendar_metrics", f"day {day} appears twice")
631
+ seen_days.add(day)
632
+ raw_pids = g.get("pids")
633
+ if not isinstance(raw_pids, list):
634
+ raise err(400, "bad_calendar_metrics", "every group needs a pids list")
635
+ try:
636
+ wanted = {int(p) for p in raw_pids}
637
+ except (TypeError, ValueError):
638
+ raise err(400, "bad_calendar_metrics", "pids must be integers")
639
+ # NARROW-ONLY, per group. The client sends what its calendar placed; the server can
640
+ # only ever remove from that, never add.
641
+ groups.append((day, d, frozenset(wanted & book)))
642
+ if sum(len(p) for _, _, p in groups) > _TS_MAX_PIDS:
643
+ raise err(400, "bad_calendar_metrics",
644
+ f"at most {_TS_MAX_PIDS} customer references per request")
645
+
646
+ rt = session.runtime
647
+ if not rt.available():
648
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
649
+ import modules.customer_data as cl_mod
650
+ ws = cl_mod.table_workspace(session.uname, consume_corrections=False)
651
+ fdefs = ws.get("fields") or {}
652
+ keys, dropped, seen = [], [], set()
653
+ for k in raw_fields:
654
+ k = str(k or "")
655
+ if not k or k in seen:
656
+ continue
657
+ seen.add(k)
658
+ fd = fdefs.get(k)
659
+ if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict):
660
+ # The TS channel's vocabulary, deliberately reused rather than re-coined: the client
661
+ # already knows how to say these three words to a reader.
662
+ dropped.append({"field": k, "reason": "not_a_measure_field"})
663
+ continue
664
+ if ((fd["measure"].get("window") or {}).get("kind") == "custom"):
665
+ dropped.append({"field": k, "reason": "window_fixed"})
666
+ continue
667
+ keys.append(k)
668
+ if not keys:
669
+ raise err(400, "bad_calendar_metrics",
670
+ "none of the requested fields are measure fields with a window that can "
671
+ "slide to a day")
672
+
673
+ today = time.strftime("%Y-%m-%d")
674
+ t = dt.date.fromisoformat(today)
675
+ team_id, agent = _team_agent(session)
676
+ stamp = _pool_stamp(rt, team_id, agent)
677
+ problems = []
678
+ values = {k: {} for k in keys}
679
+ for day, d, pids in groups:
680
+ # A day that has not happened is unanswered for EVERY aggregate kind — the unstarted
681
+ # bucket law at day grain. Answering 0 would say "we sold nothing", which is a claim
682
+ # about a day nobody has lived through yet.
683
+ if d > t or not pids:
684
+ for k in keys:
685
+ values[k][day] = None
686
+ continue
687
+ answers = measure_resolve.series_values(
688
+ [{"key": k, "measure": dict(fdefs[k]["measure"])} for k in keys],
689
+ "day", [(day, day)], team_id, pids, today, stamp, rt.series_memo,
690
+ on_error=lambda tag, e: problems.append(str(e)[:200]))
691
+ for k in keys:
692
+ ans = answers.get(k)
693
+ if ans is None:
694
+ values[k][day] = None
695
+ continue
696
+ vals_by = ans.get("values") or {}
697
+ zero_fill = str(ans.get("agg") or "sum") in ("sum", "count")
698
+ values[k][day] = vals_by.get(day, 0 if zero_fill else None)
699
+
700
+ for k in keys:
701
+ if all(v is None for v in values[k].values()):
702
+ # Every day unanswerable is a FIELD-level failure, and saying so is the difference
703
+ # between "no activity that month" and "this metric could not be computed".
704
+ dropped.append({"field": k, "reason": "unresolvable"})
705
+ out = {"values": values, "today": today,
706
+ "windows": {k: {"kind": str((_wn.normalize(
707
+ (fdefs[k].get("measure") or {}).get("window")) or {}).get("kind") or ""),
708
+ "label": _wn.label((fdefs[k].get("measure") or {}).get("window"))}
709
+ for k in keys}}
710
+ if dropped:
711
+ out["dropped"] = dropped
712
+ if problems:
713
+ out["problems"] = problems[:5]
714
+ return out
715
+
716
+
717
+ @router.post("/grid/events")
718
+ def grid_events_route(body: dict = Body(default=None),
719
+ session: Session = Depends(require_session)):
720
+ """`{events: [<component event objects, verbatim>]}` → `{results, doc?, toast?}`.
721
+
722
+ Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the
723
+ same sessions the read door (`/workspace`) admits, or a tenant without the customer module
724
+ can SEE its own user tables and not write to them."""
725
+ from core import grid_events
726
+ from routes_customers import grid_assembly
727
+
728
+ events = (body or {}).get("events")
729
+ if events is None and isinstance(body, dict) and body.get("type"):
730
+ events = [body] # a single event object, the legacy shape
731
+ if not isinstance(events, list):
732
+ raise err(400, "bad_events", "expected {events: [...]}")
733
+ if len(events) > _MAX_EVENTS:
734
+ raise err(400, "too_many_events",
735
+ f"at most {_MAX_EVENTS} events per request (the client's resend window)")
736
+
737
+ # Validated with the SAME predicate as the read route. An unrecognised scopeKey used to be
738
+ # passed through verbatim, and `core.grid_events` only ever compares it to 'cohort' — so a
739
+ # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on
740
+ # what a scope is, or the surface you read is not the surface you wrote.
741
+ scope = _scope_or_400((body or {}).get("scopeKey"))
742
+ # Wave 21 C4 — same per-scope gate as /workspace (read and write doors must agree).
743
+ if not (scope == "product" or scope.startswith("ut_")):
744
+ session.require(MODULE)
745
+ # This assembly validates the write; it is not a payload the browser will render. Leave
746
+ # one-shot field-name correction acks queued for the subsequent /workspace refresh.
747
+ # Wave 16 C-TOPIC: the PRODUCT topic gets the product assembly — product field contract,
748
+ # product pids, and (below) the product TABLE OPS, so a product event is validated against
749
+ # and lands in the product bucket. The measure/cohort context is honestly EMPTY there:
750
+ # `clean_measure_field` refuses measure creates on this surface by construction (the
751
+ # customer-grain descope), which is the fail-closed shape, not an accident.
752
+ if scope == "product":
753
+ from routes_products import MODULE as PRODUCT_MODULE, product_assembly
754
+
755
+ session.require(PRODUCT_MODULE) # the product wall (dependency is session-only, C4)
756
+ g = product_assembly(session, consume_corrections=False)
757
+ elif scope.startswith("ut_"):
758
+ # Wave 18 C3-UT — the user-table wall (creator/admin) is inside the assembly; the
759
+ # measure/cohort context is honestly EMPTY (customer-grain machinery, no meaning here).
760
+ from routes_tables import ut_assembly
761
+
762
+ g = ut_assembly(session, scope, consume_corrections=False)
763
+ else:
764
+ g = grid_assembly(session, scope=scope, consume_corrections=False)
765
+ # ⚠ THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`,
766
+ # `clean_measure_field` had an empty admission list and every measure-column create over
767
+ # HTTP was silently refused; without `measure_keys`, `clean_filter_tree` stripped every
768
+ # measure CONDITION out of a saved view. The embed always passed these; the API adapter
769
+ # simply had not been given them — the standalone shell could read measures it could
770
+ # never write.
771
+ ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope,
772
+ measure_keys=frozenset(m["key"] for m in g["measures"]),
773
+ resolved_ids=frozenset(g["measure_sets"]),
774
+ cohort_ids=frozenset(c["id"] for c in g["lists"]),
775
+ measure_offer=tuple(g["measures"]),
776
+ visible_views=tuple(g["views"]))
777
+
778
+ # Per-event results so the client can tell which of a batch landed — the component's own
779
+ # bridge has no response channel at all, so this is strictly more than the embed gets.
780
+ results = []
781
+ try:
782
+ for one in events:
783
+ eid = str(one.get("id") or "") if isinstance(one, dict) else ""
784
+ rerender = grid_events.handle_one(one, ctx)
785
+ results.append({"id": eid, "rerender": bool(rerender)})
786
+ except grid_events.StoreUnavailable:
787
+ raise err(503, "store_unavailable",
788
+ "the tenant store is unavailable — none of your changes were saved")
789
+
790
+ out = {"results": results, "rerender": any(r["rerender"] for r in results)}
791
+ if ctx.out.doc is not None:
792
+ out["doc"] = ctx.out.doc
793
+ if ctx.out.toast is not None:
794
+ out["toast"] = ctx.out.toast
795
+
796
+ # ── ⭐ owner item 2 (2026-08-03): THE NEW MEASURE COLUMN'S VALUES, ONE ROUND TRIP SOONER ──
797
+ #
798
+ # Creating a measure column cost the browser TWO sequential trips before a single number
799
+ # appeared: this one to persist the field, then a whole `/workspace` to compute it. The
800
+ # second cannot start until the first lands (the resolver reads the PERSISTED field), so the
801
+ # wait was structural, not slow code — the owner's "it takes some time for the data to
802
+ # populate". The values are computed here instead, immediately after the write, and ride
803
+ # this response.
804
+ #
805
+ # ⚠ IT COSTS NOTHING EXTRA TO COMPUTE. The expensive part is one DuckDB aggregate over the
806
+ # book, and `rt.measure_memo` is keyed on (pool stamp, scope, pool, measure, window) — so
807
+ # the `/workspace` re-read that still follows HITS the memo instead of doing this work. The
808
+ # query happens once either way; only its position moved.
809
+ #
810
+ # ⚠ NARROW ON PURPOSE. Gated to an actual measure-column write, so an overlay edit or a
811
+ # cohort add — the overwhelming majority of events — never pays for a second assembly.
812
+ #
813
+ # ⚠ AND IT IS A SHORTCUT, NOT A PATH. Any failure is swallowed: `WORKSPACE_STALE` still
814
+ # fires from `rerender`, and the re-read still delivers these values exactly as it does
815
+ # today. Nothing depends on this having worked.
816
+ if out["rerender"] and scope in ("customer", "cohort") and any(
817
+ isinstance(e, dict) and e.get("type") == "field_upsert"
818
+ and str(((e.get("field") or {}) if isinstance(e.get("field"), dict) else {})
819
+ .get("key") or "").startswith("measure_")
820
+ for e in events):
821
+ try:
822
+ fresh = grid_assembly(session, scope=scope, consume_corrections=False)
823
+ out["derived"] = {str(pid): cells for pid, cells in fresh["derived"].items()}
824
+ except Exception:
825
+ pass
826
+ # ⚠ NOTHING IS INVALIDATED HERE, on purpose. The runtime cache holds ONLY the scope-shaped
827
+ # Odoo pool (see `routes_customers._pool_rows`), and no event on this route can change an
828
+ # Odoo column — Odoo is read-only. Everything an event DOES change (overlays, fields, views,
829
+ # folders, cohorts) is re-read from the store on the next request. An earlier version cleared
830
+ # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh
831
+ # data that was never in it.
832
+ return out
api/routes_keychain.py CHANGED
@@ -1,278 +1,278 @@
1
- """routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3).
2
-
3
- Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER
4
- return a decrypted field — list rows carry a masked preview, and the decrypt function is a
5
- connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's
6
- env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records**
7
- count (rows holding overlay data whose pids the current pool no longer serves; counted and
8
- drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly
9
- in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with
10
- the keychain cutover wave R3 staged.
11
- """
12
- import os
13
-
14
- from fastapi import Body, Depends
15
- from fastapi import APIRouter
16
-
17
- from deps import Session, err
18
- from routes_admin import admin_gate
19
-
20
- router = APIRouter(prefix="/api/v1")
21
-
22
- #: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by
23
- #: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another
24
- #: freezes nothing while reporting success, which is the shape of the bug D-10 books.
25
- from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402
26
- ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY)
27
- #: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED
28
- #: odoo connector is paused; read by the pool path while paused; survives a Space restart.
29
- SNAPSHOT_KEY = "connector_snapshots"
30
-
31
-
32
- def _kc():
33
- import core.keychain as keychain
34
- return keychain
35
-
36
-
37
- def _resolved_odoo_key(rt):
38
- """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the
39
- shapes this module needs: the display string (`env` / `keychain:<id>`) and the key the pause
40
- flag is stored under.
41
-
42
- ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`,
43
- beside the `odoo_source()` it must agree with. This function had its own copy of the same
44
- three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and
45
- a second copy is exactly how a pause flag comes to be written against one resolution and read
46
- against another, freezing nothing while the UI reports success. The two SHAPES stay here
47
- because they are this module's presentation concern; the DECISION does not.
48
- """
49
- flag_key = rt.odoo_flag_key()
50
- if not flag_key:
51
- return None, None
52
- return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key
53
-
54
-
55
- def odoo_paused(rt):
56
- """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry
57
- that is not the resolved source freezes nothing — it serves nothing.
58
-
59
- ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the
60
- measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question
61
- the customer pool does. This name stays because `routes_customers`, `routes_products` and
62
- `verify_api` all call it — moving the logic without moving the door keeps one answer and
63
- costs no caller a change.
64
- """
65
- return bool(rt.odoo_paused())
66
-
67
-
68
- def _snap_scope_key(team_id, agent):
69
- return f"t={team_id}|a={agent}"
70
-
71
-
72
- def load_pool_snapshot(rt, team_id, agent):
73
- """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider
74
- scope's rows — serving the consolidated snapshot to a scoped user would widen their book."""
75
- try:
76
- snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {}
77
- e = snap.get(_snap_scope_key(team_id, agent))
78
- if isinstance(e, dict) and isinstance(e.get("rows"), list):
79
- return float(e.get("ts") or 0), e["rows"]
80
- except Exception:
81
- pass
82
- return None
83
-
84
-
85
- def save_pool_snapshots(rt, taken_by=""):
86
- """Persist every currently-cached pool scope as the pause-time snapshot ('the last
87
- successful sync', made concrete). Ensures the consolidated default scope exists first so
88
- a pause on a cold process still captures something to serve."""
89
- import time as _time
90
- import routes_customers as _rc
91
- try:
92
- _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on
93
- except Exception:
94
- pass # cold + Odoo down: persist whatever IS cached
95
- pools = {}
96
- for key, entry in list(rt.pool_cache.items()):
97
- if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool"
98
- and isinstance(entry, tuple) and len(entry) == 2
99
- and isinstance(entry[1], list)):
100
- pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]}
101
- if not pools:
102
- return 0
103
-
104
- def _up(cur):
105
- cur["odoo_pool"] = pools
106
- cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S")
107
- cur["takenBy"] = str(taken_by or "")
108
- return cur
109
-
110
- rt.update(SNAPSHOT_KEY, _up, flush="sync")
111
- return len(pools)
112
-
113
-
114
- @router.get("/admin/keychain")
115
- def list_keychain(session: Session = Depends(admin_gate)):
116
- kc = _kc()
117
- return {"entries": kc.list_entries(session.runtime), "locked": not kc.unlocked()}
118
-
119
-
120
- @router.post("/admin/keychain", status_code=201)
121
- def add_key(body: dict = Body(default=None), session: Session = Depends(admin_gate)):
122
- kc = _kc()
123
- body = body or {}
124
- if not session.runtime.available():
125
- raise err(503, "store_unavailable", "the tenant store is unavailable")
126
- try:
127
- row = kc.add_entry(session.runtime, body.get("label"), body.get("type"),
128
- body.get("fields"), session.uname)
129
- except kc.KeychainLocked as e:
130
- raise err(503, "keychain_locked",
131
- f"the keychain is locked — {e}. A secret is never stored unencrypted.")
132
- except ValueError as e:
133
- raise err(400, "bad_entry", str(e))
134
- except Exception:
135
- raise err(503, "store_unavailable", "the entry was not saved — try again")
136
- return {"entry": row}
137
-
138
-
139
- @router.delete("/admin/keychain/{entry_id}")
140
- def delete_key(entry_id: str, session: Session = Depends(admin_gate)):
141
- try:
142
- _kc().delete_entry(session.runtime, entry_id)
143
- except Exception:
144
- raise err(503, "store_unavailable", "the delete did not land — try again")
145
- return {"ok": True}
146
-
147
-
148
- @router.post("/admin/keychain/{entry_id}/test")
149
- def test_key(entry_id: str, session: Session = Depends(admin_gate)):
150
- return _kc().test_entry(session.runtime, entry_id)
151
-
152
-
153
- def _unsynced_customer_records(session):
154
- """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no
155
- longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a
156
- tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the
157
- answer is `known: False`, never a fabricated zero."""
158
- try:
159
- import core.table_store as table_store
160
- bucket = session.runtime.get("customer_table_workspace") or {}
161
- overlay_pids = {}
162
- for uname, ws in bucket.items():
163
- if uname == table_store.SHARED_KEY or not isinstance(ws, dict):
164
- continue
165
- for pid, cells in (ws.get("overlays") or {}).items():
166
- if isinstance(cells, dict) and cells:
167
- overlay_pids.setdefault(str(pid), cells)
168
- if not overlay_pids:
169
- return {"known": True, "count": 0, "rows": []}
170
- from routes_customers import allowed_pids
171
- pool = {str(p) for p in allowed_pids(session)}
172
- orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x)
173
- if str(x).isdigit() else 0)
174
- rows = []
175
- for p in orphans[:50]:
176
- cells = overlay_pids[p]
177
- hint = next((str(v) for v in cells.values() if str(v).strip()), "")
178
- rows.append({"pid": int(p) if str(p).isdigit() else p,
179
- "fields": len(cells), "hint": hint[:80]})
180
- return {"known": True, "count": len(orphans), "rows": rows,
181
- "shown": min(len(orphans), 50)}
182
- except Exception as e:
183
- return {"known": False, "count": None, "rows": [],
184
- "note": f"pool unavailable — {type(e).__name__}"}
185
-
186
-
187
- @router.get("/admin/connectors")
188
- def connectors(session: Session = Depends(admin_gate)):
189
- kc = _kc()
190
- flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {}
191
- entries = kc.list_entries(session.runtime)
192
- # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries —
193
- # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env
194
- # for tenant #0 only, else nothing (fail closed — never another tenant's environment).
195
- first_odoo = next((e["id"] for e in entries if e["type"] == "odoo"), None)
196
- if first_odoo and kc.unlocked():
197
- resolved = f"keychain:{first_odoo}"
198
- elif session.tenant == "royal-imports" and os.environ.get("ODOO_URL"):
199
- resolved = "env"
200
- else:
201
- resolved = None
202
- rows = []
203
- if session.tenant == "royal-imports" and os.environ.get("ODOO_URL"):
204
- rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo",
205
- "source": "env", "active": resolved == "env",
206
- "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))})
207
- for e in entries:
208
- rows.append({"key": e["id"], "label": e["label"], "type": e["type"],
209
- "source": "keychain", "preview": e["preview"],
210
- "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"),
211
- "paused": bool((flags.get(e["id"]) or {}).get("paused"))})
212
- out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved,
213
- # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before.
214
- # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the
215
- # hole it left: "measures not already computed may still reach the source". D-10
216
- # closed that hole — `harness/datastore.py` (the mirror every measure column is
217
- # answered from) refuses to sync while paused, and `routes_products._pool_for` got the
218
- # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather
219
- # than left standing, because a warning that outlives its defect teaches the reader to
220
- # ignore warnings.
221
- # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely
222
- # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and
223
- # collapsing them into "everything freezes" would be the kind of tidy summary that
224
- # stops being true the first time one of them changes.
225
- # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the
226
- # correction is recorded here rather than silently applied. D-62 said this note
227
- # "promises a behaviour on a dashboard measure path that has been dead since W16".
228
- # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from
229
- # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while
230
- # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is
231
- # `/api/v1/pages/{key}` (D-52), which this note never mentioned.
232
- #
233
- # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures
234
- # stop moving rather than going blank". BOTH pool paths answer **503** when they have
235
- # no copy to serve — the customer path for a scope with no snapshot
236
- # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because
237
- # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says
238
- # so in as many words). So a paused connector plus a restarted server is exactly the
239
- # blank screen this sentence promised could not happen. A warning that over-promises
240
- # is worse than none: it is the sentence somebody quotes when the screen disagrees.
241
- "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and "
242
- "views stay, and nothing reaches the source while it is paused. "
243
- "Anything this server has already read keeps showing: the customer "
244
- "workspace serves its pause-time snapshot, the product list serves "
245
- "the last copy read since startup, and measure columns keep answering "
246
- "from the mirror as it stood when you paused. What has NOT been read "
247
- "cannot be shown — a scope with no snapshot, or the product list after "
248
- "a restart, reports that the source is paused instead of showing "
249
- "figures. Resume to start reading live again.")}
250
- if session.tenant == "royal-imports":
251
- out["unsynced"] = _unsynced_customer_records(session)
252
- return out
253
-
254
-
255
- @router.post("/admin/connectors/{key}/pause")
256
- def pause_connector(key: str, body: dict = Body(default=None),
257
- session: Session = Depends(admin_gate)):
258
- paused = bool((body or {}).get("paused"))
259
- if not session.runtime.available():
260
- raise err(503, "store_unavailable", "the tenant store is unavailable")
261
-
262
- # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a
263
- # "last successful sync" to serve before the freeze takes effect. Capturing before the
264
- # flag flips means a failed capture leaves the connector live (never paused-with-nothing).
265
- snapshots = 0
266
- _, flag_key = _resolved_odoo_key(session.runtime)
267
- if paused and flag_key and str(key) == flag_key:
268
- snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname)
269
-
270
- def _up(cur):
271
- cur[str(key)] = {"paused": paused}
272
- return cur
273
-
274
- try:
275
- session.runtime.update(_CONNECTOR_FLAGS_KEY, _up)
276
- except Exception:
277
- raise err(503, "store_unavailable", "the change was not saved — try again")
278
- return {"key": key, "paused": paused, "snapshots": snapshots}
 
1
+ """routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3).
2
+
3
+ Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER
4
+ return a decrypted field — list rows carry a masked preview, and the decrypt function is a
5
+ connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's
6
+ env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records**
7
+ count (rows holding overlay data whose pids the current pool no longer serves; counted and
8
+ drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly
9
+ in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with
10
+ the keychain cutover wave R3 staged.
11
+ """
12
+ import os
13
+
14
+ from fastapi import Body, Depends
15
+ from fastapi import APIRouter
16
+
17
+ from deps import Session, err
18
+ from routes_admin import admin_gate
19
+
20
+ router = APIRouter(prefix="/api/v1")
21
+
22
+ #: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by
23
+ #: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another
24
+ #: freezes nothing while reporting success, which is the shape of the bug D-10 books.
25
+ from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402
26
+ ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY)
27
+ #: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED
28
+ #: odoo connector is paused; read by the pool path while paused; survives a Space restart.
29
+ SNAPSHOT_KEY = "connector_snapshots"
30
+
31
+
32
+ def _kc():
33
+ import core.keychain as keychain
34
+ return keychain
35
+
36
+
37
+ def _resolved_odoo_key(rt):
38
+ """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the
39
+ shapes this module needs: the display string (`env` / `keychain:<id>`) and the key the pause
40
+ flag is stored under.
41
+
42
+ ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`,
43
+ beside the `odoo_source()` it must agree with. This function had its own copy of the same
44
+ three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and
45
+ a second copy is exactly how a pause flag comes to be written against one resolution and read
46
+ against another, freezing nothing while the UI reports success. The two SHAPES stay here
47
+ because they are this module's presentation concern; the DECISION does not.
48
+ """
49
+ flag_key = rt.odoo_flag_key()
50
+ if not flag_key:
51
+ return None, None
52
+ return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key
53
+
54
+
55
+ def odoo_paused(rt):
56
+ """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry
57
+ that is not the resolved source freezes nothing — it serves nothing.
58
+
59
+ ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the
60
+ measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question
61
+ the customer pool does. This name stays because `routes_customers`, `routes_products` and
62
+ `verify_api` all call it — moving the logic without moving the door keeps one answer and
63
+ costs no caller a change.
64
+ """
65
+ return bool(rt.odoo_paused())
66
+
67
+
68
+ def _snap_scope_key(team_id, agent):
69
+ return f"t={team_id}|a={agent}"
70
+
71
+
72
+ def load_pool_snapshot(rt, team_id, agent):
73
+ """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider
74
+ scope's rows — serving the consolidated snapshot to a scoped user would widen their book."""
75
+ try:
76
+ snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {}
77
+ e = snap.get(_snap_scope_key(team_id, agent))
78
+ if isinstance(e, dict) and isinstance(e.get("rows"), list):
79
+ return float(e.get("ts") or 0), e["rows"]
80
+ except Exception:
81
+ pass
82
+ return None
83
+
84
+
85
+ def save_pool_snapshots(rt, taken_by=""):
86
+ """Persist every currently-cached pool scope as the pause-time snapshot ('the last
87
+ successful sync', made concrete). Ensures the consolidated default scope exists first so
88
+ a pause on a cold process still captures something to serve."""
89
+ import time as _time
90
+ import routes_customers as _rc
91
+ try:
92
+ _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on
93
+ except Exception:
94
+ pass # cold + Odoo down: persist whatever IS cached
95
+ pools = {}
96
+ for key, entry in list(rt.pool_cache.items()):
97
+ if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool"
98
+ and isinstance(entry, tuple) and len(entry) == 2
99
+ and isinstance(entry[1], list)):
100
+ pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]}
101
+ if not pools:
102
+ return 0
103
+
104
+ def _up(cur):
105
+ cur["odoo_pool"] = pools
106
+ cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S")
107
+ cur["takenBy"] = str(taken_by or "")
108
+ return cur
109
+
110
+ rt.update(SNAPSHOT_KEY, _up, flush="sync")
111
+ return len(pools)
112
+
113
+
114
+ @router.get("/admin/keychain")
115
+ def list_keychain(session: Session = Depends(admin_gate)):
116
+ kc = _kc()
117
+ return {"entries": kc.list_entries(session.runtime), "locked": not kc.unlocked()}
118
+
119
+
120
+ @router.post("/admin/keychain", status_code=201)
121
+ def add_key(body: dict = Body(default=None), session: Session = Depends(admin_gate)):
122
+ kc = _kc()
123
+ body = body or {}
124
+ if not session.runtime.available():
125
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
126
+ try:
127
+ row = kc.add_entry(session.runtime, body.get("label"), body.get("type"),
128
+ body.get("fields"), session.uname)
129
+ except kc.KeychainLocked as e:
130
+ raise err(503, "keychain_locked",
131
+ f"the keychain is locked — {e}. A secret is never stored unencrypted.")
132
+ except ValueError as e:
133
+ raise err(400, "bad_entry", str(e))
134
+ except Exception:
135
+ raise err(503, "store_unavailable", "the entry was not saved — try again")
136
+ return {"entry": row}
137
+
138
+
139
+ @router.delete("/admin/keychain/{entry_id}")
140
+ def delete_key(entry_id: str, session: Session = Depends(admin_gate)):
141
+ try:
142
+ _kc().delete_entry(session.runtime, entry_id)
143
+ except Exception:
144
+ raise err(503, "store_unavailable", "the delete did not land — try again")
145
+ return {"ok": True}
146
+
147
+
148
+ @router.post("/admin/keychain/{entry_id}/test")
149
+ def test_key(entry_id: str, session: Session = Depends(admin_gate)):
150
+ return _kc().test_entry(session.runtime, entry_id)
151
+
152
+
153
+ def _unsynced_customer_records(session):
154
+ """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no
155
+ longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a
156
+ tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the
157
+ answer is `known: False`, never a fabricated zero."""
158
+ try:
159
+ import core.table_store as table_store
160
+ bucket = session.runtime.get("customer_table_workspace") or {}
161
+ overlay_pids = {}
162
+ for uname, ws in bucket.items():
163
+ if uname == table_store.SHARED_KEY or not isinstance(ws, dict):
164
+ continue
165
+ for pid, cells in (ws.get("overlays") or {}).items():
166
+ if isinstance(cells, dict) and cells:
167
+ overlay_pids.setdefault(str(pid), cells)
168
+ if not overlay_pids:
169
+ return {"known": True, "count": 0, "rows": []}
170
+ from routes_customers import allowed_pids
171
+ pool = {str(p) for p in allowed_pids(session)}
172
+ orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x)
173
+ if str(x).isdigit() else 0)
174
+ rows = []
175
+ for p in orphans[:50]:
176
+ cells = overlay_pids[p]
177
+ hint = next((str(v) for v in cells.values() if str(v).strip()), "")
178
+ rows.append({"pid": int(p) if str(p).isdigit() else p,
179
+ "fields": len(cells), "hint": hint[:80]})
180
+ return {"known": True, "count": len(orphans), "rows": rows,
181
+ "shown": min(len(orphans), 50)}
182
+ except Exception as e:
183
+ return {"known": False, "count": None, "rows": [],
184
+ "note": f"pool unavailable — {type(e).__name__}"}
185
+
186
+
187
+ @router.get("/admin/connectors")
188
+ def connectors(session: Session = Depends(admin_gate)):
189
+ kc = _kc()
190
+ flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {}
191
+ entries = kc.list_entries(session.runtime)
192
+ # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries —
193
+ # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env
194
+ # for tenant #0 only, else nothing (fail closed — never another tenant's environment).
195
+ first_odoo = next((e["id"] for e in entries if e["type"] == "odoo"), None)
196
+ if first_odoo and kc.unlocked():
197
+ resolved = f"keychain:{first_odoo}"
198
+ elif session.tenant == "royal-imports" and os.environ.get("ODOO_URL"):
199
+ resolved = "env"
200
+ else:
201
+ resolved = None
202
+ rows = []
203
+ if session.tenant == "royal-imports" and os.environ.get("ODOO_URL"):
204
+ rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo",
205
+ "source": "env", "active": resolved == "env",
206
+ "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))})
207
+ for e in entries:
208
+ rows.append({"key": e["id"], "label": e["label"], "type": e["type"],
209
+ "source": "keychain", "preview": e["preview"],
210
+ "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"),
211
+ "paused": bool((flags.get(e["id"]) or {}).get("paused"))})
212
+ out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved,
213
+ # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before.
214
+ # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the
215
+ # hole it left: "measures not already computed may still reach the source". D-10
216
+ # closed that hole — `harness/datastore.py` (the mirror every measure column is
217
+ # answered from) refuses to sync while paused, and `routes_products._pool_for` got the
218
+ # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather
219
+ # than left standing, because a warning that outlives its defect teaches the reader to
220
+ # ignore warnings.
221
+ # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely
222
+ # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and
223
+ # collapsing them into "everything freezes" would be the kind of tidy summary that
224
+ # stops being true the first time one of them changes.
225
+ # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the
226
+ # correction is recorded here rather than silently applied. D-62 said this note
227
+ # "promises a behaviour on a dashboard measure path that has been dead since W16".
228
+ # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from
229
+ # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while
230
+ # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is
231
+ # `/api/v1/pages/{key}` (D-52), which this note never mentioned.
232
+ #
233
+ # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures
234
+ # stop moving rather than going blank". BOTH pool paths answer **503** when they have
235
+ # no copy to serve — the customer path for a scope with no snapshot
236
+ # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because
237
+ # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says
238
+ # so in as many words). So a paused connector plus a restarted server is exactly the
239
+ # blank screen this sentence promised could not happen. A warning that over-promises
240
+ # is worse than none: it is the sentence somebody quotes when the screen disagrees.
241
+ "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and "
242
+ "views stay, and nothing reaches the source while it is paused. "
243
+ "Anything this server has already read keeps showing: the customer "
244
+ "workspace serves its pause-time snapshot, the product list serves "
245
+ "the last copy read since startup, and measure columns keep answering "
246
+ "from the mirror as it stood when you paused. What has NOT been read "
247
+ "cannot be shown — a scope with no snapshot, or the product list after "
248
+ "a restart, reports that the source is paused instead of showing "
249
+ "figures. Resume to start reading live again.")}
250
+ if session.tenant == "royal-imports":
251
+ out["unsynced"] = _unsynced_customer_records(session)
252
+ return out
253
+
254
+
255
+ @router.post("/admin/connectors/{key}/pause")
256
+ def pause_connector(key: str, body: dict = Body(default=None),
257
+ session: Session = Depends(admin_gate)):
258
+ paused = bool((body or {}).get("paused"))
259
+ if not session.runtime.available():
260
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
261
+
262
+ # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a
263
+ # "last successful sync" to serve before the freeze takes effect. Capturing before the
264
+ # flag flips means a failed capture leaves the connector live (never paused-with-nothing).
265
+ snapshots = 0
266
+ _, flag_key = _resolved_odoo_key(session.runtime)
267
+ if paused and flag_key and str(key) == flag_key:
268
+ snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname)
269
+
270
+ def _up(cur):
271
+ cur[str(key)] = {"paused": paused}
272
+ return cur
273
+
274
+ try:
275
+ session.runtime.update(_CONNECTOR_FLAGS_KEY, _up)
276
+ except Exception:
277
+ raise err(503, "store_unavailable", "the change was not saved — try again")
278
+ return {"key": key, "paused": paused, "snapshots": snapshots}
api/routes_platform_admin.py CHANGED
@@ -1,618 +1,618 @@
1
- """routes_platform_admin.py — THE LOOPABLE ADMIN PLANE (wave 19, owner item 13 / R3+R4).
2
-
3
- Every other admin surface in this product answers questions about ONE tenant. This one answers
4
- questions about the PLATFORM: who our customers are, what they are running, whether their data
5
- sources are alive, and what the automation fleet costs. It is the first cross-tenant reader that
6
- has ever existed here, which is why it is also the most carefully walled.
7
-
8
- ⛔ THE WALL. `padmin_gate` is `core.platform_admin.is_platform_admin` — a DOUBLE lock (the record
9
- flag AND the `loopable` tenant), fail-closed, applied to every single route in this file through
10
- one dependency. A tenant admin is NOT admitted: `role: 'admin'` is Royal's or Nurilab's authority
11
- over their own workspace and it must never widen into a view of each other. `verify_api.py`'s
12
- W19-ADMIN section proves that by ENUMERATING this router and having a tenant admin try every path
13
- it declares, so a route added later cannot quietly ship without the wall.
14
-
15
- ⛔ CROSS-TENANT READS GO THROUGH EACH TENANT'S OWN RUNTIME. `runtime.get_runtime(slug)` per tenant,
16
- then `rt.get(...)` — never `core.store.get("<raw key>")`. The runtime is what applies the store
17
- NAMESPACE (`t/<slug>/…`) or binds the tenant's OWN dataset repo (R2), so reading raw keys would
18
- silently return tenant #0's data labelled as somebody else's — a wrong answer that looks right,
19
- which is the worst failure this plane could have.
20
-
21
- HONEST DEGRADATION IS THE WHOLE DESIGN, NOT AN ERROR PATH. This plane reads six subsystems across
22
- N tenants; on any given day one of them can be unreachable (a suspended tenant record, a locked
23
- keychain, an HF repo hiccup, no AWS credentials in this container). A 500 would take the entire
24
- dashboard down because one cell could not be filled. So every collector catches, and every row can
25
- carry an `error` string that the pane RENDERS — "unknown" is a real answer and it is never
26
- rendered as a zero. ([[gate-can-report-green-on-nothing]]: a fabricated 0 and a true 0 must not
27
- look alike.)
28
-
29
- EVERY COUNT DRILLS TO ROWS. `/overview`'s per-tenant counts are computed by the SAME collector
30
- functions the `/users`, `/databases`, `/connectors` and `/automations` routes serve rows from, so
31
- a number and its drill-down cannot disagree — they are one computation, projected twice
32
- ([[no-unverifiable-aggregates]]).
33
- """
34
- import json
35
- import os
36
- import subprocess
37
- import sys
38
- import time
39
- from pathlib import Path
40
-
41
- from fastapi import APIRouter, Depends
42
-
43
- import core.platform_admin as platform_admin
44
- import core.store as store
45
- from deps import Session, err, require_session, users
46
-
47
- router = APIRouter(prefix="/api/v1/platform-admin")
48
-
49
-
50
- def _now_iso():
51
- """UTC, offset-bearing — the one stamp format anything client-side may subtract from."""
52
- import datetime as _dt
53
- return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
54
-
55
-
56
- def padmin_gate(session: Session = Depends(require_session)) -> Session:
57
- """401 without a session, 403 unless this account is a Loopable platform operator.
58
-
59
- ONE dependency for the whole router. A per-route check is a per-route chance to forget, and
60
- the thing being forgotten here would be every customer's data at once.
61
-
62
- The message deliberately does not confirm that a platform plane exists for somebody else —
63
- a tenant admin who pokes at this URL learns only that their account cannot open it.
64
- """
65
- if not platform_admin.is_platform_admin(session.user):
66
- raise err(403, "forbidden", "your account does not have access to this surface")
67
- return session
68
-
69
-
70
- # ══════════════════════════════════════════════════════════════════════════ tenants
71
-
72
-
73
- def _tenant_bucket():
74
- """The control-plane `tenants` records, or {} — a platform fact, in the DEFAULT store."""
75
- from harness import runtime
76
- try:
77
- recs = store.get(runtime.TENANTS_KEY) or {}
78
- return {str(k).strip().lower(): v for k, v in recs.items() if isinstance(v, dict)}
79
- except Exception:
80
- return {}
81
-
82
-
83
- def _slugs():
84
- """Every tenant this deployment knows: compiled builders + control-plane records.
85
-
86
- `runtime.known_tenants()` is the union and it is the same list login resolves against, so
87
- this plane cannot show a customer the door does not recognise (or miss one it does).
88
- """
89
- from harness import runtime
90
- try:
91
- return list(runtime.known_tenants())
92
- except Exception:
93
- return sorted(_tenant_bucket())
94
-
95
-
96
- def _runtime_for(slug):
97
- """(runtime, error). A tenant whose runtime will not build is a ROW WITH A NOTE, never a 500.
98
-
99
- Two real cases produce one: a SUSPENDED record (`get_runtime` raises KeyError by design —
100
- "which tenants exist" is not a question the login path answers), and tenant #0's builder,
101
- which makes a LIVE Odoo call (`harness.tenants.royal_imports` → `excluded_customer_ids`) and
102
- therefore fails on a box that cannot reach the ERP. Neither may take the dashboard down.
103
- """
104
- from harness import runtime
105
- try:
106
- return runtime.get_runtime(slug), ""
107
- except KeyError:
108
- return None, "not resolvable (suspended, or no builder and no active record)"
109
- except Exception as e: # noqa: BLE001
110
- return None, f"runtime unavailable ({type(e).__name__})"
111
-
112
-
113
- def _scan(want=""):
114
- """[(row, runtime)] — every tenant (or one), with its runtime RESOLVED EXACTLY ONCE.
115
-
116
- The one-resolution rule is not tidiness. `get_runtime` is LRU-cached on success, but a tenant
117
- that FAILS to build is not cached, and tenant #0's builder makes a live Odoo call — so a route
118
- that asked twice would pay the timeout twice, and `/overview` (which asks about six
119
- subsystems) would pay it six times. Resolve once, pass the handle down.
120
- """
121
- bucket = _tenant_bucket()
122
- want = str(want or "").strip().lower()
123
- out = []
124
- for slug in _slugs():
125
- if want and slug != want:
126
- continue
127
- rec = bucket.get(slug) or {}
128
- rt, error = _runtime_for(slug)
129
- out.append(({
130
- "slug": slug,
131
- # The record's name, else the built runtime's, else the slug. Tenant #0 is compiled
132
- # and has no record, so without the runtime fallback it would render as "royal-imports".
133
- "name": str(rec.get("name") or (getattr(rt, "name", "") if rt else "") or slug),
134
- "source": "record" if rec else "compiled",
135
- "status": str(rec.get("status") or ("active" if rt else "unknown")),
136
- "domains": list(rec.get("domains") or []),
137
- "modules": rec.get("modules", "all" if not rec else []),
138
- # R2: the isolation shape. "own repo" and "shared repo + prefix" are genuinely
139
- # different blast radii and an operator should be able to see which is which.
140
- "storeRepo": rec.get("store_repo") or ("shared (tenant #0 repo)" if not rec else ""),
141
- "storePrefix": getattr(rt, "store_namespace", "") if rt else "",
142
- "error": error,
143
- }, rt))
144
- return out
145
-
146
-
147
- # ══════════════════════════════════════════════════════════════════════════ users
148
-
149
-
150
- def _user_rows(tenant=None):
151
- """Accounts across every tenant, from the GLOBAL registry (`users.json` is control-plane).
152
-
153
- ⛔ NEVER `salt` OR `hash`. This projection is the only one these routes use, mirroring
154
- `routes_admin._view`'s discipline: one function that can leak, and it does not.
155
-
156
- R4's two new fields ride here — `lastLogin` / `lastActive`, absent on every pre-wave record,
157
- rendered as "never" rather than as a fabricated date.
158
- """
159
- try:
160
- reg = users.registry() or {}
161
- except Exception:
162
- return []
163
- want = str(tenant or "").strip().lower()
164
- rows = []
165
- for uname, rec in sorted(reg.items()):
166
- if not isinstance(rec, dict):
167
- continue
168
- slug = str(rec.get("tenant") or "royal-imports").strip().lower()
169
- if want and slug != want:
170
- continue
171
- rows.append({
172
- "username": uname,
173
- "name": rec.get("name") or uname,
174
- "email": rec.get("email") or "",
175
- "tenant": slug,
176
- "role": rec.get("role", "user"),
177
- "active": bool(rec.get("active", True)),
178
- "lastLogin": rec.get("last_login") or "",
179
- "lastActive": rec.get("last_active") or "",
180
- "platformAdmin": rec.get("platform_admin") is True,
181
- })
182
- return rows
183
-
184
-
185
- # ══════════════════════════════════════════════════════════════════════════ databases
186
-
187
-
188
- def _database_rows(slug, rt):
189
- """A tenant's user-created databases (`ut_*`) with their row counts.
190
-
191
- Via `core.user_tables.all_tables(st=rt)` — the TenantRuntime, never the module-global, which
192
- the engine's own header (`automation_engine.py:49-52`) flags as a cross-tenant defect for
193
- every R2 tenant. That booked defect is exactly the mistake a cross-tenant reader would make
194
- most easily, so it is stated at the call site too.
195
- """
196
- try:
197
- import core.user_tables as ut
198
- tables = ut.all_tables(st=rt) or {}
199
- except Exception as e: # noqa: BLE001
200
- return [], f"databases unreadable ({type(e).__name__})"
201
- rows = []
202
- for key, t in sorted(tables.items(), key=lambda kv: (kv[1].get("label") or "").lower()):
203
- if not isinstance(t, dict):
204
- continue
205
- rows.append({
206
- "tenant": slug,
207
- "key": key,
208
- "label": t.get("label") or key,
209
- "source": t.get("source") or "Blank",
210
- "createdBy": t.get("createdBy") or "",
211
- "created": t.get("created") or "",
212
- "fields": len(t.get("fields") or []),
213
- "rowCount": len(t.get("rows") or {}),
214
- })
215
- return rows, ""
216
-
217
-
218
- # ══════════════════════════════════════════════════════════════════════════ connectors
219
-
220
-
221
- def _connector_rows(slug, rt):
222
- """A tenant's data sources and which one is actually RESOLVED (i.e. would serve a query).
223
-
224
- `_resolved_odoo_key` is imported from `routes_keychain` rather than re-derived. It is the
225
- route-level mirror of `TenantRuntime.odoo_source()`, it takes a runtime (not a session), and
226
- a second copy of that resolution here would be a copy that drifts — at which point this plane
227
- would confidently name the wrong live connector. Reused, not restated.
228
- """
229
- try:
230
- import core.keychain as keychain
231
- from routes_keychain import _CONNECTOR_FLAGS_KEY, _resolved_odoo_key
232
- entries = keychain.list_entries(rt)
233
- locked = not keychain.unlocked()
234
- resolved, _flag = _resolved_odoo_key(rt)
235
- flags = rt.get(_CONNECTOR_FLAGS_KEY) or {}
236
- except Exception as e: # noqa: BLE001
237
- return [], f"connectors unreadable ({type(e).__name__})", False
238
-
239
- rows = []
240
- if slug == "royal-imports" and os.environ.get("ODOO_URL"):
241
- rows.append({"tenant": slug, "key": "odoo-env", "label": "Odoo (environment)",
242
- "type": "odoo", "source": "env", "active": resolved == "env",
243
- "paused": bool((flags.get("odoo-env") or {}).get("paused"))})
244
- for e in entries:
245
- rows.append({"tenant": slug, "key": e["id"], "label": e["label"], "type": e["type"],
246
- "source": "keychain",
247
- "active": e["type"] == "odoo" and resolved == f"keychain:{e['id']}",
248
- "paused": bool((flags.get(e["id"]) or {}).get("paused"))})
249
- return rows, "", locked
250
-
251
-
252
- # ══════════════════════════════════════════════════════════════════════════ automations + cost
253
-
254
- #: THE UNIT MODEL, from `ops/provision_automation_cron.py` (its cost note at :25-28 and the
255
- #: function it actually provisions at :211 / :269). Restated as constants rather than as prose so
256
- #: the arithmetic below is re-checkable against the thing that was really deployed:
257
- #: EventBridge Scheduler `rate(15 minutes)` → a 128 MB, 30 s-timeout Lambda that POSTs the tick.
258
- TICK_CADENCE = "rate(15 minutes)"
259
- TICK_PER_DAY = 96 # 1440 / 15
260
- LAMBDA_MB = 128
261
- FREE_LAMBDA_REQUESTS = 1_000_000 # AWS always-free, per month
262
- FREE_SCHEDULER_INVOCATIONS = 14_000_000
263
- DAYS_PER_MONTH = 30.4
264
-
265
-
266
- def _runs_per_day(cron):
267
- """Scheduled runs/day for the five-field crons this product offers, or None.
268
-
269
- ⚠ DELIBERATELY NARROW. It answers exactly the shapes `automation_engine.CRON_PRESETS` can
270
- produce (every-N-minutes, hourly, daily, weekly, monthly) and returns None for anything else
271
- — an unparsed cadence is reported as "custom", never as a guessed number that would then be
272
- multiplied into a cost. A wrong denominator is worse than an absent one.
273
-
274
- ⚠ AND IT IS NOT MEASURED FROM HISTORY, on purpose: `automation_engine.MAX_RUNS` trims run
275
- history to 20 entries, so a 15-minute automation retains ~5 hours of it. Deriving runs/day
276
- from that window would understate by ~5x. History is reported as what it is — the last N runs.
277
- """
278
- parts = str(cron or "").split()
279
- if len(parts) != 5:
280
- return None
281
- minute, hour, dom, _mon, dow = parts
282
- if minute.startswith("*/") and hour == "*":
283
- try:
284
- step = int(minute[2:])
285
- except ValueError:
286
- return None
287
- return (1440.0 / step) if step > 0 else None
288
- if minute.isdigit() and hour == "*":
289
- return 24.0
290
- if minute.isdigit() and hour.isdigit():
291
- if dom.isdigit():
292
- return 1.0 / DAYS_PER_MONTH # monthly
293
- if dow.isdigit():
294
- return 1.0 / 7.0 # weekly
295
- return 1.0 # daily
296
- return None
297
-
298
-
299
- def _automation_rows(slug, rt):
300
- """A tenant's automations, their schedules, and their retained run history."""
301
- try:
302
- import automation_engine as engine
303
- defs = engine.all_definitions(rt) or {}
304
- max_runs = int(getattr(engine, "MAX_RUNS", 20))
305
- except Exception as e: # noqa: BLE001
306
- return [], f"automations unreadable ({type(e).__name__})", 20
307
- rows = []
308
- for auto_id, d in sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower()):
309
- if not isinstance(d, dict):
310
- continue
311
- sched = d.get("schedule") or {}
312
- status = d.get("status") or {}
313
- runs = list(d.get("runs") or [])
314
- cron = sched.get("cron") or ""
315
- per_day = _runs_per_day(cron) if sched.get("enabled") else 0.0
316
- rows.append({
317
- "tenant": slug,
318
- "id": auto_id,
319
- "name": d.get("name") or auto_id,
320
- "kind": d.get("kind") or "",
321
- "enabled": bool(sched.get("enabled")),
322
- "cron": cron,
323
- "runsPerDay": round(per_day, 2) if per_day is not None else None,
324
- "state": status.get("state") or "idle",
325
- "lastRunAt": status.get("lastRunAt") or "",
326
- "lastSummary": status.get("lastSummary") or "",
327
- # The retained window, named as such — see `_runs_per_day`'s second warning.
328
- "runsRetained": len(runs),
329
- "failedRetained": sum(1 for r in runs if isinstance(r, dict) and not r.get("ok")),
330
- "createdBy": d.get("createdBy") or "",
331
- "created": d.get("created") or "",
332
- })
333
- return rows, "", max_runs
334
-
335
-
336
- def _automation_cost(auto_rows):
337
- """The estimated monthly cost of running the automation fleet — and the honest shape of it.
338
-
339
- THE POINT THIS BLOCK EXISTS TO MAKE, which is not intuitive: **the external cron does not
340
- scale with automations or tenants.** One EventBridge schedule fires one Lambda, which POSTs
341
- one tick, and that tick runs every DUE automation for every tenant (`provision_automation_cron`
342
- states this as the reason it stays $0 "at ten tenants"). So the AWS bill is a function of the
343
- CADENCE alone — the fleet below adds work inside the API container, which is already paid for.
344
-
345
- WHAT IS NOT COMPUTED HERE, deliberately: Lambda GB-seconds. That needs the real average
346
- duration of the function, which only CloudWatch knows; assuming one would be inventing the
347
- larger half of the free-tier calculation. `/aws` reports the measured figure when credentials
348
- are available, and this block says so rather than filling the gap with a plausible number.
349
- """
350
- invocations = TICK_PER_DAY * DAYS_PER_MONTH
351
- fleet_runs = sum(r["runsPerDay"] or 0 for r in auto_rows if r.get("enabled"))
352
- unknown_cadence = sum(1 for r in auto_rows if r.get("enabled") and r.get("runsPerDay") is None)
353
- return {
354
- "cadence": TICK_CADENCE,
355
- "invocationsPerMonth": int(round(invocations)),
356
- "freeRequestsPct": round(100.0 * invocations / FREE_LAMBDA_REQUESTS, 3),
357
- "freeSchedulerPct": round(100.0 * invocations / FREE_SCHEDULER_INVOCATIONS, 4),
358
- "lambdaMb": LAMBDA_MB,
359
- "usd": 0.0,
360
- "fleetRunsPerDay": round(fleet_runs, 2),
361
- "unknownCadence": unknown_cadence,
362
- "basis": (
363
- f"One EventBridge schedule ({TICK_CADENCE}) fires one {LAMBDA_MB} MB Lambda that "
364
- f"POSTs the tick; that ONE tick runs every due automation for every tenant, so the "
365
- f"AWS cost is set by the cadence and does not grow with the fleet. "
366
- f"{int(round(invocations)):,} invocations/month is "
367
- f"{round(100.0 * invocations / FREE_LAMBDA_REQUESTS, 3)}% of the 1,000,000-request "
368
- f"always-free tier, so the marginal cost of the automations below is $0.00. "
369
- f"Compute (GB-seconds) depends on measured durations and is NOT estimated here — "
370
- f"the AWS report reads the real figure when credentials are available."
371
- ),
372
- }
373
-
374
-
375
- # ═════════════════════════════════════════════════════════════════════════ the AWS report
376
-
377
- #: `ops/aws_usage_report.py`, relative to this file: aios-web/api/ -> repo root -> ops/.
378
- _AWS_REPORT = Path(__file__).resolve().parents[2] / "ops" / "aws_usage_report.py"
379
- _AWS_TIMEOUT = 60
380
-
381
-
382
- def _aws_report(days=7):
383
- """The AWS usage report as text, or an honest block saying why there is none.
384
-
385
- ⛔ SUBPROCESS, NEVER `import`. Two concrete reasons, both found by reading that file rather
386
- than by running it:
387
- * it reassigns `sys.stdout` to a UTF-8 wrapper AT MODULE IMPORT (a cp1252 fix for this
388
- box) — importing it would mutate the API process's stdout as a side effect;
389
- * its `main()` calls `argparse.parse_args()` with no argv, so inside a server it would
390
- parse UVICORN's arguments and can `sys.exit(2)` — and `SystemExit` is a `BaseException`,
391
- which `except Exception` does not catch. A route that "cannot crash" would crash.
392
-
393
- ⚠ ON THE SPACE THIS IS THE NORMAL PATH, NOT THE ERROR PATH. `deploy_web.py` ships `api/*.py`
394
- and `web/`; `ops/` is not in the image, and AWS credentials live in a local `.env` that is
395
- never deployed. So the honest block below is what the deployed plane shows, and it is written
396
- to be read by an operator as information ("run it here") rather than as a fault.
397
- """
398
- if not _AWS_REPORT.is_file():
399
- return {"available": False, "text": "",
400
- "note": ("The AWS usage report is not part of this deployment — `ops/` ships "
401
- "with the repository, not with the container image. Run "
402
- "`python ops/aws_usage_report.py` locally for the live figures.")}
403
- try:
404
- proc = subprocess.run(
405
- [sys.executable, str(_AWS_REPORT), "--days", str(int(days))],
406
- capture_output=True, text=True, timeout=_AWS_TIMEOUT,
407
- cwd=str(_AWS_REPORT.parent.parent))
408
- except subprocess.TimeoutExpired:
409
- return {"available": False, "text": "",
410
- "note": (f"The AWS report did not answer within {_AWS_TIMEOUT}s — CloudWatch may "
411
- f"be unreachable from here. Nothing was assumed about usage.")}
412
- except Exception as e: # noqa: BLE001
413
- return {"available": False, "text": "",
414
- "note": f"The AWS report could not be run here ({type(e).__name__})."}
415
- out = (proc.stdout or "").strip()
416
- if proc.returncode != 0 or not out:
417
- detail = (proc.stderr or "").strip().splitlines()
418
- return {"available": False, "text": out,
419
- "note": ("AWS credentials are not available here, or boto3 is not installed — "
420
- "no usage could be read, and none is guessed. "
421
- + (detail[-1][:200] if detail else ""))}
422
- return {"available": True, "text": out, "note": ""}
423
-
424
-
425
- # ══════════════════════════════════════════════════════════════════════════ routes
426
-
427
-
428
- @router.get("/overview")
429
- def overview(session: Session = Depends(padmin_gate)):
430
- """THE PLANE'S ONE TABLE: every tenant, with the counts that drill to the rows below.
431
-
432
- Each count is produced by the same collector its `/…` route serves, so the number and its
433
- drill-down are one computation projected twice. A subsystem that cannot be read contributes
434
- an `errors` entry on that tenant's row and a NULL count — never a zero, which would read as
435
- "this customer has no databases" when the truth is "we could not look".
436
- """
437
- t0 = time.time()
438
- all_users = _user_rows()
439
- users_by_tenant = {}
440
- for u in all_users:
441
- users_by_tenant.setdefault(u["tenant"], []).append(u)
442
-
443
- rows = []
444
- for t, rt in _scan():
445
- slug = t["slug"]
446
- row = dict(t)
447
- row["users"] = len(users_by_tenant.get(slug, []))
448
- row["admins"] = sum(1 for u in users_by_tenant.get(slug, []) if u["role"] == "admin")
449
- errors = [t["error"]] if t["error"] else []
450
- if rt is None:
451
- # An unresolvable tenant still shows its ACCOUNTS (they live in the global registry,
452
- # which is readable regardless) — everything tenant-store-shaped is honestly unknown.
453
- row.update({"databases": None, "rows": None, "connectors": None,
454
- "automations": None, "keychainLocked": None, "errors": errors})
455
- rows.append(row)
456
- continue
457
- dbs, db_err = _database_rows(slug, rt)
458
- conns, conn_err, locked = _connector_rows(slug, rt)
459
- autos, auto_err, _mr = _automation_rows(slug, rt)
460
- errors += [e for e in (db_err, conn_err, auto_err) if e]
461
- row.update({
462
- "databases": None if db_err else len(dbs),
463
- "rows": None if db_err else sum(d["rowCount"] for d in dbs),
464
- "connectors": None if conn_err else len(conns),
465
- "connectorsPaused": None if conn_err else sum(1 for c in conns if c["paused"]),
466
- "automations": None if auto_err else len(autos),
467
- "automationsEnabled": None if auto_err else sum(1 for a in autos if a["enabled"]),
468
- "keychainLocked": None if conn_err else locked,
469
- "errors": errors,
470
- })
471
- rows.append(row)
472
-
473
- return {
474
- "tenants": rows,
475
- "totals": {
476
- "tenants": len(rows),
477
- # EVERY account, not the sum of the rows: an account whose `tenant` names a slug this
478
- # deployment no longer knows belongs in the platform total and would vanish from a
479
- # per-row sum. `orphanUsers` names that gap instead of hiding it.
480
- "users": len(all_users),
481
- "orphanUsers": len(all_users) - sum(r["users"] for r in rows),
482
- # Sums SKIP unknowns rather than treating them as 0, and say how many were skipped —
483
- # a total that silently absorbs an unreadable tenant is a fabricated total.
484
- "databases": sum(r["databases"] or 0 for r in rows),
485
- "rows": sum(r["rows"] or 0 for r in rows),
486
- "automations": sum(r["automations"] or 0 for r in rows),
487
- "unknownTenants": sum(1 for r in rows if r["databases"] is None),
488
- },
489
- "storeAvailable": bool(_store_ok()),
490
- # ⚠ OFFSET-BEARING, and that is not pedantry. The client renders this as "read 2 minutes
491
- # ago" via `Date.parse`, which reads a NAIVE stamp as browser-local — so a UTC container
492
- # and a US viewer would turn "just now" into "5 hours ago", or into a future date that
493
- # renders as "just now" forever. The stamps written by `core.users` carry an offset for
494
- # the same reason; anything a browser subtracts from `Date.now()` must say what zone it
495
- # is in. `verify_api` asserts the offset so this cannot regress to `strftime`.
496
- "generatedAt": _now_iso(),
497
- "tookMs": int((time.time() - t0) * 1000),
498
- }
499
-
500
-
501
- def _store_ok():
502
- try:
503
- return store.available()
504
- except Exception:
505
- return False
506
-
507
-
508
- @router.get("/users")
509
- def platform_users(tenant: str = "", session: Session = Depends(padmin_gate)):
510
- """Every account on the platform, or one tenant's — the drill behind the Users count.
511
-
512
- R4's stamps are the columns that did not exist before this wave: `lastLogin` is written by
513
- `routes_auth.login`, `lastActive` by `deps.require_session` (throttled to once an hour per
514
- account per process). Absent means never seen, and the pane renders it as "never".
515
- """
516
- rows = _user_rows(tenant)
517
- return {"users": rows, "count": len(rows),
518
- "stampsNote": ("Login and activity stamps started with this release — accounts that "
519
- "have not signed in since show no date rather than an invented one. "
520
- "Activity is recorded at most once an hour per account.")}
521
-
522
-
523
- @router.get("/databases")
524
- def platform_databases(tenant: str = "", session: Session = Depends(padmin_gate)):
525
- """Every tenant's user-created databases and row counts — the drill behind Databases/Rows."""
526
- out, errors = [], {}
527
- for t, rt in _scan(tenant):
528
- slug = t["slug"]
529
- if rt is None:
530
- errors[slug] = t["error"]
531
- continue
532
- rows, err = _database_rows(slug, rt)
533
- if err:
534
- errors[slug] = err
535
- continue
536
- out += rows
537
- return {"databases": out, "count": len(out),
538
- "rows": sum(d["rowCount"] for d in out), "errors": errors}
539
-
540
-
541
- @router.get("/connectors")
542
- def platform_connectors(tenant: str = "", session: Session = Depends(padmin_gate)):
543
- """Every tenant's data sources, which one is live, and which are paused.
544
-
545
- METADATA ONLY — `keychain.list_entries` never decrypts, so no credential, and no masked
546
- preview either: a platform operator needs to know a source EXISTS and whether it serves, not
547
- what the secret looks like. The tenant's own admin surface is where previews belong.
548
- """
549
- out, errors, locked_any = [], {}, {}
550
- for t, rt in _scan(tenant):
551
- slug = t["slug"]
552
- if rt is None:
553
- errors[slug] = t["error"]
554
- continue
555
- rows, err, locked = _connector_rows(slug, rt)
556
- if err:
557
- errors[slug] = err
558
- continue
559
- locked_any[slug] = locked
560
- out += rows
561
- return {"connectors": out, "count": len(out), "keychainLocked": locked_any,
562
- "errors": errors,
563
- "note": ("A locked keychain means this container has no `AIOS_KEYCHAIN_KEY` — stored "
564
- "credentials cannot be read, so a tenant's sources fail closed rather than "
565
- "falling back to anyone else's.")}
566
-
567
-
568
- @router.get("/automations")
569
- def platform_automations(tenant: str = "", session: Session = Depends(padmin_gate)):
570
- """Every tenant's automations, their schedules and run history, plus the fleet cost model."""
571
- out, errors, retained = [], {}, 20
572
- for t, rt in _scan(tenant):
573
- slug = t["slug"]
574
- if rt is None:
575
- errors[slug] = t["error"]
576
- continue
577
- rows, err, max_runs = _automation_rows(slug, rt)
578
- if err:
579
- errors[slug] = err
580
- continue
581
- retained = max_runs
582
- out += rows
583
- return {"automations": out, "count": len(out),
584
- "enabled": sum(1 for a in out if a["enabled"]),
585
- "historyRetained": retained,
586
- "cost": _automation_cost(out), "errors": errors,
587
- "tickEnabled": os.environ.get("AIOS_AUTOMATIONS") == "1"}
588
-
589
-
590
- @router.get("/aws")
591
- def platform_aws(days: int = 7, session: Session = Depends(padmin_gate)):
592
- """The AWS cron's usage report, verbatim, or an honest note explaining its absence."""
593
- days = max(1, min(int(days or 7), 90))
594
- return {"days": days, "report": _aws_report(days)}
595
-
596
-
597
- @router.get("/releases")
598
- def platform_releases(session: Session = Depends(padmin_gate)):
599
- """⭐ WAVE 20 (owner item 12, ruling R6) — WHAT IS RUNNING WHERE, and what else could be.
600
-
601
- Owner: *"Make this part of the deploy skill. Also ability to revert to any version we want.
602
- Make sure App versioning is something that our company admin (loopable, non-tenant) can
603
- easily see."* This is the SEEING half; promoting stays a CLI command by R6, so nothing here
604
- writes and no web session can move production.
605
-
606
- ⛔ THE HUB READS LIVE IN `core/releases.py`, NOT HERE. `ops/verify_portability.py` B2 forbids
607
- the HuggingFace SDK anywhere under `aios-web/api/` — the API process is host-agnostic by
608
- design — and it caught this endpoint's first draft doing exactly that. Same delegation shape
609
- as `routes_assets` -> `core/assets.py`.
610
- """
611
- import core.releases as releases
612
-
613
- return {"here": os.environ.get("AIOS_VERSION") or "unknown",
614
- "environments": releases.environments(),
615
- "releases": releases.history(),
616
- # The runbook, in the payload, because the panel is read-only BY DESIGN and a user
617
- # looking at it is exactly the person who needs to know how to move a version.
618
- "promote": "python aios-web/deploy_web.py --promote=vN (or --promote=staging)"}
 
1
+ """routes_platform_admin.py — THE LOOPABLE ADMIN PLANE (wave 19, owner item 13 / R3+R4).
2
+
3
+ Every other admin surface in this product answers questions about ONE tenant. This one answers
4
+ questions about the PLATFORM: who our customers are, what they are running, whether their data
5
+ sources are alive, and what the automation fleet costs. It is the first cross-tenant reader that
6
+ has ever existed here, which is why it is also the most carefully walled.
7
+
8
+ ⛔ THE WALL. `padmin_gate` is `core.platform_admin.is_platform_admin` — a DOUBLE lock (the record
9
+ flag AND the `loopable` tenant), fail-closed, applied to every single route in this file through
10
+ one dependency. A tenant admin is NOT admitted: `role: 'admin'` is Royal's or Nurilab's authority
11
+ over their own workspace and it must never widen into a view of each other. `verify_api.py`'s
12
+ W19-ADMIN section proves that by ENUMERATING this router and having a tenant admin try every path
13
+ it declares, so a route added later cannot quietly ship without the wall.
14
+
15
+ ⛔ CROSS-TENANT READS GO THROUGH EACH TENANT'S OWN RUNTIME. `runtime.get_runtime(slug)` per tenant,
16
+ then `rt.get(...)` — never `core.store.get("<raw key>")`. The runtime is what applies the store
17
+ NAMESPACE (`t/<slug>/…`) or binds the tenant's OWN dataset repo (R2), so reading raw keys would
18
+ silently return tenant #0's data labelled as somebody else's — a wrong answer that looks right,
19
+ which is the worst failure this plane could have.
20
+
21
+ HONEST DEGRADATION IS THE WHOLE DESIGN, NOT AN ERROR PATH. This plane reads six subsystems across
22
+ N tenants; on any given day one of them can be unreachable (a suspended tenant record, a locked
23
+ keychain, an HF repo hiccup, no AWS credentials in this container). A 500 would take the entire
24
+ dashboard down because one cell could not be filled. So every collector catches, and every row can
25
+ carry an `error` string that the pane RENDERS — "unknown" is a real answer and it is never
26
+ rendered as a zero. ([[gate-can-report-green-on-nothing]]: a fabricated 0 and a true 0 must not
27
+ look alike.)
28
+
29
+ EVERY COUNT DRILLS TO ROWS. `/overview`'s per-tenant counts are computed by the SAME collector
30
+ functions the `/users`, `/databases`, `/connectors` and `/automations` routes serve rows from, so
31
+ a number and its drill-down cannot disagree — they are one computation, projected twice
32
+ ([[no-unverifiable-aggregates]]).
33
+ """
34
+ import json
35
+ import os
36
+ import subprocess
37
+ import sys
38
+ import time
39
+ from pathlib import Path
40
+
41
+ from fastapi import APIRouter, Depends
42
+
43
+ import core.platform_admin as platform_admin
44
+ import core.store as store
45
+ from deps import Session, err, require_session, users
46
+
47
+ router = APIRouter(prefix="/api/v1/platform-admin")
48
+
49
+
50
+ def _now_iso():
51
+ """UTC, offset-bearing — the one stamp format anything client-side may subtract from."""
52
+ import datetime as _dt
53
+ return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
54
+
55
+
56
+ def padmin_gate(session: Session = Depends(require_session)) -> Session:
57
+ """401 without a session, 403 unless this account is a Loopable platform operator.
58
+
59
+ ONE dependency for the whole router. A per-route check is a per-route chance to forget, and
60
+ the thing being forgotten here would be every customer's data at once.
61
+
62
+ The message deliberately does not confirm that a platform plane exists for somebody else —
63
+ a tenant admin who pokes at this URL learns only that their account cannot open it.
64
+ """
65
+ if not platform_admin.is_platform_admin(session.user):
66
+ raise err(403, "forbidden", "your account does not have access to this surface")
67
+ return session
68
+
69
+
70
+ # ══════════════════════════════════════════════════════════════════════════ tenants
71
+
72
+
73
+ def _tenant_bucket():
74
+ """The control-plane `tenants` records, or {} — a platform fact, in the DEFAULT store."""
75
+ from harness import runtime
76
+ try:
77
+ recs = store.get(runtime.TENANTS_KEY) or {}
78
+ return {str(k).strip().lower(): v for k, v in recs.items() if isinstance(v, dict)}
79
+ except Exception:
80
+ return {}
81
+
82
+
83
+ def _slugs():
84
+ """Every tenant this deployment knows: compiled builders + control-plane records.
85
+
86
+ `runtime.known_tenants()` is the union and it is the same list login resolves against, so
87
+ this plane cannot show a customer the door does not recognise (or miss one it does).
88
+ """
89
+ from harness import runtime
90
+ try:
91
+ return list(runtime.known_tenants())
92
+ except Exception:
93
+ return sorted(_tenant_bucket())
94
+
95
+
96
+ def _runtime_for(slug):
97
+ """(runtime, error). A tenant whose runtime will not build is a ROW WITH A NOTE, never a 500.
98
+
99
+ Two real cases produce one: a SUSPENDED record (`get_runtime` raises KeyError by design —
100
+ "which tenants exist" is not a question the login path answers), and tenant #0's builder,
101
+ which makes a LIVE Odoo call (`harness.tenants.royal_imports` → `excluded_customer_ids`) and
102
+ therefore fails on a box that cannot reach the ERP. Neither may take the dashboard down.
103
+ """
104
+ from harness import runtime
105
+ try:
106
+ return runtime.get_runtime(slug), ""
107
+ except KeyError:
108
+ return None, "not resolvable (suspended, or no builder and no active record)"
109
+ except Exception as e: # noqa: BLE001
110
+ return None, f"runtime unavailable ({type(e).__name__})"
111
+
112
+
113
+ def _scan(want=""):
114
+ """[(row, runtime)] — every tenant (or one), with its runtime RESOLVED EXACTLY ONCE.
115
+
116
+ The one-resolution rule is not tidiness. `get_runtime` is LRU-cached on success, but a tenant
117
+ that FAILS to build is not cached, and tenant #0's builder makes a live Odoo call — so a route
118
+ that asked twice would pay the timeout twice, and `/overview` (which asks about six
119
+ subsystems) would pay it six times. Resolve once, pass the handle down.
120
+ """
121
+ bucket = _tenant_bucket()
122
+ want = str(want or "").strip().lower()
123
+ out = []
124
+ for slug in _slugs():
125
+ if want and slug != want:
126
+ continue
127
+ rec = bucket.get(slug) or {}
128
+ rt, error = _runtime_for(slug)
129
+ out.append(({
130
+ "slug": slug,
131
+ # The record's name, else the built runtime's, else the slug. Tenant #0 is compiled
132
+ # and has no record, so without the runtime fallback it would render as "royal-imports".
133
+ "name": str(rec.get("name") or (getattr(rt, "name", "") if rt else "") or slug),
134
+ "source": "record" if rec else "compiled",
135
+ "status": str(rec.get("status") or ("active" if rt else "unknown")),
136
+ "domains": list(rec.get("domains") or []),
137
+ "modules": rec.get("modules", "all" if not rec else []),
138
+ # R2: the isolation shape. "own repo" and "shared repo + prefix" are genuinely
139
+ # different blast radii and an operator should be able to see which is which.
140
+ "storeRepo": rec.get("store_repo") or ("shared (tenant #0 repo)" if not rec else ""),
141
+ "storePrefix": getattr(rt, "store_namespace", "") if rt else "",
142
+ "error": error,
143
+ }, rt))
144
+ return out
145
+
146
+
147
+ # ══════════════════════════════════════════════════════════════════════════ users
148
+
149
+
150
+ def _user_rows(tenant=None):
151
+ """Accounts across every tenant, from the GLOBAL registry (`users.json` is control-plane).
152
+
153
+ ⛔ NEVER `salt` OR `hash`. This projection is the only one these routes use, mirroring
154
+ `routes_admin._view`'s discipline: one function that can leak, and it does not.
155
+
156
+ R4's two new fields ride here — `lastLogin` / `lastActive`, absent on every pre-wave record,
157
+ rendered as "never" rather than as a fabricated date.
158
+ """
159
+ try:
160
+ reg = users.registry() or {}
161
+ except Exception:
162
+ return []
163
+ want = str(tenant or "").strip().lower()
164
+ rows = []
165
+ for uname, rec in sorted(reg.items()):
166
+ if not isinstance(rec, dict):
167
+ continue
168
+ slug = str(rec.get("tenant") or "royal-imports").strip().lower()
169
+ if want and slug != want:
170
+ continue
171
+ rows.append({
172
+ "username": uname,
173
+ "name": rec.get("name") or uname,
174
+ "email": rec.get("email") or "",
175
+ "tenant": slug,
176
+ "role": rec.get("role", "user"),
177
+ "active": bool(rec.get("active", True)),
178
+ "lastLogin": rec.get("last_login") or "",
179
+ "lastActive": rec.get("last_active") or "",
180
+ "platformAdmin": rec.get("platform_admin") is True,
181
+ })
182
+ return rows
183
+
184
+
185
+ # ══════════════════════════════════════════════════════════════════════════ databases
186
+
187
+
188
+ def _database_rows(slug, rt):
189
+ """A tenant's user-created databases (`ut_*`) with their row counts.
190
+
191
+ Via `core.user_tables.all_tables(st=rt)` — the TenantRuntime, never the module-global, which
192
+ the engine's own header (`automation_engine.py:49-52`) flags as a cross-tenant defect for
193
+ every R2 tenant. That booked defect is exactly the mistake a cross-tenant reader would make
194
+ most easily, so it is stated at the call site too.
195
+ """
196
+ try:
197
+ import core.user_tables as ut
198
+ tables = ut.all_tables(st=rt) or {}
199
+ except Exception as e: # noqa: BLE001
200
+ return [], f"databases unreadable ({type(e).__name__})"
201
+ rows = []
202
+ for key, t in sorted(tables.items(), key=lambda kv: (kv[1].get("label") or "").lower()):
203
+ if not isinstance(t, dict):
204
+ continue
205
+ rows.append({
206
+ "tenant": slug,
207
+ "key": key,
208
+ "label": t.get("label") or key,
209
+ "source": t.get("source") or "Blank",
210
+ "createdBy": t.get("createdBy") or "",
211
+ "created": t.get("created") or "",
212
+ "fields": len(t.get("fields") or []),
213
+ "rowCount": len(t.get("rows") or {}),
214
+ })
215
+ return rows, ""
216
+
217
+
218
+ # ══════════════════════════════════════════════════════════════════════════ connectors
219
+
220
+
221
+ def _connector_rows(slug, rt):
222
+ """A tenant's data sources and which one is actually RESOLVED (i.e. would serve a query).
223
+
224
+ `_resolved_odoo_key` is imported from `routes_keychain` rather than re-derived. It is the
225
+ route-level mirror of `TenantRuntime.odoo_source()`, it takes a runtime (not a session), and
226
+ a second copy of that resolution here would be a copy that drifts — at which point this plane
227
+ would confidently name the wrong live connector. Reused, not restated.
228
+ """
229
+ try:
230
+ import core.keychain as keychain
231
+ from routes_keychain import _CONNECTOR_FLAGS_KEY, _resolved_odoo_key
232
+ entries = keychain.list_entries(rt)
233
+ locked = not keychain.unlocked()
234
+ resolved, _flag = _resolved_odoo_key(rt)
235
+ flags = rt.get(_CONNECTOR_FLAGS_KEY) or {}
236
+ except Exception as e: # noqa: BLE001
237
+ return [], f"connectors unreadable ({type(e).__name__})", False
238
+
239
+ rows = []
240
+ if slug == "royal-imports" and os.environ.get("ODOO_URL"):
241
+ rows.append({"tenant": slug, "key": "odoo-env", "label": "Odoo (environment)",
242
+ "type": "odoo", "source": "env", "active": resolved == "env",
243
+ "paused": bool((flags.get("odoo-env") or {}).get("paused"))})
244
+ for e in entries:
245
+ rows.append({"tenant": slug, "key": e["id"], "label": e["label"], "type": e["type"],
246
+ "source": "keychain",
247
+ "active": e["type"] == "odoo" and resolved == f"keychain:{e['id']}",
248
+ "paused": bool((flags.get(e["id"]) or {}).get("paused"))})
249
+ return rows, "", locked
250
+
251
+
252
+ # ══════════════════════════════════════════════════════════════════════════ automations + cost
253
+
254
+ #: THE UNIT MODEL, from `ops/provision_automation_cron.py` (its cost note at :25-28 and the
255
+ #: function it actually provisions at :211 / :269). Restated as constants rather than as prose so
256
+ #: the arithmetic below is re-checkable against the thing that was really deployed:
257
+ #: EventBridge Scheduler `rate(15 minutes)` → a 128 MB, 30 s-timeout Lambda that POSTs the tick.
258
+ TICK_CADENCE = "rate(15 minutes)"
259
+ TICK_PER_DAY = 96 # 1440 / 15
260
+ LAMBDA_MB = 128
261
+ FREE_LAMBDA_REQUESTS = 1_000_000 # AWS always-free, per month
262
+ FREE_SCHEDULER_INVOCATIONS = 14_000_000
263
+ DAYS_PER_MONTH = 30.4
264
+
265
+
266
+ def _runs_per_day(cron):
267
+ """Scheduled runs/day for the five-field crons this product offers, or None.
268
+
269
+ ⚠ DELIBERATELY NARROW. It answers exactly the shapes `automation_engine.CRON_PRESETS` can
270
+ produce (every-N-minutes, hourly, daily, weekly, monthly) and returns None for anything else
271
+ — an unparsed cadence is reported as "custom", never as a guessed number that would then be
272
+ multiplied into a cost. A wrong denominator is worse than an absent one.
273
+
274
+ ⚠ AND IT IS NOT MEASURED FROM HISTORY, on purpose: `automation_engine.MAX_RUNS` trims run
275
+ history to 20 entries, so a 15-minute automation retains ~5 hours of it. Deriving runs/day
276
+ from that window would understate by ~5x. History is reported as what it is — the last N runs.
277
+ """
278
+ parts = str(cron or "").split()
279
+ if len(parts) != 5:
280
+ return None
281
+ minute, hour, dom, _mon, dow = parts
282
+ if minute.startswith("*/") and hour == "*":
283
+ try:
284
+ step = int(minute[2:])
285
+ except ValueError:
286
+ return None
287
+ return (1440.0 / step) if step > 0 else None
288
+ if minute.isdigit() and hour == "*":
289
+ return 24.0
290
+ if minute.isdigit() and hour.isdigit():
291
+ if dom.isdigit():
292
+ return 1.0 / DAYS_PER_MONTH # monthly
293
+ if dow.isdigit():
294
+ return 1.0 / 7.0 # weekly
295
+ return 1.0 # daily
296
+ return None
297
+
298
+
299
+ def _automation_rows(slug, rt):
300
+ """A tenant's automations, their schedules, and their retained run history."""
301
+ try:
302
+ import automation_engine as engine
303
+ defs = engine.all_definitions(rt) or {}
304
+ max_runs = int(getattr(engine, "MAX_RUNS", 20))
305
+ except Exception as e: # noqa: BLE001
306
+ return [], f"automations unreadable ({type(e).__name__})", 20
307
+ rows = []
308
+ for auto_id, d in sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower()):
309
+ if not isinstance(d, dict):
310
+ continue
311
+ sched = d.get("schedule") or {}
312
+ status = d.get("status") or {}
313
+ runs = list(d.get("runs") or [])
314
+ cron = sched.get("cron") or ""
315
+ per_day = _runs_per_day(cron) if sched.get("enabled") else 0.0
316
+ rows.append({
317
+ "tenant": slug,
318
+ "id": auto_id,
319
+ "name": d.get("name") or auto_id,
320
+ "kind": d.get("kind") or "",
321
+ "enabled": bool(sched.get("enabled")),
322
+ "cron": cron,
323
+ "runsPerDay": round(per_day, 2) if per_day is not None else None,
324
+ "state": status.get("state") or "idle",
325
+ "lastRunAt": status.get("lastRunAt") or "",
326
+ "lastSummary": status.get("lastSummary") or "",
327
+ # The retained window, named as such — see `_runs_per_day`'s second warning.
328
+ "runsRetained": len(runs),
329
+ "failedRetained": sum(1 for r in runs if isinstance(r, dict) and not r.get("ok")),
330
+ "createdBy": d.get("createdBy") or "",
331
+ "created": d.get("created") or "",
332
+ })
333
+ return rows, "", max_runs
334
+
335
+
336
+ def _automation_cost(auto_rows):
337
+ """The estimated monthly cost of running the automation fleet — and the honest shape of it.
338
+
339
+ THE POINT THIS BLOCK EXISTS TO MAKE, which is not intuitive: **the external cron does not
340
+ scale with automations or tenants.** One EventBridge schedule fires one Lambda, which POSTs
341
+ one tick, and that tick runs every DUE automation for every tenant (`provision_automation_cron`
342
+ states this as the reason it stays $0 "at ten tenants"). So the AWS bill is a function of the
343
+ CADENCE alone — the fleet below adds work inside the API container, which is already paid for.
344
+
345
+ WHAT IS NOT COMPUTED HERE, deliberately: Lambda GB-seconds. That needs the real average
346
+ duration of the function, which only CloudWatch knows; assuming one would be inventing the
347
+ larger half of the free-tier calculation. `/aws` reports the measured figure when credentials
348
+ are available, and this block says so rather than filling the gap with a plausible number.
349
+ """
350
+ invocations = TICK_PER_DAY * DAYS_PER_MONTH
351
+ fleet_runs = sum(r["runsPerDay"] or 0 for r in auto_rows if r.get("enabled"))
352
+ unknown_cadence = sum(1 for r in auto_rows if r.get("enabled") and r.get("runsPerDay") is None)
353
+ return {
354
+ "cadence": TICK_CADENCE,
355
+ "invocationsPerMonth": int(round(invocations)),
356
+ "freeRequestsPct": round(100.0 * invocations / FREE_LAMBDA_REQUESTS, 3),
357
+ "freeSchedulerPct": round(100.0 * invocations / FREE_SCHEDULER_INVOCATIONS, 4),
358
+ "lambdaMb": LAMBDA_MB,
359
+ "usd": 0.0,
360
+ "fleetRunsPerDay": round(fleet_runs, 2),
361
+ "unknownCadence": unknown_cadence,
362
+ "basis": (
363
+ f"One EventBridge schedule ({TICK_CADENCE}) fires one {LAMBDA_MB} MB Lambda that "
364
+ f"POSTs the tick; that ONE tick runs every due automation for every tenant, so the "
365
+ f"AWS cost is set by the cadence and does not grow with the fleet. "
366
+ f"{int(round(invocations)):,} invocations/month is "
367
+ f"{round(100.0 * invocations / FREE_LAMBDA_REQUESTS, 3)}% of the 1,000,000-request "
368
+ f"always-free tier, so the marginal cost of the automations below is $0.00. "
369
+ f"Compute (GB-seconds) depends on measured durations and is NOT estimated here — "
370
+ f"the AWS report reads the real figure when credentials are available."
371
+ ),
372
+ }
373
+
374
+
375
+ # ════════════════════════════════���═════════════════════════════════════════ the AWS report
376
+
377
+ #: `ops/aws_usage_report.py`, relative to this file: aios-web/api/ -> repo root -> ops/.
378
+ _AWS_REPORT = Path(__file__).resolve().parents[2] / "ops" / "aws_usage_report.py"
379
+ _AWS_TIMEOUT = 60
380
+
381
+
382
+ def _aws_report(days=7):
383
+ """The AWS usage report as text, or an honest block saying why there is none.
384
+
385
+ ⛔ SUBPROCESS, NEVER `import`. Two concrete reasons, both found by reading that file rather
386
+ than by running it:
387
+ * it reassigns `sys.stdout` to a UTF-8 wrapper AT MODULE IMPORT (a cp1252 fix for this
388
+ box) — importing it would mutate the API process's stdout as a side effect;
389
+ * its `main()` calls `argparse.parse_args()` with no argv, so inside a server it would
390
+ parse UVICORN's arguments and can `sys.exit(2)` — and `SystemExit` is a `BaseException`,
391
+ which `except Exception` does not catch. A route that "cannot crash" would crash.
392
+
393
+ ⚠ ON THE SPACE THIS IS THE NORMAL PATH, NOT THE ERROR PATH. `deploy_web.py` ships `api/*.py`
394
+ and `web/`; `ops/` is not in the image, and AWS credentials live in a local `.env` that is
395
+ never deployed. So the honest block below is what the deployed plane shows, and it is written
396
+ to be read by an operator as information ("run it here") rather than as a fault.
397
+ """
398
+ if not _AWS_REPORT.is_file():
399
+ return {"available": False, "text": "",
400
+ "note": ("The AWS usage report is not part of this deployment — `ops/` ships "
401
+ "with the repository, not with the container image. Run "
402
+ "`python ops/aws_usage_report.py` locally for the live figures.")}
403
+ try:
404
+ proc = subprocess.run(
405
+ [sys.executable, str(_AWS_REPORT), "--days", str(int(days))],
406
+ capture_output=True, text=True, timeout=_AWS_TIMEOUT,
407
+ cwd=str(_AWS_REPORT.parent.parent))
408
+ except subprocess.TimeoutExpired:
409
+ return {"available": False, "text": "",
410
+ "note": (f"The AWS report did not answer within {_AWS_TIMEOUT}s — CloudWatch may "
411
+ f"be unreachable from here. Nothing was assumed about usage.")}
412
+ except Exception as e: # noqa: BLE001
413
+ return {"available": False, "text": "",
414
+ "note": f"The AWS report could not be run here ({type(e).__name__})."}
415
+ out = (proc.stdout or "").strip()
416
+ if proc.returncode != 0 or not out:
417
+ detail = (proc.stderr or "").strip().splitlines()
418
+ return {"available": False, "text": out,
419
+ "note": ("AWS credentials are not available here, or boto3 is not installed — "
420
+ "no usage could be read, and none is guessed. "
421
+ + (detail[-1][:200] if detail else ""))}
422
+ return {"available": True, "text": out, "note": ""}
423
+
424
+
425
+ # ══════════════════════════════════════════════════════════════════════════ routes
426
+
427
+
428
+ @router.get("/overview")
429
+ def overview(session: Session = Depends(padmin_gate)):
430
+ """THE PLANE'S ONE TABLE: every tenant, with the counts that drill to the rows below.
431
+
432
+ Each count is produced by the same collector its `/…` route serves, so the number and its
433
+ drill-down are one computation projected twice. A subsystem that cannot be read contributes
434
+ an `errors` entry on that tenant's row and a NULL count — never a zero, which would read as
435
+ "this customer has no databases" when the truth is "we could not look".
436
+ """
437
+ t0 = time.time()
438
+ all_users = _user_rows()
439
+ users_by_tenant = {}
440
+ for u in all_users:
441
+ users_by_tenant.setdefault(u["tenant"], []).append(u)
442
+
443
+ rows = []
444
+ for t, rt in _scan():
445
+ slug = t["slug"]
446
+ row = dict(t)
447
+ row["users"] = len(users_by_tenant.get(slug, []))
448
+ row["admins"] = sum(1 for u in users_by_tenant.get(slug, []) if u["role"] == "admin")
449
+ errors = [t["error"]] if t["error"] else []
450
+ if rt is None:
451
+ # An unresolvable tenant still shows its ACCOUNTS (they live in the global registry,
452
+ # which is readable regardless) — everything tenant-store-shaped is honestly unknown.
453
+ row.update({"databases": None, "rows": None, "connectors": None,
454
+ "automations": None, "keychainLocked": None, "errors": errors})
455
+ rows.append(row)
456
+ continue
457
+ dbs, db_err = _database_rows(slug, rt)
458
+ conns, conn_err, locked = _connector_rows(slug, rt)
459
+ autos, auto_err, _mr = _automation_rows(slug, rt)
460
+ errors += [e for e in (db_err, conn_err, auto_err) if e]
461
+ row.update({
462
+ "databases": None if db_err else len(dbs),
463
+ "rows": None if db_err else sum(d["rowCount"] for d in dbs),
464
+ "connectors": None if conn_err else len(conns),
465
+ "connectorsPaused": None if conn_err else sum(1 for c in conns if c["paused"]),
466
+ "automations": None if auto_err else len(autos),
467
+ "automationsEnabled": None if auto_err else sum(1 for a in autos if a["enabled"]),
468
+ "keychainLocked": None if conn_err else locked,
469
+ "errors": errors,
470
+ })
471
+ rows.append(row)
472
+
473
+ return {
474
+ "tenants": rows,
475
+ "totals": {
476
+ "tenants": len(rows),
477
+ # EVERY account, not the sum of the rows: an account whose `tenant` names a slug this
478
+ # deployment no longer knows belongs in the platform total and would vanish from a
479
+ # per-row sum. `orphanUsers` names that gap instead of hiding it.
480
+ "users": len(all_users),
481
+ "orphanUsers": len(all_users) - sum(r["users"] for r in rows),
482
+ # Sums SKIP unknowns rather than treating them as 0, and say how many were skipped —
483
+ # a total that silently absorbs an unreadable tenant is a fabricated total.
484
+ "databases": sum(r["databases"] or 0 for r in rows),
485
+ "rows": sum(r["rows"] or 0 for r in rows),
486
+ "automations": sum(r["automations"] or 0 for r in rows),
487
+ "unknownTenants": sum(1 for r in rows if r["databases"] is None),
488
+ },
489
+ "storeAvailable": bool(_store_ok()),
490
+ # ⚠ OFFSET-BEARING, and that is not pedantry. The client renders this as "read 2 minutes
491
+ # ago" via `Date.parse`, which reads a NAIVE stamp as browser-local — so a UTC container
492
+ # and a US viewer would turn "just now" into "5 hours ago", or into a future date that
493
+ # renders as "just now" forever. The stamps written by `core.users` carry an offset for
494
+ # the same reason; anything a browser subtracts from `Date.now()` must say what zone it
495
+ # is in. `verify_api` asserts the offset so this cannot regress to `strftime`.
496
+ "generatedAt": _now_iso(),
497
+ "tookMs": int((time.time() - t0) * 1000),
498
+ }
499
+
500
+
501
+ def _store_ok():
502
+ try:
503
+ return store.available()
504
+ except Exception:
505
+ return False
506
+
507
+
508
+ @router.get("/users")
509
+ def platform_users(tenant: str = "", session: Session = Depends(padmin_gate)):
510
+ """Every account on the platform, or one tenant's — the drill behind the Users count.
511
+
512
+ R4's stamps are the columns that did not exist before this wave: `lastLogin` is written by
513
+ `routes_auth.login`, `lastActive` by `deps.require_session` (throttled to once an hour per
514
+ account per process). Absent means never seen, and the pane renders it as "never".
515
+ """
516
+ rows = _user_rows(tenant)
517
+ return {"users": rows, "count": len(rows),
518
+ "stampsNote": ("Login and activity stamps started with this release — accounts that "
519
+ "have not signed in since show no date rather than an invented one. "
520
+ "Activity is recorded at most once an hour per account.")}
521
+
522
+
523
+ @router.get("/databases")
524
+ def platform_databases(tenant: str = "", session: Session = Depends(padmin_gate)):
525
+ """Every tenant's user-created databases and row counts — the drill behind Databases/Rows."""
526
+ out, errors = [], {}
527
+ for t, rt in _scan(tenant):
528
+ slug = t["slug"]
529
+ if rt is None:
530
+ errors[slug] = t["error"]
531
+ continue
532
+ rows, err = _database_rows(slug, rt)
533
+ if err:
534
+ errors[slug] = err
535
+ continue
536
+ out += rows
537
+ return {"databases": out, "count": len(out),
538
+ "rows": sum(d["rowCount"] for d in out), "errors": errors}
539
+
540
+
541
+ @router.get("/connectors")
542
+ def platform_connectors(tenant: str = "", session: Session = Depends(padmin_gate)):
543
+ """Every tenant's data sources, which one is live, and which are paused.
544
+
545
+ METADATA ONLY — `keychain.list_entries` never decrypts, so no credential, and no masked
546
+ preview either: a platform operator needs to know a source EXISTS and whether it serves, not
547
+ what the secret looks like. The tenant's own admin surface is where previews belong.
548
+ """
549
+ out, errors, locked_any = [], {}, {}
550
+ for t, rt in _scan(tenant):
551
+ slug = t["slug"]
552
+ if rt is None:
553
+ errors[slug] = t["error"]
554
+ continue
555
+ rows, err, locked = _connector_rows(slug, rt)
556
+ if err:
557
+ errors[slug] = err
558
+ continue
559
+ locked_any[slug] = locked
560
+ out += rows
561
+ return {"connectors": out, "count": len(out), "keychainLocked": locked_any,
562
+ "errors": errors,
563
+ "note": ("A locked keychain means this container has no `AIOS_KEYCHAIN_KEY` — stored "
564
+ "credentials cannot be read, so a tenant's sources fail closed rather than "
565
+ "falling back to anyone else's.")}
566
+
567
+
568
+ @router.get("/automations")
569
+ def platform_automations(tenant: str = "", session: Session = Depends(padmin_gate)):
570
+ """Every tenant's automations, their schedules and run history, plus the fleet cost model."""
571
+ out, errors, retained = [], {}, 20
572
+ for t, rt in _scan(tenant):
573
+ slug = t["slug"]
574
+ if rt is None:
575
+ errors[slug] = t["error"]
576
+ continue
577
+ rows, err, max_runs = _automation_rows(slug, rt)
578
+ if err:
579
+ errors[slug] = err
580
+ continue
581
+ retained = max_runs
582
+ out += rows
583
+ return {"automations": out, "count": len(out),
584
+ "enabled": sum(1 for a in out if a["enabled"]),
585
+ "historyRetained": retained,
586
+ "cost": _automation_cost(out), "errors": errors,
587
+ "tickEnabled": os.environ.get("AIOS_AUTOMATIONS") == "1"}
588
+
589
+
590
+ @router.get("/aws")
591
+ def platform_aws(days: int = 7, session: Session = Depends(padmin_gate)):
592
+ """The AWS cron's usage report, verbatim, or an honest note explaining its absence."""
593
+ days = max(1, min(int(days or 7), 90))
594
+ return {"days": days, "report": _aws_report(days)}
595
+
596
+
597
+ @router.get("/releases")
598
+ def platform_releases(session: Session = Depends(padmin_gate)):
599
+ """⭐ WAVE 20 (owner item 12, ruling R6) — WHAT IS RUNNING WHERE, and what else could be.
600
+
601
+ Owner: *"Make this part of the deploy skill. Also ability to revert to any version we want.
602
+ Make sure App versioning is something that our company admin (loopable, non-tenant) can
603
+ easily see."* This is the SEEING half; promoting stays a CLI command by R6, so nothing here
604
+ writes and no web session can move production.
605
+
606
+ ⛔ THE HUB READS LIVE IN `core/releases.py`, NOT HERE. `ops/verify_portability.py` B2 forbids
607
+ the HuggingFace SDK anywhere under `aios-web/api/` — the API process is host-agnostic by
608
+ design — and it caught this endpoint's first draft doing exactly that. Same delegation shape
609
+ as `routes_assets` -> `core/assets.py`.
610
+ """
611
+ import core.releases as releases
612
+
613
+ return {"here": os.environ.get("AIOS_VERSION") or "unknown",
614
+ "environments": releases.environments(),
615
+ "releases": releases.history(),
616
+ # The runbook, in the payload, because the panel is read-only BY DESIGN and a user
617
+ # looking at it is exactly the person who needs to know how to move a version.
618
+ "promote": "python aios-web/deploy_web.py --promote=vN (or --promote=staging)"}
api/routes_products.py CHANGED
@@ -1,294 +1,294 @@
1
- """routes_products.py — the PRODUCT table's read seam (wave 15 item 9/10, contract C-TOPIC).
2
-
3
- The proof that C-TOPIC's bet is real: **a second table object is a pool module plus a route, not
4
- a second component tree**. Everything structural here is borrowed rather than re-implemented —
5
- the permission wall (`core.perm_scope`), the row builder (`aios_grid.rows_from_pool`), the
6
- scope-keyed stale-while-refresh cache (`scope_cache`), the error shape, the module gate.
7
-
8
- Landed nav-less in wave 15 ON PURPOSE (no registry row, so no user could navigate to a
9
- half-built surface while the server half hardened); wave 16 shipped the `product_data`
10
- registry row, the client `topic` prop and the write path, so the surface has been REACHABLE
11
- from the nav since — this header stayed stale until 2026-08-04 (the wave-18 debt sweep) and
12
- cost readers a false "you can't get there from here".
13
-
14
- Wave 16 (C-TOPIC's second half): the WRITE PATH exists now, and it is exactly what the wave-15
15
- header demanded before one could — its OWN table workspace bucket
16
- (`modules.product_data.TABLE_OPS`, store key 'product_table_workspace') and its own event
17
- validation (the events route builds the ctx over the PRODUCT field contract + product pids +
18
- the product ops). ⛔ The separate bucket is load-bearing, not tidy: product pids are CRC32
19
- hashes and customer pids are Odoo partner ids — one shared overlay bucket and a hash collision
20
- silently writes a product note onto somebody's customer.
21
- """
22
- import time
23
-
24
- from fastapi import Body, Depends
25
-
26
- from fastapi import APIRouter
27
-
28
- import scope_cache
29
- from deps import Session, err, module_gate
30
-
31
- router = APIRouter(prefix="/api/v1")
32
-
33
- #: The registry key this surface WILL carry. The gate is live now even though the nav is not, so
34
- #: the day the registry row appears the wall is already the one that was tested.
35
- MODULE = "product_data"
36
-
37
- _CACHE_TTL = 900
38
-
39
-
40
- def _pool_for(rt, team_id):
41
- """The cached SKU pool for one scope. Same stale-while-refresh discipline as the customer
42
- pool: only a scope's FIRST-ever build blocks, and the cache key is the SCOPE, never the user
43
- — the customer route learned that the hard way (a per-user payload cached on a scope key
44
- served one user's private columns to another)."""
45
- import modules.product_data as pd
46
-
47
- # ⚠ Signature is (cache, key, ttl, build) — the wave-15 version passed `ttl=` as a keyword
48
- # after the build lambda and DIED on every call ("multiple values for 'ttl'"). Latent until
49
- # wave 16 because nothing drove this route end-to-end: verify_perm_scope's section H stubs
50
- # the pool a layer below. verify_api's W16 section now exercises the real call.
51
- key = ("product_pool", team_id)
52
-
53
- # ⭐ D-10 (wave 24) — THE PRODUCT POOL HONOURS THE CONNECTOR PAUSE. It did not, and that was
54
- # a shipped defect on a live surface: the CUSTOMER pool got this guard at DEBT-2 and its
55
- # sibling — written from the same template, four files away — never did. So pausing Odoo
56
- # froze the Customer grid and the Product grid kept reading live, with the Settings toggle
57
- # reporting success. Sibling caches diverge exactly this way and nothing greps for
58
- # "the other one".
59
- #
60
- # ⚠ NO SNAPSHOT LEG, and that is the honest difference from the customer path rather than an
61
- # omission: `save_pool_snapshots` persists the `odoo_pool` scopes only — there is no product
62
- # snapshot bucket to serve. So this serves the in-process copy AT ANY AGE, and answers 503
63
- # naming the pause when there is none. Minting a product snapshot here would be a new
64
- # persistence format at a wave tail; refusing with a sentence is the answer the user can act
65
- # on, and it is the same 503 the customer path already gives for an unsnapshotted scope.
66
- import routes_keychain
67
- if routes_keychain.odoo_paused(rt):
68
- hit = rt.pool_cache.get(key)
69
- if hit:
70
- return hit[1]
71
- raise err(503, "connector_paused",
72
- "this data source is paused and nothing has been read since — "
73
- "an admin can resume it under Settings → Connectors")
74
-
75
- return scope_cache.get(rt.pool_cache, key, _CACHE_TTL,
76
- lambda: pd.pool(team_id=team_id))
77
-
78
-
79
- def scoped_pool(session: Session):
80
- """`(pids, team_id, rows_src, fields_base)` — THE PRODUCT WALL, on its own.
81
-
82
- Extracted from `product_assembly` (wave 19, item 12) so a caller that needs only "which
83
- products may this session touch" — record comments, say — asks the SAME question in the same
84
- order rather than re-deriving it: pool scope from the permanent filter, rows, THEN the row
85
- wall, THEN the pids. Re-deriving it is how a second wall drifts from the first, and the
86
- walls are the whole point of this route.
87
-
88
- ⚠ The caller is responsible for the GRANT (`session.require(MODULE)`); this is the row half.
89
- """
90
- import core.perm_scope as perm_scope
91
-
92
- team_id, _agent = perm_scope.derive_pool_scope(session.user, MODULE)
93
- try:
94
- rows_src = _pool_for(session.runtime, team_id)
95
- except Exception as e:
96
- raise err(503, "pool_unavailable",
97
- f"the product catalogue could not be built — {str(e)[:160]}")
98
- fields_base = pd_fields(consolidated=team_id is None)
99
- # The SAME wall the customer assembly applies, in the same order: rows first (before pids
100
- # are taken, so an out-of-filter row never enters allowed_pids), then the field closure,
101
- # then the values stripped from the rows as well as the field list.
102
- rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, fields_base)
103
- pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
104
- return pids, team_id, rows_src, fields_base
105
-
106
-
107
- def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
108
- consume_corrections: bool = True):
109
- """The product topic's mirror of `routes_customers.grid_assembly` — SAME g-dict keys, so
110
- the /workspace and events routes consume either interchangeably.
111
-
112
- One deliberate absence, a topic fact rather than a gap:
113
- * `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is CUSTOMER-grain (the
114
- C-TOPIC v1 descope, booked in the wave doc); the events ctx therefore refuses measure
115
- creates on this surface, which is the correct fail-closed shape.
116
-
117
- ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. Wave 16 passed `with_cohorts=False` because
118
- there was one customer-keyed cohort bucket and product pids are CRC32 hashes of SKU codes;
119
- intersecting the two id spaces would have printed a plausible, meaningless member count. The
120
- owner's ruling is that a cohort belongs to its database, so `modules.cohort` grew a bucket
121
- per topic and the product surface reads `product_cohorts` — ids from this pool, resolved
122
- against this pool. `derived` carries their membership cells (the Cohorts column) for the same
123
- reason it does on the customer surface; the measure half of that channel stays empty.
124
- """
125
- import aios_grid
126
- import core.perm_scope as perm_scope
127
- import modules.product_data as pd
128
- from core import grid_events
129
-
130
- pids, team_id, rows_src, fields_base = scoped_pool(session)
131
-
132
- ctx = grid_events.EventCtx(
133
- uname=session.uname, allowed_pids=pids, fields=[],
134
- hidden_keys=perm_scope.hidden_keys(session.user, MODULE, fields_base),
135
- admin=session.admin, fallback_ws=None, seen_ids={},
136
- scope_key="product", table=pd.TABLE_OPS)
137
- ws = grid_events.table_workspace(ctx, allowed_pids=pids,
138
- consume_corrections=consume_corrections)
139
- workspace, fields, views, lists = aios_grid.workspace_wire(
140
- ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key,
141
- fields_base=fields_base)
142
-
143
- hidden = perm_scope.hidden_keys(session.user, MODULE, fields)
144
- if hidden:
145
- fields = [f for f in fields if f.get("key") not in hidden]
146
- rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src]
147
-
148
- return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
149
- "fields": fields, "views": views, "lists": lists,
150
- # R9: the Cohorts column's cells, built from THIS topic's lists. Same read-only
151
- # `derived` channel the customer assembly uses — the measure half stays empty.
152
- "derived": aios_grid.cohort_cells(lists),
153
- "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"),
154
- "team_id": team_id}
155
-
156
-
157
- @router.get("/products")
158
- def products(session: Session = Depends(module_gate(MODULE))):
159
- """The product table for this session's scope — the /customers envelope byte-for-byte
160
- (`{fields, rows, today, pulled_at}`) plus two additive keys (`identity`, `scope`).
161
-
162
- The BU scope is DERIVED FROM THE PERMANENT FILTER (`perm_scope.derive_pool_scope`), exactly
163
- as the customer route derives it — and for the same reason, which is worth restating because
164
- it is the wave's central lesson: `team_id` shapes the revenue VALUES on each row, so a BU
165
- enforced as a post-filter yields a correct row list carrying both units' numbers.
166
- """
167
- import aios_grid
168
-
169
- g = product_assembly(session)
170
- rows = _seed_image(aios_grid.rows_from_pool(
171
- g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]),
172
- g["fields"])
173
- return {"fields": g["fields"], "rows": rows,
174
- "today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
175
- "identity": {"pid": "pid", "businessKey": "code"},
176
- "scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}}
177
-
178
-
179
- @router.patch("/products/{pid}")
180
- def patch_product(pid: int, body: dict = Body(default=None),
181
- session: Session = Depends(module_gate(MODULE))):
182
- """Write the product table's EDITABLE overlay stratum — the customers PATCH, on the product
183
- topic's ctx. Routed through `core.grid_events.handle_one` so the per-key `permissions.edit`
184
- wall, the pid wall and the truncation rules stay ONE implementation; the ctx's `table` ops
185
- aim the write at the PRODUCT bucket."""
186
- import modules.product_data as pd
187
- from core import grid_events
188
-
189
- updates = dict(body or {})
190
- if not updates:
191
- raise err(400, "empty_patch", "no fields to update")
192
- g = product_assembly(session, consume_corrections=False)
193
- if pid not in g["pids"]:
194
- # 403, not 404 — the code may exist; it is simply not in this session's catalogue.
195
- raise err(403, "out_of_scope", "that product is not in your catalogue")
196
- import core.perm_scope as perm_scope
197
-
198
- ctx = grid_events.EventCtx(
199
- uname=session.uname, allowed_pids=g["pids"], fields=g["fields"],
200
- admin=session.admin, fallback_ws=None, seen_ids={},
201
- hidden_keys=perm_scope.hidden_keys(
202
- session.user, MODULE, pd_fields(consolidated=g["team_id"] is None)),
203
- scope_key="product", table=pd.TABLE_OPS)
204
- try:
205
- grid_events.handle_one(
206
- {"id": f"patch:product:{pid}:{time.time_ns()}", "type": "overlay_patch",
207
- "pid": pid, "updates": updates}, ctx)
208
- except grid_events.StoreUnavailable:
209
- raise err(503, "store_unavailable",
210
- "the tenant store is unavailable — your change was not saved")
211
-
212
- # What actually landed, read back from the PRODUCT bucket rather than echoed from the
213
- # request: a refused key or a truncated value must not be reported as accepted.
214
- stored = (grid_events.table_workspace(ctx, allowed_pids=None)
215
- .get("overlays") or {}).get(str(pid)) or {}
216
- accepted = {k: stored.get(k) for k in updates if k in stored}
217
- refused = sorted(k for k in updates
218
- if k not in accepted or stored.get(k) != str(updates[k]))
219
- out = {"pid": pid, "updates": accepted}
220
- if refused:
221
- out["refused"] = refused
222
- return out
223
-
224
-
225
- #: ⭐ WAVE 19 R7 — the RI Product table's built-in picture column. `source: 'overlay'` (editable,
226
- #: so a user can upload a different shot for one SKU) with its DEFAULT supplied per render from
227
- #: the row's `code` — see `_seed_image`. A stamped overlay value would have been the other way to
228
- #: do it and it is the wrong one twice over: it is a 1,142-row migration that has to be right
229
- #: once, and it FREEZES the code into the cell, so a re-coded SKU would keep pointing at the old
230
- #: master with nothing saying why.
231
- PRODUCT_IMAGE_KEY = "image"
232
-
233
-
234
- def _seed_image(rows, fields):
235
- """R7's "auto-seeded from SKU `code`", as a per-render DEFAULT rather than stored data.
236
-
237
- Royal's 1,142 masters are named for `default_code`, so an untouched product row already names
238
- its own picture; this is what makes them appear with nothing uploaded. A user's own value is
239
- NON-EMPTY and therefore wins — the fallback only fills a cell nobody has set.
240
-
241
- ⚠ Clearing the cell restores the SKU's own picture rather than blanking it, and that is the
242
- documented meaning of empty on this column ("no override"). A product with no master on file
243
- still shows an empty frame, because the reference resolves to a 404 — the honest outcome, and
244
- the record modal names the failing reference in words.
245
-
246
- ⛔ GATED ON THE SERVED FIELD LIST, and that is C-PERM, not tidiness. `product_assembly` strips
247
- a hidden field from BOTH wires — the field list and the row payload — because narrowing only
248
- the first leaves the value sitting in the second where anything can read it. An unconditional
249
- seed would put the key straight back onto every row AFTER that strip, re-creating exactly the
250
- shape the rule forbids. (The value here is the visible `code`, so nothing new escapes today;
251
- the contract is the point, and a future non-code default would escape.)
252
-
253
- ⚠ STATED CONSEQUENCE: hiding `code` blanks this column, because the reference IS the code.
254
- That coupling is inherent to seeding from a business key, not a bug — and it fails in the safe
255
- direction (an empty frame, never another row's picture).
256
- """
257
- if not any(f.get("key") == PRODUCT_IMAGE_KEY for f in fields or ()):
258
- return rows
259
- for row in rows:
260
- if not row.get(PRODUCT_IMAGE_KEY):
261
- row[PRODUCT_IMAGE_KEY] = row.get("code") or ""
262
- return rows
263
-
264
-
265
- def pd_fields(consolidated=True):
266
- """The product field contract, minus the CONSOLIDATED-only columns when the caller is scoped.
267
-
268
- ⛔ The omission is the SCOPE RULE, not a display preference (see `modules/product_data`
269
- decision 2): `inventory.sku_inventory` is brand-independent, so serving those columns to a
270
- BU-scoped caller would put company-wide stock beside BU-shaped revenue in one row. A column
271
- that is absent asks a question; one that is silently company-wide answers a different one.
272
- """
273
- import json
274
- from pathlib import Path
275
-
276
- import aios_grid
277
-
278
- doc = json.loads((Path(aios_grid.__file__).resolve().parent /
279
- "aios_grid_fields.json").read_text(encoding="utf-8"))
280
- fields = list((doc.get("product_data") or {}).get("fields") or [])
281
- # R7: the Image column is BUILT IN on this table, injected here rather than added to
282
- # `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a
283
- # column `modules.product_data.pool()` reads off a SKU — and this one is neither read from
284
- # Odoo nor written to it. Injected after the JSON so the canonical file stays the answer to
285
- # "what does Odoo give us", which is the question `verify_fields_contract` referees.
286
- fields = fields + [{
287
- "key": PRODUCT_IMAGE_KEY, "label": "Image", "type": "image", "source": "overlay",
288
- "note": "The product's picture. Empty shows the SKU's own master image; upload one "
289
- "from the record panel to override it.",
290
- }]
291
- if consolidated:
292
- return fields
293
- import modules.product_data as pd
294
- return [f for f in fields if f.get("key") not in pd.CONSOLIDATED_ONLY]
 
1
+ """routes_products.py — the PRODUCT table's read seam (wave 15 item 9/10, contract C-TOPIC).
2
+
3
+ The proof that C-TOPIC's bet is real: **a second table object is a pool module plus a route, not
4
+ a second component tree**. Everything structural here is borrowed rather than re-implemented —
5
+ the permission wall (`core.perm_scope`), the row builder (`aios_grid.rows_from_pool`), the
6
+ scope-keyed stale-while-refresh cache (`scope_cache`), the error shape, the module gate.
7
+
8
+ Landed nav-less in wave 15 ON PURPOSE (no registry row, so no user could navigate to a
9
+ half-built surface while the server half hardened); wave 16 shipped the `product_data`
10
+ registry row, the client `topic` prop and the write path, so the surface has been REACHABLE
11
+ from the nav since — this header stayed stale until 2026-08-04 (the wave-18 debt sweep) and
12
+ cost readers a false "you can't get there from here".
13
+
14
+ Wave 16 (C-TOPIC's second half): the WRITE PATH exists now, and it is exactly what the wave-15
15
+ header demanded before one could — its OWN table workspace bucket
16
+ (`modules.product_data.TABLE_OPS`, store key 'product_table_workspace') and its own event
17
+ validation (the events route builds the ctx over the PRODUCT field contract + product pids +
18
+ the product ops). ⛔ The separate bucket is load-bearing, not tidy: product pids are CRC32
19
+ hashes and customer pids are Odoo partner ids — one shared overlay bucket and a hash collision
20
+ silently writes a product note onto somebody's customer.
21
+ """
22
+ import time
23
+
24
+ from fastapi import Body, Depends
25
+
26
+ from fastapi import APIRouter
27
+
28
+ import scope_cache
29
+ from deps import Session, err, module_gate
30
+
31
+ router = APIRouter(prefix="/api/v1")
32
+
33
+ #: The registry key this surface WILL carry. The gate is live now even though the nav is not, so
34
+ #: the day the registry row appears the wall is already the one that was tested.
35
+ MODULE = "product_data"
36
+
37
+ _CACHE_TTL = 900
38
+
39
+
40
+ def _pool_for(rt, team_id):
41
+ """The cached SKU pool for one scope. Same stale-while-refresh discipline as the customer
42
+ pool: only a scope's FIRST-ever build blocks, and the cache key is the SCOPE, never the user
43
+ — the customer route learned that the hard way (a per-user payload cached on a scope key
44
+ served one user's private columns to another)."""
45
+ import modules.product_data as pd
46
+
47
+ # ⚠ Signature is (cache, key, ttl, build) — the wave-15 version passed `ttl=` as a keyword
48
+ # after the build lambda and DIED on every call ("multiple values for 'ttl'"). Latent until
49
+ # wave 16 because nothing drove this route end-to-end: verify_perm_scope's section H stubs
50
+ # the pool a layer below. verify_api's W16 section now exercises the real call.
51
+ key = ("product_pool", team_id)
52
+
53
+ # ⭐ D-10 (wave 24) — THE PRODUCT POOL HONOURS THE CONNECTOR PAUSE. It did not, and that was
54
+ # a shipped defect on a live surface: the CUSTOMER pool got this guard at DEBT-2 and its
55
+ # sibling — written from the same template, four files away — never did. So pausing Odoo
56
+ # froze the Customer grid and the Product grid kept reading live, with the Settings toggle
57
+ # reporting success. Sibling caches diverge exactly this way and nothing greps for
58
+ # "the other one".
59
+ #
60
+ # ⚠ NO SNAPSHOT LEG, and that is the honest difference from the customer path rather than an
61
+ # omission: `save_pool_snapshots` persists the `odoo_pool` scopes only — there is no product
62
+ # snapshot bucket to serve. So this serves the in-process copy AT ANY AGE, and answers 503
63
+ # naming the pause when there is none. Minting a product snapshot here would be a new
64
+ # persistence format at a wave tail; refusing with a sentence is the answer the user can act
65
+ # on, and it is the same 503 the customer path already gives for an unsnapshotted scope.
66
+ import routes_keychain
67
+ if routes_keychain.odoo_paused(rt):
68
+ hit = rt.pool_cache.get(key)
69
+ if hit:
70
+ return hit[1]
71
+ raise err(503, "connector_paused",
72
+ "this data source is paused and nothing has been read since — "
73
+ "an admin can resume it under Settings → Connectors")
74
+
75
+ return scope_cache.get(rt.pool_cache, key, _CACHE_TTL,
76
+ lambda: pd.pool(team_id=team_id))
77
+
78
+
79
+ def scoped_pool(session: Session):
80
+ """`(pids, team_id, rows_src, fields_base)` — THE PRODUCT WALL, on its own.
81
+
82
+ Extracted from `product_assembly` (wave 19, item 12) so a caller that needs only "which
83
+ products may this session touch" — record comments, say — asks the SAME question in the same
84
+ order rather than re-deriving it: pool scope from the permanent filter, rows, THEN the row
85
+ wall, THEN the pids. Re-deriving it is how a second wall drifts from the first, and the
86
+ walls are the whole point of this route.
87
+
88
+ ⚠ The caller is responsible for the GRANT (`session.require(MODULE)`); this is the row half.
89
+ """
90
+ import core.perm_scope as perm_scope
91
+
92
+ team_id, _agent = perm_scope.derive_pool_scope(session.user, MODULE)
93
+ try:
94
+ rows_src = _pool_for(session.runtime, team_id)
95
+ except Exception as e:
96
+ raise err(503, "pool_unavailable",
97
+ f"the product catalogue could not be built — {str(e)[:160]}")
98
+ fields_base = pd_fields(consolidated=team_id is None)
99
+ # The SAME wall the customer assembly applies, in the same order: rows first (before pids
100
+ # are taken, so an out-of-filter row never enters allowed_pids), then the field closure,
101
+ # then the values stripped from the rows as well as the field list.
102
+ rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, fields_base)
103
+ pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None)
104
+ return pids, team_id, rows_src, fields_base
105
+
106
+
107
+ def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
108
+ consume_corrections: bool = True):
109
+ """The product topic's mirror of `routes_customers.grid_assembly` — SAME g-dict keys, so
110
+ the /workspace and events routes consume either interchangeably.
111
+
112
+ One deliberate absence, a topic fact rather than a gap:
113
+ * `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is CUSTOMER-grain (the
114
+ C-TOPIC v1 descope, booked in the wave doc); the events ctx therefore refuses measure
115
+ creates on this surface, which is the correct fail-closed shape.
116
+
117
+ ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. Wave 16 passed `with_cohorts=False` because
118
+ there was one customer-keyed cohort bucket and product pids are CRC32 hashes of SKU codes;
119
+ intersecting the two id spaces would have printed a plausible, meaningless member count. The
120
+ owner's ruling is that a cohort belongs to its database, so `modules.cohort` grew a bucket
121
+ per topic and the product surface reads `product_cohorts` — ids from this pool, resolved
122
+ against this pool. `derived` carries their membership cells (the Cohorts column) for the same
123
+ reason it does on the customer surface; the measure half of that channel stays empty.
124
+ """
125
+ import aios_grid
126
+ import core.perm_scope as perm_scope
127
+ import modules.product_data as pd
128
+ from core import grid_events
129
+
130
+ pids, team_id, rows_src, fields_base = scoped_pool(session)
131
+
132
+ ctx = grid_events.EventCtx(
133
+ uname=session.uname, allowed_pids=pids, fields=[],
134
+ hidden_keys=perm_scope.hidden_keys(session.user, MODULE, fields_base),
135
+ admin=session.admin, fallback_ws=None, seen_ids={},
136
+ scope_key="product", table=pd.TABLE_OPS)
137
+ ws = grid_events.table_workspace(ctx, allowed_pids=pids,
138
+ consume_corrections=consume_corrections)
139
+ workspace, fields, views, lists = aios_grid.workspace_wire(
140
+ ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key,
141
+ fields_base=fields_base)
142
+
143
+ hidden = perm_scope.hidden_keys(session.user, MODULE, fields)
144
+ if hidden:
145
+ fields = [f for f in fields if f.get("key") not in hidden]
146
+ rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src]
147
+
148
+ return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace,
149
+ "fields": fields, "views": views, "lists": lists,
150
+ # R9: the Cohorts column's cells, built from THIS topic's lists. Same read-only
151
+ # `derived` channel the customer assembly uses — the measure half stays empty.
152
+ "derived": aios_grid.cohort_cells(lists),
153
+ "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"),
154
+ "team_id": team_id}
155
+
156
+
157
+ @router.get("/products")
158
+ def products(session: Session = Depends(module_gate(MODULE))):
159
+ """The product table for this session's scope — the /customers envelope byte-for-byte
160
+ (`{fields, rows, today, pulled_at}`) plus two additive keys (`identity`, `scope`).
161
+
162
+ The BU scope is DERIVED FROM THE PERMANENT FILTER (`perm_scope.derive_pool_scope`), exactly
163
+ as the customer route derives it — and for the same reason, which is worth restating because
164
+ it is the wave's central lesson: `team_id` shapes the revenue VALUES on each row, so a BU
165
+ enforced as a post-filter yields a correct row list carrying both units' numbers.
166
+ """
167
+ import aios_grid
168
+
169
+ g = product_assembly(session)
170
+ rows = _seed_image(aios_grid.rows_from_pool(
171
+ g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]),
172
+ g["fields"])
173
+ return {"fields": g["fields"], "rows": rows,
174
+ "today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
175
+ "identity": {"pid": "pid", "businessKey": "code"},
176
+ "scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}}
177
+
178
+
179
+ @router.patch("/products/{pid}")
180
+ def patch_product(pid: int, body: dict = Body(default=None),
181
+ session: Session = Depends(module_gate(MODULE))):
182
+ """Write the product table's EDITABLE overlay stratum — the customers PATCH, on the product
183
+ topic's ctx. Routed through `core.grid_events.handle_one` so the per-key `permissions.edit`
184
+ wall, the pid wall and the truncation rules stay ONE implementation; the ctx's `table` ops
185
+ aim the write at the PRODUCT bucket."""
186
+ import modules.product_data as pd
187
+ from core import grid_events
188
+
189
+ updates = dict(body or {})
190
+ if not updates:
191
+ raise err(400, "empty_patch", "no fields to update")
192
+ g = product_assembly(session, consume_corrections=False)
193
+ if pid not in g["pids"]:
194
+ # 403, not 404 — the code may exist; it is simply not in this session's catalogue.
195
+ raise err(403, "out_of_scope", "that product is not in your catalogue")
196
+ import core.perm_scope as perm_scope
197
+
198
+ ctx = grid_events.EventCtx(
199
+ uname=session.uname, allowed_pids=g["pids"], fields=g["fields"],
200
+ admin=session.admin, fallback_ws=None, seen_ids={},
201
+ hidden_keys=perm_scope.hidden_keys(
202
+ session.user, MODULE, pd_fields(consolidated=g["team_id"] is None)),
203
+ scope_key="product", table=pd.TABLE_OPS)
204
+ try:
205
+ grid_events.handle_one(
206
+ {"id": f"patch:product:{pid}:{time.time_ns()}", "type": "overlay_patch",
207
+ "pid": pid, "updates": updates}, ctx)
208
+ except grid_events.StoreUnavailable:
209
+ raise err(503, "store_unavailable",
210
+ "the tenant store is unavailable — your change was not saved")
211
+
212
+ # What actually landed, read back from the PRODUCT bucket rather than echoed from the
213
+ # request: a refused key or a truncated value must not be reported as accepted.
214
+ stored = (grid_events.table_workspace(ctx, allowed_pids=None)
215
+ .get("overlays") or {}).get(str(pid)) or {}
216
+ accepted = {k: stored.get(k) for k in updates if k in stored}
217
+ refused = sorted(k for k in updates
218
+ if k not in accepted or stored.get(k) != str(updates[k]))
219
+ out = {"pid": pid, "updates": accepted}
220
+ if refused:
221
+ out["refused"] = refused
222
+ return out
223
+
224
+
225
+ #: ⭐ WAVE 19 R7 — the RI Product table's built-in picture column. `source: 'overlay'` (editable,
226
+ #: so a user can upload a different shot for one SKU) with its DEFAULT supplied per render from
227
+ #: the row's `code` — see `_seed_image`. A stamped overlay value would have been the other way to
228
+ #: do it and it is the wrong one twice over: it is a 1,142-row migration that has to be right
229
+ #: once, and it FREEZES the code into the cell, so a re-coded SKU would keep pointing at the old
230
+ #: master with nothing saying why.
231
+ PRODUCT_IMAGE_KEY = "image"
232
+
233
+
234
+ def _seed_image(rows, fields):
235
+ """R7's "auto-seeded from SKU `code`", as a per-render DEFAULT rather than stored data.
236
+
237
+ Royal's 1,142 masters are named for `default_code`, so an untouched product row already names
238
+ its own picture; this is what makes them appear with nothing uploaded. A user's own value is
239
+ NON-EMPTY and therefore wins — the fallback only fills a cell nobody has set.
240
+
241
+ ⚠ Clearing the cell restores the SKU's own picture rather than blanking it, and that is the
242
+ documented meaning of empty on this column ("no override"). A product with no master on file
243
+ still shows an empty frame, because the reference resolves to a 404 — the honest outcome, and
244
+ the record modal names the failing reference in words.
245
+
246
+ ⛔ GATED ON THE SERVED FIELD LIST, and that is C-PERM, not tidiness. `product_assembly` strips
247
+ a hidden field from BOTH wires — the field list and the row payload — because narrowing only
248
+ the first leaves the value sitting in the second where anything can read it. An unconditional
249
+ seed would put the key straight back onto every row AFTER that strip, re-creating exactly the
250
+ shape the rule forbids. (The value here is the visible `code`, so nothing new escapes today;
251
+ the contract is the point, and a future non-code default would escape.)
252
+
253
+ ⚠ STATED CONSEQUENCE: hiding `code` blanks this column, because the reference IS the code.
254
+ That coupling is inherent to seeding from a business key, not a bug — and it fails in the safe
255
+ direction (an empty frame, never another row's picture).
256
+ """
257
+ if not any(f.get("key") == PRODUCT_IMAGE_KEY for f in fields or ()):
258
+ return rows
259
+ for row in rows:
260
+ if not row.get(PRODUCT_IMAGE_KEY):
261
+ row[PRODUCT_IMAGE_KEY] = row.get("code") or ""
262
+ return rows
263
+
264
+
265
+ def pd_fields(consolidated=True):
266
+ """The product field contract, minus the CONSOLIDATED-only columns when the caller is scoped.
267
+
268
+ ⛔ The omission is the SCOPE RULE, not a display preference (see `modules/product_data`
269
+ decision 2): `inventory.sku_inventory` is brand-independent, so serving those columns to a
270
+ BU-scoped caller would put company-wide stock beside BU-shaped revenue in one row. A column
271
+ that is absent asks a question; one that is silently company-wide answers a different one.
272
+ """
273
+ import json
274
+ from pathlib import Path
275
+
276
+ import aios_grid
277
+
278
+ doc = json.loads((Path(aios_grid.__file__).resolve().parent /
279
+ "aios_grid_fields.json").read_text(encoding="utf-8"))
280
+ fields = list((doc.get("product_data") or {}).get("fields") or [])
281
+ # R7: the Image column is BUILT IN on this table, injected here rather than added to
282
+ # `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a
283
+ # column `modules.product_data.pool()` reads off a SKU — and this one is neither read from
284
+ # Odoo nor written to it. Injected after the JSON so the canonical file stays the answer to
285
+ # "what does Odoo give us", which is the question `verify_fields_contract` referees.
286
+ fields = fields + [{
287
+ "key": PRODUCT_IMAGE_KEY, "label": "Image", "type": "image", "source": "overlay",
288
+ "note": "The product's picture. Empty shows the SKU's own master image; upload one "
289
+ "from the record panel to override it.",
290
+ }]
291
+ if consolidated:
292
+ return fields
293
+ import modules.product_data as pd
294
+ return [f for f in fields if f.get("key") not in pd.CONSOLIDATED_ONLY]
platform/aios_grid.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/core/grid_events.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/core/links.py CHANGED
@@ -1,59 +1,59 @@
1
- """Deep links into the app — ?page=<key>&entity=<kind>:<id>&src=<source>.
2
-
3
- The app (app.py) consumes these params once per session right after login and routes to the page,
4
- opening the entity drawer when an entity is present. Digest/alert/mention emails build their links
5
- here so every emailed number lands the reader on the exact row it came from. `src` tags where the
6
- click came from (digest / alert / mention) — the app logs it as the click-through read receipt.
7
- """
8
- import os
9
- import sys
10
- from urllib.parse import quote
11
-
12
- #: ⛔ THE ONE HOST LITERAL LEFT IN THE RUNTIME PATH, and it is a MIGRATION HAZARD — named here
13
- #: rather than buried in a default argument, because the day we change hosts this is what silently
14
- #: keeps mailing customers a link to the DEAD one. Found by `ops/verify_portability.py` on its
15
- #: first run (2026-07-30).
16
- #:
17
- #: Why it is still here instead of raising: `deeplink()` has exactly one family of callers
18
- #: (`modules/digest.py`, 7 sites, all building email hrefs), `DIGEST_ENABLED` defaults to ON, and
19
- #: no deploy sets `APP_BASE_URL` today — so refusing to guess would break a LIVE feature to fix a
20
- #: latent one. Instead: both deploy scripts now push `APP_BASE_URL`, which makes the fallback
21
- #: unreachable in a deployed environment, and reaching it prints an operator warning.
22
- #:
23
- #: ⇒ EXIT CONDITION for this constant: once a deploy has set `APP_BASE_URL` and it is confirmed
24
- #: present in the running environment, delete the fallback and let `deeplink()` raise instead. It
25
- #: is declared in the portability gate's RESIDUALS so the gate reports it honestly rather than
26
- #: reporting green over it.
27
- _LEGACY_HF_FALLBACK = 'https://royal-imports-cfo-os.hf.space'
28
-
29
- BASE_URL = (os.environ.get('APP_BASE_URL') or _LEGACY_HF_FALLBACK).rstrip('/')
30
-
31
- #: Warned LAZILY and ONCE — at the first link actually built, not at import. An import-time print
32
- #: fires in every gate, every subprocess and every unit test that so much as touches `core`, which
33
- #: is how a real warning becomes noise nobody reads.
34
- _WARNED = [False]
35
-
36
-
37
- def _warn_once_if_defaulted():
38
- if _WARNED[0] or os.environ.get('APP_BASE_URL'):
39
- return
40
- _WARNED[0] = True
41
- print(f"[links] APP_BASE_URL is not set - emailed deep links will point at "
42
- f"{_LEGACY_HF_FALLBACK}, which is correct ONLY while that is where the app lives. "
43
- f"Set APP_BASE_URL in the deployed environment.", file=sys.stderr)
44
-
45
- # entity kinds the URL scheme supports (must stay in sync with app.py's _desc_from_param)
46
- LINK_KINDS = ('customer', 'sku', 'agent', 'account')
47
-
48
-
49
- def deeplink(page=None, kind=None, ident=None, src=None):
50
- """An absolute app URL. deeplink('customer_data', 'customer', 1234, src='digest')."""
51
- _warn_once_if_defaulted()
52
- parts = []
53
- if page:
54
- parts.append(f"page={quote(str(page), safe='')}")
55
- if kind in LINK_KINDS and ident not in (None, ''):
56
- parts.append(f"entity={quote(f'{kind}:{ident}', safe=':')}")
57
- if src:
58
- parts.append(f"src={quote(str(src), safe='')}")
59
- return BASE_URL + ('/?' + '&'.join(parts) if parts else '/')
 
1
+ """Deep links into the app — ?page=<key>&entity=<kind>:<id>&src=<source>.
2
+
3
+ The app (app.py) consumes these params once per session right after login and routes to the page,
4
+ opening the entity drawer when an entity is present. Digest/alert/mention emails build their links
5
+ here so every emailed number lands the reader on the exact row it came from. `src` tags where the
6
+ click came from (digest / alert / mention) — the app logs it as the click-through read receipt.
7
+ """
8
+ import os
9
+ import sys
10
+ from urllib.parse import quote
11
+
12
+ #: ⛔ THE ONE HOST LITERAL LEFT IN THE RUNTIME PATH, and it is a MIGRATION HAZARD — named here
13
+ #: rather than buried in a default argument, because the day we change hosts this is what silently
14
+ #: keeps mailing customers a link to the DEAD one. Found by `ops/verify_portability.py` on its
15
+ #: first run (2026-07-30).
16
+ #:
17
+ #: Why it is still here instead of raising: `deeplink()` has exactly one family of callers
18
+ #: (`modules/digest.py`, 7 sites, all building email hrefs), `DIGEST_ENABLED` defaults to ON, and
19
+ #: no deploy sets `APP_BASE_URL` today — so refusing to guess would break a LIVE feature to fix a
20
+ #: latent one. Instead: both deploy scripts now push `APP_BASE_URL`, which makes the fallback
21
+ #: unreachable in a deployed environment, and reaching it prints an operator warning.
22
+ #:
23
+ #: ⇒ EXIT CONDITION for this constant: once a deploy has set `APP_BASE_URL` and it is confirmed
24
+ #: present in the running environment, delete the fallback and let `deeplink()` raise instead. It
25
+ #: is declared in the portability gate's RESIDUALS so the gate reports it honestly rather than
26
+ #: reporting green over it.
27
+ _LEGACY_HF_FALLBACK = 'https://royal-imports-cfo-os.hf.space'
28
+
29
+ BASE_URL = (os.environ.get('APP_BASE_URL') or _LEGACY_HF_FALLBACK).rstrip('/')
30
+
31
+ #: Warned LAZILY and ONCE — at the first link actually built, not at import. An import-time print
32
+ #: fires in every gate, every subprocess and every unit test that so much as touches `core`, which
33
+ #: is how a real warning becomes noise nobody reads.
34
+ _WARNED = [False]
35
+
36
+
37
+ def _warn_once_if_defaulted():
38
+ if _WARNED[0] or os.environ.get('APP_BASE_URL'):
39
+ return
40
+ _WARNED[0] = True
41
+ print(f"[links] APP_BASE_URL is not set - emailed deep links will point at "
42
+ f"{_LEGACY_HF_FALLBACK}, which is correct ONLY while that is where the app lives. "
43
+ f"Set APP_BASE_URL in the deployed environment.", file=sys.stderr)
44
+
45
+ # entity kinds the URL scheme supports (must stay in sync with app.py's _desc_from_param)
46
+ LINK_KINDS = ('customer', 'sku', 'agent', 'account')
47
+
48
+
49
+ def deeplink(page=None, kind=None, ident=None, src=None):
50
+ """An absolute app URL. deeplink('customer_data', 'customer', 1234, src='digest')."""
51
+ _warn_once_if_defaulted()
52
+ parts = []
53
+ if page:
54
+ parts.append(f"page={quote(str(page), safe='')}")
55
+ if kind in LINK_KINDS and ident not in (None, ''):
56
+ parts.append(f"entity={quote(f'{kind}:{ident}', safe=':')}")
57
+ if src:
58
+ parts.append(f"src={quote(str(src), safe='')}")
59
+ return BASE_URL + ('/?' + '&'.join(parts) if parts else '/')
platform/core/registry.py CHANGED
@@ -1,199 +1,199 @@
1
- """Module registry — the single source of truth for the dashboards.
2
-
3
- `app.py` builds the sidebar nav from this, and decides whether the global Brand (DBA) filter
4
- applies to a page. `validate.py` runs each module's validate() from this. Adding a module =
5
- add one row here + a page function in app.py + a module in modules/.
6
-
7
- Fields:
8
- key stable id (matches the modules/<key>.py file and the app page_<key> function)
9
- label sidebar/title text (no emojis)
10
- brand True -> DBA-filterable (honours the Fisch/Royal selector)
11
- False -> consolidated/company-level (ignores the brand selector)
12
- hq True -> HQ / company-level (cash flow, balance sheet, AR/AP, consolidated)
13
- validate True -> module exposes validate() and is included in validate.py
14
- parent set -> this row is a SUB-MODULE of <parent key>: it leaves the Workflows dropdown and
15
- renders as a sub-nav button when its parent (or a sibling) is the active module; a
16
- parent grant covers its sub-modules (owner nav redesign 2026-07-23)
17
- archived True -> RETIRED from the UI for EVERYONE (owner 2026-07-27, supersedes the
18
- 2026-07-23 per-user-restore semantics): no nav entry, no Settings>Modules checkbox,
19
- prewarm and validate.py skip it. Code and data layer stay in the repo but are NOT
20
- maintained — spend no effort on archived modules until the owner un-archives.
21
- 2026-07-23 pass: backorders, pricecomp, o2c, bookings, spend, expenses.
22
- 2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement +
23
- Collections + Analyst/plumbing): products, assortment, financial, management,
24
- pricing, inventory, vendor, warehouse, returns, outreach, complexity, health.
25
- note one-line scope note shown in the UI / docs
26
- """
27
- # WAVE-9 I8 — `'source'`: which connected system a module's data comes from.
28
- #
29
- # The owner's model: every nav entry is "a piece of database from a source", so the nav shows the
30
- # source beside the name ("Sales · Odoo"). It is a REGISTRY FACT rather than a string in the nav
31
- # code precisely so the nav needs no edit when a second connector lands — a user-created blank
32
- # table simply carries a different source, and a module with no `source` (Metric Dictionary,
33
- # Settings, the Analyst) renders no badge because it is not a database at all.
34
- # See [[connector-onboarding]]: the plug-in seam is datastore.py, and Odoo has no delegated
35
- # OAuth, so the "+ New" connect flow is the credential-form shape, not an OAuth redirect.
36
- REGISTRY = [
37
- # ARCHIVED wave 16 (owner item 7, R3/R11, 2026-08-02): the PAGE went on the strength of
38
- # C-CHARTCAP — every Sales block shape (YoY compare series, KPI delta tiles, group-by
39
- # tables) is now assemblable BY HAND from the grid's chart/dashboard views, which is what
40
- # R3 required before deletion. `modules/sales.py` (the data layer) STAYS — drawers and the
41
- # briefing still read it, and pages_sales.py survives UNREGISTERED as the Y1 envelope's
42
- # template + verify_api's fixture.
43
- {'key': 'sales', 'label': 'Sales', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'archived': True,
44
- 'note': 'Revenue, YoY, seasonality, reps, customers, SKUs — by BU. Retired wave 16: '
45
- 'rebuild as grid chart/dashboard views (compare series, KPI deltas, tables).'},
46
- # ARCHIVED AS A PAGE (owner item 14, wave 8): "archive the current form of the Customer
47
- # dashboard completely, i want to redesign it, exactly with the backend we have". So the
48
- # PAGE goes and the DATA LAYER stays — modules/customers.py still powers the Customer table's
49
- # metrics and still validates against Odoo, which is what the redesign will be built on.
50
- #
51
- # ⚠ NOT 'archived': True. Archived means invisible to everyone, and this row is the parent of
52
- # customer_data + cohort — archiving it would take the whole family out of the nav and strand
53
- # its children. 'group_only' says exactly what is true: this key names a FOLDER, never a
54
- # destination. It has no PAGE_FUNCS entry, the nav never renders it as a leaf, and a stale
55
- # deep link to it redirects to the family's first visible member.
56
- {'key': 'customers', 'label': 'Customers', 'brand': True, 'hq': False, 'validate': True,
57
- 'group_only': True,
58
- 'note': 'Folder for the customer surfaces (Customer table, Cohort). The old Customers '
59
- 'dashboard was retired for redesign (owner item 14, 2026-07-28); its metric layer '
60
- '(modules/customers.py) still backs the table and still validates against Odoo.'},
61
- # LABEL renamed 'Customer List' -> 'Data' (owner 2026-07-25) -> 'Customer' (owner item 5,
62
- # 2026-07-27); KEY renamed 'customer_list' -> 'customer_data' (owner 2026-07-26). The key
63
- # rename is safe ONLY because _LEGACY_KEYS (ui/session.py) migrates grants, Library prefs
64
- # and ?page= deep links ON READ — stored user records are never edited, exactly as
65
- # myday->customer_list was done. The STORE keys are deliberately NOT renamed:
66
- # 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them
67
- # would orphan every saved view. Label-only changes need none of that machinery.
68
- {'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
69
- 'note': 'Build, filter and save customer lists: the whole scoped book with per-customer '
70
- 'metrics; saved lists are visible, editable formulas (Call list and Win-back ship '
71
- 'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'},
72
- # Cohort (owner item 6, 2026-07-26): the SAME table as Data over a FIXED set. The
73
- # difference is membership semantics, not layout — a saved view re-populates as the
74
- # data moves, a cohort only changes when a person edits it. validate:False because
75
- # there is no Odoo aggregate to reconcile a hand-picked list against; its metrics are
76
- # the customers module's, already validated there.
77
- # ARCHIVED wave 16 (owner item 5, R10 — the fold-in): every cohort now projects as a
78
- # LOCKED VIEW in the Customer rail's "Cohorts" section (wave-15 C-LOCK), so the separate
79
- # nav destination is gone. ALL cohort machinery stays: `scope=cohort` still works
80
- # (routes_grid._SCOPES, storage keys, events), page_cohort remains in app.py, and
81
- # _LEGACY_KEYS maps cohort→customer_data so grants + ?page= deep links land on the grid
82
- # that now hosts the cohorts.
83
- {'key': 'cohort', 'label': 'Cohort', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', 'archived': True,
84
- 'note': 'Hand-curated, unchanging customer lists — folded into the Customer rail as '
85
- 'locked views (wave 16). The set is fixed: it changes only when someone adds '
86
- 'or removes a member. Open them from Customer > Views > Cohorts.'},
87
- # ARCHIVED wave 16 (owner item 7, R11, 2026-08-02) beside Sales. The agent DRAWER and the
88
- # data layer (modules/agent.py, agent_* caches) STAY — agent entities still open from
89
- # customer surfaces; an own-book agent login's home is the Customer grid narrowed by the
90
- # R1 permanent filter ("Agent is X"), not this page.
91
- {'key': 'agent', 'label': 'Agents', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'archived': True,
92
- 'note': 'Per-agent book: sales (custom dates), returns, top SKUs, full customer list incl. inactive. '
93
- 'Retired wave 16: agent drawers + the R1 own-book filter carry the use case.'},
94
- # Wave 16 C-TOPIC (owner items 9+10, R4): the PRODUCT table family — the second object on
95
- # the table-page factory. `products_family` is the folder head (the `customers` pattern:
96
- # group_only, never a destination); `product_data` is the grid over the SKU dataset, with
97
- # its OWN workspace bucket ('product_table_workspace') and the /products + scope=product
98
- # seams. The archived Streamlit `products` (SKU) row below stays archived and untouched,
99
- # exactly as R4 rules.
100
- {'key': 'products_family', 'label': 'Products', 'brand': True, 'hq': False, 'validate': False,
101
- 'group_only': True,
102
- 'note': 'Folder for the product surfaces (Product table). The archived SKU dashboard is '
103
- 'not part of this family.'},
104
- {'key': 'product_data', 'label': 'Product', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family',
105
- 'note': 'The SKU catalogue as a grid: per-product revenue, units and (consolidated) '
106
- 'stock columns, with saved views and custom fields. Identity = the SKU code.'},
107
- {'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
108
- 'note': 'SKU movers, zombies, velocity, coverage, drawers — by BU (sales-derived).'},
109
- {'key': 'assortment', 'label': 'Assortment', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
110
- 'note': 'Facet-level performance (category/color/occasion/collection/season) + season readiness — by BU.'},
111
- {'key': 'financial', 'label': 'Financial', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
112
- 'note': 'Gross margin by BU/category/SKU. Cash-conversion cycle is HQ-consolidated.'},
113
- {'key': 'pricing', 'label': 'Pricing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
114
- 'note': 'Per-SKU margin/markup/LTM sales + allocated net P&L (channel-rate cost-to-SKU) for pricing decisions.'},
115
- {'key': 'management', 'label': 'Management P&L', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
116
- 'nav': False, # rendered INSIDE the Financial module (view toggle) — archived WITH financial
117
- 'note': 'Operating-model management income statements (FFS/RI/GD/BU-M) reproduced from raw Odoo, Sep-2024+.'},
118
- {'key': 'inventory', 'label': 'Inventory', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
119
- 'note': 'On-hand stock is physically consolidated (shared warehouse); not DBA-split.'},
120
- {'key': 'procurement', 'label': 'Procurement', 'brand': False, 'hq': False, 'validate': True, 'source': 'Odoo',
121
- 'validate_only': True,
122
- 'note': 'PAGE RETIRED 2026-08-03 (owner wave-17 item 13: "We should be able to replace '
123
- 'Procurement completely, and add it as part of the Product database"). The buy '
124
- 'list is now a saved VIEW on the Product grid, filtered on a FORMULA field over '
125
- 'the supplier/lead-time columns and the demand measure — the owner\'s ruling R3: '
126
- '"Buy list is just a View, with a Filter from a Formula field that taps into '
127
- 'Metrics Fields... the 8-month demand baseline etc. is just math in a Formula '
128
- 'field." The ROW STAYS validate_only: `modules/procurement.py` still owns the '
129
- 'supplier master map (procurement_suppliers.json, 3,963 SKUs) that seeds those '
130
- 'columns, and its validate() is the proof they reconcile.'},
131
- {'key': 'vendor', 'label': 'Vendors', 'brand': False, 'hq': False, 'validate': False, 'archived': True,
132
- 'note': 'Vendor master data + the SKUs each supplies; multiple-vendor / cheapest-price view.'},
133
- {'key': 'warehouse', 'label': 'Warehouse', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
134
- 'note': 'Movement & efficiency: on-time ship, Pick-Pack-Ship cycle times, throughput, backlog age, shrinkage. Physical ops (consolidated).'},
135
- {'key': 'ar', 'label': 'Collections', 'brand': False, 'hq': True, 'validate': True,
136
- 'source': 'Odoo', 'api_surface': False, 'admin_only': True,
137
- 'note': 'PAGE RETIRED 2026-08-03 (owner wave-17 item 15: "Collections should also be '
138
- 'entirely replicable as just a View under Customer"). The worklist is the shared '
139
- '"Collections" view on the Customer grid — same numbers, from this module\'s own '
140
- 'reconciled blocks (ar_open/ar_overdue/ar_exposure/days_to_pay + the four aging '
141
- 'buckets). The ROW STAYS validate_only so validate.py keeps running ar.validate(), '
142
- 'which is the proof those columns rest on; archiving it would have skipped the '
143
- 'reconciliation while the numbers kept shipping. ⚠ NOT `validate_only`, and the '
144
- 'difference is a LIVE WORKFLOW: this page also mounts the admin-gated STATEMENTS '
145
- 'sender (app._collections_statements -> modules/collections_send), the ONE '
146
- 'sanctioned Odoo writer, which the same ruling says stays untouched. So the row '
147
- 'keeps a Streamlit page for ADMINS ONLY (`admin_only`) and leaves the API payload '
148
- 'entirely (`api_surface: False`) — no "Collections" in the React nav, no second '
149
- 'worklist, and the biweekly send keeps its door.'},
150
- {'key': 'returns', 'label': 'Returns', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
151
- 'note': 'Credit-note lens: refund concentration by SKU (quality) and customer (behavior). Company-level.'},
152
- {'key': 'outreach', 'label': 'Outreach', 'brand': True, 'hq': False, 'validate': False, 'archived': True,
153
- 'note': 'Campaign emails to customer segments: templates, suppression, send log, revenue attribution. Sending is admin-gated behind SAFE_MODE.'},
154
- {'key': 'backorders', 'label': 'Backorders', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
155
- 'note': 'Confirmed-undelivered order lines aged vs promise date, valued, with a supply-aware next action per row (ship / expedite / call). Wholesale scope.'},
156
- {'key': 'pricecomp', 'label': 'Price Compliance', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
157
- 'note': 'Selling below the customer pricelist tier (LTM, per customer x SKU): the pocket-price floor worklist with annualized leak $. Sub-30% ratios usually mean a stale or pack-basis RULE — fix the rule, not the rep.'},
158
- {'key': 'o2c', 'label': 'Cash Timing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
159
- 'note': 'Order-to-cash stage decomposition (order-to-ship / ship-to-invoice / invoice-to-paid, each with its owner) + the terms-gap rollup: contractual vs actual days per payment term with the free-credit $ it strands.'},
160
- {'key': 'bookings', 'label': 'Order Book', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
161
- 'note': 'Pre-season booking coverage: cumulative booked $ for Aug-Dec delivery vs last year same-week; category TY-vs-LY-as-of-date. The container-program decision, months ahead.'},
162
- {'key': 'spend', 'label': 'Spend & Payables', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
163
- 'note': 'Vendor-bill side: payment-terms capture (early-pay cash surrendered), duplicate-bill review, the opex spend cube (fragmentation, consolidate+rebid) and freight recovery. Company-level.'},
164
- {'key': 'complexity', 'label': 'SKU Complexity', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
165
- 'note': 'BCG tail rationalization: every SKU re-costed by ACTIVITY (lines, picks, returns) + 25%/yr carrying; KILL candidates rest on hard math (GM minus carrying), REVIEW on the pooled activity estimate. All-channel.'},
166
- {'key': 'expenses', 'label': 'Expenses', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
167
- 'note': 'Operating expense from the GL (all expense-type accounts; COGS excluded): trend, operating leverage (opex % of revenue), the YoY cost bridge, XmR control-limit spike watch list, fixed/variable split and a drill-to-ledger category directory. Company-level.'},
168
- {'key': 'health', 'label': 'Data Health', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
169
- 'note': 'Close/reconciliation scan; period-filtered, mostly company-level. ARCHITECTURE '
170
- '(owner 2026-07-23): every discrepancy / potential-error marker lives here — '
171
- 'procurement mapping gaps, count-trust (unverified on-hand counts), untracked-on-'
172
- 'order — so operational workflows stay clean for doing the work.'},
173
- {'key': 'dictionary', 'label': 'Metric Dictionary', 'brand': False, 'hq': True, 'validate': True,
174
- 'nav': False, 'validate_only': True,
175
- 'note': 'PAGE RETIRED 2026-07-23 (owner: broken/not customer-facing) — row kept ONLY so '
176
- 'validate.py keeps running the semantic-layer contracts (the Analyst grounding '
177
- 'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does — delete '
178
- 'them") made that literal: `validate_only` takes it out of the account menu too, '
179
- 'which is the last place it was still visible. The proof survives; the door does not.'},
180
- {'key': 'automation', 'label': 'Automation', 'brand': False, 'hq': True, 'validate': False,
181
- 'nav': True,
182
- 'note': 'Wave 18 (C-AUTONAV): the Automation surface — scheduled jobs that create and '
183
- 'refresh user databases (website scrape-to-DB, the Instagram field). React-only '
184
- 'surface (no PAGE_FUNCS entry, the no-new-Streamlit rule); admins hold it via '
185
- '"all", other users need the explicit grant — fail-closed default.'},
186
- {'key': 'settings', 'label': 'Settings', 'brand': False, 'hq': True, 'validate': False,
187
- 'nav': False, # sidebar sentinel (account group) — owner IA 2026-07-12
188
- 'note': 'User scope settings: the Business Unit toggle (strict isolation) and the Data basis '
189
- 'toggle (Orders vs Invoiced) moved here from the sidebar.'},
190
- {'key': 'analyst', 'label': 'AIOS Analyst', 'brand': False, 'hq': True, 'validate': False,
191
- 'nav': False, # sidebar sentinel button; eval gate = harness/evals.py (run pre-ship, NOT in validate.py — live LLM cost)
192
- 'note': 'Ask the business a question in plain language: a small AI model calls governed tools over the semantic layer — answers carry their query trace and drill links. AI-generated output (Art. 50 labeled).'},
193
- ]
194
-
195
- BY_KEY = {m['key']: m for m in REGISTRY}
196
-
197
-
198
- def brand_filterable(key) -> bool:
199
- return bool(BY_KEY.get(key, {}).get('brand'))
 
1
+ """Module registry — the single source of truth for the dashboards.
2
+
3
+ `app.py` builds the sidebar nav from this, and decides whether the global Brand (DBA) filter
4
+ applies to a page. `validate.py` runs each module's validate() from this. Adding a module =
5
+ add one row here + a page function in app.py + a module in modules/.
6
+
7
+ Fields:
8
+ key stable id (matches the modules/<key>.py file and the app page_<key> function)
9
+ label sidebar/title text (no emojis)
10
+ brand True -> DBA-filterable (honours the Fisch/Royal selector)
11
+ False -> consolidated/company-level (ignores the brand selector)
12
+ hq True -> HQ / company-level (cash flow, balance sheet, AR/AP, consolidated)
13
+ validate True -> module exposes validate() and is included in validate.py
14
+ parent set -> this row is a SUB-MODULE of <parent key>: it leaves the Workflows dropdown and
15
+ renders as a sub-nav button when its parent (or a sibling) is the active module; a
16
+ parent grant covers its sub-modules (owner nav redesign 2026-07-23)
17
+ archived True -> RETIRED from the UI for EVERYONE (owner 2026-07-27, supersedes the
18
+ 2026-07-23 per-user-restore semantics): no nav entry, no Settings>Modules checkbox,
19
+ prewarm and validate.py skip it. Code and data layer stay in the repo but are NOT
20
+ maintained — spend no effort on archived modules until the owner un-archives.
21
+ 2026-07-23 pass: backorders, pricecomp, o2c, bookings, spend, expenses.
22
+ 2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement +
23
+ Collections + Analyst/plumbing): products, assortment, financial, management,
24
+ pricing, inventory, vendor, warehouse, returns, outreach, complexity, health.
25
+ note one-line scope note shown in the UI / docs
26
+ """
27
+ # WAVE-9 I8 — `'source'`: which connected system a module's data comes from.
28
+ #
29
+ # The owner's model: every nav entry is "a piece of database from a source", so the nav shows the
30
+ # source beside the name ("Sales · Odoo"). It is a REGISTRY FACT rather than a string in the nav
31
+ # code precisely so the nav needs no edit when a second connector lands — a user-created blank
32
+ # table simply carries a different source, and a module with no `source` (Metric Dictionary,
33
+ # Settings, the Analyst) renders no badge because it is not a database at all.
34
+ # See [[connector-onboarding]]: the plug-in seam is datastore.py, and Odoo has no delegated
35
+ # OAuth, so the "+ New" connect flow is the credential-form shape, not an OAuth redirect.
36
+ REGISTRY = [
37
+ # ARCHIVED wave 16 (owner item 7, R3/R11, 2026-08-02): the PAGE went on the strength of
38
+ # C-CHARTCAP — every Sales block shape (YoY compare series, KPI delta tiles, group-by
39
+ # tables) is now assemblable BY HAND from the grid's chart/dashboard views, which is what
40
+ # R3 required before deletion. `modules/sales.py` (the data layer) STAYS — drawers and the
41
+ # briefing still read it, and pages_sales.py survives UNREGISTERED as the Y1 envelope's
42
+ # template + verify_api's fixture.
43
+ {'key': 'sales', 'label': 'Sales', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'archived': True,
44
+ 'note': 'Revenue, YoY, seasonality, reps, customers, SKUs — by BU. Retired wave 16: '
45
+ 'rebuild as grid chart/dashboard views (compare series, KPI deltas, tables).'},
46
+ # ARCHIVED AS A PAGE (owner item 14, wave 8): "archive the current form of the Customer
47
+ # dashboard completely, i want to redesign it, exactly with the backend we have". So the
48
+ # PAGE goes and the DATA LAYER stays — modules/customers.py still powers the Customer table's
49
+ # metrics and still validates against Odoo, which is what the redesign will be built on.
50
+ #
51
+ # ⚠ NOT 'archived': True. Archived means invisible to everyone, and this row is the parent of
52
+ # customer_data + cohort — archiving it would take the whole family out of the nav and strand
53
+ # its children. 'group_only' says exactly what is true: this key names a FOLDER, never a
54
+ # destination. It has no PAGE_FUNCS entry, the nav never renders it as a leaf, and a stale
55
+ # deep link to it redirects to the family's first visible member.
56
+ {'key': 'customers', 'label': 'Customers', 'brand': True, 'hq': False, 'validate': True,
57
+ 'group_only': True,
58
+ 'note': 'Folder for the customer surfaces (Customer table, Cohort). The old Customers '
59
+ 'dashboard was retired for redesign (owner item 14, 2026-07-28); its metric layer '
60
+ '(modules/customers.py) still backs the table and still validates against Odoo.'},
61
+ # LABEL renamed 'Customer List' -> 'Data' (owner 2026-07-25) -> 'Customer' (owner item 5,
62
+ # 2026-07-27); KEY renamed 'customer_list' -> 'customer_data' (owner 2026-07-26). The key
63
+ # rename is safe ONLY because _LEGACY_KEYS (ui/session.py) migrates grants, Library prefs
64
+ # and ?page= deep links ON READ — stored user records are never edited, exactly as
65
+ # myday->customer_list was done. The STORE keys are deliberately NOT renamed:
66
+ # 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them
67
+ # would orphan every saved view. Label-only changes need none of that machinery.
68
+ {'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
69
+ 'note': 'Build, filter and save customer lists: the whole scoped book with per-customer '
70
+ 'metrics; saved lists are visible, editable formulas (Call list and Win-back ship '
71
+ 'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'},
72
+ # Cohort (owner item 6, 2026-07-26): the SAME table as Data over a FIXED set. The
73
+ # difference is membership semantics, not layout — a saved view re-populates as the
74
+ # data moves, a cohort only changes when a person edits it. validate:False because
75
+ # there is no Odoo aggregate to reconcile a hand-picked list against; its metrics are
76
+ # the customers module's, already validated there.
77
+ # ARCHIVED wave 16 (owner item 5, R10 — the fold-in): every cohort now projects as a
78
+ # LOCKED VIEW in the Customer rail's "Cohorts" section (wave-15 C-LOCK), so the separate
79
+ # nav destination is gone. ALL cohort machinery stays: `scope=cohort` still works
80
+ # (routes_grid._SCOPES, storage keys, events), page_cohort remains in app.py, and
81
+ # _LEGACY_KEYS maps cohort→customer_data so grants + ?page= deep links land on the grid
82
+ # that now hosts the cohorts.
83
+ {'key': 'cohort', 'label': 'Cohort', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', 'archived': True,
84
+ 'note': 'Hand-curated, unchanging customer lists — folded into the Customer rail as '
85
+ 'locked views (wave 16). The set is fixed: it changes only when someone adds '
86
+ 'or removes a member. Open them from Customer > Views > Cohorts.'},
87
+ # ARCHIVED wave 16 (owner item 7, R11, 2026-08-02) beside Sales. The agent DRAWER and the
88
+ # data layer (modules/agent.py, agent_* caches) STAY — agent entities still open from
89
+ # customer surfaces; an own-book agent login's home is the Customer grid narrowed by the
90
+ # R1 permanent filter ("Agent is X"), not this page.
91
+ {'key': 'agent', 'label': 'Agents', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'archived': True,
92
+ 'note': 'Per-agent book: sales (custom dates), returns, top SKUs, full customer list incl. inactive. '
93
+ 'Retired wave 16: agent drawers + the R1 own-book filter carry the use case.'},
94
+ # Wave 16 C-TOPIC (owner items 9+10, R4): the PRODUCT table family — the second object on
95
+ # the table-page factory. `products_family` is the folder head (the `customers` pattern:
96
+ # group_only, never a destination); `product_data` is the grid over the SKU dataset, with
97
+ # its OWN workspace bucket ('product_table_workspace') and the /products + scope=product
98
+ # seams. The archived Streamlit `products` (SKU) row below stays archived and untouched,
99
+ # exactly as R4 rules.
100
+ {'key': 'products_family', 'label': 'Products', 'brand': True, 'hq': False, 'validate': False,
101
+ 'group_only': True,
102
+ 'note': 'Folder for the product surfaces (Product table). The archived SKU dashboard is '
103
+ 'not part of this family.'},
104
+ {'key': 'product_data', 'label': 'Product', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family',
105
+ 'note': 'The SKU catalogue as a grid: per-product revenue, units and (consolidated) '
106
+ 'stock columns, with saved views and custom fields. Identity = the SKU code.'},
107
+ {'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
108
+ 'note': 'SKU movers, zombies, velocity, coverage, drawers — by BU (sales-derived).'},
109
+ {'key': 'assortment', 'label': 'Assortment', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
110
+ 'note': 'Facet-level performance (category/color/occasion/collection/season) + season readiness — by BU.'},
111
+ {'key': 'financial', 'label': 'Financial', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
112
+ 'note': 'Gross margin by BU/category/SKU. Cash-conversion cycle is HQ-consolidated.'},
113
+ {'key': 'pricing', 'label': 'Pricing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
114
+ 'note': 'Per-SKU margin/markup/LTM sales + allocated net P&L (channel-rate cost-to-SKU) for pricing decisions.'},
115
+ {'key': 'management', 'label': 'Management P&L', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
116
+ 'nav': False, # rendered INSIDE the Financial module (view toggle) — archived WITH financial
117
+ 'note': 'Operating-model management income statements (FFS/RI/GD/BU-M) reproduced from raw Odoo, Sep-2024+.'},
118
+ {'key': 'inventory', 'label': 'Inventory', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
119
+ 'note': 'On-hand stock is physically consolidated (shared warehouse); not DBA-split.'},
120
+ {'key': 'procurement', 'label': 'Procurement', 'brand': False, 'hq': False, 'validate': True, 'source': 'Odoo',
121
+ 'validate_only': True,
122
+ 'note': 'PAGE RETIRED 2026-08-03 (owner wave-17 item 13: "We should be able to replace '
123
+ 'Procurement completely, and add it as part of the Product database"). The buy '
124
+ 'list is now a saved VIEW on the Product grid, filtered on a FORMULA field over '
125
+ 'the supplier/lead-time columns and the demand measure — the owner\'s ruling R3: '
126
+ '"Buy list is just a View, with a Filter from a Formula field that taps into '
127
+ 'Metrics Fields... the 8-month demand baseline etc. is just math in a Formula '
128
+ 'field." The ROW STAYS validate_only: `modules/procurement.py` still owns the '
129
+ 'supplier master map (procurement_suppliers.json, 3,963 SKUs) that seeds those '
130
+ 'columns, and its validate() is the proof they reconcile.'},
131
+ {'key': 'vendor', 'label': 'Vendors', 'brand': False, 'hq': False, 'validate': False, 'archived': True,
132
+ 'note': 'Vendor master data + the SKUs each supplies; multiple-vendor / cheapest-price view.'},
133
+ {'key': 'warehouse', 'label': 'Warehouse', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
134
+ 'note': 'Movement & efficiency: on-time ship, Pick-Pack-Ship cycle times, throughput, backlog age, shrinkage. Physical ops (consolidated).'},
135
+ {'key': 'ar', 'label': 'Collections', 'brand': False, 'hq': True, 'validate': True,
136
+ 'source': 'Odoo', 'api_surface': False, 'admin_only': True,
137
+ 'note': 'PAGE RETIRED 2026-08-03 (owner wave-17 item 15: "Collections should also be '
138
+ 'entirely replicable as just a View under Customer"). The worklist is the shared '
139
+ '"Collections" view on the Customer grid — same numbers, from this module\'s own '
140
+ 'reconciled blocks (ar_open/ar_overdue/ar_exposure/days_to_pay + the four aging '
141
+ 'buckets). The ROW STAYS validate_only so validate.py keeps running ar.validate(), '
142
+ 'which is the proof those columns rest on; archiving it would have skipped the '
143
+ 'reconciliation while the numbers kept shipping. ⚠ NOT `validate_only`, and the '
144
+ 'difference is a LIVE WORKFLOW: this page also mounts the admin-gated STATEMENTS '
145
+ 'sender (app._collections_statements -> modules/collections_send), the ONE '
146
+ 'sanctioned Odoo writer, which the same ruling says stays untouched. So the row '
147
+ 'keeps a Streamlit page for ADMINS ONLY (`admin_only`) and leaves the API payload '
148
+ 'entirely (`api_surface: False`) — no "Collections" in the React nav, no second '
149
+ 'worklist, and the biweekly send keeps its door.'},
150
+ {'key': 'returns', 'label': 'Returns', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
151
+ 'note': 'Credit-note lens: refund concentration by SKU (quality) and customer (behavior). Company-level.'},
152
+ {'key': 'outreach', 'label': 'Outreach', 'brand': True, 'hq': False, 'validate': False, 'archived': True,
153
+ 'note': 'Campaign emails to customer segments: templates, suppression, send log, revenue attribution. Sending is admin-gated behind SAFE_MODE.'},
154
+ {'key': 'backorders', 'label': 'Backorders', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
155
+ 'note': 'Confirmed-undelivered order lines aged vs promise date, valued, with a supply-aware next action per row (ship / expedite / call). Wholesale scope.'},
156
+ {'key': 'pricecomp', 'label': 'Price Compliance', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
157
+ 'note': 'Selling below the customer pricelist tier (LTM, per customer x SKU): the pocket-price floor worklist with annualized leak $. Sub-30% ratios usually mean a stale or pack-basis RULE — fix the rule, not the rep.'},
158
+ {'key': 'o2c', 'label': 'Cash Timing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
159
+ 'note': 'Order-to-cash stage decomposition (order-to-ship / ship-to-invoice / invoice-to-paid, each with its owner) + the terms-gap rollup: contractual vs actual days per payment term with the free-credit $ it strands.'},
160
+ {'key': 'bookings', 'label': 'Order Book', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
161
+ 'note': 'Pre-season booking coverage: cumulative booked $ for Aug-Dec delivery vs last year same-week; category TY-vs-LY-as-of-date. The container-program decision, months ahead.'},
162
+ {'key': 'spend', 'label': 'Spend & Payables', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
163
+ 'note': 'Vendor-bill side: payment-terms capture (early-pay cash surrendered), duplicate-bill review, the opex spend cube (fragmentation, consolidate+rebid) and freight recovery. Company-level.'},
164
+ {'key': 'complexity', 'label': 'SKU Complexity', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
165
+ 'note': 'BCG tail rationalization: every SKU re-costed by ACTIVITY (lines, picks, returns) + 25%/yr carrying; KILL candidates rest on hard math (GM minus carrying), REVIEW on the pooled activity estimate. All-channel.'},
166
+ {'key': 'expenses', 'label': 'Expenses', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
167
+ 'note': 'Operating expense from the GL (all expense-type accounts; COGS excluded): trend, operating leverage (opex % of revenue), the YoY cost bridge, XmR control-limit spike watch list, fixed/variable split and a drill-to-ledger category directory. Company-level.'},
168
+ {'key': 'health', 'label': 'Data Health', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
169
+ 'note': 'Close/reconciliation scan; period-filtered, mostly company-level. ARCHITECTURE '
170
+ '(owner 2026-07-23): every discrepancy / potential-error marker lives here — '
171
+ 'procurement mapping gaps, count-trust (unverified on-hand counts), untracked-on-'
172
+ 'order — so operational workflows stay clean for doing the work.'},
173
+ {'key': 'dictionary', 'label': 'Metric Dictionary', 'brand': False, 'hq': True, 'validate': True,
174
+ 'nav': False, 'validate_only': True,
175
+ 'note': 'PAGE RETIRED 2026-07-23 (owner: broken/not customer-facing) — row kept ONLY so '
176
+ 'validate.py keeps running the semantic-layer contracts (the Analyst grounding '
177
+ 'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does — delete '
178
+ 'them") made that literal: `validate_only` takes it out of the account menu too, '
179
+ 'which is the last place it was still visible. The proof survives; the door does not.'},
180
+ {'key': 'automation', 'label': 'Automation', 'brand': False, 'hq': True, 'validate': False,
181
+ 'nav': True,
182
+ 'note': 'Wave 18 (C-AUTONAV): the Automation surface — scheduled jobs that create and '
183
+ 'refresh user databases (website scrape-to-DB, the Instagram field). React-only '
184
+ 'surface (no PAGE_FUNCS entry, the no-new-Streamlit rule); admins hold it via '
185
+ '"all", other users need the explicit grant — fail-closed default.'},
186
+ {'key': 'settings', 'label': 'Settings', 'brand': False, 'hq': True, 'validate': False,
187
+ 'nav': False, # sidebar sentinel (account group) — owner IA 2026-07-12
188
+ 'note': 'User scope settings: the Business Unit toggle (strict isolation) and the Data basis '
189
+ 'toggle (Orders vs Invoiced) moved here from the sidebar.'},
190
+ {'key': 'analyst', 'label': 'AIOS Analyst', 'brand': False, 'hq': True, 'validate': False,
191
+ 'nav': False, # sidebar sentinel button; eval gate = harness/evals.py (run pre-ship, NOT in validate.py — live LLM cost)
192
+ 'note': 'Ask the business a question in plain language: a small AI model calls governed tools over the semantic layer — answers carry their query trace and drill links. AI-generated output (Art. 50 labeled).'},
193
+ ]
194
+
195
+ BY_KEY = {m['key']: m for m in REGISTRY}
196
+
197
+
198
+ def brand_filterable(key) -> bool:
199
+ return bool(BY_KEY.get(key, {}).get('brand'))
platform/core/store_pg.py CHANGED
@@ -1,325 +1,325 @@
1
- """core/store_pg.py — `core/store.py`'s interface, backed by Postgres (X4 / EXIT-2b, 2026-07-30).
2
-
3
- THE INTERFACE IS THE CONTRACT, and it is `core/store.py`'s nine functions:
4
-
5
- available() · get(name, fresh=False) · exists(name) · put(name, data)
6
- update(name, fn, flush='sync') · upload_bytes(path, data, message=None)
7
- download_bytes(path) · delete_path(path) · flush(name=None, timeout=30.0)
8
-
9
- Same names, same signatures, same return types, same failure semantics — so switching backends is
10
- an env var (`STORE_BACKEND=hf|pg`) and not a rewrite of every caller. `core/store_backend.py`
11
- does the selection; nothing above it needs to know which store it is talking to.
12
-
13
- ✅ VERIFIED against a real managed Postgres 2026-08-04 (W19: Neon us-east-2; `verify_store_pg.py`
14
- 80/80 INCLUDING the integration half — schema apply, tenant provisioning, jsonb/bytea round-trips,
15
- 80 concurrent FOR-UPDATE writes — run from HF egress; the owner's local network resets TLS:5432).
16
- The CUTOVER remains parked behind C1e's triggers and this module is not the default backend.
17
- Without `DATABASE_URL` the gate still SKIPS its integration half LOUDLY, never silently.
18
- The schema is `harness/pg/schema.sql`. Sequenced around the blocker, never stubbed past it.
19
-
20
- WHAT POSTGRES FIXES, precisely (C1c — these are the reasons, not a preference):
21
- * **`get()` is cache-first on the HF store**, so a write from another process is invisible to a
22
- running app until restart. Observed live in wave 11. Here a read is a SELECT: cross-process
23
- coherence is the default rather than a thing to remember.
24
- * **`update()` was read-modify-write with no concurrency control** — two writers raced and the
25
- loser vanished silently. Here it runs inside ONE transaction with `SELECT … FOR UPDATE`, so
26
- the second writer waits and then sees the first writer's value.
27
- * **A write was an HTTP commit** against a 256-commits/hour repo budget, which is why the hot
28
- path needed a coalescing async flusher at all. A Postgres write is a write; `flush='async'`
29
- stays in the signature and becomes a no-op, because there is nothing to coalesce.
30
-
31
- ⚠ ONE DELIBERATE DIFFERENCE FROM THE HF BACKEND, and it is a fix. `store.get()` swallows transient
32
- read failures and hands back a cached-or-empty dict, because a display read must not break a page.
33
- That leniency is exactly what wiped the user registry once (an empty read merged into a write), and
34
- `_read_strict` exists to opt out of it. Postgres has no equivalent "the network blipped" state
35
- worth hiding: a connection failure RAISES from `get()` here. Callers that must not break on a read
36
- should catch it — silently returning `{}` from a database that is simply unreachable is how an
37
- empty dict gets written back over real data.
38
- """
39
- import json
40
- import os
41
- import threading
42
-
43
- _URL_ENV = 'DATABASE_URL'
44
- _LOCK = threading.RLock()
45
- _POOL = {'pool': None, 'url': None, 'schema': None}
46
-
47
-
48
- def _url():
49
- return os.environ.get(_URL_ENV) or None
50
-
51
-
52
- def _schema_for(tenant_slug=None):
53
- """`t_<slug>` with `-` normalised to `_` (a hyphen is not legal in an unquoted identifier).
54
-
55
- Mirrors `control.provision_tenant_schema` exactly — the two MUST agree or a write lands in a
56
- schema the DDL never created. `AIOS_TENANT` names the tenant this process serves; it defaults
57
- to tenant #0 so the default behaviour is unchanged, the same rule `harness.datastore.path_for`
58
- follows for the DuckDB filename.
59
- """
60
- slug = (tenant_slug or os.environ.get('AIOS_TENANT') or 'royal-imports').strip().lower()
61
- return 't_' + slug.replace('-', '_')
62
-
63
-
64
- def _pool():
65
- """The psycopg connection pool, opened once per process.
66
-
67
- psycopg is imported LAZILY and only here: `STORE_BACKEND=hf` must not require the dependency
68
- to be installed at all, which is what keeps this file safe to commit before B-3 unblocks.
69
- """
70
- with _LOCK:
71
- url = _url()
72
- if not url:
73
- raise RuntimeError(
74
- f'{_URL_ENV} is not set — the Postgres store backend has nothing to connect to.')
75
- if _POOL['pool'] is not None and _POOL['url'] == url:
76
- return _POOL['pool']
77
- try:
78
- from psycopg_pool import ConnectionPool
79
- except ImportError as e: # pragma: no cover - depends on B-3
80
- raise RuntimeError(
81
- 'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in '
82
- 'requirements.txt until the database exists (owner blocker B-3), so that the '
83
- 'default hf backend installs nothing it does not use.') from e
84
- if _POOL['pool'] is not None:
85
- try:
86
- _POOL['pool'].close()
87
- except Exception:
88
- pass
89
- # min_size=0 so a process that never touches the store opens no connection at all — the
90
- # shared API serves plenty of requests (health, static, a cached payload) that never read.
91
- _POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0)
92
- _POOL['url'] = url
93
- return _POOL['pool']
94
-
95
-
96
- def available():
97
- """True when a connection can actually be MADE, not merely when a URL is configured.
98
-
99
- `core.store.available()` only checks for a token, which is all it can cheaply do. Here the
100
- honest answer needs a round-trip, so the result is memoised per process: callers use this to
101
- decide whether to degrade the UI, and a `SELECT 1` per render would be absurd.
102
- """
103
- if not _url():
104
- return False
105
- if _POOL.get('ok'):
106
- return True
107
- try:
108
- with _pool().connection() as con:
109
- con.execute('SELECT 1')
110
- _POOL['ok'] = True
111
- return True
112
- except Exception:
113
- return False
114
-
115
-
116
- def _table(kind, tenant_slug=None):
117
- """`"t_slug"."store_kv"` — the qualified table, safely quoted."""
118
- from psycopg import sql
119
- return sql.SQL('{}.{}').format(sql.Identifier(_schema_for(tenant_slug)),
120
- sql.Identifier(kind))
121
-
122
-
123
- def _q(template, kind, tenant_slug=None):
124
- """Compose `template` with `{tbl}` = the QUALIFIED table and `{bare}` = the table name alone.
125
-
126
- ⚠ THE TWO ARE NOT INTERCHANGEABLE, and getting it wrong is a runtime syntax error nothing
127
- local would catch (no database here — owner blocker B-3). Inside `ON CONFLICT … DO UPDATE`,
128
- Postgres refers to the conflicting row by the table's ALIAS, which defaults to the bare
129
- table name: `store_kv.rev` is valid, `t_royal_imports.store_kv.rev` is not. So the FROM
130
- position takes `{tbl}` and the DO-UPDATE position takes `{bare}`.
131
-
132
- Identifiers go through `psycopg.sql.Identifier`, so the schema name — derived from a tenant
133
- slug — cannot be an injection vector even though it is interpolated. The slug is ALSO
134
- regex-constrained at its source (`control.tenants`), which is the belt to this brace.
135
- """
136
- from psycopg import sql
137
- return sql.SQL(template).format(tbl=_table(kind, tenant_slug),
138
- bare=sql.Identifier(kind))
139
-
140
-
141
- def get(name, fresh=False, tenant_slug=None):
142
- """The value stored under `name`, or `{}` when the key is genuinely absent.
143
-
144
- `fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and
145
- there is no cache here — every read is a SELECT. Keeping the parameter means callers do not
146
- branch on the backend.
147
- """
148
- with _pool().connection() as con:
149
- row = con.execute(
150
- _q('SELECT value FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
151
- (str(name),)).fetchone()
152
- return dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
153
-
154
-
155
- def exists(name, tenant_slug=None):
156
- """True/False — and unlike the HF backend this is NEVER "True on uncertainty".
157
-
158
- `core.store.exists` returns True on any error so a caller that would seed the store cannot
159
- clobber a registry it merely failed to reach. That guard exists because a FILE HOST cannot
160
- distinguish "absent" from "unreachable". Postgres can: an absent row is a fact, and an
161
- unreachable server raises instead of answering.
162
- """
163
- with _pool().connection() as con:
164
- row = con.execute(
165
- _q('SELECT 1 FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
166
- (str(name),)).fetchone()
167
- return row is not None
168
-
169
-
170
- def put(name, data, tenant_slug=None):
171
- with _pool().connection() as con:
172
- con.execute(
173
- _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
174
- 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
175
- 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
176
- (str(name), json.dumps(data)))
177
- return data
178
-
179
-
180
- def update(name, fn, flush='sync', tenant_slug=None):
181
- """Read-modify-write inside ONE transaction, with the row LOCKED for the duration.
182
-
183
- This is the operation the HF backend could not make safe. There, `update` read over HTTP,
184
- applied `fn`, and uploaded — so two concurrent writers both read the old value and the second
185
- upload silently discarded the first writer's change. `SELECT … FOR UPDATE` makes the second
186
- writer WAIT and then apply `fn` to the first writer's result, which is what "read-modify-write"
187
- was always supposed to mean.
188
-
189
- `flush` is accepted and ignored (see the module docstring): there is no commit budget to
190
- coalesce against, so 'async' has nothing to defer.
191
- """
192
- with _pool().connection() as con:
193
- with con.transaction():
194
- row = con.execute(
195
- _q('SELECT value FROM {tbl} WHERE key = %s FOR UPDATE',
196
- 'store_kv', tenant_slug),
197
- (str(name),)).fetchone()
198
- data = dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
199
- result = fn(data)
200
- data = result if result is not None else data
201
- con.execute(
202
- _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
203
- 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
204
- 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
205
- (str(name), json.dumps(data)))
206
- return data
207
-
208
-
209
- def upload_bytes(path_in_repo, data, message=None, tenant_slug=None):
210
- with _pool().connection() as con:
211
- con.execute(
212
- _q('INSERT INTO {tbl} (path, bytes, message) VALUES (%s, %s, %s) '
213
- 'ON CONFLICT (path) DO UPDATE SET bytes = EXCLUDED.bytes, '
214
- 'message = EXCLUDED.message, updated_at = now()', 'store_blobs', tenant_slug),
215
- (str(path_in_repo), bytes(data), message))
216
-
217
-
218
- def download_bytes(path_in_repo, tenant_slug=None):
219
- """Raw bytes, or None when absent — the same contract as the HF backend: a missing attachment
220
- degrades to "that file is gone", never to a broken page."""
221
- with _pool().connection() as con:
222
- row = con.execute(
223
- _q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
224
- (str(path_in_repo),)).fetchone()
225
- return bytes(row[0]) if row else None
226
-
227
-
228
- def delete_path(path_in_repo, tenant_slug=None):
229
- """Absent is SUCCESS — deleting what is already gone is the goal (HF backend's contract)."""
230
- with _pool().connection() as con:
231
- con.execute(_q('DELETE FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
232
- (str(path_in_repo),))
233
- return True
234
-
235
-
236
- def flush(name=None, timeout=30.0):
237
- """A no-op that returns True: every write above is already committed when it returns. Kept in
238
- the interface because QA gates and shutdown hooks call it and must not branch on the backend."""
239
- return True
240
-
241
-
242
- def close():
243
- """Release the pool (tests, and a process that is shutting down cleanly)."""
244
- with _LOCK:
245
- if _POOL['pool'] is not None:
246
- try:
247
- _POOL['pool'].close()
248
- finally:
249
- _POOL['pool'] = None
250
- _POOL['url'] = None
251
- _POOL.pop('ok', None)
252
-
253
-
254
- # ---------------------------------------------------------------------------------------------
255
- # WAVE 20 (R1 / D-4) — THE TENANT-BOUND HANDLE. This is what made the cutover a small change.
256
- #
257
- # `core/store.py` grew a CLASS in wave 18 because a tenant can own its own dataset REPO, and
258
- # `harness.runtime.TenantRuntime` carries one bound instance per tenant. Postgres isolates by
259
- # SCHEMA instead (`t_<slug>`, see harness/pg/schema.sql's own argument for schema-over-RLS), so
260
- # the two models meet here: one object, bound to one slug, exposing `core.store.Store`'s methods.
261
- #
262
- # ⚠ THE BINDING IS THE POINT. The module functions above default their schema from `AIOS_TENANT`,
263
- # which is a PROCESS-wide answer to a PER-REQUEST question — correct for a single-tenant worker,
264
- # wrong for the shared API that serves four tenants from one process. A handle carries its slug
265
- # explicitly, so a write cannot land in another tenant's schema because some env var was right
266
- # for the last request. That is the same property `TenantRuntime.store_handle` already gives the
267
- # HF backend (EXIT-4b proof #3), reached a different way.
268
- # ---------------------------------------------------------------------------------------------
269
- class PgStore:
270
- """`core.store.Store`'s interface for ONE tenant's schema.
271
-
272
- Deliberately a thin binder over the module functions rather than a reimplementation: every
273
- behaviour argument (FOR UPDATE in `update`, absent-is-success in `delete_path`, `fresh`
274
- ignored because a SELECT has no cache to bypass) is stated once, up there, and cannot drift
275
- between the two entry points.
276
- """
277
-
278
- def __init__(self, tenant_slug):
279
- self.tenant_slug = str(tenant_slug or '').strip().lower() or 'royal-imports'
280
- #: Kept so `Store`-shaped debugging (`repr`, telemetry, the admin panes) reads the same
281
- #: on both backends. There is no repo here; the schema is the address.
282
- self.repo = f'pg:{_schema_for(self.tenant_slug)}'
283
-
284
- def __repr__(self):
285
- return f'<PgStore {self.repo}>'
286
-
287
- def available(self):
288
- return available()
289
-
290
- def get(self, name, fresh=False):
291
- return get(name, fresh=fresh, tenant_slug=self.tenant_slug)
292
-
293
- def _read_strict(self, name):
294
- """Interface parity with the HF Store. There, `get` is lenient (swallows a transient
295
- failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
296
- of merging into `{}`. Here `get` ALREADY raises — the leniency it opts out of does not
297
- exist — so the strict read is the ordinary one, and saying so beats a second code path."""
298
- return get(name, tenant_slug=self.tenant_slug)
299
-
300
- def exists(self, name):
301
- return exists(name, tenant_slug=self.tenant_slug)
302
-
303
- def put(self, name, data):
304
- return put(name, data, tenant_slug=self.tenant_slug)
305
-
306
- def update(self, name, fn, flush='sync'):
307
- return update(name, fn, flush=flush, tenant_slug=self.tenant_slug)
308
-
309
- def upload_bytes(self, path_in_repo, data, message=None):
310
- return upload_bytes(path_in_repo, data, message=message, tenant_slug=self.tenant_slug)
311
-
312
- def download_bytes(self, path_in_repo):
313
- return download_bytes(path_in_repo, tenant_slug=self.tenant_slug)
314
-
315
- def delete_path(self, path_in_repo):
316
- return delete_path(path_in_repo, tenant_slug=self.tenant_slug)
317
-
318
- def flush(self, name=None, timeout=30.0):
319
- return True
320
-
321
- def _flush_now_at_exit(self):
322
- """The atexit hook `core.store` registers for its instances. Nothing is buffered here —
323
- every write above committed before it returned — so this is honestly a no-op rather than
324
- an unimplemented method that would raise during interpreter shutdown."""
325
- return True
 
1
+ """core/store_pg.py — `core/store.py`'s interface, backed by Postgres (X4 / EXIT-2b, 2026-07-30).
2
+
3
+ THE INTERFACE IS THE CONTRACT, and it is `core/store.py`'s nine functions:
4
+
5
+ available() · get(name, fresh=False) · exists(name) · put(name, data)
6
+ update(name, fn, flush='sync') · upload_bytes(path, data, message=None)
7
+ download_bytes(path) · delete_path(path) · flush(name=None, timeout=30.0)
8
+
9
+ Same names, same signatures, same return types, same failure semantics — so switching backends is
10
+ an env var (`STORE_BACKEND=hf|pg`) and not a rewrite of every caller. `core/store_backend.py`
11
+ does the selection; nothing above it needs to know which store it is talking to.
12
+
13
+ ✅ VERIFIED against a real managed Postgres 2026-08-04 (W19: Neon us-east-2; `verify_store_pg.py`
14
+ 80/80 INCLUDING the integration half — schema apply, tenant provisioning, jsonb/bytea round-trips,
15
+ 80 concurrent FOR-UPDATE writes — run from HF egress; the owner's local network resets TLS:5432).
16
+ The CUTOVER remains parked behind C1e's triggers and this module is not the default backend.
17
+ Without `DATABASE_URL` the gate still SKIPS its integration half LOUDLY, never silently.
18
+ The schema is `harness/pg/schema.sql`. Sequenced around the blocker, never stubbed past it.
19
+
20
+ WHAT POSTGRES FIXES, precisely (C1c — these are the reasons, not a preference):
21
+ * **`get()` is cache-first on the HF store**, so a write from another process is invisible to a
22
+ running app until restart. Observed live in wave 11. Here a read is a SELECT: cross-process
23
+ coherence is the default rather than a thing to remember.
24
+ * **`update()` was read-modify-write with no concurrency control** — two writers raced and the
25
+ loser vanished silently. Here it runs inside ONE transaction with `SELECT … FOR UPDATE`, so
26
+ the second writer waits and then sees the first writer's value.
27
+ * **A write was an HTTP commit** against a 256-commits/hour repo budget, which is why the hot
28
+ path needed a coalescing async flusher at all. A Postgres write is a write; `flush='async'`
29
+ stays in the signature and becomes a no-op, because there is nothing to coalesce.
30
+
31
+ ⚠ ONE DELIBERATE DIFFERENCE FROM THE HF BACKEND, and it is a fix. `store.get()` swallows transient
32
+ read failures and hands back a cached-or-empty dict, because a display read must not break a page.
33
+ That leniency is exactly what wiped the user registry once (an empty read merged into a write), and
34
+ `_read_strict` exists to opt out of it. Postgres has no equivalent "the network blipped" state
35
+ worth hiding: a connection failure RAISES from `get()` here. Callers that must not break on a read
36
+ should catch it — silently returning `{}` from a database that is simply unreachable is how an
37
+ empty dict gets written back over real data.
38
+ """
39
+ import json
40
+ import os
41
+ import threading
42
+
43
+ _URL_ENV = 'DATABASE_URL'
44
+ _LOCK = threading.RLock()
45
+ _POOL = {'pool': None, 'url': None, 'schema': None}
46
+
47
+
48
+ def _url():
49
+ return os.environ.get(_URL_ENV) or None
50
+
51
+
52
+ def _schema_for(tenant_slug=None):
53
+ """`t_<slug>` with `-` normalised to `_` (a hyphen is not legal in an unquoted identifier).
54
+
55
+ Mirrors `control.provision_tenant_schema` exactly — the two MUST agree or a write lands in a
56
+ schema the DDL never created. `AIOS_TENANT` names the tenant this process serves; it defaults
57
+ to tenant #0 so the default behaviour is unchanged, the same rule `harness.datastore.path_for`
58
+ follows for the DuckDB filename.
59
+ """
60
+ slug = (tenant_slug or os.environ.get('AIOS_TENANT') or 'royal-imports').strip().lower()
61
+ return 't_' + slug.replace('-', '_')
62
+
63
+
64
+ def _pool():
65
+ """The psycopg connection pool, opened once per process.
66
+
67
+ psycopg is imported LAZILY and only here: `STORE_BACKEND=hf` must not require the dependency
68
+ to be installed at all, which is what keeps this file safe to commit before B-3 unblocks.
69
+ """
70
+ with _LOCK:
71
+ url = _url()
72
+ if not url:
73
+ raise RuntimeError(
74
+ f'{_URL_ENV} is not set — the Postgres store backend has nothing to connect to.')
75
+ if _POOL['pool'] is not None and _POOL['url'] == url:
76
+ return _POOL['pool']
77
+ try:
78
+ from psycopg_pool import ConnectionPool
79
+ except ImportError as e: # pragma: no cover - depends on B-3
80
+ raise RuntimeError(
81
+ 'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in '
82
+ 'requirements.txt until the database exists (owner blocker B-3), so that the '
83
+ 'default hf backend installs nothing it does not use.') from e
84
+ if _POOL['pool'] is not None:
85
+ try:
86
+ _POOL['pool'].close()
87
+ except Exception:
88
+ pass
89
+ # min_size=0 so a process that never touches the store opens no connection at all — the
90
+ # shared API serves plenty of requests (health, static, a cached payload) that never read.
91
+ _POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0)
92
+ _POOL['url'] = url
93
+ return _POOL['pool']
94
+
95
+
96
+ def available():
97
+ """True when a connection can actually be MADE, not merely when a URL is configured.
98
+
99
+ `core.store.available()` only checks for a token, which is all it can cheaply do. Here the
100
+ honest answer needs a round-trip, so the result is memoised per process: callers use this to
101
+ decide whether to degrade the UI, and a `SELECT 1` per render would be absurd.
102
+ """
103
+ if not _url():
104
+ return False
105
+ if _POOL.get('ok'):
106
+ return True
107
+ try:
108
+ with _pool().connection() as con:
109
+ con.execute('SELECT 1')
110
+ _POOL['ok'] = True
111
+ return True
112
+ except Exception:
113
+ return False
114
+
115
+
116
+ def _table(kind, tenant_slug=None):
117
+ """`"t_slug"."store_kv"` — the qualified table, safely quoted."""
118
+ from psycopg import sql
119
+ return sql.SQL('{}.{}').format(sql.Identifier(_schema_for(tenant_slug)),
120
+ sql.Identifier(kind))
121
+
122
+
123
+ def _q(template, kind, tenant_slug=None):
124
+ """Compose `template` with `{tbl}` = the QUALIFIED table and `{bare}` = the table name alone.
125
+
126
+ ⚠ THE TWO ARE NOT INTERCHANGEABLE, and getting it wrong is a runtime syntax error nothing
127
+ local would catch (no database here — owner blocker B-3). Inside `ON CONFLICT … DO UPDATE`,
128
+ Postgres refers to the conflicting row by the table's ALIAS, which defaults to the bare
129
+ table name: `store_kv.rev` is valid, `t_royal_imports.store_kv.rev` is not. So the FROM
130
+ position takes `{tbl}` and the DO-UPDATE position takes `{bare}`.
131
+
132
+ Identifiers go through `psycopg.sql.Identifier`, so the schema name — derived from a tenant
133
+ slug — cannot be an injection vector even though it is interpolated. The slug is ALSO
134
+ regex-constrained at its source (`control.tenants`), which is the belt to this brace.
135
+ """
136
+ from psycopg import sql
137
+ return sql.SQL(template).format(tbl=_table(kind, tenant_slug),
138
+ bare=sql.Identifier(kind))
139
+
140
+
141
+ def get(name, fresh=False, tenant_slug=None):
142
+ """The value stored under `name`, or `{}` when the key is genuinely absent.
143
+
144
+ `fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and
145
+ there is no cache here — every read is a SELECT. Keeping the parameter means callers do not
146
+ branch on the backend.
147
+ """
148
+ with _pool().connection() as con:
149
+ row = con.execute(
150
+ _q('SELECT value FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
151
+ (str(name),)).fetchone()
152
+ return dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
153
+
154
+
155
+ def exists(name, tenant_slug=None):
156
+ """True/False — and unlike the HF backend this is NEVER "True on uncertainty".
157
+
158
+ `core.store.exists` returns True on any error so a caller that would seed the store cannot
159
+ clobber a registry it merely failed to reach. That guard exists because a FILE HOST cannot
160
+ distinguish "absent" from "unreachable". Postgres can: an absent row is a fact, and an
161
+ unreachable server raises instead of answering.
162
+ """
163
+ with _pool().connection() as con:
164
+ row = con.execute(
165
+ _q('SELECT 1 FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
166
+ (str(name),)).fetchone()
167
+ return row is not None
168
+
169
+
170
+ def put(name, data, tenant_slug=None):
171
+ with _pool().connection() as con:
172
+ con.execute(
173
+ _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
174
+ 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
175
+ 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
176
+ (str(name), json.dumps(data)))
177
+ return data
178
+
179
+
180
+ def update(name, fn, flush='sync', tenant_slug=None):
181
+ """Read-modify-write inside ONE transaction, with the row LOCKED for the duration.
182
+
183
+ This is the operation the HF backend could not make safe. There, `update` read over HTTP,
184
+ applied `fn`, and uploaded — so two concurrent writers both read the old value and the second
185
+ upload silently discarded the first writer's change. `SELECT … FOR UPDATE` makes the second
186
+ writer WAIT and then apply `fn` to the first writer's result, which is what "read-modify-write"
187
+ was always supposed to mean.
188
+
189
+ `flush` is accepted and ignored (see the module docstring): there is no commit budget to
190
+ coalesce against, so 'async' has nothing to defer.
191
+ """
192
+ with _pool().connection() as con:
193
+ with con.transaction():
194
+ row = con.execute(
195
+ _q('SELECT value FROM {tbl} WHERE key = %s FOR UPDATE',
196
+ 'store_kv', tenant_slug),
197
+ (str(name),)).fetchone()
198
+ data = dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
199
+ result = fn(data)
200
+ data = result if result is not None else data
201
+ con.execute(
202
+ _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
203
+ 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
204
+ 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
205
+ (str(name), json.dumps(data)))
206
+ return data
207
+
208
+
209
+ def upload_bytes(path_in_repo, data, message=None, tenant_slug=None):
210
+ with _pool().connection() as con:
211
+ con.execute(
212
+ _q('INSERT INTO {tbl} (path, bytes, message) VALUES (%s, %s, %s) '
213
+ 'ON CONFLICT (path) DO UPDATE SET bytes = EXCLUDED.bytes, '
214
+ 'message = EXCLUDED.message, updated_at = now()', 'store_blobs', tenant_slug),
215
+ (str(path_in_repo), bytes(data), message))
216
+
217
+
218
+ def download_bytes(path_in_repo, tenant_slug=None):
219
+ """Raw bytes, or None when absent — the same contract as the HF backend: a missing attachment
220
+ degrades to "that file is gone", never to a broken page."""
221
+ with _pool().connection() as con:
222
+ row = con.execute(
223
+ _q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
224
+ (str(path_in_repo),)).fetchone()
225
+ return bytes(row[0]) if row else None
226
+
227
+
228
+ def delete_path(path_in_repo, tenant_slug=None):
229
+ """Absent is SUCCESS — deleting what is already gone is the goal (HF backend's contract)."""
230
+ with _pool().connection() as con:
231
+ con.execute(_q('DELETE FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
232
+ (str(path_in_repo),))
233
+ return True
234
+
235
+
236
+ def flush(name=None, timeout=30.0):
237
+ """A no-op that returns True: every write above is already committed when it returns. Kept in
238
+ the interface because QA gates and shutdown hooks call it and must not branch on the backend."""
239
+ return True
240
+
241
+
242
+ def close():
243
+ """Release the pool (tests, and a process that is shutting down cleanly)."""
244
+ with _LOCK:
245
+ if _POOL['pool'] is not None:
246
+ try:
247
+ _POOL['pool'].close()
248
+ finally:
249
+ _POOL['pool'] = None
250
+ _POOL['url'] = None
251
+ _POOL.pop('ok', None)
252
+
253
+
254
+ # ---------------------------------------------------------------------------------------------
255
+ # WAVE 20 (R1 / D-4) — THE TENANT-BOUND HANDLE. This is what made the cutover a small change.
256
+ #
257
+ # `core/store.py` grew a CLASS in wave 18 because a tenant can own its own dataset REPO, and
258
+ # `harness.runtime.TenantRuntime` carries one bound instance per tenant. Postgres isolates by
259
+ # SCHEMA instead (`t_<slug>`, see harness/pg/schema.sql's own argument for schema-over-RLS), so
260
+ # the two models meet here: one object, bound to one slug, exposing `core.store.Store`'s methods.
261
+ #
262
+ # ⚠ THE BINDING IS THE POINT. The module functions above default their schema from `AIOS_TENANT`,
263
+ # which is a PROCESS-wide answer to a PER-REQUEST question — correct for a single-tenant worker,
264
+ # wrong for the shared API that serves four tenants from one process. A handle carries its slug
265
+ # explicitly, so a write cannot land in another tenant's schema because some env var was right
266
+ # for the last request. That is the same property `TenantRuntime.store_handle` already gives the
267
+ # HF backend (EXIT-4b proof #3), reached a different way.
268
+ # ---------------------------------------------------------------------------------------------
269
+ class PgStore:
270
+ """`core.store.Store`'s interface for ONE tenant's schema.
271
+
272
+ Deliberately a thin binder over the module functions rather than a reimplementation: every
273
+ behaviour argument (FOR UPDATE in `update`, absent-is-success in `delete_path`, `fresh`
274
+ ignored because a SELECT has no cache to bypass) is stated once, up there, and cannot drift
275
+ between the two entry points.
276
+ """
277
+
278
+ def __init__(self, tenant_slug):
279
+ self.tenant_slug = str(tenant_slug or '').strip().lower() or 'royal-imports'
280
+ #: Kept so `Store`-shaped debugging (`repr`, telemetry, the admin panes) reads the same
281
+ #: on both backends. There is no repo here; the schema is the address.
282
+ self.repo = f'pg:{_schema_for(self.tenant_slug)}'
283
+
284
+ def __repr__(self):
285
+ return f'<PgStore {self.repo}>'
286
+
287
+ def available(self):
288
+ return available()
289
+
290
+ def get(self, name, fresh=False):
291
+ return get(name, fresh=fresh, tenant_slug=self.tenant_slug)
292
+
293
+ def _read_strict(self, name):
294
+ """Interface parity with the HF Store. There, `get` is lenient (swallows a transient
295
+ failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
296
+ of merging into `{}`. Here `get` ALREADY raises — the leniency it opts out of does not
297
+ exist — so the strict read is the ordinary one, and saying so beats a second code path."""
298
+ return get(name, tenant_slug=self.tenant_slug)
299
+
300
+ def exists(self, name):
301
+ return exists(name, tenant_slug=self.tenant_slug)
302
+
303
+ def put(self, name, data):
304
+ return put(name, data, tenant_slug=self.tenant_slug)
305
+
306
+ def update(self, name, fn, flush='sync'):
307
+ return update(name, fn, flush=flush, tenant_slug=self.tenant_slug)
308
+
309
+ def upload_bytes(self, path_in_repo, data, message=None):
310
+ return upload_bytes(path_in_repo, data, message=message, tenant_slug=self.tenant_slug)
311
+
312
+ def download_bytes(self, path_in_repo):
313
+ return download_bytes(path_in_repo, tenant_slug=self.tenant_slug)
314
+
315
+ def delete_path(self, path_in_repo):
316
+ return delete_path(path_in_repo, tenant_slug=self.tenant_slug)
317
+
318
+ def flush(self, name=None, timeout=30.0):
319
+ return True
320
+
321
+ def _flush_now_at_exit(self):
322
+ """The atexit hook `core.store` registers for its instances. Nothing is buffered here —
323
+ every write above committed before it returned — so this is honestly a no-op rather than
324
+ an unimplemented method that would raise during interpreter shutdown."""
325
+ return True
platform/core/table_store.py CHANGED
@@ -1,595 +1,595 @@
1
- """The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory.
2
-
3
- One durable store key holds one table OBJECT's per-user Airtable-style state:
4
-
5
- {username: {'views': {view_id: SavedView},
6
- 'fields': {field_key: Field}, # notes + custom_ + measure_ strata
7
- 'overlays': {str(pid): {field_key: value}}}}
8
-
9
- `make(table_key)` returns the six operations a table page's host loop needs, closed over that
10
- key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
11
- these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table
12
- pattern to a new object is a registry row + a config, not a copy of the store plumbing
13
- (owner directive: the table-page factory).
14
-
15
- A LIST (membership/formula semantics) is deliberately a different store from a VIEW
16
- (presentation/query state) — see modules/customer_data.py's customer_lists key.
17
- """
18
- import core.store as store
19
-
20
- #: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but
21
- #: their creator live here instead of in a personal workspace, under a key that cannot collide
22
- #: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
23
- #: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
24
- #: here, and a view shared is removed from its creator's workspace. Two homes would mean two
25
- #: divergent copies the moment either was edited.
26
- SHARED_KEY = '__shared__'
27
-
28
-
29
- def _may_see(view, viewer, is_admin=False):
30
- """Visibility for ONE shared view, fail-closed.
31
-
32
- 'collaborative' = everyone who can already open the module (the caller has gated that).
33
- 'users' = the named users, plus the creator, plus admins — an admin who could not
34
- see a view could not administer it either.
35
- Anything unrecognised returns False rather than defaulting open: an unreadable permission
36
- must never widen access ([[aios-permissioning]] — no fail-open defaults).
37
- """
38
- if not isinstance(view, dict):
39
- return False
40
- if view.get('createdBy') == viewer or is_admin:
41
- return True
42
- perms = view.get('permissions') or {}
43
- edit = perms.get('edit')
44
- if edit == 'collaborative':
45
- return True
46
- if edit == 'users':
47
- return viewer in set(perms.get('users') or ())
48
- return False # 'personal', absent, or junk
49
-
50
-
51
- def _may_edit(view, viewer, is_admin=False):
52
- """Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who
53
- can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
54
- separate function so they can diverge (a future read-only share) without hunting callers."""
55
- return _may_see(view, viewer, is_admin)
56
-
57
-
58
- def _may_administer(view, viewer, is_admin=False):
59
- """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
60
-
61
- Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
62
- could grant themselves sole ownership of somebody else's view, or quietly widen a
63
- users-scoped view to everyone — the classic privilege-escalation-by-edit hole.
64
- """
65
- if not isinstance(view, dict):
66
- return False
67
- return bool(is_admin) or view.get('createdBy') == viewer
68
-
69
-
70
- def is_shared(view):
71
- """A view belongs in the shared bucket when its permissions reach beyond its creator."""
72
- return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
73
-
74
-
75
- def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
76
- """Allocate one human-facing name inside a store.update transaction.
77
-
78
- Keys/ids remain structural identity. Names compare case-insensitively after collapsing
79
- whitespace, because those variants are indistinguishable in the UI. This helper belongs
80
- in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
81
- choose the same free name before either write lands.
82
- """
83
- limit = max(1, int(max_len))
84
-
85
- def _clean(value):
86
- return ' '.join(str(value or '').split())
87
-
88
- base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
89
- taken = {_clean(value).casefold() for value in existing if _clean(value)}
90
- if base.casefold() not in taken:
91
- return base
92
- index = 2
93
- while True:
94
- suffix = f' {index}'
95
- stem = base[:max(0, limit - len(suffix))].rstrip()
96
- candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
97
- if candidate.casefold() not in taken:
98
- return candidate
99
- index += 1
100
-
101
-
102
- class TableStore:
103
- """The six store operations for one table object's workspace, closed over its store key.
104
-
105
- `st` (wave 18, C3-UT) is the STORE HANDLE — anything exposing `get(name)` /
106
- `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
107
- zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
108
- apply the tenant prefix / repo binding — which is what makes a user table created by a
109
- Nurilab admin land in Nurilab's store instead of Royal's.
110
- """
111
-
112
- def __init__(self, table_key, st=None):
113
- self.table_key = table_key
114
- self._st = st if st is not None else store
115
-
116
- @property
117
- def st(self):
118
- """The bound store handle — for SIBLING registries (core/shares) that must read the
119
- same tenant's buckets this workspace lives in (wave 21, C1)."""
120
- return self._st
121
-
122
- def find_view(self, view_id):
123
- """`(owner_username, view)` for a view living in ANY personal stratum, else None.
124
-
125
- ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
126
- view means locating the OWNER's record inside this topic's bucket. Personal strata
127
- only — the `__shared__` bucket has its own read path (`shared_views`), and serving one
128
- view from two finders is how two copies drift."""
129
- vid = str(view_id or '').strip()
130
- if not vid:
131
- return None
132
- try:
133
- data = self._st.get(self.table_key) or {}
134
- except Exception:
135
- return None
136
- for username, ws in data.items():
137
- if username == SHARED_KEY or not isinstance(ws, dict):
138
- continue
139
- v = (ws.get('views') or {}).get(vid)
140
- if isinstance(v, dict):
141
- return str(username), dict(v)
142
- return None
143
-
144
- def find_folder(self, folder_id):
145
- """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
146
- personal stratum, else None. `find_view`'s sibling, and here for the same reason.
147
-
148
- ⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
149
- `folder` and has since wave 20, but only the VIEW kind was ever projected — so "share
150
- this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
151
- Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
152
- owns it, and WHICH views are filed in it. Folder membership lives in the owner's
153
- `itemFolders` map (item id -> folder id), never on the view record, so the views are
154
- found by asking that map rather than by reading a list off the folder.
155
-
156
- Views only (`folders['views']`): the cohort surface has its own store and its own
157
- sharing question, and answering both here would make one function mean two things.
158
- """
159
- fid = str(folder_id or '').strip()
160
- if not fid:
161
- return None
162
- try:
163
- data = self._st.get(self.table_key) or {}
164
- except Exception:
165
- return None
166
- for username, ws in data.items():
167
- if username == SHARED_KEY or not isinstance(ws, dict):
168
- continue
169
- rows = (ws.get('folders') or {}).get('views') or []
170
- hit = next((f for f in rows
171
- if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
172
- if not hit:
173
- continue
174
- # ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) —
175
- # reading item ids off the top level finds the surface names instead and matches
176
- # nothing, so the projection silently returns an EMPTY folder and the feature looks
177
- # exactly as broken as it was before the fix. Caught by this change's own gate,
178
- # which is the entire argument for writing one.
179
- placed = (ws.get('itemFolders') or {}).get('views') or {}
180
- views = ws.get('views') or {}
181
- inside = {str(vid): dict(v) for vid, v in views.items()
182
- if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
183
- return str(username), dict(hit), inside
184
- return None
185
-
186
- # ---------------------------------------------------------------- read
187
- def workspace(self, username, consume_corrections=True):
188
- """One user's durable workspace: always the full three-strata shape."""
189
- try:
190
- data = self._st.get(self.table_key) or {}
191
- ws = data.get(username, {}) or {}
192
- # A collision acknowledgement is protocol state, not part of a field definition.
193
- # Consume it with the first fresh workspace payload after the correcting write,
194
- # then splice a bounded copy into that payload only. Keeping it out of `fields`
195
- # prevents an old request id surviving forever and overriding a later rename.
196
- corrections = {}
197
- if consume_corrections and ws.get('fieldCorrections'):
198
- def _take(current):
199
- current_ws = current.get(username) or {}
200
- pending = current_ws.get('fieldCorrections') or {}
201
- corrections.update({
202
- str(key)[:80]: dict(value)
203
- for key, value in pending.items()
204
- if isinstance(value, dict)
205
- })
206
- current_ws.pop('fieldCorrections', None)
207
- return current
208
-
209
- data = self._st.update(self.table_key, _take, flush='async')
210
- ws = (data or {}).get(username, {}) or {}
211
- except Exception:
212
- ws = {}
213
- corrections = {}
214
- fields = {
215
- key: dict(value) if isinstance(value, dict) else value
216
- for key, value in (ws.get('fields') or {}).items()
217
- }
218
- for key, ack in corrections.items():
219
- field = fields.get(key)
220
- accepted_label = str(ack.get('label') or '')[:120]
221
- requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
222
- correction_id = str(ack.get('labelCorrectionId') or '')[:180]
223
- # A newer field write clears/replaces the pending ack in the SAME transaction.
224
- # The label check is an extra belt against ever attaching a stale ack to a newer
225
- # definition if a future store implementation weakens that ordering.
226
- if (isinstance(field, dict) and accepted_label
227
- and str(field.get('label') or '') == accepted_label
228
- and requested_label and correction_id):
229
- field['labelCorrectedFrom'] = requested_label
230
- field['labelCorrectionId'] = correction_id
231
- out = {
232
- 'views': dict(ws.get('views') or {}),
233
- 'fields': fields,
234
- 'overlays': dict(ws.get('overlays') or {}),
235
- # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
236
- # stratum rather than a key on each item — see aios_grid.clean_folders for why
237
- # (a cohort lives in another store, and filing is an organising act, not part of
238
- # what a view is). Absent for every workspace saved before this wave, which is
239
- # exactly "no folders yet".
240
- 'folders': dict(ws.get('folders') or {}),
241
- 'itemFolders': dict(ws.get('itemFolders') or {}),
242
- }
243
- # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
244
- # client's localStorage copy wins when present; this is the server's answer for a
245
- # fresh profile, which used to fall all the way to the system default view.
246
- if ws.get('activeViewId'):
247
- out['activeViewId'] = str(ws['activeViewId'])
248
- # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
249
- # stratum, absent until the user first reorders — exactly "default order".
250
- if isinstance(ws.get('recordLayout'), dict):
251
- out['recordLayout'] = dict(ws['recordLayout'])
252
- return out
253
-
254
- # ---------------------------------------------------------------- write
255
- def _update(self, username, change):
256
- def _up(data):
257
- ws = data.setdefault(username, {})
258
- ws.setdefault('views', {})
259
- ws.setdefault('fields', {})
260
- ws.setdefault('overlays', {})
261
- ws.setdefault('folders', {})
262
- ws.setdefault('itemFolders', {})
263
- change(ws)
264
- return data
265
- # flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak,
266
- # column note and typed overlay cell lands here inside the component round-trip, and
267
- # the historical synchronous hub commit cost seconds per edit. The mutation applies to
268
- # the in-process cache (read-your-writes for every subsequent render); the hub write
269
- # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
270
- return self._st.update(self.table_key, _up, flush='async')
271
-
272
- def rename_choice_values(self, username, change):
273
- """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
274
-
275
- ⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
276
- public "do anything to the workspace" method is an invitation to put write logic in
277
- callers instead of here, and every OTHER method on this class exists precisely because
278
- that logic belongs in one place. Renaming a choice is the one operation that must touch
279
- three strata AT ONCE — the field's `choices`, the cells in `overlays`, and the views that
280
- filter or colour by the old value — inside a SINGLE transaction, because a rename that
281
- updated the cells and not the filters would leave a saved view matching nothing.
282
-
283
- `change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
284
- """
285
- return self._update(username, change)
286
-
287
- def save_active_view(self, username, view_id):
288
- """Remember which view this user last opened (owner item 3, 2026-07-31).
289
-
290
- Presentation state, not authorisation: the READ side re-validates the id against what
291
- the caller may actually see, so a stale or foreign id degrades to the default view
292
- rather than granting anything. Stored per user like every other stratum.
293
- """
294
- vid = str(view_id or '').strip()[:120]
295
- if not vid or username == SHARED_KEY:
296
- return
297
-
298
- def _set(ws):
299
- ws['activeViewId'] = vid
300
- self._update(username, _set)
301
-
302
- def save_record_layout(self, username, order):
303
- """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
304
-
305
- Presentation state for ONE surface — the record modal. Deliberately not view config:
306
- the owner's ask is per-user, not per-view, and it must never reorder grid columns.
307
- The event handler validated keys against the live field set; the wire re-validates at
308
- serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
309
- An empty order clears the stratum back to "default order".
310
- """
311
- if username == SHARED_KEY:
312
- return
313
- clean, seen = [], set()
314
- for key in (order or [])[:200]:
315
- key = str(key or '').strip()[:80]
316
- if key and key not in seen:
317
- seen.add(key)
318
- clean.append(key)
319
-
320
- def _set(ws):
321
- if clean:
322
- ws['recordLayout'] = {'order': clean}
323
- else:
324
- ws.pop('recordLayout', None)
325
-
326
- self._update(username, _set)
327
-
328
- def save_folders(self, username, folders, item_folders):
329
- """Replace the folder stratum wholesale (wave-8 I11).
330
-
331
- Wholesale rather than per-folder because the caller has ALREADY validated the complete
332
- picture through aios_grid.clean_folders / clean_item_folders, and those two are
333
- interdependent: a placement is only legal while its folder exists, so committing them
334
- separately would leave a window where a reader sees an item filed into a folder that is
335
- not there yet. One write, one consistent state.
336
- """
337
- def _set(ws):
338
- ws['folders'] = dict(folders or {})
339
- ws['itemFolders'] = dict(item_folders or {})
340
- self._update(username, _set)
341
-
342
- def save_view_order(self, username, order):
343
- """⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail.
344
-
345
- Wholesale, like `save_folders` above and for the same reason: the client sends the full
346
- list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
347
- view went.
348
-
349
- ⛔ PER USER, and it belongs in this stratum rather than on the view records themselves.
350
- `aios_grid`'s own folder note argues it out for placements and every word applies: an
351
- arrangement is a per-user ORGANISING act, not part of what a view IS — so keeping it out
352
- of the view config means duplicating, sharing or exporting a view does not drag one
353
- person's rail position along with it. It also means a SHARED view can sit in a different
354
- place for each person who can see it, which is the only coherent answer once two people
355
- share one view.
356
-
357
- An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
358
- """
359
- def _set(ws):
360
- clean = []
361
- seen = set()
362
- for vid in (order or []):
363
- vid = str(vid).strip()[:120]
364
- if vid and vid not in seen:
365
- seen.add(vid)
366
- clean.append(vid)
367
- if clean:
368
- ws['viewOrder'] = clean
369
- else:
370
- ws.pop('viewOrder', None)
371
- self._update(username, _set)
372
-
373
- def shared_views(self, viewer, is_admin=False):
374
- """Every SHARED view this viewer may see, by id (wave-9 I17).
375
-
376
- Read-only and independent of the viewer's own workspace: the caller merges. Returns
377
- only what `_may_see` allows, so a caller cannot accidentally render somebody else's
378
- personal view by forgetting to filter.
379
- """
380
- try:
381
- bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
382
- except Exception:
383
- return {}
384
- return {vid: dict(v) for vid, v in bucket.items()
385
- if _may_see(v, viewer, is_admin)}
386
-
387
- def shared_view(self, view_id):
388
- """One shared view RAW — no visibility filter. For authorisation decisions only: a
389
- caller must know a view exists and who owns it before it can decide whether the actor
390
- may touch it. Never hand the result to a renderer without checking `_may_see`."""
391
- try:
392
- return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
393
- ).get('views', {}).get(str(view_id))
394
- except Exception:
395
- return None
396
-
397
- def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
398
- """Upsert a SavedView into its ONE home — personal workspace or the shared bucket.
399
-
400
- `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
401
- it lands in, the view is REMOVED from the other, so a view can never exist as two
402
- copies that diverge on the next edit.
403
-
404
- ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer
405
- moves data and does not know who is asking. `_cl_handle_one` is the wall.
406
- """
407
- view_id = str((view or {}).get('id') or '').strip()
408
- if not view_id:
409
- raise ValueError('view id is required')
410
- if username == SHARED_KEY:
411
- raise ValueError('reserved username')
412
- to_shared = is_shared(view) if shared is None else bool(shared)
413
- requested = dict(view)
414
- accepted = {}
415
-
416
- def _up(data):
417
- # View names are tenant-global: every personal workspace plus the shared bucket.
418
- # This deliberately includes views the actor cannot see. The only disclosed fact
419
- # is that a display name is already taken, while the categorical "no duplicate
420
- # view names" contract remains true when a personal view is later shared.
421
- names = list(reserved_names or ())
422
- for workspace in data.values():
423
- if not isinstance(workspace, dict):
424
- continue
425
- names.extend(
426
- value.get('name')
427
- for candidate_id, value in (workspace.get('views') or {}).items()
428
- if candidate_id != view_id and isinstance(value, dict)
429
- )
430
- payload = dict(requested)
431
- payload['name'] = _unique_name(payload.get('name'), names)
432
- accepted.clear()
433
- accepted.update(payload)
434
- if to_shared:
435
- bucket = data.setdefault(SHARED_KEY, {})
436
- bucket.setdefault('views', {})[view_id] = payload
437
- # it may have lived in the creator's workspace before being shared
438
- owner = data.get(payload.get('createdBy') or username) or {}
439
- (owner.get('views') or {}).pop(view_id, None)
440
- else:
441
- ws = data.setdefault(username, {})
442
- ws.setdefault('views', {})[view_id] = payload
443
- (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
444
- return data
445
-
446
- self._st.update(self.table_key, _up, flush='async')
447
- return dict(accepted)
448
-
449
- def delete_view(self, username, view_id):
450
- """Delete a custom/list view override. The system all-rows view is guarded by caller.
451
-
452
- Removes from BOTH homes: the caller has already authorised the delete, and leaving a
453
- stale copy in the other bucket would resurrect the view on the next read.
454
- """
455
- vid = str(view_id)
456
-
457
- def _up(data):
458
- (data.get(username, {}).get('views') or {}).pop(vid, None)
459
- (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
460
- return data
461
-
462
- self._st.update(self.table_key, _up, flush='async')
463
-
464
- def save_field(self, username, field, reserved_names=(), correction_id=None):
465
- """Persist a column note or a user-created (custom_/measure_) field definition."""
466
- key = str((field or {}).get('key') or '').strip()
467
- if not key:
468
- raise ValueError('field key is required')
469
- requested = dict(field)
470
- accepted = {}
471
-
472
- def _save(ws):
473
- names = list(reserved_names or ())
474
- names.extend(
475
- value.get('label')
476
- for candidate_key, value in (ws.get('fields') or {}).items()
477
- if candidate_key != key and isinstance(value, dict)
478
- )
479
- payload = dict(requested)
480
- payload.pop('labelCorrectedFrom', None)
481
- payload.pop('labelCorrectionId', None)
482
- requested_label = ' '.join(
483
- str(payload.get('label') or 'Untitled').split())[:120].rstrip()
484
- payload['label'] = _unique_name(requested_label, names)
485
- corrections = ws.setdefault('fieldCorrections', {})
486
- corrections.pop(key, None)
487
- if payload['label'] != requested_label and correction_id:
488
- corrections[key] = {
489
- 'label': payload['label'],
490
- 'labelCorrectedFrom': requested_label,
491
- 'labelCorrectionId': str(correction_id)[:180],
492
- }
493
- if not corrections:
494
- ws.pop('fieldCorrections', None)
495
- accepted.clear()
496
- accepted.update(payload)
497
- # A cleared note on an immutable source field returns to the canonical schema
498
- # instead of leaving a meaningless override row. Custom fields remain even with
499
- # an empty note — and so does a PRESET field carrying a measure-window override
500
- # (wave-2 item 8) or a DISPLAY-format override (wave-5 item 10): the note may be
501
- # empty but the period/format is real user state.
502
- if (not payload.get('custom') and not str(payload.get('note') or '').strip()
503
- and not isinstance(payload.get('measure'), dict)
504
- and not isinstance(payload.get('format'), dict)):
505
- ws['fields'].pop(key, None)
506
- else:
507
- ws['fields'][key] = payload
508
-
509
- self._update(username, _save)
510
- return dict(accepted)
511
-
512
- def duplicate_field(self, username, source_key, new_key, field,
513
- reserved_names=(), correction_id=None):
514
- """Clone a user-created field in ONE store transaction (wave-5 item 1): the new
515
- definition plus — for `custom_` overlay sources only — every stored cell value under
516
- the source key. One transaction, because a def without its values (or values without a
517
- def) is exactly the orphan state delete_field exists to prevent, in reverse.
518
- The caller validated both keys (same created stratum) and stamped the clone's
519
- createdBy; this layer only moves data."""
520
- source_key = str(source_key or '').strip()
521
- new_key = str(new_key or '').strip()
522
- if not source_key or not new_key or source_key == new_key:
523
- raise ValueError('duplicate_field needs two distinct keys')
524
- requested = dict(field)
525
- accepted = {}
526
-
527
- def _dup(ws):
528
- names = list(reserved_names or ())
529
- names.extend(
530
- value.get('label')
531
- for candidate_key, value in (ws.get('fields') or {}).items()
532
- if candidate_key != new_key and isinstance(value, dict)
533
- )
534
- payload = dict(requested)
535
- payload.pop('labelCorrectedFrom', None)
536
- payload.pop('labelCorrectionId', None)
537
- requested_label = ' '.join(
538
- str(payload.get('label') or 'Untitled').split())[:120].rstrip()
539
- payload['label'] = _unique_name(requested_label, names)
540
- corrections = ws.setdefault('fieldCorrections', {})
541
- corrections.pop(new_key, None)
542
- if payload['label'] != requested_label and correction_id:
543
- corrections[new_key] = {
544
- 'label': payload['label'],
545
- 'labelCorrectedFrom': requested_label,
546
- 'labelCorrectionId': str(correction_id)[:180],
547
- }
548
- if not corrections:
549
- ws.pop('fieldCorrections', None)
550
- accepted.clear()
551
- accepted.update(payload)
552
- ws['fields'][new_key] = payload
553
- if source_key.startswith('custom_'):
554
- for row in ws['overlays'].values():
555
- if isinstance(row, dict) and source_key in row:
556
- row[new_key] = row[source_key]
557
-
558
- self._update(username, _dup)
559
- return dict(accepted)
560
-
561
- def delete_field(self, username, key):
562
- """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
563
-
564
- Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
565
- columns — the caller enforces the prefix). The stored overlay VALUES for the key are
566
- scrubbed with it: a deleted column's cells must not linger as orphan data that would
567
- silently resurface if the key were ever reused. Views referencing the key self-heal on
568
- their next autosave (an unknown colId is dropped) — the rule every stale key rides.
569
- """
570
- key = str(key or '').strip()
571
- if not key:
572
- return
573
-
574
- def _drop(ws):
575
- ws['fields'].pop(key, None)
576
- for row in ws['overlays'].values():
577
- if isinstance(row, dict):
578
- row.pop(key, None)
579
-
580
- self._update(username, _drop)
581
-
582
- def patch_overlay(self, username, pid, updates):
583
- """Patch only the external editable stratum; never writes to the source system."""
584
- clean = dict(updates or {})
585
- if not clean:
586
- return
587
-
588
- def _patch(ws):
589
- ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
590
-
591
- self._update(username, _patch)
592
-
593
-
594
- def make(table_key, st=None):
595
- return TableStore(table_key, st=st)
 
1
+ """The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory.
2
+
3
+ One durable store key holds one table OBJECT's per-user Airtable-style state:
4
+
5
+ {username: {'views': {view_id: SavedView},
6
+ 'fields': {field_key: Field}, # notes + custom_ + measure_ strata
7
+ 'overlays': {str(pid): {field_key: value}}}}
8
+
9
+ `make(table_key)` returns the six operations a table page's host loop needs, closed over that
10
+ key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
11
+ these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table
12
+ pattern to a new object is a registry row + a config, not a copy of the store plumbing
13
+ (owner directive: the table-page factory).
14
+
15
+ A LIST (membership/formula semantics) is deliberately a different store from a VIEW
16
+ (presentation/query state) — see modules/customer_data.py's customer_lists key.
17
+ """
18
+ import core.store as store
19
+
20
+ #: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but
21
+ #: their creator live here instead of in a personal workspace, under a key that cannot collide
22
+ #: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
23
+ #: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
24
+ #: here, and a view shared is removed from its creator's workspace. Two homes would mean two
25
+ #: divergent copies the moment either was edited.
26
+ SHARED_KEY = '__shared__'
27
+
28
+
29
+ def _may_see(view, viewer, is_admin=False):
30
+ """Visibility for ONE shared view, fail-closed.
31
+
32
+ 'collaborative' = everyone who can already open the module (the caller has gated that).
33
+ 'users' = the named users, plus the creator, plus admins — an admin who could not
34
+ see a view could not administer it either.
35
+ Anything unrecognised returns False rather than defaulting open: an unreadable permission
36
+ must never widen access ([[aios-permissioning]] — no fail-open defaults).
37
+ """
38
+ if not isinstance(view, dict):
39
+ return False
40
+ if view.get('createdBy') == viewer or is_admin:
41
+ return True
42
+ perms = view.get('permissions') or {}
43
+ edit = perms.get('edit')
44
+ if edit == 'collaborative':
45
+ return True
46
+ if edit == 'users':
47
+ return viewer in set(perms.get('users') or ())
48
+ return False # 'personal', absent, or junk
49
+
50
+
51
+ def _may_edit(view, viewer, is_admin=False):
52
+ """Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who
53
+ can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
54
+ separate function so they can diverge (a future read-only share) without hunting callers."""
55
+ return _may_see(view, viewer, is_admin)
56
+
57
+
58
+ def _may_administer(view, viewer, is_admin=False):
59
+ """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
60
+
61
+ Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
62
+ could grant themselves sole ownership of somebody else's view, or quietly widen a
63
+ users-scoped view to everyone — the classic privilege-escalation-by-edit hole.
64
+ """
65
+ if not isinstance(view, dict):
66
+ return False
67
+ return bool(is_admin) or view.get('createdBy') == viewer
68
+
69
+
70
+ def is_shared(view):
71
+ """A view belongs in the shared bucket when its permissions reach beyond its creator."""
72
+ return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
73
+
74
+
75
+ def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
76
+ """Allocate one human-facing name inside a store.update transaction.
77
+
78
+ Keys/ids remain structural identity. Names compare case-insensitively after collapsing
79
+ whitespace, because those variants are indistinguishable in the UI. This helper belongs
80
+ in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
81
+ choose the same free name before either write lands.
82
+ """
83
+ limit = max(1, int(max_len))
84
+
85
+ def _clean(value):
86
+ return ' '.join(str(value or '').split())
87
+
88
+ base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
89
+ taken = {_clean(value).casefold() for value in existing if _clean(value)}
90
+ if base.casefold() not in taken:
91
+ return base
92
+ index = 2
93
+ while True:
94
+ suffix = f' {index}'
95
+ stem = base[:max(0, limit - len(suffix))].rstrip()
96
+ candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
97
+ if candidate.casefold() not in taken:
98
+ return candidate
99
+ index += 1
100
+
101
+
102
+ class TableStore:
103
+ """The six store operations for one table object's workspace, closed over its store key.
104
+
105
+ `st` (wave 18, C3-UT) is the STORE HANDLE — anything exposing `get(name)` /
106
+ `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
107
+ zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
108
+ apply the tenant prefix / repo binding — which is what makes a user table created by a
109
+ Nurilab admin land in Nurilab's store instead of Royal's.
110
+ """
111
+
112
+ def __init__(self, table_key, st=None):
113
+ self.table_key = table_key
114
+ self._st = st if st is not None else store
115
+
116
+ @property
117
+ def st(self):
118
+ """The bound store handle — for SIBLING registries (core/shares) that must read the
119
+ same tenant's buckets this workspace lives in (wave 21, C1)."""
120
+ return self._st
121
+
122
+ def find_view(self, view_id):
123
+ """`(owner_username, view)` for a view living in ANY personal stratum, else None.
124
+
125
+ ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
126
+ view means locating the OWNER's record inside this topic's bucket. Personal strata
127
+ only — the `__shared__` bucket has its own read path (`shared_views`), and serving one
128
+ view from two finders is how two copies drift."""
129
+ vid = str(view_id or '').strip()
130
+ if not vid:
131
+ return None
132
+ try:
133
+ data = self._st.get(self.table_key) or {}
134
+ except Exception:
135
+ return None
136
+ for username, ws in data.items():
137
+ if username == SHARED_KEY or not isinstance(ws, dict):
138
+ continue
139
+ v = (ws.get('views') or {}).get(vid)
140
+ if isinstance(v, dict):
141
+ return str(username), dict(v)
142
+ return None
143
+
144
+ def find_folder(self, folder_id):
145
+ """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
146
+ personal stratum, else None. `find_view`'s sibling, and here for the same reason.
147
+
148
+ ⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
149
+ `folder` and has since wave 20, but only the VIEW kind was ever projected — so "share
150
+ this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
151
+ Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
152
+ owns it, and WHICH views are filed in it. Folder membership lives in the owner's
153
+ `itemFolders` map (item id -> folder id), never on the view record, so the views are
154
+ found by asking that map rather than by reading a list off the folder.
155
+
156
+ Views only (`folders['views']`): the cohort surface has its own store and its own
157
+ sharing question, and answering both here would make one function mean two things.
158
+ """
159
+ fid = str(folder_id or '').strip()
160
+ if not fid:
161
+ return None
162
+ try:
163
+ data = self._st.get(self.table_key) or {}
164
+ except Exception:
165
+ return None
166
+ for username, ws in data.items():
167
+ if username == SHARED_KEY or not isinstance(ws, dict):
168
+ continue
169
+ rows = (ws.get('folders') or {}).get('views') or []
170
+ hit = next((f for f in rows
171
+ if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
172
+ if not hit:
173
+ continue
174
+ # ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) —
175
+ # reading item ids off the top level finds the surface names instead and matches
176
+ # nothing, so the projection silently returns an EMPTY folder and the feature looks
177
+ # exactly as broken as it was before the fix. Caught by this change's own gate,
178
+ # which is the entire argument for writing one.
179
+ placed = (ws.get('itemFolders') or {}).get('views') or {}
180
+ views = ws.get('views') or {}
181
+ inside = {str(vid): dict(v) for vid, v in views.items()
182
+ if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
183
+ return str(username), dict(hit), inside
184
+ return None
185
+
186
+ # ---------------------------------------------------------------- read
187
+ def workspace(self, username, consume_corrections=True):
188
+ """One user's durable workspace: always the full three-strata shape."""
189
+ try:
190
+ data = self._st.get(self.table_key) or {}
191
+ ws = data.get(username, {}) or {}
192
+ # A collision acknowledgement is protocol state, not part of a field definition.
193
+ # Consume it with the first fresh workspace payload after the correcting write,
194
+ # then splice a bounded copy into that payload only. Keeping it out of `fields`
195
+ # prevents an old request id surviving forever and overriding a later rename.
196
+ corrections = {}
197
+ if consume_corrections and ws.get('fieldCorrections'):
198
+ def _take(current):
199
+ current_ws = current.get(username) or {}
200
+ pending = current_ws.get('fieldCorrections') or {}
201
+ corrections.update({
202
+ str(key)[:80]: dict(value)
203
+ for key, value in pending.items()
204
+ if isinstance(value, dict)
205
+ })
206
+ current_ws.pop('fieldCorrections', None)
207
+ return current
208
+
209
+ data = self._st.update(self.table_key, _take, flush='async')
210
+ ws = (data or {}).get(username, {}) or {}
211
+ except Exception:
212
+ ws = {}
213
+ corrections = {}
214
+ fields = {
215
+ key: dict(value) if isinstance(value, dict) else value
216
+ for key, value in (ws.get('fields') or {}).items()
217
+ }
218
+ for key, ack in corrections.items():
219
+ field = fields.get(key)
220
+ accepted_label = str(ack.get('label') or '')[:120]
221
+ requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
222
+ correction_id = str(ack.get('labelCorrectionId') or '')[:180]
223
+ # A newer field write clears/replaces the pending ack in the SAME transaction.
224
+ # The label check is an extra belt against ever attaching a stale ack to a newer
225
+ # definition if a future store implementation weakens that ordering.
226
+ if (isinstance(field, dict) and accepted_label
227
+ and str(field.get('label') or '') == accepted_label
228
+ and requested_label and correction_id):
229
+ field['labelCorrectedFrom'] = requested_label
230
+ field['labelCorrectionId'] = correction_id
231
+ out = {
232
+ 'views': dict(ws.get('views') or {}),
233
+ 'fields': fields,
234
+ 'overlays': dict(ws.get('overlays') or {}),
235
+ # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
236
+ # stratum rather than a key on each item — see aios_grid.clean_folders for why
237
+ # (a cohort lives in another store, and filing is an organising act, not part of
238
+ # what a view is). Absent for every workspace saved before this wave, which is
239
+ # exactly "no folders yet".
240
+ 'folders': dict(ws.get('folders') or {}),
241
+ 'itemFolders': dict(ws.get('itemFolders') or {}),
242
+ }
243
+ # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
244
+ # client's localStorage copy wins when present; this is the server's answer for a
245
+ # fresh profile, which used to fall all the way to the system default view.
246
+ if ws.get('activeViewId'):
247
+ out['activeViewId'] = str(ws['activeViewId'])
248
+ # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
249
+ # stratum, absent until the user first reorders — exactly "default order".
250
+ if isinstance(ws.get('recordLayout'), dict):
251
+ out['recordLayout'] = dict(ws['recordLayout'])
252
+ return out
253
+
254
+ # ---------------------------------------------------------------- write
255
+ def _update(self, username, change):
256
+ def _up(data):
257
+ ws = data.setdefault(username, {})
258
+ ws.setdefault('views', {})
259
+ ws.setdefault('fields', {})
260
+ ws.setdefault('overlays', {})
261
+ ws.setdefault('folders', {})
262
+ ws.setdefault('itemFolders', {})
263
+ change(ws)
264
+ return data
265
+ # flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak,
266
+ # column note and typed overlay cell lands here inside the component round-trip, and
267
+ # the historical synchronous hub commit cost seconds per edit. The mutation applies to
268
+ # the in-process cache (read-your-writes for every subsequent render); the hub write
269
+ # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
270
+ return self._st.update(self.table_key, _up, flush='async')
271
+
272
+ def rename_choice_values(self, username, change):
273
+ """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
274
+
275
+ ⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
276
+ public "do anything to the workspace" method is an invitation to put write logic in
277
+ callers instead of here, and every OTHER method on this class exists precisely because
278
+ that logic belongs in one place. Renaming a choice is the one operation that must touch
279
+ three strata AT ONCE — the field's `choices`, the cells in `overlays`, and the views that
280
+ filter or colour by the old value — inside a SINGLE transaction, because a rename that
281
+ updated the cells and not the filters would leave a saved view matching nothing.
282
+
283
+ `change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
284
+ """
285
+ return self._update(username, change)
286
+
287
+ def save_active_view(self, username, view_id):
288
+ """Remember which view this user last opened (owner item 3, 2026-07-31).
289
+
290
+ Presentation state, not authorisation: the READ side re-validates the id against what
291
+ the caller may actually see, so a stale or foreign id degrades to the default view
292
+ rather than granting anything. Stored per user like every other stratum.
293
+ """
294
+ vid = str(view_id or '').strip()[:120]
295
+ if not vid or username == SHARED_KEY:
296
+ return
297
+
298
+ def _set(ws):
299
+ ws['activeViewId'] = vid
300
+ self._update(username, _set)
301
+
302
+ def save_record_layout(self, username, order):
303
+ """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
304
+
305
+ Presentation state for ONE surface — the record modal. Deliberately not view config:
306
+ the owner's ask is per-user, not per-view, and it must never reorder grid columns.
307
+ The event handler validated keys against the live field set; the wire re-validates at
308
+ serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
309
+ An empty order clears the stratum back to "default order".
310
+ """
311
+ if username == SHARED_KEY:
312
+ return
313
+ clean, seen = [], set()
314
+ for key in (order or [])[:200]:
315
+ key = str(key or '').strip()[:80]
316
+ if key and key not in seen:
317
+ seen.add(key)
318
+ clean.append(key)
319
+
320
+ def _set(ws):
321
+ if clean:
322
+ ws['recordLayout'] = {'order': clean}
323
+ else:
324
+ ws.pop('recordLayout', None)
325
+
326
+ self._update(username, _set)
327
+
328
+ def save_folders(self, username, folders, item_folders):
329
+ """Replace the folder stratum wholesale (wave-8 I11).
330
+
331
+ Wholesale rather than per-folder because the caller has ALREADY validated the complete
332
+ picture through aios_grid.clean_folders / clean_item_folders, and those two are
333
+ interdependent: a placement is only legal while its folder exists, so committing them
334
+ separately would leave a window where a reader sees an item filed into a folder that is
335
+ not there yet. One write, one consistent state.
336
+ """
337
+ def _set(ws):
338
+ ws['folders'] = dict(folders or {})
339
+ ws['itemFolders'] = dict(item_folders or {})
340
+ self._update(username, _set)
341
+
342
+ def save_view_order(self, username, order):
343
+ """⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail.
344
+
345
+ Wholesale, like `save_folders` above and for the same reason: the client sends the full
346
+ list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
347
+ view went.
348
+
349
+ ⛔ PER USER, and it belongs in this stratum rather than on the view records themselves.
350
+ `aios_grid`'s own folder note argues it out for placements and every word applies: an
351
+ arrangement is a per-user ORGANISING act, not part of what a view IS — so keeping it out
352
+ of the view config means duplicating, sharing or exporting a view does not drag one
353
+ person's rail position along with it. It also means a SHARED view can sit in a different
354
+ place for each person who can see it, which is the only coherent answer once two people
355
+ share one view.
356
+
357
+ An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
358
+ """
359
+ def _set(ws):
360
+ clean = []
361
+ seen = set()
362
+ for vid in (order or []):
363
+ vid = str(vid).strip()[:120]
364
+ if vid and vid not in seen:
365
+ seen.add(vid)
366
+ clean.append(vid)
367
+ if clean:
368
+ ws['viewOrder'] = clean
369
+ else:
370
+ ws.pop('viewOrder', None)
371
+ self._update(username, _set)
372
+
373
+ def shared_views(self, viewer, is_admin=False):
374
+ """Every SHARED view this viewer may see, by id (wave-9 I17).
375
+
376
+ Read-only and independent of the viewer's own workspace: the caller merges. Returns
377
+ only what `_may_see` allows, so a caller cannot accidentally render somebody else's
378
+ personal view by forgetting to filter.
379
+ """
380
+ try:
381
+ bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
382
+ except Exception:
383
+ return {}
384
+ return {vid: dict(v) for vid, v in bucket.items()
385
+ if _may_see(v, viewer, is_admin)}
386
+
387
+ def shared_view(self, view_id):
388
+ """One shared view RAW — no visibility filter. For authorisation decisions only: a
389
+ caller must know a view exists and who owns it before it can decide whether the actor
390
+ may touch it. Never hand the result to a renderer without checking `_may_see`."""
391
+ try:
392
+ return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
393
+ ).get('views', {}).get(str(view_id))
394
+ except Exception:
395
+ return None
396
+
397
+ def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
398
+ """Upsert a SavedView into its ONE home — personal workspace or the shared bucket.
399
+
400
+ `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
401
+ it lands in, the view is REMOVED from the other, so a view can never exist as two
402
+ copies that diverge on the next edit.
403
+
404
+ ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer
405
+ moves data and does not know who is asking. `_cl_handle_one` is the wall.
406
+ """
407
+ view_id = str((view or {}).get('id') or '').strip()
408
+ if not view_id:
409
+ raise ValueError('view id is required')
410
+ if username == SHARED_KEY:
411
+ raise ValueError('reserved username')
412
+ to_shared = is_shared(view) if shared is None else bool(shared)
413
+ requested = dict(view)
414
+ accepted = {}
415
+
416
+ def _up(data):
417
+ # View names are tenant-global: every personal workspace plus the shared bucket.
418
+ # This deliberately includes views the actor cannot see. The only disclosed fact
419
+ # is that a display name is already taken, while the categorical "no duplicate
420
+ # view names" contract remains true when a personal view is later shared.
421
+ names = list(reserved_names or ())
422
+ for workspace in data.values():
423
+ if not isinstance(workspace, dict):
424
+ continue
425
+ names.extend(
426
+ value.get('name')
427
+ for candidate_id, value in (workspace.get('views') or {}).items()
428
+ if candidate_id != view_id and isinstance(value, dict)
429
+ )
430
+ payload = dict(requested)
431
+ payload['name'] = _unique_name(payload.get('name'), names)
432
+ accepted.clear()
433
+ accepted.update(payload)
434
+ if to_shared:
435
+ bucket = data.setdefault(SHARED_KEY, {})
436
+ bucket.setdefault('views', {})[view_id] = payload
437
+ # it may have lived in the creator's workspace before being shared
438
+ owner = data.get(payload.get('createdBy') or username) or {}
439
+ (owner.get('views') or {}).pop(view_id, None)
440
+ else:
441
+ ws = data.setdefault(username, {})
442
+ ws.setdefault('views', {})[view_id] = payload
443
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
444
+ return data
445
+
446
+ self._st.update(self.table_key, _up, flush='async')
447
+ return dict(accepted)
448
+
449
+ def delete_view(self, username, view_id):
450
+ """Delete a custom/list view override. The system all-rows view is guarded by caller.
451
+
452
+ Removes from BOTH homes: the caller has already authorised the delete, and leaving a
453
+ stale copy in the other bucket would resurrect the view on the next read.
454
+ """
455
+ vid = str(view_id)
456
+
457
+ def _up(data):
458
+ (data.get(username, {}).get('views') or {}).pop(vid, None)
459
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
460
+ return data
461
+
462
+ self._st.update(self.table_key, _up, flush='async')
463
+
464
+ def save_field(self, username, field, reserved_names=(), correction_id=None):
465
+ """Persist a column note or a user-created (custom_/measure_) field definition."""
466
+ key = str((field or {}).get('key') or '').strip()
467
+ if not key:
468
+ raise ValueError('field key is required')
469
+ requested = dict(field)
470
+ accepted = {}
471
+
472
+ def _save(ws):
473
+ names = list(reserved_names or ())
474
+ names.extend(
475
+ value.get('label')
476
+ for candidate_key, value in (ws.get('fields') or {}).items()
477
+ if candidate_key != key and isinstance(value, dict)
478
+ )
479
+ payload = dict(requested)
480
+ payload.pop('labelCorrectedFrom', None)
481
+ payload.pop('labelCorrectionId', None)
482
+ requested_label = ' '.join(
483
+ str(payload.get('label') or 'Untitled').split())[:120].rstrip()
484
+ payload['label'] = _unique_name(requested_label, names)
485
+ corrections = ws.setdefault('fieldCorrections', {})
486
+ corrections.pop(key, None)
487
+ if payload['label'] != requested_label and correction_id:
488
+ corrections[key] = {
489
+ 'label': payload['label'],
490
+ 'labelCorrectedFrom': requested_label,
491
+ 'labelCorrectionId': str(correction_id)[:180],
492
+ }
493
+ if not corrections:
494
+ ws.pop('fieldCorrections', None)
495
+ accepted.clear()
496
+ accepted.update(payload)
497
+ # A cleared note on an immutable source field returns to the canonical schema
498
+ # instead of leaving a meaningless override row. Custom fields remain even with
499
+ # an empty note — and so does a PRESET field carrying a measure-window override
500
+ # (wave-2 item 8) or a DISPLAY-format override (wave-5 item 10): the note may be
501
+ # empty but the period/format is real user state.
502
+ if (not payload.get('custom') and not str(payload.get('note') or '').strip()
503
+ and not isinstance(payload.get('measure'), dict)
504
+ and not isinstance(payload.get('format'), dict)):
505
+ ws['fields'].pop(key, None)
506
+ else:
507
+ ws['fields'][key] = payload
508
+
509
+ self._update(username, _save)
510
+ return dict(accepted)
511
+
512
+ def duplicate_field(self, username, source_key, new_key, field,
513
+ reserved_names=(), correction_id=None):
514
+ """Clone a user-created field in ONE store transaction (wave-5 item 1): the new
515
+ definition plus — for `custom_` overlay sources only — every stored cell value under
516
+ the source key. One transaction, because a def without its values (or values without a
517
+ def) is exactly the orphan state delete_field exists to prevent, in reverse.
518
+ The caller validated both keys (same created stratum) and stamped the clone's
519
+ createdBy; this layer only moves data."""
520
+ source_key = str(source_key or '').strip()
521
+ new_key = str(new_key or '').strip()
522
+ if not source_key or not new_key or source_key == new_key:
523
+ raise ValueError('duplicate_field needs two distinct keys')
524
+ requested = dict(field)
525
+ accepted = {}
526
+
527
+ def _dup(ws):
528
+ names = list(reserved_names or ())
529
+ names.extend(
530
+ value.get('label')
531
+ for candidate_key, value in (ws.get('fields') or {}).items()
532
+ if candidate_key != new_key and isinstance(value, dict)
533
+ )
534
+ payload = dict(requested)
535
+ payload.pop('labelCorrectedFrom', None)
536
+ payload.pop('labelCorrectionId', None)
537
+ requested_label = ' '.join(
538
+ str(payload.get('label') or 'Untitled').split())[:120].rstrip()
539
+ payload['label'] = _unique_name(requested_label, names)
540
+ corrections = ws.setdefault('fieldCorrections', {})
541
+ corrections.pop(new_key, None)
542
+ if payload['label'] != requested_label and correction_id:
543
+ corrections[new_key] = {
544
+ 'label': payload['label'],
545
+ 'labelCorrectedFrom': requested_label,
546
+ 'labelCorrectionId': str(correction_id)[:180],
547
+ }
548
+ if not corrections:
549
+ ws.pop('fieldCorrections', None)
550
+ accepted.clear()
551
+ accepted.update(payload)
552
+ ws['fields'][new_key] = payload
553
+ if source_key.startswith('custom_'):
554
+ for row in ws['overlays'].values():
555
+ if isinstance(row, dict) and source_key in row:
556
+ row[new_key] = row[source_key]
557
+
558
+ self._update(username, _dup)
559
+ return dict(accepted)
560
+
561
+ def delete_field(self, username, key):
562
+ """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
563
+
564
+ Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
565
+ columns — the caller enforces the prefix). The stored overlay VALUES for the key are
566
+ scrubbed with it: a deleted column's cells must not linger as orphan data that would
567
+ silently resurface if the key were ever reused. Views referencing the key self-heal on
568
+ their next autosave (an unknown colId is dropped) — the rule every stale key rides.
569
+ """
570
+ key = str(key or '').strip()
571
+ if not key:
572
+ return
573
+
574
+ def _drop(ws):
575
+ ws['fields'].pop(key, None)
576
+ for row in ws['overlays'].values():
577
+ if isinstance(row, dict):
578
+ row.pop(key, None)
579
+
580
+ self._update(username, _drop)
581
+
582
+ def patch_overlay(self, username, pid, updates):
583
+ """Patch only the external editable stratum; never writes to the source system."""
584
+ clean = dict(updates or {})
585
+ if not clean:
586
+ return
587
+
588
+ def _patch(ws):
589
+ ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
590
+
591
+ self._update(username, _patch)
592
+
593
+
594
+ def make(table_key, st=None):
595
+ return TableStore(table_key, st=st)
platform/core/user_tables.py CHANGED
@@ -676,6 +676,23 @@ ROLLUP_CONDITION_OPS = ('eq', 'neq', 'contains', 'not_contains', 'is_empty', 'is
676
  'gt', 'gte', 'lt', 'lte')
677
  ROLLUP_CONDITION_CONJ = ('and', 'or')
678
  ROLLUP_MAX_CONDITIONS = 20
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
679
  #: The ceiling on `limit`. Not a performance bound — `MAX_ROWS` is that — but a refusal to let a
680
  #: column claim a window bigger than a table can hold.
681
  ROLLUP_MAX_LIMIT = MAX_ROWS
@@ -747,6 +764,41 @@ def _clean_rollup(raw):
747
  """
748
  if not isinstance(raw, dict):
749
  return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
750
  link = _field_key(raw.get('link'))
751
  field = _field_key(raw.get('field'))
752
  fn = str(raw.get('fn') or '').strip().lower()
 
676
  'gt', 'gte', 'lt', 'lte')
677
  ROLLUP_CONDITION_CONJ = ('and', 'or')
678
  ROLLUP_MAX_CONDITIONS = 20
679
+
680
+ #: The date-window KINDS a source-backed rollup may name. MIRRORS `harness/windows.py`'s
681
+ #: `WINDOW_KINDS`, deliberately as a local literal — `core` stays dependency-light and does not
682
+ #: import up, the same reason `UT_FIELD_TYPES` is a literal rather than an `aios_grid` import.
683
+ #: ⚠ This list VALIDATES; it never RESOLVES. `harness.windows.resolve(spec, today)` owns turning a
684
+ #: kind into a date pair, so there is exactly one implementation of what "ytd" means and this
685
+ #: module cannot drift into a second one. A kind added there and not here is simply not offerable
686
+ #: on a rollup yet — the fail-closed direction.
687
+ ROLLUP_SOURCE_WINDOWS = (
688
+ 'all_time', 'today', 'yesterday', 'this_week', 'last_week', 'this_month', 'last_month',
689
+ 'this_quarter', 'last_quarter', 'this_year', 'last_year', 'ytd', 'ytd_last_year',
690
+ 'ltm', 'past_week', 'past_month', 'past_year',
691
+ )
692
+ #: ⛔ DELIBERATELY ABSENT: `last_n_days`, `next_n_days`, `custom`. Each needs a PARAMETER (an `n`,
693
+ #: or a date pair) that this bag has nowhere to carry, and a parameterised kind accepted without
694
+ #: its parameter would resolve to something arbitrary — a window that looks configured and
695
+ #: measures the wrong period. They land when the bag learns to carry the parameter, not before.
696
  #: The ceiling on `limit`. Not a performance bound — `MAX_ROWS` is that — but a refusal to let a
697
  #: column claim a window bigger than a table can hold.
698
  ROLLUP_MAX_LIMIT = MAX_ROWS
 
764
  """
765
  if not isinstance(raw, dict):
766
  return None
767
+ # ⭐ A SOURCE-BACKED ROLLUP — the read-through kind (owner 2026-08-09: *"make rollup be able to
768
+ # capture up to 1 million rows of a linked database … ok to push rollup filter etc to SQL"*).
769
+ #
770
+ # ⛔ WHY IT IS A DIFFERENT SHAPE RATHER THAN A BIGGER `link`. A linked rollup folds ROWS THAT
771
+ # EXIST IN THE STORE, and `MAX_ROWS = 5000` bounds them because every `ut_*` table lives in one
772
+ # JSON blob that is parsed and deep-copied per request (MEASURED: 5.1 MB / 1,567 rows = 67 ms a
773
+ # read). 256,810 order lines cannot live there and never will. This kind never COPIES the rows
774
+ # at all: it names a governed semantic TOPIC and a METRIC KEY, and one grouped SQL query
775
+ # answers every parent row at once (MEASURED: 1,748 customers over 256,810 lines in 232 ms).
776
+ #
777
+ # ⚠ A METRIC KEY, NEVER A FILTER FRAGMENT, and that is the load-bearing decision. `model/
778
+ # metrics/*.yml` already carries each metric's scope, its `store_filter_sql` AND the matching
779
+ # `live_domain` — whose own comment says *"BOTH or store_parity compares two different
780
+ # questions"*. Binding to the key inherits the scope and the live-parity oracle; letting a
781
+ # rollup carry SQL would mint a second definition of a number the semantic layer exists to
782
+ # define once.
783
+ #
784
+ # ⚠ `window` is a date-window KIND (`ytd`, `this_month`, …) resolved against `today` at
785
+ # COMPUTE time, never a literal date pair — a hand-computed year start is correct until
786
+ # 1 January and wrong after it, with nothing to notice ([[date-window-vocabulary]]).
787
+ src = raw.get('source')
788
+ if isinstance(src, dict):
789
+ topic = _field_key(src.get('topic'))
790
+ measure = _field_key(src.get('measure'))
791
+ group_by = _field_key(src.get('groupBy'))
792
+ on = _field_key(src.get('on'))
793
+ if not (topic and measure and group_by and on):
794
+ return None
795
+ bag = {'topic': topic, 'measure': measure, 'groupBy': group_by, 'on': on}
796
+ window = str(src.get('window') or '').strip().lower()
797
+ if window:
798
+ if window not in ROLLUP_SOURCE_WINDOWS:
799
+ return None
800
+ bag['window'] = window
801
+ return {'source': bag}
802
  link = _field_key(raw.get('link'))
803
  field = _field_key(raw.get('field'))
804
  fn = str(raw.get('fn') or '').strip().lower()
platform/evals/analyst_golden.yml CHANGED
@@ -1,104 +1,104 @@
1
- # Analyst golden eval set v1 (OM-4 gate, 2026-07-11; truths REFRESHED 2026-07-17 after the
2
- # Jul-15 Odoo mass-recompute + hard-delete reconciliation — closed windows DO drift when docs
3
- # are edited/deleted; refresh truths from the parity-proven layer, never loosen tolerances).
4
- # Original header: truths computed from the PARITY-PROVEN
5
- # semantic layer over CLOSED windows (H1 2026 = 2026-01-01..2026-06-30) so live data drift cannot
6
- # break them. Checks are DETERMINISTIC (number within tolerance + required substrings + tool-use
7
- # assertions) — the validate() culture applied to AI answers. Run: harness/evals.py.
8
- # expect.value tolerance: abs or rel (fraction). expect.contains: ALL substrings (case-insensitive).
9
- # expect.tools: tool names that MUST appear in the trace (the answer must come FROM the tools).
10
- # expect.not_tools: tool names that must NOT appear (the over-refusal guard — an answerable ask
11
- # must never end in report_gap; a gap ask must never end in a forced wrong query).
12
- items:
13
- - id: rev_h1
14
- ask: "What was our wholesale revenue for the first half of 2026 (Jan 1 through Jun 30)?"
15
- expect: {value: 3501508.33, rel: 0.001, tools: [run_semantic_query]}
16
- - id: rev_h1_fisch
17
- ask: "Fisch revenue only, H1 2026 (Jan 1 - Jun 30)?"
18
- expect: {value: 2494244.70, rel: 0.001, tools: [run_semantic_query]}
19
- - id: rev_h1_royal
20
- ask: "And Royal's revenue for the same H1 2026 window?"
21
- expect: {value: 1007263.63, rel: 0.001, tools: [run_semantic_query]}
22
- - id: rev_june
23
- ask: "Revenue for June 2026?"
24
- expect: {value: 535046.06, rel: 0.001, tools: [run_semantic_query]}
25
- - id: margin_h1
26
- ask: "Gross margin dollars for H1 2026 (Jan-Jun), wholesale?"
27
- expect: {value: 2067509.45, rel: 0.001, tools: [run_semantic_query]}
28
- - id: margin_pct_h1
29
- ask: "What gross margin percentage did we run in H1 2026?"
30
- expect: {value: 0.59, abs: 0.005, allow_pct_form: true, tools: [run_semantic_query]}
31
- - id: orders_h1
32
- ask: "How many confirmed wholesale orders did we take in H1 2026 (Jan 1 - Jun 30)?"
33
- expect: {value: 4989, abs: 1, tools: [run_semantic_query]}
34
- - id: customers_h1
35
- ask: "How many distinct customers bought from us in H1 2026?"
36
- expect: {value: 1037, abs: 1, tools: [run_semantic_query]}
37
- - id: top_customer
38
- ask: "Who was our biggest customer by revenue in H1 2026?"
39
- expect: {contains: ["Poppy Flowers"], tools: [run_semantic_query], not_tools: [report_gap]}
40
- - id: opex_h1
41
- ask: "Total operating expense on the GL for H1 2026 (Jan 1 - Jun 30)?"
42
- expect: {value: 4683428.81, rel: 0.001, tools: [run_semantic_query]}
43
- - id: top_expense
44
- ask: "What is our single largest expense account in H1 2026?"
45
- expect: {contains: ["FBA"], tools: [run_semantic_query]}
46
- - id: chart_request
47
- ask: "Make me a line chart of monthly wholesale revenue for H1 2026."
48
- expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
49
- - id: bu_guard
50
- ask: "Show one single revenue line combining Fisch and Royal together per month for H1 2026."
51
- expect: {tools: [run_semantic_query]} # combined-total ok; the guard is it must NOT error
52
- - id: name_resolution
53
- ask: "How much revenue did the customer 'flower' — I don't remember the exact name, something with FLOWERS in it — bring in H1 2026? Pick the top match."
54
- expect: {tools: [get_field_values, run_semantic_query]}
55
- - id: map_request
56
- ask: "Put my top 20 customers by H1 2026 revenue on a map."
57
- expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
58
- # --- the gap loop + location/YoY competence (added 2026-07-16 with the debug fixes) ---
59
- - id: city_filter
60
- ask: "Which customer in Brooklyn brought us the most revenue in H1 2026 (Jan 1 - Jun 30)?"
61
- expect: {value: 48785.00, rel: 0.001, contains: ["J KAY"],
62
- tools: [get_field_values, run_semantic_query], not_tools: [report_gap]}
63
- - id: yoy_decliners
64
- ask: "Which Fisch customer declined the most in revenue in H1 2026 (Jan 1 - Jun 30) versus the same window of 2025? Give the dollar decline."
65
- expect: {value: 16451.25, rel: 0.005, contains: ["JUNE"],
66
- tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
67
- # --- the agent dimension (added 2026-07-17: res.partner.agent_ids m2m sync — the #1 logged
68
- # gap, closed). agent_book is VERBATIM the old gap_refusal ask: yesterday's honest refusal
69
- # must become today's answer.
70
- - id: agent_book
71
- ask: "Who are the top customers handled by our salesperson Naomi in H1 2026?"
72
- expect: {contains: ["Poppy Flowers"], tools: [get_field_values, run_semantic_query],
73
- not_tools: [report_gap]}
74
- - id: agent_revenue
75
- ask: "How much revenue did agent Naomi's book of customers bring in H1 2026 (Jan 1 - Jun 30)?"
76
- expect: {value: 872869.98, rel: 0.005, tools: [run_semantic_query], not_tools: [report_gap]}
77
- # --- the TWO agent sources (added 2026-07-28). Odoo holds a customer-master book AND a
78
- # per-invoice-line commission agent; they name different people, and the commission table also
79
- # carries internal SALESPEOPLE who are not agents. These three lock the distinction: the
80
- # unallocated bucket must be answerable at all, and Anna/Shantal must never be called agents.
81
- - id: agent_unallocated
82
- ask: "Using the invoice-line basis, how much of Fisch's 2025 sales was NOT allocated to an agent?"
83
- expect: {value: 1447579.59, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
84
- - id: agent_not_salesperson
85
- ask: "Who are our agents on the Fisch invoice lines in 2025, and what did each generate?"
86
- expect: {contains: ["Pasternak"], not_contains: ["Anna", "Shantal"],
87
- tools: [run_semantic_query], not_tools: [report_gap]}
88
- - id: agent_allocated_2025
89
- ask: "On the invoice-line basis, how much of Fisch's 2025 sales was allocated to a real agent?"
90
- expect: {value: 2995664.75, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
91
- - id: gap_refusal
92
- ask: "How many units of our best-selling SKU do we physically have in stock right now?"
93
- expect: {tools: [report_gap]} # stock-on-hand has NO topic in the store — must self-report, never guess
94
- # --- the analytics transform library (added 2026-07-18: 60-op Tableau-class set). Truths from
95
- # the parity-proven semantic layer over the CLOSED H1 2026 window.
96
- - id: abc_a_tier
97
- ask: "Using an ABC / Pareto analysis of H1 2026 revenue (Jan 1 - Jun 30), how many customers fall in the A tier — the vital few that make up the first 80% of revenue?"
98
- expect: {value: 311, abs: 6, tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
99
- - id: ytd_customers
100
- ask: "How many distinct customers had we sold to cumulatively from Jan 1 through the end of March 2026?"
101
- expect: {value: 790, abs: 1, tools: [run_semantic_query], not_tools: [report_gap]}
102
- - id: concentration_top5
103
- ask: "What percent of our H1 2026 revenue (Jan 1 - Jun 30) is accounted for by our top 5 customers?"
104
- expect: {value: 7.84, abs: 0.5, tools: [run_semantic_query], not_tools: [report_gap]}
 
1
+ # Analyst golden eval set v1 (OM-4 gate, 2026-07-11; truths REFRESHED 2026-07-17 after the
2
+ # Jul-15 Odoo mass-recompute + hard-delete reconciliation — closed windows DO drift when docs
3
+ # are edited/deleted; refresh truths from the parity-proven layer, never loosen tolerances).
4
+ # Original header: truths computed from the PARITY-PROVEN
5
+ # semantic layer over CLOSED windows (H1 2026 = 2026-01-01..2026-06-30) so live data drift cannot
6
+ # break them. Checks are DETERMINISTIC (number within tolerance + required substrings + tool-use
7
+ # assertions) — the validate() culture applied to AI answers. Run: harness/evals.py.
8
+ # expect.value tolerance: abs or rel (fraction). expect.contains: ALL substrings (case-insensitive).
9
+ # expect.tools: tool names that MUST appear in the trace (the answer must come FROM the tools).
10
+ # expect.not_tools: tool names that must NOT appear (the over-refusal guard — an answerable ask
11
+ # must never end in report_gap; a gap ask must never end in a forced wrong query).
12
+ items:
13
+ - id: rev_h1
14
+ ask: "What was our wholesale revenue for the first half of 2026 (Jan 1 through Jun 30)?"
15
+ expect: {value: 3501508.33, rel: 0.001, tools: [run_semantic_query]}
16
+ - id: rev_h1_fisch
17
+ ask: "Fisch revenue only, H1 2026 (Jan 1 - Jun 30)?"
18
+ expect: {value: 2494244.70, rel: 0.001, tools: [run_semantic_query]}
19
+ - id: rev_h1_royal
20
+ ask: "And Royal's revenue for the same H1 2026 window?"
21
+ expect: {value: 1007263.63, rel: 0.001, tools: [run_semantic_query]}
22
+ - id: rev_june
23
+ ask: "Revenue for June 2026?"
24
+ expect: {value: 535046.06, rel: 0.001, tools: [run_semantic_query]}
25
+ - id: margin_h1
26
+ ask: "Gross margin dollars for H1 2026 (Jan-Jun), wholesale?"
27
+ expect: {value: 2067509.45, rel: 0.001, tools: [run_semantic_query]}
28
+ - id: margin_pct_h1
29
+ ask: "What gross margin percentage did we run in H1 2026?"
30
+ expect: {value: 0.59, abs: 0.005, allow_pct_form: true, tools: [run_semantic_query]}
31
+ - id: orders_h1
32
+ ask: "How many confirmed wholesale orders did we take in H1 2026 (Jan 1 - Jun 30)?"
33
+ expect: {value: 4989, abs: 1, tools: [run_semantic_query]}
34
+ - id: customers_h1
35
+ ask: "How many distinct customers bought from us in H1 2026?"
36
+ expect: {value: 1037, abs: 1, tools: [run_semantic_query]}
37
+ - id: top_customer
38
+ ask: "Who was our biggest customer by revenue in H1 2026?"
39
+ expect: {contains: ["Poppy Flowers"], tools: [run_semantic_query], not_tools: [report_gap]}
40
+ - id: opex_h1
41
+ ask: "Total operating expense on the GL for H1 2026 (Jan 1 - Jun 30)?"
42
+ expect: {value: 4683428.81, rel: 0.001, tools: [run_semantic_query]}
43
+ - id: top_expense
44
+ ask: "What is our single largest expense account in H1 2026?"
45
+ expect: {contains: ["FBA"], tools: [run_semantic_query]}
46
+ - id: chart_request
47
+ ask: "Make me a line chart of monthly wholesale revenue for H1 2026."
48
+ expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
49
+ - id: bu_guard
50
+ ask: "Show one single revenue line combining Fisch and Royal together per month for H1 2026."
51
+ expect: {tools: [run_semantic_query]} # combined-total ok; the guard is it must NOT error
52
+ - id: name_resolution
53
+ ask: "How much revenue did the customer 'flower' — I don't remember the exact name, something with FLOWERS in it — bring in H1 2026? Pick the top match."
54
+ expect: {tools: [get_field_values, run_semantic_query]}
55
+ - id: map_request
56
+ ask: "Put my top 20 customers by H1 2026 revenue on a map."
57
+ expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
58
+ # --- the gap loop + location/YoY competence (added 2026-07-16 with the debug fixes) ---
59
+ - id: city_filter
60
+ ask: "Which customer in Brooklyn brought us the most revenue in H1 2026 (Jan 1 - Jun 30)?"
61
+ expect: {value: 48785.00, rel: 0.001, contains: ["J KAY"],
62
+ tools: [get_field_values, run_semantic_query], not_tools: [report_gap]}
63
+ - id: yoy_decliners
64
+ ask: "Which Fisch customer declined the most in revenue in H1 2026 (Jan 1 - Jun 30) versus the same window of 2025? Give the dollar decline."
65
+ expect: {value: 16451.25, rel: 0.005, contains: ["JUNE"],
66
+ tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
67
+ # --- the agent dimension (added 2026-07-17: res.partner.agent_ids m2m sync — the #1 logged
68
+ # gap, closed). agent_book is VERBATIM the old gap_refusal ask: yesterday's honest refusal
69
+ # must become today's answer.
70
+ - id: agent_book
71
+ ask: "Who are the top customers handled by our salesperson Naomi in H1 2026?"
72
+ expect: {contains: ["Poppy Flowers"], tools: [get_field_values, run_semantic_query],
73
+ not_tools: [report_gap]}
74
+ - id: agent_revenue
75
+ ask: "How much revenue did agent Naomi's book of customers bring in H1 2026 (Jan 1 - Jun 30)?"
76
+ expect: {value: 872869.98, rel: 0.005, tools: [run_semantic_query], not_tools: [report_gap]}
77
+ # --- the TWO agent sources (added 2026-07-28). Odoo holds a customer-master book AND a
78
+ # per-invoice-line commission agent; they name different people, and the commission table also
79
+ # carries internal SALESPEOPLE who are not agents. These three lock the distinction: the
80
+ # unallocated bucket must be answerable at all, and Anna/Shantal must never be called agents.
81
+ - id: agent_unallocated
82
+ ask: "Using the invoice-line basis, how much of Fisch's 2025 sales was NOT allocated to an agent?"
83
+ expect: {value: 1447579.59, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
84
+ - id: agent_not_salesperson
85
+ ask: "Who are our agents on the Fisch invoice lines in 2025, and what did each generate?"
86
+ expect: {contains: ["Pasternak"], not_contains: ["Anna", "Shantal"],
87
+ tools: [run_semantic_query], not_tools: [report_gap]}
88
+ - id: agent_allocated_2025
89
+ ask: "On the invoice-line basis, how much of Fisch's 2025 sales was allocated to a real agent?"
90
+ expect: {value: 2995664.75, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
91
+ - id: gap_refusal
92
+ ask: "How many units of our best-selling SKU do we physically have in stock right now?"
93
+ expect: {tools: [report_gap]} # stock-on-hand has NO topic in the store — must self-report, never guess
94
+ # --- the analytics transform library (added 2026-07-18: 60-op Tableau-class set). Truths from
95
+ # the parity-proven semantic layer over the CLOSED H1 2026 window.
96
+ - id: abc_a_tier
97
+ ask: "Using an ABC / Pareto analysis of H1 2026 revenue (Jan 1 - Jun 30), how many customers fall in the A tier — the vital few that make up the first 80% of revenue?"
98
+ expect: {value: 311, abs: 6, tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
99
+ - id: ytd_customers
100
+ ask: "How many distinct customers had we sold to cumulatively from Jan 1 through the end of March 2026?"
101
+ expect: {value: 790, abs: 1, tools: [run_semantic_query], not_tools: [report_gap]}
102
+ - id: concentration_top5
103
+ ask: "What percent of our H1 2026 revenue (Jan 1 - Jun 30) is accounted for by our top 5 customers?"
104
+ expect: {value: 7.84, abs: 0.5, tools: [run_semantic_query], not_tools: [report_gap]}
platform/harness/filter_sql.py CHANGED
@@ -1,673 +1,673 @@
1
- """harness/filter_sql.py — the grid's filter/sort engine, compiled to SQL (CG-1).
2
-
3
- THE PORT. `customer-grid/useVisibleRows.ts` evaluates the Airtable-parity filter TREE and the
4
- multi-level sort in TypeScript, per row, in the browser. That works at customer grain (1,550 rows,
5
- 576 KB) and does NOT work at line grain, so a line-grain table has to filter server-side. This
6
- module is the second implementation of that contract, emitting parameterized SQL instead of
7
- booleans. `aios_grid.clean_filter_tree` (the validator) is the third. All three are held in
8
- lock-step by `aios-web/verify_filter_engine.py`, which runs the SAME cases through the TS engine,
9
- this compiler (executed on DuckDB), and the validator, and compares row sets.
10
-
11
- Design: this file is PURE — no model, no store, no app imports. It takes a filter tree plus a
12
- `columns` spec and returns SQL + params. `harness/semantic.store_columns` builds the spec from a
13
- topic; nothing else here knows what a topic is.
14
-
15
- columns = {colId: {"sql": <SQL expression>, "type": <field type>, "aggregate": bool}}
16
-
17
- `sql` is a trusted IDENTIFIER/expression from the semantic model (never user input); every VALUE
18
- is a bound parameter. That split is the whole injection story.
19
-
20
- --------------------------------------------------------------------------------------------
21
- TRI-STATE, AND WHY IT BECOMES COMPILE-TIME PRUNING
22
- --------------------------------------------------------------------------------------------
23
- `evalNode` in TS returns true | false | null, where null means "this node is INACTIVE and its
24
- parent must IGNORE it". Get that wrong and a half-typed condition returns true, makes its `or`
25
- group true, and shows every record.
26
-
27
- Activeness reads ONLY the rule (op, value, value2, is-the-column-known) — never the row. So in
28
- SQL it is not a third truth value at all: it is decided while COMPILING. An inactive node emits
29
- no SQL, a group with no active children emits no SQL, and a root with nothing active emits NO
30
- PREDICATE AT ALL (`compile_filter_tree` returns None) — which is exactly `matchFilterTree`
31
- returning true for every row.
32
-
33
- --------------------------------------------------------------------------------------------
34
- THE FOUR PLACES A NAIVE PORT SILENTLY DIVERGES
35
- --------------------------------------------------------------------------------------------
36
- JavaScript coerces null to 0 / "" before comparing; SQL propagates NULL and drops the row.
37
-
38
- 1. `toNum(null)` is 0, so a NULL revenue PASSES `>= 0`. SQL: NULL >= 0 -> NULL -> dropped.
39
- 2. `String(null)` is "", so a NULL state PASSES `is not FL`. SQL: NULL <> 'fl' -> dropped.
40
- 3. `"" <= "2026-01-01"` is true, so a NULL date PASSES `is before`. SQL: dropped.
41
- 4. `contains` is a LITERAL substring test. `LIKE '%v%'` treats % and _ as wildcards, so a
42
- value containing either matches the wrong rows. Hence `strpos`, not LIKE.
43
-
44
- Every column is therefore read through `_value_sql`, which COALESCEs to the same zero-value
45
- JavaScript would have coerced to. The one thing that must NOT be coalesced away is the
46
- blank test itself — `isBlank` looks at the RAW value, and numeric 0 is NOT blank.
47
-
48
- --------------------------------------------------------------------------------------------
49
- AND THE FIFTH: `default: return true` IS ACTIVE, NOT INACTIVE
50
- --------------------------------------------------------------------------------------------
51
- `matchFilter`'s per-type switch ends in `default: return true`, reachable whenever an operator
52
- does not apply to the field type — `contains` on a currency/date field, or `gt`/`between` on a
53
- text field. A persisted tree can hold such a pair. That rule is ACTIVE and evaluates TRUE for
54
- every row, which is NOT the same as inactive: under `or` it makes the whole group true and shows
55
- everything. We emit `1=1` for those, and the gate asserts the difference is observable.
56
-
57
- --------------------------------------------------------------------------------------------
58
- INPUT CONTRACT
59
- --------------------------------------------------------------------------------------------
60
- Feed this CLEANED trees (`aios_grid.clean_filter_tree`). It is deliberately faithful to the TS
61
- engine rather than fail-closed on garbage — an unknown OP compiles to `1=1` exactly as TS's
62
- `default:` branch returns true — because the validator's job is to reject garbage and this
63
- module's job is to mean the same thing as the client. The gate proves the composition.
64
-
65
- Ported ahead of their consumer (CG-2) and gated at the same fidelity as the tree:
66
- `compile_search` (`matchSearch`) and `member_ids` (the hand-picked-members union in
67
- `useVisibleRows`).
68
- """
69
- import math
70
- import re
71
- from collections import namedtuple
72
-
73
- # Mirrors customer-grid/types.ts FilterOp. verify_filter_engine.py asserts this set is
74
- # byte-identical to aios_grid.FILTER_OPS — three copies of a vocabulary need a drift gate.
75
- FILTER_OPS = frozenset({
76
- 'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty',
77
- 'gt', 'gte', 'lt', 'lte', 'between', 'within',
78
- 'topN', 'bottomN', 'inTopPct', 'inBottomPct',
79
- 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile',
80
- })
81
- #: Wave 2026-08-02 (C-OPS): the RANK subset — set-ranked over the sibling-filtered domain.
82
- #: Vocabulary here, execution NOWHERE in this module: a per-row WHERE cannot express Top-N,
83
- #: and compiling to 1=0 would answer "nothing" to a question that has an answer, under a
84
- #: row_count that still looks authoritative. `_compile_leaf` therefore REFUSES (ValueError),
85
- #: the same server-side posture as a relative anchor with no `today`. The TS engine
86
- #: evaluates these as a set pass in useVisibleRows (gated by web/verify_rank_filters.py).
87
- RANK_OPS = frozenset({'topN', 'bottomN', 'inTopPct', 'inBottomPct',
88
- 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'})
89
- #: The RESERVED pseudo-column of a cohort-membership leaf. Mirrors types.ts COHORT_FIELD.
90
- COHORT_FIELD = '__cohort__'
91
- #: Set operators over a SET of cohorts. Mirrors types.ts COHORT_OPS; DISJOINT from FILTER_OPS.
92
- COHORT_OPS = frozenset({'anyOf', 'allOf', 'noneOf'})
93
- #: The single-cohort ops the leaf shipped with, kept as permanent aliases. types.ts mirrors this.
94
- COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'}
95
- #: Mirrors types.ts MAX_COHORT_IDS.
96
- MAX_COHORT_IDS = 20
97
-
98
-
99
- def parse_cohort_ids(value):
100
- """The cohorts a leaf names. Mirrors types.ts `cohortIds()` and aios_grid.parse_cohort_ids."""
101
- out = []
102
- for raw in ('' if value is None else str(value)).split(','):
103
- cid = raw.strip()[:120]
104
- if not cid or cid in out:
105
- continue
106
- out.append(cid)
107
- if len(out) >= MAX_COHORT_IDS:
108
- break
109
- return out
110
- # types.ts VALUE_FREE_OPS: no value control is rendered, and they must never be judged
111
- # "inactive because the value is blank" — that would turn them into no-ops.
112
- VALUE_FREE_OPS = frozenset({'isEmpty', 'isNotEmpty', 'aboveAvg', 'belowAvg'})
113
- # types.ts isNumericType()
114
- NUMERIC_TYPES = frozenset({'currency', 'int', 'pct'})
115
- TEXTUAL_TYPES = frozenset({'text', 'status'})
116
-
117
- SQL_TRUE = '1=1'
118
-
119
- _NUM_CMP = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'}
120
- _DATE_CMP = dict(_NUM_CMP)
121
-
122
- #: sql = the predicate (already parenthesised); params = bound values, positional;
123
- #: uses_aggregate = the predicate references an aggregate column, so it belongs in HAVING
124
- #: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
125
- #: always-true `1=1` branch references nothing, and must not force HAVING routing).
126
- Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
127
-
128
-
129
- # --------------------------------------------------------------------------- value coercion
130
-
131
- # JS `Number(string)` trims exactly this set (WhiteSpace + LineTerminator). Python's float() and
132
- # str.strip() use a DIFFERENT set -- notably U+0085 NEL, which Python treats as whitespace and JS
133
- # does not -- so the set is spelled out rather than left to strip().
134
- _JS_WS = "".join(chr(c) for c in (9, 10, 11, 12, 13, 32, 160, 5760, 8232, 8233,
135
- 8239, 8287, 12288, 65279)) + \
136
- "".join(chr(c) for c in range(8192, 8203))
137
- # ASCII-ONLY on purpose: `\d` in a Python str pattern matches every Unicode decimal digit, so
138
- # `\d` here would parse the fullwidth "250" that JS rejects -- reintroducing the exact bug
139
- # this function exists to avoid.
140
- _JS_DEC = re.compile(r"^[+-]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$")
141
- _JS_INF = re.compile(r"^[+-]?Infinity$")
142
- _JS_RADIX = ((re.compile(r"^0[xX][0-9a-fA-F]+$"), 16),
143
- (re.compile(r"^0[oO][0-7]+$"), 8),
144
- (re.compile(r"^0[bB][01]+$"), 2))
145
-
146
-
147
- def to_num(v):
148
- """Port of `toNum`: JS `Number(v)`, then `Number.isFinite` or 0.
149
-
150
- NOT `float(v)`. Python's float() is MORE permissive than JS's Number() in ways that make SQL
151
- narrow where the client shows everything -- the worse direction, because at line grain that
152
- is a silent under-fetch sitting beside an honest-looking row_count:
153
-
154
- "1_000" float() -> 1000.0 (PEP-515 underscores) Number() -> NaN -> 0
155
- "250" float() -> 250.0 (any Unicode Nd) Number() -> NaN -> 0
156
- "\\u0085" + "12" float() -> 12.0 (NEL is Python whitespace) Number() -> NaN -> 0
157
-
158
- and LESS permissive in one family, which errs the other way:
159
-
160
- "0x10" float() -> raises -> 0 Number() -> 16 (also 0b, 0o)
161
-
162
- A tree carrying any of these is reachable: the React builder renders <input type="number">,
163
- but persisted / seeded / imported view JSON goes through `clean_filter_tree`, which passes
164
- the value string through untouched by design (it validates structure, not numeric syntax).
165
-
166
- Agreed by construction on the rest: ""/None -> 0, "abc"/"12,000" -> 0, " 250 " -> 250,
167
- ".5"/"5."/"+5", "1e400" -> Infinity -> 0, "-0" -> 0, and float64 truncation of
168
- "9007199254740993" -> 9007199254740992.
169
- """
170
- if isinstance(v, bool):
171
- return 1.0 if v else 0.0 # JS Number(true) === 1
172
- if isinstance(v, (int, float)):
173
- n = float(v)
174
- return n if math.isfinite(n) else 0.0
175
- if v is None:
176
- return 0.0 # JS Number(null) === 0
177
- s = str(v).strip(_JS_WS)
178
- if s == "":
179
- return 0.0 # JS Number("") === 0
180
- if _JS_INF.match(s):
181
- return 0.0 # Infinity is not finite -> 0
182
- for pattern, base in _JS_RADIX:
183
- if pattern.match(s):
184
- return float(int(s[2:], base))
185
- if not _JS_DEC.match(s):
186
- return 0.0 # JS NaN -> 0
187
- n = float(s)
188
- return n if math.isfinite(n) else 0.0
189
-
190
-
191
- def _value_sql(spec):
192
- """The ONE expression for a column, used by BOTH the filter and the sort.
193
-
194
- Two expressions would let two rows tie in the grid and not tie in SQL (dates truncated for
195
- comparison but ordered by the raw timestamp is the obvious way to get that wrong), so the
196
- next sort key would break the tie in one engine and not the other.
197
-
198
- numeric: `round_even(x, 0)` reproduces `aios_grid._round`, which is Python's round() =
199
- BANKER'S rounding. DuckDB's plain ROUND() rounds half AWAY from zero, so ROUND(1234.5)=1235
200
- while the grid shows 1234 — `eq 1234` would then match on screen and miss in SQL. The grid
201
- displays rounded values; filters must agree with what is on screen.
202
- date: truncate to the 10-char ISO shape the payload carries, so lexical compare is
203
- chronological and a timestamp column cannot smuggle ' 00:00:00' into the comparison.
204
- text: lowercased, matching `String(raw ?? "").toLowerCase()`.
205
- """
206
- e = spec['sql']
207
- t = spec['type']
208
- if t in NUMERIC_TYPES:
209
- return f"COALESCE(round_even(TRY_CAST({e} AS DOUBLE), 0), 0)"
210
- if t == 'date':
211
- return f"COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '')"
212
- return f"LOWER(COALESCE(CAST({e} AS VARCHAR), ''))"
213
-
214
-
215
- def _blank_sql(spec):
216
- """Port of `isBlank`: null/undefined/"" are blank — numeric 0 is NOT.
217
-
218
- Reads the RAW column, never `_value_sql` (which has already coalesced blanks into 0/"",
219
- the very distinction this test exists to make).
220
- """
221
- e = spec['sql']
222
- if spec['type'] in NUMERIC_TYPES:
223
- # 0 stays non-blank: CAST(0 AS VARCHAR) is '0'. The '' arm covers a text-typed column
224
- # holding "", which `_round` passes through unchanged into the payload.
225
- return f"(({e}) IS NULL OR CAST({e} AS VARCHAR) = '')"
226
- if spec['type'] == 'date':
227
- return f"(COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '') = '')"
228
- return f"(COALESCE(CAST({e} AS VARCHAR), '') = '')"
229
-
230
-
231
- # --------------------------------------------------------------------------- activeness
232
-
233
- def is_rule_active(rule, columns):
234
- """Port of `isRuleActive` + `evalNode`'s two leaf guards. Row-independent by construction.
235
-
236
- Unknown column -> inactive (evalNode returns null before matchFilter is ever called).
237
- Value-free ops are always active. `between` needs BOTH bounds.
238
-
239
- ⚠ The four VALUE-FREE ANCHOR MODES (`is before [today]`) carry no value, and the historical
240
- "blank value = inactive" rule would IGNORE them — widening the result under a count nobody
241
- would doubt. Same shape as the isEmpty/isNotEmpty no-op bug, arriving by a different door.
242
- """
243
- if not isinstance(rule, dict):
244
- return False
245
- col = rule.get('colId')
246
- if col == COHORT_FIELD: # a cohort leaf has no column to look up
247
- # Driven by the SET it names, so a value of "," is INACTIVE rather than active-and-
248
- # unanswerable. Mirrors the cohort branch at the top of types.ts isRuleActive.
249
- return bool(parse_cohort_ids(rule.get('value')))
250
- if col not in columns:
251
- return False
252
- op = rule.get('op')
253
- if op in VALUE_FREE_OPS:
254
- return True
255
- if op == 'within':
256
- return _window_bounds(rule.get('dateWindow'), None, probe=True)
257
- if rule.get('rhs') is not None:
258
- rhs = rule['rhs']
259
- # An rhs naming a column this table does not have is IGNORED, by the same rule as an
260
- # unknown left column two lines above — for EITHER kind. Ignoring one side and hiding
261
- # every row for the other would make the comparison asymmetric in a way nobody could
262
- # predict. Doing it here rather than in _compile_leaf keeps activeness the single place
263
- # that decides, which is what makes the compiled branch below provably unreachable.
264
- if not isinstance(rhs, dict) or rhs.get('colId') not in columns:
265
- return False
266
- return _rhs_complete(rhs)
267
- if rule.get('dateMode') in _ANCHOR_VALUE_FREE:
268
- return True
269
- value = _as_str(rule.get('value'))
270
- if op == 'between':
271
- v2 = rule.get('value2')
272
- return value != '' and v2 is not None and _as_str(v2) != ''
273
- return value != ''
274
-
275
-
276
- def _rhs_complete(rhs):
277
- """Port of types.ts isRhsComplete. An incomplete rhs makes the rule INACTIVE, exactly as a
278
- half-typed value does."""
279
- if not isinstance(rhs, dict) or not rhs.get('colId'):
280
- return False
281
- if rhs.get('kind') == 'measure':
282
- return _windows().normalize(rhs.get('window')) is not None
283
- return rhs.get('kind') == 'field'
284
-
285
-
286
- def _windows():
287
- """`harness.windows` imported lazily, so this module stays importable on its own.
288
-
289
- filter_sql is otherwise PURE (no model/store/app imports) and that property is what lets
290
- aios_grid embed the grid anywhere. windows.py is the one exception and it is a deliberate
291
- one: the date vocabulary is a two-ENGINE contract, and a third copy of "what the past month
292
- means" is precisely the drift verify_windows.py exists to prevent. It imports nothing but
293
- datetime, so purity is preserved in substance.
294
- """
295
- from harness import windows as _w
296
- return _w
297
-
298
-
299
- #: Mirrors windows.ANCHOR_VALUE_FREE, resolved lazily on first use (see _windows).
300
- class _AnchorValueFree:
301
- def __contains__(self, mode):
302
- return mode in _windows().ANCHOR_VALUE_FREE
303
-
304
-
305
- _ANCHOR_VALUE_FREE = _AnchorValueFree()
306
-
307
-
308
- def _window_bounds(spec, today, probe=False):
309
- """Resolve a date-condition RANGE, or refuse.
310
-
311
- `probe=True` answers only "is this a resolvable window?" without needing `today` — which is
312
- what activeness has to know while compiling, before any date is in hand.
313
-
314
- ⚠ A missing `today` on the real path RAISES rather than widening. The client's answer to the
315
- same gap is to match nothing; the server's must be to refuse, because a server that quietly
316
- drops a condition reports an authoritative row_count for a query nobody asked for. Same rule
317
- as the sibling/depth caps: truncate on the client, refuse on the server.
318
- """
319
- w = _windows()
320
- if probe:
321
- return w.normalize(spec) is not None
322
- if today is None:
323
- raise ValueError(
324
- 'a relative-date condition needs `today` — compile_filter_tree(today=...). '
325
- 'Refusing rather than dropping the condition, which would widen the result while '
326
- 'the count beside it still looked authoritative.')
327
- return w.resolve(spec, today)
328
-
329
-
330
- def _as_str(v):
331
- """The builder stores values as raw input strings; be tolerant of a number that survived
332
- an older payload. `str(500) != ''` is active and `to_num` reads it back, matching JS where
333
- `500 === ""` is false."""
334
- return '' if v is None else str(v)
335
-
336
-
337
- # --------------------------------------------------------------------------- leaf
338
-
339
- def _compile_leaf(rule, columns, state):
340
- """Port of `matchFilter`. Returns (sql, params) or None when the rule is INACTIVE."""
341
- if not is_rule_active(rule, columns):
342
- return None
343
- col_id = rule['colId']
344
- op = rule.get('op')
345
- value = _as_str(rule.get('value'))
346
-
347
- # --- owner item 5: a COHORT leaf, before any column lookup -------------------------------
348
- # `__cohort__` is not a column. Handled first for the same reason evalNode handles it first:
349
- # falling through to the column path would make it INACTIVE, and inactive means every row.
350
- if col_id == COHORT_FIELD:
351
- return _compile_cohort(rule, state)
352
-
353
- # --- C-OPS (wave 2026-08-02): a RANK leaf is structurally unanswerable at row grain ------
354
- # Top-N / quantile membership is a property of the SET, not the row; the client engine
355
- # answers it with a set pass over the sibling-filtered domain. Refuse loudly rather than
356
- # compile something that means a different question (see RANK_OPS).
357
- if op in RANK_OPS:
358
- raise ValueError(
359
- f'rank operator {op!r} cannot compile to row SQL - evaluate it as a set pass '
360
- f'(the client engine) or pre-resolve the matching ids before compiling')
361
-
362
- spec = columns[col_id]
363
-
364
- def used(sql, params=()):
365
- state['used'].add(col_id)
366
- if spec.get('aggregate'):
367
- state['aggregate'] = True
368
- return (sql, list(params))
369
-
370
- # --- value-free ops, BEFORE any value-based short-circuit (the no-op bug) ---------------
371
- if op == 'isEmpty':
372
- return used(_blank_sql(spec))
373
- if op == 'isNotEmpty':
374
- return used(f"NOT {_blank_sql(spec)}")
375
-
376
- v = _value_sql(spec)
377
- t = spec['type']
378
-
379
- # --- CG-9: the comparand is another COLUMN, coerced through the LEFT field's type --------
380
- # `_value_sql` on a spec whose `sql` is the RIGHT column and whose `type` is the LEFT one —
381
- # exactly what TS does (`toNum(other)` for a numeric LHS, `String(other ?? "")` for a date,
382
- # `.toLowerCase()` for text). Reading the right column through its OWN type would compare a
383
- # rounded number against an unrounded one and disagree on the ties.
384
- rhs_v = None
385
- if rule.get('rhs') is not None:
386
- # No `rspec is None` branch: `is_rule_active` already rejected an rhs naming a column
387
- # this table does not have, so it cannot be missing here. A defensive fallback would be
388
- # unreachable, and the negative control showed exactly that — deleting it changed
389
- # nothing, which is how dead code hides behind a careful-looking line.
390
- rspec = columns[rule['rhs']['colId']]
391
- rhs_v = _value_sql({'sql': rspec['sql'], 'type': t})
392
- state['used'].add(rule['rhs']['colId'])
393
- if rspec.get('aggregate'):
394
- state['aggregate'] = True
395
-
396
- if t in NUMERIC_TYPES:
397
- if rhs_v is not None:
398
- return used(f"({v} {_NUM_CMP[op]} {rhs_v})") if op in _NUM_CMP else (SQL_TRUE, [])
399
- if op == 'between':
400
- a, b = to_num(value), to_num(rule.get('value2'))
401
- return used(f"({v} >= ? AND {v} <= ?)", [min(a, b), max(a, b)])
402
- if op in _NUM_CMP:
403
- return used(f"({v} {_NUM_CMP[op]} ?)", [to_num(value)])
404
- return (SQL_TRUE, []) # contains/doesNotContain on a number -> default: true
405
-
406
- if t == 'date':
407
- if op == 'within':
408
- bounds = _window_bounds(rule.get('dateWindow'), state.get('today'))
409
- if bounds is None:
410
- return ('1=0', []) # unreachable: activeness probed it. Never widen.
411
- lo, hi = bounds
412
- # `{v} <> ''` is NOT redundant. `_value_sql` COALESCEs a NULL date to '', and
413
- # '' <= any upper bound is true — so an open-ended window would sweep in every
414
- # never-ordered customer. TS guards it with `if (a === "") return false`.
415
- parts, params = [f"{v} <> ''"], []
416
- if lo is not None:
417
- parts.append(f"{v} >= ?")
418
- params.append(lo)
419
- if hi is not None:
420
- parts.append(f"{v} <= ?")
421
- params.append(hi)
422
- return used('(' + ' AND '.join(parts) + ')', params)
423
- if rhs_v is not None:
424
- return used(f"({v} {_DATE_CMP[op]} {rhs_v})") if op in _DATE_CMP else (SQL_TRUE, [])
425
- if op == 'between':
426
- v2 = _as_str(rule.get('value2'))
427
- lo, hi = (value, v2) if value < v2 else (v2, value)
428
- return used(f"({v} >= ? AND {v} <= ?)", [lo, hi])
429
- if op in _DATE_CMP:
430
- anchor = _resolve_anchor(rule.get('dateMode'), value, state.get('today'))
431
- if anchor is None:
432
- return ('1=0', []) # unresolvable anchor -> matches nothing (TS: false)
433
- return used(f"({v} {_DATE_CMP[op]} ?)", [anchor])
434
- return (SQL_TRUE, []) # contains/doesNotContain on a date -> default: true
435
-
436
- # text / status — case-insensitive. strpos, not LIKE: `includes` is a LITERAL substring
437
- # test, so a value holding % or _ must not act as a wildcard.
438
- #
439
- # BOTH SIDES fold in SQL (`LOWER(?)`, never Python's str.lower()). Python and JS agree on
440
- # case folding; DuckDB does NOT — LOWER('ISTANBUL' with a dotted capital I) drops the dot
441
- # while both languages keep it as i + combining dot. Folding the needle in Python and the
442
- # haystack in DuckDB therefore produced two different strings, and a value could fail to
443
- # match ITSELF. One folder for both operands is the only version that cannot do that.
444
- if rhs_v is not None:
445
- # Both operands are already folded by `_value_sql` (text branch), so no LOWER(?) here.
446
- if op == 'contains':
447
- return used(f"(strpos({v}, {rhs_v}) > 0)")
448
- if op == 'doesNotContain':
449
- return used(f"(strpos({v}, {rhs_v}) = 0)")
450
- if op == 'eq':
451
- return used(f"({v} = {rhs_v})")
452
- if op == 'neq':
453
- return used(f"({v} <> {rhs_v})")
454
- return (SQL_TRUE, [])
455
- if op == 'contains':
456
- return used(f"(strpos({v}, LOWER(?)) > 0)", [value])
457
- if op == 'doesNotContain':
458
- return used(f"(strpos({v}, LOWER(?)) = 0)", [value])
459
- if op == 'eq':
460
- return used(f"({v} = LOWER(?))", [value])
461
- if op == 'neq':
462
- return used(f"({v} <> LOWER(?))", [value])
463
- return (SQL_TRUE, []) # gt/gte/lt/lte/between/within on text -> default: true
464
-
465
-
466
- def _resolve_anchor(mode, value, today):
467
- """A date condition's anchor -> one ISO date, or None. `today` may be absent ONLY for the
468
- `exact` mode, which does not use it.
469
-
470
- The sentinel is deliberate and load-bearing for BACK-COMPAT: every date filter that existed
471
- before anchors is `exact`, and every caller that compiles one (store_query, store_rows)
472
- passes no `today`. Requiring it unconditionally would turn a working filter into an
473
- exception. A RELATIVE anchor still refuses — the server never widens.
474
- """
475
- mode = mode or 'exact'
476
- if today is None:
477
- if mode != 'exact':
478
- raise ValueError(
479
- f"the date anchor {mode!r} is relative and needs `today` — "
480
- f"compile_filter_tree(today=...). Refusing rather than dropping the condition, "
481
- f"which would widen the result under an authoritative-looking count.")
482
- # `exact` ignores `today` entirely (verify_windows resolves it across 11 probe dates and
483
- # gets the same answer every time), so any valid date serves as the unused argument.
484
- today = '1970-01-01'
485
- return _windows().resolve_anchor(mode, value, today)
486
-
487
-
488
- def _compile_cohort(rule, state):
489
- """Port of evalNode's cohort branch: membership in a SET of hand-curated sets of row ids.
490
-
491
- Owner 2026-07-27: the leaf names one OR MORE cohorts and asks `anyOf` / `allOf` / `noneOf`.
492
- A single id is a one-element set, so the shipped `is part of` compiles down this same path.
493
-
494
- States, mirroring TS exactly:
495
- every set present -> the union (anyOf / noneOf) or the intersection (allOf)
496
- a set present but EMPTY -> that member contributes 1=0, because nobody is in an empty
497
- cohort. So `allOf` collapses to 1=0 and `noneOf` to 1=1 —
498
- which is what `set.has(pid)` on an empty Set does in TS.
499
- ANY set ABSENT -> 1=0 for ALL THREE ops. The condition is unanswerable, and
500
- unanswerable matches nothing: answering "everything" to
501
- `is none of [a cohort we cannot see]` is the widening sin
502
- wearing a plausible face, and answering it for the members we
503
- DO have would silently ask a different question.
504
-
505
- COALESCE(... , FALSE) because a NULL row id makes `id IN (...)` NULL, and `NOT NULL` is NULL
506
- — SQL would drop the row where TS keeps it.
507
- """
508
- sets = state.get('cohort_sets')
509
- if sets is None:
510
- raise ValueError(
511
- 'a cohort condition needs cohort_sets — compile_filter_tree(cohort_sets=...). '
512
- 'Refusing rather than dropping it: a dropped condition widens the result while the '
513
- 'count beside it still looks authoritative.')
514
- id_sql = state.get('id_sql')
515
- if not id_sql:
516
- raise ValueError('a cohort condition needs id_sql (the row-identity column)')
517
- op = COHORT_OP_ALIASES.get(rule.get('op'), rule.get('op'))
518
- named = parse_cohort_ids(rule.get('value'))
519
- members = [sets.get(cid) for cid in named]
520
- if not named or any(m is None for m in members):
521
- return ('1=0', [])
522
-
523
- parts, params = [], []
524
- for ids in members:
525
- ids = [int(x) for x in ids]
526
- if not ids:
527
- parts.append('1=0')
528
- continue
529
- parts.append(f"COALESCE({id_sql} IN ({','.join('?' for _ in ids)}), FALSE)")
530
- params.extend(ids)
531
- if op == 'allOf':
532
- return ('(' + ' AND '.join(parts) + ')', params)
533
- union = '(' + ' OR '.join(parts) + ')'
534
- return ((f"(NOT {union})" if op == 'noneOf' else union), params)
535
-
536
-
537
- # --------------------------------------------------------------------------- tree
538
-
539
- def _compile_node(node, columns, state):
540
- """Port of `evalNode`. Returns (sql, params), or None when the node is INACTIVE.
541
-
542
- A group combines only its ACTIVE children with its OWN conjunction; a group with no active
543
- children is itself inactive and disappears. That is what stops a half-built group from
544
- either zeroing the grid (`and`) or disabling it (`or`).
545
- """
546
- if not isinstance(node, dict):
547
- return None
548
- if isinstance(node.get('children'), list):
549
- parts, params = [], []
550
- for child in node['children']:
551
- got = _compile_node(child, columns, state)
552
- if got is None:
553
- continue # inactive -> ignored entirely
554
- parts.append(got[0])
555
- params.extend(got[1])
556
- if not parts:
557
- return None
558
- joiner = ' OR ' if node.get('conj') == 'or' else ' AND '
559
- return ('(' + joiner.join(parts) + ')', params)
560
- return _compile_leaf(node, columns, state)
561
-
562
-
563
- def compile_filter_tree(nodes, conj='and', columns=None, member_ids=None, id_sql=None,
564
- today=None, cohort_sets=None):
565
- """Port of `matchFilterTree` (+ the `memberPids` union in `useVisibleRows`).
566
-
567
- Returns a `Compiled`, or **None when nothing is active** — meaning no predicate at all, not
568
- a false one. `matchFilterTree` returns true when the tree is inactive, so emitting anything
569
- here would narrow rows the client would have shown.
570
-
571
- `member_ids` reproduces `members.has(r.pid) || matchFilterTree(...)`. Note the union lives
572
- INSIDE `if (filters.length)` and is OR'd with a tree that is true for everything when
573
- inactive — so an inactive tree must emit NOTHING, never `pid IN (...)`, which would narrow
574
- to just the hand-picked members. Same family of bug as the tri-state one, inverted.
575
-
576
- `today` is the TENANT'S today, required by any relative date condition (`is within …`,
577
- `is before [one month ago]`) and by nothing else — an `exact` date needs none, which is what
578
- keeps every pre-anchor caller working unchanged. `cohort_sets` is `{cohortId: [pid, ...]}`
579
- for cohort-membership leaves. Both REFUSE when needed and missing; neither defaults.
580
- """
581
- state = {'aggregate': False, 'used': set(),
582
- 'today': today, 'cohort_sets': cohort_sets, 'id_sql': id_sql}
583
- got = _compile_node({'conj': conj, 'children': list(nodes or [])}, columns or {}, state)
584
- if got is None:
585
- return None
586
- sql, params = got
587
- if member_ids:
588
- if not id_sql:
589
- raise ValueError('member_ids requires id_sql (the row-identity column)')
590
- ids = [int(x) for x in member_ids]
591
- sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))"
592
- params = list(params) + ids
593
- return Compiled(sql, list(params), state['aggregate'], frozenset(state['used']))
594
-
595
-
596
- def compile_search(q, columns):
597
- """Port of `matchSearch`: the narrowing search box.
598
-
599
- True if ANY human-readable (text/status) column contains `q`, case-insensitively. NUMERIC
600
- columns are skipped on purpose so a query like "10" does not match revenue or order counts.
601
- Dates are skipped too — `matchSearch` tests only text and status.
602
-
603
- The caller ANDs this with the filter predicate, matching the TS pipeline order
604
- (filter -> search). It is deliberately OUTSIDE the `member_ids` union: in `useVisibleRows`
605
- the union applies to the filter step only, so a hand-picked member is still searchable away.
606
- """
607
- # `.strip(_JS_WS)`, not `.strip()`: the client trims with JS String.trim(), whose whitespace
608
- # set differs from Python's in BOTH directions (Python strips U+0085 NEL and U+001C..1F,
609
- # JS does not; JS strips U+FEFF, Python does not). A query of just those characters is
610
- # blank to one engine and a real search to the other. Folding is left to SQL — see the
611
- # LOWER(?) note in _compile_leaf.
612
- q = (q or '').strip(_JS_WS)
613
- if not q:
614
- return None
615
- parts, params, used = [], [], set()
616
- aggregate = False
617
- for col_id, spec in (columns or {}).items():
618
- if spec['type'] not in TEXTUAL_TYPES:
619
- continue
620
- parts.append(f"(strpos({_value_sql(spec)}, LOWER(?)) > 0)")
621
- params.append(q)
622
- used.add(col_id)
623
- aggregate = aggregate or bool(spec.get('aggregate'))
624
- if not parts:
625
- # No searchable column: `matchSearch` returns false for every row, so the search
626
- # narrows to nothing. That is a real predicate, not an absent one.
627
- return Compiled('1=0', [], False, frozenset())
628
- return Compiled('(' + ' OR '.join(parts) + ')', params, aggregate, frozenset(used))
629
-
630
-
631
- def compile_order_by(sorts, columns, tiebreak_sql=None):
632
- """Port of `makeComparator`. Returns an ORDER BY body, or None when nothing sorts.
633
-
634
- `tiebreak_sql` (the row-identity column) is appended as a final ASC key and is REQUIRED for
635
- any windowed fetch. `Array.prototype.sort` is stable, so TS leaves tied rows in their
636
- incoming order; SQL guarantees NOTHING for ties, so LIMIT/OFFSET paging over a non-total
637
- order can repeat a row on one page and drop it from another. Appending row identity makes
638
- the order total. It is a STRENGTHENING of the TS guarantee, not a divergence: the payload a
639
- windowed fetch produces is itself in this order, so the client never re-sorts it differently.
640
-
641
- BLANKS SORT LAST IN BOTH DIRECTIONS (Airtable's behavior, and the reason descending
642
- "Last order" leads with the most recent rather than with never-ordered customers). Each key
643
- contributes TWO terms: an always-ASC blank flag, then the value in the requested direction.
644
- Two blanks tie on both terms, so the NEXT key decides — matching the comparator's
645
- `if (aBlank && bBlank) continue`.
646
-
647
- An unknown column is skipped: in TS its values are `undefined` on both sides, so both are
648
- blank and the key is a no-op tiebreak.
649
-
650
- Direction: `s.dir === "asc" ? c : -c` — only the exact string "asc" ascends, anything else
651
- descends. Faithfully reproduced; do not "fix" it to default ASC.
652
-
653
- Residual divergence, deliberate: TS uses `localeCompare`, we use LOWER() + DuckDB's binary
654
- collation. They agree on case-insensitive ordering of ASCII and on ISO dates (unaffected by
655
- LOWER), and differ on locale tie-breaks between case-variants of the same string ("a" vs
656
- "A" sort adjacent under localeCompare, tie under LOWER) and on accent ordering.
657
- """
658
- terms = []
659
- for s in (sorts or []):
660
- if not isinstance(s, dict):
661
- continue
662
- spec = (columns or {}).get(s.get('colId'))
663
- if spec is None:
664
- continue
665
- terms.append(f"CASE WHEN {_blank_sql(spec)} THEN 1 ELSE 0 END ASC")
666
- terms.append(f"{_value_sql(spec)} {'ASC' if s.get('dir') == 'asc' else 'DESC'}")
667
- # Append the tiebreak even when NO sort key survived (all unknown, or none given). A
668
- # windowed fetch needs a total order regardless of what the user sorted by — an unsorted
669
- # LIMIT/OFFSET is exactly as unstable as a tied one. Callers wanting pure client parity
670
- # pass no tiebreak_sql and still get None.
671
- if tiebreak_sql:
672
- terms.append(f"{tiebreak_sql} ASC")
673
- return ', '.join(terms) or None
 
1
+ """harness/filter_sql.py — the grid's filter/sort engine, compiled to SQL (CG-1).
2
+
3
+ THE PORT. `customer-grid/useVisibleRows.ts` evaluates the Airtable-parity filter TREE and the
4
+ multi-level sort in TypeScript, per row, in the browser. That works at customer grain (1,550 rows,
5
+ 576 KB) and does NOT work at line grain, so a line-grain table has to filter server-side. This
6
+ module is the second implementation of that contract, emitting parameterized SQL instead of
7
+ booleans. `aios_grid.clean_filter_tree` (the validator) is the third. All three are held in
8
+ lock-step by `aios-web/verify_filter_engine.py`, which runs the SAME cases through the TS engine,
9
+ this compiler (executed on DuckDB), and the validator, and compares row sets.
10
+
11
+ Design: this file is PURE — no model, no store, no app imports. It takes a filter tree plus a
12
+ `columns` spec and returns SQL + params. `harness/semantic.store_columns` builds the spec from a
13
+ topic; nothing else here knows what a topic is.
14
+
15
+ columns = {colId: {"sql": <SQL expression>, "type": <field type>, "aggregate": bool}}
16
+
17
+ `sql` is a trusted IDENTIFIER/expression from the semantic model (never user input); every VALUE
18
+ is a bound parameter. That split is the whole injection story.
19
+
20
+ --------------------------------------------------------------------------------------------
21
+ TRI-STATE, AND WHY IT BECOMES COMPILE-TIME PRUNING
22
+ --------------------------------------------------------------------------------------------
23
+ `evalNode` in TS returns true | false | null, where null means "this node is INACTIVE and its
24
+ parent must IGNORE it". Get that wrong and a half-typed condition returns true, makes its `or`
25
+ group true, and shows every record.
26
+
27
+ Activeness reads ONLY the rule (op, value, value2, is-the-column-known) — never the row. So in
28
+ SQL it is not a third truth value at all: it is decided while COMPILING. An inactive node emits
29
+ no SQL, a group with no active children emits no SQL, and a root with nothing active emits NO
30
+ PREDICATE AT ALL (`compile_filter_tree` returns None) — which is exactly `matchFilterTree`
31
+ returning true for every row.
32
+
33
+ --------------------------------------------------------------------------------------------
34
+ THE FOUR PLACES A NAIVE PORT SILENTLY DIVERGES
35
+ --------------------------------------------------------------------------------------------
36
+ JavaScript coerces null to 0 / "" before comparing; SQL propagates NULL and drops the row.
37
+
38
+ 1. `toNum(null)` is 0, so a NULL revenue PASSES `>= 0`. SQL: NULL >= 0 -> NULL -> dropped.
39
+ 2. `String(null)` is "", so a NULL state PASSES `is not FL`. SQL: NULL <> 'fl' -> dropped.
40
+ 3. `"" <= "2026-01-01"` is true, so a NULL date PASSES `is before`. SQL: dropped.
41
+ 4. `contains` is a LITERAL substring test. `LIKE '%v%'` treats % and _ as wildcards, so a
42
+ value containing either matches the wrong rows. Hence `strpos`, not LIKE.
43
+
44
+ Every column is therefore read through `_value_sql`, which COALESCEs to the same zero-value
45
+ JavaScript would have coerced to. The one thing that must NOT be coalesced away is the
46
+ blank test itself — `isBlank` looks at the RAW value, and numeric 0 is NOT blank.
47
+
48
+ --------------------------------------------------------------------------------------------
49
+ AND THE FIFTH: `default: return true` IS ACTIVE, NOT INACTIVE
50
+ --------------------------------------------------------------------------------------------
51
+ `matchFilter`'s per-type switch ends in `default: return true`, reachable whenever an operator
52
+ does not apply to the field type — `contains` on a currency/date field, or `gt`/`between` on a
53
+ text field. A persisted tree can hold such a pair. That rule is ACTIVE and evaluates TRUE for
54
+ every row, which is NOT the same as inactive: under `or` it makes the whole group true and shows
55
+ everything. We emit `1=1` for those, and the gate asserts the difference is observable.
56
+
57
+ --------------------------------------------------------------------------------------------
58
+ INPUT CONTRACT
59
+ --------------------------------------------------------------------------------------------
60
+ Feed this CLEANED trees (`aios_grid.clean_filter_tree`). It is deliberately faithful to the TS
61
+ engine rather than fail-closed on garbage — an unknown OP compiles to `1=1` exactly as TS's
62
+ `default:` branch returns true — because the validator's job is to reject garbage and this
63
+ module's job is to mean the same thing as the client. The gate proves the composition.
64
+
65
+ Ported ahead of their consumer (CG-2) and gated at the same fidelity as the tree:
66
+ `compile_search` (`matchSearch`) and `member_ids` (the hand-picked-members union in
67
+ `useVisibleRows`).
68
+ """
69
+ import math
70
+ import re
71
+ from collections import namedtuple
72
+
73
+ # Mirrors customer-grid/types.ts FilterOp. verify_filter_engine.py asserts this set is
74
+ # byte-identical to aios_grid.FILTER_OPS — three copies of a vocabulary need a drift gate.
75
+ FILTER_OPS = frozenset({
76
+ 'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty',
77
+ 'gt', 'gte', 'lt', 'lte', 'between', 'within',
78
+ 'topN', 'bottomN', 'inTopPct', 'inBottomPct',
79
+ 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile',
80
+ })
81
+ #: Wave 2026-08-02 (C-OPS): the RANK subset — set-ranked over the sibling-filtered domain.
82
+ #: Vocabulary here, execution NOWHERE in this module: a per-row WHERE cannot express Top-N,
83
+ #: and compiling to 1=0 would answer "nothing" to a question that has an answer, under a
84
+ #: row_count that still looks authoritative. `_compile_leaf` therefore REFUSES (ValueError),
85
+ #: the same server-side posture as a relative anchor with no `today`. The TS engine
86
+ #: evaluates these as a set pass in useVisibleRows (gated by web/verify_rank_filters.py).
87
+ RANK_OPS = frozenset({'topN', 'bottomN', 'inTopPct', 'inBottomPct',
88
+ 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'})
89
+ #: The RESERVED pseudo-column of a cohort-membership leaf. Mirrors types.ts COHORT_FIELD.
90
+ COHORT_FIELD = '__cohort__'
91
+ #: Set operators over a SET of cohorts. Mirrors types.ts COHORT_OPS; DISJOINT from FILTER_OPS.
92
+ COHORT_OPS = frozenset({'anyOf', 'allOf', 'noneOf'})
93
+ #: The single-cohort ops the leaf shipped with, kept as permanent aliases. types.ts mirrors this.
94
+ COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'}
95
+ #: Mirrors types.ts MAX_COHORT_IDS.
96
+ MAX_COHORT_IDS = 20
97
+
98
+
99
+ def parse_cohort_ids(value):
100
+ """The cohorts a leaf names. Mirrors types.ts `cohortIds()` and aios_grid.parse_cohort_ids."""
101
+ out = []
102
+ for raw in ('' if value is None else str(value)).split(','):
103
+ cid = raw.strip()[:120]
104
+ if not cid or cid in out:
105
+ continue
106
+ out.append(cid)
107
+ if len(out) >= MAX_COHORT_IDS:
108
+ break
109
+ return out
110
+ # types.ts VALUE_FREE_OPS: no value control is rendered, and they must never be judged
111
+ # "inactive because the value is blank" — that would turn them into no-ops.
112
+ VALUE_FREE_OPS = frozenset({'isEmpty', 'isNotEmpty', 'aboveAvg', 'belowAvg'})
113
+ # types.ts isNumericType()
114
+ NUMERIC_TYPES = frozenset({'currency', 'int', 'pct'})
115
+ TEXTUAL_TYPES = frozenset({'text', 'status'})
116
+
117
+ SQL_TRUE = '1=1'
118
+
119
+ _NUM_CMP = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'}
120
+ _DATE_CMP = dict(_NUM_CMP)
121
+
122
+ #: sql = the predicate (already parenthesised); params = bound values, positional;
123
+ #: uses_aggregate = the predicate references an aggregate column, so it belongs in HAVING
124
+ #: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
125
+ #: always-true `1=1` branch references nothing, and must not force HAVING routing).
126
+ Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
127
+
128
+
129
+ # --------------------------------------------------------------------------- value coercion
130
+
131
+ # JS `Number(string)` trims exactly this set (WhiteSpace + LineTerminator). Python's float() and
132
+ # str.strip() use a DIFFERENT set -- notably U+0085 NEL, which Python treats as whitespace and JS
133
+ # does not -- so the set is spelled out rather than left to strip().
134
+ _JS_WS = "".join(chr(c) for c in (9, 10, 11, 12, 13, 32, 160, 5760, 8232, 8233,
135
+ 8239, 8287, 12288, 65279)) + \
136
+ "".join(chr(c) for c in range(8192, 8203))
137
+ # ASCII-ONLY on purpose: `\d` in a Python str pattern matches every Unicode decimal digit, so
138
+ # `\d` here would parse the fullwidth "250" that JS rejects -- reintroducing the exact bug
139
+ # this function exists to avoid.
140
+ _JS_DEC = re.compile(r"^[+-]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$")
141
+ _JS_INF = re.compile(r"^[+-]?Infinity$")
142
+ _JS_RADIX = ((re.compile(r"^0[xX][0-9a-fA-F]+$"), 16),
143
+ (re.compile(r"^0[oO][0-7]+$"), 8),
144
+ (re.compile(r"^0[bB][01]+$"), 2))
145
+
146
+
147
+ def to_num(v):
148
+ """Port of `toNum`: JS `Number(v)`, then `Number.isFinite` or 0.
149
+
150
+ NOT `float(v)`. Python's float() is MORE permissive than JS's Number() in ways that make SQL
151
+ narrow where the client shows everything -- the worse direction, because at line grain that
152
+ is a silent under-fetch sitting beside an honest-looking row_count:
153
+
154
+ "1_000" float() -> 1000.0 (PEP-515 underscores) Number() -> NaN -> 0
155
+ "250" float() -> 250.0 (any Unicode Nd) Number() -> NaN -> 0
156
+ "\\u0085" + "12" float() -> 12.0 (NEL is Python whitespace) Number() -> NaN -> 0
157
+
158
+ and LESS permissive in one family, which errs the other way:
159
+
160
+ "0x10" float() -> raises -> 0 Number() -> 16 (also 0b, 0o)
161
+
162
+ A tree carrying any of these is reachable: the React builder renders <input type="number">,
163
+ but persisted / seeded / imported view JSON goes through `clean_filter_tree`, which passes
164
+ the value string through untouched by design (it validates structure, not numeric syntax).
165
+
166
+ Agreed by construction on the rest: ""/None -> 0, "abc"/"12,000" -> 0, " 250 " -> 250,
167
+ ".5"/"5."/"+5", "1e400" -> Infinity -> 0, "-0" -> 0, and float64 truncation of
168
+ "9007199254740993" -> 9007199254740992.
169
+ """
170
+ if isinstance(v, bool):
171
+ return 1.0 if v else 0.0 # JS Number(true) === 1
172
+ if isinstance(v, (int, float)):
173
+ n = float(v)
174
+ return n if math.isfinite(n) else 0.0
175
+ if v is None:
176
+ return 0.0 # JS Number(null) === 0
177
+ s = str(v).strip(_JS_WS)
178
+ if s == "":
179
+ return 0.0 # JS Number("") === 0
180
+ if _JS_INF.match(s):
181
+ return 0.0 # Infinity is not finite -> 0
182
+ for pattern, base in _JS_RADIX:
183
+ if pattern.match(s):
184
+ return float(int(s[2:], base))
185
+ if not _JS_DEC.match(s):
186
+ return 0.0 # JS NaN -> 0
187
+ n = float(s)
188
+ return n if math.isfinite(n) else 0.0
189
+
190
+
191
+ def _value_sql(spec):
192
+ """The ONE expression for a column, used by BOTH the filter and the sort.
193
+
194
+ Two expressions would let two rows tie in the grid and not tie in SQL (dates truncated for
195
+ comparison but ordered by the raw timestamp is the obvious way to get that wrong), so the
196
+ next sort key would break the tie in one engine and not the other.
197
+
198
+ numeric: `round_even(x, 0)` reproduces `aios_grid._round`, which is Python's round() =
199
+ BANKER'S rounding. DuckDB's plain ROUND() rounds half AWAY from zero, so ROUND(1234.5)=1235
200
+ while the grid shows 1234 — `eq 1234` would then match on screen and miss in SQL. The grid
201
+ displays rounded values; filters must agree with what is on screen.
202
+ date: truncate to the 10-char ISO shape the payload carries, so lexical compare is
203
+ chronological and a timestamp column cannot smuggle ' 00:00:00' into the comparison.
204
+ text: lowercased, matching `String(raw ?? "").toLowerCase()`.
205
+ """
206
+ e = spec['sql']
207
+ t = spec['type']
208
+ if t in NUMERIC_TYPES:
209
+ return f"COALESCE(round_even(TRY_CAST({e} AS DOUBLE), 0), 0)"
210
+ if t == 'date':
211
+ return f"COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '')"
212
+ return f"LOWER(COALESCE(CAST({e} AS VARCHAR), ''))"
213
+
214
+
215
+ def _blank_sql(spec):
216
+ """Port of `isBlank`: null/undefined/"" are blank — numeric 0 is NOT.
217
+
218
+ Reads the RAW column, never `_value_sql` (which has already coalesced blanks into 0/"",
219
+ the very distinction this test exists to make).
220
+ """
221
+ e = spec['sql']
222
+ if spec['type'] in NUMERIC_TYPES:
223
+ # 0 stays non-blank: CAST(0 AS VARCHAR) is '0'. The '' arm covers a text-typed column
224
+ # holding "", which `_round` passes through unchanged into the payload.
225
+ return f"(({e}) IS NULL OR CAST({e} AS VARCHAR) = '')"
226
+ if spec['type'] == 'date':
227
+ return f"(COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '') = '')"
228
+ return f"(COALESCE(CAST({e} AS VARCHAR), '') = '')"
229
+
230
+
231
+ # --------------------------------------------------------------------------- activeness
232
+
233
+ def is_rule_active(rule, columns):
234
+ """Port of `isRuleActive` + `evalNode`'s two leaf guards. Row-independent by construction.
235
+
236
+ Unknown column -> inactive (evalNode returns null before matchFilter is ever called).
237
+ Value-free ops are always active. `between` needs BOTH bounds.
238
+
239
+ ⚠ The four VALUE-FREE ANCHOR MODES (`is before [today]`) carry no value, and the historical
240
+ "blank value = inactive" rule would IGNORE them — widening the result under a count nobody
241
+ would doubt. Same shape as the isEmpty/isNotEmpty no-op bug, arriving by a different door.
242
+ """
243
+ if not isinstance(rule, dict):
244
+ return False
245
+ col = rule.get('colId')
246
+ if col == COHORT_FIELD: # a cohort leaf has no column to look up
247
+ # Driven by the SET it names, so a value of "," is INACTIVE rather than active-and-
248
+ # unanswerable. Mirrors the cohort branch at the top of types.ts isRuleActive.
249
+ return bool(parse_cohort_ids(rule.get('value')))
250
+ if col not in columns:
251
+ return False
252
+ op = rule.get('op')
253
+ if op in VALUE_FREE_OPS:
254
+ return True
255
+ if op == 'within':
256
+ return _window_bounds(rule.get('dateWindow'), None, probe=True)
257
+ if rule.get('rhs') is not None:
258
+ rhs = rule['rhs']
259
+ # An rhs naming a column this table does not have is IGNORED, by the same rule as an
260
+ # unknown left column two lines above — for EITHER kind. Ignoring one side and hiding
261
+ # every row for the other would make the comparison asymmetric in a way nobody could
262
+ # predict. Doing it here rather than in _compile_leaf keeps activeness the single place
263
+ # that decides, which is what makes the compiled branch below provably unreachable.
264
+ if not isinstance(rhs, dict) or rhs.get('colId') not in columns:
265
+ return False
266
+ return _rhs_complete(rhs)
267
+ if rule.get('dateMode') in _ANCHOR_VALUE_FREE:
268
+ return True
269
+ value = _as_str(rule.get('value'))
270
+ if op == 'between':
271
+ v2 = rule.get('value2')
272
+ return value != '' and v2 is not None and _as_str(v2) != ''
273
+ return value != ''
274
+
275
+
276
+ def _rhs_complete(rhs):
277
+ """Port of types.ts isRhsComplete. An incomplete rhs makes the rule INACTIVE, exactly as a
278
+ half-typed value does."""
279
+ if not isinstance(rhs, dict) or not rhs.get('colId'):
280
+ return False
281
+ if rhs.get('kind') == 'measure':
282
+ return _windows().normalize(rhs.get('window')) is not None
283
+ return rhs.get('kind') == 'field'
284
+
285
+
286
+ def _windows():
287
+ """`harness.windows` imported lazily, so this module stays importable on its own.
288
+
289
+ filter_sql is otherwise PURE (no model/store/app imports) and that property is what lets
290
+ aios_grid embed the grid anywhere. windows.py is the one exception and it is a deliberate
291
+ one: the date vocabulary is a two-ENGINE contract, and a third copy of "what the past month
292
+ means" is precisely the drift verify_windows.py exists to prevent. It imports nothing but
293
+ datetime, so purity is preserved in substance.
294
+ """
295
+ from harness import windows as _w
296
+ return _w
297
+
298
+
299
+ #: Mirrors windows.ANCHOR_VALUE_FREE, resolved lazily on first use (see _windows).
300
+ class _AnchorValueFree:
301
+ def __contains__(self, mode):
302
+ return mode in _windows().ANCHOR_VALUE_FREE
303
+
304
+
305
+ _ANCHOR_VALUE_FREE = _AnchorValueFree()
306
+
307
+
308
+ def _window_bounds(spec, today, probe=False):
309
+ """Resolve a date-condition RANGE, or refuse.
310
+
311
+ `probe=True` answers only "is this a resolvable window?" without needing `today` — which is
312
+ what activeness has to know while compiling, before any date is in hand.
313
+
314
+ ⚠ A missing `today` on the real path RAISES rather than widening. The client's answer to the
315
+ same gap is to match nothing; the server's must be to refuse, because a server that quietly
316
+ drops a condition reports an authoritative row_count for a query nobody asked for. Same rule
317
+ as the sibling/depth caps: truncate on the client, refuse on the server.
318
+ """
319
+ w = _windows()
320
+ if probe:
321
+ return w.normalize(spec) is not None
322
+ if today is None:
323
+ raise ValueError(
324
+ 'a relative-date condition needs `today` — compile_filter_tree(today=...). '
325
+ 'Refusing rather than dropping the condition, which would widen the result while '
326
+ 'the count beside it still looked authoritative.')
327
+ return w.resolve(spec, today)
328
+
329
+
330
+ def _as_str(v):
331
+ """The builder stores values as raw input strings; be tolerant of a number that survived
332
+ an older payload. `str(500) != ''` is active and `to_num` reads it back, matching JS where
333
+ `500 === ""` is false."""
334
+ return '' if v is None else str(v)
335
+
336
+
337
+ # --------------------------------------------------------------------------- leaf
338
+
339
+ def _compile_leaf(rule, columns, state):
340
+ """Port of `matchFilter`. Returns (sql, params) or None when the rule is INACTIVE."""
341
+ if not is_rule_active(rule, columns):
342
+ return None
343
+ col_id = rule['colId']
344
+ op = rule.get('op')
345
+ value = _as_str(rule.get('value'))
346
+
347
+ # --- owner item 5: a COHORT leaf, before any column lookup -------------------------------
348
+ # `__cohort__` is not a column. Handled first for the same reason evalNode handles it first:
349
+ # falling through to the column path would make it INACTIVE, and inactive means every row.
350
+ if col_id == COHORT_FIELD:
351
+ return _compile_cohort(rule, state)
352
+
353
+ # --- C-OPS (wave 2026-08-02): a RANK leaf is structurally unanswerable at row grain ------
354
+ # Top-N / quantile membership is a property of the SET, not the row; the client engine
355
+ # answers it with a set pass over the sibling-filtered domain. Refuse loudly rather than
356
+ # compile something that means a different question (see RANK_OPS).
357
+ if op in RANK_OPS:
358
+ raise ValueError(
359
+ f'rank operator {op!r} cannot compile to row SQL - evaluate it as a set pass '
360
+ f'(the client engine) or pre-resolve the matching ids before compiling')
361
+
362
+ spec = columns[col_id]
363
+
364
+ def used(sql, params=()):
365
+ state['used'].add(col_id)
366
+ if spec.get('aggregate'):
367
+ state['aggregate'] = True
368
+ return (sql, list(params))
369
+
370
+ # --- value-free ops, BEFORE any value-based short-circuit (the no-op bug) ---------------
371
+ if op == 'isEmpty':
372
+ return used(_blank_sql(spec))
373
+ if op == 'isNotEmpty':
374
+ return used(f"NOT {_blank_sql(spec)}")
375
+
376
+ v = _value_sql(spec)
377
+ t = spec['type']
378
+
379
+ # --- CG-9: the comparand is another COLUMN, coerced through the LEFT field's type --------
380
+ # `_value_sql` on a spec whose `sql` is the RIGHT column and whose `type` is the LEFT one —
381
+ # exactly what TS does (`toNum(other)` for a numeric LHS, `String(other ?? "")` for a date,
382
+ # `.toLowerCase()` for text). Reading the right column through its OWN type would compare a
383
+ # rounded number against an unrounded one and disagree on the ties.
384
+ rhs_v = None
385
+ if rule.get('rhs') is not None:
386
+ # No `rspec is None` branch: `is_rule_active` already rejected an rhs naming a column
387
+ # this table does not have, so it cannot be missing here. A defensive fallback would be
388
+ # unreachable, and the negative control showed exactly that — deleting it changed
389
+ # nothing, which is how dead code hides behind a careful-looking line.
390
+ rspec = columns[rule['rhs']['colId']]
391
+ rhs_v = _value_sql({'sql': rspec['sql'], 'type': t})
392
+ state['used'].add(rule['rhs']['colId'])
393
+ if rspec.get('aggregate'):
394
+ state['aggregate'] = True
395
+
396
+ if t in NUMERIC_TYPES:
397
+ if rhs_v is not None:
398
+ return used(f"({v} {_NUM_CMP[op]} {rhs_v})") if op in _NUM_CMP else (SQL_TRUE, [])
399
+ if op == 'between':
400
+ a, b = to_num(value), to_num(rule.get('value2'))
401
+ return used(f"({v} >= ? AND {v} <= ?)", [min(a, b), max(a, b)])
402
+ if op in _NUM_CMP:
403
+ return used(f"({v} {_NUM_CMP[op]} ?)", [to_num(value)])
404
+ return (SQL_TRUE, []) # contains/doesNotContain on a number -> default: true
405
+
406
+ if t == 'date':
407
+ if op == 'within':
408
+ bounds = _window_bounds(rule.get('dateWindow'), state.get('today'))
409
+ if bounds is None:
410
+ return ('1=0', []) # unreachable: activeness probed it. Never widen.
411
+ lo, hi = bounds
412
+ # `{v} <> ''` is NOT redundant. `_value_sql` COALESCEs a NULL date to '', and
413
+ # '' <= any upper bound is true — so an open-ended window would sweep in every
414
+ # never-ordered customer. TS guards it with `if (a === "") return false`.
415
+ parts, params = [f"{v} <> ''"], []
416
+ if lo is not None:
417
+ parts.append(f"{v} >= ?")
418
+ params.append(lo)
419
+ if hi is not None:
420
+ parts.append(f"{v} <= ?")
421
+ params.append(hi)
422
+ return used('(' + ' AND '.join(parts) + ')', params)
423
+ if rhs_v is not None:
424
+ return used(f"({v} {_DATE_CMP[op]} {rhs_v})") if op in _DATE_CMP else (SQL_TRUE, [])
425
+ if op == 'between':
426
+ v2 = _as_str(rule.get('value2'))
427
+ lo, hi = (value, v2) if value < v2 else (v2, value)
428
+ return used(f"({v} >= ? AND {v} <= ?)", [lo, hi])
429
+ if op in _DATE_CMP:
430
+ anchor = _resolve_anchor(rule.get('dateMode'), value, state.get('today'))
431
+ if anchor is None:
432
+ return ('1=0', []) # unresolvable anchor -> matches nothing (TS: false)
433
+ return used(f"({v} {_DATE_CMP[op]} ?)", [anchor])
434
+ return (SQL_TRUE, []) # contains/doesNotContain on a date -> default: true
435
+
436
+ # text / status — case-insensitive. strpos, not LIKE: `includes` is a LITERAL substring
437
+ # test, so a value holding % or _ must not act as a wildcard.
438
+ #
439
+ # BOTH SIDES fold in SQL (`LOWER(?)`, never Python's str.lower()). Python and JS agree on
440
+ # case folding; DuckDB does NOT — LOWER('ISTANBUL' with a dotted capital I) drops the dot
441
+ # while both languages keep it as i + combining dot. Folding the needle in Python and the
442
+ # haystack in DuckDB therefore produced two different strings, and a value could fail to
443
+ # match ITSELF. One folder for both operands is the only version that cannot do that.
444
+ if rhs_v is not None:
445
+ # Both operands are already folded by `_value_sql` (text branch), so no LOWER(?) here.
446
+ if op == 'contains':
447
+ return used(f"(strpos({v}, {rhs_v}) > 0)")
448
+ if op == 'doesNotContain':
449
+ return used(f"(strpos({v}, {rhs_v}) = 0)")
450
+ if op == 'eq':
451
+ return used(f"({v} = {rhs_v})")
452
+ if op == 'neq':
453
+ return used(f"({v} <> {rhs_v})")
454
+ return (SQL_TRUE, [])
455
+ if op == 'contains':
456
+ return used(f"(strpos({v}, LOWER(?)) > 0)", [value])
457
+ if op == 'doesNotContain':
458
+ return used(f"(strpos({v}, LOWER(?)) = 0)", [value])
459
+ if op == 'eq':
460
+ return used(f"({v} = LOWER(?))", [value])
461
+ if op == 'neq':
462
+ return used(f"({v} <> LOWER(?))", [value])
463
+ return (SQL_TRUE, []) # gt/gte/lt/lte/between/within on text -> default: true
464
+
465
+
466
+ def _resolve_anchor(mode, value, today):
467
+ """A date condition's anchor -> one ISO date, or None. `today` may be absent ONLY for the
468
+ `exact` mode, which does not use it.
469
+
470
+ The sentinel is deliberate and load-bearing for BACK-COMPAT: every date filter that existed
471
+ before anchors is `exact`, and every caller that compiles one (store_query, store_rows)
472
+ passes no `today`. Requiring it unconditionally would turn a working filter into an
473
+ exception. A RELATIVE anchor still refuses — the server never widens.
474
+ """
475
+ mode = mode or 'exact'
476
+ if today is None:
477
+ if mode != 'exact':
478
+ raise ValueError(
479
+ f"the date anchor {mode!r} is relative and needs `today` — "
480
+ f"compile_filter_tree(today=...). Refusing rather than dropping the condition, "
481
+ f"which would widen the result under an authoritative-looking count.")
482
+ # `exact` ignores `today` entirely (verify_windows resolves it across 11 probe dates and
483
+ # gets the same answer every time), so any valid date serves as the unused argument.
484
+ today = '1970-01-01'
485
+ return _windows().resolve_anchor(mode, value, today)
486
+
487
+
488
+ def _compile_cohort(rule, state):
489
+ """Port of evalNode's cohort branch: membership in a SET of hand-curated sets of row ids.
490
+
491
+ Owner 2026-07-27: the leaf names one OR MORE cohorts and asks `anyOf` / `allOf` / `noneOf`.
492
+ A single id is a one-element set, so the shipped `is part of` compiles down this same path.
493
+
494
+ States, mirroring TS exactly:
495
+ every set present -> the union (anyOf / noneOf) or the intersection (allOf)
496
+ a set present but EMPTY -> that member contributes 1=0, because nobody is in an empty
497
+ cohort. So `allOf` collapses to 1=0 and `noneOf` to 1=1 —
498
+ which is what `set.has(pid)` on an empty Set does in TS.
499
+ ANY set ABSENT -> 1=0 for ALL THREE ops. The condition is unanswerable, and
500
+ unanswerable matches nothing: answering "everything" to
501
+ `is none of [a cohort we cannot see]` is the widening sin
502
+ wearing a plausible face, and answering it for the members we
503
+ DO have would silently ask a different question.
504
+
505
+ COALESCE(... , FALSE) because a NULL row id makes `id IN (...)` NULL, and `NOT NULL` is NULL
506
+ — SQL would drop the row where TS keeps it.
507
+ """
508
+ sets = state.get('cohort_sets')
509
+ if sets is None:
510
+ raise ValueError(
511
+ 'a cohort condition needs cohort_sets — compile_filter_tree(cohort_sets=...). '
512
+ 'Refusing rather than dropping it: a dropped condition widens the result while the '
513
+ 'count beside it still looks authoritative.')
514
+ id_sql = state.get('id_sql')
515
+ if not id_sql:
516
+ raise ValueError('a cohort condition needs id_sql (the row-identity column)')
517
+ op = COHORT_OP_ALIASES.get(rule.get('op'), rule.get('op'))
518
+ named = parse_cohort_ids(rule.get('value'))
519
+ members = [sets.get(cid) for cid in named]
520
+ if not named or any(m is None for m in members):
521
+ return ('1=0', [])
522
+
523
+ parts, params = [], []
524
+ for ids in members:
525
+ ids = [int(x) for x in ids]
526
+ if not ids:
527
+ parts.append('1=0')
528
+ continue
529
+ parts.append(f"COALESCE({id_sql} IN ({','.join('?' for _ in ids)}), FALSE)")
530
+ params.extend(ids)
531
+ if op == 'allOf':
532
+ return ('(' + ' AND '.join(parts) + ')', params)
533
+ union = '(' + ' OR '.join(parts) + ')'
534
+ return ((f"(NOT {union})" if op == 'noneOf' else union), params)
535
+
536
+
537
+ # --------------------------------------------------------------------------- tree
538
+
539
+ def _compile_node(node, columns, state):
540
+ """Port of `evalNode`. Returns (sql, params), or None when the node is INACTIVE.
541
+
542
+ A group combines only its ACTIVE children with its OWN conjunction; a group with no active
543
+ children is itself inactive and disappears. That is what stops a half-built group from
544
+ either zeroing the grid (`and`) or disabling it (`or`).
545
+ """
546
+ if not isinstance(node, dict):
547
+ return None
548
+ if isinstance(node.get('children'), list):
549
+ parts, params = [], []
550
+ for child in node['children']:
551
+ got = _compile_node(child, columns, state)
552
+ if got is None:
553
+ continue # inactive -> ignored entirely
554
+ parts.append(got[0])
555
+ params.extend(got[1])
556
+ if not parts:
557
+ return None
558
+ joiner = ' OR ' if node.get('conj') == 'or' else ' AND '
559
+ return ('(' + joiner.join(parts) + ')', params)
560
+ return _compile_leaf(node, columns, state)
561
+
562
+
563
+ def compile_filter_tree(nodes, conj='and', columns=None, member_ids=None, id_sql=None,
564
+ today=None, cohort_sets=None):
565
+ """Port of `matchFilterTree` (+ the `memberPids` union in `useVisibleRows`).
566
+
567
+ Returns a `Compiled`, or **None when nothing is active** — meaning no predicate at all, not
568
+ a false one. `matchFilterTree` returns true when the tree is inactive, so emitting anything
569
+ here would narrow rows the client would have shown.
570
+
571
+ `member_ids` reproduces `members.has(r.pid) || matchFilterTree(...)`. Note the union lives
572
+ INSIDE `if (filters.length)` and is OR'd with a tree that is true for everything when
573
+ inactive — so an inactive tree must emit NOTHING, never `pid IN (...)`, which would narrow
574
+ to just the hand-picked members. Same family of bug as the tri-state one, inverted.
575
+
576
+ `today` is the TENANT'S today, required by any relative date condition (`is within …`,
577
+ `is before [one month ago]`) and by nothing else — an `exact` date needs none, which is what
578
+ keeps every pre-anchor caller working unchanged. `cohort_sets` is `{cohortId: [pid, ...]}`
579
+ for cohort-membership leaves. Both REFUSE when needed and missing; neither defaults.
580
+ """
581
+ state = {'aggregate': False, 'used': set(),
582
+ 'today': today, 'cohort_sets': cohort_sets, 'id_sql': id_sql}
583
+ got = _compile_node({'conj': conj, 'children': list(nodes or [])}, columns or {}, state)
584
+ if got is None:
585
+ return None
586
+ sql, params = got
587
+ if member_ids:
588
+ if not id_sql:
589
+ raise ValueError('member_ids requires id_sql (the row-identity column)')
590
+ ids = [int(x) for x in member_ids]
591
+ sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))"
592
+ params = list(params) + ids
593
+ return Compiled(sql, list(params), state['aggregate'], frozenset(state['used']))
594
+
595
+
596
+ def compile_search(q, columns):
597
+ """Port of `matchSearch`: the narrowing search box.
598
+
599
+ True if ANY human-readable (text/status) column contains `q`, case-insensitively. NUMERIC
600
+ columns are skipped on purpose so a query like "10" does not match revenue or order counts.
601
+ Dates are skipped too — `matchSearch` tests only text and status.
602
+
603
+ The caller ANDs this with the filter predicate, matching the TS pipeline order
604
+ (filter -> search). It is deliberately OUTSIDE the `member_ids` union: in `useVisibleRows`
605
+ the union applies to the filter step only, so a hand-picked member is still searchable away.
606
+ """
607
+ # `.strip(_JS_WS)`, not `.strip()`: the client trims with JS String.trim(), whose whitespace
608
+ # set differs from Python's in BOTH directions (Python strips U+0085 NEL and U+001C..1F,
609
+ # JS does not; JS strips U+FEFF, Python does not). A query of just those characters is
610
+ # blank to one engine and a real search to the other. Folding is left to SQL — see the
611
+ # LOWER(?) note in _compile_leaf.
612
+ q = (q or '').strip(_JS_WS)
613
+ if not q:
614
+ return None
615
+ parts, params, used = [], [], set()
616
+ aggregate = False
617
+ for col_id, spec in (columns or {}).items():
618
+ if spec['type'] not in TEXTUAL_TYPES:
619
+ continue
620
+ parts.append(f"(strpos({_value_sql(spec)}, LOWER(?)) > 0)")
621
+ params.append(q)
622
+ used.add(col_id)
623
+ aggregate = aggregate or bool(spec.get('aggregate'))
624
+ if not parts:
625
+ # No searchable column: `matchSearch` returns false for every row, so the search
626
+ # narrows to nothing. That is a real predicate, not an absent one.
627
+ return Compiled('1=0', [], False, frozenset())
628
+ return Compiled('(' + ' OR '.join(parts) + ')', params, aggregate, frozenset(used))
629
+
630
+
631
+ def compile_order_by(sorts, columns, tiebreak_sql=None):
632
+ """Port of `makeComparator`. Returns an ORDER BY body, or None when nothing sorts.
633
+
634
+ `tiebreak_sql` (the row-identity column) is appended as a final ASC key and is REQUIRED for
635
+ any windowed fetch. `Array.prototype.sort` is stable, so TS leaves tied rows in their
636
+ incoming order; SQL guarantees NOTHING for ties, so LIMIT/OFFSET paging over a non-total
637
+ order can repeat a row on one page and drop it from another. Appending row identity makes
638
+ the order total. It is a STRENGTHENING of the TS guarantee, not a divergence: the payload a
639
+ windowed fetch produces is itself in this order, so the client never re-sorts it differently.
640
+
641
+ BLANKS SORT LAST IN BOTH DIRECTIONS (Airtable's behavior, and the reason descending
642
+ "Last order" leads with the most recent rather than with never-ordered customers). Each key
643
+ contributes TWO terms: an always-ASC blank flag, then the value in the requested direction.
644
+ Two blanks tie on both terms, so the NEXT key decides — matching the comparator's
645
+ `if (aBlank && bBlank) continue`.
646
+
647
+ An unknown column is skipped: in TS its values are `undefined` on both sides, so both are
648
+ blank and the key is a no-op tiebreak.
649
+
650
+ Direction: `s.dir === "asc" ? c : -c` — only the exact string "asc" ascends, anything else
651
+ descends. Faithfully reproduced; do not "fix" it to default ASC.
652
+
653
+ Residual divergence, deliberate: TS uses `localeCompare`, we use LOWER() + DuckDB's binary
654
+ collation. They agree on case-insensitive ordering of ASCII and on ISO dates (unaffected by
655
+ LOWER), and differ on locale tie-breaks between case-variants of the same string ("a" vs
656
+ "A" sort adjacent under localeCompare, tie under LOWER) and on accent ordering.
657
+ """
658
+ terms = []
659
+ for s in (sorts or []):
660
+ if not isinstance(s, dict):
661
+ continue
662
+ spec = (columns or {}).get(s.get('colId'))
663
+ if spec is None:
664
+ continue
665
+ terms.append(f"CASE WHEN {_blank_sql(spec)} THEN 1 ELSE 0 END ASC")
666
+ terms.append(f"{_value_sql(spec)} {'ASC' if s.get('dir') == 'asc' else 'DESC'}")
667
+ # Append the tiebreak even when NO sort key survived (all unknown, or none given). A
668
+ # windowed fetch needs a total order regardless of what the user sorted by — an unsorted
669
+ # LIMIT/OFFSET is exactly as unstable as a tied one. Callers wanting pure client parity
670
+ # pass no tiebreak_sql and still get None.
671
+ if tiebreak_sql:
672
+ terms.append(f"{tiebreak_sql} ASC")
673
+ return ', '.join(terms) or None
platform/harness/semantic.py CHANGED
@@ -608,6 +608,12 @@ def store_columns(topic, include_measures=True, grain="aggregate"):
608
  return cols
609
 
610
 
 
 
 
 
 
 
611
  def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
612
  team_id=None, filters=None, sort=None, limit=1000, exclude_services=False,
613
  filter_tree=None, filter_conj="and", today=None):
@@ -619,7 +625,24 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date
619
  s = t["store"]
620
  alias = s["alias"]
621
  dims = s.get("dims") or {}
622
- limit = max(1, min(int(limit or 1000), 5000))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
623
 
624
  base_keys, requested = _expand_measures(list(measures or []))
625
  if not base_keys:
@@ -722,7 +745,9 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date
722
  sql += f" ORDER BY {key} {'DESC' if sort.startswith('-') else 'ASC'}"
723
  elif grain:
724
  sql += " ORDER BY period"
725
- sql += f" LIMIT {limit}"
 
 
726
 
727
  con = _store_con()
728
  try:
@@ -738,8 +763,14 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date
738
  row[k] = _post_compute(row, k)
739
  if "period" in row and row["period"] is not None:
740
  row["period"] = str(row["period"])[:10]
 
 
 
741
  return {"topic": topic, "rows": rows, "row_count": len(rows), "sql": sql,
742
- "measures": requested, "group_by": gb, "grain": grain}
 
 
 
743
 
744
 
745
  def store_rows(topic, date_from=None, date_to=None, team_id=None, filter_tree=None,
 
608
  return cols
609
 
610
 
611
+ #: The ceiling for a GROUPED `store_query`. One row per group, so this bounds DIMENSION
612
+ #: CARDINALITY, not payload — a tenant would need 200,000 distinct customers (or products, or
613
+ #: cities) to reach it. Deliberately NOT unbounded: a runaway group-by should fail, not swap.
614
+ MAX_GROUPS = 200_000
615
+
616
+
617
  def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
618
  team_id=None, filters=None, sort=None, limit=1000, exclude_services=False,
619
  filter_tree=None, filter_conj="and", today=None):
 
625
  s = t["store"]
626
  alias = s["alias"]
627
  dims = s.get("dims") or {}
628
+ # A GROUPED QUERY IS BOUNDED BY GROUPS, NOT BY ROWS — and the 5,000 row ceiling applied to
629
+ # both, which made it a SILENT TRUNCATION of the answer rather than of a payload.
630
+ #
631
+ # ⛔ MEASURED 2026-08-09 on the Royal mirror, grouping 256,810 order lines by customer:
632
+ # limit=100 -> 100 groups, total 1,644,181.65
633
+ # limit=1000 -> 1000 groups, total 9,042,614.10
634
+ # limit=5000 -> 1748 groups, total 14,567,929.72
635
+ # The TOTAL MOVES WITH THE CAP. A row window is honest because counts and totals are computed
636
+ # over the full scope beside it (`store_rows`' whole design); a GROUP window has no such
637
+ # companion — the groups ARE the answer, so dropping one is dropping data with nothing to
638
+ # notice. Royal has 1,943 customers so it passes today and would have passed every test,
639
+ # then gone quietly wrong for the first tenant with more ([[no-unverifiable-aggregates]]).
640
+ #
641
+ # ⚠ The ceiling is not removed, because unbounded is its own failure. It is raised to a bound
642
+ # no realistic dimension reaches, and — the load-bearing half — truncation is now a FACT the
643
+ # caller can read rather than something it must infer from `len(rows)`.
644
+ grouped = bool(group_by)
645
+ limit = max(1, min(int(limit or 1000), MAX_GROUPS if grouped else 5000))
646
 
647
  base_keys, requested = _expand_measures(list(measures or []))
648
  if not base_keys:
 
745
  sql += f" ORDER BY {key} {'DESC' if sort.startswith('-') else 'ASC'}"
746
  elif grain:
747
  sql += " ORDER BY period"
748
+ # ⚠ `limit + 1` — ONE extra row, so "did this truncate" is a FACT and not the guess
749
+ # `len(rows) == limit` makes (which is wrong exactly when the count lands on the cap).
750
+ sql += f" LIMIT {limit + 1}"
751
 
752
  con = _store_con()
753
  try:
 
763
  row[k] = _post_compute(row, k)
764
  if "period" in row and row["period"] is not None:
765
  row["period"] = str(row["period"])[:10]
766
+ truncated = len(rows) > limit
767
+ if truncated:
768
+ rows = rows[:limit]
769
  return {"topic": topic, "rows": rows, "row_count": len(rows), "sql": sql,
770
+ "measures": requested, "group_by": gb, "grain": grain,
771
+ # ⛔ A CALLER THAT AGGREGATES THESE ROWS MUST CHECK THIS. For a grouped query the
772
+ # groups ARE the answer, so a truncated result is a WRONG NUMBER, not a short list.
773
+ "truncated": truncated}
774
 
775
 
776
  def store_rows(topic, date_from=None, date_to=None, team_id=None, filter_tree=None,
platform/harness/tables.py CHANGED
@@ -1,129 +1,129 @@
1
- """harness/tables.py — a topic-backed TABLE for the grid (CG-2).
2
-
3
- The grid's first table (customers) is built from `modules.customer_data.pool()`: one row per
4
- customer, the whole book shipped as component args, filtered client-side. That works at 1,550
5
- rows and does not work at line grain — `sales_lines` is 201,558 rows / ~89 MB.
6
-
7
- This module builds the SECOND table: rows come from `semantic.store_rows`, so the fetch is
8
- WINDOWED and the filtering is SQL. Its whole job is to hand the grid a payload in the same
9
- host-neutral shape the React component already speaks, plus the one thing a windowed table must
10
- carry that a whole-book table never needed:
11
-
12
- counts = {matched, total, shown, windowed}
13
-
14
- `matched` and `total` come from their own queries over the FULL scope; `shown` is the size of
15
- this window. The component renders "showing SHOWN of MATCHED (filtered from TOTAL)" from those
16
- numbers and NEVER from `rows.length` — which is the difference between an honest window and a
17
- silent `[:N]` wearing a total ([[no-unverifiable-aggregates]]).
18
-
19
- The field CONTRACT is derived from the topic, not hand-written: labels and types come from the
20
- model's dims and metrics, so a new metric cannot arrive as an untyped, unlabelled column and the
21
- grid can never disagree with the semantic layer about what a column means.
22
- """
23
- from harness import semantic as _sem
24
-
25
- #: Topic-backed tables the grid can open. `key` is the storage key its saved views live under —
26
- #: separate from the customer table's, so the two tables' views can never collide.
27
- TOPIC_TABLES = {
28
- "sales_lines": {
29
- "label": "Sales lines",
30
- "storage_key": "sales_lines_table_workspace",
31
- # the columns worth defaulting to visible, in order; everything else stays available
32
- "default_visible": ["date", "order_partner", "product", "category", "state", "agent",
33
- "team", "revenue", "units", "margin"],
34
- # a line-grain table is unusable sorted by nothing in particular
35
- "default_sorts": [{"colId": "date", "dir": "desc"}],
36
- },
37
- }
38
-
39
- #: how many rows one window carries. Small enough that the payload stays a few hundred KB at
40
- #: ~440 B/row, large enough to scroll. The honest counts make the window visible rather than
41
- #: silent, so this is a performance knob and not a correctness one.
42
- PAGE_ROWS = 200
43
-
44
-
45
- def _labels(topic):
46
- """colId -> human label, taken from the MODEL (dim labels, metric labels)."""
47
- t = _sem.topics()[topic]
48
- out = {}
49
- for key, d in ((t.get("store") or {}).get("dims") or {}).items():
50
- out[key] = d.get("label") or key.replace("_", " ").title()
51
- out[f"{key}_id"] = f"{out[key]} id"
52
- out["date"] = "Date"
53
- for k, m in _sem.metrics().items():
54
- if m.get("topic") == topic:
55
- out[k] = m.get("label") or k
56
- return out
57
-
58
-
59
- def table_fields(topic):
60
- """The grid field contract for a topic, derived from the semantic model.
61
-
62
- Every column is `source='odoo'` — read-only. A line is a fact from the ERP; unlike the
63
- customer table there is no overlay stratum here (notes/tags hang off a CUSTOMER, not off an
64
- individual order line), so the two-strata split simply does not arise.
65
- """
66
- cfg = TOPIC_TABLES[topic]
67
- cols = _sem.store_columns(topic, grain="row")
68
- labels = _labels(topic)
69
- order = [k for k in cfg["default_visible"] if k in cols]
70
- order += [k for k in cols if k not in order]
71
- return [{
72
- "key": k,
73
- "label": labels.get(k, k.replace("_", " ").title()),
74
- "type": cols[k]["type"],
75
- "source": "odoo",
76
- "default": k in cfg["default_visible"],
77
- } for k in order]
78
-
79
-
80
- def table_payload(topic, config=None, offset=0, limit=None, team_id=None,
81
- date_from=None, date_to=None):
82
- """Fetch ONE window of a topic-backed table plus its scope-wide counts and totals.
83
-
84
- `config` is a grid ViewConfig (filters / filterConj / sorts / search) — the same shape the
85
- customer table persists, so a saved view means the same thing on either table.
86
-
87
- Returns the component payload: {fields, rows, counts, aggregates, storageKey}.
88
- """
89
- if topic not in TOPIC_TABLES:
90
- raise _sem.ModelError(f"no topic-backed table registered for {topic!r}")
91
- cfg = TOPIC_TABLES[topic]
92
- config = config or {}
93
- limit = PAGE_ROWS if limit is None else limit
94
-
95
- got = _sem.store_rows(
96
- topic,
97
- filter_tree=config.get("filters") or [],
98
- filter_conj=config.get("filterConj") or "and",
99
- search=config.get("search") or None,
100
- sorts=config.get("sorts") or cfg["default_sorts"],
101
- member_ids=config.get("memberPids") or None,
102
- limit=limit, offset=offset,
103
- team_id=team_id, date_from=date_from, date_to=date_to,
104
- )
105
-
106
- rows = []
107
- for i, r in enumerate(got["rows"]):
108
- # `pid` is the grid's row identity contract (selection, detail, overlay all key on it).
109
- # At line grain that identity is the order LINE id, which store_rows returns as _rid.
110
- row = {k: v for k, v in r.items() if k != "_rid"}
111
- row["pid"] = r["_rid"]
112
- rows.append(row)
113
-
114
- return {
115
- "fields": table_fields(topic),
116
- "rows": rows,
117
- # THE honest window. `matched`/`total` are scope-wide queries; `shown` is this window.
118
- # A caller must never recompute `matched` from len(rows).
119
- "counts": {
120
- "shown": len(rows),
121
- "matched": got["row_count"],
122
- "total": got["total_count"],
123
- "windowed": True,
124
- },
125
- "aggregates": got["aggregates"], # over the whole FILTERED scope, not the window
126
- "storageKey": cfg["storage_key"],
127
- "label": cfg["label"],
128
- "window": got["window"],
129
- }
 
1
+ """harness/tables.py — a topic-backed TABLE for the grid (CG-2).
2
+
3
+ The grid's first table (customers) is built from `modules.customer_data.pool()`: one row per
4
+ customer, the whole book shipped as component args, filtered client-side. That works at 1,550
5
+ rows and does not work at line grain — `sales_lines` is 201,558 rows / ~89 MB.
6
+
7
+ This module builds the SECOND table: rows come from `semantic.store_rows`, so the fetch is
8
+ WINDOWED and the filtering is SQL. Its whole job is to hand the grid a payload in the same
9
+ host-neutral shape the React component already speaks, plus the one thing a windowed table must
10
+ carry that a whole-book table never needed:
11
+
12
+ counts = {matched, total, shown, windowed}
13
+
14
+ `matched` and `total` come from their own queries over the FULL scope; `shown` is the size of
15
+ this window. The component renders "showing SHOWN of MATCHED (filtered from TOTAL)" from those
16
+ numbers and NEVER from `rows.length` — which is the difference between an honest window and a
17
+ silent `[:N]` wearing a total ([[no-unverifiable-aggregates]]).
18
+
19
+ The field CONTRACT is derived from the topic, not hand-written: labels and types come from the
20
+ model's dims and metrics, so a new metric cannot arrive as an untyped, unlabelled column and the
21
+ grid can never disagree with the semantic layer about what a column means.
22
+ """
23
+ from harness import semantic as _sem
24
+
25
+ #: Topic-backed tables the grid can open. `key` is the storage key its saved views live under —
26
+ #: separate from the customer table's, so the two tables' views can never collide.
27
+ TOPIC_TABLES = {
28
+ "sales_lines": {
29
+ "label": "Sales lines",
30
+ "storage_key": "sales_lines_table_workspace",
31
+ # the columns worth defaulting to visible, in order; everything else stays available
32
+ "default_visible": ["date", "order_partner", "product", "category", "state", "agent",
33
+ "team", "revenue", "units", "margin"],
34
+ # a line-grain table is unusable sorted by nothing in particular
35
+ "default_sorts": [{"colId": "date", "dir": "desc"}],
36
+ },
37
+ }
38
+
39
+ #: how many rows one window carries. Small enough that the payload stays a few hundred KB at
40
+ #: ~440 B/row, large enough to scroll. The honest counts make the window visible rather than
41
+ #: silent, so this is a performance knob and not a correctness one.
42
+ PAGE_ROWS = 200
43
+
44
+
45
+ def _labels(topic):
46
+ """colId -> human label, taken from the MODEL (dim labels, metric labels)."""
47
+ t = _sem.topics()[topic]
48
+ out = {}
49
+ for key, d in ((t.get("store") or {}).get("dims") or {}).items():
50
+ out[key] = d.get("label") or key.replace("_", " ").title()
51
+ out[f"{key}_id"] = f"{out[key]} id"
52
+ out["date"] = "Date"
53
+ for k, m in _sem.metrics().items():
54
+ if m.get("topic") == topic:
55
+ out[k] = m.get("label") or k
56
+ return out
57
+
58
+
59
+ def table_fields(topic):
60
+ """The grid field contract for a topic, derived from the semantic model.
61
+
62
+ Every column is `source='odoo'` — read-only. A line is a fact from the ERP; unlike the
63
+ customer table there is no overlay stratum here (notes/tags hang off a CUSTOMER, not off an
64
+ individual order line), so the two-strata split simply does not arise.
65
+ """
66
+ cfg = TOPIC_TABLES[topic]
67
+ cols = _sem.store_columns(topic, grain="row")
68
+ labels = _labels(topic)
69
+ order = [k for k in cfg["default_visible"] if k in cols]
70
+ order += [k for k in cols if k not in order]
71
+ return [{
72
+ "key": k,
73
+ "label": labels.get(k, k.replace("_", " ").title()),
74
+ "type": cols[k]["type"],
75
+ "source": "odoo",
76
+ "default": k in cfg["default_visible"],
77
+ } for k in order]
78
+
79
+
80
+ def table_payload(topic, config=None, offset=0, limit=None, team_id=None,
81
+ date_from=None, date_to=None):
82
+ """Fetch ONE window of a topic-backed table plus its scope-wide counts and totals.
83
+
84
+ `config` is a grid ViewConfig (filters / filterConj / sorts / search) — the same shape the
85
+ customer table persists, so a saved view means the same thing on either table.
86
+
87
+ Returns the component payload: {fields, rows, counts, aggregates, storageKey}.
88
+ """
89
+ if topic not in TOPIC_TABLES:
90
+ raise _sem.ModelError(f"no topic-backed table registered for {topic!r}")
91
+ cfg = TOPIC_TABLES[topic]
92
+ config = config or {}
93
+ limit = PAGE_ROWS if limit is None else limit
94
+
95
+ got = _sem.store_rows(
96
+ topic,
97
+ filter_tree=config.get("filters") or [],
98
+ filter_conj=config.get("filterConj") or "and",
99
+ search=config.get("search") or None,
100
+ sorts=config.get("sorts") or cfg["default_sorts"],
101
+ member_ids=config.get("memberPids") or None,
102
+ limit=limit, offset=offset,
103
+ team_id=team_id, date_from=date_from, date_to=date_to,
104
+ )
105
+
106
+ rows = []
107
+ for i, r in enumerate(got["rows"]):
108
+ # `pid` is the grid's row identity contract (selection, detail, overlay all key on it).
109
+ # At line grain that identity is the order LINE id, which store_rows returns as _rid.
110
+ row = {k: v for k, v in r.items() if k != "_rid"}
111
+ row["pid"] = r["_rid"]
112
+ rows.append(row)
113
+
114
+ return {
115
+ "fields": table_fields(topic),
116
+ "rows": rows,
117
+ # THE honest window. `matched`/`total` are scope-wide queries; `shown` is this window.
118
+ # A caller must never recompute `matched` from len(rows).
119
+ "counts": {
120
+ "shown": len(rows),
121
+ "matched": got["row_count"],
122
+ "total": got["total_count"],
123
+ "windowed": True,
124
+ },
125
+ "aggregates": got["aggregates"], # over the whole FILTERED scope, not the window
126
+ "storageKey": cfg["storage_key"],
127
+ "label": cfg["label"],
128
+ "window": got["window"],
129
+ }
platform/harness/tools.py CHANGED
@@ -1,510 +1,510 @@
1
- """harness/tools.py — the compounding TOOL REGISTRY (OM-4 spine, 2026-07-11).
2
-
3
- The formal tool surface the (small) model calls — the productization directive's "tools we keep
4
- compounding". Every tool wraps the SEMANTIC layer (harness/semantic.py): the model navigates by
5
- registry keys and recipe plans (model/skills/*.skill.yml — recipes and these tools version
6
- TOGETHER), never by SQL. Adding a connector/topic extends what the SAME tools reach — that is the
7
- compounding. Exported in OpenAI function-calling format (`openai_tools()`) — OpenRouter-compatible,
8
- so any cheap model with tool-calling drives the platform.
9
-
10
- Correctness posture (Part VI of the plan): whitelisted keys only; values parameterized downstream;
11
- every query result carries a result_id + drill note; artifact tools (save/compose/schedule/alert)
12
- are explicit and confirm-gated by recipe. Errors return a uniform envelope the model can read.
13
- """
14
- import json
15
- import time
16
- import uuid
17
- from pathlib import Path
18
-
19
- import harness.semantic as SEM
20
-
21
- VIEWS_PATH = Path(__file__).resolve().parents[1] / "data" / "store" / "views.json"
22
-
23
- _RESULTS = {} # result_id -> query result (session-scoped working memory for chart tools)
24
- _RESULTS_CAP = 40
25
-
26
- # The exhaustive chart vocabulary (2026-07-16): every Zelazny comparison form has a kind, so the
27
- # Analyst never lacks a shape. The model picks by the CHART PICKER guide (analyst.py) + the
28
- # charting skill recipes; the platform owns every pixel (app._render_analyst_artifact).
29
- CHART_KINDS = (
30
- "line", "bar", "area", "scatter", "kpi", "map", # the original six
31
- "pie", "donut", # part-to-whole (≤6 slices)
32
- "stacked_bar", "grouped_bar", "ranked_bar", "stacked_pct", # composition / rank forms
33
- "combo", "yoy_bars", # level+rate; this-vs-last-year
34
- "waterfall", "pareto", "histogram", "heatmap", "treemap", # bridge / concentration / distribution
35
- "funnel", "bullet", "bubble", "sparkline", # stages / target / 3-measure / mini
36
- )
37
-
38
- # Per-kind param contract (beyond x): what else the spec must carry to be renderable.
39
- _KIND_NEEDS = {
40
- "combo": ("y", "y2"), "bullet": ("y", "y2"), "bubble": ("y", "size"),
41
- "heatmap": ("y", "value"), "histogram": (), # histogram bins x itself
42
- "stacked_bar": ("y", "series"), "grouped_bar": ("y", "series"),
43
- "stacked_pct": ("y", "series"),
44
- }
45
- _QUERY_KEYS = ("topic", "measures", "group_by", "grain", "date_from", "date_to",
46
- "team_id", "filters", "sort", "limit", "exclude_services")
47
-
48
-
49
- def _remember(res):
50
- rid = uuid.uuid4().hex[:10]
51
- _RESULTS[rid] = res
52
- while len(_RESULTS) > _RESULTS_CAP:
53
- _RESULTS.pop(next(iter(_RESULTS)))
54
- return rid
55
-
56
-
57
- def _ok(data):
58
- return {"ok": True, "data": data}
59
-
60
-
61
- def _err(msg):
62
- return {"ok": False, "error": str(msg)[:400]}
63
-
64
-
65
- # ------------------------------------------------------------------ schema tools
66
-
67
- def list_topics():
68
- """The 'what data exists' tool."""
69
- out = []
70
- for k, t in SEM.topics().items():
71
- out.append({"topic": k, "label": t.get("label"), "entity": t.get("entity"),
72
- "grain": t.get("grain"),
73
- "dims": list((t.get("store") or {}).get("dims") or {}),
74
- "metrics": [m for m, d in SEM.metrics().items() if d["topic"] == k]})
75
- return out
76
-
77
-
78
- def describe_topic(topic):
79
- """The schema-learning tool: scope, grain, dims, metrics w/ definitions, and ai_context."""
80
- t = SEM.topics().get(topic)
81
- if not t:
82
- raise SEM.ModelError(f"unknown topic {topic!r} (use list_topics)")
83
- mets = {k: {"label": m.get("label"), "description": m.get("description"),
84
- "format": m.get("format"), "ai_context": m.get("ai_context")}
85
- for k, m in SEM.metrics().items() if m["topic"] == topic}
86
- return {"topic": topic, "label": t.get("label"), "scope": t.get("scope"),
87
- "grain": t.get("grain"), "ai_context": t.get("ai_context"),
88
- "dims": {k: v.get("label") for k, v in ((t.get("store") or {}).get("dims") or {}).items()},
89
- "metrics": mets}
90
-
91
-
92
- # ------------------------------------------------------------------ query tools
93
-
94
- def run_semantic_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
95
- team_id=None, filters=None, sort=None, limit=1000, exclude_services=False):
96
- # limit defaults HIGH (1000): transforms/charts operate on the FULL result while the model
97
- # only ever sees rows[:100] — a small default silently truncated per-customer analytics
98
- # (the BELLA FLORIST wrong-decliner incident, 2026-07-17).
99
- res = SEM.store_query(topic, measures, group_by=group_by, grain=grain, date_from=date_from,
100
- date_to=date_to, team_id=team_id, filters=filters, sort=sort,
101
- limit=limit, exclude_services=exclude_services)
102
- # Echo the full query onto the result: chart specs built from it carry the query, so a SAVED
103
- # view is a re-runnable QUERY (the OM-3 viewer re-executes it live), never a stale snapshot.
104
- res["query"] = {"topic": topic, "measures": list(measures or []), "group_by": group_by,
105
- "grain": grain, "date_from": date_from, "date_to": date_to, "team_id": team_id,
106
- "filters": filters, "sort": sort, "limit": limit,
107
- "exclude_services": exclude_services}
108
- rid = _remember(res)
109
- # The EFFECTIVE window, stated by the platform — the model must repeat this, never guess
110
- # (a query without dates covers all recorded history; there is no hidden default window).
111
- if date_from and date_to:
112
- window = f"{date_from} to {date_to}"
113
- elif date_from or date_to:
114
- window = f"{'from ' + date_from if date_from else 'through ' + date_to}"
115
- else:
116
- window = "ALL recorded history (no date filter was applied)"
117
- out = {"result_id": rid, "rows": res["rows"][:100], "row_count": res["row_count"],
118
- "measures": res["measures"], "group_by": res["group_by"], "grain": res["grain"],
119
- "window": window,
120
- "note": "every number here is drillable; cite result_id when charting"}
121
- if res["row_count"] >= (limit or 1000): # surface every truncation (plan hard line)
122
- out["warning"] = (f"TRUNCATED: the result hit limit={limit} — the full set is larger. "
123
- "Re-run with a higher limit (max 5000) BEFORE ranking, comparing or "
124
- "aggregating, or your answer will be computed on a partial set.")
125
- return out
126
-
127
-
128
- def get_field_values(topic, dim, search=None):
129
- return SEM.store_field_values(topic, dim, search=search)
130
-
131
-
132
- # ------------------------------------------------------------------ transform tool (governed)
133
-
134
- def transform_result(result_id, transforms):
135
- """Apply governed ANALYTICS TRANSFORMS to a query result -> a NEW result_id to chart/table.
136
- The chain is recorded on the derived result, so saved views replay query -> transforms live."""
137
- res = _RESULTS.get(result_id)
138
- if not res:
139
- raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first")
140
- import harness.transforms as TR
141
- rows, applied = TR.apply(res, transforms, run_query=_run_query)
142
- new = {**res, "rows": rows, "row_count": len(rows),
143
- "transforms": (res.get("transforms") or []) + applied}
144
- rid = _remember(new)
145
- return {"result_id": rid, "rows": rows[:100], "row_count": len(rows),
146
- "columns": sorted(rows[0]) if rows else [],
147
- "note": "derived result — chart THIS result_id to show the transform"}
148
-
149
-
150
- def _run_query(q):
151
- return SEM.store_query(**{k: q.get(k) for k in _QUERY_KEYS if q.get(k) is not None})
152
-
153
-
154
- # ------------------------------------------------------------------ viz tools (emit OUR specs)
155
-
156
- def make_chart(result_id, kind, x, y=None, title=None, series=None, y2=None, size=None,
157
- value=None, facet=None):
158
- """Returns a validated CHART SPEC the platform renders with its own primitives (design system
159
- enforced — the model never emits HTML/vega). Extra encodings per kind: combo/bullet need y2
160
- (line/target), bubble needs size, heatmap needs value (the colour measure); facet (a dim
161
- column) turns line|bar|area|scatter into small multiples."""
162
- res = _RESULTS.get(result_id)
163
- if not res:
164
- raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first")
165
- if kind not in CHART_KINDS:
166
- raise SEM.ModelError(f"kind must be one of {CHART_KINDS}")
167
- cols = set(res["rows"][0]) if res["rows"] else set()
168
- if y is None and kind != "histogram":
169
- raise SEM.ModelError(f"kind={kind!r} needs y (only histogram bins x by itself)")
170
- for ref, nm in ((x, "x"), (y, "y"), (series, "series"), (y2, "y2"), (size, "size"),
171
- (value, "value"), (facet, "facet")):
172
- if ref and ref not in cols:
173
- raise SEM.ModelError(f"{nm}={ref!r} not in result columns {sorted(cols)}")
174
- given = {"y": y, "y2": y2, "size": size, "value": value, "series": series}
175
- missing = [p for p in _KIND_NEEDS.get(kind, ()) if not given.get(p)]
176
- if missing:
177
- raise SEM.ModelError(f"kind={kind!r} also needs {missing} "
178
- f"(pick from result columns {sorted(cols)})")
179
- if facet and kind not in ("line", "bar", "area", "scatter"):
180
- raise SEM.ModelError("facet (small multiples) works with line|bar|area|scatter only")
181
- if kind == "yoy_bars" and f"{y}_ly" not in cols:
182
- raise SEM.ModelError(f"yoy_bars needs a {y}_ly column — run transform_result "
183
- "[{'op':'yoy'}] on the result first")
184
- spec = {"kind": kind, "x": x, "y": y, "series": series,
185
- "title": title or f"{y or x} by {x}", "result_id": result_id,
186
- "query": res.get("query"), "rows": res["rows"]}
187
- for k, v in (("y2", y2), ("size", size), ("value", value), ("facet", facet),
188
- ("transforms", res.get("transforms"))):
189
- if v:
190
- spec[k] = v
191
- out = {"chart": spec}
192
- if kind in ("pie", "donut") and len(res["rows"]) > 6:
193
- out["note"] = (f"{len(res['rows'])} slices — the platform will show the top 5 plus an "
194
- "'Other' bucket; for a cleaner story run transform_result top_n first")
195
- return out
196
-
197
-
198
- def make_table(result_id, columns=None, title=None):
199
- """First-class TABLE artifact: the exact rows, house-formatted (sortable, totals row, the
200
- drill IS the table). columns (optional) picks and orders a subset."""
201
- res = _RESULTS.get(result_id)
202
- if not res:
203
- raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first")
204
- rows = res["rows"]
205
- if columns:
206
- cols = set(rows[0]) if rows else set()
207
- bad = [c for c in columns if c not in cols]
208
- if bad:
209
- raise SEM.ModelError(f"columns {bad} not in result columns {sorted(cols)}")
210
- rows = [{c: r.get(c) for c in columns} for r in rows]
211
- return {"table": {"kind": "table", "title": title, "columns": columns,
212
- "result_id": result_id, "query": res.get("query"),
213
- "transforms": res.get("transforms"), "rows": rows}}
214
-
215
-
216
- def make_kpi_card(result_id, metric, compare_result_id=None):
217
- res = _RESULTS.get(result_id)
218
- if not res or not res["rows"]:
219
- raise SEM.ModelError("result_id missing/empty — run a scalar run_semantic_query first")
220
- val = res["rows"][0].get(metric)
221
- if val is None:
222
- raise SEM.ModelError(f"{metric!r} not in result")
223
- card = {"kpi": {"metric": metric, "value": val, "result_id": result_id,
224
- "query": res.get("query")}}
225
- if compare_result_id and _RESULTS.get(compare_result_id, {}).get("rows"):
226
- prev = _RESULTS[compare_result_id]["rows"][0].get(metric)
227
- if prev:
228
- card["kpi"]["delta_pct"] = (val - prev) / abs(prev)
229
- card["kpi"]["compare_result_id"] = compare_result_id
230
- card["kpi"]["compare_query"] = _RESULTS[compare_result_id].get("query")
231
- return card
232
-
233
-
234
- # ------------------------------------------------------------------ artifact tools (v0: local)
235
-
236
- def _load_views():
237
- if VIEWS_PATH.exists():
238
- return json.loads(VIEWS_PATH.read_text(encoding="utf-8"))
239
- return {"views": {}, "dashboards": {}}
240
-
241
-
242
- def _save_views(d):
243
- VIEWS_PATH.parent.mkdir(parents=True, exist_ok=True)
244
- VIEWS_PATH.write_text(json.dumps(d, indent=1), encoding="utf-8")
245
-
246
-
247
- def save_view(name, chart):
248
- """Persist a chart/KPI spec (from make_chart / make_kpi_card) as a named view. Specs persist
249
- WITH their semantic query and WITHOUT rows — the OM-3 viewer re-executes the query live, so a
250
- saved view is always current, never a snapshot."""
251
- d = _load_views()
252
- spec = chart.get("chart") or chart.get("kpi") or chart.get("table") or chart
253
- spec = {k: v for k, v in spec.items() if k != "rows"}
254
- if "kpi" in chart and not spec.get("kind"):
255
- spec["kind"] = "kpi"
256
- if "table" in chart and not spec.get("kind"):
257
- spec["kind"] = "table"
258
- if not spec.get("query"):
259
- raise SEM.ModelError("spec carries no query — pass the exact object returned by "
260
- "make_chart / make_kpi_card (from a fresh run_semantic_query)")
261
- d["views"][name] = {"chart": spec, "saved_at": time.strftime("%Y-%m-%d %H:%M")}
262
- _save_views(d)
263
- return {"saved": name, "views": list(d["views"])}
264
-
265
-
266
- def compose_dashboard(name, views):
267
- """'Spawn a dashboard': compose saved views into a named dashboard spec (rendered at OM-3)."""
268
- d = _load_views()
269
- missing = [v for v in views if v not in d["views"]]
270
- if missing:
271
- raise SEM.ModelError(f"unknown views {missing} — save_view them first")
272
- d["dashboards"][name] = {"views": views, "created_at": time.strftime("%Y-%m-%d %H:%M")}
273
- _save_views(d)
274
- return {"dashboard": name, "views": views}
275
-
276
-
277
- # ------------------------------------------------------------------ the GAP LOOP (rung 4)
278
-
279
- GAP_KINDS = ("dimension", "metric", "transform", "chart_kind", "data_source", "other")
280
- GAPS_KEY = "analyst_gaps"
281
- GAPS_CAP = 500
282
-
283
-
284
- def report_gap(kind, missing, question, workaround=None):
285
- """Log a CAPABILITY GAP: the model determined (after checking the schema) that no registered
286
- dim/metric/transform/kind can answer. The entry lands in telemetry AND the durable store —
287
- the admin Gaps view aggregates them into the platform build backlog. This is how every
288
- honest 'I can't' becomes the next dim, transform, or recipe."""
289
- if kind not in GAP_KINDS:
290
- raise SEM.ModelError(f"kind must be one of {GAP_KINDS}")
291
- entry = {"ts": time.strftime("%Y-%m-%d %H:%M:%S"), "kind": kind,
292
- "missing": str(missing)[:120], "question": str(question)[:300],
293
- "workaround": (str(workaround)[:200] if workaround else None)}
294
- import harness.telemetry as TEL
295
- TEL.log("analyst_gap", **{("gap_kind" if k == "kind" else k): v for k, v in entry.items()})
296
- try: # durable copy — async, the chatlog write posture
297
- import threading
298
-
299
- import core.store as store
300
- if store.available():
301
- def _fn(data):
302
- data = list(data or [])
303
- data.append(entry)
304
- return data[-GAPS_CAP:]
305
- threading.Thread(target=lambda: store.update(GAPS_KEY, _fn),
306
- daemon=True, name="analyst-gap").start()
307
- except Exception:
308
- pass
309
- return {"logged": True,
310
- "note": "gap recorded for the platform backlog — now tell the user in ONE sentence "
311
- "what is missing and offer the nearest ask that IS answerable"}
312
-
313
-
314
- # ------------------------------------------------------------------ workspace tools (OM-3/P11)
315
-
316
- def list_workspace():
317
- """Everything in the tenant workspace (uniform modular objects) + available templates."""
318
- import harness.workspace as W
319
- return {"objects": W.items(), "templates": W.templates()}
320
-
321
-
322
- def instantiate_template(filename, new_name=None):
323
- """Stamp a tenant-agnostic template into this workspace as NEW objects (never overwrites)."""
324
- import harness.workspace as W
325
- return W.instantiate_template(filename, new_name)
326
-
327
-
328
- def update_view(name, changes):
329
- """Patch an existing view (spec and/or query) — validated by re-execution before saving."""
330
- import harness.views as V
331
- return V.update_view(name, changes)
332
-
333
-
334
- def update_workbook(name, views=None, new_name=None):
335
- """Recompose and/or rename a workbook (dashboard); renames follow into schedules."""
336
- import harness.views as V
337
- return V.update_dashboard(name, views_list=views, new_name=new_name)
338
-
339
-
340
- def delete_object(kind, name):
341
- """Delete any workspace object. DESTRUCTIVE — the recipe requires explicit confirmation."""
342
- import harness.workspace as W
343
- W.delete(kind, name)
344
- return {"deleted": f"{kind} · {name}"}
345
-
346
-
347
- # ------------------------------------------------------------------ registry + dispatch
348
-
349
- def _p(props, required):
350
- return {"type": "object", "properties": props, "required": required}
351
-
352
-
353
- TOOLS = {
354
- "list_topics": {"fn": lambda **kw: list_topics(),
355
- "description": "List the datasets (topics) available: their metrics, dims, and grain. Start here.",
356
- "parameters": _p({}, [])},
357
- "describe_topic": {"fn": lambda **kw: describe_topic(kw["topic"]),
358
- "description": "Full schema of one topic: scope rules, metric definitions, dims, and the business context you must respect.",
359
- "parameters": _p({"topic": {"type": "string"}}, ["topic"])},
360
- "get_field_values": {"fn": lambda **kw: get_field_values(kw["topic"], kw["dim"], kw.get("search")),
361
- "description": "Resolve real filter values (ids+names) for a dim. ALWAYS use before filtering by a typed name.",
362
- "parameters": _p({"topic": {"type": "string"}, "dim": {"type": "string"},
363
- "search": {"type": "string"}}, ["topic", "dim"])},
364
- "run_semantic_query": {"fn": lambda **kw: run_semantic_query(**kw),
365
- "description": "Run a governed query: registered measures over a topic, optional group_by dims / time grain / filters. The ONLY way to read data.",
366
- "parameters": _p({"topic": {"type": "string"},
367
- "measures": {"type": "array", "items": {"type": "string"}},
368
- "group_by": {"type": "array", "items": {"type": "string"}},
369
- "grain": {"type": "string", "enum": ["month", "week", "day"]},
370
- "date_from": {"type": "string"}, "date_to": {"type": "string"},
371
- "team_id": {"type": "integer"},
372
- "filters": {"type": "object"},
373
- "sort": {"type": "string"}, "limit": {"type": "integer"},
374
- "exclude_services": {"type": "boolean"}},
375
- ["topic", "measures"])},
376
- "transform_result": {"fn": lambda **kw: transform_result(kw["result_id"], kw["transforms"]),
377
- "description": "Apply governed analytics transforms to a result -> a NEW result_id (a "
378
- "CHAINABLE list of {op, ...}). The Tableau-class analytics library — pick "
379
- "op names (full catalog + recipes in the analytics skill). Families: "
380
- "ordering/rank (sort, head, bottom_n, rank, rank_pct, ntile, top_n, "
381
- "add_total) · part-to-whole (share_of_total, cum_share) · running/moving "
382
- "(running_total/avg/max/min, running_count, moving_average/sum/median, "
383
- "rolling_std) · period-over-period (diff, pct_change, lag, lead, "
384
- "diff_from_first, index_to_100, percent_of_max, compare) · distribution/"
385
- "stats (bin, describe, zscore, outliers, winsorize, clip, normalize, "
386
- "correlate, weighted_average, safe_ratio, product) · business (abc_classify, "
387
- "concentration, contribution_to_change, rfm, funnel_rates) · modeling "
388
- "(trend_line, regression, cagr, growth_rate) · reference lines as columns "
389
- "(reference_line, reference_band, target_line, xmr_limits) · reshape (pivot, "
390
- "unpivot, filter_rows, dedupe, resample) · re-query windows (yoy, ytd, "
391
- "rolling, forecast). ADDITIVITY LAW: accumulating ops "
392
- "(running_total/share_of_total/cum_share/moving_sum/abc_classify/"
393
- "concentration) work on ADDITIVE measures (revenue/units/margin/orders); "
394
- "for a cumulative/trailing DISTINCT count (customers) or ratio use ytd/"
395
- "rolling (they re-query) — never running_total. NEVER compute any of these "
396
- "yourself; transform, then chart/table the new result_id.",
397
- "parameters": _p({"result_id": {"type": "string"},
398
- "transforms": {"type": "array", "items": {"type": "object"}}},
399
- ["result_id", "transforms"])},
400
- "make_chart": {"fn": lambda **kw: make_chart(**kw),
401
- "description": "Turn a query result into a platform chart. kinds: line|bar|area|scatter|"
402
- "map|pie|donut|stacked_bar|grouped_bar|ranked_bar|stacked_pct|combo|"
403
- "yoy_bars|waterfall|pareto|histogram|heatmap|treemap|funnel|bullet|bubble|"
404
- "sparkline. x/y/series must be result columns. Extra encodings: combo "
405
- "(bars y + line y2, dual axis), bullet (value y vs target y2), bubble "
406
- "(scatter + size), heatmap (dims x,y + colour value), histogram (bins x, "
407
- "no y), facet (a dim column -> small multiples of line|bar|area|scatter). "
408
- "yoy_bars needs the yoy transform first. kind='map' plots customers "
409
- "geographically: x = the customer dim, dot size = y.",
410
- "parameters": _p({"result_id": {"type": "string"}, "kind": {"type": "string", "enum": list(CHART_KINDS)},
411
- "x": {"type": "string"}, "y": {"type": "string"},
412
- "title": {"type": "string"}, "series": {"type": "string"},
413
- "y2": {"type": "string"}, "size": {"type": "string"},
414
- "value": {"type": "string"}, "facet": {"type": "string"}},
415
- ["result_id", "kind", "x"])},
416
- "make_table": {"fn": lambda **kw: make_table(kw["result_id"], kw.get("columns"),
417
- kw.get("title")),
418
- "description": "Turn a query result into a first-class TABLE artifact (house-formatted, "
419
- "totals row, drillable). Use when the user wants exact figures, many "
420
- "columns, or a list — not a shape. columns (optional) picks and orders.",
421
- "parameters": _p({"result_id": {"type": "string"},
422
- "columns": {"type": "array", "items": {"type": "string"}},
423
- "title": {"type": "string"}}, ["result_id"])},
424
- "make_kpi_card": {"fn": lambda **kw: make_kpi_card(**kw),
425
- "description": "Turn a scalar query result into a KPI card; optional compare_result_id adds a YoY delta.",
426
- "parameters": _p({"result_id": {"type": "string"}, "metric": {"type": "string"},
427
- "compare_result_id": {"type": "string"}}, ["result_id", "metric"])},
428
- "report_gap": {"fn": lambda **kw: report_gap(kw["kind"], kw["missing"], kw["question"],
429
- kw.get("workaround")),
430
- "description": "LAST RESORT — log a capability gap. Call ONLY after list_topics/"
431
- "describe_topic confirm that NO registered dimension, metric, transform "
432
- "or chart kind can answer the user's question (e.g. stock on hand, "
433
- "which has no topic). NOT a gap: YoY/decline/growth compares "
434
- "(transform_result yoy), rankings/top-N, shares, running totals, "
435
- "distributions — those are ANSWERABLE via transform_result. Then tell "
436
- "the user plainly what is missing and offer the nearest answerable ask. "
437
- "NEVER call this for something the tools support, and NEVER guess "
438
- "instead of calling it.",
439
- "parameters": _p({"kind": {"type": "string", "enum": list(GAP_KINDS)},
440
- "missing": {"type": "string",
441
- "description": "what does not exist, short (e.g. 'inventory/stock-on-hand topic')"},
442
- "question": {"type": "string",
443
- "description": "the user's question, verbatim"},
444
- "workaround": {"type": "string",
445
- "description": "the nearest answerable alternative you offered"}},
446
- ["kind", "missing", "question"])},
447
- "save_view": {"fn": lambda **kw: save_view(kw["name"], kw["chart"]),
448
- "description": "Save a chart as a named view (confirm with the user first).",
449
- "parameters": _p({"name": {"type": "string"}, "chart": {"type": "object"}}, ["name", "chart"])},
450
- "compose_dashboard": {"fn": lambda **kw: compose_dashboard(kw["name"], kw["views"]),
451
- "description": "Compose saved views into a named dashboard (confirm with the user first).",
452
- "parameters": _p({"name": {"type": "string"},
453
- "views": {"type": "array", "items": {"type": "string"}}}, ["name", "views"])},
454
- "list_workspace": {"fn": lambda **kw: list_workspace(),
455
- "description": "List the tenant workspace: every saved view/dashboard/alert/report "
456
- "(modular objects) plus available templates. Use when the user asks what "
457
- "exists, wants to reuse/manage artifacts, or before composing.",
458
- "parameters": _p({}, [])},
459
- "instantiate_template": {"fn": lambda **kw: instantiate_template(kw["filename"],
460
- kw.get("new_name")),
461
- "description": "Stamp a tenant-agnostic template (from list_workspace) into the "
462
- "workspace as NEW objects — never overwrites. Confirm with the user first.",
463
- "parameters": _p({"filename": {"type": "string"}, "new_name": {"type": "string"}},
464
- ["filename"])},
465
- "update_view": {"fn": lambda **kw: update_view(kw["name"], kw.get("changes")),
466
- "description": "UPDATE an existing saved view: patch spec keys (title/kind/x/y/series) "
467
- "and/or 'query' subkeys (measures, group_by, grain, date_from, date_to, "
468
- "team_id, filters, sort, limit; null REMOVES a key). The patched query is "
469
- "re-executed before saving — invalid updates are rejected. Confirm first.",
470
- "parameters": _p({"name": {"type": "string"}, "changes": {"type": "object"}},
471
- ["name", "changes"])},
472
- "update_workbook": {"fn": lambda **kw: update_workbook(kw["name"], kw.get("views"),
473
- kw.get("new_name")),
474
- "description": "UPDATE a workbook (dashboard): recompose its views (list = new display "
475
- "order; add/remove by including/omitting) and/or rename it. Confirm first.",
476
- "parameters": _p({"name": {"type": "string"},
477
- "views": {"type": "array", "items": {"type": "string"}},
478
- "new_name": {"type": "string"}}, ["name"])},
479
- "delete_object": {"fn": lambda **kw: delete_object(kw["kind"], kw["name"]),
480
- "description": "DELETE a workspace object (view|dashboard|alert|report). DESTRUCTIVE — "
481
- "requires the user's explicit confirmation in this conversation first.",
482
- "parameters": _p({"kind": {"type": "string",
483
- "enum": ["view", "dashboard", "alert", "report"]},
484
- "name": {"type": "string"}}, ["kind", "name"])},
485
- }
486
-
487
-
488
- def openai_tools():
489
- """The registry in OpenAI function-calling format (OpenRouter-compatible)."""
490
- return [{"type": "function",
491
- "function": {"name": k, "description": v["description"], "parameters": v["parameters"]}}
492
- for k, v in TOOLS.items()]
493
-
494
-
495
- def dispatch(name, arguments):
496
- """Uniform tool execution for the Analyst loop: JSON-safe result or a readable error the
497
- model can act on. Never raises."""
498
- t = TOOLS.get(name)
499
- if not t:
500
- return _err(f"unknown tool {name!r} (tools: {list(TOOLS)})")
501
- try:
502
- args = json.loads(arguments) if isinstance(arguments, str) else dict(arguments or {})
503
- missing = [r for r in t["parameters"].get("required", []) if r not in args]
504
- if missing:
505
- return _err(f"missing required arguments: {missing}")
506
- return _ok(t["fn"](**args))
507
- except SEM.ModelError as e:
508
- return _err(e)
509
- except Exception as e:
510
- return _err(f"{type(e).__name__}: {e}")
 
1
+ """harness/tools.py — the compounding TOOL REGISTRY (OM-4 spine, 2026-07-11).
2
+
3
+ The formal tool surface the (small) model calls — the productization directive's "tools we keep
4
+ compounding". Every tool wraps the SEMANTIC layer (harness/semantic.py): the model navigates by
5
+ registry keys and recipe plans (model/skills/*.skill.yml — recipes and these tools version
6
+ TOGETHER), never by SQL. Adding a connector/topic extends what the SAME tools reach — that is the
7
+ compounding. Exported in OpenAI function-calling format (`openai_tools()`) — OpenRouter-compatible,
8
+ so any cheap model with tool-calling drives the platform.
9
+
10
+ Correctness posture (Part VI of the plan): whitelisted keys only; values parameterized downstream;
11
+ every query result carries a result_id + drill note; artifact tools (save/compose/schedule/alert)
12
+ are explicit and confirm-gated by recipe. Errors return a uniform envelope the model can read.
13
+ """
14
+ import json
15
+ import time
16
+ import uuid
17
+ from pathlib import Path
18
+
19
+ import harness.semantic as SEM
20
+
21
+ VIEWS_PATH = Path(__file__).resolve().parents[1] / "data" / "store" / "views.json"
22
+
23
+ _RESULTS = {} # result_id -> query result (session-scoped working memory for chart tools)
24
+ _RESULTS_CAP = 40
25
+
26
+ # The exhaustive chart vocabulary (2026-07-16): every Zelazny comparison form has a kind, so the
27
+ # Analyst never lacks a shape. The model picks by the CHART PICKER guide (analyst.py) + the
28
+ # charting skill recipes; the platform owns every pixel (app._render_analyst_artifact).
29
+ CHART_KINDS = (
30
+ "line", "bar", "area", "scatter", "kpi", "map", # the original six
31
+ "pie", "donut", # part-to-whole (≤6 slices)
32
+ "stacked_bar", "grouped_bar", "ranked_bar", "stacked_pct", # composition / rank forms
33
+ "combo", "yoy_bars", # level+rate; this-vs-last-year
34
+ "waterfall", "pareto", "histogram", "heatmap", "treemap", # bridge / concentration / distribution
35
+ "funnel", "bullet", "bubble", "sparkline", # stages / target / 3-measure / mini
36
+ )
37
+
38
+ # Per-kind param contract (beyond x): what else the spec must carry to be renderable.
39
+ _KIND_NEEDS = {
40
+ "combo": ("y", "y2"), "bullet": ("y", "y2"), "bubble": ("y", "size"),
41
+ "heatmap": ("y", "value"), "histogram": (), # histogram bins x itself
42
+ "stacked_bar": ("y", "series"), "grouped_bar": ("y", "series"),
43
+ "stacked_pct": ("y", "series"),
44
+ }
45
+ _QUERY_KEYS = ("topic", "measures", "group_by", "grain", "date_from", "date_to",
46
+ "team_id", "filters", "sort", "limit", "exclude_services")
47
+
48
+
49
+ def _remember(res):
50
+ rid = uuid.uuid4().hex[:10]
51
+ _RESULTS[rid] = res
52
+ while len(_RESULTS) > _RESULTS_CAP:
53
+ _RESULTS.pop(next(iter(_RESULTS)))
54
+ return rid
55
+
56
+
57
+ def _ok(data):
58
+ return {"ok": True, "data": data}
59
+
60
+
61
+ def _err(msg):
62
+ return {"ok": False, "error": str(msg)[:400]}
63
+
64
+
65
+ # ------------------------------------------------------------------ schema tools
66
+
67
+ def list_topics():
68
+ """The 'what data exists' tool."""
69
+ out = []
70
+ for k, t in SEM.topics().items():
71
+ out.append({"topic": k, "label": t.get("label"), "entity": t.get("entity"),
72
+ "grain": t.get("grain"),
73
+ "dims": list((t.get("store") or {}).get("dims") or {}),
74
+ "metrics": [m for m, d in SEM.metrics().items() if d["topic"] == k]})
75
+ return out
76
+
77
+
78
+ def describe_topic(topic):
79
+ """The schema-learning tool: scope, grain, dims, metrics w/ definitions, and ai_context."""
80
+ t = SEM.topics().get(topic)
81
+ if not t:
82
+ raise SEM.ModelError(f"unknown topic {topic!r} (use list_topics)")
83
+ mets = {k: {"label": m.get("label"), "description": m.get("description"),
84
+ "format": m.get("format"), "ai_context": m.get("ai_context")}
85
+ for k, m in SEM.metrics().items() if m["topic"] == topic}
86
+ return {"topic": topic, "label": t.get("label"), "scope": t.get("scope"),
87
+ "grain": t.get("grain"), "ai_context": t.get("ai_context"),
88
+ "dims": {k: v.get("label") for k, v in ((t.get("store") or {}).get("dims") or {}).items()},
89
+ "metrics": mets}
90
+
91
+
92
+ # ------------------------------------------------------------------ query tools
93
+
94
+ def run_semantic_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
95
+ team_id=None, filters=None, sort=None, limit=1000, exclude_services=False):
96
+ # limit defaults HIGH (1000): transforms/charts operate on the FULL result while the model
97
+ # only ever sees rows[:100] — a small default silently truncated per-customer analytics
98
+ # (the BELLA FLORIST wrong-decliner incident, 2026-07-17).
99
+ res = SEM.store_query(topic, measures, group_by=group_by, grain=grain, date_from=date_from,
100
+ date_to=date_to, team_id=team_id, filters=filters, sort=sort,
101
+ limit=limit, exclude_services=exclude_services)
102
+ # Echo the full query onto the result: chart specs built from it carry the query, so a SAVED
103
+ # view is a re-runnable QUERY (the OM-3 viewer re-executes it live), never a stale snapshot.
104
+ res["query"] = {"topic": topic, "measures": list(measures or []), "group_by": group_by,
105
+ "grain": grain, "date_from": date_from, "date_to": date_to, "team_id": team_id,
106
+ "filters": filters, "sort": sort, "limit": limit,
107
+ "exclude_services": exclude_services}
108
+ rid = _remember(res)
109
+ # The EFFECTIVE window, stated by the platform — the model must repeat this, never guess
110
+ # (a query without dates covers all recorded history; there is no hidden default window).
111
+ if date_from and date_to:
112
+ window = f"{date_from} to {date_to}"
113
+ elif date_from or date_to:
114
+ window = f"{'from ' + date_from if date_from else 'through ' + date_to}"
115
+ else:
116
+ window = "ALL recorded history (no date filter was applied)"
117
+ out = {"result_id": rid, "rows": res["rows"][:100], "row_count": res["row_count"],
118
+ "measures": res["measures"], "group_by": res["group_by"], "grain": res["grain"],
119
+ "window": window,
120
+ "note": "every number here is drillable; cite result_id when charting"}
121
+ if res["row_count"] >= (limit or 1000): # surface every truncation (plan hard line)
122
+ out["warning"] = (f"TRUNCATED: the result hit limit={limit} — the full set is larger. "
123
+ "Re-run with a higher limit (max 5000) BEFORE ranking, comparing or "
124
+ "aggregating, or your answer will be computed on a partial set.")
125
+ return out
126
+
127
+
128
+ def get_field_values(topic, dim, search=None):
129
+ return SEM.store_field_values(topic, dim, search=search)
130
+
131
+
132
+ # ------------------------------------------------------------------ transform tool (governed)
133
+
134
+ def transform_result(result_id, transforms):
135
+ """Apply governed ANALYTICS TRANSFORMS to a query result -> a NEW result_id to chart/table.
136
+ The chain is recorded on the derived result, so saved views replay query -> transforms live."""
137
+ res = _RESULTS.get(result_id)
138
+ if not res:
139
+ raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first")
140
+ import harness.transforms as TR
141
+ rows, applied = TR.apply(res, transforms, run_query=_run_query)
142
+ new = {**res, "rows": rows, "row_count": len(rows),
143
+ "transforms": (res.get("transforms") or []) + applied}
144
+ rid = _remember(new)
145
+ return {"result_id": rid, "rows": rows[:100], "row_count": len(rows),
146
+ "columns": sorted(rows[0]) if rows else [],
147
+ "note": "derived result — chart THIS result_id to show the transform"}
148
+
149
+
150
+ def _run_query(q):
151
+ return SEM.store_query(**{k: q.get(k) for k in _QUERY_KEYS if q.get(k) is not None})
152
+
153
+
154
+ # ------------------------------------------------------------------ viz tools (emit OUR specs)
155
+
156
+ def make_chart(result_id, kind, x, y=None, title=None, series=None, y2=None, size=None,
157
+ value=None, facet=None):
158
+ """Returns a validated CHART SPEC the platform renders with its own primitives (design system
159
+ enforced — the model never emits HTML/vega). Extra encodings per kind: combo/bullet need y2
160
+ (line/target), bubble needs size, heatmap needs value (the colour measure); facet (a dim
161
+ column) turns line|bar|area|scatter into small multiples."""
162
+ res = _RESULTS.get(result_id)
163
+ if not res:
164
+ raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first")
165
+ if kind not in CHART_KINDS:
166
+ raise SEM.ModelError(f"kind must be one of {CHART_KINDS}")
167
+ cols = set(res["rows"][0]) if res["rows"] else set()
168
+ if y is None and kind != "histogram":
169
+ raise SEM.ModelError(f"kind={kind!r} needs y (only histogram bins x by itself)")
170
+ for ref, nm in ((x, "x"), (y, "y"), (series, "series"), (y2, "y2"), (size, "size"),
171
+ (value, "value"), (facet, "facet")):
172
+ if ref and ref not in cols:
173
+ raise SEM.ModelError(f"{nm}={ref!r} not in result columns {sorted(cols)}")
174
+ given = {"y": y, "y2": y2, "size": size, "value": value, "series": series}
175
+ missing = [p for p in _KIND_NEEDS.get(kind, ()) if not given.get(p)]
176
+ if missing:
177
+ raise SEM.ModelError(f"kind={kind!r} also needs {missing} "
178
+ f"(pick from result columns {sorted(cols)})")
179
+ if facet and kind not in ("line", "bar", "area", "scatter"):
180
+ raise SEM.ModelError("facet (small multiples) works with line|bar|area|scatter only")
181
+ if kind == "yoy_bars" and f"{y}_ly" not in cols:
182
+ raise SEM.ModelError(f"yoy_bars needs a {y}_ly column — run transform_result "
183
+ "[{'op':'yoy'}] on the result first")
184
+ spec = {"kind": kind, "x": x, "y": y, "series": series,
185
+ "title": title or f"{y or x} by {x}", "result_id": result_id,
186
+ "query": res.get("query"), "rows": res["rows"]}
187
+ for k, v in (("y2", y2), ("size", size), ("value", value), ("facet", facet),
188
+ ("transforms", res.get("transforms"))):
189
+ if v:
190
+ spec[k] = v
191
+ out = {"chart": spec}
192
+ if kind in ("pie", "donut") and len(res["rows"]) > 6:
193
+ out["note"] = (f"{len(res['rows'])} slices — the platform will show the top 5 plus an "
194
+ "'Other' bucket; for a cleaner story run transform_result top_n first")
195
+ return out
196
+
197
+
198
+ def make_table(result_id, columns=None, title=None):
199
+ """First-class TABLE artifact: the exact rows, house-formatted (sortable, totals row, the
200
+ drill IS the table). columns (optional) picks and orders a subset."""
201
+ res = _RESULTS.get(result_id)
202
+ if not res:
203
+ raise SEM.ModelError(f"unknown result_id {result_id!r} — run run_semantic_query first")
204
+ rows = res["rows"]
205
+ if columns:
206
+ cols = set(rows[0]) if rows else set()
207
+ bad = [c for c in columns if c not in cols]
208
+ if bad:
209
+ raise SEM.ModelError(f"columns {bad} not in result columns {sorted(cols)}")
210
+ rows = [{c: r.get(c) for c in columns} for r in rows]
211
+ return {"table": {"kind": "table", "title": title, "columns": columns,
212
+ "result_id": result_id, "query": res.get("query"),
213
+ "transforms": res.get("transforms"), "rows": rows}}
214
+
215
+
216
+ def make_kpi_card(result_id, metric, compare_result_id=None):
217
+ res = _RESULTS.get(result_id)
218
+ if not res or not res["rows"]:
219
+ raise SEM.ModelError("result_id missing/empty — run a scalar run_semantic_query first")
220
+ val = res["rows"][0].get(metric)
221
+ if val is None:
222
+ raise SEM.ModelError(f"{metric!r} not in result")
223
+ card = {"kpi": {"metric": metric, "value": val, "result_id": result_id,
224
+ "query": res.get("query")}}
225
+ if compare_result_id and _RESULTS.get(compare_result_id, {}).get("rows"):
226
+ prev = _RESULTS[compare_result_id]["rows"][0].get(metric)
227
+ if prev:
228
+ card["kpi"]["delta_pct"] = (val - prev) / abs(prev)
229
+ card["kpi"]["compare_result_id"] = compare_result_id
230
+ card["kpi"]["compare_query"] = _RESULTS[compare_result_id].get("query")
231
+ return card
232
+
233
+
234
+ # ------------------------------------------------------------------ artifact tools (v0: local)
235
+
236
+ def _load_views():
237
+ if VIEWS_PATH.exists():
238
+ return json.loads(VIEWS_PATH.read_text(encoding="utf-8"))
239
+ return {"views": {}, "dashboards": {}}
240
+
241
+
242
+ def _save_views(d):
243
+ VIEWS_PATH.parent.mkdir(parents=True, exist_ok=True)
244
+ VIEWS_PATH.write_text(json.dumps(d, indent=1), encoding="utf-8")
245
+
246
+
247
+ def save_view(name, chart):
248
+ """Persist a chart/KPI spec (from make_chart / make_kpi_card) as a named view. Specs persist
249
+ WITH their semantic query and WITHOUT rows — the OM-3 viewer re-executes the query live, so a
250
+ saved view is always current, never a snapshot."""
251
+ d = _load_views()
252
+ spec = chart.get("chart") or chart.get("kpi") or chart.get("table") or chart
253
+ spec = {k: v for k, v in spec.items() if k != "rows"}
254
+ if "kpi" in chart and not spec.get("kind"):
255
+ spec["kind"] = "kpi"
256
+ if "table" in chart and not spec.get("kind"):
257
+ spec["kind"] = "table"
258
+ if not spec.get("query"):
259
+ raise SEM.ModelError("spec carries no query — pass the exact object returned by "
260
+ "make_chart / make_kpi_card (from a fresh run_semantic_query)")
261
+ d["views"][name] = {"chart": spec, "saved_at": time.strftime("%Y-%m-%d %H:%M")}
262
+ _save_views(d)
263
+ return {"saved": name, "views": list(d["views"])}
264
+
265
+
266
+ def compose_dashboard(name, views):
267
+ """'Spawn a dashboard': compose saved views into a named dashboard spec (rendered at OM-3)."""
268
+ d = _load_views()
269
+ missing = [v for v in views if v not in d["views"]]
270
+ if missing:
271
+ raise SEM.ModelError(f"unknown views {missing} — save_view them first")
272
+ d["dashboards"][name] = {"views": views, "created_at": time.strftime("%Y-%m-%d %H:%M")}
273
+ _save_views(d)
274
+ return {"dashboard": name, "views": views}
275
+
276
+
277
+ # ------------------------------------------------------------------ the GAP LOOP (rung 4)
278
+
279
+ GAP_KINDS = ("dimension", "metric", "transform", "chart_kind", "data_source", "other")
280
+ GAPS_KEY = "analyst_gaps"
281
+ GAPS_CAP = 500
282
+
283
+
284
+ def report_gap(kind, missing, question, workaround=None):
285
+ """Log a CAPABILITY GAP: the model determined (after checking the schema) that no registered
286
+ dim/metric/transform/kind can answer. The entry lands in telemetry AND the durable store —
287
+ the admin Gaps view aggregates them into the platform build backlog. This is how every
288
+ honest 'I can't' becomes the next dim, transform, or recipe."""
289
+ if kind not in GAP_KINDS:
290
+ raise SEM.ModelError(f"kind must be one of {GAP_KINDS}")
291
+ entry = {"ts": time.strftime("%Y-%m-%d %H:%M:%S"), "kind": kind,
292
+ "missing": str(missing)[:120], "question": str(question)[:300],
293
+ "workaround": (str(workaround)[:200] if workaround else None)}
294
+ import harness.telemetry as TEL
295
+ TEL.log("analyst_gap", **{("gap_kind" if k == "kind" else k): v for k, v in entry.items()})
296
+ try: # durable copy — async, the chatlog write posture
297
+ import threading
298
+
299
+ import core.store as store
300
+ if store.available():
301
+ def _fn(data):
302
+ data = list(data or [])
303
+ data.append(entry)
304
+ return data[-GAPS_CAP:]
305
+ threading.Thread(target=lambda: store.update(GAPS_KEY, _fn),
306
+ daemon=True, name="analyst-gap").start()
307
+ except Exception:
308
+ pass
309
+ return {"logged": True,
310
+ "note": "gap recorded for the platform backlog — now tell the user in ONE sentence "
311
+ "what is missing and offer the nearest ask that IS answerable"}
312
+
313
+
314
+ # ------------------------------------------------------------------ workspace tools (OM-3/P11)
315
+
316
+ def list_workspace():
317
+ """Everything in the tenant workspace (uniform modular objects) + available templates."""
318
+ import harness.workspace as W
319
+ return {"objects": W.items(), "templates": W.templates()}
320
+
321
+
322
+ def instantiate_template(filename, new_name=None):
323
+ """Stamp a tenant-agnostic template into this workspace as NEW objects (never overwrites)."""
324
+ import harness.workspace as W
325
+ return W.instantiate_template(filename, new_name)
326
+
327
+
328
+ def update_view(name, changes):
329
+ """Patch an existing view (spec and/or query) — validated by re-execution before saving."""
330
+ import harness.views as V
331
+ return V.update_view(name, changes)
332
+
333
+
334
+ def update_workbook(name, views=None, new_name=None):
335
+ """Recompose and/or rename a workbook (dashboard); renames follow into schedules."""
336
+ import harness.views as V
337
+ return V.update_dashboard(name, views_list=views, new_name=new_name)
338
+
339
+
340
+ def delete_object(kind, name):
341
+ """Delete any workspace object. DESTRUCTIVE — the recipe requires explicit confirmation."""
342
+ import harness.workspace as W
343
+ W.delete(kind, name)
344
+ return {"deleted": f"{kind} · {name}"}
345
+
346
+
347
+ # ------------------------------------------------------------------ registry + dispatch
348
+
349
+ def _p(props, required):
350
+ return {"type": "object", "properties": props, "required": required}
351
+
352
+
353
+ TOOLS = {
354
+ "list_topics": {"fn": lambda **kw: list_topics(),
355
+ "description": "List the datasets (topics) available: their metrics, dims, and grain. Start here.",
356
+ "parameters": _p({}, [])},
357
+ "describe_topic": {"fn": lambda **kw: describe_topic(kw["topic"]),
358
+ "description": "Full schema of one topic: scope rules, metric definitions, dims, and the business context you must respect.",
359
+ "parameters": _p({"topic": {"type": "string"}}, ["topic"])},
360
+ "get_field_values": {"fn": lambda **kw: get_field_values(kw["topic"], kw["dim"], kw.get("search")),
361
+ "description": "Resolve real filter values (ids+names) for a dim. ALWAYS use before filtering by a typed name.",
362
+ "parameters": _p({"topic": {"type": "string"}, "dim": {"type": "string"},
363
+ "search": {"type": "string"}}, ["topic", "dim"])},
364
+ "run_semantic_query": {"fn": lambda **kw: run_semantic_query(**kw),
365
+ "description": "Run a governed query: registered measures over a topic, optional group_by dims / time grain / filters. The ONLY way to read data.",
366
+ "parameters": _p({"topic": {"type": "string"},
367
+ "measures": {"type": "array", "items": {"type": "string"}},
368
+ "group_by": {"type": "array", "items": {"type": "string"}},
369
+ "grain": {"type": "string", "enum": ["month", "week", "day"]},
370
+ "date_from": {"type": "string"}, "date_to": {"type": "string"},
371
+ "team_id": {"type": "integer"},
372
+ "filters": {"type": "object"},
373
+ "sort": {"type": "string"}, "limit": {"type": "integer"},
374
+ "exclude_services": {"type": "boolean"}},
375
+ ["topic", "measures"])},
376
+ "transform_result": {"fn": lambda **kw: transform_result(kw["result_id"], kw["transforms"]),
377
+ "description": "Apply governed analytics transforms to a result -> a NEW result_id (a "
378
+ "CHAINABLE list of {op, ...}). The Tableau-class analytics library — pick "
379
+ "op names (full catalog + recipes in the analytics skill). Families: "
380
+ "ordering/rank (sort, head, bottom_n, rank, rank_pct, ntile, top_n, "
381
+ "add_total) · part-to-whole (share_of_total, cum_share) · running/moving "
382
+ "(running_total/avg/max/min, running_count, moving_average/sum/median, "
383
+ "rolling_std) · period-over-period (diff, pct_change, lag, lead, "
384
+ "diff_from_first, index_to_100, percent_of_max, compare) · distribution/"
385
+ "stats (bin, describe, zscore, outliers, winsorize, clip, normalize, "
386
+ "correlate, weighted_average, safe_ratio, product) · business (abc_classify, "
387
+ "concentration, contribution_to_change, rfm, funnel_rates) · modeling "
388
+ "(trend_line, regression, cagr, growth_rate) · reference lines as columns "
389
+ "(reference_line, reference_band, target_line, xmr_limits) · reshape (pivot, "
390
+ "unpivot, filter_rows, dedupe, resample) · re-query windows (yoy, ytd, "
391
+ "rolling, forecast). ADDITIVITY LAW: accumulating ops "
392
+ "(running_total/share_of_total/cum_share/moving_sum/abc_classify/"
393
+ "concentration) work on ADDITIVE measures (revenue/units/margin/orders); "
394
+ "for a cumulative/trailing DISTINCT count (customers) or ratio use ytd/"
395
+ "rolling (they re-query) — never running_total. NEVER compute any of these "
396
+ "yourself; transform, then chart/table the new result_id.",
397
+ "parameters": _p({"result_id": {"type": "string"},
398
+ "transforms": {"type": "array", "items": {"type": "object"}}},
399
+ ["result_id", "transforms"])},
400
+ "make_chart": {"fn": lambda **kw: make_chart(**kw),
401
+ "description": "Turn a query result into a platform chart. kinds: line|bar|area|scatter|"
402
+ "map|pie|donut|stacked_bar|grouped_bar|ranked_bar|stacked_pct|combo|"
403
+ "yoy_bars|waterfall|pareto|histogram|heatmap|treemap|funnel|bullet|bubble|"
404
+ "sparkline. x/y/series must be result columns. Extra encodings: combo "
405
+ "(bars y + line y2, dual axis), bullet (value y vs target y2), bubble "
406
+ "(scatter + size), heatmap (dims x,y + colour value), histogram (bins x, "
407
+ "no y), facet (a dim column -> small multiples of line|bar|area|scatter). "
408
+ "yoy_bars needs the yoy transform first. kind='map' plots customers "
409
+ "geographically: x = the customer dim, dot size = y.",
410
+ "parameters": _p({"result_id": {"type": "string"}, "kind": {"type": "string", "enum": list(CHART_KINDS)},
411
+ "x": {"type": "string"}, "y": {"type": "string"},
412
+ "title": {"type": "string"}, "series": {"type": "string"},
413
+ "y2": {"type": "string"}, "size": {"type": "string"},
414
+ "value": {"type": "string"}, "facet": {"type": "string"}},
415
+ ["result_id", "kind", "x"])},
416
+ "make_table": {"fn": lambda **kw: make_table(kw["result_id"], kw.get("columns"),
417
+ kw.get("title")),
418
+ "description": "Turn a query result into a first-class TABLE artifact (house-formatted, "
419
+ "totals row, drillable). Use when the user wants exact figures, many "
420
+ "columns, or a list — not a shape. columns (optional) picks and orders.",
421
+ "parameters": _p({"result_id": {"type": "string"},
422
+ "columns": {"type": "array", "items": {"type": "string"}},
423
+ "title": {"type": "string"}}, ["result_id"])},
424
+ "make_kpi_card": {"fn": lambda **kw: make_kpi_card(**kw),
425
+ "description": "Turn a scalar query result into a KPI card; optional compare_result_id adds a YoY delta.",
426
+ "parameters": _p({"result_id": {"type": "string"}, "metric": {"type": "string"},
427
+ "compare_result_id": {"type": "string"}}, ["result_id", "metric"])},
428
+ "report_gap": {"fn": lambda **kw: report_gap(kw["kind"], kw["missing"], kw["question"],
429
+ kw.get("workaround")),
430
+ "description": "LAST RESORT — log a capability gap. Call ONLY after list_topics/"
431
+ "describe_topic confirm that NO registered dimension, metric, transform "
432
+ "or chart kind can answer the user's question (e.g. stock on hand, "
433
+ "which has no topic). NOT a gap: YoY/decline/growth compares "
434
+ "(transform_result yoy), rankings/top-N, shares, running totals, "
435
+ "distributions — those are ANSWERABLE via transform_result. Then tell "
436
+ "the user plainly what is missing and offer the nearest answerable ask. "
437
+ "NEVER call this for something the tools support, and NEVER guess "
438
+ "instead of calling it.",
439
+ "parameters": _p({"kind": {"type": "string", "enum": list(GAP_KINDS)},
440
+ "missing": {"type": "string",
441
+ "description": "what does not exist, short (e.g. 'inventory/stock-on-hand topic')"},
442
+ "question": {"type": "string",
443
+ "description": "the user's question, verbatim"},
444
+ "workaround": {"type": "string",
445
+ "description": "the nearest answerable alternative you offered"}},
446
+ ["kind", "missing", "question"])},
447
+ "save_view": {"fn": lambda **kw: save_view(kw["name"], kw["chart"]),
448
+ "description": "Save a chart as a named view (confirm with the user first).",
449
+ "parameters": _p({"name": {"type": "string"}, "chart": {"type": "object"}}, ["name", "chart"])},
450
+ "compose_dashboard": {"fn": lambda **kw: compose_dashboard(kw["name"], kw["views"]),
451
+ "description": "Compose saved views into a named dashboard (confirm with the user first).",
452
+ "parameters": _p({"name": {"type": "string"},
453
+ "views": {"type": "array", "items": {"type": "string"}}}, ["name", "views"])},
454
+ "list_workspace": {"fn": lambda **kw: list_workspace(),
455
+ "description": "List the tenant workspace: every saved view/dashboard/alert/report "
456
+ "(modular objects) plus available templates. Use when the user asks what "
457
+ "exists, wants to reuse/manage artifacts, or before composing.",
458
+ "parameters": _p({}, [])},
459
+ "instantiate_template": {"fn": lambda **kw: instantiate_template(kw["filename"],
460
+ kw.get("new_name")),
461
+ "description": "Stamp a tenant-agnostic template (from list_workspace) into the "
462
+ "workspace as NEW objects — never overwrites. Confirm with the user first.",
463
+ "parameters": _p({"filename": {"type": "string"}, "new_name": {"type": "string"}},
464
+ ["filename"])},
465
+ "update_view": {"fn": lambda **kw: update_view(kw["name"], kw.get("changes")),
466
+ "description": "UPDATE an existing saved view: patch spec keys (title/kind/x/y/series) "
467
+ "and/or 'query' subkeys (measures, group_by, grain, date_from, date_to, "
468
+ "team_id, filters, sort, limit; null REMOVES a key). The patched query is "
469
+ "re-executed before saving — invalid updates are rejected. Confirm first.",
470
+ "parameters": _p({"name": {"type": "string"}, "changes": {"type": "object"}},
471
+ ["name", "changes"])},
472
+ "update_workbook": {"fn": lambda **kw: update_workbook(kw["name"], kw.get("views"),
473
+ kw.get("new_name")),
474
+ "description": "UPDATE a workbook (dashboard): recompose its views (list = new display "
475
+ "order; add/remove by including/omitting) and/or rename it. Confirm first.",
476
+ "parameters": _p({"name": {"type": "string"},
477
+ "views": {"type": "array", "items": {"type": "string"}},
478
+ "new_name": {"type": "string"}}, ["name"])},
479
+ "delete_object": {"fn": lambda **kw: delete_object(kw["kind"], kw["name"]),
480
+ "description": "DELETE a workspace object (view|dashboard|alert|report). DESTRUCTIVE — "
481
+ "requires the user's explicit confirmation in this conversation first.",
482
+ "parameters": _p({"kind": {"type": "string",
483
+ "enum": ["view", "dashboard", "alert", "report"]},
484
+ "name": {"type": "string"}}, ["kind", "name"])},
485
+ }
486
+
487
+
488
+ def openai_tools():
489
+ """The registry in OpenAI function-calling format (OpenRouter-compatible)."""
490
+ return [{"type": "function",
491
+ "function": {"name": k, "description": v["description"], "parameters": v["parameters"]}}
492
+ for k, v in TOOLS.items()]
493
+
494
+
495
+ def dispatch(name, arguments):
496
+ """Uniform tool execution for the Analyst loop: JSON-safe result or a readable error the
497
+ model can act on. Never raises."""
498
+ t = TOOLS.get(name)
499
+ if not t:
500
+ return _err(f"unknown tool {name!r} (tools: {list(TOOLS)})")
501
+ try:
502
+ args = json.loads(arguments) if isinstance(arguments, str) else dict(arguments or {})
503
+ missing = [r for r in t["parameters"].get("required", []) if r not in args]
504
+ if missing:
505
+ return _err(f"missing required arguments: {missing}")
506
+ return _ok(t["fn"](**args))
507
+ except SEM.ModelError as e:
508
+ return _err(e)
509
+ except Exception as e:
510
+ return _err(f"{type(e).__name__}: {e}")
platform/harness/windows.py CHANGED
@@ -1,390 +1,390 @@
1
- """harness/windows.py — the DATE WINDOW vocabulary (CG-7, owner item 2, 2026-07-26).
2
-
3
- A filter condition on a measure reads:
4
-
5
- Where [Sales] [in the last 90 days] [>] [5,000]
6
-
7
- The middle bracket is this module. It turns a window SPEC — a small JSON object a saved view can
8
- persist — into a concrete `(date_from, date_to)` pair of ISO dates.
9
-
10
- WHY IT IS ITS OWN MODULE, AND WHY IT TAKES `today` AS AN ARGUMENT
11
- -----------------------------------------------------------------------------------------------
12
- Two engines must agree on what "this quarter" means: this one (which compiles the SQL) and
13
- `customer-grid/windows.ts` (which renders the label the user reads and the bounds the client
14
- engine would use). If they disagree by one day, the grid shows a number the label denies, and
15
- nothing errors. `aios-web/verify_windows.py` holds them in lock-step over a fixed set of probe
16
- dates, exactly as verify_filter_engine.py does for the filter tree.
17
-
18
- `today` is a PARAMETER, never `date.today()` read inside. Three reasons, all learned:
19
- - a gate cannot compare two engines on "now" — the two runs are milliseconds apart and land on
20
- different sides of midnight roughly once every few thousand runs, which is the worst kind of
21
- flake because it looks like a real divergence
22
- - a window resolved during a request must not shift between the count query and the row query
23
- - the tenant's day boundary is the tenant's, not the server's
24
-
25
- DESIGN NOTES THAT ARE EASY TO GET WRONG
26
- -----------------------------------------------------------------------------------------------
27
- - Bounds are INCLUSIVE at both ends and are DATES, not timestamps. `store_query` casts both
28
- sides to TIMESTAMP and appends 23:59:59 to the upper bound, so a bare date is correct there;
29
- the client compares the 10-char ISO prefix, so a bare date is correct there too.
30
- - "Last N days" INCLUDES today. Airtable's "the past week" means the last 7 days up to and
31
- including now, not the 7 days before yesterday. Off-by-one here silently drops today's
32
- orders from every "recent" filter — invisible until someone asks why a sale they just
33
- entered is missing.
34
- - Week starts MONDAY (ISO 8601). Stated rather than defaulted: Python's weekday() is
35
- Monday=0 and JavaScript's getDay() is Sunday=0, so the two engines disagree unless one of
36
- them is explicitly corrected. That correction is the single most likely divergence in this
37
- file, and the gate probes a Sunday and a Monday for exactly that reason.
38
- - A `custom` window with only one bound is legal and means open-ended on the other side.
39
- - An UNRESOLVABLE window returns None rather than a default. A window that quietly becomes
40
- "all time" would widen a filter while still reporting an authoritative count.
41
- """
42
- import datetime as _dt
43
-
44
- #: Every window kind the UI may offer. Mirrors WINDOW_KINDS in customer-grid/windows.ts;
45
- #: verify_windows.py asserts the two lists are identical, because a kind the client can emit and
46
- #: the server cannot resolve is a filter that silently stops narrowing.
47
- WINDOW_KINDS = (
48
- "all_time",
49
- "today",
50
- "yesterday",
51
- "this_week",
52
- "last_week",
53
- "this_month",
54
- "last_month",
55
- "this_quarter",
56
- "last_quarter",
57
- "this_year",
58
- "last_year",
59
- "ytd",
60
- "ytd_last_year",
61
- "ltm",
62
- "past_week",
63
- "past_month",
64
- "past_year",
65
- "last_n_days",
66
- "next_n_days",
67
- "custom",
68
- )
69
-
70
- #: Kinds that carry an integer `n`.
71
- N_KINDS = frozenset({"last_n_days", "next_n_days"})
72
-
73
- #: How each kind reads in a sentence. The UI renders these; they are here so the label and the
74
- #: arithmetic cannot drift apart in the one place a user would never think to check.
75
- WINDOW_LABELS = {
76
- "all_time": "all time",
77
- "today": "today",
78
- "yesterday": "yesterday",
79
- "this_week": "this week",
80
- "last_week": "last week",
81
- "this_month": "this month",
82
- "last_month": "last month",
83
- "this_quarter": "this quarter",
84
- "last_quarter": "last quarter",
85
- "this_year": "this year",
86
- "last_year": "last year",
87
- "ytd": "year to date",
88
- # "last year to date" (LYTD), not "year to date, last year": the longer phrasing CLIPPED to
89
- # "year to date, last ye" in the 146px window select — and this vocabulary is CLOSED, so
90
- # the rule for it is that it stays readable rather than that it fits. Seen in a live
91
- # screenshot of the owner's own comparison; every assertion in that run was green.
92
- "ytd_last_year": "last year to date",
93
- "ltm": "the last 12 months",
94
- "past_week": "the past week",
95
- "past_month": "the past month",
96
- "past_year": "the past year",
97
- "last_n_days": "the last {n} days",
98
- "next_n_days": "the next {n} days",
99
- "custom": "a custom range",
100
- }
101
-
102
- MAX_N = 3650 # ten years; a bound, not a business rule
103
-
104
- # --- the DATE-VALUE anchors (owner item 3, 2026-07-26) ----------------------------------------
105
- # The second half of a date CONDITION, as distinct from a measure's window:
106
- #
107
- # Where [Last order] [is before] [one month ago]
108
- # ^ op ^ THIS
109
- #
110
- # A window answers "over what period do I sum"; an anchor answers "which single date am I
111
- # comparing against". Both live in this module for one reason: they are the same two-engine
112
- # contract, resolved from the same `today`, and a gate that holds one in lock-step and not the
113
- # other would leave half the sentence free to drift.
114
-
115
- #: Anchor modes, in the order the picker offers them. `exact` is the historical behaviour — a
116
- #: rule with NO mode is an `exact` rule whose value is an ISO date, which is what every view
117
- #: saved before this change carries.
118
- ANCHOR_MODES = (
119
- "today",
120
- "yesterday",
121
- "one_week_ago",
122
- "one_month_ago",
123
- "n_days_ago",
124
- "exact",
125
- )
126
-
127
- #: Modes that carry NO value. ⚠ This set is load-bearing far outside this module: a rule whose
128
- #: value is blank normally reads as INACTIVE, and an inactive rule is IGNORED — which WIDENS the
129
- #: result under a count nobody would doubt. Every activeness check (`isRuleActive` in TS,
130
- #: `filter_sql.is_rule_active`, and the validator's value handling) has to know these four are
131
- #: active with an empty value.
132
- ANCHOR_VALUE_FREE = frozenset({"today", "yesterday", "one_week_ago", "one_month_ago"})
133
-
134
- ANCHOR_LABELS = {
135
- "today": "today",
136
- "yesterday": "yesterday",
137
- "one_week_ago": "one week ago",
138
- "one_month_ago": "one month ago",
139
- "n_days_ago": "{n} days ago",
140
- "exact": "an exact date",
141
- }
142
-
143
-
144
- def _iso(d):
145
- return d.isoformat()
146
-
147
-
148
- def _month_start(d):
149
- return d.replace(day=1)
150
-
151
-
152
- def _month_end(d):
153
- return _next_month(d.replace(day=1)) - _dt.timedelta(days=1)
154
-
155
-
156
- def _next_month(d):
157
- return (d.replace(day=28) + _dt.timedelta(days=4)).replace(day=1)
158
-
159
-
160
- def _quarter_start(d):
161
- return _dt.date(d.year, 3 * ((d.month - 1) // 3) + 1, 1)
162
-
163
-
164
- def _days_in_month(y, m):
165
- return (_dt.date(y + (m == 12), 1 if m == 12 else m + 1, 1) - _dt.timedelta(days=1)).day
166
-
167
-
168
- def _shift_months(d, n):
169
- """`d` moved `n` months, CLAMPING the day to the target month's length.
170
-
171
- Jan 31 back one month is Dec 31, but Mar 31 back one month is Feb 28 (or 29) — there is no
172
- Feb 31 to land on. Both engines must clamp identically or "the past month" differs by up to
173
- three days for a third of the calendar; the gate probes a 31st and a leap day for exactly it.
174
- """
175
- total = (d.year * 12 + (d.month - 1)) + n
176
- y, m = divmod(total, 12)
177
- m += 1
178
- return _dt.date(y, m, min(d.day, _days_in_month(y, m)))
179
-
180
-
181
- def _parse_date(v):
182
- """Accept 'YYYY-MM-DD' (and tolerate a longer ISO timestamp by taking its date part)."""
183
- if isinstance(v, _dt.date):
184
- return v
185
- s = str(v or "").strip()[:10]
186
- if not s:
187
- return None
188
- try:
189
- return _dt.date.fromisoformat(s)
190
- except ValueError:
191
- return None
192
-
193
-
194
- def normalize(spec):
195
- """Coerce an untrusted window spec to `{kind, n?, from?, to?}` or None.
196
-
197
- Fail-closed on the KIND (an unknown kind is not a window), tolerant on the rest — the same
198
- split `clean_filter_tree` uses, so one malformed window cannot cost a user their saved view.
199
- """
200
- if not isinstance(spec, dict):
201
- return None
202
- kind = spec.get("kind")
203
- if kind not in WINDOW_KINDS:
204
- return None
205
- out = {"kind": kind}
206
- if kind in N_KINDS:
207
- try:
208
- n = int(spec.get("n"))
209
- except (TypeError, ValueError):
210
- return None
211
- if n < 1 or n > MAX_N:
212
- return None
213
- out["n"] = n
214
- if kind == "custom":
215
- f, t = _parse_date(spec.get("from")), _parse_date(spec.get("to"))
216
- if f is None and t is None:
217
- return None # a custom range with no bounds is not a window
218
- if f is not None and t is not None and f > t:
219
- f, t = t, f # the builder cannot enforce order; the engine can
220
- if f is not None:
221
- out["from"] = _iso(f)
222
- if t is not None:
223
- out["to"] = _iso(t)
224
- return out
225
-
226
-
227
- def resolve(spec, today):
228
- """Window spec + the tenant's today -> `(date_from, date_to)`, both inclusive ISO dates.
229
-
230
- Either side may be None, meaning open-ended. Returns None when the spec is not a window at
231
- all — callers MUST treat that as "this condition cannot be evaluated" and refuse, never as
232
- "no window", which would silently widen the result to all time.
233
- """
234
- spec = normalize(spec)
235
- if spec is None:
236
- return None
237
- kind = spec["kind"]
238
- d = _parse_date(today)
239
- if d is None:
240
- raise ValueError("resolve() needs an explicit `today` — see this module's docstring")
241
-
242
- if kind == "all_time":
243
- return (None, None)
244
- if kind == "today":
245
- return (_iso(d), _iso(d))
246
- if kind == "yesterday":
247
- y = d - _dt.timedelta(days=1)
248
- return (_iso(y), _iso(y))
249
-
250
- # ISO 8601: the week starts MONDAY. Python's weekday() is already Monday=0; the TS mirror
251
- # has to correct getDay(), which is Sunday=0. The gate probes both a Sunday and a Monday.
252
- if kind == "this_week":
253
- start = d - _dt.timedelta(days=d.weekday())
254
- return (_iso(start), _iso(start + _dt.timedelta(days=6)))
255
- if kind == "last_week":
256
- start = d - _dt.timedelta(days=d.weekday() + 7)
257
- return (_iso(start), _iso(start + _dt.timedelta(days=6)))
258
-
259
- if kind == "this_month":
260
- return (_iso(_month_start(d)), _iso(_month_end(d)))
261
- if kind == "last_month":
262
- prev = _month_start(d) - _dt.timedelta(days=1)
263
- return (_iso(_month_start(prev)), _iso(prev))
264
-
265
- if kind == "this_quarter":
266
- qs = _quarter_start(d)
267
- qe = _month_end(_dt.date(qs.year, qs.month + 2, 1))
268
- return (_iso(qs), _iso(qe))
269
- if kind == "last_quarter":
270
- prev_end = _quarter_start(d) - _dt.timedelta(days=1)
271
- qs = _quarter_start(prev_end)
272
- return (_iso(qs), _iso(prev_end))
273
-
274
- if kind == "this_year":
275
- return (_iso(_dt.date(d.year, 1, 1)), _iso(_dt.date(d.year, 12, 31)))
276
- if kind == "last_year":
277
- return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_dt.date(d.year - 1, 12, 31)))
278
- if kind == "ytd":
279
- # year-to-date ENDS TODAY, unlike this_year which runs to Dec 31. The distinction is the
280
- # whole reason both exist: comparing "this year" against last year double-counts the
281
- # months that have not happened yet.
282
- return (_iso(_dt.date(d.year, 1, 1)), _iso(d))
283
- if kind == "ytd_last_year":
284
- # SAME PERIOD last year — Jan 1 LY through today's month/day LY. This is the honest
285
- # partner of `ytd`, and it is not a nicety: it mirrors `core.periods.ytd_last_year`
286
- # EXACTLY (including the Feb 29 -> Feb 28 clamp), which is what `pool()` computes the
287
- # `revenue_ly` column from. `Sales[ytd] < Sales[ytd_last_year]` therefore reproduces the
288
- # retired `at_risk > 0` condition rather than approximating it. Comparing `ytd` against
289
- # `last_year` instead would pit seven months against twelve.
290
- return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_shift_months(d, -12)))
291
- if kind == "ltm":
292
- # the last twelve months INCLUDING today: 365 days back, both ends inclusive
293
- return (_iso(d - _dt.timedelta(days=364)), _iso(d))
294
-
295
- # "The past week/month/year" are ROLLING and end TODAY — Airtable's wording, and NOT the
296
- # calendar kinds above: `last_week` is the previous Monday-Sunday, `past_week` is the seven
297
- # days ending today. Both are offered because a user means different things by them, and
298
- # picking one to serve both would silently answer the other question.
299
- # past_week = the same rule as last_n_days(7) — one period back, PLUS ONE DAY, so today
300
- # is included and the span is exactly 7 days rather than 8.
301
- # past_year agrees with `ltm` except across a leap day, where the calendar shift keeps the
302
- # same month/day and the 364-day subtraction cannot.
303
- if kind == "past_week":
304
- return (_iso(d - _dt.timedelta(days=6)), _iso(d))
305
- if kind == "past_month":
306
- return (_iso(_shift_months(d, -1) + _dt.timedelta(days=1)), _iso(d))
307
- if kind == "past_year":
308
- return (_iso(_shift_months(d, -12) + _dt.timedelta(days=1)), _iso(d))
309
-
310
- if kind == "last_n_days":
311
- # INCLUSIVE of today — "the last 7 days" is today and the 6 before it, not the 7 before
312
- # yesterday. Getting this wrong drops today's orders from every recent-activity filter.
313
- return (_iso(d - _dt.timedelta(days=spec["n"] - 1)), _iso(d))
314
- if kind == "next_n_days":
315
- return (_iso(d), _iso(d + _dt.timedelta(days=spec["n"] - 1)))
316
-
317
- if kind == "custom":
318
- return (spec.get("from"), spec.get("to"))
319
- raise AssertionError(f"unhandled window kind {kind!r}") # unreachable; WINDOW_KINDS is closed
320
-
321
-
322
- def label(spec):
323
- """How a window reads in the condition sentence ('in the last 90 days')."""
324
- spec = normalize(spec)
325
- if spec is None:
326
- return "an invalid range"
327
- return WINDOW_LABELS[spec["kind"]].format(n=spec.get("n"))
328
-
329
-
330
- def resolve_anchor(mode, value, today):
331
- """A date condition's right-hand side -> ONE inclusive ISO date, or None.
332
-
333
- `mode` is an entry of ANCHOR_MODES; None/'' means `exact`, which is what every view saved
334
- before anchors existed carries (op + an ISO date in `value`). `value` supplies the number for
335
- `n_days_ago` and the date for `exact`, and is ignored by the four value-free modes.
336
-
337
- Returns None when the anchor cannot be resolved — an unknown mode, a non-numeric N, a value
338
- that is not a date. The caller must then treat the CONDITION as unanswerable and match
339
- nothing. It must NOT fall through to "no condition": that widens the result while the count
340
- beside it still looks authoritative, which is the whole reason this returns None rather than
341
- a best guess (same contract as `resolve()` above).
342
-
343
- `n_days_ago` accepts n = 0 where a WINDOW requires n >= 1. A zero-day window is empty and
344
- could only be a mistake; "0 days ago" is today, which is a date a person can mean.
345
- """
346
- m = mode or "exact"
347
- if m not in ANCHOR_MODES:
348
- return None
349
- d = _parse_date(today)
350
- if d is None:
351
- raise ValueError("resolve_anchor() needs an explicit `today` — see this module's "
352
- "docstring on why the clock is never read here")
353
- if m == "today":
354
- return _iso(d)
355
- if m == "yesterday":
356
- return _iso(d - _dt.timedelta(days=1))
357
- if m == "one_week_ago":
358
- # A week ago is a DATE (today minus 7), not the past-week RANGE. The two read almost
359
- # identically in English and mean different things in a comparison.
360
- return _iso(d - _dt.timedelta(days=7))
361
- if m == "one_month_ago":
362
- return _iso(_shift_months(d, -1))
363
- if m == "n_days_ago":
364
- s = str("" if value is None else value).strip()
365
- # `[0-9]` not `\d`: a Python \d matches every Unicode decimal digit, so a fullwidth
366
- # '30' would resolve here and be rejected by the TS mirror's Number() — the exact
367
- # two-engine divergence filter_sql.to_num was rewritten to avoid.
368
- if not s or not all("0" <= ch <= "9" for ch in s):
369
- return None
370
- n = int(s)
371
- # No `n < 0` guard: the digit test above already excludes a sign, so it would be
372
- # unreachable. The negative control proved that — removing it changed nothing, which is
373
- # how dead code hides in a defensive-looking line. The CAP is the live guard.
374
- if n > MAX_N:
375
- return None
376
- return _iso(d - _dt.timedelta(days=n))
377
- parsed = _parse_date(value) # exact
378
- return None if parsed is None else _iso(parsed)
379
-
380
-
381
- def anchor_label(mode, value):
382
- """How an anchor reads in the condition sentence ('one month ago', '30 days ago')."""
383
- m = mode or "exact"
384
- if m not in ANCHOR_MODES:
385
- return "an invalid date"
386
- if m == "n_days_ago":
387
- return ANCHOR_LABELS[m].format(n=str("" if value is None else value).strip() or "N")
388
- if m == "exact":
389
- return str("" if value is None else value).strip() or ANCHOR_LABELS[m]
390
- return ANCHOR_LABELS[m]
 
1
+ """harness/windows.py — the DATE WINDOW vocabulary (CG-7, owner item 2, 2026-07-26).
2
+
3
+ A filter condition on a measure reads:
4
+
5
+ Where [Sales] [in the last 90 days] [>] [5,000]
6
+
7
+ The middle bracket is this module. It turns a window SPEC — a small JSON object a saved view can
8
+ persist — into a concrete `(date_from, date_to)` pair of ISO dates.
9
+
10
+ WHY IT IS ITS OWN MODULE, AND WHY IT TAKES `today` AS AN ARGUMENT
11
+ -----------------------------------------------------------------------------------------------
12
+ Two engines must agree on what "this quarter" means: this one (which compiles the SQL) and
13
+ `customer-grid/windows.ts` (which renders the label the user reads and the bounds the client
14
+ engine would use). If they disagree by one day, the grid shows a number the label denies, and
15
+ nothing errors. `aios-web/verify_windows.py` holds them in lock-step over a fixed set of probe
16
+ dates, exactly as verify_filter_engine.py does for the filter tree.
17
+
18
+ `today` is a PARAMETER, never `date.today()` read inside. Three reasons, all learned:
19
+ - a gate cannot compare two engines on "now" — the two runs are milliseconds apart and land on
20
+ different sides of midnight roughly once every few thousand runs, which is the worst kind of
21
+ flake because it looks like a real divergence
22
+ - a window resolved during a request must not shift between the count query and the row query
23
+ - the tenant's day boundary is the tenant's, not the server's
24
+
25
+ DESIGN NOTES THAT ARE EASY TO GET WRONG
26
+ -----------------------------------------------------------------------------------------------
27
+ - Bounds are INCLUSIVE at both ends and are DATES, not timestamps. `store_query` casts both
28
+ sides to TIMESTAMP and appends 23:59:59 to the upper bound, so a bare date is correct there;
29
+ the client compares the 10-char ISO prefix, so a bare date is correct there too.
30
+ - "Last N days" INCLUDES today. Airtable's "the past week" means the last 7 days up to and
31
+ including now, not the 7 days before yesterday. Off-by-one here silently drops today's
32
+ orders from every "recent" filter — invisible until someone asks why a sale they just
33
+ entered is missing.
34
+ - Week starts MONDAY (ISO 8601). Stated rather than defaulted: Python's weekday() is
35
+ Monday=0 and JavaScript's getDay() is Sunday=0, so the two engines disagree unless one of
36
+ them is explicitly corrected. That correction is the single most likely divergence in this
37
+ file, and the gate probes a Sunday and a Monday for exactly that reason.
38
+ - A `custom` window with only one bound is legal and means open-ended on the other side.
39
+ - An UNRESOLVABLE window returns None rather than a default. A window that quietly becomes
40
+ "all time" would widen a filter while still reporting an authoritative count.
41
+ """
42
+ import datetime as _dt
43
+
44
+ #: Every window kind the UI may offer. Mirrors WINDOW_KINDS in customer-grid/windows.ts;
45
+ #: verify_windows.py asserts the two lists are identical, because a kind the client can emit and
46
+ #: the server cannot resolve is a filter that silently stops narrowing.
47
+ WINDOW_KINDS = (
48
+ "all_time",
49
+ "today",
50
+ "yesterday",
51
+ "this_week",
52
+ "last_week",
53
+ "this_month",
54
+ "last_month",
55
+ "this_quarter",
56
+ "last_quarter",
57
+ "this_year",
58
+ "last_year",
59
+ "ytd",
60
+ "ytd_last_year",
61
+ "ltm",
62
+ "past_week",
63
+ "past_month",
64
+ "past_year",
65
+ "last_n_days",
66
+ "next_n_days",
67
+ "custom",
68
+ )
69
+
70
+ #: Kinds that carry an integer `n`.
71
+ N_KINDS = frozenset({"last_n_days", "next_n_days"})
72
+
73
+ #: How each kind reads in a sentence. The UI renders these; they are here so the label and the
74
+ #: arithmetic cannot drift apart in the one place a user would never think to check.
75
+ WINDOW_LABELS = {
76
+ "all_time": "all time",
77
+ "today": "today",
78
+ "yesterday": "yesterday",
79
+ "this_week": "this week",
80
+ "last_week": "last week",
81
+ "this_month": "this month",
82
+ "last_month": "last month",
83
+ "this_quarter": "this quarter",
84
+ "last_quarter": "last quarter",
85
+ "this_year": "this year",
86
+ "last_year": "last year",
87
+ "ytd": "year to date",
88
+ # "last year to date" (LYTD), not "year to date, last year": the longer phrasing CLIPPED to
89
+ # "year to date, last ye" in the 146px window select — and this vocabulary is CLOSED, so
90
+ # the rule for it is that it stays readable rather than that it fits. Seen in a live
91
+ # screenshot of the owner's own comparison; every assertion in that run was green.
92
+ "ytd_last_year": "last year to date",
93
+ "ltm": "the last 12 months",
94
+ "past_week": "the past week",
95
+ "past_month": "the past month",
96
+ "past_year": "the past year",
97
+ "last_n_days": "the last {n} days",
98
+ "next_n_days": "the next {n} days",
99
+ "custom": "a custom range",
100
+ }
101
+
102
+ MAX_N = 3650 # ten years; a bound, not a business rule
103
+
104
+ # --- the DATE-VALUE anchors (owner item 3, 2026-07-26) ----------------------------------------
105
+ # The second half of a date CONDITION, as distinct from a measure's window:
106
+ #
107
+ # Where [Last order] [is before] [one month ago]
108
+ # ^ op ^ THIS
109
+ #
110
+ # A window answers "over what period do I sum"; an anchor answers "which single date am I
111
+ # comparing against". Both live in this module for one reason: they are the same two-engine
112
+ # contract, resolved from the same `today`, and a gate that holds one in lock-step and not the
113
+ # other would leave half the sentence free to drift.
114
+
115
+ #: Anchor modes, in the order the picker offers them. `exact` is the historical behaviour — a
116
+ #: rule with NO mode is an `exact` rule whose value is an ISO date, which is what every view
117
+ #: saved before this change carries.
118
+ ANCHOR_MODES = (
119
+ "today",
120
+ "yesterday",
121
+ "one_week_ago",
122
+ "one_month_ago",
123
+ "n_days_ago",
124
+ "exact",
125
+ )
126
+
127
+ #: Modes that carry NO value. ⚠ This set is load-bearing far outside this module: a rule whose
128
+ #: value is blank normally reads as INACTIVE, and an inactive rule is IGNORED — which WIDENS the
129
+ #: result under a count nobody would doubt. Every activeness check (`isRuleActive` in TS,
130
+ #: `filter_sql.is_rule_active`, and the validator's value handling) has to know these four are
131
+ #: active with an empty value.
132
+ ANCHOR_VALUE_FREE = frozenset({"today", "yesterday", "one_week_ago", "one_month_ago"})
133
+
134
+ ANCHOR_LABELS = {
135
+ "today": "today",
136
+ "yesterday": "yesterday",
137
+ "one_week_ago": "one week ago",
138
+ "one_month_ago": "one month ago",
139
+ "n_days_ago": "{n} days ago",
140
+ "exact": "an exact date",
141
+ }
142
+
143
+
144
+ def _iso(d):
145
+ return d.isoformat()
146
+
147
+
148
+ def _month_start(d):
149
+ return d.replace(day=1)
150
+
151
+
152
+ def _month_end(d):
153
+ return _next_month(d.replace(day=1)) - _dt.timedelta(days=1)
154
+
155
+
156
+ def _next_month(d):
157
+ return (d.replace(day=28) + _dt.timedelta(days=4)).replace(day=1)
158
+
159
+
160
+ def _quarter_start(d):
161
+ return _dt.date(d.year, 3 * ((d.month - 1) // 3) + 1, 1)
162
+
163
+
164
+ def _days_in_month(y, m):
165
+ return (_dt.date(y + (m == 12), 1 if m == 12 else m + 1, 1) - _dt.timedelta(days=1)).day
166
+
167
+
168
+ def _shift_months(d, n):
169
+ """`d` moved `n` months, CLAMPING the day to the target month's length.
170
+
171
+ Jan 31 back one month is Dec 31, but Mar 31 back one month is Feb 28 (or 29) — there is no
172
+ Feb 31 to land on. Both engines must clamp identically or "the past month" differs by up to
173
+ three days for a third of the calendar; the gate probes a 31st and a leap day for exactly it.
174
+ """
175
+ total = (d.year * 12 + (d.month - 1)) + n
176
+ y, m = divmod(total, 12)
177
+ m += 1
178
+ return _dt.date(y, m, min(d.day, _days_in_month(y, m)))
179
+
180
+
181
+ def _parse_date(v):
182
+ """Accept 'YYYY-MM-DD' (and tolerate a longer ISO timestamp by taking its date part)."""
183
+ if isinstance(v, _dt.date):
184
+ return v
185
+ s = str(v or "").strip()[:10]
186
+ if not s:
187
+ return None
188
+ try:
189
+ return _dt.date.fromisoformat(s)
190
+ except ValueError:
191
+ return None
192
+
193
+
194
+ def normalize(spec):
195
+ """Coerce an untrusted window spec to `{kind, n?, from?, to?}` or None.
196
+
197
+ Fail-closed on the KIND (an unknown kind is not a window), tolerant on the rest — the same
198
+ split `clean_filter_tree` uses, so one malformed window cannot cost a user their saved view.
199
+ """
200
+ if not isinstance(spec, dict):
201
+ return None
202
+ kind = spec.get("kind")
203
+ if kind not in WINDOW_KINDS:
204
+ return None
205
+ out = {"kind": kind}
206
+ if kind in N_KINDS:
207
+ try:
208
+ n = int(spec.get("n"))
209
+ except (TypeError, ValueError):
210
+ return None
211
+ if n < 1 or n > MAX_N:
212
+ return None
213
+ out["n"] = n
214
+ if kind == "custom":
215
+ f, t = _parse_date(spec.get("from")), _parse_date(spec.get("to"))
216
+ if f is None and t is None:
217
+ return None # a custom range with no bounds is not a window
218
+ if f is not None and t is not None and f > t:
219
+ f, t = t, f # the builder cannot enforce order; the engine can
220
+ if f is not None:
221
+ out["from"] = _iso(f)
222
+ if t is not None:
223
+ out["to"] = _iso(t)
224
+ return out
225
+
226
+
227
+ def resolve(spec, today):
228
+ """Window spec + the tenant's today -> `(date_from, date_to)`, both inclusive ISO dates.
229
+
230
+ Either side may be None, meaning open-ended. Returns None when the spec is not a window at
231
+ all — callers MUST treat that as "this condition cannot be evaluated" and refuse, never as
232
+ "no window", which would silently widen the result to all time.
233
+ """
234
+ spec = normalize(spec)
235
+ if spec is None:
236
+ return None
237
+ kind = spec["kind"]
238
+ d = _parse_date(today)
239
+ if d is None:
240
+ raise ValueError("resolve() needs an explicit `today` — see this module's docstring")
241
+
242
+ if kind == "all_time":
243
+ return (None, None)
244
+ if kind == "today":
245
+ return (_iso(d), _iso(d))
246
+ if kind == "yesterday":
247
+ y = d - _dt.timedelta(days=1)
248
+ return (_iso(y), _iso(y))
249
+
250
+ # ISO 8601: the week starts MONDAY. Python's weekday() is already Monday=0; the TS mirror
251
+ # has to correct getDay(), which is Sunday=0. The gate probes both a Sunday and a Monday.
252
+ if kind == "this_week":
253
+ start = d - _dt.timedelta(days=d.weekday())
254
+ return (_iso(start), _iso(start + _dt.timedelta(days=6)))
255
+ if kind == "last_week":
256
+ start = d - _dt.timedelta(days=d.weekday() + 7)
257
+ return (_iso(start), _iso(start + _dt.timedelta(days=6)))
258
+
259
+ if kind == "this_month":
260
+ return (_iso(_month_start(d)), _iso(_month_end(d)))
261
+ if kind == "last_month":
262
+ prev = _month_start(d) - _dt.timedelta(days=1)
263
+ return (_iso(_month_start(prev)), _iso(prev))
264
+
265
+ if kind == "this_quarter":
266
+ qs = _quarter_start(d)
267
+ qe = _month_end(_dt.date(qs.year, qs.month + 2, 1))
268
+ return (_iso(qs), _iso(qe))
269
+ if kind == "last_quarter":
270
+ prev_end = _quarter_start(d) - _dt.timedelta(days=1)
271
+ qs = _quarter_start(prev_end)
272
+ return (_iso(qs), _iso(prev_end))
273
+
274
+ if kind == "this_year":
275
+ return (_iso(_dt.date(d.year, 1, 1)), _iso(_dt.date(d.year, 12, 31)))
276
+ if kind == "last_year":
277
+ return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_dt.date(d.year - 1, 12, 31)))
278
+ if kind == "ytd":
279
+ # year-to-date ENDS TODAY, unlike this_year which runs to Dec 31. The distinction is the
280
+ # whole reason both exist: comparing "this year" against last year double-counts the
281
+ # months that have not happened yet.
282
+ return (_iso(_dt.date(d.year, 1, 1)), _iso(d))
283
+ if kind == "ytd_last_year":
284
+ # SAME PERIOD last year — Jan 1 LY through today's month/day LY. This is the honest
285
+ # partner of `ytd`, and it is not a nicety: it mirrors `core.periods.ytd_last_year`
286
+ # EXACTLY (including the Feb 29 -> Feb 28 clamp), which is what `pool()` computes the
287
+ # `revenue_ly` column from. `Sales[ytd] < Sales[ytd_last_year]` therefore reproduces the
288
+ # retired `at_risk > 0` condition rather than approximating it. Comparing `ytd` against
289
+ # `last_year` instead would pit seven months against twelve.
290
+ return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_shift_months(d, -12)))
291
+ if kind == "ltm":
292
+ # the last twelve months INCLUDING today: 365 days back, both ends inclusive
293
+ return (_iso(d - _dt.timedelta(days=364)), _iso(d))
294
+
295
+ # "The past week/month/year" are ROLLING and end TODAY — Airtable's wording, and NOT the
296
+ # calendar kinds above: `last_week` is the previous Monday-Sunday, `past_week` is the seven
297
+ # days ending today. Both are offered because a user means different things by them, and
298
+ # picking one to serve both would silently answer the other question.
299
+ # past_week = the same rule as last_n_days(7) — one period back, PLUS ONE DAY, so today
300
+ # is included and the span is exactly 7 days rather than 8.
301
+ # past_year agrees with `ltm` except across a leap day, where the calendar shift keeps the
302
+ # same month/day and the 364-day subtraction cannot.
303
+ if kind == "past_week":
304
+ return (_iso(d - _dt.timedelta(days=6)), _iso(d))
305
+ if kind == "past_month":
306
+ return (_iso(_shift_months(d, -1) + _dt.timedelta(days=1)), _iso(d))
307
+ if kind == "past_year":
308
+ return (_iso(_shift_months(d, -12) + _dt.timedelta(days=1)), _iso(d))
309
+
310
+ if kind == "last_n_days":
311
+ # INCLUSIVE of today — "the last 7 days" is today and the 6 before it, not the 7 before
312
+ # yesterday. Getting this wrong drops today's orders from every recent-activity filter.
313
+ return (_iso(d - _dt.timedelta(days=spec["n"] - 1)), _iso(d))
314
+ if kind == "next_n_days":
315
+ return (_iso(d), _iso(d + _dt.timedelta(days=spec["n"] - 1)))
316
+
317
+ if kind == "custom":
318
+ return (spec.get("from"), spec.get("to"))
319
+ raise AssertionError(f"unhandled window kind {kind!r}") # unreachable; WINDOW_KINDS is closed
320
+
321
+
322
+ def label(spec):
323
+ """How a window reads in the condition sentence ('in the last 90 days')."""
324
+ spec = normalize(spec)
325
+ if spec is None:
326
+ return "an invalid range"
327
+ return WINDOW_LABELS[spec["kind"]].format(n=spec.get("n"))
328
+
329
+
330
+ def resolve_anchor(mode, value, today):
331
+ """A date condition's right-hand side -> ONE inclusive ISO date, or None.
332
+
333
+ `mode` is an entry of ANCHOR_MODES; None/'' means `exact`, which is what every view saved
334
+ before anchors existed carries (op + an ISO date in `value`). `value` supplies the number for
335
+ `n_days_ago` and the date for `exact`, and is ignored by the four value-free modes.
336
+
337
+ Returns None when the anchor cannot be resolved — an unknown mode, a non-numeric N, a value
338
+ that is not a date. The caller must then treat the CONDITION as unanswerable and match
339
+ nothing. It must NOT fall through to "no condition": that widens the result while the count
340
+ beside it still looks authoritative, which is the whole reason this returns None rather than
341
+ a best guess (same contract as `resolve()` above).
342
+
343
+ `n_days_ago` accepts n = 0 where a WINDOW requires n >= 1. A zero-day window is empty and
344
+ could only be a mistake; "0 days ago" is today, which is a date a person can mean.
345
+ """
346
+ m = mode or "exact"
347
+ if m not in ANCHOR_MODES:
348
+ return None
349
+ d = _parse_date(today)
350
+ if d is None:
351
+ raise ValueError("resolve_anchor() needs an explicit `today` — see this module's "
352
+ "docstring on why the clock is never read here")
353
+ if m == "today":
354
+ return _iso(d)
355
+ if m == "yesterday":
356
+ return _iso(d - _dt.timedelta(days=1))
357
+ if m == "one_week_ago":
358
+ # A week ago is a DATE (today minus 7), not the past-week RANGE. The two read almost
359
+ # identically in English and mean different things in a comparison.
360
+ return _iso(d - _dt.timedelta(days=7))
361
+ if m == "one_month_ago":
362
+ return _iso(_shift_months(d, -1))
363
+ if m == "n_days_ago":
364
+ s = str("" if value is None else value).strip()
365
+ # `[0-9]` not `\d`: a Python \d matches every Unicode decimal digit, so a fullwidth
366
+ # '30' would resolve here and be rejected by the TS mirror's Number() — the exact
367
+ # two-engine divergence filter_sql.to_num was rewritten to avoid.
368
+ if not s or not all("0" <= ch <= "9" for ch in s):
369
+ return None
370
+ n = int(s)
371
+ # No `n < 0` guard: the digit test above already excludes a sign, so it would be
372
+ # unreachable. The negative control proved that — removing it changed nothing, which is
373
+ # how dead code hides in a defensive-looking line. The CAP is the live guard.
374
+ if n > MAX_N:
375
+ return None
376
+ return _iso(d - _dt.timedelta(days=n))
377
+ parsed = _parse_date(value) # exact
378
+ return None if parsed is None else _iso(parsed)
379
+
380
+
381
+ def anchor_label(mode, value):
382
+ """How an anchor reads in the condition sentence ('one month ago', '30 days ago')."""
383
+ m = mode or "exact"
384
+ if m not in ANCHOR_MODES:
385
+ return "an invalid date"
386
+ if m == "n_days_ago":
387
+ return ANCHOR_LABELS[m].format(n=str("" if value is None else value).strip() or "N")
388
+ if m == "exact":
389
+ return str("" if value is None else value).strip() or ANCHOR_LABELS[m]
390
+ return ANCHOR_LABELS[m]
platform/model/metrics/returns.yml CHANGED
@@ -1,55 +1,55 @@
1
- # Metrics: returns — the credit-note lens, defined once (2026-07-27, owner's ask).
2
- #
3
- # `modules/returns.py` has read these numbers from Odoo since it shipped; this file makes them
4
- # MODEL metrics so the Analyst, the metric dictionary and the customer-grid condition builder all
5
- # resolve the same definition instead of three.
6
- topic: credit_notes
7
- metrics:
8
- - key: returns
9
- label: Returns $
10
- agg: sum
11
- field: amount_untaxed_signed
12
- negate: true
13
- format: usd
14
- description: >-
15
- Untaxed value of posted customer credit notes (refunds), reported POSITIVE.
16
- ai_context: >-
17
- ⚠ NEGATE IS LOAD-BEARING. `amount_untaxed_signed` is negative on a credit note, so the raw
18
- sum is negative and this metric flips it. A returns figure that arrives negative means the
19
- negation was lost somewhere — and it will read as "no returns" rather than as an error.
20
- Company-level: credit notes are not business-unit tagged (they all carry team_id 1), so
21
- never attribute a returns number to Fisch or Royal. Attribute by CUSTOMER instead.
22
- validate:
23
- method: returns_sign
24
- note: >-
25
- Independent check: the metric must equal the ABSOLUTE untaxed total of the same posted
26
- credit notes read straight from Odoo, and must be positive. Catches a lost or doubled
27
- negation, which self-consistency cannot see.
28
-
29
- - key: invoiced
30
- label: Invoiced $
31
- topic: customer_invoices
32
- agg: sum
33
- field: amount_untaxed_signed
34
- format: usd
35
- description: >-
36
- Untaxed value of posted customer invoices — the denominator the business rates returns
37
- against.
38
- ai_context: >-
39
- NOT `revenue`. `revenue` is order-LINE subtotals of confirmed sales orders; this is posted
40
- billing documents, and the two differ by unbilled orders and by billing timing. Use
41
- `revenue` for "sales" and this only as the return-rate denominator.
42
-
43
- - key: returns_pct
44
- label: Return %
45
- agg: ratio
46
- numerator: returns
47
- denominator: invoiced
48
- format: pct
49
- description: "Returns as a share of invoiced sales — the rate modules/returns.py reports."
50
- ai_context: >-
51
- A FRACTION (0-1), like every other pct metric here — 0.0159 is 1.59%. The condition builder
52
- divides a typed percentage by 100 before comparing (`input_scale`), so a user types 5 and
53
- means 5%. Guarded against zero invoiced (returns 0). Cross-topic: the numerator is on
54
- credit_notes and the denominator on customer_invoices, so it is resolved per customer by
55
- computing both and combining, never by one query.
 
1
+ # Metrics: returns — the credit-note lens, defined once (2026-07-27, owner's ask).
2
+ #
3
+ # `modules/returns.py` has read these numbers from Odoo since it shipped; this file makes them
4
+ # MODEL metrics so the Analyst, the metric dictionary and the customer-grid condition builder all
5
+ # resolve the same definition instead of three.
6
+ topic: credit_notes
7
+ metrics:
8
+ - key: returns
9
+ label: Returns $
10
+ agg: sum
11
+ field: amount_untaxed_signed
12
+ negate: true
13
+ format: usd
14
+ description: >-
15
+ Untaxed value of posted customer credit notes (refunds), reported POSITIVE.
16
+ ai_context: >-
17
+ ⚠ NEGATE IS LOAD-BEARING. `amount_untaxed_signed` is negative on a credit note, so the raw
18
+ sum is negative and this metric flips it. A returns figure that arrives negative means the
19
+ negation was lost somewhere — and it will read as "no returns" rather than as an error.
20
+ Company-level: credit notes are not business-unit tagged (they all carry team_id 1), so
21
+ never attribute a returns number to Fisch or Royal. Attribute by CUSTOMER instead.
22
+ validate:
23
+ method: returns_sign
24
+ note: >-
25
+ Independent check: the metric must equal the ABSOLUTE untaxed total of the same posted
26
+ credit notes read straight from Odoo, and must be positive. Catches a lost or doubled
27
+ negation, which self-consistency cannot see.
28
+
29
+ - key: invoiced
30
+ label: Invoiced $
31
+ topic: customer_invoices
32
+ agg: sum
33
+ field: amount_untaxed_signed
34
+ format: usd
35
+ description: >-
36
+ Untaxed value of posted customer invoices — the denominator the business rates returns
37
+ against.
38
+ ai_context: >-
39
+ NOT `revenue`. `revenue` is order-LINE subtotals of confirmed sales orders; this is posted
40
+ billing documents, and the two differ by unbilled orders and by billing timing. Use
41
+ `revenue` for "sales" and this only as the return-rate denominator.
42
+
43
+ - key: returns_pct
44
+ label: Return %
45
+ agg: ratio
46
+ numerator: returns
47
+ denominator: invoiced
48
+ format: pct
49
+ description: "Returns as a share of invoiced sales — the rate modules/returns.py reports."
50
+ ai_context: >-
51
+ A FRACTION (0-1), like every other pct metric here — 0.0159 is 1.59%. The condition builder
52
+ divides a typed percentage by 100 before comparing (`input_scale`), so a user types 5 and
53
+ means 5%. Guarded against zero invoiced (returns 0). Cross-topic: the numerator is on
54
+ credit_notes and the denominator on customer_invoices, so it is resolved per customer by
55
+ computing both and combining, never by one query.
platform/model/skills/sales.skill.yml CHANGED
@@ -1,184 +1,184 @@
1
- # In-product SKILL: sales analysis recipes (v0, 2026-07-11) — the first "strong skills for
2
- # small-model AI" (the productization directive; omni-adoption addendum §3).
3
- #
4
- # WHAT THIS IS: short task recipes a SMALL model loads before acting. Each recipe maps a user ask
5
- # to an explicit TOOL-CALL PLAN over the semantic layer — the model navigates by recipe + changes
6
- # parameters, it never invents joins, scope, or SQL. This file is ALSO the binding spec for the
7
- # OM-4 tool registry: the tools named here (run_semantic_query, make_chart, make_kpi_card,
8
- # get_field_values, compose_dashboard) are the signatures OM-4
9
- # implements — recipes and tools ship and version TOGETHER (drift between them = a build error).
10
- #
11
- # Recipe fields: ask (the intent pattern, [brackets] = user-changeable parameters) · plan (the
12
- # tool calls, in order) · params (what the model may vary + allowed values) · guard (what it must
13
- # never do here). Every numeric answer inherits the platform rule: it must carry its drill link.
14
- key: sales
15
- label: Sales analysis
16
- topic: sales_lines # default topic; recipes may name another (sales_orders)
17
- metrics: [revenue, units, margin, cogs, margin_pct, orders, customers, aov]
18
-
19
- recipes:
20
- - ask: "revenue trend [monthly|weekly] [this year|LTM|a date range] [for Fisch|Royal|both]"
21
- plan: >
22
- run_semantic_query(topic=sales_lines, measures=[revenue], grain=month|week,
23
- date_from=?, date_to=?, team_id=5|6|null) -> make_chart(kind=line, x=grain, y=revenue)
24
- params: {grain: [month, week], team_id: [5, 6, null], window: "any date range"}
25
- guard: "one BU per series; if the user asks 'both', emit two series (team_id=5 and 6), never a mixed line"
26
-
27
- - ask: "top [N] customers [by revenue|margin] [window] [BU]"
28
- plan: >
29
- run_semantic_query(topic=sales_lines, measures=[revenue, margin], group_by=order_partner,
30
- date_from=?, date_to=?, team_id=?, sort=-revenue, limit=N) -> make_chart(kind=bar)
31
- params: {N: "5-50 (default 15)", sort: [revenue, margin]}
32
- guard: "group_by uses DIRECT m2o fields only (order_partner, product) — dot-path groupings fail on this platform"
33
-
34
- - ask: "top [N] products/SKUs [window] [BU]"
35
- plan: >
36
- run_semantic_query(topic=sales_lines, measures=[revenue, units, margin], group_by=product,
37
- date_from=?, date_to=?, team_id=?, sort=-revenue, limit=N, exclude_services=true)
38
- -> make_chart(kind=bar)
39
- guard: "always exclude_services=true for SKU rankings (Delivery Charges pollutes top movers)"
40
-
41
- - ask: "what gross margin [%] did we run [window]? (NO business-unit words in the ask)"
42
- plan: >
43
- run_semantic_query(topic=sales_lines, measures=[revenue, margin, margin_pct],
44
- date_from=?, date_to=?) # NO group_by — a plain margin question wants the CONSOLIDATED figure
45
- guard: "only split by BU when the user says 'by business unit' / 'per BU' / 'Fisch vs Royal'"
46
-
47
- - ask: "gross margin [%] BY BUSINESS UNIT / per BU / Fisch vs Royal [window]"
48
- plan: >
49
- run_semantic_query(topic=sales_lines, measures=[revenue, margin, margin_pct],
50
- group_by=team, date_from=?, date_to=?) -> make_chart(kind=bar, y=margin_pct)
51
- guard: "margin_pct is a ratio — computed at scope, never averaged from sub-rows"
52
-
53
- - ask: "how are sales doing [window]? / sales KPIs"
54
- plan: >
55
- run_semantic_query(topic=sales_lines, measures=[revenue, margin_pct, orders, aov],
56
- date_from=?, date_to=?, team_id=?) -> make_kpi_card(each metric, with same-window-last-year
57
- deltas via a second query shifted -1 year)
58
- guard: "deltas compare IDENTICAL windows year-over-year; never partial-vs-full periods"
59
-
60
- - ask: "revenue by state / by product category / where do we sell the most"
61
- plan: >
62
- run_semantic_query(topic=sales_lines, measures=[revenue], group_by=[state|category],
63
- sort=-revenue) -> make_chart(kind=bar, x=state|category, y=revenue)
64
- guard: >
65
- 'state' = the customer's billing state; 'category' = the product's category, where the
66
- root bucket literally named "All" is UNCATEGORIZED product — say so when it ranks
67
-
68
- - ask: "top customers BASED IN / located in [New York|a state|a city]"
69
- plan: >
70
- get_field_values(topic=sales_lines, dim=state|city, search=<place>) -> run_semantic_query(
71
- measures=[revenue], group_by=[order_partner], filters={state|city: [ids]}, sort=-revenue,
72
- limit=N, date_from=?, date_to=?) -> make_table or ranked_bar
73
- guard: >
74
- location = the state/city dims, NEVER a customer-name search ('WEST NEW YORK FLORIST' is
75
- not necessarily in New York). "New York" is ambiguous — check BOTH: if the state and the
76
- city both match, ask which one (or default to the state and say so). City master data is
77
- MESSY (case/trailing-space variants: 'BROOKLYN'/'Brooklyn'/'Brooklyn ') — include EVERY
78
- matching variant from get_field_values in the filter. Always pass an explicit window and
79
- state it.
80
-
81
- - ask: "[salesperson/agent]'s customers / sales / top customers for [Naomi|an agent] [window]"
82
- plan: >
83
- get_field_values(topic=sales_lines, dim=agent, search=<name>) -> run_semantic_query(
84
- topic=sales_lines, measures=[revenue], group_by=[order_partner], filters={agent: [id]},
85
- date_from=?, date_to=?, sort=-revenue, limit=N) -> make_table or ranked_bar
86
- guard: >
87
- agent = the CUSTOMER's assigned sales agent (book attribution: every order of that
88
- customer counts toward their agent) — resolve the agent's real name via get_field_values
89
- on the agent dim FIRST, never from customer names. For the agent's TOTAL (not per
90
- customer) drop the group_by. State the window.
91
-
92
- - ask: "revenue [or orders] BY AGENT / how did the agents do / rank the agents [window]"
93
- plan: >
94
- run_semantic_query(topic=sales_lines, measures=[revenue, margin], group_by=[agent],
95
- date_from=?, date_to=?, sort=-revenue) -> make_chart(kind=ranked_bar) or make_table
96
- guard: >
97
- the NULL agent group is customers with NO agent assigned — present it as "(no agent)"
98
- and keep it in totals (it is usually large: house accounts). Order counts by agent run
99
- on topic=sales_orders with the same agent dim.
100
-
101
- - ask: "which customers are DECLINING / negative YoY / dropped the most [BU] [window]"
102
- plan: >
103
- run_semantic_query(measures=[revenue], group_by=[order_partner], team_id=?, date_from=?,
104
- date_to=?, limit=1000) -> transform_result([{op: yoy}, {op: sort, by: revenue_delta,
105
- direction: asc}, {op: head, n: 20}]) -> make_table(columns=[order_partner, revenue,
106
- revenue_ly, revenue_delta, revenue_yoy_pct])
107
- guard: >
108
- YoY per customer comes ONLY from the yoy transform (adds revenue_ly, revenue_delta,
109
- revenue_yoy_pct per row) — never join two queries by hand. sort revenue_delta asc + head
110
- = biggest $ decline first. yoy needs explicit date_from/date_to on the source query.
111
-
112
- - ask: "export to Excel / send me a spreadsheet / CSV"
113
- plan: >
114
- run_semantic_query(...) -> make_table(result_id, title=?) — then tell the user: the table
115
- below has a 'Download CSV (opens in Excel)' button under it
116
- guard: >
117
- you cannot SEND files or email; the platform renders a download button under every chart's
118
- data expander and every table. Never claim to have sent anything.
119
-
120
- - ask: "map my customers / where are my [top] customers [by revenue]"
121
- plan: >
122
- run_semantic_query(topic=sales_lines, measures=[revenue], group_by=[order_partner],
123
- sort=-revenue, limit=20..100) -> make_chart(kind=map, x=order_partner, y=revenue)
124
- guard: >
125
- the platform places each customer at its geocoded address, dot size = the measure; rows
126
- without a geocoded address are skipped and counted on screen — no need to fetch
127
- coordinates yourself
128
-
129
- - ask: "compare this year to last year [monthly] [BU]"
130
- plan: >
131
- run_semantic_query(... grain=month, window=this year) + run_semantic_query(... window=last
132
- year shifted) -> make_chart(kind=line, two series: this year, last year)
133
-
134
- - ask: "which customer/product values are valid? (before filtering by name)"
135
- plan: >
136
- get_field_values(topic=sales_lines, field=order_partner|product, search=?) — resolve the
137
- user's spelling to the real record BEFORE querying ('NYC Florist' vs the actual name)
138
- guard: "never filter on a free-typed name; always resolve via get_field_values first"
139
-
140
- - ask: "save this as a dashboard / add to my dashboard"
141
- plan: >
142
- save_view(each query above, name=?) -> compose_dashboard(views=[...], name=?)
143
- guard: >
144
- pass save_view the EXACT object make_chart/make_kpi_card returned (it carries the query —
145
- saved views RE-RUN live; a spec without its query is rejected). After compose_dashboard,
146
- tell the user the dashboard is now in the sidebar Workflows menu as 'Saved · <name>'
147
- and stays current (every open re-runs the queries).
148
-
149
- - ask: "change/edit that view or workbook — different window, add a breakdown, rename, reorder"
150
- plan: >
151
- list_workspace() to find the exact name -> update_view(name, changes={title?/kind?/x?/y?/
152
- series?, query: {date_from?/date_to?/group_by?/grain?/filters?/limit?/...}}) for a view |
153
- update_workbook(name, views=[new order], new_name=?) for a workbook (dashboard)
154
- guard: >
155
- UPDATE tools modify existing objects — state exactly what will change and confirm first;
156
- update_view re-runs the patched query before saving, so report its row_count back. To
157
- REMOVE a query key (e.g. drop a group_by), pass it as null.
158
-
159
- - ask: "delete that view/dashboard/alert/report"
160
- plan: "list_workspace() to resolve the exact kind+name -> delete_object(kind, name)"
161
- guard: >
162
- DESTRUCTIVE — never call delete_object unless the user explicitly confirmed THE NAMED
163
- object in this conversation; deleting a view also removes it from workbooks that use it
164
-
165
- - ask: "what dashboards/views/alerts do I have? / reuse that template"
166
- plan: >
167
- list_workspace() -> answer from the objects list; to reuse a template:
168
- instantiate_template(filename, new_name=?) — it stamps NEW objects, never overwrites
169
- guard: >
170
- instantiate_template is an ARTIFACT tool — confirm with the user first; afterwards point
171
- them to the new dashboard in the sidebar Workflows menu ('Saved · <name>')
172
-
173
- - ask: "send me this every [Monday|month-start] / watch this KPI / alert me about X"
174
- plan: >
175
- (retired 2026-07-23 with the Routines module) say scheduled reports and KPI alerts are not
176
- offered right now; offer the live alternative — save the view/dashboard so it re-runs
177
- fresh on every open, and answer the underlying question NOW with run_semantic_query
178
- guard: "never claim a schedule or alert was created; the schedule_report/create_alert/schedule_insight tools no longer exist"
179
-
180
- context: >
181
- All recipes inherit the sales_lines topic scope (confirmed wholesale orders, untaxed amounts,
182
- Amazon/GIFTWARE excluded, BUs strictly isolated — see the topic's ai_context). Order COUNTS come
183
- from the sales_orders topic (order grain). Every number answered must link its drill-down rows;
184
- if a query returns something that cannot be drilled, say so instead of presenting it.
 
1
+ # In-product SKILL: sales analysis recipes (v0, 2026-07-11) — the first "strong skills for
2
+ # small-model AI" (the productization directive; omni-adoption addendum §3).
3
+ #
4
+ # WHAT THIS IS: short task recipes a SMALL model loads before acting. Each recipe maps a user ask
5
+ # to an explicit TOOL-CALL PLAN over the semantic layer — the model navigates by recipe + changes
6
+ # parameters, it never invents joins, scope, or SQL. This file is ALSO the binding spec for the
7
+ # OM-4 tool registry: the tools named here (run_semantic_query, make_chart, make_kpi_card,
8
+ # get_field_values, compose_dashboard) are the signatures OM-4
9
+ # implements — recipes and tools ship and version TOGETHER (drift between them = a build error).
10
+ #
11
+ # Recipe fields: ask (the intent pattern, [brackets] = user-changeable parameters) · plan (the
12
+ # tool calls, in order) · params (what the model may vary + allowed values) · guard (what it must
13
+ # never do here). Every numeric answer inherits the platform rule: it must carry its drill link.
14
+ key: sales
15
+ label: Sales analysis
16
+ topic: sales_lines # default topic; recipes may name another (sales_orders)
17
+ metrics: [revenue, units, margin, cogs, margin_pct, orders, customers, aov]
18
+
19
+ recipes:
20
+ - ask: "revenue trend [monthly|weekly] [this year|LTM|a date range] [for Fisch|Royal|both]"
21
+ plan: >
22
+ run_semantic_query(topic=sales_lines, measures=[revenue], grain=month|week,
23
+ date_from=?, date_to=?, team_id=5|6|null) -> make_chart(kind=line, x=grain, y=revenue)
24
+ params: {grain: [month, week], team_id: [5, 6, null], window: "any date range"}
25
+ guard: "one BU per series; if the user asks 'both', emit two series (team_id=5 and 6), never a mixed line"
26
+
27
+ - ask: "top [N] customers [by revenue|margin] [window] [BU]"
28
+ plan: >
29
+ run_semantic_query(topic=sales_lines, measures=[revenue, margin], group_by=order_partner,
30
+ date_from=?, date_to=?, team_id=?, sort=-revenue, limit=N) -> make_chart(kind=bar)
31
+ params: {N: "5-50 (default 15)", sort: [revenue, margin]}
32
+ guard: "group_by uses DIRECT m2o fields only (order_partner, product) — dot-path groupings fail on this platform"
33
+
34
+ - ask: "top [N] products/SKUs [window] [BU]"
35
+ plan: >
36
+ run_semantic_query(topic=sales_lines, measures=[revenue, units, margin], group_by=product,
37
+ date_from=?, date_to=?, team_id=?, sort=-revenue, limit=N, exclude_services=true)
38
+ -> make_chart(kind=bar)
39
+ guard: "always exclude_services=true for SKU rankings (Delivery Charges pollutes top movers)"
40
+
41
+ - ask: "what gross margin [%] did we run [window]? (NO business-unit words in the ask)"
42
+ plan: >
43
+ run_semantic_query(topic=sales_lines, measures=[revenue, margin, margin_pct],
44
+ date_from=?, date_to=?) # NO group_by — a plain margin question wants the CONSOLIDATED figure
45
+ guard: "only split by BU when the user says 'by business unit' / 'per BU' / 'Fisch vs Royal'"
46
+
47
+ - ask: "gross margin [%] BY BUSINESS UNIT / per BU / Fisch vs Royal [window]"
48
+ plan: >
49
+ run_semantic_query(topic=sales_lines, measures=[revenue, margin, margin_pct],
50
+ group_by=team, date_from=?, date_to=?) -> make_chart(kind=bar, y=margin_pct)
51
+ guard: "margin_pct is a ratio — computed at scope, never averaged from sub-rows"
52
+
53
+ - ask: "how are sales doing [window]? / sales KPIs"
54
+ plan: >
55
+ run_semantic_query(topic=sales_lines, measures=[revenue, margin_pct, orders, aov],
56
+ date_from=?, date_to=?, team_id=?) -> make_kpi_card(each metric, with same-window-last-year
57
+ deltas via a second query shifted -1 year)
58
+ guard: "deltas compare IDENTICAL windows year-over-year; never partial-vs-full periods"
59
+
60
+ - ask: "revenue by state / by product category / where do we sell the most"
61
+ plan: >
62
+ run_semantic_query(topic=sales_lines, measures=[revenue], group_by=[state|category],
63
+ sort=-revenue) -> make_chart(kind=bar, x=state|category, y=revenue)
64
+ guard: >
65
+ 'state' = the customer's billing state; 'category' = the product's category, where the
66
+ root bucket literally named "All" is UNCATEGORIZED product — say so when it ranks
67
+
68
+ - ask: "top customers BASED IN / located in [New York|a state|a city]"
69
+ plan: >
70
+ get_field_values(topic=sales_lines, dim=state|city, search=<place>) -> run_semantic_query(
71
+ measures=[revenue], group_by=[order_partner], filters={state|city: [ids]}, sort=-revenue,
72
+ limit=N, date_from=?, date_to=?) -> make_table or ranked_bar
73
+ guard: >
74
+ location = the state/city dims, NEVER a customer-name search ('WEST NEW YORK FLORIST' is
75
+ not necessarily in New York). "New York" is ambiguous — check BOTH: if the state and the
76
+ city both match, ask which one (or default to the state and say so). City master data is
77
+ MESSY (case/trailing-space variants: 'BROOKLYN'/'Brooklyn'/'Brooklyn ') — include EVERY
78
+ matching variant from get_field_values in the filter. Always pass an explicit window and
79
+ state it.
80
+
81
+ - ask: "[salesperson/agent]'s customers / sales / top customers for [Naomi|an agent] [window]"
82
+ plan: >
83
+ get_field_values(topic=sales_lines, dim=agent, search=<name>) -> run_semantic_query(
84
+ topic=sales_lines, measures=[revenue], group_by=[order_partner], filters={agent: [id]},
85
+ date_from=?, date_to=?, sort=-revenue, limit=N) -> make_table or ranked_bar
86
+ guard: >
87
+ agent = the CUSTOMER's assigned sales agent (book attribution: every order of that
88
+ customer counts toward their agent) — resolve the agent's real name via get_field_values
89
+ on the agent dim FIRST, never from customer names. For the agent's TOTAL (not per
90
+ customer) drop the group_by. State the window.
91
+
92
+ - ask: "revenue [or orders] BY AGENT / how did the agents do / rank the agents [window]"
93
+ plan: >
94
+ run_semantic_query(topic=sales_lines, measures=[revenue, margin], group_by=[agent],
95
+ date_from=?, date_to=?, sort=-revenue) -> make_chart(kind=ranked_bar) or make_table
96
+ guard: >
97
+ the NULL agent group is customers with NO agent assigned — present it as "(no agent)"
98
+ and keep it in totals (it is usually large: house accounts). Order counts by agent run
99
+ on topic=sales_orders with the same agent dim.
100
+
101
+ - ask: "which customers are DECLINING / negative YoY / dropped the most [BU] [window]"
102
+ plan: >
103
+ run_semantic_query(measures=[revenue], group_by=[order_partner], team_id=?, date_from=?,
104
+ date_to=?, limit=1000) -> transform_result([{op: yoy}, {op: sort, by: revenue_delta,
105
+ direction: asc}, {op: head, n: 20}]) -> make_table(columns=[order_partner, revenue,
106
+ revenue_ly, revenue_delta, revenue_yoy_pct])
107
+ guard: >
108
+ YoY per customer comes ONLY from the yoy transform (adds revenue_ly, revenue_delta,
109
+ revenue_yoy_pct per row) — never join two queries by hand. sort revenue_delta asc + head
110
+ = biggest $ decline first. yoy needs explicit date_from/date_to on the source query.
111
+
112
+ - ask: "export to Excel / send me a spreadsheet / CSV"
113
+ plan: >
114
+ run_semantic_query(...) -> make_table(result_id, title=?) — then tell the user: the table
115
+ below has a 'Download CSV (opens in Excel)' button under it
116
+ guard: >
117
+ you cannot SEND files or email; the platform renders a download button under every chart's
118
+ data expander and every table. Never claim to have sent anything.
119
+
120
+ - ask: "map my customers / where are my [top] customers [by revenue]"
121
+ plan: >
122
+ run_semantic_query(topic=sales_lines, measures=[revenue], group_by=[order_partner],
123
+ sort=-revenue, limit=20..100) -> make_chart(kind=map, x=order_partner, y=revenue)
124
+ guard: >
125
+ the platform places each customer at its geocoded address, dot size = the measure; rows
126
+ without a geocoded address are skipped and counted on screen — no need to fetch
127
+ coordinates yourself
128
+
129
+ - ask: "compare this year to last year [monthly] [BU]"
130
+ plan: >
131
+ run_semantic_query(... grain=month, window=this year) + run_semantic_query(... window=last
132
+ year shifted) -> make_chart(kind=line, two series: this year, last year)
133
+
134
+ - ask: "which customer/product values are valid? (before filtering by name)"
135
+ plan: >
136
+ get_field_values(topic=sales_lines, field=order_partner|product, search=?) — resolve the
137
+ user's spelling to the real record BEFORE querying ('NYC Florist' vs the actual name)
138
+ guard: "never filter on a free-typed name; always resolve via get_field_values first"
139
+
140
+ - ask: "save this as a dashboard / add to my dashboard"
141
+ plan: >
142
+ save_view(each query above, name=?) -> compose_dashboard(views=[...], name=?)
143
+ guard: >
144
+ pass save_view the EXACT object make_chart/make_kpi_card returned (it carries the query —
145
+ saved views RE-RUN live; a spec without its query is rejected). After compose_dashboard,
146
+ tell the user the dashboard is now in the sidebar Workflows menu as 'Saved · <name>'
147
+ and stays current (every open re-runs the queries).
148
+
149
+ - ask: "change/edit that view or workbook — different window, add a breakdown, rename, reorder"
150
+ plan: >
151
+ list_workspace() to find the exact name -> update_view(name, changes={title?/kind?/x?/y?/
152
+ series?, query: {date_from?/date_to?/group_by?/grain?/filters?/limit?/...}}) for a view |
153
+ update_workbook(name, views=[new order], new_name=?) for a workbook (dashboard)
154
+ guard: >
155
+ UPDATE tools modify existing objects — state exactly what will change and confirm first;
156
+ update_view re-runs the patched query before saving, so report its row_count back. To
157
+ REMOVE a query key (e.g. drop a group_by), pass it as null.
158
+
159
+ - ask: "delete that view/dashboard/alert/report"
160
+ plan: "list_workspace() to resolve the exact kind+name -> delete_object(kind, name)"
161
+ guard: >
162
+ DESTRUCTIVE — never call delete_object unless the user explicitly confirmed THE NAMED
163
+ object in this conversation; deleting a view also removes it from workbooks that use it
164
+
165
+ - ask: "what dashboards/views/alerts do I have? / reuse that template"
166
+ plan: >
167
+ list_workspace() -> answer from the objects list; to reuse a template:
168
+ instantiate_template(filename, new_name=?) — it stamps NEW objects, never overwrites
169
+ guard: >
170
+ instantiate_template is an ARTIFACT tool — confirm with the user first; afterwards point
171
+ them to the new dashboard in the sidebar Workflows menu ('Saved · <name>')
172
+
173
+ - ask: "send me this every [Monday|month-start] / watch this KPI / alert me about X"
174
+ plan: >
175
+ (retired 2026-07-23 with the Routines module) say scheduled reports and KPI alerts are not
176
+ offered right now; offer the live alternative — save the view/dashboard so it re-runs
177
+ fresh on every open, and answer the underlying question NOW with run_semantic_query
178
+ guard: "never claim a schedule or alert was created; the schedule_report/create_alert/schedule_insight tools no longer exist"
179
+
180
+ context: >
181
+ All recipes inherit the sales_lines topic scope (confirmed wholesale orders, untaxed amounts,
182
+ Amazon/GIFTWARE excluded, BUs strictly isolated — see the topic's ai_context). Order COUNTS come
183
+ from the sales_orders topic (order grain). Every number answered must link its drill-down rows;
184
+ if a query returns something that cannot be drilled, say so instead of presenting it.
platform/model/topics/credit_notes.yml CHANGED
@@ -1,48 +1,48 @@
1
- # Topic: credit_notes — posted customer CREDIT NOTES (returns), at document grain.
2
- #
3
- # Why its own topic rather than a filtered measure on a shared one: a FILTERED measure
4
- # (`store_filter_sql`) exists only on the store path, and `semantic.metric()` resolves a sum by
5
- # handing the metric's `field` to `O.sum_field(entity, domain, field)` with no room for a
6
- # per-measure filter. A metric that can be computed on one path and not the other cannot be
7
- # parity-checked — and parity is the admission bar for filtering on it. Two topics with two
8
- # DOMAINS give both paths the same scope by construction.
9
- key: credit_notes
10
- label: Customer credit notes (returns)
11
- entity: account.move
12
- domain_builder: credit_note_domain
13
-
14
- scope:
15
- documents: "posted customer credit notes only (move_type = out_refund, state = posted)"
16
- channels: >-
17
- COMPANY-LEVEL. Credit notes carry team_id = 1 for everything, so a business-unit filter on
18
- the document is meaningless — modules/returns.py says the same and attributes by the
19
- CUSTOMER instead. A BU-scoped caller is refused rather than given a company number wearing
20
- their name.
21
- excluded: >-
22
- Nothing. modules/returns.py — the number the business already reads — does not exclude the
23
- GIFTWARE DEALS partner the wholesale sales topics do. Matching it keeps one definition of
24
- "returns"; the customer-grid pool is wholesale anyway, so the intersection makes the
25
- difference unobservable there.
26
- basis: "untaxed document amounts (amount_untaxed_signed, NEGATED — see the metric)"
27
- date_field: "invoice_date"
28
-
29
- grain: "one row per posted credit note; time-filterable by invoice_date"
30
-
31
- store:
32
- table: account_move
33
- alias: m
34
- join: "LEFT JOIN res_partner rp ON rp.id = m.partner_id"
35
- date_col: "m.invoice_date"
36
- scope_sql: "m.state = 'posted' AND m.move_type = 'out_refund'"
37
- dims:
38
- partner: {col: "m.partner_id", name_col: "m.partner_name", label: "Customer"}
39
-
40
- ai_context: >
41
- Posted customer credit notes — refunds/returns. ⚠ `amount_untaxed_signed` is NEGATIVE on a
42
- credit note (verified on the tenant store 2026-07-27: all 3,006 posted refunds are negative,
43
- summing to -358,141.70), so the `returns` metric declares `negate: true` and reports a POSITIVE
44
- amount. A returns figure that comes out negative means the negation was lost, and it will read
45
- as "we have no returns" rather than as an error. This topic is COMPANY-LEVEL: credit notes are
46
- not business-unit tagged, so never present a returns number as belonging to Fisch or Royal.
47
- Odoo has no return-REASON field, so concentration (which customers, which SKUs) is the only
48
- diagnostic available.
 
1
+ # Topic: credit_notes — posted customer CREDIT NOTES (returns), at document grain.
2
+ #
3
+ # Why its own topic rather than a filtered measure on a shared one: a FILTERED measure
4
+ # (`store_filter_sql`) exists only on the store path, and `semantic.metric()` resolves a sum by
5
+ # handing the metric's `field` to `O.sum_field(entity, domain, field)` with no room for a
6
+ # per-measure filter. A metric that can be computed on one path and not the other cannot be
7
+ # parity-checked — and parity is the admission bar for filtering on it. Two topics with two
8
+ # DOMAINS give both paths the same scope by construction.
9
+ key: credit_notes
10
+ label: Customer credit notes (returns)
11
+ entity: account.move
12
+ domain_builder: credit_note_domain
13
+
14
+ scope:
15
+ documents: "posted customer credit notes only (move_type = out_refund, state = posted)"
16
+ channels: >-
17
+ COMPANY-LEVEL. Credit notes carry team_id = 1 for everything, so a business-unit filter on
18
+ the document is meaningless — modules/returns.py says the same and attributes by the
19
+ CUSTOMER instead. A BU-scoped caller is refused rather than given a company number wearing
20
+ their name.
21
+ excluded: >-
22
+ Nothing. modules/returns.py — the number the business already reads — does not exclude the
23
+ GIFTWARE DEALS partner the wholesale sales topics do. Matching it keeps one definition of
24
+ "returns"; the customer-grid pool is wholesale anyway, so the intersection makes the
25
+ difference unobservable there.
26
+ basis: "untaxed document amounts (amount_untaxed_signed, NEGATED — see the metric)"
27
+ date_field: "invoice_date"
28
+
29
+ grain: "one row per posted credit note; time-filterable by invoice_date"
30
+
31
+ store:
32
+ table: account_move
33
+ alias: m
34
+ join: "LEFT JOIN res_partner rp ON rp.id = m.partner_id"
35
+ date_col: "m.invoice_date"
36
+ scope_sql: "m.state = 'posted' AND m.move_type = 'out_refund'"
37
+ dims:
38
+ partner: {col: "m.partner_id", name_col: "m.partner_name", label: "Customer"}
39
+
40
+ ai_context: >
41
+ Posted customer credit notes — refunds/returns. ⚠ `amount_untaxed_signed` is NEGATIVE on a
42
+ credit note (verified on the tenant store 2026-07-27: all 3,006 posted refunds are negative,
43
+ summing to -358,141.70), so the `returns` metric declares `negate: true` and reports a POSITIVE
44
+ amount. A returns figure that comes out negative means the negation was lost, and it will read
45
+ as "we have no returns" rather than as an error. This topic is COMPANY-LEVEL: credit notes are
46
+ not business-unit tagged, so never present a returns number as belonging to Fisch or Royal.
47
+ Odoo has no return-REASON field, so concentration (which customers, which SKUs) is the only
48
+ diagnostic available.
platform/modules/customer_data.py CHANGED
@@ -1,684 +1,684 @@
1
- """Customer List — build, filter and save customer lists (replaces My Day, owner IA 2026-07-23).
2
-
3
- The page is a customer-list WORKBENCH in two parts:
4
- 1. BUILDER — the whole scoped book, one row per customer with a consistent metric surface
5
- (YTD / LY revenue, at-risk $, cadence, overdue days, estimated missed $ …): filter, sort,
6
- SELECT customers and add them to a list.
7
- 2. LISTS — saved lists whose membership is a visible, editable FORMULA (a rule set over the
8
- metric columns) plus hand-picked members. 'Call list' and 'Win-back' ship as TEMPLATES —
9
- the exact formulas the old My Day queues used — and the user can change or reset them.
10
-
11
- Composes already-validated customers-module functions; no separate validate() — the page's
12
- counts derive FROM the lists it renders (rule 8b). Persistence: HF store key 'customer_lists'
13
- ({username: {list_name: {rules, sort, members, note}}}); templates are virtual until edited.
14
-
15
- The user↔agent link (users.py 'agent' field) scopes the pool to that agent's book; users with
16
- no link (owner/CFO/admin) see the whole book.
17
- """
18
- import core.odoo as O
19
- import core.periods as P
20
- import core.store as store
21
- import core.table_store as table_store
22
- import modules.ar as ar
23
- import modules.customers as cust
24
- import modules.map as map_mod
25
- import modules.sales as S
26
-
27
- LIMIT = 500 # honest builder display cap — always shown with the full count, never silent
28
- KEY = 'customer_lists'
29
- TABLE_KEY = 'customer_table_workspace'
30
-
31
- # ------------------------------------------------------------------ the metric pool
32
- # Field metadata for the rule builder: key -> (label, kind). 'num' fields take >=/<=/>/</=,
33
- # 'text' fields take contains/=. Every field is a column of pool() rows.
34
- FIELDS = {
35
- 'customer': ('Customer', 'text'),
36
- 'status': ('Status', 'text'),
37
- 'agent': ('Agent', 'text'),
38
- 'city': ('City', 'text'),
39
- 'state': ('State', 'text'),
40
- 'revenue_ytd': ('YTD $', 'num'),
41
- 'revenue_ly': ('LY (same period) $', 'num'),
42
- 'yoy_pct': ('YoY %', 'num'),
43
- 'at_risk': ('At risk $', 'num'),
44
- 'ltm_rev': ('LTM $', 'num'),
45
- 'orders_24m': ('Orders (24M)', 'num'),
46
- 'aov': ('Avg order $', 'num'),
47
- 'days_since': ('Days since last order', 'num'),
48
- 'typical_gap_days': ('Typical gap (days)', 'num'),
49
- 'overdue_days': ('Overdue vs cadence (days)', 'num'),
50
- 'est_missed': ('Est. missed $', 'num'),
51
- }
52
- NUM_OPS = ('>=', '<=', '>', '<', '=')
53
- TEXT_OPS = ('contains', '=')
54
-
55
-
56
- def agent_pids(agent_name):
57
- """Partner ids of an agent's book (None = unscoped/whole book)."""
58
- return cust.agent_partner_ids(agent_name) if agent_name else None
59
-
60
-
61
- #: Wave 17 R2 — `ar` bucket label -> the grid's column key. ONE mapping, read by the attribute
62
- #: builder, the empty-row default and validate(), so a bucket cannot exist under two spellings.
63
- AR_BUCKET_FIELDS = {'1-30': 'ar_aged_1_30', '31-60': 'ar_aged_31_60',
64
- '61-90': 'ar_aged_61_90', '90+': 'ar_aged_90_plus'}
65
-
66
-
67
- def _ar_attrs(t=None):
68
- """{pid: {ar_open, ar_overdue, ar_outstanding, ar_exposure}} — the AR family.
69
-
70
- Wave 21 R1: `ar_open` is the NOT-YET-DUE half of a disjoint split (label "AR current $" —
71
- key kept so saved views keep working); `ar_outstanding` is the total (current + overdue),
72
- which is what "open AR" means everywhere else.
73
-
74
- Composed from `modules/ar.credit_exposure()`, NOT re-derived: it is already reconciled to
75
- Odoo by `ar.validate()`, so the Customer table and the Collections page cannot disagree about
76
- what a customer owes. Read from the UNFILTERED `_all_rows` — `rows` is a display-truncated
77
- top-40, and a column built off that would silently blank everybody else.
78
-
79
- ⚠ `credit_limit` is deliberately NOT a column: 20 of 1,548 customers have one. EXPOSURE is
80
- the number that answers "how much are we out on this customer", and it needs no limit.
81
-
82
- ⚠ `days_to_pay` is NOT COMPUTED here — it is READ from a nightly snapshot. MEASURED
83
- 2026-07-27: `ar.credit_exposure()` costs 9s; `ar.days_to_pay()` costs **301s**, because
84
- settlement date means walking three years of reconciled receivable lines and their
85
- full-reconcile groups. Computing it in `pool()` would add five minutes to every Customer-table
86
- build and every Space container start; computing it LAZILY would just move those five minutes
87
- onto whoever opened the page first. So the expensive half runs on the app's existing
88
- background store-sync thread (`ar.refresh_days_to_pay_snapshot`) and this reads the answer
89
- out of the store for nothing. A missing snapshot yields a blank column, never a slow page.
90
- """
91
- out = {}
92
- try:
93
- exp = ar.credit_exposure(t)
94
- for r in exp.get('_all_rows') or []:
95
- out[r['pid']] = {
96
- 'ar_open': r.get('open', 0.0) or 0.0,
97
- 'ar_overdue': r.get('overdue', 0.0) or 0.0,
98
- # Wave 21 R1 — the TOTAL owed. Sum of the two halves ar.credit_exposure already
99
- # reconciles to Odoo's residual read_group (ar.py validate), so no second oracle.
100
- 'ar_outstanding': (r.get('open', 0.0) or 0.0) + (r.get('overdue', 0.0) or 0.0),
101
- 'ar_exposure': r.get('exposure', 0.0) or 0.0,
102
- # Wave 17 R2 — the aging split, so a saved VIEW can be the collections
103
- # worklist. Keys mirror `ar.OVERDUE_BUCKETS` through AR_BUCKET_FIELDS; the four
104
- # sum EXACTLY to `ar_overdue` (gated in validate()).
105
- **{k: r.get(f'aged_{b}', 0.0) or 0.0
106
- for b, k in AR_BUCKET_FIELDS.items()},
107
- }
108
- except Exception:
109
- pass
110
- for pid, avg in ar.days_to_pay_snapshot().items():
111
- # A customer with an AR row but no settled invoice in the window keeps `None` — blank,
112
- # not 0. "Pays in 0 days" is a claim, and a false one.
113
- out.setdefault(pid, {})['days_to_pay'] = avg
114
- return out
115
-
116
-
117
- def _mix_attrs(pids=None, t=None):
118
- """{pid: {top_category, sku_count, top_sku, top_category_pct}} — the product-mix family.
119
-
120
- ONE read_group over sale.order.line by (partner, product) across the LTM window, then the
121
- product->category map, then aggregation in Python. Line grain is the only grain that can
122
- answer "what does this customer buy", and it is deliberately ALL-CHANNEL scope-wise for the
123
- same reason inventory analysis is (see [[ri-channel-scope-amazon]]): a customer's product mix
124
- is a fact about the customer, not about a sales team.
125
-
126
- `top_category_pct` is that category's share of the customer's LTM line revenue — the number
127
- that says whether "top" means dominant or merely first.
128
-
129
- ⚠ There is NO `team_id` parameter, and its absence is the point rather than an omission: for
130
- a BU-scoped viewer the revenue columns beside this one ARE team-scoped while these are not,
131
- so "top category" can name products bought through a channel that viewer's revenue figures
132
- exclude. That is the same trade the inventory modules make, and it is the right one — a
133
- customer's product mix is a fact about the customer.
134
- """
135
- t = t or P.today()
136
- mf, mt = P.ltm(t)
137
- # ⚠ `pids` NARROWS THE RESULT, NEVER THE DOMAIN — and that is not a style choice. Putting
138
- # the caller's ~1,500 partner ids into an `in` clause on a LINE-grain read_group makes Odoo
139
- # answer **502 Bad Gateway**; the same query unfiltered returns in 22s. Measured 2026-07-27,
140
- # after the filtered form took the live Customer page past 20 MINUTES without rendering.
141
- # The grouped result is per-partner anyway, so dropping the extras in Python costs nothing.
142
- dom = [('order_id.state', 'in', ['sale', 'done']),
143
- ('order_id.date_order', '>=', str(mf)), ('order_id.date_order', '<=', str(mt))]
144
- keep = set(pids) if pids else None
145
- rows = O.read_group('sale.order.line', dom, ['price_subtotal:sum'],
146
- ['order_partner_id', 'product_id'], lazy=False)
147
- prod_ids = {O.m2o_id(r.get('product_id')) for r in rows if r.get('product_id')}
148
- prod_ids.discard(None)
149
- cats = {}
150
- plist = list(prod_ids)
151
- for i in range(0, len(plist), 5000):
152
- for p in O.search_read('product.product', [('id', 'in', plist[i:i + 5000])], ['categ_id']):
153
- # `categ_id`'s display name is the full PATH ("All / Foams & Finishes / Styrofoam").
154
- # The column takes the LEAF, because a column and a group label have to be readable —
155
- # and the path's own root ("All") carries no information. ⚠ Two categories that share
156
- # a leaf name would merge in a group-by; measured against this catalogue that does not
157
- # happen, and readability is worth more than defending against a rename.
158
- full = O.m2o_name(p.get('categ_id')) or ''
159
- cats[p['id']] = (full.split(' / ')[-1].strip() or '(none)') if full else '(none)'
160
-
161
- per = {}
162
- for r in rows:
163
- pid = O.m2o_id(r.get('order_partner_id'))
164
- prod = O.m2o_id(r.get('product_id'))
165
- if pid is None or prod is None or (keep is not None and pid not in keep):
166
- continue
167
- val = r.get('price_subtotal', 0.0) or 0.0
168
- e = per.setdefault(pid, {'skus': set(), 'by_cat': {}, 'by_sku': {}, 'total': 0.0})
169
- e['skus'].add(prod)
170
- e['by_cat'][cats.get(prod, '(none)')] = e['by_cat'].get(cats.get(prod, '(none)'), 0.0) + val
171
- e['by_sku'][prod] = e['by_sku'].get(prod, 0.0) + val
172
- e['total'] += val
173
-
174
- top_prod_ids = {max(e['by_sku'], key=e['by_sku'].get) for e in per.values() if e['by_sku']}
175
- names = {}
176
- tlist = list(top_prod_ids)
177
- for i in range(0, len(tlist), 5000):
178
- for p in O.search_read('product.product', [('id', 'in', tlist[i:i + 5000])], ['name']):
179
- names[p['id']] = p.get('name') or ''
180
-
181
- out = {}
182
- for pid, e in per.items():
183
- cat = max(e['by_cat'], key=e['by_cat'].get) if e['by_cat'] else '(none)'
184
- sku = max(e['by_sku'], key=e['by_sku'].get) if e['by_sku'] else None
185
- out[pid] = {
186
- 'top_category': cat,
187
- 'top_category_pct': (e['by_cat'][cat] / e['total']) if e['total'] else None,
188
- 'sku_count': len(e['skus']),
189
- 'top_sku': names.get(sku, '(none)') if sku is not None else '(none)',
190
- }
191
- return out
192
-
193
-
194
- def _salesperson_attrs(pids=None, t=None):
195
- """{pid: salesperson} — the DOMINANT order-taker over LTM, by order count.
196
-
197
- ⚠ Salesperson is NOT the Agent, and they are not interchangeable (owner, 2026-07-27):
198
- the SALESPERSON is whoever entered and took the order, the AGENT is the person the customer
199
- is assigned to and who earns the commission — including when the customer orders through the
200
- office and somebody else keys it in. So this reads `sale.order.user_id` (the ORDER's taker,
201
- fully populated: 20 people over $6.2M LTM) and NOT `res.partner.user_id`, which is a
202
- different, near-empty field.
203
-
204
- A customer can have several over a year, so the column takes the one who took the MOST of
205
- their orders — a single value the table can group and filter cleanly. Ties break toward the
206
- larger revenue.
207
- """
208
- t = t or P.today()
209
- mf, mt = P.ltm(t)
210
- # Same rule as `_mix_attrs`: the pid list narrows the RESULT, not the domain. Order grain is
211
- # far smaller than line grain so this one never actually 502'd, but a caller's id list has no
212
- # business being pushed into a query whose grouping already answers per partner.
213
- keep = set(pids) if pids else None
214
- dom = S.order_domain(str(mf), str(mt), None)
215
- rows = O.read_group('sale.order', dom, ['amount_untaxed:sum'],
216
- ['partner_id', 'user_id'], lazy=False)
217
- per = {}
218
- for r in rows:
219
- pid = O.m2o_id(r.get('partner_id'))
220
- if pid is None or (keep is not None and pid not in keep):
221
- continue
222
- who = O.m2o_name(r.get('user_id')) or '(none)'
223
- n, rev = r.get('__count', 0) or 0, r.get('amount_untaxed', 0.0) or 0.0
224
- cur = per.get(pid)
225
- if cur is None or (n, rev) > (cur[1], cur[2]):
226
- per[pid] = (who, n, rev)
227
- return {pid: v[0] for pid, v in per.items()}
228
-
229
-
230
- def pool(agent_name=None, team_id=None):
231
- """One row per customer across the scoped book: the union of YTD buyers, same-period-LY
232
- buyers and anyone with an order in the 24-month cadence window — so lapsed/win-back
233
- accounts are in the pool, not just YTD actives. Reuses the customers module's validated
234
- building blocks (_cust_rev / _cadence_bulk / _partner_attrs) so every metric matches the
235
- Customers page definitions; est_missed is the Call-list formula (min(cycles missed, 3) × AOV)."""
236
- return _pool_build(agent_name, team_id, limit=None)['rows']
237
-
238
-
239
- def pool_first(agent_name=None, team_id=None, limit=100):
240
- """Wave-7 W1 (C1 as amended): the cold-load FAST SLICE — the pool's first `limit` rows in
241
- its own default order + the true total, built from the CHEAP families only (revenue ×3,
242
- cadence, status, est_missed, partner attributes ≈ 12s measured). The AR / product-mix /
243
- salesperson / coords columns stay BLANK until the full build swaps in: `_mix_attrs` alone
244
- is a whole-book 20s read_group whose domain must never be pid-narrowed (502 — see its
245
- docstring), so "all columns on 100 rows" measured 44.6s vs 45.1s full — no win. A partial
246
- payload is a UI PHASE, never a reporting basis — validate() reconciles the full pool only."""
247
- return _pool_build(agent_name, team_id, limit=limit, fast=True)
248
-
249
-
250
- def _dba_attrs(pids, t):
251
- """{pid: 'Fisch' | 'Royal' | 'Both'} — which brand(s) a customer's confirmed orders carry
252
- over the pool's own 24-month universe (C-DBA, wave 2026-08-02).
253
-
254
- GIFTWARE DEALS (the Amazon channel) is deliberately NOT a DBA — a customer buying only
255
- through it stays blank here, exactly as ARCHITECTURE.md §4's wholesale scope treats that
256
- team. Blank also covers the ledger-retained customers whose activity predates the window:
257
- blank means "no brand attributable in 24 months", never a guess. One read_group, grouped
258
- (partner, team); degrades to {} like the other attr families — a blank column beats a
259
- page that cannot render.
260
- """
261
- try:
262
- import datetime as _dt
263
- d_from = (_dt.date.fromisoformat(str(t)) - _dt.timedelta(days=731)).isoformat()
264
- pairs = O.read_group(
265
- 'sale.order',
266
- [('state', 'in', ('sale', 'done')), ('date_order', '>=', d_from),
267
- ('partner_id', 'in', list(pids)), ('team_id', 'in', (5, 6))],
268
- ['partner_id'], ['partner_id', 'team_id'], lazy=False)
269
- teams = {}
270
- for r in pairs:
271
- pid = O.m2o_id(r.get('partner_id'))
272
- tid = O.m2o_id(r.get('team_id'))
273
- if pid and tid in (5, 6):
274
- teams.setdefault(pid, set()).add(tid)
275
- label = {5: 'Fisch', 6: 'Royal'}
276
- return {pid: ('Both' if len(ts) == 2 else label[next(iter(ts))])
277
- for pid, ts in teams.items()}
278
- except Exception:
279
- return {}
280
-
281
-
282
- def _book_pids(team_id=None, agent_pids=None):
283
- """⭐ WAVE 20 (owner items 8 + 27, ruling R3) — THE WHOLE BOOK, not just who bought recently.
284
-
285
- Owner: *"Martin Pasternak showing only 313 accounts when I filter agent, whereas in Odoo it is
286
- 494 … I want our App to show complete source of truth, even if no sale.order at all or not.
287
- Its important to see what customer is getting assigned to Martin or whether we can retarget
288
- them."* The pool was a 24-MONTH SALES universe (YTD ∪ LY-YTD ∪ cadence-window buyers), so a
289
- customer assigned to an agent who has never ordered — precisely a retargeting target — did not
290
- exist in the app at all.
291
-
292
- R3's definition, MEASURED against live Odoo 2026-08-05:
293
- customer_rank>0 active .............. 3,614
294
- ever ordered (confirmed, team 5/6) ... 1,748
295
- UNION ............................... 3,723 (vs 1,555 in the old 24-month pool)
296
- Martin's book under this rule is **494** — the owner's Odoo number, to the account.
297
-
298
- ⛔ **THE "agent-assigned" LEG OF R3 IS DELIBERATELY NOT A THIRD TERM, and that is a
299
- measurement, not a shortcut.** Taken literally it added 174 partners and pushed Martin to 503;
300
- every one of the 9 extras on his book was an ODOO ADDRESS RECORD rather than an account —
301
- `type` in (`delivery`, `other`), most carrying a `parent_id`, and TWO with `name: False`.
302
- Shipping them would have put nameless rows in the customer table and made "how many customers
303
- does Martin have" answer 503 against an Odoo screen that says 494.
304
- Excluding address types wholesale is also wrong (`type not in (delivery, invoice, other,
305
- private)` measured **488** — it drops 6 genuine accounts that happen to carry a delivery
306
- type). `customer_rank > 0` is the predicate that means "this partner is a customer record",
307
- it reproduces the owner's number exactly, and every agent-assigned ACCOUNT already satisfies
308
- it — so the leg is subsumed rather than dropped. Item 27's actual ask (the 126 assigned
309
- partners with no `sale.order` at all) is fully served: they are rank>0 and they are in.
310
-
311
- ⚠ ACTIVE ONLY. Archived partners stay out (R3): they are ex-customers, and putting them in
312
- every count would make "how many customers do we have" unanswerable. The agent-LOGIN scope
313
- (`customers.agent_partner_ids`) deliberately still includes archived — an agent's own book is
314
- their whole history — and the two remain compatible because that set is INTERSECTED with this
315
- pool, so archived rows drop out of the table without narrowing the agent's own permissions.
316
-
317
- ⚠ BU SCOPE IS APPLIED THROUGH ORDERS, NOT THROUGH THE PARTNER. `res.partner` carries no team,
318
- so a BU-scoped caller gets the partners who have ORDERED in that BU (plus their own agent
319
- book). Widening it to every partner for a scoped user would cross the BU isolation rule that
320
- the whole permissioning model rests on.
321
- """
322
- try:
323
- if team_id:
324
- # Scoped: partners with confirmed orders in THIS BU, ever. `read_group` on partner_id
325
- # rather than a search_read of orders — the group is the distinct set, and it is one
326
- # round trip instead of paging tens of thousands of order rows.
327
- rows = O.read_group('sale.order',
328
- [('state', 'in', ('sale', 'done')), ('team_id', '=', team_id)],
329
- ['partner_id'], ['partner_id'], lazy=False)
330
- book = {O.m2o_id(r.get('partner_id')) for r in rows}
331
- book.discard(None)
332
- else:
333
- rank = O.search_read('res.partner',
334
- [('customer_rank', '>', 0), ('active', '=', True)],
335
- ['id'], limit=200000)
336
- # `ever` is NOT redundant with rank>0: a partner can be archived-then-reactivated, or
337
- # have had its rank reset, and an account that demonstrably bought from us belongs in
338
- # the book whatever its flags say now. An ORDER is a fact; a rank is a setting.
339
- rows = O.read_group('sale.order',
340
- [('state', 'in', ('sale', 'done')), ('team_id', 'in', (5, 6))],
341
- ['partner_id'], ['partner_id'], lazy=False)
342
- ever = {O.m2o_id(r.get('partner_id')) for r in rows}
343
- ever.discard(None)
344
- book = {r['id'] for r in rank} | ever
345
- # An agent filter NARROWS the book to that agent's partners — never widens it. The
346
- # intersection is what keeps an agent-login user inside their own book while still
347
- # gaining every no-order account assigned to them, which is the whole point of item 27.
348
- return (book & set(agent_pids)) if agent_pids is not None else book
349
- except Exception as e:
350
- # Degrade to the sales-derived universe rather than failing the page — but LOUDLY.
351
- #
352
- # ⛔ THIS DEGRADATION IS NOT LIKE THE OTHERS IN THIS MODULE, and the difference is what
353
- # makes silence wrong here. Every other family (`_ar_attrs`, `_mix_attrs`, …) degrades to
354
- # a blank COLUMN, which is visible: the user sees an empty column and asks why. This one
355
- # degrades to absent ROWS. The pool still builds from the sales legs, so the page renders
356
- # perfectly, the counts look plausible, and Martin is quietly back at 313 with nothing
357
- # anywhere saying the book leg failed. An invisible degradation of a COUNT is the failure
358
- # mode this codebase keeps writing rules against ([[no-unverifiable-aggregates]]).
359
- try:
360
- import harness.telemetry as _tel
361
- _tel.error('customer_data:_book_pids', e, fallback='sales-derived pool only')
362
- except Exception:
363
- pass
364
- print(f"[aios] WARNING customer_data._book_pids failed ({type(e).__name__}: {e}) - the "
365
- f"customer pool is falling back to the 24-month SALES universe, so accounts with "
366
- f"no recent orders (owner item 27) are MISSING from this build.")
367
- return set()
368
-
369
-
370
- def _pool_build(agent_name, team_id, limit, fast=False):
371
- t = P.today()
372
- yf, yt = P.ytd(t)
373
- lf, lt = P.ytd_last_year(t)
374
- mf, mt = P.ltm(t)
375
- pids = agent_pids(agent_name)
376
- this = cust._cust_rev(yf, yt, team_id, pids)
377
- last = cust._cust_rev(lf, lt, team_id, pids)
378
- ltm = cust._cust_rev(mf, mt, team_id, pids)
379
- cad = cust._cadence_bulk(t, team_id, agent_pids=pids)
380
- all_pids = set(this) | set(last) | set(cad) | _book_pids(team_id, pids)
381
- total = len(all_pids)
382
- if limit is not None and total > limit:
383
- # the SAME ordering the assembled pool ships (see the sort below) — the slice must be
384
- # the first page of the very list the full build renders, or the swap-in reshuffles.
385
- ranked = sorted(all_pids,
386
- key=lambda p: -((this.get(p) or {}).get('rev', 0.0)
387
- + (last.get(p) or {}).get('rev', 0.0)))
388
- all_pids = set(ranked[:limit])
389
- attrs = cust._partner_attrs(list(all_pids))
390
- if fast:
391
- # C1 fast phase: the slow families stay empty — their columns render blank behind the
392
- # toolbar's partial indicator until the full build swaps in.
393
- ar_a, mix_a, sp_a, geo_a, dba_a = {}, {}, {}, {}, {}
394
- else:
395
- # The three families the owner asked for (2026-07-27). Each degrades to {} rather than
396
- # raising: a customer table that cannot render because AR is momentarily unreachable is
397
- # a worse failure than one with a blank column.
398
- ar_a = _ar_attrs(t)
399
- mix_a = _mix_attrs(all_pids, t)
400
- sp_a = _salesperson_attrs(all_pids, t)
401
- # W11 (wave-7): coordinates for the Map VIEW — Odoo coords else the geocode cache,
402
- # read-only reuse (coords_for never geocodes); missing → null lat/lon, and the
403
- # client's map view discloses the count.
404
- geo_a = map_mod.coords_for(list(all_pids))
405
- # C-DBA (2026-08-02): the brand attribute rides the full build only, like the rest.
406
- dba_a = _dba_attrs(all_pids, t)
407
- # customers whose only activity sits in the older half of the 24-month cadence window
408
- # appear in no revenue map — fetch their names directly (else the lapsed cohort, exactly
409
- # the win-back targets this union exists for, would render as '?')
410
- unnamed = [pid for pid in all_pids
411
- if not ((this.get(pid) or {}).get('name') or (last.get(pid) or {}).get('name')
412
- or (ltm.get(pid) or {}).get('name'))]
413
- extra_names = ({r['id']: r['name'] for r in
414
- O.search_read('res.partner', [('id', 'in', unnamed)], ['name'])}
415
- if unnamed else {})
416
- rows = []
417
- for pid in all_pids:
418
- tv = this.get(pid) or {}
419
- lv = last.get(pid) or {}
420
- c = cad.get(pid) or {}
421
- rev, ly = tv.get('rev', 0.0), lv.get('rev', 0.0)
422
- gap = c.get('typical_gap_days')
423
- overdue = c.get('overdue_days')
424
- aov = c.get('aov', 0.0)
425
- est = (min(overdue / gap, 3.0) * aov) if (gap and overdue and overdue > 0) else 0.0
426
- status = ('New' if rev > 0 and ly <= 0 else
427
- 'Lost' if ly > 0 and rev <= 0 else
428
- 'Declining' if 0 < rev < ly else
429
- 'Growing' if rev > 0 else 'Dormant')
430
- a = attrs.get(pid) or {}
431
- rows.append({
432
- 'pid': pid,
433
- 'customer': tv.get('name') or lv.get('name')
434
- or (ltm.get(pid) or {}).get('name') or extra_names.get(pid) or '?',
435
- 'status': status,
436
- 'agent': a.get('agent', '(none)'), 'city': a.get('city', '(none)'),
437
- # C-DBA: blank (not '(none)') — a select's blank is its own honest empty state.
438
- 'dba': dba_a.get(pid, ''),
439
- 'state': a.get('state', '(none)'),
440
- # Owner 2026-07-27, the partner-attribute family. `.get(..., default)` is not
441
- # defensive habit here: search_read returned 1,548 of 1,550 pool pids — two partners
442
- # in the sales history are archived or deleted — so a customer with NO attrs row is
443
- # a real case, and it must render as '(none)' rather than KeyError the whole page.
444
- 'country': a.get('country', '(none)'), 'zip': a.get('zip', '(none)'),
445
- 'payment_terms': a.get('payment_terms', '(none)'),
446
- 'customer_since': a.get('customer_since', ''),
447
- # Row-level datum for the `created_time` field type (wave-5 item 11) — rides every
448
- # row like pid, no column of its own until a user creates one.
449
- '_created': a.get('created_at', ''),
450
- 'tags': a.get('tags', '(none)'), 'pricelist': a.get('pricelist', '(none)'),
451
- # AR / credit exposure — composed from the already-reconciled ar.py blocks, so the
452
- # Customer table and the Collections page cannot disagree.
453
- 'ar_open': (ar_a.get(pid) or {}).get('ar_open', 0.0),
454
- 'ar_overdue': (ar_a.get(pid) or {}).get('ar_overdue', 0.0),
455
- 'ar_outstanding': (ar_a.get(pid) or {}).get('ar_outstanding', 0.0),
456
- 'ar_exposure': (ar_a.get(pid) or {}).get('ar_exposure', 0.0),
457
- 'days_to_pay': (ar_a.get(pid) or {}).get('days_to_pay'),
458
- **{k: (ar_a.get(pid) or {}).get(k, 0.0) for k in AR_BUCKET_FIELDS.values()},
459
- # Product mix, LTM, line grain, ALL-channel.
460
- 'top_category': (mix_a.get(pid) or {}).get('top_category', '(none)'),
461
- 'top_category_pct': (mix_a.get(pid) or {}).get('top_category_pct'),
462
- 'sku_count': (mix_a.get(pid) or {}).get('sku_count', 0),
463
- 'top_sku': (mix_a.get(pid) or {}).get('top_sku', '(none)'),
464
- # The ORDER-TAKER, not the agent. Different people, different questions.
465
- 'salesperson': sp_a.get(pid, '(none)'),
466
- # W11: nullable — the Map view pins what it can and counts what it cannot.
467
- 'lat': (geo_a.get(pid) or {}).get('lat'),
468
- 'lon': (geo_a.get(pid) or {}).get('lon'),
469
- 'revenue_ytd': rev, 'revenue_ly': ly,
470
- 'yoy_pct': P.yoy_pct(rev, ly),
471
- 'at_risk': max(ly - rev, 0.0),
472
- 'ltm_rev': (ltm.get(pid) or {}).get('rev', 0.0),
473
- 'orders_24m': c.get('n_orders', 0), 'aov': aov,
474
- 'last_order': c.get('last_order', ''),
475
- 'days_since': c.get('days_since'),
476
- 'typical_gap_days': gap, 'overdue_days': overdue,
477
- 'est_missed': est,
478
- })
479
- for r in rows:
480
- r['odoo_status'] = 'Active'
481
- # I13 (owner item 13, contract C6): RETAIN customers that disappear from Odoo.
482
- # Only the FULL build reconciles the ledger — the fast slice is 100 rows by construction,
483
- # so letting it write would mark ~1,450 living customers archived on every cold load.
484
- if not fast:
485
- rows, total = _reconcile_ledger(rows, total, agent_name, team_id)
486
- rows.sort(key=lambda r: -(r['revenue_ytd'] + r['revenue_ly']))
487
- return {'rows': rows, 'total': total}
488
-
489
-
490
- LEDGER_KEY = 'customer_ledger'
491
-
492
-
493
- def _reconcile_ledger(rows, total, agent_name, team_id):
494
- """Remember every customer we have ever seen, and keep serving the ones Odoo stopped
495
- returning (owner item 13: "if it is deleted, that it goes to an archived tag immediately —
496
- so stored in our platform even if deleted in Odoo").
497
-
498
- Why a ledger at all: this pool is built from SALES HISTORY, not from a partner list, so a
499
- partner deleted in Odoo usually keeps appearing (their order lines remain) — until the day
500
- the lines go too, when the row would silently vanish along with every note, tag and cohort
501
- membership attached to it. Nothing else in the app would notice. The ledger is the only
502
- record that the customer ever existed, which is why it starts recording now rather than
503
- when the surfacing is finished: history you did not write down is not recoverable later.
504
-
505
- ⚠ SCOPE-SAFE. The ledger is written ONLY from an UNSCOPED build (no agent, no BU). A scoped
506
- build legitimately sees a fraction of the book, and reconciling from it would mark every
507
- customer outside that scope 'Archived' for everyone — the failure would look exactly like
508
- the data loss this is meant to prevent. Scoped builds READ the ledger and resurrect nothing
509
- they cannot prove they should see.
510
- """
511
- try:
512
- ledger = dict(store.get(LEDGER_KEY) or {})
513
- except Exception:
514
- return rows, total # a ledger we cannot read must not break the table
515
- today = str(P.today())
516
- live = {r['pid'] for r in rows}
517
- unscoped = agent_name is None and team_id is None
518
-
519
- if unscoped:
520
- for r in rows:
521
- e = ledger.get(str(r['pid'])) or {}
522
- e.update({'name': r['customer'], 'last_seen': today})
523
- e.setdefault('first_seen', today)
524
- ledger[str(r['pid'])] = e
525
-
526
- # Rows the ledger knows and this build did not return. On an unscoped build that means
527
- # "gone from Odoo"; on a scoped one it usually just means "not in this agent's book", so
528
- # only the unscoped build may surface them.
529
- revived = []
530
- if unscoped:
531
- for key, e in ledger.items():
532
- try:
533
- pid = int(key)
534
- except (TypeError, ValueError):
535
- continue
536
- if pid in live:
537
- continue
538
- revived.append({
539
- 'pid': pid, 'customer': e.get('name') or '?', 'status': 'Dormant',
540
- 'odoo_status': 'Archived',
541
- # Everything else is genuinely UNKNOWN now — the record is gone. Blank is the
542
- # honest rendering; carrying the last-known numbers forward would state figures
543
- # as current that nothing can reconcile ([[no-unverifiable-aggregates]]).
544
- 'agent': '(none)', 'city': '(none)', 'state': '(none)', 'country': '(none)',
545
- 'zip': '(none)', 'payment_terms': '(none)', 'customer_since': '',
546
- '_created': '', 'tags': '(none)', 'pricelist': '(none)',
547
- 'ar_open': 0.0, 'ar_overdue': 0.0, 'ar_outstanding': 0.0, 'ar_exposure': 0.0,
548
- 'days_to_pay': None,
549
- **{k: 0.0 for k in AR_BUCKET_FIELDS.values()},
550
- 'top_category': '(none)', 'top_category_pct': None, 'sku_count': 0,
551
- 'top_sku': '(none)', 'salesperson': '(none)', 'lat': None, 'lon': None,
552
- 'revenue_ytd': 0.0, 'revenue_ly': 0.0, 'yoy_pct': None, 'at_risk': 0.0,
553
- 'ltm_rev': 0.0, 'orders_24m': 0, 'aov': 0.0, 'last_order': '',
554
- 'days_since': None, 'typical_gap_days': None, 'overdue_days': None,
555
- 'est_missed': 0.0,
556
- })
557
- if store.available():
558
- try:
559
- store.update(LEDGER_KEY, lambda cur: {**cur, **ledger}, flush='async')
560
- except Exception:
561
- pass # recording is best-effort; the table is not
562
- return rows + revived, total + len(revived)
563
-
564
-
565
- # ------------------------------------------------------------------ the rule engine
566
- # A list's formula = [{'field','op','value'}, ...] over FIELDS + a 'sort' key ('-x' = desc).
567
- # The two templates ARE the old My Day queue formulas, expressed as visible rules.
568
- #
569
- # 'note' is a SEED, not the source of truth (owner 2026-07-25). It is what a list reads
570
- # BEFORE anyone edits it; the moment a user saves a description the stored one wins, and
571
- # an empty stored description stays empty — the template must never resurrect. That
572
- # precedence lives in aios_grid.views_from_defs (a saved view replaces its template-derived
573
- # one wholesale) and is pinned by _qa_grid_view_note.py. Edit the prose here only to change
574
- # what a NEW user starts with.
575
- TEMPLATES = {
576
- 'Call list': {
577
- 'rules': [{'field': 'orders_24m', 'op': '>=', 'value': 3},
578
- {'field': 'overdue_days', 'op': '>', 'value': 0},
579
- {'field': 'days_since', 'op': '<=', 'value': 365}],
580
- 'sort': '-est_missed',
581
- 'note': 'Customers overdue against their OWN reorder cadence (3+ orders, a real gap '
582
- 'pattern, ordered within 365d), ranked by estimated missed revenue.'},
583
- 'Win-back': {
584
- 'rules': [{'field': 'revenue_ly', 'op': '>=', 'value': 2000},
585
- {'field': 'at_risk', 'op': '>', 'value': 0}],
586
- 'sort': '-at_risk',
587
- 'note': 'Bought materially last year (over $2,000), down or gone this year, ranked by '
588
- 'dollars at risk.'},
589
- }
590
-
591
-
592
- def _match(row, rule):
593
- """One rule against one row. None/missing numeric values fail every numeric comparison
594
- (a customer with no cadence is never 'overdue'); text ops are case-insensitive."""
595
- field, op, val = rule.get('field'), rule.get('op'), rule.get('value')
596
- if field not in FIELDS:
597
- return True # unknown field → rule is inert, not a crash
598
- v = row.get(field)
599
- if FIELDS[field][1] == 'text':
600
- s, q = str(v or '').lower(), str(val or '').lower()
601
- return q in s if op == 'contains' else s == q
602
- try:
603
- v, val = float(v), float(val)
604
- except (TypeError, ValueError):
605
- return False
606
- return {'>=': v >= val, '<=': v <= val, '>': v > val, '<': v < val,
607
- '=': v == val}.get(op, False)
608
-
609
-
610
- def apply_rules(rows, rules, sort=None):
611
- """Filter the pool through a formula (AND of all rules) and apply the list's sort."""
612
- out = [r for r in rows if all(_match(r, ru) for ru in (rules or []))]
613
- if sort:
614
- key = sort.lstrip('-')
615
- out.sort(key=lambda r: (r.get(key) is None,
616
- -(r.get(key) or 0) if sort.startswith('-') else (r.get(key) or 0))
617
- if FIELDS.get(key, ('', 'num'))[1] == 'num'
618
- else str(r.get(key) or '').lower(), reverse=False)
619
- return out
620
-
621
-
622
- # ------------------------------------------------------------------ persistence (HF store)
623
- def saved_lists(username):
624
- """{list_name: {'rules','sort','members','note'}} for one user ({} when none / store down)."""
625
- try:
626
- return (store.get(KEY) or {}).get(username, {}) or {}
627
- except Exception:
628
- return {}
629
-
630
-
631
- def save_list(username, name, definition):
632
- def _up(d):
633
- d.setdefault(username, {})[name] = definition
634
- return d
635
- store.update(KEY, _up)
636
-
637
-
638
- def delete_list(username, name):
639
- def _up(d):
640
- (d.get(username) or {}).pop(name, None)
641
- return d
642
- store.update(KEY, _up)
643
-
644
-
645
- # ------------------------------------------------------------------ Airtable-style table workspace
646
- # This is intentionally separate from `customer_lists`: a LIST owns membership/formula
647
- # semantics, while a VIEW owns presentation/query state (filters, multi-sort, grouping,
648
- # visible fields, widths, color, row height). List views can reference the same formula
649
- # without conflating the two persistence contracts.
650
- #
651
- # The store logic MOVED to core/table_store.py 2026-07-27 (the table-page factory): these
652
- # names are the Customer table's instance of the generic per-object workspace store, kept as
653
- # module functions so every existing caller (app.py's host loop, QA gates) is untouched.
654
- # A NEW table object gets its own `core.table_store.make('<its>_table_workspace')`.
655
- _TSTORE = table_store.make(TABLE_KEY)
656
-
657
- #: Wave 16 C-TOPIC — the ops OBJECT, for callers that select a table by TOPIC (grid_events'
658
- #: `EventCtx.table`). The module-level re-exports below stay for every existing caller.
659
- TABLE_OPS = _TSTORE
660
-
661
- table_workspace = _TSTORE.workspace
662
- shared_table_views = _TSTORE.shared_views # wave-9 I17: views shared WITH this viewer
663
- shared_table_view = _TSTORE.shared_view # ...and one raw, for authorisation only
664
- save_table_view = _TSTORE.save_view
665
- delete_table_view = _TSTORE.delete_view
666
- save_table_field = _TSTORE.save_field
667
- delete_table_field = _TSTORE.delete_field
668
- duplicate_table_field = _TSTORE.duplicate_field
669
- patch_table_overlay = _TSTORE.patch_overlay
670
- save_table_folders = _TSTORE.save_folders
671
- save_table_active_view = _TSTORE.save_active_view # owner item 3: last-opened view, per user
672
- save_table_record_layout = _TSTORE.save_record_layout # 2026-08-02 C-LAYOUT: record-detail order
673
-
674
-
675
- # ------------------------------------------------------------------ digest compatibility
676
- def queues(agent_name=None, team_id=None, limit=200):
677
- """The two canonical morning lists (call list + win-back) for the daily digest email —
678
- unchanged formulas via the customers module (the digest always uses the CANONICAL templates,
679
- not a user's edited copy, so every rep's email means the same thing)."""
680
- pids = agent_pids(agent_name)
681
- return {
682
- 'calls': cust.contact_recommendations(team_id=team_id, limit=limit, agent_pids=pids),
683
- 'risk': cust.at_risk(team_id=team_id, limit=limit, agent_pids=pids),
684
- }
 
1
+ """Customer List — build, filter and save customer lists (replaces My Day, owner IA 2026-07-23).
2
+
3
+ The page is a customer-list WORKBENCH in two parts:
4
+ 1. BUILDER — the whole scoped book, one row per customer with a consistent metric surface
5
+ (YTD / LY revenue, at-risk $, cadence, overdue days, estimated missed $ …): filter, sort,
6
+ SELECT customers and add them to a list.
7
+ 2. LISTS — saved lists whose membership is a visible, editable FORMULA (a rule set over the
8
+ metric columns) plus hand-picked members. 'Call list' and 'Win-back' ship as TEMPLATES —
9
+ the exact formulas the old My Day queues used — and the user can change or reset them.
10
+
11
+ Composes already-validated customers-module functions; no separate validate() — the page's
12
+ counts derive FROM the lists it renders (rule 8b). Persistence: HF store key 'customer_lists'
13
+ ({username: {list_name: {rules, sort, members, note}}}); templates are virtual until edited.
14
+
15
+ The user↔agent link (users.py 'agent' field) scopes the pool to that agent's book; users with
16
+ no link (owner/CFO/admin) see the whole book.
17
+ """
18
+ import core.odoo as O
19
+ import core.periods as P
20
+ import core.store as store
21
+ import core.table_store as table_store
22
+ import modules.ar as ar
23
+ import modules.customers as cust
24
+ import modules.map as map_mod
25
+ import modules.sales as S
26
+
27
+ LIMIT = 500 # honest builder display cap — always shown with the full count, never silent
28
+ KEY = 'customer_lists'
29
+ TABLE_KEY = 'customer_table_workspace'
30
+
31
+ # ------------------------------------------------------------------ the metric pool
32
+ # Field metadata for the rule builder: key -> (label, kind). 'num' fields take >=/<=/>/</=,
33
+ # 'text' fields take contains/=. Every field is a column of pool() rows.
34
+ FIELDS = {
35
+ 'customer': ('Customer', 'text'),
36
+ 'status': ('Status', 'text'),
37
+ 'agent': ('Agent', 'text'),
38
+ 'city': ('City', 'text'),
39
+ 'state': ('State', 'text'),
40
+ 'revenue_ytd': ('YTD $', 'num'),
41
+ 'revenue_ly': ('LY (same period) $', 'num'),
42
+ 'yoy_pct': ('YoY %', 'num'),
43
+ 'at_risk': ('At risk $', 'num'),
44
+ 'ltm_rev': ('LTM $', 'num'),
45
+ 'orders_24m': ('Orders (24M)', 'num'),
46
+ 'aov': ('Avg order $', 'num'),
47
+ 'days_since': ('Days since last order', 'num'),
48
+ 'typical_gap_days': ('Typical gap (days)', 'num'),
49
+ 'overdue_days': ('Overdue vs cadence (days)', 'num'),
50
+ 'est_missed': ('Est. missed $', 'num'),
51
+ }
52
+ NUM_OPS = ('>=', '<=', '>', '<', '=')
53
+ TEXT_OPS = ('contains', '=')
54
+
55
+
56
+ def agent_pids(agent_name):
57
+ """Partner ids of an agent's book (None = unscoped/whole book)."""
58
+ return cust.agent_partner_ids(agent_name) if agent_name else None
59
+
60
+
61
+ #: Wave 17 R2 — `ar` bucket label -> the grid's column key. ONE mapping, read by the attribute
62
+ #: builder, the empty-row default and validate(), so a bucket cannot exist under two spellings.
63
+ AR_BUCKET_FIELDS = {'1-30': 'ar_aged_1_30', '31-60': 'ar_aged_31_60',
64
+ '61-90': 'ar_aged_61_90', '90+': 'ar_aged_90_plus'}
65
+
66
+
67
+ def _ar_attrs(t=None):
68
+ """{pid: {ar_open, ar_overdue, ar_outstanding, ar_exposure}} — the AR family.
69
+
70
+ Wave 21 R1: `ar_open` is the NOT-YET-DUE half of a disjoint split (label "AR current $" —
71
+ key kept so saved views keep working); `ar_outstanding` is the total (current + overdue),
72
+ which is what "open AR" means everywhere else.
73
+
74
+ Composed from `modules/ar.credit_exposure()`, NOT re-derived: it is already reconciled to
75
+ Odoo by `ar.validate()`, so the Customer table and the Collections page cannot disagree about
76
+ what a customer owes. Read from the UNFILTERED `_all_rows` — `rows` is a display-truncated
77
+ top-40, and a column built off that would silently blank everybody else.
78
+
79
+ ⚠ `credit_limit` is deliberately NOT a column: 20 of 1,548 customers have one. EXPOSURE is
80
+ the number that answers "how much are we out on this customer", and it needs no limit.
81
+
82
+ ⚠ `days_to_pay` is NOT COMPUTED here — it is READ from a nightly snapshot. MEASURED
83
+ 2026-07-27: `ar.credit_exposure()` costs 9s; `ar.days_to_pay()` costs **301s**, because
84
+ settlement date means walking three years of reconciled receivable lines and their
85
+ full-reconcile groups. Computing it in `pool()` would add five minutes to every Customer-table
86
+ build and every Space container start; computing it LAZILY would just move those five minutes
87
+ onto whoever opened the page first. So the expensive half runs on the app's existing
88
+ background store-sync thread (`ar.refresh_days_to_pay_snapshot`) and this reads the answer
89
+ out of the store for nothing. A missing snapshot yields a blank column, never a slow page.
90
+ """
91
+ out = {}
92
+ try:
93
+ exp = ar.credit_exposure(t)
94
+ for r in exp.get('_all_rows') or []:
95
+ out[r['pid']] = {
96
+ 'ar_open': r.get('open', 0.0) or 0.0,
97
+ 'ar_overdue': r.get('overdue', 0.0) or 0.0,
98
+ # Wave 21 R1 — the TOTAL owed. Sum of the two halves ar.credit_exposure already
99
+ # reconciles to Odoo's residual read_group (ar.py validate), so no second oracle.
100
+ 'ar_outstanding': (r.get('open', 0.0) or 0.0) + (r.get('overdue', 0.0) or 0.0),
101
+ 'ar_exposure': r.get('exposure', 0.0) or 0.0,
102
+ # Wave 17 R2 — the aging split, so a saved VIEW can be the collections
103
+ # worklist. Keys mirror `ar.OVERDUE_BUCKETS` through AR_BUCKET_FIELDS; the four
104
+ # sum EXACTLY to `ar_overdue` (gated in validate()).
105
+ **{k: r.get(f'aged_{b}', 0.0) or 0.0
106
+ for b, k in AR_BUCKET_FIELDS.items()},
107
+ }
108
+ except Exception:
109
+ pass
110
+ for pid, avg in ar.days_to_pay_snapshot().items():
111
+ # A customer with an AR row but no settled invoice in the window keeps `None` — blank,
112
+ # not 0. "Pays in 0 days" is a claim, and a false one.
113
+ out.setdefault(pid, {})['days_to_pay'] = avg
114
+ return out
115
+
116
+
117
+ def _mix_attrs(pids=None, t=None):
118
+ """{pid: {top_category, sku_count, top_sku, top_category_pct}} — the product-mix family.
119
+
120
+ ONE read_group over sale.order.line by (partner, product) across the LTM window, then the
121
+ product->category map, then aggregation in Python. Line grain is the only grain that can
122
+ answer "what does this customer buy", and it is deliberately ALL-CHANNEL scope-wise for the
123
+ same reason inventory analysis is (see [[ri-channel-scope-amazon]]): a customer's product mix
124
+ is a fact about the customer, not about a sales team.
125
+
126
+ `top_category_pct` is that category's share of the customer's LTM line revenue — the number
127
+ that says whether "top" means dominant or merely first.
128
+
129
+ ⚠ There is NO `team_id` parameter, and its absence is the point rather than an omission: for
130
+ a BU-scoped viewer the revenue columns beside this one ARE team-scoped while these are not,
131
+ so "top category" can name products bought through a channel that viewer's revenue figures
132
+ exclude. That is the same trade the inventory modules make, and it is the right one — a
133
+ customer's product mix is a fact about the customer.
134
+ """
135
+ t = t or P.today()
136
+ mf, mt = P.ltm(t)
137
+ # ⚠ `pids` NARROWS THE RESULT, NEVER THE DOMAIN — and that is not a style choice. Putting
138
+ # the caller's ~1,500 partner ids into an `in` clause on a LINE-grain read_group makes Odoo
139
+ # answer **502 Bad Gateway**; the same query unfiltered returns in 22s. Measured 2026-07-27,
140
+ # after the filtered form took the live Customer page past 20 MINUTES without rendering.
141
+ # The grouped result is per-partner anyway, so dropping the extras in Python costs nothing.
142
+ dom = [('order_id.state', 'in', ['sale', 'done']),
143
+ ('order_id.date_order', '>=', str(mf)), ('order_id.date_order', '<=', str(mt))]
144
+ keep = set(pids) if pids else None
145
+ rows = O.read_group('sale.order.line', dom, ['price_subtotal:sum'],
146
+ ['order_partner_id', 'product_id'], lazy=False)
147
+ prod_ids = {O.m2o_id(r.get('product_id')) for r in rows if r.get('product_id')}
148
+ prod_ids.discard(None)
149
+ cats = {}
150
+ plist = list(prod_ids)
151
+ for i in range(0, len(plist), 5000):
152
+ for p in O.search_read('product.product', [('id', 'in', plist[i:i + 5000])], ['categ_id']):
153
+ # `categ_id`'s display name is the full PATH ("All / Foams & Finishes / Styrofoam").
154
+ # The column takes the LEAF, because a column and a group label have to be readable —
155
+ # and the path's own root ("All") carries no information. ⚠ Two categories that share
156
+ # a leaf name would merge in a group-by; measured against this catalogue that does not
157
+ # happen, and readability is worth more than defending against a rename.
158
+ full = O.m2o_name(p.get('categ_id')) or ''
159
+ cats[p['id']] = (full.split(' / ')[-1].strip() or '(none)') if full else '(none)'
160
+
161
+ per = {}
162
+ for r in rows:
163
+ pid = O.m2o_id(r.get('order_partner_id'))
164
+ prod = O.m2o_id(r.get('product_id'))
165
+ if pid is None or prod is None or (keep is not None and pid not in keep):
166
+ continue
167
+ val = r.get('price_subtotal', 0.0) or 0.0
168
+ e = per.setdefault(pid, {'skus': set(), 'by_cat': {}, 'by_sku': {}, 'total': 0.0})
169
+ e['skus'].add(prod)
170
+ e['by_cat'][cats.get(prod, '(none)')] = e['by_cat'].get(cats.get(prod, '(none)'), 0.0) + val
171
+ e['by_sku'][prod] = e['by_sku'].get(prod, 0.0) + val
172
+ e['total'] += val
173
+
174
+ top_prod_ids = {max(e['by_sku'], key=e['by_sku'].get) for e in per.values() if e['by_sku']}
175
+ names = {}
176
+ tlist = list(top_prod_ids)
177
+ for i in range(0, len(tlist), 5000):
178
+ for p in O.search_read('product.product', [('id', 'in', tlist[i:i + 5000])], ['name']):
179
+ names[p['id']] = p.get('name') or ''
180
+
181
+ out = {}
182
+ for pid, e in per.items():
183
+ cat = max(e['by_cat'], key=e['by_cat'].get) if e['by_cat'] else '(none)'
184
+ sku = max(e['by_sku'], key=e['by_sku'].get) if e['by_sku'] else None
185
+ out[pid] = {
186
+ 'top_category': cat,
187
+ 'top_category_pct': (e['by_cat'][cat] / e['total']) if e['total'] else None,
188
+ 'sku_count': len(e['skus']),
189
+ 'top_sku': names.get(sku, '(none)') if sku is not None else '(none)',
190
+ }
191
+ return out
192
+
193
+
194
+ def _salesperson_attrs(pids=None, t=None):
195
+ """{pid: salesperson} — the DOMINANT order-taker over LTM, by order count.
196
+
197
+ ⚠ Salesperson is NOT the Agent, and they are not interchangeable (owner, 2026-07-27):
198
+ the SALESPERSON is whoever entered and took the order, the AGENT is the person the customer
199
+ is assigned to and who earns the commission — including when the customer orders through the
200
+ office and somebody else keys it in. So this reads `sale.order.user_id` (the ORDER's taker,
201
+ fully populated: 20 people over $6.2M LTM) and NOT `res.partner.user_id`, which is a
202
+ different, near-empty field.
203
+
204
+ A customer can have several over a year, so the column takes the one who took the MOST of
205
+ their orders — a single value the table can group and filter cleanly. Ties break toward the
206
+ larger revenue.
207
+ """
208
+ t = t or P.today()
209
+ mf, mt = P.ltm(t)
210
+ # Same rule as `_mix_attrs`: the pid list narrows the RESULT, not the domain. Order grain is
211
+ # far smaller than line grain so this one never actually 502'd, but a caller's id list has no
212
+ # business being pushed into a query whose grouping already answers per partner.
213
+ keep = set(pids) if pids else None
214
+ dom = S.order_domain(str(mf), str(mt), None)
215
+ rows = O.read_group('sale.order', dom, ['amount_untaxed:sum'],
216
+ ['partner_id', 'user_id'], lazy=False)
217
+ per = {}
218
+ for r in rows:
219
+ pid = O.m2o_id(r.get('partner_id'))
220
+ if pid is None or (keep is not None and pid not in keep):
221
+ continue
222
+ who = O.m2o_name(r.get('user_id')) or '(none)'
223
+ n, rev = r.get('__count', 0) or 0, r.get('amount_untaxed', 0.0) or 0.0
224
+ cur = per.get(pid)
225
+ if cur is None or (n, rev) > (cur[1], cur[2]):
226
+ per[pid] = (who, n, rev)
227
+ return {pid: v[0] for pid, v in per.items()}
228
+
229
+
230
+ def pool(agent_name=None, team_id=None):
231
+ """One row per customer across the scoped book: the union of YTD buyers, same-period-LY
232
+ buyers and anyone with an order in the 24-month cadence window — so lapsed/win-back
233
+ accounts are in the pool, not just YTD actives. Reuses the customers module's validated
234
+ building blocks (_cust_rev / _cadence_bulk / _partner_attrs) so every metric matches the
235
+ Customers page definitions; est_missed is the Call-list formula (min(cycles missed, 3) × AOV)."""
236
+ return _pool_build(agent_name, team_id, limit=None)['rows']
237
+
238
+
239
+ def pool_first(agent_name=None, team_id=None, limit=100):
240
+ """Wave-7 W1 (C1 as amended): the cold-load FAST SLICE — the pool's first `limit` rows in
241
+ its own default order + the true total, built from the CHEAP families only (revenue ×3,
242
+ cadence, status, est_missed, partner attributes ≈ 12s measured). The AR / product-mix /
243
+ salesperson / coords columns stay BLANK until the full build swaps in: `_mix_attrs` alone
244
+ is a whole-book 20s read_group whose domain must never be pid-narrowed (502 — see its
245
+ docstring), so "all columns on 100 rows" measured 44.6s vs 45.1s full — no win. A partial
246
+ payload is a UI PHASE, never a reporting basis — validate() reconciles the full pool only."""
247
+ return _pool_build(agent_name, team_id, limit=limit, fast=True)
248
+
249
+
250
+ def _dba_attrs(pids, t):
251
+ """{pid: 'Fisch' | 'Royal' | 'Both'} — which brand(s) a customer's confirmed orders carry
252
+ over the pool's own 24-month universe (C-DBA, wave 2026-08-02).
253
+
254
+ GIFTWARE DEALS (the Amazon channel) is deliberately NOT a DBA — a customer buying only
255
+ through it stays blank here, exactly as ARCHITECTURE.md §4's wholesale scope treats that
256
+ team. Blank also covers the ledger-retained customers whose activity predates the window:
257
+ blank means "no brand attributable in 24 months", never a guess. One read_group, grouped
258
+ (partner, team); degrades to {} like the other attr families — a blank column beats a
259
+ page that cannot render.
260
+ """
261
+ try:
262
+ import datetime as _dt
263
+ d_from = (_dt.date.fromisoformat(str(t)) - _dt.timedelta(days=731)).isoformat()
264
+ pairs = O.read_group(
265
+ 'sale.order',
266
+ [('state', 'in', ('sale', 'done')), ('date_order', '>=', d_from),
267
+ ('partner_id', 'in', list(pids)), ('team_id', 'in', (5, 6))],
268
+ ['partner_id'], ['partner_id', 'team_id'], lazy=False)
269
+ teams = {}
270
+ for r in pairs:
271
+ pid = O.m2o_id(r.get('partner_id'))
272
+ tid = O.m2o_id(r.get('team_id'))
273
+ if pid and tid in (5, 6):
274
+ teams.setdefault(pid, set()).add(tid)
275
+ label = {5: 'Fisch', 6: 'Royal'}
276
+ return {pid: ('Both' if len(ts) == 2 else label[next(iter(ts))])
277
+ for pid, ts in teams.items()}
278
+ except Exception:
279
+ return {}
280
+
281
+
282
+ def _book_pids(team_id=None, agent_pids=None):
283
+ """⭐ WAVE 20 (owner items 8 + 27, ruling R3) — THE WHOLE BOOK, not just who bought recently.
284
+
285
+ Owner: *"Martin Pasternak showing only 313 accounts when I filter agent, whereas in Odoo it is
286
+ 494 … I want our App to show complete source of truth, even if no sale.order at all or not.
287
+ Its important to see what customer is getting assigned to Martin or whether we can retarget
288
+ them."* The pool was a 24-MONTH SALES universe (YTD ∪ LY-YTD ∪ cadence-window buyers), so a
289
+ customer assigned to an agent who has never ordered — precisely a retargeting target — did not
290
+ exist in the app at all.
291
+
292
+ R3's definition, MEASURED against live Odoo 2026-08-05:
293
+ customer_rank>0 active .............. 3,614
294
+ ever ordered (confirmed, team 5/6) ... 1,748
295
+ UNION ............................... 3,723 (vs 1,555 in the old 24-month pool)
296
+ Martin's book under this rule is **494** — the owner's Odoo number, to the account.
297
+
298
+ ⛔ **THE "agent-assigned" LEG OF R3 IS DELIBERATELY NOT A THIRD TERM, and that is a
299
+ measurement, not a shortcut.** Taken literally it added 174 partners and pushed Martin to 503;
300
+ every one of the 9 extras on his book was an ODOO ADDRESS RECORD rather than an account —
301
+ `type` in (`delivery`, `other`), most carrying a `parent_id`, and TWO with `name: False`.
302
+ Shipping them would have put nameless rows in the customer table and made "how many customers
303
+ does Martin have" answer 503 against an Odoo screen that says 494.
304
+ Excluding address types wholesale is also wrong (`type not in (delivery, invoice, other,
305
+ private)` measured **488** — it drops 6 genuine accounts that happen to carry a delivery
306
+ type). `customer_rank > 0` is the predicate that means "this partner is a customer record",
307
+ it reproduces the owner's number exactly, and every agent-assigned ACCOUNT already satisfies
308
+ it — so the leg is subsumed rather than dropped. Item 27's actual ask (the 126 assigned
309
+ partners with no `sale.order` at all) is fully served: they are rank>0 and they are in.
310
+
311
+ ⚠ ACTIVE ONLY. Archived partners stay out (R3): they are ex-customers, and putting them in
312
+ every count would make "how many customers do we have" unanswerable. The agent-LOGIN scope
313
+ (`customers.agent_partner_ids`) deliberately still includes archived — an agent's own book is
314
+ their whole history — and the two remain compatible because that set is INTERSECTED with this
315
+ pool, so archived rows drop out of the table without narrowing the agent's own permissions.
316
+
317
+ ⚠ BU SCOPE IS APPLIED THROUGH ORDERS, NOT THROUGH THE PARTNER. `res.partner` carries no team,
318
+ so a BU-scoped caller gets the partners who have ORDERED in that BU (plus their own agent
319
+ book). Widening it to every partner for a scoped user would cross the BU isolation rule that
320
+ the whole permissioning model rests on.
321
+ """
322
+ try:
323
+ if team_id:
324
+ # Scoped: partners with confirmed orders in THIS BU, ever. `read_group` on partner_id
325
+ # rather than a search_read of orders — the group is the distinct set, and it is one
326
+ # round trip instead of paging tens of thousands of order rows.
327
+ rows = O.read_group('sale.order',
328
+ [('state', 'in', ('sale', 'done')), ('team_id', '=', team_id)],
329
+ ['partner_id'], ['partner_id'], lazy=False)
330
+ book = {O.m2o_id(r.get('partner_id')) for r in rows}
331
+ book.discard(None)
332
+ else:
333
+ rank = O.search_read('res.partner',
334
+ [('customer_rank', '>', 0), ('active', '=', True)],
335
+ ['id'], limit=200000)
336
+ # `ever` is NOT redundant with rank>0: a partner can be archived-then-reactivated, or
337
+ # have had its rank reset, and an account that demonstrably bought from us belongs in
338
+ # the book whatever its flags say now. An ORDER is a fact; a rank is a setting.
339
+ rows = O.read_group('sale.order',
340
+ [('state', 'in', ('sale', 'done')), ('team_id', 'in', (5, 6))],
341
+ ['partner_id'], ['partner_id'], lazy=False)
342
+ ever = {O.m2o_id(r.get('partner_id')) for r in rows}
343
+ ever.discard(None)
344
+ book = {r['id'] for r in rank} | ever
345
+ # An agent filter NARROWS the book to that agent's partners — never widens it. The
346
+ # intersection is what keeps an agent-login user inside their own book while still
347
+ # gaining every no-order account assigned to them, which is the whole point of item 27.
348
+ return (book & set(agent_pids)) if agent_pids is not None else book
349
+ except Exception as e:
350
+ # Degrade to the sales-derived universe rather than failing the page — but LOUDLY.
351
+ #
352
+ # ⛔ THIS DEGRADATION IS NOT LIKE THE OTHERS IN THIS MODULE, and the difference is what
353
+ # makes silence wrong here. Every other family (`_ar_attrs`, `_mix_attrs`, …) degrades to
354
+ # a blank COLUMN, which is visible: the user sees an empty column and asks why. This one
355
+ # degrades to absent ROWS. The pool still builds from the sales legs, so the page renders
356
+ # perfectly, the counts look plausible, and Martin is quietly back at 313 with nothing
357
+ # anywhere saying the book leg failed. An invisible degradation of a COUNT is the failure
358
+ # mode this codebase keeps writing rules against ([[no-unverifiable-aggregates]]).
359
+ try:
360
+ import harness.telemetry as _tel
361
+ _tel.error('customer_data:_book_pids', e, fallback='sales-derived pool only')
362
+ except Exception:
363
+ pass
364
+ print(f"[aios] WARNING customer_data._book_pids failed ({type(e).__name__}: {e}) - the "
365
+ f"customer pool is falling back to the 24-month SALES universe, so accounts with "
366
+ f"no recent orders (owner item 27) are MISSING from this build.")
367
+ return set()
368
+
369
+
370
+ def _pool_build(agent_name, team_id, limit, fast=False):
371
+ t = P.today()
372
+ yf, yt = P.ytd(t)
373
+ lf, lt = P.ytd_last_year(t)
374
+ mf, mt = P.ltm(t)
375
+ pids = agent_pids(agent_name)
376
+ this = cust._cust_rev(yf, yt, team_id, pids)
377
+ last = cust._cust_rev(lf, lt, team_id, pids)
378
+ ltm = cust._cust_rev(mf, mt, team_id, pids)
379
+ cad = cust._cadence_bulk(t, team_id, agent_pids=pids)
380
+ all_pids = set(this) | set(last) | set(cad) | _book_pids(team_id, pids)
381
+ total = len(all_pids)
382
+ if limit is not None and total > limit:
383
+ # the SAME ordering the assembled pool ships (see the sort below) — the slice must be
384
+ # the first page of the very list the full build renders, or the swap-in reshuffles.
385
+ ranked = sorted(all_pids,
386
+ key=lambda p: -((this.get(p) or {}).get('rev', 0.0)
387
+ + (last.get(p) or {}).get('rev', 0.0)))
388
+ all_pids = set(ranked[:limit])
389
+ attrs = cust._partner_attrs(list(all_pids))
390
+ if fast:
391
+ # C1 fast phase: the slow families stay empty — their columns render blank behind the
392
+ # toolbar's partial indicator until the full build swaps in.
393
+ ar_a, mix_a, sp_a, geo_a, dba_a = {}, {}, {}, {}, {}
394
+ else:
395
+ # The three families the owner asked for (2026-07-27). Each degrades to {} rather than
396
+ # raising: a customer table that cannot render because AR is momentarily unreachable is
397
+ # a worse failure than one with a blank column.
398
+ ar_a = _ar_attrs(t)
399
+ mix_a = _mix_attrs(all_pids, t)
400
+ sp_a = _salesperson_attrs(all_pids, t)
401
+ # W11 (wave-7): coordinates for the Map VIEW — Odoo coords else the geocode cache,
402
+ # read-only reuse (coords_for never geocodes); missing → null lat/lon, and the
403
+ # client's map view discloses the count.
404
+ geo_a = map_mod.coords_for(list(all_pids))
405
+ # C-DBA (2026-08-02): the brand attribute rides the full build only, like the rest.
406
+ dba_a = _dba_attrs(all_pids, t)
407
+ # customers whose only activity sits in the older half of the 24-month cadence window
408
+ # appear in no revenue map — fetch their names directly (else the lapsed cohort, exactly
409
+ # the win-back targets this union exists for, would render as '?')
410
+ unnamed = [pid for pid in all_pids
411
+ if not ((this.get(pid) or {}).get('name') or (last.get(pid) or {}).get('name')
412
+ or (ltm.get(pid) or {}).get('name'))]
413
+ extra_names = ({r['id']: r['name'] for r in
414
+ O.search_read('res.partner', [('id', 'in', unnamed)], ['name'])}
415
+ if unnamed else {})
416
+ rows = []
417
+ for pid in all_pids:
418
+ tv = this.get(pid) or {}
419
+ lv = last.get(pid) or {}
420
+ c = cad.get(pid) or {}
421
+ rev, ly = tv.get('rev', 0.0), lv.get('rev', 0.0)
422
+ gap = c.get('typical_gap_days')
423
+ overdue = c.get('overdue_days')
424
+ aov = c.get('aov', 0.0)
425
+ est = (min(overdue / gap, 3.0) * aov) if (gap and overdue and overdue > 0) else 0.0
426
+ status = ('New' if rev > 0 and ly <= 0 else
427
+ 'Lost' if ly > 0 and rev <= 0 else
428
+ 'Declining' if 0 < rev < ly else
429
+ 'Growing' if rev > 0 else 'Dormant')
430
+ a = attrs.get(pid) or {}
431
+ rows.append({
432
+ 'pid': pid,
433
+ 'customer': tv.get('name') or lv.get('name')
434
+ or (ltm.get(pid) or {}).get('name') or extra_names.get(pid) or '?',
435
+ 'status': status,
436
+ 'agent': a.get('agent', '(none)'), 'city': a.get('city', '(none)'),
437
+ # C-DBA: blank (not '(none)') — a select's blank is its own honest empty state.
438
+ 'dba': dba_a.get(pid, ''),
439
+ 'state': a.get('state', '(none)'),
440
+ # Owner 2026-07-27, the partner-attribute family. `.get(..., default)` is not
441
+ # defensive habit here: search_read returned 1,548 of 1,550 pool pids — two partners
442
+ # in the sales history are archived or deleted — so a customer with NO attrs row is
443
+ # a real case, and it must render as '(none)' rather than KeyError the whole page.
444
+ 'country': a.get('country', '(none)'), 'zip': a.get('zip', '(none)'),
445
+ 'payment_terms': a.get('payment_terms', '(none)'),
446
+ 'customer_since': a.get('customer_since', ''),
447
+ # Row-level datum for the `created_time` field type (wave-5 item 11) — rides every
448
+ # row like pid, no column of its own until a user creates one.
449
+ '_created': a.get('created_at', ''),
450
+ 'tags': a.get('tags', '(none)'), 'pricelist': a.get('pricelist', '(none)'),
451
+ # AR / credit exposure — composed from the already-reconciled ar.py blocks, so the
452
+ # Customer table and the Collections page cannot disagree.
453
+ 'ar_open': (ar_a.get(pid) or {}).get('ar_open', 0.0),
454
+ 'ar_overdue': (ar_a.get(pid) or {}).get('ar_overdue', 0.0),
455
+ 'ar_outstanding': (ar_a.get(pid) or {}).get('ar_outstanding', 0.0),
456
+ 'ar_exposure': (ar_a.get(pid) or {}).get('ar_exposure', 0.0),
457
+ 'days_to_pay': (ar_a.get(pid) or {}).get('days_to_pay'),
458
+ **{k: (ar_a.get(pid) or {}).get(k, 0.0) for k in AR_BUCKET_FIELDS.values()},
459
+ # Product mix, LTM, line grain, ALL-channel.
460
+ 'top_category': (mix_a.get(pid) or {}).get('top_category', '(none)'),
461
+ 'top_category_pct': (mix_a.get(pid) or {}).get('top_category_pct'),
462
+ 'sku_count': (mix_a.get(pid) or {}).get('sku_count', 0),
463
+ 'top_sku': (mix_a.get(pid) or {}).get('top_sku', '(none)'),
464
+ # The ORDER-TAKER, not the agent. Different people, different questions.
465
+ 'salesperson': sp_a.get(pid, '(none)'),
466
+ # W11: nullable — the Map view pins what it can and counts what it cannot.
467
+ 'lat': (geo_a.get(pid) or {}).get('lat'),
468
+ 'lon': (geo_a.get(pid) or {}).get('lon'),
469
+ 'revenue_ytd': rev, 'revenue_ly': ly,
470
+ 'yoy_pct': P.yoy_pct(rev, ly),
471
+ 'at_risk': max(ly - rev, 0.0),
472
+ 'ltm_rev': (ltm.get(pid) or {}).get('rev', 0.0),
473
+ 'orders_24m': c.get('n_orders', 0), 'aov': aov,
474
+ 'last_order': c.get('last_order', ''),
475
+ 'days_since': c.get('days_since'),
476
+ 'typical_gap_days': gap, 'overdue_days': overdue,
477
+ 'est_missed': est,
478
+ })
479
+ for r in rows:
480
+ r['odoo_status'] = 'Active'
481
+ # I13 (owner item 13, contract C6): RETAIN customers that disappear from Odoo.
482
+ # Only the FULL build reconciles the ledger — the fast slice is 100 rows by construction,
483
+ # so letting it write would mark ~1,450 living customers archived on every cold load.
484
+ if not fast:
485
+ rows, total = _reconcile_ledger(rows, total, agent_name, team_id)
486
+ rows.sort(key=lambda r: -(r['revenue_ytd'] + r['revenue_ly']))
487
+ return {'rows': rows, 'total': total}
488
+
489
+
490
+ LEDGER_KEY = 'customer_ledger'
491
+
492
+
493
+ def _reconcile_ledger(rows, total, agent_name, team_id):
494
+ """Remember every customer we have ever seen, and keep serving the ones Odoo stopped
495
+ returning (owner item 13: "if it is deleted, that it goes to an archived tag immediately —
496
+ so stored in our platform even if deleted in Odoo").
497
+
498
+ Why a ledger at all: this pool is built from SALES HISTORY, not from a partner list, so a
499
+ partner deleted in Odoo usually keeps appearing (their order lines remain) — until the day
500
+ the lines go too, when the row would silently vanish along with every note, tag and cohort
501
+ membership attached to it. Nothing else in the app would notice. The ledger is the only
502
+ record that the customer ever existed, which is why it starts recording now rather than
503
+ when the surfacing is finished: history you did not write down is not recoverable later.
504
+
505
+ ⚠ SCOPE-SAFE. The ledger is written ONLY from an UNSCOPED build (no agent, no BU). A scoped
506
+ build legitimately sees a fraction of the book, and reconciling from it would mark every
507
+ customer outside that scope 'Archived' for everyone — the failure would look exactly like
508
+ the data loss this is meant to prevent. Scoped builds READ the ledger and resurrect nothing
509
+ they cannot prove they should see.
510
+ """
511
+ try:
512
+ ledger = dict(store.get(LEDGER_KEY) or {})
513
+ except Exception:
514
+ return rows, total # a ledger we cannot read must not break the table
515
+ today = str(P.today())
516
+ live = {r['pid'] for r in rows}
517
+ unscoped = agent_name is None and team_id is None
518
+
519
+ if unscoped:
520
+ for r in rows:
521
+ e = ledger.get(str(r['pid'])) or {}
522
+ e.update({'name': r['customer'], 'last_seen': today})
523
+ e.setdefault('first_seen', today)
524
+ ledger[str(r['pid'])] = e
525
+
526
+ # Rows the ledger knows and this build did not return. On an unscoped build that means
527
+ # "gone from Odoo"; on a scoped one it usually just means "not in this agent's book", so
528
+ # only the unscoped build may surface them.
529
+ revived = []
530
+ if unscoped:
531
+ for key, e in ledger.items():
532
+ try:
533
+ pid = int(key)
534
+ except (TypeError, ValueError):
535
+ continue
536
+ if pid in live:
537
+ continue
538
+ revived.append({
539
+ 'pid': pid, 'customer': e.get('name') or '?', 'status': 'Dormant',
540
+ 'odoo_status': 'Archived',
541
+ # Everything else is genuinely UNKNOWN now — the record is gone. Blank is the
542
+ # honest rendering; carrying the last-known numbers forward would state figures
543
+ # as current that nothing can reconcile ([[no-unverifiable-aggregates]]).
544
+ 'agent': '(none)', 'city': '(none)', 'state': '(none)', 'country': '(none)',
545
+ 'zip': '(none)', 'payment_terms': '(none)', 'customer_since': '',
546
+ '_created': '', 'tags': '(none)', 'pricelist': '(none)',
547
+ 'ar_open': 0.0, 'ar_overdue': 0.0, 'ar_outstanding': 0.0, 'ar_exposure': 0.0,
548
+ 'days_to_pay': None,
549
+ **{k: 0.0 for k in AR_BUCKET_FIELDS.values()},
550
+ 'top_category': '(none)', 'top_category_pct': None, 'sku_count': 0,
551
+ 'top_sku': '(none)', 'salesperson': '(none)', 'lat': None, 'lon': None,
552
+ 'revenue_ytd': 0.0, 'revenue_ly': 0.0, 'yoy_pct': None, 'at_risk': 0.0,
553
+ 'ltm_rev': 0.0, 'orders_24m': 0, 'aov': 0.0, 'last_order': '',
554
+ 'days_since': None, 'typical_gap_days': None, 'overdue_days': None,
555
+ 'est_missed': 0.0,
556
+ })
557
+ if store.available():
558
+ try:
559
+ store.update(LEDGER_KEY, lambda cur: {**cur, **ledger}, flush='async')
560
+ except Exception:
561
+ pass # recording is best-effort; the table is not
562
+ return rows + revived, total + len(revived)
563
+
564
+
565
+ # ------------------------------------------------------------------ the rule engine
566
+ # A list's formula = [{'field','op','value'}, ...] over FIELDS + a 'sort' key ('-x' = desc).
567
+ # The two templates ARE the old My Day queue formulas, expressed as visible rules.
568
+ #
569
+ # 'note' is a SEED, not the source of truth (owner 2026-07-25). It is what a list reads
570
+ # BEFORE anyone edits it; the moment a user saves a description the stored one wins, and
571
+ # an empty stored description stays empty — the template must never resurrect. That
572
+ # precedence lives in aios_grid.views_from_defs (a saved view replaces its template-derived
573
+ # one wholesale) and is pinned by _qa_grid_view_note.py. Edit the prose here only to change
574
+ # what a NEW user starts with.
575
+ TEMPLATES = {
576
+ 'Call list': {
577
+ 'rules': [{'field': 'orders_24m', 'op': '>=', 'value': 3},
578
+ {'field': 'overdue_days', 'op': '>', 'value': 0},
579
+ {'field': 'days_since', 'op': '<=', 'value': 365}],
580
+ 'sort': '-est_missed',
581
+ 'note': 'Customers overdue against their OWN reorder cadence (3+ orders, a real gap '
582
+ 'pattern, ordered within 365d), ranked by estimated missed revenue.'},
583
+ 'Win-back': {
584
+ 'rules': [{'field': 'revenue_ly', 'op': '>=', 'value': 2000},
585
+ {'field': 'at_risk', 'op': '>', 'value': 0}],
586
+ 'sort': '-at_risk',
587
+ 'note': 'Bought materially last year (over $2,000), down or gone this year, ranked by '
588
+ 'dollars at risk.'},
589
+ }
590
+
591
+
592
+ def _match(row, rule):
593
+ """One rule against one row. None/missing numeric values fail every numeric comparison
594
+ (a customer with no cadence is never 'overdue'); text ops are case-insensitive."""
595
+ field, op, val = rule.get('field'), rule.get('op'), rule.get('value')
596
+ if field not in FIELDS:
597
+ return True # unknown field → rule is inert, not a crash
598
+ v = row.get(field)
599
+ if FIELDS[field][1] == 'text':
600
+ s, q = str(v or '').lower(), str(val or '').lower()
601
+ return q in s if op == 'contains' else s == q
602
+ try:
603
+ v, val = float(v), float(val)
604
+ except (TypeError, ValueError):
605
+ return False
606
+ return {'>=': v >= val, '<=': v <= val, '>': v > val, '<': v < val,
607
+ '=': v == val}.get(op, False)
608
+
609
+
610
+ def apply_rules(rows, rules, sort=None):
611
+ """Filter the pool through a formula (AND of all rules) and apply the list's sort."""
612
+ out = [r for r in rows if all(_match(r, ru) for ru in (rules or []))]
613
+ if sort:
614
+ key = sort.lstrip('-')
615
+ out.sort(key=lambda r: (r.get(key) is None,
616
+ -(r.get(key) or 0) if sort.startswith('-') else (r.get(key) or 0))
617
+ if FIELDS.get(key, ('', 'num'))[1] == 'num'
618
+ else str(r.get(key) or '').lower(), reverse=False)
619
+ return out
620
+
621
+
622
+ # ------------------------------------------------------------------ persistence (HF store)
623
+ def saved_lists(username):
624
+ """{list_name: {'rules','sort','members','note'}} for one user ({} when none / store down)."""
625
+ try:
626
+ return (store.get(KEY) or {}).get(username, {}) or {}
627
+ except Exception:
628
+ return {}
629
+
630
+
631
+ def save_list(username, name, definition):
632
+ def _up(d):
633
+ d.setdefault(username, {})[name] = definition
634
+ return d
635
+ store.update(KEY, _up)
636
+
637
+
638
+ def delete_list(username, name):
639
+ def _up(d):
640
+ (d.get(username) or {}).pop(name, None)
641
+ return d
642
+ store.update(KEY, _up)
643
+
644
+
645
+ # ------------------------------------------------------------------ Airtable-style table workspace
646
+ # This is intentionally separate from `customer_lists`: a LIST owns membership/formula
647
+ # semantics, while a VIEW owns presentation/query state (filters, multi-sort, grouping,
648
+ # visible fields, widths, color, row height). List views can reference the same formula
649
+ # without conflating the two persistence contracts.
650
+ #
651
+ # The store logic MOVED to core/table_store.py 2026-07-27 (the table-page factory): these
652
+ # names are the Customer table's instance of the generic per-object workspace store, kept as
653
+ # module functions so every existing caller (app.py's host loop, QA gates) is untouched.
654
+ # A NEW table object gets its own `core.table_store.make('<its>_table_workspace')`.
655
+ _TSTORE = table_store.make(TABLE_KEY)
656
+
657
+ #: Wave 16 C-TOPIC — the ops OBJECT, for callers that select a table by TOPIC (grid_events'
658
+ #: `EventCtx.table`). The module-level re-exports below stay for every existing caller.
659
+ TABLE_OPS = _TSTORE
660
+
661
+ table_workspace = _TSTORE.workspace
662
+ shared_table_views = _TSTORE.shared_views # wave-9 I17: views shared WITH this viewer
663
+ shared_table_view = _TSTORE.shared_view # ...and one raw, for authorisation only
664
+ save_table_view = _TSTORE.save_view
665
+ delete_table_view = _TSTORE.delete_view
666
+ save_table_field = _TSTORE.save_field
667
+ delete_table_field = _TSTORE.delete_field
668
+ duplicate_table_field = _TSTORE.duplicate_field
669
+ patch_table_overlay = _TSTORE.patch_overlay
670
+ save_table_folders = _TSTORE.save_folders
671
+ save_table_active_view = _TSTORE.save_active_view # owner item 3: last-opened view, per user
672
+ save_table_record_layout = _TSTORE.save_record_layout # 2026-08-02 C-LAYOUT: record-detail order
673
+
674
+
675
+ # ------------------------------------------------------------------ digest compatibility
676
+ def queues(agent_name=None, team_id=None, limit=200):
677
+ """The two canonical morning lists (call list + win-back) for the daily digest email —
678
+ unchanged formulas via the customers module (the digest always uses the CANONICAL templates,
679
+ not a user's edited copy, so every rep's email means the same thing)."""
680
+ pids = agent_pids(agent_name)
681
+ return {
682
+ 'calls': cust.contact_recommendations(team_id=team_id, limit=limit, agent_pids=pids),
683
+ 'risk': cust.at_risk(team_id=team_id, limit=limit, agent_pids=pids),
684
+ }
platform/modules/customers.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/modules/digest.py CHANGED
@@ -1,227 +1,227 @@
1
- """Daily agent digest — "Your book this morning": one email per user, their queue only.
2
-
3
- Delivery reuses the ONE sanctioned Odoo write path (modules/collections_send.py: mail.mail
4
- create through the Office 365 relay) including its SAFE_MODE allow-list — while the guardrail
5
- is ON no digest can leave the allow-list. Content rules (adoption canon, engagement-adoption
6
- brief): ≤10 lines, only non-empty sections, every line deep-links back to the exact row
7
- (?src=digest), and the email is SKIPPED entirely when there is nothing to act on — silence
8
- keeps opens high. Internal mail: no model/res_id (these must NOT land in customer chatter).
9
-
10
- Scheduling: app.py starts one daemon ticker (in the cache_resource singleton) that calls
11
- run_due() every few minutes; run_due sends at most once per (weekday, user) after
12
- DIGEST_UTC_HOUR (default 11:00 UTC = 7am ET in summer), idempotent across Space restarts via
13
- the store key 'digest_log' (empty-skips are logged too, so a quiet day isn't rebuilt all day).
14
-
15
- BU isolation carries into email: a single-BU user's digest is scoped to that BU's team_id.
16
- """
17
- import os
18
- import html as _html
19
- import datetime as dt
20
-
21
- import core.store as store
22
- import core.users as users
23
- import core.links as links
24
- import modules.collections_send as cs
25
- import modules.customer_data as md # call-list / win-back queues (ex-myday, ex-customer_list)
26
- import modules.tasks as tasks
27
- import modules.customers as cust
28
-
29
- KEY = 'digest_log'
30
- UTC_HOUR = int(os.environ.get('DIGEST_UTC_HOUR', '11'))
31
- WEEKDAYS_ONLY = os.environ.get('DIGEST_WEEKDAYS_ONLY', '1').strip().lower() \
32
- not in ('0', 'false', 'no', '')
33
- ENABLED = os.environ.get('DIGEST_ENABLED', '1').strip().lower() not in ('0', 'false', 'no', '')
34
-
35
- _NAVY, _GOLD, _MUTED = '#5B8FD9', '#D9A93A', '#6B7280'
36
-
37
-
38
- def _team_for(user):
39
- """Single-BU users get a digest scoped to that BU — isolation carries into email."""
40
- bus = (user or {}).get('bus', 'all')
41
- if isinstance(bus, (list, tuple)) and len(bus) == 1:
42
- return bus[0]
43
- return None
44
-
45
-
46
- def _m(x):
47
- try:
48
- return f"${float(x):,.0f}"
49
- except (TypeError, ValueError):
50
- return '—'
51
-
52
-
53
- def _line(text, href, meta=''):
54
- t = _html.escape(str(text))
55
- m = f" <span style='color:{_MUTED}'>{_html.escape(str(meta))}</span>" if meta else ''
56
- return (f"<tr><td style='padding:6px 0;border-bottom:1px solid #ECEFF3;font-size:14px'>"
57
- f"<a href='{href}' style='color:{_NAVY};text-decoration:none;font-weight:600'>{t}</a>"
58
- f"{m}</td></tr>")
59
-
60
-
61
- def _sec(title):
62
- return (f"<tr><td style='padding:14px 0 4px;font-size:12.5px;font-weight:700;"
63
- f"color:{_MUTED}'>{_html.escape(title)}</td></tr>")
64
-
65
-
66
- def build(user, t=None):
67
- """(subject, html, n_actionable) for one user. n_actionable == 0 → skip the send."""
68
- t = t or dt.date.today()
69
- uname = user.get('username', '')
70
- agent = user.get('agent')
71
- team_id = _team_for(user)
72
- pids = md.agent_pids(agent)
73
-
74
- today = t.isoformat()
75
- my = [x for x in tasks.for_owner(uname)] if store.available() else []
76
- due = [x for x in my if (x.get('due') or '9999-12-31') <= today]
77
- q = md.queues(agent, team_id, limit=25)
78
- calls, risk = q['calls'][:5], q['risk'][:3]
79
-
80
- # yesterday's orders for the book (one read_group; skip silently on a transport error —
81
- # the digest must never fail because one section couldn't be pulled)
82
- yday = (t - dt.timedelta(days=1)).isoformat()
83
- y_rev = y_orders = 0
84
- try:
85
- rev_map = cust._cust_rev(yday, yday, team_id, pids)
86
- y_rev = sum(v.get('rev', 0.0) for v in rev_map.values())
87
- y_orders = sum(v.get('orders', 0) for v in rev_map.values())
88
- except Exception:
89
- pass
90
-
91
- n = len(due) + len(calls) + len(risk)
92
- if n == 0 and y_orders == 0:
93
- return None, None, 0
94
-
95
- rows = []
96
- if due:
97
- rows.append(_sec(f'Tasks due ({len(due)})'))
98
- for x in due[:5]:
99
- ent = x.get('entity') or {}
100
- href = links.deeplink('customer_data', src='digest')
101
- if ent.get('kind') and ent.get('id') is not None:
102
- href = links.deeplink('customer_data', ent['kind'], ent['id'], src='digest')
103
- meta = f"due {x.get('due') or '—'}"
104
- rows.append(_line(x.get('title', 'Task'), href, meta))
105
- if calls:
106
- rows.append(_sec('Call today — overdue vs their own cadence'))
107
- for r in calls:
108
- href = links.deeplink('customer_data', 'customer', r['pid'], src='digest')
109
- meta = (f"{int(r.get('overdue_days') or 0)}d past cycle · "
110
- f"est. missed {_m(r.get('est_missed'))}")
111
- rows.append(_line(r.get('customer', '?'), href, meta))
112
- if risk:
113
- rows.append(_sec('Win-back — down vs last year'))
114
- for r in risk:
115
- href = links.deeplink('customer_data', 'customer', r['pid'], src='digest')
116
- meta = f"{_m(r.get('at_risk'))} at risk · {r.get('status', '')}"
117
- rows.append(_line(r.get('customer', '?'), href, meta))
118
- if y_orders:
119
- rows.append(_sec('Yesterday'))
120
- # Wave 16: the Sales page is retired (registry row archived) — yesterday's orders now
121
- # link to the Customer grid, the surface that carries the book. A link to an archived
122
- # page would fall through to the landing and read as a broken link.
123
- rows.append(_line(f"{y_orders} orders · {_m(y_rev)}",
124
- links.deeplink('customer_data', src='digest')))
125
-
126
- scope = f"{agent}'s book" if agent else 'the whole book'
127
- list_url = links.deeplink('customer_data', src='digest')
128
- html_body = f"""
129
- <div style='font-family:Calibri,Arial,sans-serif;color:#1A2332;max-width:600px;margin:0 auto'>
130
- <div style='border-top:3px solid {_GOLD};padding:16px 0 4px'>
131
- <div style='font-size:18px;font-weight:700;color:{_NAVY}'>Your book this morning</div>
132
- <div style='font-size:12px;color:{_MUTED}'>{t.strftime('%A, %B %d')} · {scope}</div>
133
- </div>
134
- <table style='width:100%;border-collapse:collapse'>{''.join(rows)}</table>
135
- <div style='padding:16px 0;font-size:13px'>
136
- <a href='{list_url}' style='color:{_NAVY};font-weight:700'>Open your Customer List →</a>
137
- </div>
138
- <div style='font-size:11px;color:{_MUTED};border-top:1px solid #ECEFF3;padding-top:8px'>
139
- Internal daily digest from Loopable.
140
- Sent only on days with something to act on.
141
- </div>
142
- </div>"""
143
- subject = f"Your book this morning — {n} to act on · {t.strftime('%b %d')}"
144
- return subject, html_body, n
145
-
146
-
147
- def send_one(user, odoo=None, override_to=None):
148
- """Build + queue one digest. Returns a short result string (never raises past itself)."""
149
- to = override_to or user.get('email')
150
- if not to:
151
- return 'no-email'
152
- subject, body, n = build(user)
153
- if n == 0:
154
- return 'empty-skip'
155
- if not cs.safe_recipient_ok(to):
156
- return 'safe-mode-blocked'
157
- odoo = odoo or cs.Odoo()
158
- mid = odoo.queue_mail({
159
- 'subject': subject, 'body_html': body,
160
- 'email_to': to, 'email_from': cs.SENDER_DISPLAY, 'reply_to': cs.REPLY_TO,
161
- 'mail_server_id': cs.ROYAL_MAIL_SERVER_ID, 'author_id': cs.ROYAL_AUTHOR_ID,
162
- 'state': 'outgoing', 'auto_delete': False,
163
- # This Odoo runs the custom rt_multi_dba module whose mail-create hook browses
164
- # env[mail.model].browse(mail.res_id) — a mail WITHOUT model/res_id crashes with
165
- # KeyError: False (verified 2026-07-05). Anchor internal mail on the house partner
166
- # (Royal Imports, cs.ROYAL_AUTHOR_ID) so nothing lands in a CUSTOMER's chatter.
167
- 'model': 'res.partner', 'res_id': cs.ROYAL_AUTHOR_ID,
168
- })
169
- return f'queued:{mid}'
170
-
171
-
172
- def run_due(force_user=None, override_to=None):
173
- """The ticker entry point. Normal path: after UTC_HOUR on weekdays, send each active
174
- emailed user their digest at most once per day (dedup via digest_log, including
175
- empty-skips). force_user bypasses the schedule/dedup (the 'Send my digest now' button)
176
- but NEVER the SAFE_MODE guardrail; forced sends are not logged (the 7am run still runs)."""
177
- if not store.available():
178
- return {}
179
- now = dt.datetime.utcnow()
180
- today = now.date().isoformat()
181
- if force_user is None:
182
- if not ENABLED:
183
- return {}
184
- if WEEKDAYS_ONLY and now.weekday() >= 5:
185
- return {}
186
- if now.hour < UTC_HOUR:
187
- return {}
188
- try:
189
- reg = users.registry()
190
- except Exception:
191
- return {}
192
- already = (store.get(KEY) or {}).get(today, {}) if force_user is None else {}
193
- results = {}
194
- odoo = None
195
- for un, u in sorted(reg.items()):
196
- if force_user is not None and un != force_user:
197
- continue
198
- if not u.get('active', True) or not u.get('email'):
199
- continue
200
- if force_user is None and un in already:
201
- continue
202
- pub = users._public(un, u)
203
- try:
204
- if odoo is None:
205
- odoo = cs.Odoo()
206
- results[un] = send_one(pub, odoo, override_to=override_to)
207
- except cs.SafeModeBlocked:
208
- results[un] = 'safe-mode-blocked'
209
- except Exception as e:
210
- results[un] = f'error:{str(e)[:80]}'
211
- if force_user is None and results:
212
- stamp = now.strftime('%H:%M')
213
-
214
- def _rec(d):
215
- day = d.setdefault(today, {})
216
- for un, r in results.items():
217
- day[un] = {'at': stamp, 'result': r}
218
- return d
219
- try:
220
- store.update(KEY, _rec)
221
- except Exception:
222
- pass
223
- return results
224
-
225
-
226
- def today_log():
227
- return (store.get(KEY) or {}).get(dt.date.today().isoformat(), {})
 
1
+ """Daily agent digest — "Your book this morning": one email per user, their queue only.
2
+
3
+ Delivery reuses the ONE sanctioned Odoo write path (modules/collections_send.py: mail.mail
4
+ create through the Office 365 relay) including its SAFE_MODE allow-list — while the guardrail
5
+ is ON no digest can leave the allow-list. Content rules (adoption canon, engagement-adoption
6
+ brief): ≤10 lines, only non-empty sections, every line deep-links back to the exact row
7
+ (?src=digest), and the email is SKIPPED entirely when there is nothing to act on — silence
8
+ keeps opens high. Internal mail: no model/res_id (these must NOT land in customer chatter).
9
+
10
+ Scheduling: app.py starts one daemon ticker (in the cache_resource singleton) that calls
11
+ run_due() every few minutes; run_due sends at most once per (weekday, user) after
12
+ DIGEST_UTC_HOUR (default 11:00 UTC = 7am ET in summer), idempotent across Space restarts via
13
+ the store key 'digest_log' (empty-skips are logged too, so a quiet day isn't rebuilt all day).
14
+
15
+ BU isolation carries into email: a single-BU user's digest is scoped to that BU's team_id.
16
+ """
17
+ import os
18
+ import html as _html
19
+ import datetime as dt
20
+
21
+ import core.store as store
22
+ import core.users as users
23
+ import core.links as links
24
+ import modules.collections_send as cs
25
+ import modules.customer_data as md # call-list / win-back queues (ex-myday, ex-customer_list)
26
+ import modules.tasks as tasks
27
+ import modules.customers as cust
28
+
29
+ KEY = 'digest_log'
30
+ UTC_HOUR = int(os.environ.get('DIGEST_UTC_HOUR', '11'))
31
+ WEEKDAYS_ONLY = os.environ.get('DIGEST_WEEKDAYS_ONLY', '1').strip().lower() \
32
+ not in ('0', 'false', 'no', '')
33
+ ENABLED = os.environ.get('DIGEST_ENABLED', '1').strip().lower() not in ('0', 'false', 'no', '')
34
+
35
+ _NAVY, _GOLD, _MUTED = '#5B8FD9', '#D9A93A', '#6B7280'
36
+
37
+
38
+ def _team_for(user):
39
+ """Single-BU users get a digest scoped to that BU — isolation carries into email."""
40
+ bus = (user or {}).get('bus', 'all')
41
+ if isinstance(bus, (list, tuple)) and len(bus) == 1:
42
+ return bus[0]
43
+ return None
44
+
45
+
46
+ def _m(x):
47
+ try:
48
+ return f"${float(x):,.0f}"
49
+ except (TypeError, ValueError):
50
+ return '—'
51
+
52
+
53
+ def _line(text, href, meta=''):
54
+ t = _html.escape(str(text))
55
+ m = f" <span style='color:{_MUTED}'>{_html.escape(str(meta))}</span>" if meta else ''
56
+ return (f"<tr><td style='padding:6px 0;border-bottom:1px solid #ECEFF3;font-size:14px'>"
57
+ f"<a href='{href}' style='color:{_NAVY};text-decoration:none;font-weight:600'>{t}</a>"
58
+ f"{m}</td></tr>")
59
+
60
+
61
+ def _sec(title):
62
+ return (f"<tr><td style='padding:14px 0 4px;font-size:12.5px;font-weight:700;"
63
+ f"color:{_MUTED}'>{_html.escape(title)}</td></tr>")
64
+
65
+
66
+ def build(user, t=None):
67
+ """(subject, html, n_actionable) for one user. n_actionable == 0 → skip the send."""
68
+ t = t or dt.date.today()
69
+ uname = user.get('username', '')
70
+ agent = user.get('agent')
71
+ team_id = _team_for(user)
72
+ pids = md.agent_pids(agent)
73
+
74
+ today = t.isoformat()
75
+ my = [x for x in tasks.for_owner(uname)] if store.available() else []
76
+ due = [x for x in my if (x.get('due') or '9999-12-31') <= today]
77
+ q = md.queues(agent, team_id, limit=25)
78
+ calls, risk = q['calls'][:5], q['risk'][:3]
79
+
80
+ # yesterday's orders for the book (one read_group; skip silently on a transport error —
81
+ # the digest must never fail because one section couldn't be pulled)
82
+ yday = (t - dt.timedelta(days=1)).isoformat()
83
+ y_rev = y_orders = 0
84
+ try:
85
+ rev_map = cust._cust_rev(yday, yday, team_id, pids)
86
+ y_rev = sum(v.get('rev', 0.0) for v in rev_map.values())
87
+ y_orders = sum(v.get('orders', 0) for v in rev_map.values())
88
+ except Exception:
89
+ pass
90
+
91
+ n = len(due) + len(calls) + len(risk)
92
+ if n == 0 and y_orders == 0:
93
+ return None, None, 0
94
+
95
+ rows = []
96
+ if due:
97
+ rows.append(_sec(f'Tasks due ({len(due)})'))
98
+ for x in due[:5]:
99
+ ent = x.get('entity') or {}
100
+ href = links.deeplink('customer_data', src='digest')
101
+ if ent.get('kind') and ent.get('id') is not None:
102
+ href = links.deeplink('customer_data', ent['kind'], ent['id'], src='digest')
103
+ meta = f"due {x.get('due') or '—'}"
104
+ rows.append(_line(x.get('title', 'Task'), href, meta))
105
+ if calls:
106
+ rows.append(_sec('Call today — overdue vs their own cadence'))
107
+ for r in calls:
108
+ href = links.deeplink('customer_data', 'customer', r['pid'], src='digest')
109
+ meta = (f"{int(r.get('overdue_days') or 0)}d past cycle · "
110
+ f"est. missed {_m(r.get('est_missed'))}")
111
+ rows.append(_line(r.get('customer', '?'), href, meta))
112
+ if risk:
113
+ rows.append(_sec('Win-back — down vs last year'))
114
+ for r in risk:
115
+ href = links.deeplink('customer_data', 'customer', r['pid'], src='digest')
116
+ meta = f"{_m(r.get('at_risk'))} at risk · {r.get('status', '')}"
117
+ rows.append(_line(r.get('customer', '?'), href, meta))
118
+ if y_orders:
119
+ rows.append(_sec('Yesterday'))
120
+ # Wave 16: the Sales page is retired (registry row archived) — yesterday's orders now
121
+ # link to the Customer grid, the surface that carries the book. A link to an archived
122
+ # page would fall through to the landing and read as a broken link.
123
+ rows.append(_line(f"{y_orders} orders · {_m(y_rev)}",
124
+ links.deeplink('customer_data', src='digest')))
125
+
126
+ scope = f"{agent}'s book" if agent else 'the whole book'
127
+ list_url = links.deeplink('customer_data', src='digest')
128
+ html_body = f"""
129
+ <div style='font-family:Calibri,Arial,sans-serif;color:#1A2332;max-width:600px;margin:0 auto'>
130
+ <div style='border-top:3px solid {_GOLD};padding:16px 0 4px'>
131
+ <div style='font-size:18px;font-weight:700;color:{_NAVY}'>Your book this morning</div>
132
+ <div style='font-size:12px;color:{_MUTED}'>{t.strftime('%A, %B %d')} · {scope}</div>
133
+ </div>
134
+ <table style='width:100%;border-collapse:collapse'>{''.join(rows)}</table>
135
+ <div style='padding:16px 0;font-size:13px'>
136
+ <a href='{list_url}' style='color:{_NAVY};font-weight:700'>Open your Customer List →</a>
137
+ </div>
138
+ <div style='font-size:11px;color:{_MUTED};border-top:1px solid #ECEFF3;padding-top:8px'>
139
+ Internal daily digest from Loopable.
140
+ Sent only on days with something to act on.
141
+ </div>
142
+ </div>"""
143
+ subject = f"Your book this morning — {n} to act on · {t.strftime('%b %d')}"
144
+ return subject, html_body, n
145
+
146
+
147
+ def send_one(user, odoo=None, override_to=None):
148
+ """Build + queue one digest. Returns a short result string (never raises past itself)."""
149
+ to = override_to or user.get('email')
150
+ if not to:
151
+ return 'no-email'
152
+ subject, body, n = build(user)
153
+ if n == 0:
154
+ return 'empty-skip'
155
+ if not cs.safe_recipient_ok(to):
156
+ return 'safe-mode-blocked'
157
+ odoo = odoo or cs.Odoo()
158
+ mid = odoo.queue_mail({
159
+ 'subject': subject, 'body_html': body,
160
+ 'email_to': to, 'email_from': cs.SENDER_DISPLAY, 'reply_to': cs.REPLY_TO,
161
+ 'mail_server_id': cs.ROYAL_MAIL_SERVER_ID, 'author_id': cs.ROYAL_AUTHOR_ID,
162
+ 'state': 'outgoing', 'auto_delete': False,
163
+ # This Odoo runs the custom rt_multi_dba module whose mail-create hook browses
164
+ # env[mail.model].browse(mail.res_id) — a mail WITHOUT model/res_id crashes with
165
+ # KeyError: False (verified 2026-07-05). Anchor internal mail on the house partner
166
+ # (Royal Imports, cs.ROYAL_AUTHOR_ID) so nothing lands in a CUSTOMER's chatter.
167
+ 'model': 'res.partner', 'res_id': cs.ROYAL_AUTHOR_ID,
168
+ })
169
+ return f'queued:{mid}'
170
+
171
+
172
+ def run_due(force_user=None, override_to=None):
173
+ """The ticker entry point. Normal path: after UTC_HOUR on weekdays, send each active
174
+ emailed user their digest at most once per day (dedup via digest_log, including
175
+ empty-skips). force_user bypasses the schedule/dedup (the 'Send my digest now' button)
176
+ but NEVER the SAFE_MODE guardrail; forced sends are not logged (the 7am run still runs)."""
177
+ if not store.available():
178
+ return {}
179
+ now = dt.datetime.utcnow()
180
+ today = now.date().isoformat()
181
+ if force_user is None:
182
+ if not ENABLED:
183
+ return {}
184
+ if WEEKDAYS_ONLY and now.weekday() >= 5:
185
+ return {}
186
+ if now.hour < UTC_HOUR:
187
+ return {}
188
+ try:
189
+ reg = users.registry()
190
+ except Exception:
191
+ return {}
192
+ already = (store.get(KEY) or {}).get(today, {}) if force_user is None else {}
193
+ results = {}
194
+ odoo = None
195
+ for un, u in sorted(reg.items()):
196
+ if force_user is not None and un != force_user:
197
+ continue
198
+ if not u.get('active', True) or not u.get('email'):
199
+ continue
200
+ if force_user is None and un in already:
201
+ continue
202
+ pub = users._public(un, u)
203
+ try:
204
+ if odoo is None:
205
+ odoo = cs.Odoo()
206
+ results[un] = send_one(pub, odoo, override_to=override_to)
207
+ except cs.SafeModeBlocked:
208
+ results[un] = 'safe-mode-blocked'
209
+ except Exception as e:
210
+ results[un] = f'error:{str(e)[:80]}'
211
+ if force_user is None and results:
212
+ stamp = now.strftime('%H:%M')
213
+
214
+ def _rec(d):
215
+ day = d.setdefault(today, {})
216
+ for un, r in results.items():
217
+ day[un] = {'at': stamp, 'result': r}
218
+ return d
219
+ try:
220
+ store.update(KEY, _rec)
221
+ except Exception:
222
+ pass
223
+ return results
224
+
225
+
226
+ def today_log():
227
+ return (store.get(KEY) or {}).get(dt.date.today().isoformat(), {})
platform/modules/pricing.py CHANGED
@@ -1,395 +1,395 @@
1
- """Pricing module — per-SKU economics for pricing decisions.
2
-
3
- For every SKU (LTM): units, revenue, unit cost, avg selling price, GROSS margin $/%, markup %
4
- (= revenue/COGS − 1), and a fully-loaded NET margin after allocating operating expenses to the SKU.
5
-
6
- COST-TO-SKU ALLOCATION (tiered, channel-scoped, SELECTABLE driver):
7
- COGS is per-SKU already (Odoo Margin module). Operating expenses live per analytic
8
- (Fisch/Royal/Amazon/HQ). Each channel pool = its analytic opex; we distribute it across that
9
- channel's SKUs by a chosen DRIVER, and HQ overhead across all SKUs by the same driver:
10
- pool_C = channel_rate_C * Σ(channel revenue) # rate-based magnitude (robust to Amazon
11
- opex(sku)= Σ_C pool_C * driverC(sku)/Σ driverC + pool_HQ * driver(sku)/Σ driver
12
- # invoice revenue not fully visible per-SKU)
13
- Driver options:
14
- - 'cogs' : cost-weighted (default) — higher-cost items bear more overhead.
15
- - 'cbm' : physical size — units × volume (m³); BIGGER/bulkier SKUs absorb more (storage/freight/
16
- FBA scale with size). Volume is on ~24% of SKUs in Odoo; the rest are imputed at the
17
- category (else global) median volume. Weight is unusable (~0% populated).
18
- - 'revenue': % of sale (reduces to channel_rate × revenue).
19
- - 'units' : per-unit.
20
- net(sku) = gross_margin(sku) − opex(sku). NET is a decision estimate; GROSS (margin/markup) is exact.
21
-
22
- Brand-filterable: team_id None = all channels (incl Amazon); 5 = Fisch, 6 = Royal (wholesale scope).
23
- READ-ONLY.
24
- """
25
- import sys
26
- import statistics
27
- import datetime as dt
28
- from pathlib import Path
29
- sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
30
- import core.odoo as O
31
- import core.periods as P
32
-
33
- _ANA = {1: 'Fisch', 2: 'Royal', 3: 'Amazon', 4: 'HQ', 5: 'Internal'}
34
- # Period: calendar 2025 (not LTM) so the pricing P&L ties to the 2025 P&L / GL / vendor analyses.
35
- _FY = ('2025-01-01', '2025-12-31')
36
- _FY_LABEL = 'FY2025'
37
- DRIVERS = {'cogs': 'COGS (cost-weighted)', 'cbm': 'CBM / volume (size)', 'revenue': 'Revenue', 'units': 'Units'}
38
- _DRIVER_SHORT = {'cogs': 'COGS', 'cbm': 'CBM', 'units': 'Units', 'revenue': 'Revenue'}
39
-
40
-
41
- def has_driver_data(row, driver):
42
- """True if the SKU has REAL (not imputed / defaulted) data for the chosen allocation driver."""
43
- if driver == 'cbm':
44
- return bool(row.get('vol_known')) # volume on file in Odoo (else category-median imputed)
45
- if driver == 'cogs':
46
- return (row.get('cogs') or 0) > 0 # has a real product cost (else uncosted)
47
- if driver == 'units':
48
- return (row.get('units') or 0) > 0
49
- return (row.get('revenue') or 0) > 0 # revenue: always present in the table
50
-
51
-
52
- def rates(t=None):
53
- """LTM opex rates per channel + HQ, from the analytic ledger. {channel: opex/revenue}."""
54
- lf, lt = _FY
55
- o = O.get_odoo()
56
- rev = {_ANA.get(O.m2o_id(r['account_id'])): (r['amount'] or 0.0) for r in o.read_group(
57
- 'account.analytic.line', [('date', '>=', lf), ('date', '<=', lt),
58
- ('general_account_id.account_type', 'in', ['income', 'income_other'])],
59
- ['amount:sum', 'account_id'], ['account_id'], lazy=False) if O.m2o_id(r['account_id']) in _ANA}
60
- opx = {_ANA.get(O.m2o_id(r['account_id'])): -(r['amount'] or 0.0) for r in o.read_group(
61
- 'account.analytic.line', [('date', '>=', lf), ('date', '<=', lt),
62
- ('general_account_id.account_type', 'in', ['expense', 'expense_depreciation'])],
63
- ['amount:sum', 'account_id'], ['account_id'], lazy=False) if O.m2o_id(r['account_id']) in _ANA}
64
- tot_rev = sum(rev.values()) or 1.0
65
- def rate(k):
66
- return (opx.get(k, 0.0) / rev[k]) if rev.get(k) else 0.0
67
- return {'Fisch': rate('Fisch'), 'Royal': rate('Royal'), 'Amazon': rate('Amazon'),
68
- 'HQ': (opx.get('HQ', 0.0) + opx.get('Internal', 0.0)) / tot_rev,
69
- 'window': _FY_LABEL, '_rev': rev, '_opex': opx, '_tot_rev': tot_rev}
70
-
71
-
72
- def _meta(pids):
73
- prods = O.search_read('product.product', [('id', 'in', pids), ('active', 'in', [True, False])],
74
- ['default_code', 'name', 'categ_id', 'volume'])
75
- cats = {}
76
- for c in O.search_read('product.category', [], ['id', 'complete_name']):
77
- parts = [x.strip() for x in (c['complete_name'] or '').split('/')]
78
- cats[c['id']] = parts[1] if len(parts) >= 2 else (parts[0] if parts else None)
79
- # median volume per category + global, to impute the SKUs without volume on file
80
- by_cat = {}
81
- allv = []
82
- for p in prods:
83
- v = p.get('volume') or 0.0
84
- if v > 0:
85
- allv.append(v)
86
- by_cat.setdefault(O.m2o_id(p.get('categ_id')), []).append(v)
87
- gmed = statistics.median(allv) if allv else 0.0
88
- cmed = {c: statistics.median(vs) for c, vs in by_cat.items()}
89
- out = {}
90
- for p in prods:
91
- cid = O.m2o_id(p.get('categ_id'))
92
- v = p.get('volume') or 0.0
93
- out[p['id']] = {'sku': p.get('default_code') or f"#{p['id']}", 'name': p.get('name') or '',
94
- 'category': cats.get(cid) or '(uncategorized)',
95
- 'volume': v, 'vol_used': v if v > 0 else (cmed.get(cid) or gmed), 'vol_known': v > 0}
96
- return out
97
-
98
-
99
- def _build(team_id=None, driver='cogs', t=None):
100
- t = t or P.today()
101
- lf, lt = _FY
102
- o = O.get_odoo()
103
- rt = rates(t)
104
- gift = list(O.excluded_partner_ids())
105
- win = [('order_id.date_order', '>=', f'{lf} 00:00:00'), ('order_id.date_order', '<=', f'{lt} 23:59:59')]
106
- sbase = [('order_id.state', 'in', ['sale', 'done']), ('product_id.type', '!=', 'service')] + win
107
-
108
- def by_prod(extra, fields):
109
- return {O.m2o_id(r['product_id']): r for r in o.read_group('sale.order.line', sbase + extra,
110
- fields + ['product_id'], ['product_id'], lazy=False) if r.get('product_id')}
111
-
112
- if team_id in (5, 6): # one wholesale BU
113
- allc = by_prod([('order_id.team_id', '=', team_id), ('order_partner_id', 'not in', gift)],
114
- ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'])
115
- chan_of = {pid: ('Fisch' if team_id == 5 else 'Royal') for pid in allc}
116
- chan_rev = {pid: {('Fisch' if team_id == 5 else 'Royal'): (a['price_subtotal'] or 0.0)} for pid, a in allc.items()}
117
- else: # all channels
118
- allc = by_prod([], ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'])
119
- rf = by_prod([('order_id.team_id', '=', 5), ('order_partner_id', 'not in', gift)], ['price_subtotal:sum'])
120
- rr = by_prod([('order_id.team_id', '=', 6), ('order_partner_id', 'not in', gift)], ['price_subtotal:sum'])
121
- ra = by_prod([('order_partner_id', 'in', gift)] if gift else [('id', '=', -1)], ['price_subtotal:sum'])
122
- chan_rev = {pid: {'Fisch': (rf.get(pid, {}).get('price_subtotal') or 0.0),
123
- 'Royal': (rr.get(pid, {}).get('price_subtotal') or 0.0),
124
- 'Amazon': (ra.get(pid, {}).get('price_subtotal') or 0.0)} for pid in allc}
125
-
126
- meta = _meta(list(allc))
127
- # assemble per-SKU base
128
- skus = {}
129
- for pid, a in allc.items():
130
- rev = a['price_subtotal'] or 0.0
131
- if rev <= 0:
132
- continue
133
- units = a['product_uom_qty'] or 0.0
134
- gm = a['margin'] or 0.0
135
- cogs = rev - gm
136
- m = meta.get(pid, {})
137
- skus[pid] = {'rev': rev, 'units': units, 'gm': gm, 'cogs': cogs, 'cr': chan_rev.get(pid, {}),
138
- 'cbm': units * (m.get('vol_used') or 0.0), 'm': m}
139
-
140
- # driver value per SKU + its apportionment to each channel (by revenue mix)
141
- def dval(s):
142
- return {'cogs': s['cogs'], 'cbm': s['cbm'], 'units': s['units'], 'revenue': s['rev']}.get(driver, s['cogs'])
143
- chans = ['Fisch', 'Royal', 'Amazon']
144
- pool = {C: rt[C] * sum(s['cr'].get(C, 0.0) for s in skus.values()) for C in chans} # rate × visible channel rev
145
- pool_hq = rt['HQ'] * sum(s['rev'] for s in skus.values())
146
- sumdrvC = {C: sum(dval(s) * (s['cr'].get(C, 0.0) / s['rev']) for s in skus.values() if s['rev']) for C in chans}
147
- sumdrv = sum(dval(s) for s in skus.values()) or 1.0
148
-
149
- rows = []
150
- for pid, s in skus.items():
151
- load = pool_hq * (dval(s) / sumdrv)
152
- for C in chans:
153
- if sumdrvC[C] > 0 and s['rev']:
154
- load += pool[C] * (dval(s) * (s['cr'].get(C, 0.0) / s['rev'])) / sumdrvC[C]
155
- rev, gm, cogs, units = s['rev'], s['gm'], s['cogs'], s['units']
156
- net = gm - load
157
- m = s['m']
158
- rows.append({
159
- 'product_id': pid, 'sku': m.get('sku', f'#{pid}'), 'code': m.get('sku', f'#{pid}'),
160
- 'product': m.get('name', ''), 'name': m.get('name', ''), 'category': m.get('category', '(uncategorized)'),
161
- 'units': units, 'revenue': rev, 'cogs': cogs,
162
- 'unit_cost': (cogs / units) if units else 0.0, 'avg_price': (rev / units) if units else 0.0,
163
- 'cbm_unit': m.get('volume', 0.0), 'cbm_used': m.get('vol_used', 0.0), 'cbm_total': s['cbm'],
164
- 'vol_known': m.get('vol_known', False), 'cbm_src': ('on file' if m.get('vol_known') else 'imputed'),
165
- 'gm_dollars': gm, 'gm_pct': (gm / rev * 100) if rev else 0.0,
166
- 'markup_pct': (gm / cogs * 100) if cogs > 0 else None,
167
- 'opex_load': load, 'opex_pct': (load / rev * 100) if rev else 0.0,
168
- 'net_dollars': net, 'net_pct': (net / rev * 100) if rev else 0.0,
169
- 'status': ('Below cost' if gm < 0 else 'Net-negative' if net < 0 else 'Thin (<10% net)' if (net / rev) < 0.10 else 'Healthy'),
170
- })
171
- rows.sort(key=lambda r: -r['revenue'])
172
- return rows, rt
173
-
174
-
175
- def table(team_id=None, driver='cogs', t=None):
176
- return _build(team_id, driver, t)[0]
177
-
178
-
179
- def summary(team_id=None, driver='cogs', t=None, built=None):
180
- rows, rt = built or _build(team_id, driver, t)
181
- rev = sum(r['revenue'] for r in rows)
182
- gm = sum(r['gm_dollars'] for r in rows)
183
- net = sum(r['net_dollars'] for r in rows)
184
- mk = [r['markup_pct'] for r in rows if r['markup_pct'] is not None]
185
- vol_known = sum(1 for r in rows if r['vol_known'])
186
- driver_known = sum(1 for r in rows if has_driver_data(r, driver))
187
- return {
188
- 'window': rt['window'], 'skus': len(rows), 'revenue': rev, 'driver': driver, 'driver_label': DRIVERS.get(driver, driver),
189
- 'driver_short': _DRIVER_SHORT.get(driver, driver),
190
- 'gm_dollars': gm, 'gm_pct': (gm / rev * 100) if rev else 0.0,
191
- 'net_dollars': net, 'net_pct': (net / rev * 100) if rev else 0.0,
192
- 'avg_markup': (sum(mk) / len(mk)) if mk else 0.0,
193
- 'below_cost_skus': sum(1 for r in rows if r['gm_dollars'] < 0),
194
- 'net_negative_skus': sum(1 for r in rows if r['net_dollars'] < 0),
195
- 'net_negative_rev': sum(r['revenue'] for r in rows if r['net_dollars'] < 0),
196
- 'vol_coverage': (vol_known / len(rows) * 100) if rows else 0.0,
197
- 'driver_known': driver_known, 'driver_coverage': (driver_known / len(rows) * 100) if rows else 0.0,
198
- 'rates': {k: round(rt[k] * 100, 1) for k in ('Fisch', 'Royal', 'Amazon', 'HQ')},
199
- }
200
-
201
-
202
- def cost_drift(t=None, team_id=None):
203
- """Replacement-cost drift from confirmed PO lines: the net price paid per unit in the LAST
204
- 12 months vs the 12 months BEFORE, per product, joined to what the SELL price did over the
205
- same two windows (BU-scoped sell side; cost is company-wide). ERODING = cost up >5% while
206
- the selling price followed by less than half — the margin leaks silently until repriced.
207
- cost_impact_12m = (cost_now − cost_prior) × units sold last 12m (the annualized $ at stake)."""
208
- t = t or P.today()
209
- d24 = (t - dt.timedelta(days=730)).isoformat()
210
- d12 = (t - dt.timedelta(days=365)).isoformat()
211
- lines = O.search_read('purchase.order.line',
212
- [('order_id.state', 'in', ('purchase', 'done')),
213
- ('order_id.date_order', '>=', d24),
214
- ('product_qty', '>', 0), ('price_unit', '>', 0)],
215
- ['product_id', 'product_qty', 'product_uom_qty', 'price_subtotal',
216
- 'order_id'])
217
- oids = list({O.m2o_id(l['order_id']) for l in lines if l.get('order_id')})
218
- od = {}
219
- for i in range(0, len(oids), 5000):
220
- for o_ in O.search_read('purchase.order', [('id', 'in', oids[i:i + 5000])], ['date_order']):
221
- od[o_['id']] = str(o_['date_order'])[:10]
222
- cur, prior = {}, {} # pid -> [net spend, qty]
223
- for l in lines:
224
- pid = O.m2o_id(l.get('product_id'))
225
- d = od.get(O.m2o_id(l.get('order_id')))
226
- if not pid or not d:
227
- continue
228
- e = (cur if d >= d12 else prior).setdefault(pid, [0.0, 0.0])
229
- e[0] += l.get('price_subtotal') or 0.0
230
- # BASE-UoM qty, so a piece→case purchase-UoM switch doesn't fake a price spike
231
- e[1] += l.get('product_uom_qty') or l.get('product_qty') or 0.0
232
- both = [p for p in cur if p in prior and prior[p][1] > 0 and cur[p][1] > 0]
233
-
234
- def _sell(a, b_):
235
- out = {}
236
- for g in O.read_group('sale.order.line', O.sale_line_domain(a, b_, team_id),
237
- ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'],
238
- lazy=False):
239
- pid = O.m2o_id(g.get('product_id'))
240
- if pid:
241
- out[pid] = (g.get('price_subtotal') or 0.0, g.get('product_uom_qty') or 0.0)
242
- return out
243
- s_now, s_pri = _sell(d12, t.isoformat()), _sell(d24, d12)
244
-
245
- meta = {}
246
- for i in range(0, len(both), 5000):
247
- for p in O.search_read('product.product',
248
- [('id', 'in', both[i:i + 5000]), ('active', 'in', [True, False])],
249
- ['default_code', 'name', 'standard_price']):
250
- meta[p['id']] = p
251
- rows = []
252
- for pid in both:
253
- c_now, c_pri = cur[pid][0] / cur[pid][1], prior[pid][0] / prior[pid][1]
254
- if c_pri <= 0:
255
- continue
256
- rn, qn = s_now.get(pid, (0.0, 0.0))
257
- rp, qp = s_pri.get(pid, (0.0, 0.0))
258
- asp_now = rn / qn if qn else None
259
- asp_pri = rp / qp if qp else None
260
- p = meta.get(pid, {})
261
- rows.append({'pid': pid, 'code': (p.get('default_code') or '').strip(),
262
- 'product': p.get('name') or '',
263
- 'cost_prior': c_pri, 'cost_now': c_now,
264
- 'drift_pct': (c_now / c_pri - 1) * 100,
265
- 'buy_qty_12m': cur[pid][1], 'std_cost': p.get('standard_price') or 0.0,
266
- 'asp_now': asp_now, 'asp_prior': asp_pri,
267
- 'price_chg_pct': ((asp_now / asp_pri - 1) * 100)
268
- if (asp_now and asp_pri) else None,
269
- 'units_12m': qn, 'cost_impact_12m': (c_now - c_pri) * qn,
270
- 'gm_pct_now': ((asp_now - c_now) / asp_now * 100) if asp_now else None})
271
- # a >3x (or <1/3) per-base-unit move is a UoM/master-data break, not market inflation —
272
- # surfaced as its own data-quality list so it can't pollute the erosion signal
273
- breaks = sorted((r for r in rows if not (1 / 3 <= (r['cost_now'] / r['cost_prior']) <= 3)),
274
- key=lambda r: -abs(r['drift_pct']))
275
- broken = {r['pid'] for r in breaks}
276
- eroding = sorted((r for r in rows
277
- if r['pid'] not in broken
278
- and r['drift_pct'] > 5 and (r['units_12m'] or 0) > 0
279
- and (r['price_chg_pct'] is None or r['price_chg_pct'] < r['drift_pct'] / 2)),
280
- key=lambda r: -(r['cost_impact_12m'] or 0))
281
- improving = sorted((r for r in rows if r['pid'] not in broken
282
- and r['drift_pct'] < -5 and (r['units_12m'] or 0) > 0),
283
- key=lambda r: r['cost_impact_12m'])
284
- return {'rows': rows, 'eroding': eroding, 'improving': improving, 'breaks': breaks,
285
- 'n_products': len(rows),
286
- 'erosion_total': sum(r['cost_impact_12m'] for r in eroding),
287
- 'tailwind_total': sum(r['cost_impact_12m'] for r in improving),
288
- '_cur_spend': sum(e[0] for e in cur.values()),
289
- '_cur_domain_from': d12, 'windows': (d24, d12, t.isoformat())}
290
-
291
-
292
- def cost_drift_validate(cd, t=None):
293
- """The 12m PO spend our per-product weighting is built on == the server-side sum over the
294
- identical domain (two independent aggregation paths)."""
295
- t = t or P.today()
296
- dom = [('order_id.state', 'in', ('purchase', 'done')),
297
- ('order_id.date_order', '>=', cd['_cur_domain_from']),
298
- ('product_qty', '>', 0), ('price_unit', '>', 0)]
299
- srv = O.sum_field('purchase.order.line', dom, 'price_subtotal')
300
- return [{'check': 'Cost drift: Σ(per-product 12m PO spend) == server Σ(line subtotal), same domain',
301
- 'a': round(cd['_cur_spend'], 2), 'b': round(srv, 2),
302
- 'gap': round(cd['_cur_spend'] - srv, 2),
303
- 'ok': abs(cd['_cur_spend'] - srv) <= max(1.0, abs(srv) * 0.001)}]
304
-
305
-
306
- def _pnl_entities(t=None):
307
- """Actual LTM P&L per analytic entity (the basis the Management P&L is built from)."""
308
- lf, lt = _FY
309
- o = O.get_odoo()
310
- def grp(types):
311
- return {_ANA.get(O.m2o_id(r['account_id'])): (r['amount'] or 0.0) for r in o.read_group(
312
- 'account.analytic.line', [('date', '>=', lf), ('date', '<=', lt),
313
- ('general_account_id.account_type', 'in', types)], ['amount:sum', 'account_id'], ['account_id'], lazy=False)
314
- if O.m2o_id(r['account_id']) in _ANA}
315
- inc, cog, opx = grp(['income', 'income_other']), grp(['expense_direct_cost']), grp(['expense', 'expense_depreciation'])
316
- return {k: {'revenue': inc.get(k, 0.0), 'cogs': -cog.get(k, 0.0), 'opex': -opx.get(k, 0.0)} for k in _ANA.values()}
317
-
318
-
319
- def gl_pnl(t=None):
320
- """Actual LTM P&L straight from the posted GL (the official books) — the independent reconciliation
321
- target. Revenue − COGS − Opex = Net."""
322
- lf, lt = _FY
323
- def s(types):
324
- return O.sum_field('account.move.line', [('parent_state', '=', 'posted'), ('date', '>=', lf),
325
- ('date', '<=', lt), ('account_id.account_type', 'in', types)], 'balance')
326
- rev = -s(['income', 'income_other']) # income is credit → flip to positive
327
- cogs = s(['expense_direct_cost'])
328
- opex = s(['expense', 'expense_depreciation'])
329
- return {'revenue': rev, 'cogs': cogs, 'gm': rev - cogs, 'opex': opex, 'net': rev - cogs - opex}
330
-
331
-
332
- def reconcile(team_id=None, driver='cogs', t=None, built=None):
333
- """Bridge the per-SKU P&L to the ACTUAL P&L: attributed SKUs + unattributed (Amazon-direct, not
334
- booked per-SKU) = the displayed P&L. Ties by construction; the unattributed line is the residual."""
335
- rows, rt = built or _build(team_id, driver, t)
336
- ent = _pnl_entities(t)
337
- tot_rev = sum(e['revenue'] for e in ent.values()) or 1.0
338
- hq_rate = (ent['HQ']['opex'] + ent['Internal']['opex']) / tot_rev
339
- if team_id in (5, 6):
340
- k = 'Fisch' if team_id == 5 else 'Royal'
341
- rev_t, cogs_t = ent[k]['revenue'], ent[k]['cogs']
342
- opex_t = ent[k]['opex'] + hq_rate * ent[k]['revenue'] # BU opex + its share of HQ
343
- else:
344
- rev_t = sum(e['revenue'] for e in ent.values())
345
- cogs_t = sum(e['cogs'] for e in ent.values())
346
- opex_t = sum(e['opex'] for e in ent.values()) # all opex incl HQ
347
- net_t = rev_t - cogs_t - opex_t
348
- rev_s = sum(r['revenue'] for r in rows)
349
- gm_s = sum(r['gm_dollars'] for r in rows)
350
- cogs_s, opex_s = rev_s - gm_s, sum(r['opex_load'] for r in rows)
351
- net_s = gm_s - opex_s
352
- una = {'revenue': rev_t - rev_s, 'cogs': cogs_t - cogs_s, 'gm': (rev_t - rev_s) - (cogs_t - cogs_s),
353
- 'opex': opex_t - opex_s, 'net': net_t - net_s}
354
- pnl = {'revenue': rev_t, 'cogs': cogs_t, 'gm': rev_t - cogs_t, 'opex': opex_t, 'net': net_t}
355
- sku = {'revenue': rev_s, 'cogs': cogs_s, 'gm': gm_s, 'opex': opex_s, 'net': net_s}
356
- return {'pnl': pnl, 'sku': sku, 'unattrib': una, 'ties': abs((net_s + una['net']) - net_t) < 1.0}
357
-
358
-
359
- def page_data(team_id=None, driver='cogs', t=None):
360
- """One build → rows + summary + reconciliation + validation (avoids rebuilding 4×)."""
361
- built = _build(team_id, driver, t)
362
- return {'rows': built[0], 'summary': summary(team_id, driver, t, built=built),
363
- 'reconcile': reconcile(team_id, driver, t, built=built),
364
- 'validation': validate(team_id, driver, t, built=built)}
365
-
366
-
367
- def validate(team_id=None, driver='cogs', t=None, built=None):
368
- rows, rt = built or _build(team_id, driver, t)
369
- checks = []
370
- lf, lt = _FY
371
- gift = list(O.excluded_partner_ids())
372
- dom = [('order_id.state', 'in', ['sale', 'done']), ('order_id.date_order', '>=', f'{lf} 00:00:00'),
373
- ('order_id.date_order', '<=', f'{lt} 23:59:59'), ('product_id.type', '!=', 'service'), ('price_subtotal', '>', 0)]
374
- if team_id in (5, 6):
375
- dom += [('order_id.team_id', '=', team_id), ('order_partner_id', 'not in', gift)]
376
- indep = O.sum_field('sale.order.line', dom, 'price_subtotal')
377
- ours = sum(r['revenue'] for r in rows)
378
- checks.append({'check': 'Σ per-SKU revenue == scoped FY2025 (positive lines)', 'a': round(ours, 0),
379
- 'b': round(indep, 0), 'gap': round(ours - indep, 0), 'ok': abs(ours - indep) <= max(50.0, indep * 0.01)})
380
- if rows:
381
- s = rows[0]
382
- checks.append({'check': f"GM == revenue−COGS (sample {s['sku']})", 'a': round(s['gm_dollars'], 2),
383
- 'b': round(s['revenue'] - s['cogs'], 2), 'gap': round(s['gm_dollars'] - (s['revenue'] - s['cogs']), 2),
384
- 'ok': abs(s['gm_dollars'] - (s['revenue'] - s['cogs'])) <= 0.5})
385
- # THE finance check — per-SKU P&L reconciles to the actual P&L (and that target ties to the posted GL)
386
- rc = reconcile(team_id, driver, t, built=(rows, rt))
387
- checks.append({'check': 'Attributed SKUs + unattributed net == P&L net (reconciles)',
388
- 'a': round(rc['sku']['net'] + rc['unattrib']['net'], 0), 'b': round(rc['pnl']['net'], 0),
389
- 'gap': round(rc['sku']['net'] + rc['unattrib']['net'] - rc['pnl']['net'], 0), 'ok': rc['ties']})
390
- if team_id is None:
391
- gl = gl_pnl(t)
392
- checks.append({'check': 'P&L target (analytic) == posted GL net (actual books)',
393
- 'a': round(rc['pnl']['net'], 0), 'b': round(gl['net'], 0),
394
- 'gap': round(rc['pnl']['net'] - gl['net'], 0), 'ok': abs(rc['pnl']['net'] - gl['net']) <= 2.0})
395
- return checks
 
1
+ """Pricing module — per-SKU economics for pricing decisions.
2
+
3
+ For every SKU (LTM): units, revenue, unit cost, avg selling price, GROSS margin $/%, markup %
4
+ (= revenue/COGS − 1), and a fully-loaded NET margin after allocating operating expenses to the SKU.
5
+
6
+ COST-TO-SKU ALLOCATION (tiered, channel-scoped, SELECTABLE driver):
7
+ COGS is per-SKU already (Odoo Margin module). Operating expenses live per analytic
8
+ (Fisch/Royal/Amazon/HQ). Each channel pool = its analytic opex; we distribute it across that
9
+ channel's SKUs by a chosen DRIVER, and HQ overhead across all SKUs by the same driver:
10
+ pool_C = channel_rate_C * Σ(channel revenue) # rate-based magnitude (robust to Amazon
11
+ opex(sku)= Σ_C pool_C * driverC(sku)/Σ driverC + pool_HQ * driver(sku)/Σ driver
12
+ # invoice revenue not fully visible per-SKU)
13
+ Driver options:
14
+ - 'cogs' : cost-weighted (default) — higher-cost items bear more overhead.
15
+ - 'cbm' : physical size — units × volume (m³); BIGGER/bulkier SKUs absorb more (storage/freight/
16
+ FBA scale with size). Volume is on ~24% of SKUs in Odoo; the rest are imputed at the
17
+ category (else global) median volume. Weight is unusable (~0% populated).
18
+ - 'revenue': % of sale (reduces to channel_rate × revenue).
19
+ - 'units' : per-unit.
20
+ net(sku) = gross_margin(sku) − opex(sku). NET is a decision estimate; GROSS (margin/markup) is exact.
21
+
22
+ Brand-filterable: team_id None = all channels (incl Amazon); 5 = Fisch, 6 = Royal (wholesale scope).
23
+ READ-ONLY.
24
+ """
25
+ import sys
26
+ import statistics
27
+ import datetime as dt
28
+ from pathlib import Path
29
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
30
+ import core.odoo as O
31
+ import core.periods as P
32
+
33
+ _ANA = {1: 'Fisch', 2: 'Royal', 3: 'Amazon', 4: 'HQ', 5: 'Internal'}
34
+ # Period: calendar 2025 (not LTM) so the pricing P&L ties to the 2025 P&L / GL / vendor analyses.
35
+ _FY = ('2025-01-01', '2025-12-31')
36
+ _FY_LABEL = 'FY2025'
37
+ DRIVERS = {'cogs': 'COGS (cost-weighted)', 'cbm': 'CBM / volume (size)', 'revenue': 'Revenue', 'units': 'Units'}
38
+ _DRIVER_SHORT = {'cogs': 'COGS', 'cbm': 'CBM', 'units': 'Units', 'revenue': 'Revenue'}
39
+
40
+
41
+ def has_driver_data(row, driver):
42
+ """True if the SKU has REAL (not imputed / defaulted) data for the chosen allocation driver."""
43
+ if driver == 'cbm':
44
+ return bool(row.get('vol_known')) # volume on file in Odoo (else category-median imputed)
45
+ if driver == 'cogs':
46
+ return (row.get('cogs') or 0) > 0 # has a real product cost (else uncosted)
47
+ if driver == 'units':
48
+ return (row.get('units') or 0) > 0
49
+ return (row.get('revenue') or 0) > 0 # revenue: always present in the table
50
+
51
+
52
+ def rates(t=None):
53
+ """LTM opex rates per channel + HQ, from the analytic ledger. {channel: opex/revenue}."""
54
+ lf, lt = _FY
55
+ o = O.get_odoo()
56
+ rev = {_ANA.get(O.m2o_id(r['account_id'])): (r['amount'] or 0.0) for r in o.read_group(
57
+ 'account.analytic.line', [('date', '>=', lf), ('date', '<=', lt),
58
+ ('general_account_id.account_type', 'in', ['income', 'income_other'])],
59
+ ['amount:sum', 'account_id'], ['account_id'], lazy=False) if O.m2o_id(r['account_id']) in _ANA}
60
+ opx = {_ANA.get(O.m2o_id(r['account_id'])): -(r['amount'] or 0.0) for r in o.read_group(
61
+ 'account.analytic.line', [('date', '>=', lf), ('date', '<=', lt),
62
+ ('general_account_id.account_type', 'in', ['expense', 'expense_depreciation'])],
63
+ ['amount:sum', 'account_id'], ['account_id'], lazy=False) if O.m2o_id(r['account_id']) in _ANA}
64
+ tot_rev = sum(rev.values()) or 1.0
65
+ def rate(k):
66
+ return (opx.get(k, 0.0) / rev[k]) if rev.get(k) else 0.0
67
+ return {'Fisch': rate('Fisch'), 'Royal': rate('Royal'), 'Amazon': rate('Amazon'),
68
+ 'HQ': (opx.get('HQ', 0.0) + opx.get('Internal', 0.0)) / tot_rev,
69
+ 'window': _FY_LABEL, '_rev': rev, '_opex': opx, '_tot_rev': tot_rev}
70
+
71
+
72
+ def _meta(pids):
73
+ prods = O.search_read('product.product', [('id', 'in', pids), ('active', 'in', [True, False])],
74
+ ['default_code', 'name', 'categ_id', 'volume'])
75
+ cats = {}
76
+ for c in O.search_read('product.category', [], ['id', 'complete_name']):
77
+ parts = [x.strip() for x in (c['complete_name'] or '').split('/')]
78
+ cats[c['id']] = parts[1] if len(parts) >= 2 else (parts[0] if parts else None)
79
+ # median volume per category + global, to impute the SKUs without volume on file
80
+ by_cat = {}
81
+ allv = []
82
+ for p in prods:
83
+ v = p.get('volume') or 0.0
84
+ if v > 0:
85
+ allv.append(v)
86
+ by_cat.setdefault(O.m2o_id(p.get('categ_id')), []).append(v)
87
+ gmed = statistics.median(allv) if allv else 0.0
88
+ cmed = {c: statistics.median(vs) for c, vs in by_cat.items()}
89
+ out = {}
90
+ for p in prods:
91
+ cid = O.m2o_id(p.get('categ_id'))
92
+ v = p.get('volume') or 0.0
93
+ out[p['id']] = {'sku': p.get('default_code') or f"#{p['id']}", 'name': p.get('name') or '',
94
+ 'category': cats.get(cid) or '(uncategorized)',
95
+ 'volume': v, 'vol_used': v if v > 0 else (cmed.get(cid) or gmed), 'vol_known': v > 0}
96
+ return out
97
+
98
+
99
+ def _build(team_id=None, driver='cogs', t=None):
100
+ t = t or P.today()
101
+ lf, lt = _FY
102
+ o = O.get_odoo()
103
+ rt = rates(t)
104
+ gift = list(O.excluded_partner_ids())
105
+ win = [('order_id.date_order', '>=', f'{lf} 00:00:00'), ('order_id.date_order', '<=', f'{lt} 23:59:59')]
106
+ sbase = [('order_id.state', 'in', ['sale', 'done']), ('product_id.type', '!=', 'service')] + win
107
+
108
+ def by_prod(extra, fields):
109
+ return {O.m2o_id(r['product_id']): r for r in o.read_group('sale.order.line', sbase + extra,
110
+ fields + ['product_id'], ['product_id'], lazy=False) if r.get('product_id')}
111
+
112
+ if team_id in (5, 6): # one wholesale BU
113
+ allc = by_prod([('order_id.team_id', '=', team_id), ('order_partner_id', 'not in', gift)],
114
+ ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'])
115
+ chan_of = {pid: ('Fisch' if team_id == 5 else 'Royal') for pid in allc}
116
+ chan_rev = {pid: {('Fisch' if team_id == 5 else 'Royal'): (a['price_subtotal'] or 0.0)} for pid, a in allc.items()}
117
+ else: # all channels
118
+ allc = by_prod([], ['price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'])
119
+ rf = by_prod([('order_id.team_id', '=', 5), ('order_partner_id', 'not in', gift)], ['price_subtotal:sum'])
120
+ rr = by_prod([('order_id.team_id', '=', 6), ('order_partner_id', 'not in', gift)], ['price_subtotal:sum'])
121
+ ra = by_prod([('order_partner_id', 'in', gift)] if gift else [('id', '=', -1)], ['price_subtotal:sum'])
122
+ chan_rev = {pid: {'Fisch': (rf.get(pid, {}).get('price_subtotal') or 0.0),
123
+ 'Royal': (rr.get(pid, {}).get('price_subtotal') or 0.0),
124
+ 'Amazon': (ra.get(pid, {}).get('price_subtotal') or 0.0)} for pid in allc}
125
+
126
+ meta = _meta(list(allc))
127
+ # assemble per-SKU base
128
+ skus = {}
129
+ for pid, a in allc.items():
130
+ rev = a['price_subtotal'] or 0.0
131
+ if rev <= 0:
132
+ continue
133
+ units = a['product_uom_qty'] or 0.0
134
+ gm = a['margin'] or 0.0
135
+ cogs = rev - gm
136
+ m = meta.get(pid, {})
137
+ skus[pid] = {'rev': rev, 'units': units, 'gm': gm, 'cogs': cogs, 'cr': chan_rev.get(pid, {}),
138
+ 'cbm': units * (m.get('vol_used') or 0.0), 'm': m}
139
+
140
+ # driver value per SKU + its apportionment to each channel (by revenue mix)
141
+ def dval(s):
142
+ return {'cogs': s['cogs'], 'cbm': s['cbm'], 'units': s['units'], 'revenue': s['rev']}.get(driver, s['cogs'])
143
+ chans = ['Fisch', 'Royal', 'Amazon']
144
+ pool = {C: rt[C] * sum(s['cr'].get(C, 0.0) for s in skus.values()) for C in chans} # rate × visible channel rev
145
+ pool_hq = rt['HQ'] * sum(s['rev'] for s in skus.values())
146
+ sumdrvC = {C: sum(dval(s) * (s['cr'].get(C, 0.0) / s['rev']) for s in skus.values() if s['rev']) for C in chans}
147
+ sumdrv = sum(dval(s) for s in skus.values()) or 1.0
148
+
149
+ rows = []
150
+ for pid, s in skus.items():
151
+ load = pool_hq * (dval(s) / sumdrv)
152
+ for C in chans:
153
+ if sumdrvC[C] > 0 and s['rev']:
154
+ load += pool[C] * (dval(s) * (s['cr'].get(C, 0.0) / s['rev'])) / sumdrvC[C]
155
+ rev, gm, cogs, units = s['rev'], s['gm'], s['cogs'], s['units']
156
+ net = gm - load
157
+ m = s['m']
158
+ rows.append({
159
+ 'product_id': pid, 'sku': m.get('sku', f'#{pid}'), 'code': m.get('sku', f'#{pid}'),
160
+ 'product': m.get('name', ''), 'name': m.get('name', ''), 'category': m.get('category', '(uncategorized)'),
161
+ 'units': units, 'revenue': rev, 'cogs': cogs,
162
+ 'unit_cost': (cogs / units) if units else 0.0, 'avg_price': (rev / units) if units else 0.0,
163
+ 'cbm_unit': m.get('volume', 0.0), 'cbm_used': m.get('vol_used', 0.0), 'cbm_total': s['cbm'],
164
+ 'vol_known': m.get('vol_known', False), 'cbm_src': ('on file' if m.get('vol_known') else 'imputed'),
165
+ 'gm_dollars': gm, 'gm_pct': (gm / rev * 100) if rev else 0.0,
166
+ 'markup_pct': (gm / cogs * 100) if cogs > 0 else None,
167
+ 'opex_load': load, 'opex_pct': (load / rev * 100) if rev else 0.0,
168
+ 'net_dollars': net, 'net_pct': (net / rev * 100) if rev else 0.0,
169
+ 'status': ('Below cost' if gm < 0 else 'Net-negative' if net < 0 else 'Thin (<10% net)' if (net / rev) < 0.10 else 'Healthy'),
170
+ })
171
+ rows.sort(key=lambda r: -r['revenue'])
172
+ return rows, rt
173
+
174
+
175
+ def table(team_id=None, driver='cogs', t=None):
176
+ return _build(team_id, driver, t)[0]
177
+
178
+
179
+ def summary(team_id=None, driver='cogs', t=None, built=None):
180
+ rows, rt = built or _build(team_id, driver, t)
181
+ rev = sum(r['revenue'] for r in rows)
182
+ gm = sum(r['gm_dollars'] for r in rows)
183
+ net = sum(r['net_dollars'] for r in rows)
184
+ mk = [r['markup_pct'] for r in rows if r['markup_pct'] is not None]
185
+ vol_known = sum(1 for r in rows if r['vol_known'])
186
+ driver_known = sum(1 for r in rows if has_driver_data(r, driver))
187
+ return {
188
+ 'window': rt['window'], 'skus': len(rows), 'revenue': rev, 'driver': driver, 'driver_label': DRIVERS.get(driver, driver),
189
+ 'driver_short': _DRIVER_SHORT.get(driver, driver),
190
+ 'gm_dollars': gm, 'gm_pct': (gm / rev * 100) if rev else 0.0,
191
+ 'net_dollars': net, 'net_pct': (net / rev * 100) if rev else 0.0,
192
+ 'avg_markup': (sum(mk) / len(mk)) if mk else 0.0,
193
+ 'below_cost_skus': sum(1 for r in rows if r['gm_dollars'] < 0),
194
+ 'net_negative_skus': sum(1 for r in rows if r['net_dollars'] < 0),
195
+ 'net_negative_rev': sum(r['revenue'] for r in rows if r['net_dollars'] < 0),
196
+ 'vol_coverage': (vol_known / len(rows) * 100) if rows else 0.0,
197
+ 'driver_known': driver_known, 'driver_coverage': (driver_known / len(rows) * 100) if rows else 0.0,
198
+ 'rates': {k: round(rt[k] * 100, 1) for k in ('Fisch', 'Royal', 'Amazon', 'HQ')},
199
+ }
200
+
201
+
202
+ def cost_drift(t=None, team_id=None):
203
+ """Replacement-cost drift from confirmed PO lines: the net price paid per unit in the LAST
204
+ 12 months vs the 12 months BEFORE, per product, joined to what the SELL price did over the
205
+ same two windows (BU-scoped sell side; cost is company-wide). ERODING = cost up >5% while
206
+ the selling price followed by less than half — the margin leaks silently until repriced.
207
+ cost_impact_12m = (cost_now − cost_prior) × units sold last 12m (the annualized $ at stake)."""
208
+ t = t or P.today()
209
+ d24 = (t - dt.timedelta(days=730)).isoformat()
210
+ d12 = (t - dt.timedelta(days=365)).isoformat()
211
+ lines = O.search_read('purchase.order.line',
212
+ [('order_id.state', 'in', ('purchase', 'done')),
213
+ ('order_id.date_order', '>=', d24),
214
+ ('product_qty', '>', 0), ('price_unit', '>', 0)],
215
+ ['product_id', 'product_qty', 'product_uom_qty', 'price_subtotal',
216
+ 'order_id'])
217
+ oids = list({O.m2o_id(l['order_id']) for l in lines if l.get('order_id')})
218
+ od = {}
219
+ for i in range(0, len(oids), 5000):
220
+ for o_ in O.search_read('purchase.order', [('id', 'in', oids[i:i + 5000])], ['date_order']):
221
+ od[o_['id']] = str(o_['date_order'])[:10]
222
+ cur, prior = {}, {} # pid -> [net spend, qty]
223
+ for l in lines:
224
+ pid = O.m2o_id(l.get('product_id'))
225
+ d = od.get(O.m2o_id(l.get('order_id')))
226
+ if not pid or not d:
227
+ continue
228
+ e = (cur if d >= d12 else prior).setdefault(pid, [0.0, 0.0])
229
+ e[0] += l.get('price_subtotal') or 0.0
230
+ # BASE-UoM qty, so a piece→case purchase-UoM switch doesn't fake a price spike
231
+ e[1] += l.get('product_uom_qty') or l.get('product_qty') or 0.0
232
+ both = [p for p in cur if p in prior and prior[p][1] > 0 and cur[p][1] > 0]
233
+
234
+ def _sell(a, b_):
235
+ out = {}
236
+ for g in O.read_group('sale.order.line', O.sale_line_domain(a, b_, team_id),
237
+ ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'],
238
+ lazy=False):
239
+ pid = O.m2o_id(g.get('product_id'))
240
+ if pid:
241
+ out[pid] = (g.get('price_subtotal') or 0.0, g.get('product_uom_qty') or 0.0)
242
+ return out
243
+ s_now, s_pri = _sell(d12, t.isoformat()), _sell(d24, d12)
244
+
245
+ meta = {}
246
+ for i in range(0, len(both), 5000):
247
+ for p in O.search_read('product.product',
248
+ [('id', 'in', both[i:i + 5000]), ('active', 'in', [True, False])],
249
+ ['default_code', 'name', 'standard_price']):
250
+ meta[p['id']] = p
251
+ rows = []
252
+ for pid in both:
253
+ c_now, c_pri = cur[pid][0] / cur[pid][1], prior[pid][0] / prior[pid][1]
254
+ if c_pri <= 0:
255
+ continue
256
+ rn, qn = s_now.get(pid, (0.0, 0.0))
257
+ rp, qp = s_pri.get(pid, (0.0, 0.0))
258
+ asp_now = rn / qn if qn else None
259
+ asp_pri = rp / qp if qp else None
260
+ p = meta.get(pid, {})
261
+ rows.append({'pid': pid, 'code': (p.get('default_code') or '').strip(),
262
+ 'product': p.get('name') or '',
263
+ 'cost_prior': c_pri, 'cost_now': c_now,
264
+ 'drift_pct': (c_now / c_pri - 1) * 100,
265
+ 'buy_qty_12m': cur[pid][1], 'std_cost': p.get('standard_price') or 0.0,
266
+ 'asp_now': asp_now, 'asp_prior': asp_pri,
267
+ 'price_chg_pct': ((asp_now / asp_pri - 1) * 100)
268
+ if (asp_now and asp_pri) else None,
269
+ 'units_12m': qn, 'cost_impact_12m': (c_now - c_pri) * qn,
270
+ 'gm_pct_now': ((asp_now - c_now) / asp_now * 100) if asp_now else None})
271
+ # a >3x (or <1/3) per-base-unit move is a UoM/master-data break, not market inflation —
272
+ # surfaced as its own data-quality list so it can't pollute the erosion signal
273
+ breaks = sorted((r for r in rows if not (1 / 3 <= (r['cost_now'] / r['cost_prior']) <= 3)),
274
+ key=lambda r: -abs(r['drift_pct']))
275
+ broken = {r['pid'] for r in breaks}
276
+ eroding = sorted((r for r in rows
277
+ if r['pid'] not in broken
278
+ and r['drift_pct'] > 5 and (r['units_12m'] or 0) > 0
279
+ and (r['price_chg_pct'] is None or r['price_chg_pct'] < r['drift_pct'] / 2)),
280
+ key=lambda r: -(r['cost_impact_12m'] or 0))
281
+ improving = sorted((r for r in rows if r['pid'] not in broken
282
+ and r['drift_pct'] < -5 and (r['units_12m'] or 0) > 0),
283
+ key=lambda r: r['cost_impact_12m'])
284
+ return {'rows': rows, 'eroding': eroding, 'improving': improving, 'breaks': breaks,
285
+ 'n_products': len(rows),
286
+ 'erosion_total': sum(r['cost_impact_12m'] for r in eroding),
287
+ 'tailwind_total': sum(r['cost_impact_12m'] for r in improving),
288
+ '_cur_spend': sum(e[0] for e in cur.values()),
289
+ '_cur_domain_from': d12, 'windows': (d24, d12, t.isoformat())}
290
+
291
+
292
+ def cost_drift_validate(cd, t=None):
293
+ """The 12m PO spend our per-product weighting is built on == the server-side sum over the
294
+ identical domain (two independent aggregation paths)."""
295
+ t = t or P.today()
296
+ dom = [('order_id.state', 'in', ('purchase', 'done')),
297
+ ('order_id.date_order', '>=', cd['_cur_domain_from']),
298
+ ('product_qty', '>', 0), ('price_unit', '>', 0)]
299
+ srv = O.sum_field('purchase.order.line', dom, 'price_subtotal')
300
+ return [{'check': 'Cost drift: Σ(per-product 12m PO spend) == server Σ(line subtotal), same domain',
301
+ 'a': round(cd['_cur_spend'], 2), 'b': round(srv, 2),
302
+ 'gap': round(cd['_cur_spend'] - srv, 2),
303
+ 'ok': abs(cd['_cur_spend'] - srv) <= max(1.0, abs(srv) * 0.001)}]
304
+
305
+
306
+ def _pnl_entities(t=None):
307
+ """Actual LTM P&L per analytic entity (the basis the Management P&L is built from)."""
308
+ lf, lt = _FY
309
+ o = O.get_odoo()
310
+ def grp(types):
311
+ return {_ANA.get(O.m2o_id(r['account_id'])): (r['amount'] or 0.0) for r in o.read_group(
312
+ 'account.analytic.line', [('date', '>=', lf), ('date', '<=', lt),
313
+ ('general_account_id.account_type', 'in', types)], ['amount:sum', 'account_id'], ['account_id'], lazy=False)
314
+ if O.m2o_id(r['account_id']) in _ANA}
315
+ inc, cog, opx = grp(['income', 'income_other']), grp(['expense_direct_cost']), grp(['expense', 'expense_depreciation'])
316
+ return {k: {'revenue': inc.get(k, 0.0), 'cogs': -cog.get(k, 0.0), 'opex': -opx.get(k, 0.0)} for k in _ANA.values()}
317
+
318
+
319
+ def gl_pnl(t=None):
320
+ """Actual LTM P&L straight from the posted GL (the official books) — the independent reconciliation
321
+ target. Revenue − COGS − Opex = Net."""
322
+ lf, lt = _FY
323
+ def s(types):
324
+ return O.sum_field('account.move.line', [('parent_state', '=', 'posted'), ('date', '>=', lf),
325
+ ('date', '<=', lt), ('account_id.account_type', 'in', types)], 'balance')
326
+ rev = -s(['income', 'income_other']) # income is credit → flip to positive
327
+ cogs = s(['expense_direct_cost'])
328
+ opex = s(['expense', 'expense_depreciation'])
329
+ return {'revenue': rev, 'cogs': cogs, 'gm': rev - cogs, 'opex': opex, 'net': rev - cogs - opex}
330
+
331
+
332
+ def reconcile(team_id=None, driver='cogs', t=None, built=None):
333
+ """Bridge the per-SKU P&L to the ACTUAL P&L: attributed SKUs + unattributed (Amazon-direct, not
334
+ booked per-SKU) = the displayed P&L. Ties by construction; the unattributed line is the residual."""
335
+ rows, rt = built or _build(team_id, driver, t)
336
+ ent = _pnl_entities(t)
337
+ tot_rev = sum(e['revenue'] for e in ent.values()) or 1.0
338
+ hq_rate = (ent['HQ']['opex'] + ent['Internal']['opex']) / tot_rev
339
+ if team_id in (5, 6):
340
+ k = 'Fisch' if team_id == 5 else 'Royal'
341
+ rev_t, cogs_t = ent[k]['revenue'], ent[k]['cogs']
342
+ opex_t = ent[k]['opex'] + hq_rate * ent[k]['revenue'] # BU opex + its share of HQ
343
+ else:
344
+ rev_t = sum(e['revenue'] for e in ent.values())
345
+ cogs_t = sum(e['cogs'] for e in ent.values())
346
+ opex_t = sum(e['opex'] for e in ent.values()) # all opex incl HQ
347
+ net_t = rev_t - cogs_t - opex_t
348
+ rev_s = sum(r['revenue'] for r in rows)
349
+ gm_s = sum(r['gm_dollars'] for r in rows)
350
+ cogs_s, opex_s = rev_s - gm_s, sum(r['opex_load'] for r in rows)
351
+ net_s = gm_s - opex_s
352
+ una = {'revenue': rev_t - rev_s, 'cogs': cogs_t - cogs_s, 'gm': (rev_t - rev_s) - (cogs_t - cogs_s),
353
+ 'opex': opex_t - opex_s, 'net': net_t - net_s}
354
+ pnl = {'revenue': rev_t, 'cogs': cogs_t, 'gm': rev_t - cogs_t, 'opex': opex_t, 'net': net_t}
355
+ sku = {'revenue': rev_s, 'cogs': cogs_s, 'gm': gm_s, 'opex': opex_s, 'net': net_s}
356
+ return {'pnl': pnl, 'sku': sku, 'unattrib': una, 'ties': abs((net_s + una['net']) - net_t) < 1.0}
357
+
358
+
359
+ def page_data(team_id=None, driver='cogs', t=None):
360
+ """One build → rows + summary + reconciliation + validation (avoids rebuilding 4×)."""
361
+ built = _build(team_id, driver, t)
362
+ return {'rows': built[0], 'summary': summary(team_id, driver, t, built=built),
363
+ 'reconcile': reconcile(team_id, driver, t, built=built),
364
+ 'validation': validate(team_id, driver, t, built=built)}
365
+
366
+
367
+ def validate(team_id=None, driver='cogs', t=None, built=None):
368
+ rows, rt = built or _build(team_id, driver, t)
369
+ checks = []
370
+ lf, lt = _FY
371
+ gift = list(O.excluded_partner_ids())
372
+ dom = [('order_id.state', 'in', ['sale', 'done']), ('order_id.date_order', '>=', f'{lf} 00:00:00'),
373
+ ('order_id.date_order', '<=', f'{lt} 23:59:59'), ('product_id.type', '!=', 'service'), ('price_subtotal', '>', 0)]
374
+ if team_id in (5, 6):
375
+ dom += [('order_id.team_id', '=', team_id), ('order_partner_id', 'not in', gift)]
376
+ indep = O.sum_field('sale.order.line', dom, 'price_subtotal')
377
+ ours = sum(r['revenue'] for r in rows)
378
+ checks.append({'check': 'Σ per-SKU revenue == scoped FY2025 (positive lines)', 'a': round(ours, 0),
379
+ 'b': round(indep, 0), 'gap': round(ours - indep, 0), 'ok': abs(ours - indep) <= max(50.0, indep * 0.01)})
380
+ if rows:
381
+ s = rows[0]
382
+ checks.append({'check': f"GM == revenue−COGS (sample {s['sku']})", 'a': round(s['gm_dollars'], 2),
383
+ 'b': round(s['revenue'] - s['cogs'], 2), 'gap': round(s['gm_dollars'] - (s['revenue'] - s['cogs']), 2),
384
+ 'ok': abs(s['gm_dollars'] - (s['revenue'] - s['cogs'])) <= 0.5})
385
+ # THE finance check — per-SKU P&L reconciles to the actual P&L (and that target ties to the posted GL)
386
+ rc = reconcile(team_id, driver, t, built=(rows, rt))
387
+ checks.append({'check': 'Attributed SKUs + unattributed net == P&L net (reconciles)',
388
+ 'a': round(rc['sku']['net'] + rc['unattrib']['net'], 0), 'b': round(rc['pnl']['net'], 0),
389
+ 'gap': round(rc['sku']['net'] + rc['unattrib']['net'] - rc['pnl']['net'], 0), 'ok': rc['ties']})
390
+ if team_id is None:
391
+ gl = gl_pnl(t)
392
+ checks.append({'check': 'P&L target (analytic) == posted GL net (actual books)',
393
+ 'a': round(rc['pnl']['net'], 0), 'b': round(gl['net'], 0),
394
+ 'gap': round(rc['pnl']['net'] - gl['net'], 0), 'ok': abs(rc['pnl']['net'] - gl['net']) <= 2.0})
395
+ return checks
platform/procurement_suppliers.json CHANGED
The diff for this file is too large to render. See raw diff
 
requirements.txt CHANGED
@@ -1,47 +1,47 @@
1
- # AIOS web API — the FastAPI backend + the platform data-layer deps it reuses.
2
- #
3
- # ⛔ STREAMLIT IS GONE (EXIT-6, 2026-08-04) — do not add it back. The line here read
4
- # `streamlit==1.58.0` with the comment "included only because the shared modules/core may import
5
- # it at load; it is never served". That stopped being true when the shared layer was cleaned, and
6
- # it stayed in THIS file — the one the Dockerfile actually installs — while `api/requirements.txt`
7
- # had already dropped it and a gate reported the omission "verified". ~250 MB of image for a
8
- # package nothing imported.
9
- #
10
- # The lesson is the file, not the package: the PINNED intent and the SHIPPED manifest are two
11
- # different documents, and checking only the one you wrote is how the other rots.
12
- # `api/verify_no_streamlit.py` now checks BOTH, and proves the API imports with streamlit,
13
- # plotly, altair, openpyxl, reportlab and jinja2 all blocked at sys.meta_path.
14
- # No altair/plotly/reportlab either — those were Streamlit UI/export only. openpyxl RETURNED in
15
- # wave 21 (C5) as a LAZY dep of routes_uploads.py — the boot proof above still holds because the
16
- # import lives inside the handler, and the gate now asserts this line exists here.
17
- fastapi>=0.139
18
- uvicorn[standard]>=0.30
19
- python-dotenv>=1.0
20
- pandas>=2.0
21
- requests>=2.28
22
- huggingface_hub>=0.20
23
- duckdb>=1.0
24
- pyyaml>=6.0
25
- pillow>=10.0
26
- beautifulsoup4>=4.12
27
- lxml>=5.0
28
- cryptography>=42.0
29
- # ⭐ WAVE 20 (R1 / D-4) — THE POSTGRES DRIVER, AND IT IS LOAD-BEARING IN THIS FILE SPECIFICALLY.
30
- # `core/store_pg.py` is the store backend from this wave on, and it is deliberately FAIL-CLOSED:
31
- # `_pool()` raises rather than falling back to the HF file store, so a container that gets
32
- # `STORE_BACKEND=pg` without this line does not degrade — it refuses every request that touches
33
- # the store, which is every authenticated request.
34
- #
35
- # ⛔ AND THIS IS THE FILE THAT MATTERS: the Dockerfile does `COPY requirements.txt` from the
36
- # Space root, i.e. THIS manifest, not `api/requirements.txt`. That header's own streamlit story
37
- # is the same defect in the other direction — the pinned intent and the shipped manifest are two
38
- # documents. psycopg is now in BOTH, and `ops/verify_portability.py` gates the pair.
39
- psycopg[binary,pool]>=3.2
40
- # ⭐ WAVE 21 (item 11, C5) — .xlsx preview for Select-from-file; lazy-imported in
41
- # routes_uploads.py only. verify_no_streamlit asserts this line in BOTH manifests.
42
- openpyxl>=3.1
43
- # ⛔ AND ITS TRANSPORT, learned from a RUNTIME_ERROR on the first v6 boot: FastAPI demands
44
- # python-multipart AT IMPORT TIME for any route declaring File(...)/Form(...). Every local gate
45
- # was green because the dev box happens to have it for unrelated reasons — the container
46
- # installs exactly this file. The environment-parity twin of the streamlit lesson above.
47
- python-multipart>=0.0.20
 
1
+ # AIOS web API — the FastAPI backend + the platform data-layer deps it reuses.
2
+ #
3
+ # ⛔ STREAMLIT IS GONE (EXIT-6, 2026-08-04) — do not add it back. The line here read
4
+ # `streamlit==1.58.0` with the comment "included only because the shared modules/core may import
5
+ # it at load; it is never served". That stopped being true when the shared layer was cleaned, and
6
+ # it stayed in THIS file — the one the Dockerfile actually installs — while `api/requirements.txt`
7
+ # had already dropped it and a gate reported the omission "verified". ~250 MB of image for a
8
+ # package nothing imported.
9
+ #
10
+ # The lesson is the file, not the package: the PINNED intent and the SHIPPED manifest are two
11
+ # different documents, and checking only the one you wrote is how the other rots.
12
+ # `api/verify_no_streamlit.py` now checks BOTH, and proves the API imports with streamlit,
13
+ # plotly, altair, openpyxl, reportlab and jinja2 all blocked at sys.meta_path.
14
+ # No altair/plotly/reportlab either — those were Streamlit UI/export only. openpyxl RETURNED in
15
+ # wave 21 (C5) as a LAZY dep of routes_uploads.py — the boot proof above still holds because the
16
+ # import lives inside the handler, and the gate now asserts this line exists here.
17
+ fastapi>=0.139
18
+ uvicorn[standard]>=0.30
19
+ python-dotenv>=1.0
20
+ pandas>=2.0
21
+ requests>=2.28
22
+ huggingface_hub>=0.20
23
+ duckdb>=1.0
24
+ pyyaml>=6.0
25
+ pillow>=10.0
26
+ beautifulsoup4>=4.12
27
+ lxml>=5.0
28
+ cryptography>=42.0
29
+ # ⭐ WAVE 20 (R1 / D-4) — THE POSTGRES DRIVER, AND IT IS LOAD-BEARING IN THIS FILE SPECIFICALLY.
30
+ # `core/store_pg.py` is the store backend from this wave on, and it is deliberately FAIL-CLOSED:
31
+ # `_pool()` raises rather than falling back to the HF file store, so a container that gets
32
+ # `STORE_BACKEND=pg` without this line does not degrade — it refuses every request that touches
33
+ # the store, which is every authenticated request.
34
+ #
35
+ # ⛔ AND THIS IS THE FILE THAT MATTERS: the Dockerfile does `COPY requirements.txt` from the
36
+ # Space root, i.e. THIS manifest, not `api/requirements.txt`. That header's own streamlit story
37
+ # is the same defect in the other direction — the pinned intent and the shipped manifest are two
38
+ # documents. psycopg is now in BOTH, and `ops/verify_portability.py` gates the pair.
39
+ psycopg[binary,pool]>=3.2
40
+ # ⭐ WAVE 21 (item 11, C5) — .xlsx preview for Select-from-file; lazy-imported in
41
+ # routes_uploads.py only. verify_no_streamlit asserts this line in BOTH manifests.
42
+ openpyxl>=3.1
43
+ # ⛔ AND ITS TRANSPORT, learned from a RUNTIME_ERROR on the first v6 boot: FastAPI demands
44
+ # python-multipart AT IMPORT TIME for any route declaring File(...)/Form(...). Every local gate
45
+ # was green because the dev box happens to have it for unrelated reasons — the container
46
+ # installs exactly this file. The environment-parity twin of the streamlit lesson above.
47
+ python-multipart>=0.0.20
web/public/sample_customers.json CHANGED
@@ -1,529 +1,529 @@
1
- {
2
- "fields": [
3
- {
4
- "key": "customer",
5
- "label": "Customer",
6
- "type": "text",
7
- "source": "odoo",
8
- "pinned": true,
9
- "default": true,
10
- "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
11
- },
12
- {
13
- "key": "odoo_status",
14
- "label": "Odoo record",
15
- "type": "status",
16
- "source": "odoo",
17
- "default": false,
18
- "description": "Whether this customer still exists in Odoo. Archived means deleted there."
19
- },
20
- {
21
- "key": "agent",
22
- "label": "Agent",
23
- "type": "text",
24
- "source": "odoo",
25
- "default": true,
26
- "description": "The sales agent who owns this account."
27
- },
28
- {
29
- "key": "dba",
30
- "label": "DBA",
31
- "type": "select",
32
- "source": "odoo",
33
- "default": false,
34
- "options": [
35
- "Fisch",
36
- "Royal",
37
- "Both"
38
- ],
39
- "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
40
- },
41
- {
42
- "key": "salesperson",
43
- "label": "Salesperson",
44
- "type": "text",
45
- "source": "odoo",
46
- "default": false,
47
- "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account."
48
- },
49
- {
50
- "key": "city",
51
- "label": "City",
52
- "type": "text",
53
- "source": "odoo",
54
- "default": true,
55
- "description": "City on the customer's Odoo address."
56
- },
57
- {
58
- "key": "state",
59
- "label": "State",
60
- "type": "text",
61
- "source": "odoo",
62
- "default": true,
63
- "description": "State or province on the customer's Odoo address."
64
- },
65
- {
66
- "key": "country",
67
- "label": "Country",
68
- "type": "text",
69
- "source": "odoo",
70
- "default": false,
71
- "description": "Country on the customer's Odoo address."
72
- },
73
- {
74
- "key": "zip",
75
- "label": "ZIP",
76
- "type": "text",
77
- "source": "odoo",
78
- "default": false,
79
- "description": "Postal code on the customer's Odoo address."
80
- },
81
- {
82
- "key": "customer_since",
83
- "label": "Customer since",
84
- "type": "date",
85
- "source": "odoo",
86
- "default": false,
87
- "description": "When this customer was first set up in Odoo."
88
- },
89
- {
90
- "key": "tags",
91
- "label": "Tags",
92
- "type": "text",
93
- "source": "odoo",
94
- "default": false,
95
- "description": "Odoo labels on this customer, comma-separated."
96
- },
97
- {
98
- "key": "pricelist",
99
- "label": "Price list",
100
- "type": "text",
101
- "source": "odoo",
102
- "default": false,
103
- "description": "The price list this customer buys on."
104
- },
105
- {
106
- "key": "payment_terms",
107
- "label": "Payment terms",
108
- "type": "text",
109
- "source": "odoo",
110
- "default": false,
111
- "description": "Payment terms on this customer's account — Net 30, for example."
112
- },
113
- {
114
- "key": "last_order",
115
- "label": "Last order",
116
- "type": "date",
117
- "source": "odoo",
118
- "default": true,
119
- "description": "Date of the most recent confirmed order."
120
- },
121
- {
122
- "key": "overdue_days",
123
- "label": "Overdue days",
124
- "type": "int",
125
- "source": "odoo",
126
- "default": true,
127
- "description": "How many days late this customer is running against their own usual ordering rhythm."
128
- },
129
- {
130
- "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
131
- "key": "est_missed",
132
- "label": "Est. missed $",
133
- "type": "currency",
134
- "source": "odoo",
135
- "default": true,
136
- "agg": "sum",
137
- "filterable": false,
138
- "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
139
- },
140
- {
141
- "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
142
- "key": "ar_open",
143
- "label": "AR current $",
144
- "type": "currency",
145
- "source": "odoo",
146
- "default": false,
147
- "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
148
- },
149
- {
150
- "key": "ar_overdue",
151
- "label": "AR overdue $",
152
- "type": "currency",
153
- "source": "odoo",
154
- "default": false,
155
- "description": "Invoiced money past due — same basis as the Collections page."
156
- },
157
- {
158
- "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
159
- "key": "ar_outstanding",
160
- "label": "AR outstanding $",
161
- "type": "currency",
162
- "source": "odoo",
163
- "default": false,
164
- "description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
165
- },
166
- {
167
- "key": "ar_exposure",
168
- "label": "Credit exposure $",
169
- "type": "currency",
170
- "source": "odoo",
171
- "default": false,
172
- "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
173
- },
174
- {
175
- "key": "ar_aged_1_30",
176
- "label": "1-30 days $",
177
- "type": "currency",
178
- "source": "odoo",
179
- "default": false,
180
- "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
181
- },
182
- {
183
- "key": "ar_aged_31_60",
184
- "label": "31-60 days $",
185
- "type": "currency",
186
- "source": "odoo",
187
- "default": false,
188
- "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
189
- },
190
- {
191
- "key": "ar_aged_61_90",
192
- "label": "61-90 days $",
193
- "type": "currency",
194
- "source": "odoo",
195
- "default": false,
196
- "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
197
- },
198
- {
199
- "key": "ar_aged_90_plus",
200
- "label": "90+ days $",
201
- "type": "currency",
202
- "source": "odoo",
203
- "default": false,
204
- "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
205
- },
206
- {
207
- "key": "days_to_pay",
208
- "label": "Days to pay",
209
- "type": "int",
210
- "source": "odoo",
211
- "default": false,
212
- "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
213
- },
214
- {
215
- "key": "top_category",
216
- "label": "Top category",
217
- "type": "text",
218
- "source": "odoo",
219
- "default": false,
220
- "description": "The category this customer spent the most on in the last 12 months."
221
- },
222
- {
223
- "key": "top_category_pct",
224
- "label": "Top category %",
225
- "type": "pct",
226
- "source": "odoo",
227
- "default": false,
228
- "description": "Share of last-12-months spend that went to the top category."
229
- },
230
- {
231
- "key": "sku_count",
232
- "label": "SKUs bought",
233
- "type": "int",
234
- "source": "odoo",
235
- "default": false,
236
- "description": "Distinct products bought in the last 12 months."
237
- },
238
- {
239
- "key": "top_sku",
240
- "label": "Top SKU",
241
- "type": "text",
242
- "source": "odoo",
243
- "default": false,
244
- "description": "The product this customer spent the most on in the last 12 months."
245
- },
246
- {
247
- "key": "days_since",
248
- "label": "Days since order",
249
- "type": "int",
250
- "source": "odoo",
251
- "default": false,
252
- "description": "Days since the last confirmed order."
253
- },
254
- {
255
- "key": "typical_gap_days",
256
- "label": "Typical gap days",
257
- "type": "int",
258
- "source": "odoo",
259
- "default": false,
260
- "description": "Days this customer usually goes between orders, from their own history."
261
- },
262
- {
263
- "key": "notes",
264
- "label": "Notes",
265
- "type": "text",
266
- "source": "overlay",
267
- "default": false,
268
- "description": "Your notes on this customer. Saved in this app only, visible only to you."
269
- }
270
- ],
271
- "rows": [
272
- {
273
- "pid": 101,
274
- "customer": "Poppy Flowers",
275
- "status": "New",
276
- "agent": "Naomi Linnell Rivera",
277
- "city": "Charlottesville",
278
- "state": "Virginia (US)",
279
- "last_order": "2026-07-21",
280
- "est_missed": 0,
281
- "notes": "",
282
- "country": "United States",
283
- "zip": "02720",
284
- "payment_terms": "30 Days",
285
- "pricelist": "Fisch 1 (USD)",
286
- "tags": "Royal",
287
- "customer_since": "2023-01-10",
288
- "salesperson": "Jessica",
289
- "ar_open": 0,
290
- "ar_overdue": 0,
291
- "ar_exposure": 0,
292
- "top_category": "Styrofoam",
293
- "top_category_pct": 0.47,
294
- "sku_count": 93,
295
- "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
296
- "days_to_pay": 34,
297
- "_created": "2023-01-15 09:10:00",
298
- "lat": 25.7617,
299
- "lon": -80.1918,
300
- "odoo_status": "Archived",
301
- "dba": "Royal",
302
- "ar_outstanding": 0
303
- },
304
- {
305
- "pid": 102,
306
- "customer": "Meadow & Vine Wholesale",
307
- "status": "Growing",
308
- "agent": "Carla Jimenez",
309
- "city": "Portland",
310
- "state": "Oregon (US)",
311
- "last_order": "2026-07-19",
312
- "est_missed": 0,
313
- "notes": "Expanding to a second storefront.",
314
- "country": "United States",
315
- "zip": "77041",
316
- "payment_terms": "Immediate Payment",
317
- "pricelist": "Royal 1 (USD)",
318
- "tags": "Royal, Key account",
319
- "customer_since": "2024-02-11",
320
- "salesperson": "Naomi",
321
- "ar_open": 1240.5,
322
- "ar_overdue": 0,
323
- "ar_exposure": 1740.5,
324
- "top_category": "Ribbon",
325
- "top_category_pct": 0.95,
326
- "sku_count": 4,
327
- "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
328
- "days_to_pay": null,
329
- "_created": "2023-02-15 09:11:00",
330
- "lat": 27.9506,
331
- "lon": -82.4572,
332
- "odoo_status": "Active",
333
- "dba": "Fisch",
334
- "ar_outstanding": 1240.5
335
- },
336
- {
337
- "pid": 103,
338
- "customer": "Bluestem Floral Supply",
339
- "status": "Growing",
340
- "agent": "Naomi Linnell Rivera",
341
- "city": "Kansas City",
342
- "state": "Missouri (US)",
343
- "last_order": "2026-07-17",
344
- "est_missed": 0,
345
- "notes": "",
346
- "country": "United States",
347
- "zip": "11219",
348
- "payment_terms": "60 Days",
349
- "pricelist": "Royal 1 (USD)",
350
- "tags": "Fisch",
351
- "customer_since": "2025-03-12",
352
- "salesperson": "Karen",
353
- "ar_open": 0,
354
- "ar_overdue": 5120.25,
355
- "ar_exposure": 5120.25,
356
- "top_category": "Foams & Finishes",
357
- "top_category_pct": 0.31,
358
- "sku_count": 27,
359
- "top_sku": "SATIN RIBBON 2IN",
360
- "days_to_pay": 61,
361
- "_created": "2023-03-15 09:12:00",
362
- "lat": 28.5384,
363
- "lon": -81.3789,
364
- "odoo_status": "Active",
365
- "dba": "Both",
366
- "ar_outstanding": 5120.25
367
- },
368
- {
369
- "pid": 104,
370
- "customer": "Camellia Row Florist",
371
- "status": "Declining",
372
- "agent": "Devon Marsh",
373
- "city": "Savannah",
374
- "state": "Georgia (US)",
375
- "last_order": "2026-05-30",
376
- "est_missed": 41200,
377
- "notes": "Switched some volume to a local grower.",
378
- "country": "United States",
379
- "zip": "07649",
380
- "payment_terms": "30 Days",
381
- "pricelist": "Fisch 1 (USD)",
382
- "tags": "(none)",
383
- "customer_since": "2026-04-13",
384
- "salesperson": "(none)",
385
- "ar_open": 8300,
386
- "ar_overdue": 940,
387
- "ar_exposure": 11440,
388
- "top_category": "All",
389
- "top_category_pct": 1.0,
390
- "sku_count": 1,
391
- "top_sku": "(none)",
392
- "days_to_pay": 12,
393
- "_created": "2023-04-15 09:13:00",
394
- "lat": 30.3322,
395
- "lon": -81.6557,
396
- "odoo_status": "Active",
397
- "dba": "Royal",
398
- "ar_outstanding": 9240
399
- },
400
- {
401
- "pid": 105,
402
- "customer": "Harborlight Wholesale Blooms",
403
- "status": "Growing",
404
- "agent": "Carla Jimenez",
405
- "city": "Seattle",
406
- "state": "Washington (US)",
407
- "last_order": "2026-07-22",
408
- "est_missed": 0,
409
- "notes": "Top-10 account.",
410
- "country": "United States",
411
- "zip": "33125",
412
- "payment_terms": "Immediate Payment",
413
- "pricelist": "Royal 1 (USD)",
414
- "tags": "Royal",
415
- "customer_since": "2023-05-14",
416
- "salesperson": "Jessica",
417
- "ar_open": 0,
418
- "ar_overdue": 0,
419
- "ar_exposure": 0,
420
- "top_category": "Styrofoam",
421
- "top_category_pct": 0.47,
422
- "sku_count": 93,
423
- "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
424
- "days_to_pay": 34,
425
- "_created": "2023-05-15 09:14:00",
426
- "lat": 26.1224,
427
- "lon": -80.1373,
428
- "odoo_status": "Active",
429
- "dba": "",
430
- "ar_outstanding": 0
431
- },
432
- {
433
- "pid": 106,
434
- "customer": "Dogwood & Fern Co.",
435
- "status": "Dormant",
436
- "agent": "Devon Marsh",
437
- "city": "Asheville",
438
- "state": "North Carolina (US)",
439
- "last_order": "2026-02-11",
440
- "est_missed": 52400,
441
- "notes": "No spring order this year.",
442
- "country": "United States",
443
- "zip": "90210",
444
- "payment_terms": "60 Days",
445
- "pricelist": "Royal 1 (USD)",
446
- "tags": "Royal, Key account",
447
- "customer_since": "2024-06-15",
448
- "salesperson": "Naomi",
449
- "ar_open": 1240.5,
450
- "ar_overdue": 0,
451
- "ar_exposure": 1740.5,
452
- "top_category": "Ribbon",
453
- "top_category_pct": 0.95,
454
- "sku_count": 4,
455
- "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
456
- "days_to_pay": null,
457
- "_created": "2023-06-15 09:15:00",
458
- "lat": 27.3364,
459
- "lon": -82.5307,
460
- "odoo_status": "Active",
461
- "dba": "Fisch",
462
- "ar_outstanding": 1240.5
463
- },
464
- {
465
- "pid": 107,
466
- "customer": "Verbena Market Florals",
467
- "status": "Lost",
468
- "agent": "Naomi Linnell Rivera",
469
- "city": "Austin",
470
- "state": "Texas (US)",
471
- "last_order": "2025-11-04",
472
- "est_missed": 78300,
473
- "notes": "Went with a competitor on freight terms.",
474
- "country": "United States",
475
- "zip": "08701",
476
- "payment_terms": "30 Days",
477
- "pricelist": "Fisch 1 (USD)",
478
- "tags": "Fisch",
479
- "customer_since": "2025-07-16",
480
- "salesperson": "Karen",
481
- "ar_open": 0,
482
- "ar_overdue": 5120.25,
483
- "ar_exposure": 5120.25,
484
- "top_category": "Foams & Finishes",
485
- "top_category_pct": 0.31,
486
- "sku_count": 27,
487
- "top_sku": "SATIN RIBBON 2IN",
488
- "days_to_pay": 61,
489
- "_created": "2023-07-15 09:16:00",
490
- "lat": null,
491
- "lon": null,
492
- "odoo_status": "Active",
493
- "dba": "Both",
494
- "ar_outstanding": 5120.25
495
- },
496
- {
497
- "pid": 108,
498
- "customer": "Larkspur Lane Supply",
499
- "status": "New",
500
- "agent": "Carla Jimenez",
501
- "city": "Denver",
502
- "state": "Colorado (US)",
503
- "last_order": "2026-07-14",
504
- "est_missed": 0,
505
- "notes": "First order in April.",
506
- "country": "United States",
507
- "zip": "60614",
508
- "payment_terms": "Immediate Payment",
509
- "pricelist": "Royal 1 (USD)",
510
- "tags": "(none)",
511
- "customer_since": "2026-08-17",
512
- "salesperson": "(none)",
513
- "ar_open": 8300,
514
- "ar_overdue": 940,
515
- "ar_exposure": 11440,
516
- "top_category": "All",
517
- "top_category_pct": 1.0,
518
- "sku_count": 1,
519
- "top_sku": "(none)",
520
- "days_to_pay": 12,
521
- "_created": "2023-08-15 09:17:00",
522
- "lat": 33.749,
523
- "lon": -84.388,
524
- "odoo_status": "Active",
525
- "dba": "Royal",
526
- "ar_outstanding": 9240
527
- }
528
- ]
529
- }
 
1
+ {
2
+ "fields": [
3
+ {
4
+ "key": "customer",
5
+ "label": "Customer",
6
+ "type": "text",
7
+ "source": "odoo",
8
+ "pinned": true,
9
+ "default": true,
10
+ "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
11
+ },
12
+ {
13
+ "key": "odoo_status",
14
+ "label": "Odoo record",
15
+ "type": "status",
16
+ "source": "odoo",
17
+ "default": false,
18
+ "description": "Whether this customer still exists in Odoo. Archived means deleted there."
19
+ },
20
+ {
21
+ "key": "agent",
22
+ "label": "Agent",
23
+ "type": "text",
24
+ "source": "odoo",
25
+ "default": true,
26
+ "description": "The sales agent who owns this account."
27
+ },
28
+ {
29
+ "key": "dba",
30
+ "label": "DBA",
31
+ "type": "select",
32
+ "source": "odoo",
33
+ "default": false,
34
+ "options": [
35
+ "Fisch",
36
+ "Royal",
37
+ "Both"
38
+ ],
39
+ "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
40
+ },
41
+ {
42
+ "key": "salesperson",
43
+ "label": "Salesperson",
44
+ "type": "text",
45
+ "source": "odoo",
46
+ "default": false,
47
+ "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account."
48
+ },
49
+ {
50
+ "key": "city",
51
+ "label": "City",
52
+ "type": "text",
53
+ "source": "odoo",
54
+ "default": true,
55
+ "description": "City on the customer's Odoo address."
56
+ },
57
+ {
58
+ "key": "state",
59
+ "label": "State",
60
+ "type": "text",
61
+ "source": "odoo",
62
+ "default": true,
63
+ "description": "State or province on the customer's Odoo address."
64
+ },
65
+ {
66
+ "key": "country",
67
+ "label": "Country",
68
+ "type": "text",
69
+ "source": "odoo",
70
+ "default": false,
71
+ "description": "Country on the customer's Odoo address."
72
+ },
73
+ {
74
+ "key": "zip",
75
+ "label": "ZIP",
76
+ "type": "text",
77
+ "source": "odoo",
78
+ "default": false,
79
+ "description": "Postal code on the customer's Odoo address."
80
+ },
81
+ {
82
+ "key": "customer_since",
83
+ "label": "Customer since",
84
+ "type": "date",
85
+ "source": "odoo",
86
+ "default": false,
87
+ "description": "When this customer was first set up in Odoo."
88
+ },
89
+ {
90
+ "key": "tags",
91
+ "label": "Tags",
92
+ "type": "text",
93
+ "source": "odoo",
94
+ "default": false,
95
+ "description": "Odoo labels on this customer, comma-separated."
96
+ },
97
+ {
98
+ "key": "pricelist",
99
+ "label": "Price list",
100
+ "type": "text",
101
+ "source": "odoo",
102
+ "default": false,
103
+ "description": "The price list this customer buys on."
104
+ },
105
+ {
106
+ "key": "payment_terms",
107
+ "label": "Payment terms",
108
+ "type": "text",
109
+ "source": "odoo",
110
+ "default": false,
111
+ "description": "Payment terms on this customer's account — Net 30, for example."
112
+ },
113
+ {
114
+ "key": "last_order",
115
+ "label": "Last order",
116
+ "type": "date",
117
+ "source": "odoo",
118
+ "default": true,
119
+ "description": "Date of the most recent confirmed order."
120
+ },
121
+ {
122
+ "key": "overdue_days",
123
+ "label": "Overdue days",
124
+ "type": "int",
125
+ "source": "odoo",
126
+ "default": true,
127
+ "description": "How many days late this customer is running against their own usual ordering rhythm."
128
+ },
129
+ {
130
+ "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
131
+ "key": "est_missed",
132
+ "label": "Est. missed $",
133
+ "type": "currency",
134
+ "source": "odoo",
135
+ "default": true,
136
+ "agg": "sum",
137
+ "filterable": false,
138
+ "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
139
+ },
140
+ {
141
+ "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
142
+ "key": "ar_open",
143
+ "label": "AR current $",
144
+ "type": "currency",
145
+ "source": "odoo",
146
+ "default": false,
147
+ "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
148
+ },
149
+ {
150
+ "key": "ar_overdue",
151
+ "label": "AR overdue $",
152
+ "type": "currency",
153
+ "source": "odoo",
154
+ "default": false,
155
+ "description": "Invoiced money past due — same basis as the Collections page."
156
+ },
157
+ {
158
+ "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
159
+ "key": "ar_outstanding",
160
+ "label": "AR outstanding $",
161
+ "type": "currency",
162
+ "source": "odoo",
163
+ "default": false,
164
+ "description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
165
+ },
166
+ {
167
+ "key": "ar_exposure",
168
+ "label": "Credit exposure $",
169
+ "type": "currency",
170
+ "source": "odoo",
171
+ "default": false,
172
+ "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
173
+ },
174
+ {
175
+ "key": "ar_aged_1_30",
176
+ "label": "1-30 days $",
177
+ "type": "currency",
178
+ "source": "odoo",
179
+ "default": false,
180
+ "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
181
+ },
182
+ {
183
+ "key": "ar_aged_31_60",
184
+ "label": "31-60 days $",
185
+ "type": "currency",
186
+ "source": "odoo",
187
+ "default": false,
188
+ "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
189
+ },
190
+ {
191
+ "key": "ar_aged_61_90",
192
+ "label": "61-90 days $",
193
+ "type": "currency",
194
+ "source": "odoo",
195
+ "default": false,
196
+ "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
197
+ },
198
+ {
199
+ "key": "ar_aged_90_plus",
200
+ "label": "90+ days $",
201
+ "type": "currency",
202
+ "source": "odoo",
203
+ "default": false,
204
+ "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
205
+ },
206
+ {
207
+ "key": "days_to_pay",
208
+ "label": "Days to pay",
209
+ "type": "int",
210
+ "source": "odoo",
211
+ "default": false,
212
+ "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
213
+ },
214
+ {
215
+ "key": "top_category",
216
+ "label": "Top category",
217
+ "type": "text",
218
+ "source": "odoo",
219
+ "default": false,
220
+ "description": "The category this customer spent the most on in the last 12 months."
221
+ },
222
+ {
223
+ "key": "top_category_pct",
224
+ "label": "Top category %",
225
+ "type": "pct",
226
+ "source": "odoo",
227
+ "default": false,
228
+ "description": "Share of last-12-months spend that went to the top category."
229
+ },
230
+ {
231
+ "key": "sku_count",
232
+ "label": "SKUs bought",
233
+ "type": "int",
234
+ "source": "odoo",
235
+ "default": false,
236
+ "description": "Distinct products bought in the last 12 months."
237
+ },
238
+ {
239
+ "key": "top_sku",
240
+ "label": "Top SKU",
241
+ "type": "text",
242
+ "source": "odoo",
243
+ "default": false,
244
+ "description": "The product this customer spent the most on in the last 12 months."
245
+ },
246
+ {
247
+ "key": "days_since",
248
+ "label": "Days since order",
249
+ "type": "int",
250
+ "source": "odoo",
251
+ "default": false,
252
+ "description": "Days since the last confirmed order."
253
+ },
254
+ {
255
+ "key": "typical_gap_days",
256
+ "label": "Typical gap days",
257
+ "type": "int",
258
+ "source": "odoo",
259
+ "default": false,
260
+ "description": "Days this customer usually goes between orders, from their own history."
261
+ },
262
+ {
263
+ "key": "notes",
264
+ "label": "Notes",
265
+ "type": "text",
266
+ "source": "overlay",
267
+ "default": false,
268
+ "description": "Your notes on this customer. Saved in this app only, visible only to you."
269
+ }
270
+ ],
271
+ "rows": [
272
+ {
273
+ "pid": 101,
274
+ "customer": "Poppy Flowers",
275
+ "status": "New",
276
+ "agent": "Naomi Linnell Rivera",
277
+ "city": "Charlottesville",
278
+ "state": "Virginia (US)",
279
+ "last_order": "2026-07-21",
280
+ "est_missed": 0,
281
+ "notes": "",
282
+ "country": "United States",
283
+ "zip": "02720",
284
+ "payment_terms": "30 Days",
285
+ "pricelist": "Fisch 1 (USD)",
286
+ "tags": "Royal",
287
+ "customer_since": "2023-01-10",
288
+ "salesperson": "Jessica",
289
+ "ar_open": 0,
290
+ "ar_overdue": 0,
291
+ "ar_exposure": 0,
292
+ "top_category": "Styrofoam",
293
+ "top_category_pct": 0.47,
294
+ "sku_count": 93,
295
+ "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
296
+ "days_to_pay": 34,
297
+ "_created": "2023-01-15 09:10:00",
298
+ "lat": 25.7617,
299
+ "lon": -80.1918,
300
+ "odoo_status": "Archived",
301
+ "dba": "Royal",
302
+ "ar_outstanding": 0
303
+ },
304
+ {
305
+ "pid": 102,
306
+ "customer": "Meadow & Vine Wholesale",
307
+ "status": "Growing",
308
+ "agent": "Carla Jimenez",
309
+ "city": "Portland",
310
+ "state": "Oregon (US)",
311
+ "last_order": "2026-07-19",
312
+ "est_missed": 0,
313
+ "notes": "Expanding to a second storefront.",
314
+ "country": "United States",
315
+ "zip": "77041",
316
+ "payment_terms": "Immediate Payment",
317
+ "pricelist": "Royal 1 (USD)",
318
+ "tags": "Royal, Key account",
319
+ "customer_since": "2024-02-11",
320
+ "salesperson": "Naomi",
321
+ "ar_open": 1240.5,
322
+ "ar_overdue": 0,
323
+ "ar_exposure": 1740.5,
324
+ "top_category": "Ribbon",
325
+ "top_category_pct": 0.95,
326
+ "sku_count": 4,
327
+ "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
328
+ "days_to_pay": null,
329
+ "_created": "2023-02-15 09:11:00",
330
+ "lat": 27.9506,
331
+ "lon": -82.4572,
332
+ "odoo_status": "Active",
333
+ "dba": "Fisch",
334
+ "ar_outstanding": 1240.5
335
+ },
336
+ {
337
+ "pid": 103,
338
+ "customer": "Bluestem Floral Supply",
339
+ "status": "Growing",
340
+ "agent": "Naomi Linnell Rivera",
341
+ "city": "Kansas City",
342
+ "state": "Missouri (US)",
343
+ "last_order": "2026-07-17",
344
+ "est_missed": 0,
345
+ "notes": "",
346
+ "country": "United States",
347
+ "zip": "11219",
348
+ "payment_terms": "60 Days",
349
+ "pricelist": "Royal 1 (USD)",
350
+ "tags": "Fisch",
351
+ "customer_since": "2025-03-12",
352
+ "salesperson": "Karen",
353
+ "ar_open": 0,
354
+ "ar_overdue": 5120.25,
355
+ "ar_exposure": 5120.25,
356
+ "top_category": "Foams & Finishes",
357
+ "top_category_pct": 0.31,
358
+ "sku_count": 27,
359
+ "top_sku": "SATIN RIBBON 2IN",
360
+ "days_to_pay": 61,
361
+ "_created": "2023-03-15 09:12:00",
362
+ "lat": 28.5384,
363
+ "lon": -81.3789,
364
+ "odoo_status": "Active",
365
+ "dba": "Both",
366
+ "ar_outstanding": 5120.25
367
+ },
368
+ {
369
+ "pid": 104,
370
+ "customer": "Camellia Row Florist",
371
+ "status": "Declining",
372
+ "agent": "Devon Marsh",
373
+ "city": "Savannah",
374
+ "state": "Georgia (US)",
375
+ "last_order": "2026-05-30",
376
+ "est_missed": 41200,
377
+ "notes": "Switched some volume to a local grower.",
378
+ "country": "United States",
379
+ "zip": "07649",
380
+ "payment_terms": "30 Days",
381
+ "pricelist": "Fisch 1 (USD)",
382
+ "tags": "(none)",
383
+ "customer_since": "2026-04-13",
384
+ "salesperson": "(none)",
385
+ "ar_open": 8300,
386
+ "ar_overdue": 940,
387
+ "ar_exposure": 11440,
388
+ "top_category": "All",
389
+ "top_category_pct": 1.0,
390
+ "sku_count": 1,
391
+ "top_sku": "(none)",
392
+ "days_to_pay": 12,
393
+ "_created": "2023-04-15 09:13:00",
394
+ "lat": 30.3322,
395
+ "lon": -81.6557,
396
+ "odoo_status": "Active",
397
+ "dba": "Royal",
398
+ "ar_outstanding": 9240
399
+ },
400
+ {
401
+ "pid": 105,
402
+ "customer": "Harborlight Wholesale Blooms",
403
+ "status": "Growing",
404
+ "agent": "Carla Jimenez",
405
+ "city": "Seattle",
406
+ "state": "Washington (US)",
407
+ "last_order": "2026-07-22",
408
+ "est_missed": 0,
409
+ "notes": "Top-10 account.",
410
+ "country": "United States",
411
+ "zip": "33125",
412
+ "payment_terms": "Immediate Payment",
413
+ "pricelist": "Royal 1 (USD)",
414
+ "tags": "Royal",
415
+ "customer_since": "2023-05-14",
416
+ "salesperson": "Jessica",
417
+ "ar_open": 0,
418
+ "ar_overdue": 0,
419
+ "ar_exposure": 0,
420
+ "top_category": "Styrofoam",
421
+ "top_category_pct": 0.47,
422
+ "sku_count": 93,
423
+ "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
424
+ "days_to_pay": 34,
425
+ "_created": "2023-05-15 09:14:00",
426
+ "lat": 26.1224,
427
+ "lon": -80.1373,
428
+ "odoo_status": "Active",
429
+ "dba": "",
430
+ "ar_outstanding": 0
431
+ },
432
+ {
433
+ "pid": 106,
434
+ "customer": "Dogwood & Fern Co.",
435
+ "status": "Dormant",
436
+ "agent": "Devon Marsh",
437
+ "city": "Asheville",
438
+ "state": "North Carolina (US)",
439
+ "last_order": "2026-02-11",
440
+ "est_missed": 52400,
441
+ "notes": "No spring order this year.",
442
+ "country": "United States",
443
+ "zip": "90210",
444
+ "payment_terms": "60 Days",
445
+ "pricelist": "Royal 1 (USD)",
446
+ "tags": "Royal, Key account",
447
+ "customer_since": "2024-06-15",
448
+ "salesperson": "Naomi",
449
+ "ar_open": 1240.5,
450
+ "ar_overdue": 0,
451
+ "ar_exposure": 1740.5,
452
+ "top_category": "Ribbon",
453
+ "top_category_pct": 0.95,
454
+ "sku_count": 4,
455
+ "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
456
+ "days_to_pay": null,
457
+ "_created": "2023-06-15 09:15:00",
458
+ "lat": 27.3364,
459
+ "lon": -82.5307,
460
+ "odoo_status": "Active",
461
+ "dba": "Fisch",
462
+ "ar_outstanding": 1240.5
463
+ },
464
+ {
465
+ "pid": 107,
466
+ "customer": "Verbena Market Florals",
467
+ "status": "Lost",
468
+ "agent": "Naomi Linnell Rivera",
469
+ "city": "Austin",
470
+ "state": "Texas (US)",
471
+ "last_order": "2025-11-04",
472
+ "est_missed": 78300,
473
+ "notes": "Went with a competitor on freight terms.",
474
+ "country": "United States",
475
+ "zip": "08701",
476
+ "payment_terms": "30 Days",
477
+ "pricelist": "Fisch 1 (USD)",
478
+ "tags": "Fisch",
479
+ "customer_since": "2025-07-16",
480
+ "salesperson": "Karen",
481
+ "ar_open": 0,
482
+ "ar_overdue": 5120.25,
483
+ "ar_exposure": 5120.25,
484
+ "top_category": "Foams & Finishes",
485
+ "top_category_pct": 0.31,
486
+ "sku_count": 27,
487
+ "top_sku": "SATIN RIBBON 2IN",
488
+ "days_to_pay": 61,
489
+ "_created": "2023-07-15 09:16:00",
490
+ "lat": null,
491
+ "lon": null,
492
+ "odoo_status": "Active",
493
+ "dba": "Both",
494
+ "ar_outstanding": 5120.25
495
+ },
496
+ {
497
+ "pid": 108,
498
+ "customer": "Larkspur Lane Supply",
499
+ "status": "New",
500
+ "agent": "Carla Jimenez",
501
+ "city": "Denver",
502
+ "state": "Colorado (US)",
503
+ "last_order": "2026-07-14",
504
+ "est_missed": 0,
505
+ "notes": "First order in April.",
506
+ "country": "United States",
507
+ "zip": "60614",
508
+ "payment_terms": "Immediate Payment",
509
+ "pricelist": "Royal 1 (USD)",
510
+ "tags": "(none)",
511
+ "customer_since": "2026-08-17",
512
+ "salesperson": "(none)",
513
+ "ar_open": 8300,
514
+ "ar_overdue": 940,
515
+ "ar_exposure": 11440,
516
+ "top_category": "All",
517
+ "top_category_pct": 1.0,
518
+ "sku_count": 1,
519
+ "top_sku": "(none)",
520
+ "days_to_pay": 12,
521
+ "_created": "2023-08-15 09:17:00",
522
+ "lat": 33.749,
523
+ "lon": -84.388,
524
+ "odoo_status": "Active",
525
+ "dba": "Royal",
526
+ "ar_outstanding": 9240
527
+ }
528
+ ]
529
+ }
web/src/alerts/AlertsPane.tsx CHANGED
@@ -1,329 +1,329 @@
1
- // ---------------------------------------------------------------------------
2
- // alerts/AlertsPane.tsx — WAVE 20 item 25 (C-ALERT): the inbox.
3
- //
4
- // Two lists, one panel, and the order is the point: what HAPPENED first, what is
5
- // WATCHING second. An inbox that opens on its own configuration is a settings
6
- // screen wearing a bell.
7
- //
8
- // A drawer rather than a route, for the same reason the AI note is a modal: the
9
- // nav is server-filtered and an undeclared surface is denied by design (this
10
- // shell has no client-invented pages). Alerts are not a granted module — they
11
- // are this account's own inbox over its own views.
12
- // ---------------------------------------------------------------------------
13
-
14
- import { useCallback, useEffect, useState } from "react";
15
- import {
16
- deleteAlert,
17
- fetchAlerts,
18
- fetchInbox,
19
- markRead,
20
- runAlert,
21
- } from "./alertsApi";
22
- import {
23
- EMPTY_INBOX,
24
- applyRead,
25
- inboxOrder,
26
- isAutomationOutcome,
27
- isAutomationReview,
28
- routeForTopic,
29
- stampText,
30
- } from "./alertsModel";
31
- import type { Alert, Inbox, Notification } from "./alertsModel";
32
- // WAVE 27 C5 — the ONE generated loop mark, consumed rather than redrawn (C7's rule).
33
- import { Mark } from "../shell/Brand";
34
-
35
- /** The clock face this pane uses for its one glyph — drawn, never an emoji. */
36
- function BellIcon() {
37
- return (
38
- <svg className="alerts-bell" viewBox="0 0 16 16" aria-hidden="true">
39
- <path d="M8 2.2a3.6 3.6 0 0 1 3.6 3.6v2.4l1.2 2H3.2l1.2-2V5.8A3.6 3.6 0 0 1 8 2.2Z" />
40
- <path d="M6.6 12.6a1.5 1.5 0 0 0 2.8 0" />
41
- </svg>
42
- );
43
- }
44
-
45
- export default function AlertsPane({
46
- onClose,
47
- onOpenView,
48
- onOpenAutomation,
49
- onInbox,
50
- onToast,
51
- }: {
52
- onClose: () => void;
53
- /** Navigate to the alert's view. The FRAME owns routing; this pane owns the row. */
54
- onOpenView: (topic: string, viewId: string) => void;
55
- /**
56
- * WAVE 23 (C6, wiring W23-W5) — the same division for a REVIEW notification: the frame routes
57
- * to `#/automation` and asks the surface to select this automation.
58
- *
59
- * ⛔ REQUIRED, NOT OPTIONAL, and that is the wave-20 lesson written into a type. An optional
60
- * callback the frame forgot to pass degrades to "clicking a review notification does nothing",
61
- * which is indistinguishable from "the feature was never built" and goes red in no gate. A
62
- * required prop fails `tsc` the moment it is unmounted.
63
- */
64
- onOpenAutomation: (autoId: string, stageId?: string) => void;
65
- /** Hand the freshly-read inbox back so the nav badge and the pane agree. */
66
- onInbox: (inbox: Inbox) => void;
67
- onToast: (message: string) => void;
68
- }) {
69
- const [inbox, setInbox] = useState<Inbox>(EMPTY_INBOX);
70
- const [alerts, setAlerts] = useState<Alert[]>([]);
71
- const [error, setError] = useState("");
72
- const [busy, setBusy] = useState(false);
73
-
74
- const publish = useCallback(
75
- (next: Inbox) => {
76
- setInbox(next);
77
- onInbox(next);
78
- },
79
- [onInbox]
80
- );
81
-
82
- const load = useCallback(async () => {
83
- const [inboxRes, alertRes] = await Promise.all([fetchInbox(), fetchAlerts()]);
84
- if (inboxRes.ok) publish(inboxRes.value);
85
- else setError(inboxRes.message);
86
- if (alertRes.ok) setAlerts(alertRes.value);
87
- }, [publish]);
88
-
89
- useEffect(() => {
90
- void load();
91
- }, [load]);
92
-
93
- // ⛔ Escape closes it, like every other panel in this shell — a scrim with no
94
- // keyboard way out is a trap (the wave-18 lesson, kept).
95
- useEffect(() => {
96
- const onKey = (e: KeyboardEvent) => {
97
- if (e.key === "Escape") onClose();
98
- };
99
- window.addEventListener("keydown", onKey);
100
- return () => window.removeEventListener("keydown", onKey);
101
- }, [onClose]);
102
-
103
- const toggleRead = useCallback(
104
- (n: Notification) => {
105
- // Optimistic, with the server's own convention (`ids`, `read`) sent
106
- // explicitly — and reverted out loud if the write is refused, because a
107
- // badge that silently disagrees with the list is how a reader learns to
108
- // stop trusting it.
109
- const before = inbox;
110
- publish(applyRead(inbox, [n.id], !n.read));
111
- void markRead([n.id], !n.read).then((r) => {
112
- if (!r.ok) {
113
- publish(before);
114
- onToast(r.message);
115
- }
116
- });
117
- },
118
- [inbox, publish, onToast]
119
- );
120
-
121
- const markAll = useCallback(() => {
122
- const before = inbox;
123
- publish(applyRead(inbox, null, true));
124
- void markRead(null, true).then((r) => {
125
- if (!r.ok) {
126
- publish(before);
127
- onToast(r.message);
128
- }
129
- });
130
- }, [inbox, publish, onToast]);
131
-
132
- const open = useCallback(
133
- (n: Notification) => {
134
- if (!n.read) {
135
- publish(applyRead(inbox, [n.id], true));
136
- void markRead([n.id], true);
137
- }
138
- // ⛔ WAVE 23 C6 — THE KIND BRANCH COMES FIRST, AND ITS ORDER IS THE WHOLE FIX. A review
139
- // notification has no `topic` a grid could render, so the view guard below would have
140
- // caught every one of them and told the reader their table was "no longer available" —
141
- // a confidently wrong sentence about a feature that is working. The destination decides
142
- // which guard applies, so the destination is decided first.
143
- if (isAutomationReview(n)) {
144
- onOpenAutomation(n.autoId!, n.stageId);
145
- onClose();
146
- return;
147
- }
148
- // ⭐ WAVE 27 C5 — a RUN OUTCOME routes to its automation too, and it carries the id in
149
- // `alertId` rather than `autoId` (`core.alerts.notify`'s shape, not `_queue_review`'s).
150
- // Beside the review branch and after it: a review notification satisfies BOTH predicates,
151
- // and only the review one knows about the stage to scroll to.
152
- if (isAutomationOutcome(n)) {
153
- onOpenAutomation(n.alertId, undefined);
154
- onClose();
155
- return;
156
- }
157
- // A route this client cannot resolve opens NOTHING and says so — alerts
158
- // outlive the surfaces they were made from.
159
- if (!routeForTopic(n.topic)) {
160
- onToast("That alert's table is no longer available to this account.");
161
- return;
162
- }
163
- onOpenView(n.topic, n.viewId);
164
- onClose();
165
- },
166
- [inbox, publish, onOpenView, onOpenAutomation, onClose, onToast]
167
- );
168
-
169
- const rows = inboxOrder(inbox.items);
170
-
171
- return (
172
- <div className="shell-newdb-scrim" onClick={onClose}>
173
- <aside
174
- className="alerts-pane"
175
- role="dialog"
176
- aria-label="Alerts"
177
- onClick={(e) => e.stopPropagation()}
178
- >
179
- <header className="alerts-head">
180
- <h2>
181
- <BellIcon /> Alerts
182
- </h2>
183
- <button
184
- type="button"
185
- className="alerts-markall"
186
- disabled={inbox.unread === 0}
187
- onClick={markAll}
188
- >
189
- Mark all read
190
- </button>
191
- </header>
192
-
193
- {error ? <p className="shell-newdb-err">{error}</p> : null}
194
-
195
- <div className="alerts-scroll">
196
- {rows.length === 0 ? (
197
- <p className="alerts-empty">
198
- Nothing new. An alert watches ONE view and tells you when a record it had
199
- never matched arrives in it — make one from a view's ··· menu.
200
- </p>
201
- ) : (
202
- <ul className="alerts-list">
203
- {rows.map((n) => (
204
- <li key={n.id} className={"alerts-row" + (n.read ? "" : " is-unread")}>
205
- <button
206
- type="button"
207
- className="alerts-row-main"
208
- onClick={() => open(n)}
209
- // The hover names the DESTINATION, which is now two different places.
210
- // "Open the alert's view" over a review notification described a journey
211
- // the click does not take.
212
- title={
213
- isAutomationReview(n) || isAutomationOutcome(n)
214
- ? "Open the automation"
215
- : `Open ${n.alertLabel || "the alert's view"}`
216
- }
217
- >
218
- {/* ⭐ WAVE 27 C5 — an automation's row wears the LOOP MARK.
219
- ⛔ NOT A STATUS DOT, and not a second one: wave-26 R14 deleted every
220
- status dot in the automation module on the grounds that *"these
221
- automations are basically loops"*, and a run that FAILED is already said
222
- in `n.label` (the run summary) in words. A coloured dot here would
223
- reintroduce, in the inbox, the vocabulary that was deleted from the
224
- surface it points at.
225
- ⛔ The generated `Mark`, never a redrawn path: `shell/Brand.tsx`'s header
226
- records a hand-copied version silently painting LAST WAVE'S BRAND while
227
- a comment claimed parity. */}
228
- <span className="alerts-row-label">
229
- {isAutomationOutcome(n) || isAutomationReview(n) ? (
230
- // ⚠ INSIDE the label span, not beside it. `.alerts-row-main` is a flex
231
- // COLUMN, so a sibling here becomes a third ROW stacked above the text
232
- // ([[wrong-parent-not-broken-control]]); inline, it sits on the line it
233
- // annotates and rides the same ellipsis.
234
- <span className="alerts-row-mark">
235
- <Mark size={12} />
236
- </span>
237
- ) : null}
238
- {n.label}
239
- </span>
240
- <span className="alerts-row-meta">
241
- {n.alertLabel ? `${n.alertLabel} · ` : ""}
242
- {/* Readable, but never re-derived: `stampText` is string surgery
243
- over the server's own UTC-with-offset stamp (D-18). Parsing it
244
- into a browser Date is how a tenant a day ahead gets told an
245
- event happened tomorrow. */}
246
- {stampText(n.at)}
247
- </span>
248
- </button>
249
- <button
250
- type="button"
251
- className="alerts-row-toggle"
252
- aria-label={n.read ? `Mark ${n.label} unread` : `Mark ${n.label} read`}
253
- title={n.read ? "Mark unread" : "Mark read"}
254
- onClick={() => toggleRead(n)}
255
- >
256
- {/* The ACTION, not the state: "Read" beside an unread row reads as
257
- a label for the row itself. */}
258
- {n.read ? "Mark unread" : "Mark read"}
259
- </button>
260
- </li>
261
- ))}
262
- </ul>
263
- )}
264
-
265
- {alerts.length > 0 ? (
266
- <section className="alerts-watching">
267
- <h3>Watching</h3>
268
- {alerts.map((a) => (
269
- <div key={a.id} className="alerts-watch-row">
270
- <span className="alerts-watch-label">{a.label}</span>
271
- <span className="alerts-watch-meta">
272
- {/* The remembered set's SIZE, which is what "no news" means here:
273
- an alert with 40 matches and nothing new is working. */}
274
- {a.matched.toLocaleString()} matched
275
- {a.lastError ? ` · ${a.lastError}` : ""}
276
- </span>
277
- <button
278
- type="button"
279
- className="alerts-watch-run"
280
- disabled={busy}
281
- onClick={() => {
282
- setBusy(true);
283
- void runAlert(a.id).then((r) => {
284
- setBusy(false);
285
- if (!r.ok) return onToast(r.message);
286
- const v = r.value as { new?: unknown[]; skipped?: string };
287
- onToast(
288
- v?.skipped
289
- ? `Skipped: ${v.skipped}`
290
- : `${(v?.new ?? []).length} new since the last check.`
291
- );
292
- void load();
293
- });
294
- }}
295
- >
296
- Check now
297
- </button>
298
- <button
299
- type="button"
300
- className="alerts-watch-del"
301
- disabled={busy}
302
- aria-label={`Delete the alert ${a.label}`}
303
- onClick={() => {
304
- setBusy(true);
305
- void deleteAlert(a.id).then((r) => {
306
- setBusy(false);
307
- if (!r.ok) return onToast(r.message);
308
- setAlerts((cur) => cur.filter((x) => x.id !== a.id));
309
- void load();
310
- });
311
- }}
312
- >
313
- Delete
314
- </button>
315
- </div>
316
- ))}
317
- </section>
318
- ) : null}
319
- </div>
320
-
321
- <div className="shell-newdb-actions alerts-foot">
322
- <button type="button" className="login-submit" onClick={onClose}>
323
- Close
324
- </button>
325
- </div>
326
- </aside>
327
- </div>
328
- );
329
- }
 
1
+ // ---------------------------------------------------------------------------
2
+ // alerts/AlertsPane.tsx — WAVE 20 item 25 (C-ALERT): the inbox.
3
+ //
4
+ // Two lists, one panel, and the order is the point: what HAPPENED first, what is
5
+ // WATCHING second. An inbox that opens on its own configuration is a settings
6
+ // screen wearing a bell.
7
+ //
8
+ // A drawer rather than a route, for the same reason the AI note is a modal: the
9
+ // nav is server-filtered and an undeclared surface is denied by design (this
10
+ // shell has no client-invented pages). Alerts are not a granted module — they
11
+ // are this account's own inbox over its own views.
12
+ // ---------------------------------------------------------------------------
13
+
14
+ import { useCallback, useEffect, useState } from "react";
15
+ import {
16
+ deleteAlert,
17
+ fetchAlerts,
18
+ fetchInbox,
19
+ markRead,
20
+ runAlert,
21
+ } from "./alertsApi";
22
+ import {
23
+ EMPTY_INBOX,
24
+ applyRead,
25
+ inboxOrder,
26
+ isAutomationOutcome,
27
+ isAutomationReview,
28
+ routeForTopic,
29
+ stampText,
30
+ } from "./alertsModel";
31
+ import type { Alert, Inbox, Notification } from "./alertsModel";
32
+ // WAVE 27 C5 — the ONE generated loop mark, consumed rather than redrawn (C7's rule).
33
+ import { Mark } from "../shell/Brand";
34
+
35
+ /** The clock face this pane uses for its one glyph — drawn, never an emoji. */
36
+ function BellIcon() {
37
+ return (
38
+ <svg className="alerts-bell" viewBox="0 0 16 16" aria-hidden="true">
39
+ <path d="M8 2.2a3.6 3.6 0 0 1 3.6 3.6v2.4l1.2 2H3.2l1.2-2V5.8A3.6 3.6 0 0 1 8 2.2Z" />
40
+ <path d="M6.6 12.6a1.5 1.5 0 0 0 2.8 0" />
41
+ </svg>
42
+ );
43
+ }
44
+
45
+ export default function AlertsPane({
46
+ onClose,
47
+ onOpenView,
48
+ onOpenAutomation,
49
+ onInbox,
50
+ onToast,
51
+ }: {
52
+ onClose: () => void;
53
+ /** Navigate to the alert's view. The FRAME owns routing; this pane owns the row. */
54
+ onOpenView: (topic: string, viewId: string) => void;
55
+ /**
56
+ * WAVE 23 (C6, wiring W23-W5) — the same division for a REVIEW notification: the frame routes
57
+ * to `#/automation` and asks the surface to select this automation.
58
+ *
59
+ * ⛔ REQUIRED, NOT OPTIONAL, and that is the wave-20 lesson written into a type. An optional
60
+ * callback the frame forgot to pass degrades to "clicking a review notification does nothing",
61
+ * which is indistinguishable from "the feature was never built" and goes red in no gate. A
62
+ * required prop fails `tsc` the moment it is unmounted.
63
+ */
64
+ onOpenAutomation: (autoId: string, stageId?: string) => void;
65
+ /** Hand the freshly-read inbox back so the nav badge and the pane agree. */
66
+ onInbox: (inbox: Inbox) => void;
67
+ onToast: (message: string) => void;
68
+ }) {
69
+ const [inbox, setInbox] = useState<Inbox>(EMPTY_INBOX);
70
+ const [alerts, setAlerts] = useState<Alert[]>([]);
71
+ const [error, setError] = useState("");
72
+ const [busy, setBusy] = useState(false);
73
+
74
+ const publish = useCallback(
75
+ (next: Inbox) => {
76
+ setInbox(next);
77
+ onInbox(next);
78
+ },
79
+ [onInbox]
80
+ );
81
+
82
+ const load = useCallback(async () => {
83
+ const [inboxRes, alertRes] = await Promise.all([fetchInbox(), fetchAlerts()]);
84
+ if (inboxRes.ok) publish(inboxRes.value);
85
+ else setError(inboxRes.message);
86
+ if (alertRes.ok) setAlerts(alertRes.value);
87
+ }, [publish]);
88
+
89
+ useEffect(() => {
90
+ void load();
91
+ }, [load]);
92
+
93
+ // ⛔ Escape closes it, like every other panel in this shell — a scrim with no
94
+ // keyboard way out is a trap (the wave-18 lesson, kept).
95
+ useEffect(() => {
96
+ const onKey = (e: KeyboardEvent) => {
97
+ if (e.key === "Escape") onClose();
98
+ };
99
+ window.addEventListener("keydown", onKey);
100
+ return () => window.removeEventListener("keydown", onKey);
101
+ }, [onClose]);
102
+
103
+ const toggleRead = useCallback(
104
+ (n: Notification) => {
105
+ // Optimistic, with the server's own convention (`ids`, `read`) sent
106
+ // explicitly — and reverted out loud if the write is refused, because a
107
+ // badge that silently disagrees with the list is how a reader learns to
108
+ // stop trusting it.
109
+ const before = inbox;
110
+ publish(applyRead(inbox, [n.id], !n.read));
111
+ void markRead([n.id], !n.read).then((r) => {
112
+ if (!r.ok) {
113
+ publish(before);
114
+ onToast(r.message);
115
+ }
116
+ });
117
+ },
118
+ [inbox, publish, onToast]
119
+ );
120
+
121
+ const markAll = useCallback(() => {
122
+ const before = inbox;
123
+ publish(applyRead(inbox, null, true));
124
+ void markRead(null, true).then((r) => {
125
+ if (!r.ok) {
126
+ publish(before);
127
+ onToast(r.message);
128
+ }
129
+ });
130
+ }, [inbox, publish, onToast]);
131
+
132
+ const open = useCallback(
133
+ (n: Notification) => {
134
+ if (!n.read) {
135
+ publish(applyRead(inbox, [n.id], true));
136
+ void markRead([n.id], true);
137
+ }
138
+ // ⛔ WAVE 23 C6 — THE KIND BRANCH COMES FIRST, AND ITS ORDER IS THE WHOLE FIX. A review
139
+ // notification has no `topic` a grid could render, so the view guard below would have
140
+ // caught every one of them and told the reader their table was "no longer available" —
141
+ // a confidently wrong sentence about a feature that is working. The destination decides
142
+ // which guard applies, so the destination is decided first.
143
+ if (isAutomationReview(n)) {
144
+ onOpenAutomation(n.autoId!, n.stageId);
145
+ onClose();
146
+ return;
147
+ }
148
+ // ⭐ WAVE 27 C5 — a RUN OUTCOME routes to its automation too, and it carries the id in
149
+ // `alertId` rather than `autoId` (`core.alerts.notify`'s shape, not `_queue_review`'s).
150
+ // Beside the review branch and after it: a review notification satisfies BOTH predicates,
151
+ // and only the review one knows about the stage to scroll to.
152
+ if (isAutomationOutcome(n)) {
153
+ onOpenAutomation(n.alertId, undefined);
154
+ onClose();
155
+ return;
156
+ }
157
+ // A route this client cannot resolve opens NOTHING and says so — alerts
158
+ // outlive the surfaces they were made from.
159
+ if (!routeForTopic(n.topic)) {
160
+ onToast("That alert's table is no longer available to this account.");
161
+ return;
162
+ }
163
+ onOpenView(n.topic, n.viewId);
164
+ onClose();
165
+ },
166
+ [inbox, publish, onOpenView, onOpenAutomation, onClose, onToast]
167
+ );
168
+
169
+ const rows = inboxOrder(inbox.items);
170
+
171
+ return (
172
+ <div className="shell-newdb-scrim" onClick={onClose}>
173
+ <aside
174
+ className="alerts-pane"
175
+ role="dialog"
176
+ aria-label="Alerts"
177
+ onClick={(e) => e.stopPropagation()}
178
+ >
179
+ <header className="alerts-head">
180
+ <h2>
181
+ <BellIcon /> Alerts
182
+ </h2>
183
+ <button
184
+ type="button"
185
+ className="alerts-markall"
186
+ disabled={inbox.unread === 0}
187
+ onClick={markAll}
188
+ >
189
+ Mark all read
190
+ </button>
191
+ </header>
192
+
193
+ {error ? <p className="shell-newdb-err">{error}</p> : null}
194
+
195
+ <div className="alerts-scroll">
196
+ {rows.length === 0 ? (
197
+ <p className="alerts-empty">
198
+ Nothing new. An alert watches ONE view and tells you when a record it had
199
+ never matched arrives in it — make one from a view's ··· menu.
200
+ </p>
201
+ ) : (
202
+ <ul className="alerts-list">
203
+ {rows.map((n) => (
204
+ <li key={n.id} className={"alerts-row" + (n.read ? "" : " is-unread")}>
205
+ <button
206
+ type="button"
207
+ className="alerts-row-main"
208
+ onClick={() => open(n)}
209
+ // The hover names the DESTINATION, which is now two different places.
210
+ // "Open the alert's view" over a review notification described a journey
211
+ // the click does not take.
212
+ title={
213
+ isAutomationReview(n) || isAutomationOutcome(n)
214
+ ? "Open the automation"
215
+ : `Open ${n.alertLabel || "the alert's view"}`
216
+ }
217
+ >
218
+ {/* ⭐ WAVE 27 C5 — an automation's row wears the LOOP MARK.
219
+ ⛔ NOT A STATUS DOT, and not a second one: wave-26 R14 deleted every
220
+ status dot in the automation module on the grounds that *"these
221
+ automations are basically loops"*, and a run that FAILED is already said
222
+ in `n.label` (the run summary) in words. A coloured dot here would
223
+ reintroduce, in the inbox, the vocabulary that was deleted from the
224
+ surface it points at.
225
+ ⛔ The generated `Mark`, never a redrawn path: `shell/Brand.tsx`'s header
226
+ records a hand-copied version silently painting LAST WAVE'S BRAND while
227
+ a comment claimed parity. */}
228
+ <span className="alerts-row-label">
229
+ {isAutomationOutcome(n) || isAutomationReview(n) ? (
230
+ // ⚠ INSIDE the label span, not beside it. `.alerts-row-main` is a flex
231
+ // COLUMN, so a sibling here becomes a third ROW stacked above the text
232
+ // ([[wrong-parent-not-broken-control]]); inline, it sits on the line it
233
+ // annotates and rides the same ellipsis.
234
+ <span className="alerts-row-mark">
235
+ <Mark size={12} />
236
+ </span>
237
+ ) : null}
238
+ {n.label}
239
+ </span>
240
+ <span className="alerts-row-meta">
241
+ {n.alertLabel ? `${n.alertLabel} · ` : ""}
242
+ {/* Readable, but never re-derived: `stampText` is string surgery
243
+ over the server's own UTC-with-offset stamp (D-18). Parsing it
244
+ into a browser Date is how a tenant a day ahead gets told an
245
+ event happened tomorrow. */}
246
+ {stampText(n.at)}
247
+ </span>
248
+ </button>
249
+ <button
250
+ type="button"
251
+ className="alerts-row-toggle"
252
+ aria-label={n.read ? `Mark ${n.label} unread` : `Mark ${n.label} read`}
253
+ title={n.read ? "Mark unread" : "Mark read"}
254
+ onClick={() => toggleRead(n)}
255
+ >
256
+ {/* The ACTION, not the state: "Read" beside an unread row reads as
257
+ a label for the row itself. */}
258
+ {n.read ? "Mark unread" : "Mark read"}
259
+ </button>
260
+ </li>
261
+ ))}
262
+ </ul>
263
+ )}
264
+
265
+ {alerts.length > 0 ? (
266
+ <section className="alerts-watching">
267
+ <h3>Watching</h3>
268
+ {alerts.map((a) => (
269
+ <div key={a.id} className="alerts-watch-row">
270
+ <span className="alerts-watch-label">{a.label}</span>
271
+ <span className="alerts-watch-meta">
272
+ {/* The remembered set's SIZE, which is what "no news" means here:
273
+ an alert with 40 matches and nothing new is working. */}
274
+ {a.matched.toLocaleString()} matched
275
+ {a.lastError ? ` · ${a.lastError}` : ""}
276
+ </span>
277
+ <button
278
+ type="button"
279
+ className="alerts-watch-run"
280
+ disabled={busy}
281
+ onClick={() => {
282
+ setBusy(true);
283
+ void runAlert(a.id).then((r) => {
284
+ setBusy(false);
285
+ if (!r.ok) return onToast(r.message);
286
+ const v = r.value as { new?: unknown[]; skipped?: string };
287
+ onToast(
288
+ v?.skipped
289
+ ? `Skipped: ${v.skipped}`
290
+ : `${(v?.new ?? []).length} new since the last check.`
291
+ );
292
+ void load();
293
+ });
294
+ }}
295
+ >
296
+ Check now
297
+ </button>
298
+ <button
299
+ type="button"
300
+ className="alerts-watch-del"
301
+ disabled={busy}
302
+ aria-label={`Delete the alert ${a.label}`}
303
+ onClick={() => {
304
+ setBusy(true);
305
+ void deleteAlert(a.id).then((r) => {
306
+ setBusy(false);
307
+ if (!r.ok) return onToast(r.message);
308
+ setAlerts((cur) => cur.filter((x) => x.id !== a.id));
309
+ void load();
310
+ });
311
+ }}
312
+ >
313
+ Delete
314
+ </button>
315
+ </div>
316
+ ))}
317
+ </section>
318
+ ) : null}
319
+ </div>
320
+
321
+ <div className="shell-newdb-actions alerts-foot">
322
+ <button type="button" className="login-submit" onClick={onClose}>
323
+ Close
324
+ </button>
325
+ </div>
326
+ </aside>
327
+ </div>
328
+ );
329
+ }
web/src/apiContract.ts CHANGED
@@ -1,237 +1,237 @@
1
- // ---------------------------------------------------------------------------
2
- // apiContract.ts — the handful of constants BOTH trees need, and the two
3
- // browser-level signals the data layer raises to the frame.
4
- //
5
- // WHY A ROOT-LEVEL LEAF. `shell/session.ts` and `customer-grid/apiBridge.ts`
6
- // both speak X2, so both need the version prefix and the credentials word. The
7
- // alternatives were worse: a second copy of `"/api/v1"` is a string that drifts
8
- // (this codebase keeps a lock-step GATE for exactly that class of duplication),
9
- // and importing shell code from `customer-grid/**` would point the dependency
10
- // edge the wrong way — the grid is the ASSET, the shell is the disposable
11
- // frame, and the embed bundle must never grow a reason to pull the frame in.
12
- // A leaf both import is the only arrangement with no cycle and no drift.
13
- //
14
- // It imports nothing, by construction.
15
- // ---------------------------------------------------------------------------
16
-
17
- /** X2: every route lives under this prefix. `/api/health` is the one exception
18
- * and is not ours — it is the unauthenticated liveness probe. */
19
- export const API_V1 = "/api/v1";
20
-
21
- /**
22
- * ⚠ EXPLICIT, not defaulted. Same-origin is already the browser default, but
23
- * this is the one word that decides whether the X3 session cookie rides the
24
- * request at all — and the cookie is HttpOnly, so getting it wrong produces no
25
- * client-side symptom, just a 401 from a server that never saw a session. An
26
- * implicit default is not something a reader (or a gate) can check.
27
- */
28
- export const CREDENTIALS: RequestCredentials = "same-origin";
29
-
30
- /**
31
- * THE SESSION DIED UNDER US. Raised by any authenticated call that comes back
32
- * 401, and handled by the shell (sign out, show the door).
33
- *
34
- * A DOM CustomEvent rather than an import, deliberately: `customer-grid/**` is
35
- * host-neutral and must not know a shell exists — it is the same tree the
36
- * Streamlit embed ships. The window is the one channel both sides already
37
- * share (`hostBridge` uses it for the host-render signal for the same reason).
38
- */
39
- export const UNAUTHORIZED_EVENT = "aios:unauthorized";
40
-
41
- /**
42
- * ⭐ THE TENANT THIS RESPONSE WAS SERVED FOR — the header that closes a
43
- * cross-tenant SIGHTING the 401 path structurally cannot catch.
44
- *
45
- * ⛔ THE BUG, owner-reported 2026-08-09: *"I am able to see the automation of
46
- * tenant Nurilab, when logged into Royal Import's."* MEASURED: it is not a
47
- * server leak. Automations live in per-tenant dataset REPOS, `/automations`
48
- * reads `session.runtime` = `get_runtime(claims["t"])` off the SIGNED cookie,
49
- * and `_user_for` refuses when the cookie's tenant is not the account's. Every
50
- * link holds. What does NOT hold is the browser:
51
- *
52
- * `aios_session` is ONE cookie, `path="/"`, per ORIGIN. So one browser can
53
- * hold exactly one tenant session at a time. Sign into tenant B in a second
54
- * tab and the FIRST tab is silently repointed — it keeps painting tenant A's
55
- * chrome (nav, page, the automations it already fetched) while every new
56
- * request it makes is answered for tenant B. Two tenants on one screen, and
57
- * the server was right every time.
58
- *
59
- * ⚠ AND IT IS NOT COSMETIC: a write issued from the stale tab lands in the
60
- * OTHER tenant's store, because the cookie decides. That is the same event the
61
- * owner reports as "I updated the data and it doesn't register the change" —
62
- * it registered, in the wrong tenant.
63
- *
64
- * `handledUnauthorized` cannot see this: a repointed tab gets 200s, not 401s.
65
- * Its own comment already names the neighbouring hazard ("one browser, two
66
- * accounts, and a cached book served across the boundary") — this is that rule
67
- * one level up, at the TENANT boundary rather than the user one.
68
- */
69
- export const TENANT_HEADER = "X-AIOS-Tenant";
70
-
71
- /** The tenant this frame BOOTED for; `null` until the first authenticated
72
- * answer names one. Module-level on purpose — it must outlive every component
73
- * that could be unmounted by the very reset it triggers. */
74
- let bootTenant: string | null = null;
75
-
76
- /** Test seam ONLY — `verify_login.py` drives the comparator without a browser. */
77
- export function _resetTenantGuard(): void {
78
- bootTenant = null;
79
- }
80
-
81
- export function currentTenant(): string | null {
82
- return bootTenant;
83
- }
84
-
85
- /**
86
- * Compare a response's tenant stamp against the one this frame booted with.
87
- * Returns true when it detected a SWAP and handled it.
88
- *
89
- * ⚠ RELOAD, NEVER A PARTIAL RESET. There is no correct way to re-point a live
90
- * frame at another tenant: its nav, its route, its grid caches, its localStorage
91
- * bucket and its in-flight requests were all resolved for the old one. A full
92
- * reload is the only action that cannot leave two tenants blended, and it lands
93
- * the user in the tenant they actually signed into.
94
- *
95
- * ⚠ An ABSENT header is not a mismatch. Unauthenticated routes and any older
96
- * build serve none, and treating absent as "changed" would reload the app in a
97
- * loop — the failure mode that would be worse than the bug.
98
- */
99
- export function checkTenant(res: { headers: { get(name: string): string | null } }): boolean {
100
- let seen: string | null = null;
101
- try {
102
- seen = res.headers.get(TENANT_HEADER);
103
- } catch {
104
- return false;
105
- }
106
- if (!seen) return false;
107
- if (bootTenant === null) {
108
- bootTenant = seen;
109
- return false;
110
- }
111
- if (bootTenant === seen) return false;
112
- bootTenant = seen;
113
- try {
114
- if (typeof window !== "undefined" && window.location) window.location.reload();
115
- } catch {
116
- /* node (the gate) has no window — the comparator is what is under test */
117
- }
118
- return true;
119
- }
120
-
121
- /** The authenticated read FAILED for a reason that is not authentication. The
122
- * frame says so plainly; it never substitutes sample data (see apiBridge). */
123
- export const DATA_ERROR_EVENT = "aios:data-error";
124
-
125
- /** The server sent a human-readable confirmation with a write (X2's `toast`).
126
- * `detail` is the string. */
127
- export const TOAST_EVENT = "aios:toast";
128
-
129
- /**
130
- * owner item 2 (2026-08-03) — the events response carried DERIVED CELLS with it.
131
- *
132
- * Creating a measure column used to take two sequential round trips before a number appeared:
133
- * one to persist the field, one to compute it. The server now computes right after the write
134
- * and returns the values on the same response; `detail` is `{[pid: string]: {[key]: value}}`,
135
- * exactly the `derived` shape `/workspace` sends.
136
- *
137
- * ⚠ A SHORTCUT, NEVER A PATH. `WORKSPACE_STALE_EVENT` still fires beside it and the re-read
138
- * still delivers the same values — so a browser that misses this, or a server that could not
139
- * compute it, behaves exactly as it did before. Nothing may be built on it arriving.
140
- */
141
- export const DERIVED_CELLS_EVENT = "aios:derived-cells";
142
-
143
- /** A write CHANGED durable workspace state (a cohort's membership, a list add, a
144
- * folder move) and the client's copy is now stale.
145
- *
146
- * ⛔ WHY THIS EXISTS. `/customers` carries rows; the workspace lives at its own
147
- * URL. So in the embed the Streamlit host reruns and the left panel repaints,
148
- * while standalone had NOTHING — the server's `toast` was the only evidence a
149
- * cohort add had happened, and the panel beside it still showed the old
150
- * membership. A toast is a receipt, not a refresh. */
151
- export const WORKSPACE_STALE_EVENT = "aios:workspace-stale";
152
-
153
- /** Owner item 10 (2026-07-31): the user CLICKED INTO THE WORK SURFACE (a saved view, a grid
154
- * cell) — the frame should fold its navigation rail down to the slim strip so the table gets
155
- * the width. Raised by the grid, handled by the shell; a no-op in the embed (no listener),
156
- * which is exactly the host-neutral contract the other signals follow. */
157
- export const NAV_MINIMIZE_EVENT = "aios:nav-minimize";
158
-
159
- /** Wave 18 (C3-UT): rows changed OUTSIDE the grid's own write path — a shell "Add record",
160
- * an automation run — and the current topic's rows should be refetched. The grid clears its
161
- * rows cache and reloads; senders call `clearCustomersCache()` first so the refetch cannot
162
- * be served from the 5-minute memo. */
163
- export const ROWS_STALE_EVENT = "aios:rows-stale";
164
-
165
- export function signal(name: string, detail?: unknown): void {
166
- if (typeof window === "undefined") return;
167
- window.dispatchEvent(new CustomEvent(name, { detail }));
168
- }
169
-
170
- /**
171
- * Wave 20 (item 25) — the shell→grid channel for an alert's click-through.
172
- *
173
- * A notification names `{topic, viewId}`; the SHELL routes to the table and the GRID owns view
174
- * selection, so neither has to learn the other's state. An alert for a view the reader can no
175
- * longer see must do NOTHING rather than throw — the listener checks its own view list first.
176
- */
177
- export const VIEW_OPEN_EVENT = "aios:view-open";
178
-
179
- /** `detail` of {@link VIEW_OPEN_EVENT}. */
180
- export interface ViewOpenDetail {
181
- topic: string;
182
- viewId: string;
183
- }
184
-
185
- /**
186
- * WAVE 23 (contract C6, wiring W23-W5) — the shell→automation channel for a review
187
- * notification's click-through.
188
- *
189
- * ⛔ THE SAME SHAPE AS `VIEW_OPEN_EVENT` ABOVE, FOR THE SAME REASON, and wave 20 is why both
190
- * exist as constants in this leaf rather than as a string typed twice. A card arriving at a
191
- * review stage queues an `automation_review` notification naming `{autoId, stageId, count}`;
192
- * clicking it has to do TWO things that live on opposite sides of an ownership fence — route to
193
- * `#/automation` (the SHELL's hash) and select that automation in the rail (`AutomationSurface`'s
194
- * own `activeId`, which the shell cannot see and must not learn). So the frame navigates and
195
- * then ASKS, and the surface answers if it can.
196
- *
197
- * ⚠ THE LISTENER IS THE HALF THAT CAN BE ABSENT. Wave 20 shipped item 25's click-through with
198
- * this exact shape and NO listener — the event was dispatched into nothing, every gate green,
199
- * the notification landing the reader on the right table and doing nothing else. Declared here
200
- * on the wave's first day precisely so the surface can wire the listener while it is being
201
- * built rather than at close-out; the wiring row (W23-W5) asserts both ends.
202
- *
203
- * An automation the reader can no longer open must do NOTHING rather than throw — the listener
204
- * checks its own list first, exactly as the grid does for a view it cannot see.
205
- */
206
- export const AUTOMATION_OPEN_EVENT = "aios:automation-open";
207
-
208
- /** `detail` of {@link AUTOMATION_OPEN_EVENT}. `stageId` is advisory — a surface that does not
209
- * scroll to a stage simply selects the automation. */
210
- export interface AutomationOpenDetail {
211
- autoId: string;
212
- stageId?: string;
213
- }
214
-
215
- /*
216
- * ⛔ `AUTOMATION_CREATE_EVENT` STOOD HERE AND IS DELETED WHOLE (wave 25 item 5a, ruling R8).
217
- *
218
- * It existed for ONE purpose: the "Automated database" doors on Home, in the Database flyout's
219
- * create menu and in the New-database dialog routed to `#/automation` and then raised this so the
220
- * surface would open its create flow. R8 deletes all three doors — "creating a database is one
221
- * act; pointing an automation at it is another" — and `Shell.tsx`'s `openAutomated` was this
222
- * event's ONLY signaller.
223
- *
224
- * ⛔ SO IT HAD TO GO WITH THEM, and the reason is this repo's own scar tissue rather than tidiness.
225
- * Left behind, it would be a constant with a listener and no signaller — the exact mirror of the
226
- * defect wave 24 found here (declared, signalled, consumed NOWHERE, every gate green, three doors
227
- * that navigated and then did nothing). A one-sided event is indistinguishable from a working one
228
- * from every direction except a grep, in BOTH directions, so the fix is symmetric: delete the side
229
- * that is left, never leave the half that compiles.
230
- *
231
- * ⚠ `AUTOMATION_OPEN_EVENT` above is UNAFFECTED and still two-sided (the shell signals it from
232
- * Home's automation tiles and the alerts click-through; `AutomationSurface` listens at module
233
- * scope). `verify_automation_ui.py` now DERIVES that rule instead of naming these two constants:
234
- * every `AUTOMATION_*_EVENT` declared in this file must have a signal site AND a listener site.
235
- * Creating an automation has one door and it is on the automation surface, which is where W24 put
236
- * the front door anyway (`AutomationSurface`'s rail button and its empty state).
237
- */
 
1
+ // ---------------------------------------------------------------------------
2
+ // apiContract.ts — the handful of constants BOTH trees need, and the two
3
+ // browser-level signals the data layer raises to the frame.
4
+ //
5
+ // WHY A ROOT-LEVEL LEAF. `shell/session.ts` and `customer-grid/apiBridge.ts`
6
+ // both speak X2, so both need the version prefix and the credentials word. The
7
+ // alternatives were worse: a second copy of `"/api/v1"` is a string that drifts
8
+ // (this codebase keeps a lock-step GATE for exactly that class of duplication),
9
+ // and importing shell code from `customer-grid/**` would point the dependency
10
+ // edge the wrong way — the grid is the ASSET, the shell is the disposable
11
+ // frame, and the embed bundle must never grow a reason to pull the frame in.
12
+ // A leaf both import is the only arrangement with no cycle and no drift.
13
+ //
14
+ // It imports nothing, by construction.
15
+ // ---------------------------------------------------------------------------
16
+
17
+ /** X2: every route lives under this prefix. `/api/health` is the one exception
18
+ * and is not ours — it is the unauthenticated liveness probe. */
19
+ export const API_V1 = "/api/v1";
20
+
21
+ /**
22
+ * ⚠ EXPLICIT, not defaulted. Same-origin is already the browser default, but
23
+ * this is the one word that decides whether the X3 session cookie rides the
24
+ * request at all — and the cookie is HttpOnly, so getting it wrong produces no
25
+ * client-side symptom, just a 401 from a server that never saw a session. An
26
+ * implicit default is not something a reader (or a gate) can check.
27
+ */
28
+ export const CREDENTIALS: RequestCredentials = "same-origin";
29
+
30
+ /**
31
+ * THE SESSION DIED UNDER US. Raised by any authenticated call that comes back
32
+ * 401, and handled by the shell (sign out, show the door).
33
+ *
34
+ * A DOM CustomEvent rather than an import, deliberately: `customer-grid/**` is
35
+ * host-neutral and must not know a shell exists — it is the same tree the
36
+ * Streamlit embed ships. The window is the one channel both sides already
37
+ * share (`hostBridge` uses it for the host-render signal for the same reason).
38
+ */
39
+ export const UNAUTHORIZED_EVENT = "aios:unauthorized";
40
+
41
+ /**
42
+ * ⭐ THE TENANT THIS RESPONSE WAS SERVED FOR — the header that closes a
43
+ * cross-tenant SIGHTING the 401 path structurally cannot catch.
44
+ *
45
+ * ⛔ THE BUG, owner-reported 2026-08-09: *"I am able to see the automation of
46
+ * tenant Nurilab, when logged into Royal Import's."* MEASURED: it is not a
47
+ * server leak. Automations live in per-tenant dataset REPOS, `/automations`
48
+ * reads `session.runtime` = `get_runtime(claims["t"])` off the SIGNED cookie,
49
+ * and `_user_for` refuses when the cookie's tenant is not the account's. Every
50
+ * link holds. What does NOT hold is the browser:
51
+ *
52
+ * `aios_session` is ONE cookie, `path="/"`, per ORIGIN. So one browser can
53
+ * hold exactly one tenant session at a time. Sign into tenant B in a second
54
+ * tab and the FIRST tab is silently repointed — it keeps painting tenant A's
55
+ * chrome (nav, page, the automations it already fetched) while every new
56
+ * request it makes is answered for tenant B. Two tenants on one screen, and
57
+ * the server was right every time.
58
+ *
59
+ * ⚠ AND IT IS NOT COSMETIC: a write issued from the stale tab lands in the
60
+ * OTHER tenant's store, because the cookie decides. That is the same event the
61
+ * owner reports as "I updated the data and it doesn't register the change" —
62
+ * it registered, in the wrong tenant.
63
+ *
64
+ * `handledUnauthorized` cannot see this: a repointed tab gets 200s, not 401s.
65
+ * Its own comment already names the neighbouring hazard ("one browser, two
66
+ * accounts, and a cached book served across the boundary") — this is that rule
67
+ * one level up, at the TENANT boundary rather than the user one.
68
+ */
69
+ export const TENANT_HEADER = "X-AIOS-Tenant";
70
+
71
+ /** The tenant this frame BOOTED for; `null` until the first authenticated
72
+ * answer names one. Module-level on purpose — it must outlive every component
73
+ * that could be unmounted by the very reset it triggers. */
74
+ let bootTenant: string | null = null;
75
+
76
+ /** Test seam ONLY — `verify_login.py` drives the comparator without a browser. */
77
+ export function _resetTenantGuard(): void {
78
+ bootTenant = null;
79
+ }
80
+
81
+ export function currentTenant(): string | null {
82
+ return bootTenant;
83
+ }
84
+
85
+ /**
86
+ * Compare a response's tenant stamp against the one this frame booted with.
87
+ * Returns true when it detected a SWAP and handled it.
88
+ *
89
+ * ⚠ RELOAD, NEVER A PARTIAL RESET. There is no correct way to re-point a live
90
+ * frame at another tenant: its nav, its route, its grid caches, its localStorage
91
+ * bucket and its in-flight requests were all resolved for the old one. A full
92
+ * reload is the only action that cannot leave two tenants blended, and it lands
93
+ * the user in the tenant they actually signed into.
94
+ *
95
+ * ⚠ An ABSENT header is not a mismatch. Unauthenticated routes and any older
96
+ * build serve none, and treating absent as "changed" would reload the app in a
97
+ * loop — the failure mode that would be worse than the bug.
98
+ */
99
+ export function checkTenant(res: { headers: { get(name: string): string | null } }): boolean {
100
+ let seen: string | null = null;
101
+ try {
102
+ seen = res.headers.get(TENANT_HEADER);
103
+ } catch {
104
+ return false;
105
+ }
106
+ if (!seen) return false;
107
+ if (bootTenant === null) {
108
+ bootTenant = seen;
109
+ return false;
110
+ }
111
+ if (bootTenant === seen) return false;
112
+ bootTenant = seen;
113
+ try {
114
+ if (typeof window !== "undefined" && window.location) window.location.reload();
115
+ } catch {
116
+ /* node (the gate) has no window — the comparator is what is under test */
117
+ }
118
+ return true;
119
+ }
120
+
121
+ /** The authenticated read FAILED for a reason that is not authentication. The
122
+ * frame says so plainly; it never substitutes sample data (see apiBridge). */
123
+ export const DATA_ERROR_EVENT = "aios:data-error";
124
+
125
+ /** The server sent a human-readable confirmation with a write (X2's `toast`).
126
+ * `detail` is the string. */
127
+ export const TOAST_EVENT = "aios:toast";
128
+
129
+ /**
130
+ * owner item 2 (2026-08-03) — the events response carried DERIVED CELLS with it.
131
+ *
132
+ * Creating a measure column used to take two sequential round trips before a number appeared:
133
+ * one to persist the field, one to compute it. The server now computes right after the write
134
+ * and returns the values on the same response; `detail` is `{[pid: string]: {[key]: value}}`,
135
+ * exactly the `derived` shape `/workspace` sends.
136
+ *
137
+ * ⚠ A SHORTCUT, NEVER A PATH. `WORKSPACE_STALE_EVENT` still fires beside it and the re-read
138
+ * still delivers the same values — so a browser that misses this, or a server that could not
139
+ * compute it, behaves exactly as it did before. Nothing may be built on it arriving.
140
+ */
141
+ export const DERIVED_CELLS_EVENT = "aios:derived-cells";
142
+
143
+ /** A write CHANGED durable workspace state (a cohort's membership, a list add, a
144
+ * folder move) and the client's copy is now stale.
145
+ *
146
+ * ⛔ WHY THIS EXISTS. `/customers` carries rows; the workspace lives at its own
147
+ * URL. So in the embed the Streamlit host reruns and the left panel repaints,
148
+ * while standalone had NOTHING — the server's `toast` was the only evidence a
149
+ * cohort add had happened, and the panel beside it still showed the old
150
+ * membership. A toast is a receipt, not a refresh. */
151
+ export const WORKSPACE_STALE_EVENT = "aios:workspace-stale";
152
+
153
+ /** Owner item 10 (2026-07-31): the user CLICKED INTO THE WORK SURFACE (a saved view, a grid
154
+ * cell) — the frame should fold its navigation rail down to the slim strip so the table gets
155
+ * the width. Raised by the grid, handled by the shell; a no-op in the embed (no listener),
156
+ * which is exactly the host-neutral contract the other signals follow. */
157
+ export const NAV_MINIMIZE_EVENT = "aios:nav-minimize";
158
+
159
+ /** Wave 18 (C3-UT): rows changed OUTSIDE the grid's own write path — a shell "Add record",
160
+ * an automation run — and the current topic's rows should be refetched. The grid clears its
161
+ * rows cache and reloads; senders call `clearCustomersCache()` first so the refetch cannot
162
+ * be served from the 5-minute memo. */
163
+ export const ROWS_STALE_EVENT = "aios:rows-stale";
164
+
165
+ export function signal(name: string, detail?: unknown): void {
166
+ if (typeof window === "undefined") return;
167
+ window.dispatchEvent(new CustomEvent(name, { detail }));
168
+ }
169
+
170
+ /**
171
+ * Wave 20 (item 25) — the shell→grid channel for an alert's click-through.
172
+ *
173
+ * A notification names `{topic, viewId}`; the SHELL routes to the table and the GRID owns view
174
+ * selection, so neither has to learn the other's state. An alert for a view the reader can no
175
+ * longer see must do NOTHING rather than throw — the listener checks its own view list first.
176
+ */
177
+ export const VIEW_OPEN_EVENT = "aios:view-open";
178
+
179
+ /** `detail` of {@link VIEW_OPEN_EVENT}. */
180
+ export interface ViewOpenDetail {
181
+ topic: string;
182
+ viewId: string;
183
+ }
184
+
185
+ /**
186
+ * WAVE 23 (contract C6, wiring W23-W5) — the shell→automation channel for a review
187
+ * notification's click-through.
188
+ *
189
+ * ⛔ THE SAME SHAPE AS `VIEW_OPEN_EVENT` ABOVE, FOR THE SAME REASON, and wave 20 is why both
190
+ * exist as constants in this leaf rather than as a string typed twice. A card arriving at a
191
+ * review stage queues an `automation_review` notification naming `{autoId, stageId, count}`;
192
+ * clicking it has to do TWO things that live on opposite sides of an ownership fence — route to
193
+ * `#/automation` (the SHELL's hash) and select that automation in the rail (`AutomationSurface`'s
194
+ * own `activeId`, which the shell cannot see and must not learn). So the frame navigates and
195
+ * then ASKS, and the surface answers if it can.
196
+ *
197
+ * ⚠ THE LISTENER IS THE HALF THAT CAN BE ABSENT. Wave 20 shipped item 25's click-through with
198
+ * this exact shape and NO listener — the event was dispatched into nothing, every gate green,
199
+ * the notification landing the reader on the right table and doing nothing else. Declared here
200
+ * on the wave's first day precisely so the surface can wire the listener while it is being
201
+ * built rather than at close-out; the wiring row (W23-W5) asserts both ends.
202
+ *
203
+ * An automation the reader can no longer open must do NOTHING rather than throw — the listener
204
+ * checks its own list first, exactly as the grid does for a view it cannot see.
205
+ */
206
+ export const AUTOMATION_OPEN_EVENT = "aios:automation-open";
207
+
208
+ /** `detail` of {@link AUTOMATION_OPEN_EVENT}. `stageId` is advisory — a surface that does not
209
+ * scroll to a stage simply selects the automation. */
210
+ export interface AutomationOpenDetail {
211
+ autoId: string;
212
+ stageId?: string;
213
+ }
214
+
215
+ /*
216
+ * ⛔ `AUTOMATION_CREATE_EVENT` STOOD HERE AND IS DELETED WHOLE (wave 25 item 5a, ruling R8).
217
+ *
218
+ * It existed for ONE purpose: the "Automated database" doors on Home, in the Database flyout's
219
+ * create menu and in the New-database dialog routed to `#/automation` and then raised this so the
220
+ * surface would open its create flow. R8 deletes all three doors — "creating a database is one
221
+ * act; pointing an automation at it is another" — and `Shell.tsx`'s `openAutomated` was this
222
+ * event's ONLY signaller.
223
+ *
224
+ * ⛔ SO IT HAD TO GO WITH THEM, and the reason is this repo's own scar tissue rather than tidiness.
225
+ * Left behind, it would be a constant with a listener and no signaller — the exact mirror of the
226
+ * defect wave 24 found here (declared, signalled, consumed NOWHERE, every gate green, three doors
227
+ * that navigated and then did nothing). A one-sided event is indistinguishable from a working one
228
+ * from every direction except a grep, in BOTH directions, so the fix is symmetric: delete the side
229
+ * that is left, never leave the half that compiles.
230
+ *
231
+ * ⚠ `AUTOMATION_OPEN_EVENT` above is UNAFFECTED and still two-sided (the shell signals it from
232
+ * Home's automation tiles and the alerts click-through; `AutomationSurface` listens at module
233
+ * scope). `verify_automation_ui.py` now DERIVES that rule instead of naming these two constants:
234
+ * every `AUTOMATION_*_EVENT` declared in this file must have a signal site AND a listener site.
235
+ * Creating an automation has one door and it is on the automation surface, which is where W24 put
236
+ * the front door anyway (`AutomationSurface`'s rail button and its empty state).
237
+ */
web/src/automation/AutomationBuilder.tsx CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/automation/AutomationDetail.tsx CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/automation/AutomationFind.tsx CHANGED
@@ -1,893 +1,893 @@
1
- // ---------------------------------------------------------------------------
2
- // automation/AutomationFind.tsx — the discovery filter, as TOGGLES (owner item 6,
3
- // contract C4).
4
- //
5
- // WHAT CHANGED AND WHY. It was a condition BUILDER: an empty row, a field
6
- // dropdown listing 21 names, "Add a condition". That shape asks the user to
7
- // remember what is searchable before they can look, and it hid the only fact
8
- // that decides whether a search returns anything at all — which fields actually
9
- // carry values. The corpus is 620 million profiles and every run is billed, so
10
- // "I did not know I could filter on that" and "I filtered on a field that is
11
- // empty on every row" are both expensive mistakes made in silence.
12
- //
13
- // So every searchable field is ON SCREEN, off by default, and turning one on is
14
- // what creates its condition. The list leads with the fields MEASURED to carry
15
- // values; the rest are grouped under what is known about them, and neither group
16
- // is hidden — R3 already decided that "every filter" means the 21 we can stand
17
- // behind, so all 21 are visible.
18
- //
19
- // ⛔ THE VOCABULARY IS THE SERVER'S, ALL OF IT. Field names, operators, which
20
- // operators take no value, the record ceiling and (once C4 lands it) which
21
- // fields are populated and which narrow — every one of those rides on
22
- // `GET /automations`. This file holds no list of its own. The reason is the one
23
- // `automationApi.ts:11-15` gives for cron presets: the thing that ACCEPTS a
24
- // filter is Python, and a client copy of what it accepts is a copy that can
25
- // offer a search the server refuses.
26
- //
27
- // ⛔ AND IT DOES NOT RE-IMPLEMENT THE GUARD. `guard` carries the server's
28
- // numbers, so the rows can say which fields narrow — but the refusal is the
29
- // server's to make and this surface prints it VERBATIM when it comes back
30
- // (`DiscoverGuard`'s note). A client that predicts the refusal is a second copy
31
- // of the rule, free to disagree with the first, and when they disagree the user
32
- // gets a Save button that is disabled for a reason nobody can see.
33
- // ---------------------------------------------------------------------------
34
- import { useEffect, useState } from "react";
35
- // ⭐ WAVE 27 item 29 — the overlay layer the grid already owns, consumed rather than
36
- // re-implemented: it is a body portal (so it escapes `.auto-panel`'s 380px), and it brings
37
- // the dismiss/focus/placement behaviour every other popover on this product already has.
38
- import { AnchoredOverlay } from "../customer-grid/OverlaySurface";
39
- import type { AnchorRect } from "../customer-grid/OverlaySurface";
40
-
41
- import type {
42
- DiscoverEstimate,
43
- DiscoverFieldMeta,
44
- DiscoverOperator,
45
- DiscoverVocab,
46
- Predicate,
47
- } from "./automationApi";
48
- import { discoverCategories } from "./automationApi";
49
-
50
- interface Props {
51
- /** The server-declared discovery vocabulary (absent until the list loads). */
52
- discover?: DiscoverVocab;
53
- recordsLimit: number;
54
- onRecordsLimit: (n: number) => void;
55
- joinOp: string;
56
- onJoinOp: (v: string) => void;
57
- preds: Predicate[];
58
- onPreds: (next: Predicate[]) => void;
59
- estimate: DiscoverEstimate | null;
60
- onEstimate: () => void;
61
- }
62
-
63
- /**
64
- * A FILTER FIELD'S ICON (owner item 5). Drawn, `currentColor`, never an emoji — and in the
65
- * existing `TriggerMark`/`ActionMark` language rather than a new one: same 16 viewBox, same 15px
66
- * box, same 1.4 stroke, so a row here and a card in the builder read as one product.
67
- *
68
- * ⚠ SHARED SHAPES WHERE THE FIELDS SHARE A MEANING, distinct everywhere else. `TriggerMark`'s
69
- * note settled the principle — a column of identical glyphs is decoration, the eye learns nothing
70
- * — but its converse matters just as much here: `bio_hashtags` and `post_hashtags` ARE both
71
- * hashtags, and drawing two different marks for them would invent a distinction the vendor does
72
- * not make. Twenty-one contrived glyphs would be twenty-one things to misread.
73
- *
74
- * ⛔ THE FALLBACK IS NOT A MEMBER OF THE SET, which is the `ActionMark` scar exactly: its default
75
- * used to BE the pencil, a real member, so every unmatched kind silently borrowed "edit" and a
76
- * fallback could not be told from a match. The vocabulary is the SERVER's — it can grow a field
77
- * tomorrow — so an unmatched name draws a neutral mark that is deliberately meaningless.
78
- */
79
- function FilterMark({ name }: { name: string }) {
80
- const common = {
81
- width: 15, height: 15, viewBox: "0 0 16 16", fill: "none", stroke: "currentColor",
82
- strokeWidth: 1.4, strokeLinecap: "round" as const, strokeLinejoin: "round" as const,
83
- "aria-hidden": true,
84
- };
85
- // Audience size — two figures, one behind the other.
86
- if (name === "followers")
87
- return (
88
- <svg {...common}>
89
- <circle cx="6.2" cy="5.6" r="2.4" />
90
- <path d="M2.2 13c.5-2 2.1-3.2 4-3.2s3.5 1.2 4 3.2" />
91
- <path d="M10.6 3.6a2.4 2.4 0 0 1 0 4M11.4 9.9c1.4.4 2.4 1.5 2.8 3.1" />
92
- </svg>
93
- );
94
- // Who this account follows — a figure with an outbound arrow.
95
- if (name === "following")
96
- return (
97
- <svg {...common}>
98
- <circle cx="6" cy="5.6" r="2.4" />
99
- <path d="M1.8 13c.5-2 2.1-3.2 4.2-3.2 .6 0 1.2.1 1.7.3" />
100
- <path d="M9.8 10.6h4.2M12.2 8.8l1.8 1.8-1.8 1.8" />
101
- </svg>
102
- );
103
- if (name === "posts_count")
104
- return (
105
- <svg {...common}>
106
- <rect x="2.6" y="2.6" width="4.6" height="4.6" rx="1" />
107
- <rect x="8.8" y="2.6" width="4.6" height="4.6" rx="1" />
108
- <rect x="2.6" y="8.8" width="4.6" height="4.6" rx="1" />
109
- <rect x="8.8" y="8.8" width="4.6" height="4.6" rx="1" />
110
- </svg>
111
- );
112
- // A story highlight — the ring Instagram draws around one.
113
- if (name === "highlights_count")
114
- return (
115
- <svg {...common}>
116
- <circle cx="8" cy="8" r="5.6" strokeDasharray="2.6 1.8" />
117
- <circle cx="8" cy="8" r="2.2" />
118
- </svg>
119
- );
120
- if (name === "avg_engagement")
121
- return (
122
- <svg {...common}>
123
- <path d="M8 13.2 3.4 8.8a2.9 2.9 0 0 1 4.6-3.4 2.9 2.9 0 0 1 4.6 3.4z" />
124
- </svg>
125
- );
126
- if (name === "biography")
127
- return (
128
- <svg {...common}>
129
- <rect x="2.4" y="2.8" width="11.2" height="10.4" rx="1.6" />
130
- <path d="M4.8 6h6.4M4.8 8.4h6.4M4.8 10.8h3.6" />
131
- </svg>
132
- );
133
- if (name === "category_name" || name === "business_category_name")
134
- return (
135
- <svg {...common}>
136
- <path d="M8.4 2.6H13v4.6l-6.2 6.2-4.6-4.6z" />
137
- <circle cx="10.8" cy="5.2" r="0.9" />
138
- </svg>
139
- );
140
- if (name === "is_business_account" || name === "is_professional_account")
141
- return (
142
- <svg {...common}>
143
- <rect x="2.2" y="5" width="11.6" height="8.2" rx="1.6" />
144
- <path d="M6 5V3.6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1V5" />
145
- </svg>
146
- );
147
- if (name === "is_verified")
148
- return (
149
- <svg {...common}>
150
- <path d="M8 2.2l1.7 1.2 2-.2.6 2 1.7 1.1-.9 1.9.9 1.9-1.7 1.1-.6 2-2-.2L8 14.2l-1.7-1.2-2 .2-.6-2L2 10.1l.9-1.9L2 6.3l1.7-1.1.6-2 2 .2z" />
151
- <path d="m5.8 8.1 1.5 1.5 3-3.1" />
152
- </svg>
153
- );
154
- // The handle, and the display name it is written under.
155
- if (name === "account" || name === "profile_name")
156
- return (
157
- <svg {...common}>
158
- <circle cx="8" cy="8" r="2.4" />
159
- <path d="M10.4 5.6v3.2a1.9 1.9 0 0 0 3.2 1.3A5.8 5.8 0 1 0 11 13.1" />
160
- </svg>
161
- );
162
- if (name === "full_name")
163
- return (
164
- <svg {...common}>
165
- <circle cx="8" cy="5.6" r="2.6" />
166
- <path d="M3 13.4c.7-2.3 2.6-3.6 5-3.6s4.3 1.3 5 3.6" />
167
- </svg>
168
- );
169
- if (name === "external_url" || name === "profile_url")
170
- return (
171
- <svg {...common}>
172
- <path d="M6.6 9.4 4.9 11a2.6 2.6 0 1 1-1.5-4.4" />
173
- <path d="M9.4 6.6 11.1 5a2.6 2.6 0 1 1 1.5 4.4" />
174
- <path d="M6.2 8h3.6" />
175
- </svg>
176
- );
177
- if (name === "bio_hashtags" || name === "post_hashtags")
178
- return (
179
- <svg {...common}>
180
- <path d="M6.2 2.8 4.8 13.2M11.2 2.8 9.8 13.2M2.8 6h10.4M2.2 10h10.4" />
181
- </svg>
182
- );
183
- if (name === "related_accounts")
184
- return (
185
- <svg {...common}>
186
- <circle cx="8" cy="3.6" r="1.8" />
187
- <circle cx="3.6" cy="12" r="1.8" />
188
- <circle cx="12.4" cy="12" r="1.8" />
189
- <path d="M6.8 5.2 4.6 10.4M9.2 5.2l2.2 5.2M5.4 12h5.2" />
190
- </svg>
191
- );
192
- // The vendor's own record keys — the thing you hold to open exactly one row.
193
- if (name === "id" || name === "fbid")
194
- return (
195
- <svg {...common}>
196
- <circle cx="5.4" cy="10.6" r="2.6" />
197
- <path d="M7.2 8.8 13 3M10.6 5.4l1.6 1.6M9.2 6.8l1.6 1.6" />
198
- </svg>
199
- );
200
- return (
201
- <svg {...common}>
202
- <rect x="3.2" y="3.2" width="9.6" height="9.6" rx="2.2" />
203
- <circle cx="8" cy="8" r="1.4" />
204
- </svg>
205
- );
206
- }
207
-
208
- /** One field's line in the list, with everything known about it. */
209
- interface FieldRow {
210
- name: string;
211
- /** What the row is CALLED. Falls back to the raw name only if the server sent none. */
212
- label: string;
213
- hint: string;
214
- kind: string;
215
- operators: DiscoverOperator[];
216
- options: { value: string; label: string }[];
217
- /** Every stored condition naming this field, with its index in the saved array. */
218
- at: number[];
219
- /** MEASURED to carry values. `null` = the server has not said (C4 not shipped yet). */
220
- populated: boolean | null;
221
- /** A condition here cuts the corpus down. `null` = not said. */
222
- narrowing: boolean | null;
223
- }
224
-
225
- /** A stored value → what the text box shows. A list is the "any of" shape. */
226
- function valueText(v: Predicate["value"]): string {
227
- if (Array.isArray(v)) return v.join(", ");
228
- return v === undefined || v === null ? "" : String(v);
229
- }
230
-
231
- /**
232
- * What the user typed → what is stored. Comma-separated becomes a LIST for the operators that
233
- * accept one, and stays a plain string for the ones that do not.
234
- *
235
- * ⚠ A single value stays a SCALAR rather than a one-element list — the server sends scalars
236
- * flat and lists as a nested OR group, and a one-element group is a shape nothing has been
237
- * billed against.
238
- */
239
- function parseValue(text: string, multi: boolean): Predicate["value"] {
240
- if (!multi) return text;
241
- const parts = text.split(",").map((s) => s.trim()).filter(Boolean);
242
- if (parts.length > 1) return parts;
243
- // Keep the RAW text while it is still being typed — trimming here would fight the cursor on
244
- // every keystroke, and the server trims anyway.
245
- return text;
246
- }
247
-
248
- /**
249
- * ⭐ WAVE 26 · DEBT D-69 — A THIRD VALUE COULD NOT BE TYPED, and the cause was one round trip
250
- * through this pair of functions.
251
- *
252
- * `parseValue` returns an ARRAY at 2+ values; `valueText` then renders that array as
253
- * `join(", ")` — which has NO TRAILING COMMA. So on a controlled input the separator was
254
- * normalised away between keystrokes: typing `,` onto `skincare, beauty` re-rendered as
255
- * `skincare, beauty`, and the next character landed as `skincare, beautym`. Two keywords
256
- * silently became one that matches nothing. ⚠ It hit the FIRST surface a new user meets (wave 24
257
- * made this panel the only door) and it hit precisely the 3+ keyword list the feature was built
258
- * for — the owner's *"if i want to include many keywords like floral, flower, beauty"*.
259
- *
260
- * ⛔ THE FIX IS THE ONE `parseValue`'s OWN COMMENT ALREADY DESCRIBES FOR THE 1-VALUE CASE:
261
- * keep the raw text while it is being typed. That comment was right and its scope was too narrow
262
- * — it protected the cursor at one value and handed the 2+ case to `join`. So the raw text is
263
- * held for whichever input is FOCUSED, and the stored value is still parsed on every keystroke
264
- * (so nothing depends on blur to save) and re-parsed on blur (so the display settles).
265
- *
266
- * ⚠ KEYED BY THE ROW'S REACT KEY, not by index alone: two conditions on the same field are
267
- * ordinary here ("Another condition on Bio"), and an index-only key would leak one row's draft
268
- * into its sibling on a removal.
269
- */
270
- interface Draft {
271
- key: string;
272
- text: string;
273
- }
274
-
275
- /**
276
- * The comparison a freshly toggled field starts with — ASKED FOR, never chosen here.
277
- *
278
- * ⛔ THE THIRD COSTUME OF ONE SCAR, and the reason this function no longer has an opinion.
279
- * Wave 21 made a new condition NULLARY (`is_not_null`) so it would be saveable without a
280
- * value, and called that "narrowing-in-the-right-direction". Wave 22 then wrote the narrowing
281
- * law server-side and `is_not_null` is not in it. From that moment, toggling ANY field on and
282
- * pressing Save returned "add at least one CONTENT condition…" — the error told the user to do
283
- * the exact thing they had just done. Wave 24 deleted the wizard and made this panel the only
284
- * door, so it stopped being a corner case and became the first thing a new user hits.
285
- *
286
- * The fix is not a better guess. It is that the module owning the guard also names the default
287
- * (`automation_engine.default_operator`, asserted against `predicate_narrows` in
288
- * `verify_automation.py`), and this file looks it up per FIELD. An absent one falls back to the
289
- * server's first operator — never to a nullary, which is the thing that could not narrow.
290
- */
291
- function firstOperator(v: DiscoverVocab | undefined, name: string): string {
292
- const meta = (v?.filterMeta || []).find((m) => m.name === name);
293
- return meta?.defaultOperator || (v?.operators || [])[0] || "";
294
- }
295
-
296
- /**
297
- * The rows, ordered: measured-populated first, then the rest.
298
- *
299
- * TWO SOURCES, AND THE FALLBACK IS NARROWER THAN THE REAL THING. With C4's
300
- * `filterMeta` the answer is per field and complete. Without it, `lead` names
301
- * the three fields seen carrying values and says NOTHING about the other
302
- * eighteen — so those come back `null`, not `false`, and the UI prints no claim
303
- * about them rather than an invented one.
304
- */
305
- function buildRows(preds: Predicate[], v?: DiscoverVocab): FieldRow[] {
306
- const meta = new Map<string, DiscoverFieldMeta>(
307
- (v?.filterMeta || []).map((m) => [m.name, m])
308
- );
309
- const lead = new Set(v?.lead || []);
310
- const byName = new Map<string, number[]>();
311
- preds.forEach((p, i) => {
312
- const list = byName.get(p.name);
313
- if (list) list.push(i);
314
- else byName.set(p.name, [i]);
315
- });
316
-
317
- const rows = (v?.fields || []).map((name) => {
318
- const m = meta.get(name);
319
- return {
320
- name,
321
- label: m?.label || name,
322
- hint: m?.hint || "",
323
- kind: m?.kind || "text",
324
- // ⛔ NO FALLBACK TO THE GLOBAL OPERATOR LIST. An older server that sends no per-field list
325
- // gets a row with no comparisons rather than all fourteen on every field — visibly
326
- // unfinished beats quietly offering `at least` on a yes/no column.
327
- operators: m?.operators || [],
328
- options: m?.options || [],
329
- at: byName.get(name) || [],
330
- populated: m ? !!m.populated : lead.has(name) ? true : null,
331
- narrowing: m ? !!m.narrowing : null,
332
- };
333
- });
334
- // Stable partition, never a sort: the server's order inside each half is the
335
- // order the engine lists them in, and re-alphabetising it would be this file
336
- // having an opinion about a server list.
337
- return [...rows.filter((r) => r.populated === true), ...rows.filter((r) => r.populated !== true)];
338
- }
339
-
340
- export default function AutomationFind({
341
- discover,
342
- recordsLimit,
343
- onRecordsLimit,
344
- joinOp,
345
- onJoinOp,
346
- preds,
347
- onPreds,
348
- estimate,
349
- onEstimate,
350
- }: Props) {
351
- const rows = buildRows(preds, discover);
352
- const known = new Set(discover?.fields || []);
353
- // The GLOBAL operator list is gone from this component: comparisons are per field now
354
- // (`row.operators`). `nullary` survives for STRAYS only — a stored condition on a field the
355
- // vendor dropped has no per-field row to read from, and it still has to render to be removable.
356
- const nullary = new Set(discover?.nullaryOperators || []);
357
- const withValues = rows.filter((r) => r.populated === true);
358
- const rest = rows.filter((r) => r.populated !== true);
359
- const knownMeta = !!(discover?.filterMeta || []).length;
360
- // ⚠ THE STORED FILTER CAN NAME A FIELD THE LIST NO LONGER HAS — a vocabulary
361
- // that moved, an automation copied from elsewhere. It must stay VISIBLE and
362
- // REMOVABLE: a condition the user cannot see is one they cannot delete, and
363
- // every Save keeps sending it. Same defect as a `<select>` whose value matches
364
- // no option, one level up.
365
- const strays = preds
366
- .map((p, i) => ({ p, i }))
367
- .filter(({ p }) => !known.has(p.name));
368
-
369
- const maxRecords = discover?.guard?.maxRecords || discover?.maxRecords || 500;
370
-
371
- /** D-69 — the raw text of the input a person is typing in right now (see `Draft`). */
372
- const [draft, setDraft] = useState<Draft | null>(null);
373
-
374
- /**
375
- * ⭐ WAVE 27 · OWNER ITEM 29 — which condition's value is open in the BIG editor, if any.
376
- *
377
- * ⛔ THE TEXT IS HELD HERE, NOT WRITTEN THROUGH ON EVERY KEYSTROKE, and that is the one place
378
- * this differs from the rail input beside it. The rail input writes per keystroke on purpose
379
- * (a Save pressed straight from the field must not store the previous value — D-69's note).
380
- * A modal has its own Save, so per-keystroke writes would buy nothing and would make Cancel
381
- * a lie: the value would already be in the config by the time it was pressed.
382
- * ⚠ The ANCHOR is captured at click time and stored, because the button it came from is
383
- * inside a panel that scrolls — re-reading it later would place the overlay against a rect
384
- * that has moved.
385
- */
386
- const [big, setBig] = useState<{
387
- key: string;
388
- index: number;
389
- label: string;
390
- multi: boolean;
391
- text: string;
392
- anchor: AnchorRect;
393
- } | null>(null);
394
-
395
- /**
396
- * ⭐ ITEM 16 / D-71 — THE OBSERVED CATEGORY VALUES, fetched WHEN THIS PANEL OPENS.
397
- *
398
- * ⛔ ON MOUNT, ONCE, AND NEVER ON THE POLLED PAYLOAD. This component is rendered only while the
399
- * Find panel is open, so mounting IS the panel opening — and the server moved these off
400
- * `GET /automations` after measuring that deriving them reads two user tables plus the platform
401
- * master, i.e. a network round-trip per open tab every 2.5 s. Putting the request here is the
402
- * client half of that same decision, not an optimisation.
403
- *
404
- * ⚠ AN EMPTY LIST IS AN ANSWER AND IS RENDERED AS ONE. It means this deployment has not seen a
405
- * category value yet — which is TRUE for a fresh tenant — and the field keeps its free-text
406
- * input either way. It must never read as "loading forever", and it must never be filled in
407
- * with a guess: D-59 is explicit that an invented taxonomy is worse than no dropdown.
408
- */
409
- const [cats, setCats] = useState<{ value: string; count: number }[]>([]);
410
- useEffect(() => {
411
- const ac = new AbortController();
412
- discoverCategories(ac.signal)
413
- .then((r) => setCats(Array.isArray(r?.options) ? r.options : []))
414
- // Fail-quiet, on purpose: the picker is an ASSIST over a control that works without it.
415
- // An error banner here would report a broken panel when the only thing missing is a
416
- // convenience — and the free-text input beside it is unaffected.
417
- .catch(() => setCats([]));
418
- return () => ac.abort();
419
- }, []);
420
-
421
- const setAt = (i: number, patch: Partial<Predicate>) =>
422
- onPreds(preds.map((p, j) => (j === i ? { ...p, ...patch } : p)));
423
-
424
- const removeAt = (i: number) => onPreds(preds.filter((_p, j) => j !== i));
425
-
426
- const toggleField = (row: FieldRow) => {
427
- if (row.at.length) {
428
- onPreds(preds.filter((p) => p.name !== row.name));
429
- return;
430
- }
431
- onPreds([...preds, { name: row.name, operator: firstOperator(discover, row.name) }]);
432
- };
433
-
434
- /** One condition line — the comparison, its value, and a way out. */
435
- const condition = (i: number, label: string, row?: FieldRow) => {
436
- const p = preds[i];
437
- if (!p) return null;
438
- // The field's OWN comparisons. `row` is absent only for a stray (a stored condition on a
439
- // field the vendor no longer offers), which keeps the raw token so it stays removable.
440
- const ops = row?.operators || [];
441
- const cur = ops.find((o) => o.value === p.operator);
442
- const isNullary = cur ? cur.nullary : nullary.has(p.operator);
443
- const isMulti = !!cur?.multi;
444
- const options = row?.options || [];
445
- const rowKey = `${p.name}-${i}`;
446
- // D-69: the focused input shows what was TYPED; every other one shows the stored value.
447
- const shown = draft && draft.key === rowKey ? draft.text : valueText(p.value);
448
- /*
449
- * ⭐ ITEM 16 / D-71 — WHICH FIELDS GET THE OBSERVED-VALUES PICKER, DERIVED OFF THE WIRE.
450
- *
451
- * ⛔ NOT A CLIENT-SIDE LIST OF FIELD NAMES. Naming the vendor's two category columns here
452
- * would be the D-55 class verbatim — the wave-9 silent-drop shape, a client copy of a server
453
- * vocabulary that is free to drift the day the engine adds a third one. (Their literal keys
454
- * are deliberately not written anywhere in this file: the gate asserts the tokens are ABSENT,
455
- * and an absence check cannot tell a hard-coded list from the comment forbidding one.)
456
- * The server already publishes the answer twice over: `kind: "choice"` is its own declaration
457
- * (`BD_FIELD_KINDS`), and `filter_meta` deliberately ships `options: []` for those fields
458
- * because their vocabulary is OBSERVED rather than declared and arrives on its own route.
459
- * So the rule is exactly that: a choice-kind field with no declared options is one whose
460
- * values we can only have learned by seeing them. A new choice field inherits this for free.
461
- */
462
- const wantsObserved = row?.kind === "choice" && !options.length;
463
- /** D-71 — APPEND, never replace: the picker has to compose with the comma list. */
464
- const appendValue = (v: string) => {
465
- const has = shown
466
- .split(",")
467
- .map((s) => s.trim().toLowerCase())
468
- .filter(Boolean);
469
- // Picking the same value twice is a no-op rather than a duplicate: the server expands a
470
- // multi-value condition into an OR, and `beauty OR beauty` costs a slot to say nothing.
471
- if (has.includes(v.trim().toLowerCase())) return;
472
- const next = shown.trim() ? `${shown.replace(/[\s,]+$/, "")}, ${v}` : v;
473
- setDraft(null);
474
- setAt(i, { value: parseValue(next, isMulti) });
475
- };
476
- return (
477
- <div className="autob-cond" key={rowKey}>
478
- <select
479
- className="auto-input is-small"
480
- aria-label={`Comparison for ${label}`}
481
- value={p.operator}
482
- onChange={(e) => setAt(i, { operator: e.target.value })}
483
- >
484
- {/* The stored value is ALWAYS an option. A select whose value matches no
485
- option renders the FIRST one, and the next Save writes a comparison
486
- nobody chose — this codebase has paid for that twice. */}
487
- {p.operator && !ops.some((o) => o.value === p.operator) ? (
488
- <option value={p.operator}>{p.operator}</option>
489
- ) : null}
490
- {ops.map((o) => (
491
- <option key={o.value} value={o.value}>
492
- {o.label}
493
- </option>
494
- ))}
495
- </select>
496
- {isNullary ? null : options.length ? (
497
- // A fixed vocabulary gets a dropdown. Owner item 2: a yes/no column was a box you
498
- // typed `true` into, beside a comparison list that offered "at least".
499
- <select
500
- className="auto-input is-small"
501
- aria-label={`Value for ${label}`}
502
- value={valueText(p.value)}
503
- onChange={(e) => setAt(i, { value: e.target.value })}
504
- >
505
- <option value="">Choose…</option>
506
- {options.map((o) => (
507
- <option key={o.value} value={o.value}>
508
- {o.label}
509
- </option>
510
- ))}
511
- </select>
512
- ) : (
513
- <input
514
- className="auto-input is-small"
515
- type={row?.kind === "number" ? "number" : "text"}
516
- aria-label={`Value for ${label}`}
517
- // ⭐ OWNER ITEM 3. Several comma-separated values become an "any of" condition, which
518
- // the server sends as its own OR group — so it does NOT drag Followers into a union
519
- // the way the Match dropdown would.
520
- placeholder={isMulti ? "floral, flower, beauty" : ""}
521
- // D-69: `shown` is the RAW text while this input has focus, so a trailing comma
522
- // survives long enough to type the next value after it.
523
- value={shown}
524
- onFocus={() => setDraft({ key: rowKey, text: valueText(p.value) })}
525
- onChange={(e) => {
526
- setDraft({ key: rowKey, text: e.target.value });
527
- // The STORE still updates on every keystroke. Parsing only on blur would mean a
528
- // Save pressed straight from the field wrote the previous value — trading a typing
529
- // bug for a data-loss one.
530
- setAt(i, { value: parseValue(e.target.value, isMulti) });
531
- }}
532
- onBlur={(e) => {
533
- setDraft(null);
534
- setAt(i, { value: parseValue(e.target.value, isMulti) });
535
- }}
536
- />
537
- )}
538
- {/* ⭐ WAVE 27 · OWNER ITEM 29 — A WIDE EDITOR FOR THE VALUES THAT DO NOT FIT.
539
- ⛔ THE COMPLAINT IS GEOMETRY, and the geometry is a CHAIN nothing here can widen:
540
- `.auto-panel` is 380px, `.autob-cond .auto-input` divides what is left across three
541
- controls, and `.is-small` trims it again — so a bio phrase list ("floral, flower,
542
- wedding florist, event styling") is typed six characters at a time through a box
543
- that scrolls sideways. The panel cannot grow; the EDITOR can leave it, and an
544
- overlay is drawn in a body portal, so it is not bound by that chain at all.
545
- ⚠ ALWAYS OFFERED, never "appears once the text is long": a control that materialises
546
- at some threshold is one people do not know exists at the moment they need it.
547
- ⚠ NOT on a `<select>` value (a fixed vocabulary has nothing to expand) and not on a
548
- nullary comparison (which has no value at all). */}
549
- {!isNullary && !options.length ? (
550
- <button
551
- type="button"
552
- className="auto-input is-small autoc-expand"
553
- aria-label={`Open a bigger editor for ${label}`}
554
- title="Edit in a bigger box"
555
- onClick={(e) => {
556
- // ⚠ READ SYNCHRONOUSLY from the event. `currentTarget` is null by the time React
557
- // re-invokes a state updater, and a null rect here paints the overlay at 0,0 —
558
- // measured, in this repo, on a different control ([[react-event-currenttarget-updater]]).
559
- const r = e.currentTarget.getBoundingClientRect();
560
- setBig({
561
- key: rowKey,
562
- index: i,
563
- label,
564
- multi: isMulti,
565
- text: valueText(p.value),
566
- anchor: { left: r.left, top: r.top, right: r.right, bottom: r.bottom,
567
- width: r.width, height: r.height },
568
- });
569
- }}
570
- >
571
- {/* Drawn, never a glyph: the design constitution's "no emojis in the UI" covers
572
- the arrow characters too, and every other mark in this product is a path in the
573
- same 16x16 stroke vocabulary. Two corners pulling apart = "make this bigger". */}
574
- <svg viewBox="0 0 16 16" aria-hidden className="autoc-expand-icon">
575
- <path d="M9.5 2.5h4v4M13.5 2.5 9.6 6.4" />
576
- <path d="M6.5 13.5h-4v-4M2.5 13.5l3.9-3.9" />
577
- </svg>
578
- </button>
579
- ) : null}
580
- {/* ⭐ ITEM 16 / D-71 — THE APPEND-PICKER, BESIDE the input and never instead of it.
581
- Free text stays the primary control: these are the values this deployment has SEEN,
582
- not the values that exist, so the picker is a shortcut over a superset it cannot
583
- enumerate. Appending in code also sidesteps D-69 entirely — a picked value never has
584
- to survive a round trip through the text box. */}
585
- {wantsObserved && !isNullary ? (
586
- <select
587
- className="auto-input is-small autoc-catpick"
588
- aria-label={`Add a value seen before, to ${label}`}
589
- title={
590
- cats.length
591
- ? "Values this workspace has actually seen, most-seen first. Adds to the list."
592
- : "No category values have been seen in your data yet — type one."
593
- }
594
- value=""
595
- disabled={!cats.length}
596
- onChange={(e) => {
597
- if (e.target.value) appendValue(e.target.value);
598
- }}
599
- >
600
- <option value="">{cats.length ? "Seen before…" : "None seen yet"}</option>
601
- {cats.map((c) => (
602
- // The COUNT is the honest half (D-59): a value seen once and one seen forty times
603
- // are different bets, and an alphabetical list of bare strings hides that.
604
- <option key={c.value} value={c.value}>
605
- {c.value} ({c.count.toLocaleString()})
606
- </option>
607
- ))}
608
- </select>
609
- ) : null}
610
- {isMulti && Array.isArray(p.value) && p.value.length > 1 ? (
611
- /*
612
- * ⭐ ITEM 16 / D-71 — HOW MANY VALUES THIS CONDITION CARRIES, and against what ceiling.
613
- *
614
- * The COUNT is ours and is always shown. The CEILING is the server's and is shown only
615
- * when the server sends one (`guard.maxValues` — an open `ASK ->A:`; `guard` ships
616
- * `{minNarrowing, maxRecords}` today).
617
- * ⛔ NO CLIENT-SIDE `4`. D-71's exit says "how many of D-68's 4 value slots" — but A
618
- * shipped `depth_refusal()`, which computes the depth of the shape actually emitted
619
- * rather than budgeting fixed slots, so the number 4 describes a model that no longer
620
- * exists. Printing it would be a client copy of a server rule that has already changed
621
- * once, which is the specific failure this file's header refuses to commit
622
- * ([[measure-the-real-call]]: a correct-looking answer about the wrong subject).
623
- */
624
- <span className="autob-cond-any" title="Any one of these is a match">
625
- any of {p.value.length}
626
- {discover?.guard?.maxValues ? ` of ${discover.guard.maxValues}` : ""}
627
- </span>
628
- ) : null}
629
- <button
630
- type="button"
631
- className="autob-cond-x"
632
- aria-label={`Remove this condition on ${label}`}
633
- title="Remove this condition"
634
- onClick={() => removeAt(i)}
635
- >
636
- <svg width="12" height="12" viewBox="0 0 16 16" fill="none" aria-hidden="true">
637
- <path d="M4 4l8 8M12 4l-8 8" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round" />
638
- </svg>
639
- </button>
640
- </div>
641
- );
642
- };
643
-
644
- const fieldRow = (row: FieldRow) => {
645
- const on = row.at.length > 0;
646
- return (
647
- <div className={"autob-tgl" + (on ? " is-on" : "")} key={row.name}>
648
- <div className="autob-tgl-head">
649
- <button
650
- type="button"
651
- className={"auto-step-switch autob-tgl-switch" + (on ? " is-on" : "")}
652
- aria-pressed={on}
653
- aria-label={`${on ? "Stop searching" : "Search"} on ${row.label}`}
654
- title={`${on ? "Stop searching" : "Search"} on ${row.label}`}
655
- onClick={() => toggleField(row)}
656
- >
657
- <span className="auto-step-switch-knob" />
658
- </button>
659
- <span className="autob-tgl-mark">
660
- <FilterMark name={row.name} />
661
- </span>
662
- {/* ⭐ THE HUMAN LABEL (owner item 1: "wtf is id/fbid").
663
- This used to render the vendor's own column name verbatim, and the comment
664
- defending it was RIGHT about the reason — a refusal names the field, and a label
665
- that reads differently makes the error unmatchable. So the label did not move
666
- alone: `field_label()` is what the SERVER's refusals use too, which is the only
667
- version of this change that does not trade one confusion for another. */}
668
- <span className="autob-tgl-name" title={row.hint || undefined}>
669
- {row.label}
670
- </span>
671
- {/*
672
- ⚠ ONE CHIP, AND IT MARKS THE MINORITY. The first version chipped every
673
- non-narrowing field "broad" AND every unpopulated one "no values seen" —
674
- which put two chips on 18 of the 21 rows, and one of them repeated the group
675
- heading the row was already sitting under. A mark that is on almost every row
676
- stops being a mark. Read off the screenshot; no assertion could have said it.
677
-
678
- So: the group label carries POPULATED (it is what the groups ARE), and the
679
- chip carries NARROWING, inverted to mark the five that DO narrow. That is the
680
- actionable half — the server refuses a filter with no narrowing condition, so
681
- "these are the ones that satisfy it" is the sentence a user needs, and
682
- "this one does not" is not.
683
- */}
684
- {row.narrowing === true ? (
685
- <span
686
- className="autob-tgl-flag is-narrow"
687
- title="A search needs at least one condition like this."
688
- >
689
- narrows
690
- </span>
691
- ) : null}
692
- </div>
693
- {on ? (
694
- <div className="autob-tgl-body">
695
- {row.at.map((i) => condition(i, row.label, row))}
696
- {row.hint ? <p className="autob-tgl-hint">{row.hint}</p> : null}
697
- <button
698
- type="button"
699
- className="autob-cond-add"
700
- onClick={() =>
701
- onPreds([...preds, { name: row.name, operator: firstOperator(discover, row.name) }])
702
- }
703
- >
704
- Another condition on {row.label}
705
- </button>
706
- </div>
707
- ) : null}
708
- </div>
709
- );
710
- };
711
-
712
- return (
713
- <>
714
- <div className="auto-field">
715
- <label htmlFor="auto-records">How many profiles to fetch</label>
716
- <input
717
- id="auto-records"
718
- className="auto-input"
719
- type="number"
720
- min={1}
721
- max={maxRecords}
722
- value={recordsLimit}
723
- onChange={(e) => onRecordsLimit(Number(e.target.value) || 1)}
724
- />
725
- </div>
726
- {/* Owner item 5: "Stop at the 'A search must be at most 500 profiles a run'". The two
727
- sentences that followed explained the vendor's billing model to somebody who wants a
728
- list of florists. */}
729
- <p className="auto-hint">At most {maxRecords.toLocaleString()} profiles a run.</p>
730
-
731
- <h3>Conditions</h3>
732
- <div className="auto-field autob-match">
733
- <label htmlFor="auto-joinop">Match</label>
734
- <select
735
- id="auto-joinop"
736
- className="auto-input"
737
- value={joinOp}
738
- onChange={(e) => onJoinOp(e.target.value)}
739
- >
740
- <option value="and">All of them</option>
741
- <option value="or">Any of them</option>
742
- </select>
743
- </div>
744
-
745
- {!rows.length ? (
746
- <p className="auto-note">
747
- The searchable fields have not loaded yet.
748
- </p>
749
- ) : null}
750
-
751
- {withValues.length ? (
752
- <div className="autob-tglgroup">
753
- <p
754
- className="autob-tglgroup-label"
755
- title="Most accounts have this filled in."
756
- >
757
- Usually filled in
758
- </p>
759
- {withValues.map(fieldRow)}
760
- </div>
761
- ) : null}
762
-
763
- {rest.length ? (
764
- <div className="autob-tglgroup">
765
- <p
766
- className="autob-tglgroup-label"
767
- title={
768
- knownMeta
769
- ? "Most accounts leave these blank, so a condition here can return nothing."
770
- : "We have not measured how often these are filled in."
771
- }
772
- >
773
- {knownMeta ? "Often blank" : "Not measured"}
774
- </p>
775
- {rest.map(fieldRow)}
776
- </div>
777
- ) : null}
778
-
779
- {strays.length ? (
780
- <div className="autob-tglgroup">
781
- <p className="autob-tglgroup-label">Not in the searchable list</p>
782
- {strays.map(({ p, i }) => (
783
- <div className="autob-tgl is-on is-stray" key={`stray-${i}`}>
784
- <div className="autob-tgl-head">
785
- {/* A stray gets a mark too — "every filter row" includes the ones the vendor no
786
- longer offers, and a row missing the icon its neighbours have would read as a
787
- different KIND of thing rather than as a field that fell out of the list. It
788
- draws the neutral fallback by construction: a stray is, by definition, not in
789
- the vocabulary this component has arms for. */}
790
- <span className="autob-tgl-mark">
791
- <FilterMark name={p.name} />
792
- </span>
793
- <span className="autob-tgl-name">{p.name}</span>
794
- <span className="autob-tgl-flag is-stray" title="The server will refuse this condition by name.">
795
- not searchable
796
- </span>
797
- </div>
798
- <div className="autob-tgl-body">{condition(i, p.name)}</div>
799
- </div>
800
- ))}
801
- </div>
802
- ) : null}
803
-
804
- {/* The consequence kept, the mechanism dropped: a condition on an often-blank field
805
- returning nothing is a thing that will happen to somebody and confuse them. Why the
806
- search is slow when broad is our problem, not theirs. */}
807
- <p className="auto-hint">
808
- Conditions on often-blank fields can come back with nothing.
809
- </p>
810
-
811
- <h3>What it costs</h3>
812
- <div className="auto-head-actions">
813
- <button type="button" className="auto-btn" onClick={onEstimate}>
814
- Estimate this search
815
- </button>
816
- </div>
817
- {estimate ? (
818
- <p className="auto-hint">
819
- About <strong>${estimate.usd}</strong> for {estimate.records} profiles, at $
820
- {estimate.unitUsd} each.{" "}
821
- {/*
822
- THE CAVEAT IS NOT OPTIONAL. The vendor never quotes a price before a run and this
823
- deployment cannot read its own balance, so presenting this as a billed figure
824
- would be inventing a measurement.
825
- */}
826
- <strong>This is an estimate</strong> — {estimate.note}.
827
- </p>
828
- ) : null}
829
-
830
- {/* ⭐ WAVE 27 · OWNER ITEM 29 — THE BIG EDITOR, rendered ONCE for the whole panel.
831
- ⛔ Not one overlay per condition row: `AnchoredOverlay` mounts a body portal and
832
- installs document-level dismiss/focus handlers, so N of them would be N listener
833
- stacks fighting over one Escape key. One overlay, told which row it is editing.
834
- ⚠ `role="dialog"`, not "menu": it holds a text field, and the menu layer's arrow-key
835
- handling would take the cursor keys away from the text being typed. */}
836
- {big ? (
837
- <AnchoredOverlay
838
- anchor={big.anchor}
839
- className="autoc-bigedit"
840
- placement="right-start"
841
- role="dialog"
842
- ariaLabel={`Value for ${big.label}`}
843
- onDismiss={() => setBig(null)}
844
- dataKind="find-value-editor"
845
- >
846
- <label htmlFor="autoc-bigedit-text">{big.label}</label>
847
- <textarea
848
- id="autoc-bigedit-text"
849
- className="autoc-bigedit-text"
850
- data-overlay-autofocus
851
- autoFocus
852
- rows={7}
853
- spellCheck={false}
854
- value={big.text}
855
- placeholder={big.multi ? "floral, flower, wedding florist, event styling" : ""}
856
- onChange={(e) => setBig((cur) => (cur ? { ...cur, text: e.target.value } : cur))}
857
- onKeyDown={(e) => {
858
- // Escape abandons; Ctrl/Cmd+Enter commits. A bare Enter types a NEWLINE, because
859
- // this is a textarea and the whole reason it exists is that the value is long.
860
- if (e.key === "Escape") setBig(null);
861
- if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
862
- setAt(big.index, { value: parseValue(big.text, big.multi) });
863
- setBig(null);
864
- }
865
- }}
866
- />
867
- {/* The same sentence the rail input's placeholder makes, said where there is room
868
- for it: a comma list is several values, and the server turns them into an OR. */}
869
- {big.multi ? (
870
- <p className="auto-hint">
871
- Separate values with commas — the search matches any of them.
872
- </p>
873
- ) : null}
874
- <div className="auto-head-actions">
875
- <button
876
- type="button"
877
- className="auto-btn auto-btn--primary"
878
- onClick={() => {
879
- setAt(big.index, { value: parseValue(big.text, big.multi) });
880
- setBig(null);
881
- }}
882
- >
883
- Done
884
- </button>
885
- <button type="button" className="auto-btn" onClick={() => setBig(null)}>
886
- Cancel
887
- </button>
888
- </div>
889
- </AnchoredOverlay>
890
- ) : null}
891
- </>
892
- );
893
- }
 
1
+ // ---------------------------------------------------------------------------
2
+ // automation/AutomationFind.tsx — the discovery filter, as TOGGLES (owner item 6,
3
+ // contract C4).
4
+ //
5
+ // WHAT CHANGED AND WHY. It was a condition BUILDER: an empty row, a field
6
+ // dropdown listing 21 names, "Add a condition". That shape asks the user to
7
+ // remember what is searchable before they can look, and it hid the only fact
8
+ // that decides whether a search returns anything at all — which fields actually
9
+ // carry values. The corpus is 620 million profiles and every run is billed, so
10
+ // "I did not know I could filter on that" and "I filtered on a field that is
11
+ // empty on every row" are both expensive mistakes made in silence.
12
+ //
13
+ // So every searchable field is ON SCREEN, off by default, and turning one on is
14
+ // what creates its condition. The list leads with the fields MEASURED to carry
15
+ // values; the rest are grouped under what is known about them, and neither group
16
+ // is hidden — R3 already decided that "every filter" means the 21 we can stand
17
+ // behind, so all 21 are visible.
18
+ //
19
+ // ⛔ THE VOCABULARY IS THE SERVER'S, ALL OF IT. Field names, operators, which
20
+ // operators take no value, the record ceiling and (once C4 lands it) which
21
+ // fields are populated and which narrow — every one of those rides on
22
+ // `GET /automations`. This file holds no list of its own. The reason is the one
23
+ // `automationApi.ts:11-15` gives for cron presets: the thing that ACCEPTS a
24
+ // filter is Python, and a client copy of what it accepts is a copy that can
25
+ // offer a search the server refuses.
26
+ //
27
+ // ⛔ AND IT DOES NOT RE-IMPLEMENT THE GUARD. `guard` carries the server's
28
+ // numbers, so the rows can say which fields narrow — but the refusal is the
29
+ // server's to make and this surface prints it VERBATIM when it comes back
30
+ // (`DiscoverGuard`'s note). A client that predicts the refusal is a second copy
31
+ // of the rule, free to disagree with the first, and when they disagree the user
32
+ // gets a Save button that is disabled for a reason nobody can see.
33
+ // ---------------------------------------------------------------------------
34
+ import { useEffect, useState } from "react";
35
+ // ⭐ WAVE 27 item 29 — the overlay layer the grid already owns, consumed rather than
36
+ // re-implemented: it is a body portal (so it escapes `.auto-panel`'s 380px), and it brings
37
+ // the dismiss/focus/placement behaviour every other popover on this product already has.
38
+ import { AnchoredOverlay } from "../customer-grid/OverlaySurface";
39
+ import type { AnchorRect } from "../customer-grid/OverlaySurface";
40
+
41
+ import type {
42
+ DiscoverEstimate,
43
+ DiscoverFieldMeta,
44
+ DiscoverOperator,
45
+ DiscoverVocab,
46
+ Predicate,
47
+ } from "./automationApi";
48
+ import { discoverCategories } from "./automationApi";
49
+
50
+ interface Props {
51
+ /** The server-declared discovery vocabulary (absent until the list loads). */
52
+ discover?: DiscoverVocab;
53
+ recordsLimit: number;
54
+ onRecordsLimit: (n: number) => void;
55
+ joinOp: string;
56
+ onJoinOp: (v: string) => void;
57
+ preds: Predicate[];
58
+ onPreds: (next: Predicate[]) => void;
59
+ estimate: DiscoverEstimate | null;
60
+ onEstimate: () => void;
61
+ }
62
+
63
+ /**
64
+ * A FILTER FIELD'S ICON (owner item 5). Drawn, `currentColor`, never an emoji — and in the
65
+ * existing `TriggerMark`/`ActionMark` language rather than a new one: same 16 viewBox, same 15px
66
+ * box, same 1.4 stroke, so a row here and a card in the builder read as one product.
67
+ *
68
+ * ⚠ SHARED SHAPES WHERE THE FIELDS SHARE A MEANING, distinct everywhere else. `TriggerMark`'s
69
+ * note settled the principle — a column of identical glyphs is decoration, the eye learns nothing
70
+ * — but its converse matters just as much here: `bio_hashtags` and `post_hashtags` ARE both
71
+ * hashtags, and drawing two different marks for them would invent a distinction the vendor does
72
+ * not make. Twenty-one contrived glyphs would be twenty-one things to misread.
73
+ *
74
+ * ⛔ THE FALLBACK IS NOT A MEMBER OF THE SET, which is the `ActionMark` scar exactly: its default
75
+ * used to BE the pencil, a real member, so every unmatched kind silently borrowed "edit" and a
76
+ * fallback could not be told from a match. The vocabulary is the SERVER's — it can grow a field
77
+ * tomorrow — so an unmatched name draws a neutral mark that is deliberately meaningless.
78
+ */
79
+ function FilterMark({ name }: { name: string }) {
80
+ const common = {
81
+ width: 15, height: 15, viewBox: "0 0 16 16", fill: "none", stroke: "currentColor",
82
+ strokeWidth: 1.4, strokeLinecap: "round" as const, strokeLinejoin: "round" as const,
83
+ "aria-hidden": true,
84
+ };
85
+ // Audience size — two figures, one behind the other.
86
+ if (name === "followers")
87
+ return (
88
+ <svg {...common}>
89
+ <circle cx="6.2" cy="5.6" r="2.4" />
90
+ <path d="M2.2 13c.5-2 2.1-3.2 4-3.2s3.5 1.2 4 3.2" />
91
+ <path d="M10.6 3.6a2.4 2.4 0 0 1 0 4M11.4 9.9c1.4.4 2.4 1.5 2.8 3.1" />
92
+ </svg>
93
+ );
94
+ // Who this account follows — a figure with an outbound arrow.
95
+ if (name === "following")
96
+ return (
97
+ <svg {...common}>
98
+ <circle cx="6" cy="5.6" r="2.4" />
99
+ <path d="M1.8 13c.5-2 2.1-3.2 4.2-3.2 .6 0 1.2.1 1.7.3" />
100
+ <path d="M9.8 10.6h4.2M12.2 8.8l1.8 1.8-1.8 1.8" />
101
+ </svg>
102
+ );
103
+ if (name === "posts_count")
104
+ return (
105
+ <svg {...common}>
106
+ <rect x="2.6" y="2.6" width="4.6" height="4.6" rx="1" />
107
+ <rect x="8.8" y="2.6" width="4.6" height="4.6" rx="1" />
108
+ <rect x="2.6" y="8.8" width="4.6" height="4.6" rx="1" />
109
+ <rect x="8.8" y="8.8" width="4.6" height="4.6" rx="1" />
110
+ </svg>
111
+ );
112
+ // A story highlight — the ring Instagram draws around one.
113
+ if (name === "highlights_count")
114
+ return (
115
+ <svg {...common}>
116
+ <circle cx="8" cy="8" r="5.6" strokeDasharray="2.6 1.8" />
117
+ <circle cx="8" cy="8" r="2.2" />
118
+ </svg>
119
+ );
120
+ if (name === "avg_engagement")
121
+ return (
122
+ <svg {...common}>
123
+ <path d="M8 13.2 3.4 8.8a2.9 2.9 0 0 1 4.6-3.4 2.9 2.9 0 0 1 4.6 3.4z" />
124
+ </svg>
125
+ );
126
+ if (name === "biography")
127
+ return (
128
+ <svg {...common}>
129
+ <rect x="2.4" y="2.8" width="11.2" height="10.4" rx="1.6" />
130
+ <path d="M4.8 6h6.4M4.8 8.4h6.4M4.8 10.8h3.6" />
131
+ </svg>
132
+ );
133
+ if (name === "category_name" || name === "business_category_name")
134
+ return (
135
+ <svg {...common}>
136
+ <path d="M8.4 2.6H13v4.6l-6.2 6.2-4.6-4.6z" />
137
+ <circle cx="10.8" cy="5.2" r="0.9" />
138
+ </svg>
139
+ );
140
+ if (name === "is_business_account" || name === "is_professional_account")
141
+ return (
142
+ <svg {...common}>
143
+ <rect x="2.2" y="5" width="11.6" height="8.2" rx="1.6" />
144
+ <path d="M6 5V3.6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1V5" />
145
+ </svg>
146
+ );
147
+ if (name === "is_verified")
148
+ return (
149
+ <svg {...common}>
150
+ <path d="M8 2.2l1.7 1.2 2-.2.6 2 1.7 1.1-.9 1.9.9 1.9-1.7 1.1-.6 2-2-.2L8 14.2l-1.7-1.2-2 .2-.6-2L2 10.1l.9-1.9L2 6.3l1.7-1.1.6-2 2 .2z" />
151
+ <path d="m5.8 8.1 1.5 1.5 3-3.1" />
152
+ </svg>
153
+ );
154
+ // The handle, and the display name it is written under.
155
+ if (name === "account" || name === "profile_name")
156
+ return (
157
+ <svg {...common}>
158
+ <circle cx="8" cy="8" r="2.4" />
159
+ <path d="M10.4 5.6v3.2a1.9 1.9 0 0 0 3.2 1.3A5.8 5.8 0 1 0 11 13.1" />
160
+ </svg>
161
+ );
162
+ if (name === "full_name")
163
+ return (
164
+ <svg {...common}>
165
+ <circle cx="8" cy="5.6" r="2.6" />
166
+ <path d="M3 13.4c.7-2.3 2.6-3.6 5-3.6s4.3 1.3 5 3.6" />
167
+ </svg>
168
+ );
169
+ if (name === "external_url" || name === "profile_url")
170
+ return (
171
+ <svg {...common}>
172
+ <path d="M6.6 9.4 4.9 11a2.6 2.6 0 1 1-1.5-4.4" />
173
+ <path d="M9.4 6.6 11.1 5a2.6 2.6 0 1 1 1.5 4.4" />
174
+ <path d="M6.2 8h3.6" />
175
+ </svg>
176
+ );
177
+ if (name === "bio_hashtags" || name === "post_hashtags")
178
+ return (
179
+ <svg {...common}>
180
+ <path d="M6.2 2.8 4.8 13.2M11.2 2.8 9.8 13.2M2.8 6h10.4M2.2 10h10.4" />
181
+ </svg>
182
+ );
183
+ if (name === "related_accounts")
184
+ return (
185
+ <svg {...common}>
186
+ <circle cx="8" cy="3.6" r="1.8" />
187
+ <circle cx="3.6" cy="12" r="1.8" />
188
+ <circle cx="12.4" cy="12" r="1.8" />
189
+ <path d="M6.8 5.2 4.6 10.4M9.2 5.2l2.2 5.2M5.4 12h5.2" />
190
+ </svg>
191
+ );
192
+ // The vendor's own record keys — the thing you hold to open exactly one row.
193
+ if (name === "id" || name === "fbid")
194
+ return (
195
+ <svg {...common}>
196
+ <circle cx="5.4" cy="10.6" r="2.6" />
197
+ <path d="M7.2 8.8 13 3M10.6 5.4l1.6 1.6M9.2 6.8l1.6 1.6" />
198
+ </svg>
199
+ );
200
+ return (
201
+ <svg {...common}>
202
+ <rect x="3.2" y="3.2" width="9.6" height="9.6" rx="2.2" />
203
+ <circle cx="8" cy="8" r="1.4" />
204
+ </svg>
205
+ );
206
+ }
207
+
208
+ /** One field's line in the list, with everything known about it. */
209
+ interface FieldRow {
210
+ name: string;
211
+ /** What the row is CALLED. Falls back to the raw name only if the server sent none. */
212
+ label: string;
213
+ hint: string;
214
+ kind: string;
215
+ operators: DiscoverOperator[];
216
+ options: { value: string; label: string }[];
217
+ /** Every stored condition naming this field, with its index in the saved array. */
218
+ at: number[];
219
+ /** MEASURED to carry values. `null` = the server has not said (C4 not shipped yet). */
220
+ populated: boolean | null;
221
+ /** A condition here cuts the corpus down. `null` = not said. */
222
+ narrowing: boolean | null;
223
+ }
224
+
225
+ /** A stored value → what the text box shows. A list is the "any of" shape. */
226
+ function valueText(v: Predicate["value"]): string {
227
+ if (Array.isArray(v)) return v.join(", ");
228
+ return v === undefined || v === null ? "" : String(v);
229
+ }
230
+
231
+ /**
232
+ * What the user typed → what is stored. Comma-separated becomes a LIST for the operators that
233
+ * accept one, and stays a plain string for the ones that do not.
234
+ *
235
+ * ⚠ A single value stays a SCALAR rather than a one-element list — the server sends scalars
236
+ * flat and lists as a nested OR group, and a one-element group is a shape nothing has been
237
+ * billed against.
238
+ */
239
+ function parseValue(text: string, multi: boolean): Predicate["value"] {
240
+ if (!multi) return text;
241
+ const parts = text.split(",").map((s) => s.trim()).filter(Boolean);
242
+ if (parts.length > 1) return parts;
243
+ // Keep the RAW text while it is still being typed — trimming here would fight the cursor on
244
+ // every keystroke, and the server trims anyway.
245
+ return text;
246
+ }
247
+
248
+ /**
249
+ * ⭐ WAVE 26 · DEBT D-69 — A THIRD VALUE COULD NOT BE TYPED, and the cause was one round trip
250
+ * through this pair of functions.
251
+ *
252
+ * `parseValue` returns an ARRAY at 2+ values; `valueText` then renders that array as
253
+ * `join(", ")` — which has NO TRAILING COMMA. So on a controlled input the separator was
254
+ * normalised away between keystrokes: typing `,` onto `skincare, beauty` re-rendered as
255
+ * `skincare, beauty`, and the next character landed as `skincare, beautym`. Two keywords
256
+ * silently became one that matches nothing. ⚠ It hit the FIRST surface a new user meets (wave 24
257
+ * made this panel the only door) and it hit precisely the 3+ keyword list the feature was built
258
+ * for — the owner's *"if i want to include many keywords like floral, flower, beauty"*.
259
+ *
260
+ * ⛔ THE FIX IS THE ONE `parseValue`'s OWN COMMENT ALREADY DESCRIBES FOR THE 1-VALUE CASE:
261
+ * keep the raw text while it is being typed. That comment was right and its scope was too narrow
262
+ * — it protected the cursor at one value and handed the 2+ case to `join`. So the raw text is
263
+ * held for whichever input is FOCUSED, and the stored value is still parsed on every keystroke
264
+ * (so nothing depends on blur to save) and re-parsed on blur (so the display settles).
265
+ *
266
+ * ⚠ KEYED BY THE ROW'S REACT KEY, not by index alone: two conditions on the same field are
267
+ * ordinary here ("Another condition on Bio"), and an index-only key would leak one row's draft
268
+ * into its sibling on a removal.
269
+ */
270
+ interface Draft {
271
+ key: string;
272
+ text: string;
273
+ }
274
+
275
+ /**
276
+ * The comparison a freshly toggled field starts with — ASKED FOR, never chosen here.
277
+ *
278
+ * ⛔ THE THIRD COSTUME OF ONE SCAR, and the reason this function no longer has an opinion.
279
+ * Wave 21 made a new condition NULLARY (`is_not_null`) so it would be saveable without a
280
+ * value, and called that "narrowing-in-the-right-direction". Wave 22 then wrote the narrowing
281
+ * law server-side and `is_not_null` is not in it. From that moment, toggling ANY field on and
282
+ * pressing Save returned "add at least one CONTENT condition…" — the error told the user to do
283
+ * the exact thing they had just done. Wave 24 deleted the wizard and made this panel the only
284
+ * door, so it stopped being a corner case and became the first thing a new user hits.
285
+ *
286
+ * The fix is not a better guess. It is that the module owning the guard also names the default
287
+ * (`automation_engine.default_operator`, asserted against `predicate_narrows` in
288
+ * `verify_automation.py`), and this file looks it up per FIELD. An absent one falls back to the
289
+ * server's first operator — never to a nullary, which is the thing that could not narrow.
290
+ */
291
+ function firstOperator(v: DiscoverVocab | undefined, name: string): string {
292
+ const meta = (v?.filterMeta || []).find((m) => m.name === name);
293
+ return meta?.defaultOperator || (v?.operators || [])[0] || "";
294
+ }
295
+
296
+ /**
297
+ * The rows, ordered: measured-populated first, then the rest.
298
+ *
299
+ * TWO SOURCES, AND THE FALLBACK IS NARROWER THAN THE REAL THING. With C4's
300
+ * `filterMeta` the answer is per field and complete. Without it, `lead` names
301
+ * the three fields seen carrying values and says NOTHING about the other
302
+ * eighteen — so those come back `null`, not `false`, and the UI prints no claim
303
+ * about them rather than an invented one.
304
+ */
305
+ function buildRows(preds: Predicate[], v?: DiscoverVocab): FieldRow[] {
306
+ const meta = new Map<string, DiscoverFieldMeta>(
307
+ (v?.filterMeta || []).map((m) => [m.name, m])
308
+ );
309
+ const lead = new Set(v?.lead || []);
310
+ const byName = new Map<string, number[]>();
311
+ preds.forEach((p, i) => {
312
+ const list = byName.get(p.name);
313
+ if (list) list.push(i);
314
+ else byName.set(p.name, [i]);
315
+ });
316
+
317
+ const rows = (v?.fields || []).map((name) => {
318
+ const m = meta.get(name);
319
+ return {
320
+ name,
321
+ label: m?.label || name,
322
+ hint: m?.hint || "",
323
+ kind: m?.kind || "text",
324
+ // ⛔ NO FALLBACK TO THE GLOBAL OPERATOR LIST. An older server that sends no per-field list
325
+ // gets a row with no comparisons rather than all fourteen on every field — visibly
326
+ // unfinished beats quietly offering `at least` on a yes/no column.
327
+ operators: m?.operators || [],
328
+ options: m?.options || [],
329
+ at: byName.get(name) || [],
330
+ populated: m ? !!m.populated : lead.has(name) ? true : null,
331
+ narrowing: m ? !!m.narrowing : null,
332
+ };
333
+ });
334
+ // Stable partition, never a sort: the server's order inside each half is the
335
+ // order the engine lists them in, and re-alphabetising it would be this file
336
+ // having an opinion about a server list.
337
+ return [...rows.filter((r) => r.populated === true), ...rows.filter((r) => r.populated !== true)];
338
+ }
339
+
340
+ export default function AutomationFind({
341
+ discover,
342
+ recordsLimit,
343
+ onRecordsLimit,
344
+ joinOp,
345
+ onJoinOp,
346
+ preds,
347
+ onPreds,
348
+ estimate,
349
+ onEstimate,
350
+ }: Props) {
351
+ const rows = buildRows(preds, discover);
352
+ const known = new Set(discover?.fields || []);
353
+ // The GLOBAL operator list is gone from this component: comparisons are per field now
354
+ // (`row.operators`). `nullary` survives for STRAYS only — a stored condition on a field the
355
+ // vendor dropped has no per-field row to read from, and it still has to render to be removable.
356
+ const nullary = new Set(discover?.nullaryOperators || []);
357
+ const withValues = rows.filter((r) => r.populated === true);
358
+ const rest = rows.filter((r) => r.populated !== true);
359
+ const knownMeta = !!(discover?.filterMeta || []).length;
360
+ // ⚠ THE STORED FILTER CAN NAME A FIELD THE LIST NO LONGER HAS — a vocabulary
361
+ // that moved, an automation copied from elsewhere. It must stay VISIBLE and
362
+ // REMOVABLE: a condition the user cannot see is one they cannot delete, and
363
+ // every Save keeps sending it. Same defect as a `<select>` whose value matches
364
+ // no option, one level up.
365
+ const strays = preds
366
+ .map((p, i) => ({ p, i }))
367
+ .filter(({ p }) => !known.has(p.name));
368
+
369
+ const maxRecords = discover?.guard?.maxRecords || discover?.maxRecords || 500;
370
+
371
+ /** D-69 — the raw text of the input a person is typing in right now (see `Draft`). */
372
+ const [draft, setDraft] = useState<Draft | null>(null);
373
+
374
+ /**
375
+ * ⭐ WAVE 27 · OWNER ITEM 29 — which condition's value is open in the BIG editor, if any.
376
+ *
377
+ * ⛔ THE TEXT IS HELD HERE, NOT WRITTEN THROUGH ON EVERY KEYSTROKE, and that is the one place
378
+ * this differs from the rail input beside it. The rail input writes per keystroke on purpose
379
+ * (a Save pressed straight from the field must not store the previous value — D-69's note).
380
+ * A modal has its own Save, so per-keystroke writes would buy nothing and would make Cancel
381
+ * a lie: the value would already be in the config by the time it was pressed.
382
+ * ⚠ The ANCHOR is captured at click time and stored, because the button it came from is
383
+ * inside a panel that scrolls — re-reading it later would place the overlay against a rect
384
+ * that has moved.
385
+ */
386
+ const [big, setBig] = useState<{
387
+ key: string;
388
+ index: number;
389
+ label: string;
390
+ multi: boolean;
391
+ text: string;
392
+ anchor: AnchorRect;
393
+ } | null>(null);
394
+
395
+ /**
396
+ * ⭐ ITEM 16 / D-71 — THE OBSERVED CATEGORY VALUES, fetched WHEN THIS PANEL OPENS.
397
+ *
398
+ * ⛔ ON MOUNT, ONCE, AND NEVER ON THE POLLED PAYLOAD. This component is rendered only while the
399
+ * Find panel is open, so mounting IS the panel opening — and the server moved these off
400
+ * `GET /automations` after measuring that deriving them reads two user tables plus the platform
401
+ * master, i.e. a network round-trip per open tab every 2.5 s. Putting the request here is the
402
+ * client half of that same decision, not an optimisation.
403
+ *
404
+ * ⚠ AN EMPTY LIST IS AN ANSWER AND IS RENDERED AS ONE. It means this deployment has not seen a
405
+ * category value yet — which is TRUE for a fresh tenant — and the field keeps its free-text
406
+ * input either way. It must never read as "loading forever", and it must never be filled in
407
+ * with a guess: D-59 is explicit that an invented taxonomy is worse than no dropdown.
408
+ */
409
+ const [cats, setCats] = useState<{ value: string; count: number }[]>([]);
410
+ useEffect(() => {
411
+ const ac = new AbortController();
412
+ discoverCategories(ac.signal)
413
+ .then((r) => setCats(Array.isArray(r?.options) ? r.options : []))
414
+ // Fail-quiet, on purpose: the picker is an ASSIST over a control that works without it.
415
+ // An error banner here would report a broken panel when the only thing missing is a
416
+ // convenience — and the free-text input beside it is unaffected.
417
+ .catch(() => setCats([]));
418
+ return () => ac.abort();
419
+ }, []);
420
+
421
+ const setAt = (i: number, patch: Partial<Predicate>) =>
422
+ onPreds(preds.map((p, j) => (j === i ? { ...p, ...patch } : p)));
423
+
424
+ const removeAt = (i: number) => onPreds(preds.filter((_p, j) => j !== i));
425
+
426
+ const toggleField = (row: FieldRow) => {
427
+ if (row.at.length) {
428
+ onPreds(preds.filter((p) => p.name !== row.name));
429
+ return;
430
+ }
431
+ onPreds([...preds, { name: row.name, operator: firstOperator(discover, row.name) }]);
432
+ };
433
+
434
+ /** One condition line — the comparison, its value, and a way out. */
435
+ const condition = (i: number, label: string, row?: FieldRow) => {
436
+ const p = preds[i];
437
+ if (!p) return null;
438
+ // The field's OWN comparisons. `row` is absent only for a stray (a stored condition on a
439
+ // field the vendor no longer offers), which keeps the raw token so it stays removable.
440
+ const ops = row?.operators || [];
441
+ const cur = ops.find((o) => o.value === p.operator);
442
+ const isNullary = cur ? cur.nullary : nullary.has(p.operator);
443
+ const isMulti = !!cur?.multi;
444
+ const options = row?.options || [];
445
+ const rowKey = `${p.name}-${i}`;
446
+ // D-69: the focused input shows what was TYPED; every other one shows the stored value.
447
+ const shown = draft && draft.key === rowKey ? draft.text : valueText(p.value);
448
+ /*
449
+ * ⭐ ITEM 16 / D-71 — WHICH FIELDS GET THE OBSERVED-VALUES PICKER, DERIVED OFF THE WIRE.
450
+ *
451
+ * ⛔ NOT A CLIENT-SIDE LIST OF FIELD NAMES. Naming the vendor's two category columns here
452
+ * would be the D-55 class verbatim — the wave-9 silent-drop shape, a client copy of a server
453
+ * vocabulary that is free to drift the day the engine adds a third one. (Their literal keys
454
+ * are deliberately not written anywhere in this file: the gate asserts the tokens are ABSENT,
455
+ * and an absence check cannot tell a hard-coded list from the comment forbidding one.)
456
+ * The server already publishes the answer twice over: `kind: "choice"` is its own declaration
457
+ * (`BD_FIELD_KINDS`), and `filter_meta` deliberately ships `options: []` for those fields
458
+ * because their vocabulary is OBSERVED rather than declared and arrives on its own route.
459
+ * So the rule is exactly that: a choice-kind field with no declared options is one whose
460
+ * values we can only have learned by seeing them. A new choice field inherits this for free.
461
+ */
462
+ const wantsObserved = row?.kind === "choice" && !options.length;
463
+ /** D-71 — APPEND, never replace: the picker has to compose with the comma list. */
464
+ const appendValue = (v: string) => {
465
+ const has = shown
466
+ .split(",")
467
+ .map((s) => s.trim().toLowerCase())
468
+ .filter(Boolean);
469
+ // Picking the same value twice is a no-op rather than a duplicate: the server expands a
470
+ // multi-value condition into an OR, and `beauty OR beauty` costs a slot to say nothing.
471
+ if (has.includes(v.trim().toLowerCase())) return;
472
+ const next = shown.trim() ? `${shown.replace(/[\s,]+$/, "")}, ${v}` : v;
473
+ setDraft(null);
474
+ setAt(i, { value: parseValue(next, isMulti) });
475
+ };
476
+ return (
477
+ <div className="autob-cond" key={rowKey}>
478
+ <select
479
+ className="auto-input is-small"
480
+ aria-label={`Comparison for ${label}`}
481
+ value={p.operator}
482
+ onChange={(e) => setAt(i, { operator: e.target.value })}
483
+ >
484
+ {/* The stored value is ALWAYS an option. A select whose value matches no
485
+ option renders the FIRST one, and the next Save writes a comparison
486
+ nobody chose — this codebase has paid for that twice. */}
487
+ {p.operator && !ops.some((o) => o.value === p.operator) ? (
488
+ <option value={p.operator}>{p.operator}</option>
489
+ ) : null}
490
+ {ops.map((o) => (
491
+ <option key={o.value} value={o.value}>
492
+ {o.label}
493
+ </option>
494
+ ))}
495
+ </select>
496
+ {isNullary ? null : options.length ? (
497
+ // A fixed vocabulary gets a dropdown. Owner item 2: a yes/no column was a box you
498
+ // typed `true` into, beside a comparison list that offered "at least".
499
+ <select
500
+ className="auto-input is-small"
501
+ aria-label={`Value for ${label}`}
502
+ value={valueText(p.value)}
503
+ onChange={(e) => setAt(i, { value: e.target.value })}
504
+ >
505
+ <option value="">Choose…</option>
506
+ {options.map((o) => (
507
+ <option key={o.value} value={o.value}>
508
+ {o.label}
509
+ </option>
510
+ ))}
511
+ </select>
512
+ ) : (
513
+ <input
514
+ className="auto-input is-small"
515
+ type={row?.kind === "number" ? "number" : "text"}
516
+ aria-label={`Value for ${label}`}
517
+ // ⭐ OWNER ITEM 3. Several comma-separated values become an "any of" condition, which
518
+ // the server sends as its own OR group — so it does NOT drag Followers into a union
519
+ // the way the Match dropdown would.
520
+ placeholder={isMulti ? "floral, flower, beauty" : ""}
521
+ // D-69: `shown` is the RAW text while this input has focus, so a trailing comma
522
+ // survives long enough to type the next value after it.
523
+ value={shown}
524
+ onFocus={() => setDraft({ key: rowKey, text: valueText(p.value) })}
525
+ onChange={(e) => {
526
+ setDraft({ key: rowKey, text: e.target.value });
527
+ // The STORE still updates on every keystroke. Parsing only on blur would mean a
528
+ // Save pressed straight from the field wrote the previous value — trading a typing
529
+ // bug for a data-loss one.
530
+ setAt(i, { value: parseValue(e.target.value, isMulti) });
531
+ }}
532
+ onBlur={(e) => {
533
+ setDraft(null);
534
+ setAt(i, { value: parseValue(e.target.value, isMulti) });
535
+ }}
536
+ />
537
+ )}
538
+ {/* ⭐ WAVE 27 · OWNER ITEM 29 — A WIDE EDITOR FOR THE VALUES THAT DO NOT FIT.
539
+ ⛔ THE COMPLAINT IS GEOMETRY, and the geometry is a CHAIN nothing here can widen:
540
+ `.auto-panel` is 380px, `.autob-cond .auto-input` divides what is left across three
541
+ controls, and `.is-small` trims it again — so a bio phrase list ("floral, flower,
542
+ wedding florist, event styling") is typed six characters at a time through a box
543
+ that scrolls sideways. The panel cannot grow; the EDITOR can leave it, and an
544
+ overlay is drawn in a body portal, so it is not bound by that chain at all.
545
+ ⚠ ALWAYS OFFERED, never "appears once the text is long": a control that materialises
546
+ at some threshold is one people do not know exists at the moment they need it.
547
+ ⚠ NOT on a `<select>` value (a fixed vocabulary has nothing to expand) and not on a
548
+ nullary comparison (which has no value at all). */}
549
+ {!isNullary && !options.length ? (
550
+ <button
551
+ type="button"
552
+ className="auto-input is-small autoc-expand"
553
+ aria-label={`Open a bigger editor for ${label}`}
554
+ title="Edit in a bigger box"
555
+ onClick={(e) => {
556
+ // ⚠ READ SYNCHRONOUSLY from the event. `currentTarget` is null by the time React
557
+ // re-invokes a state updater, and a null rect here paints the overlay at 0,0 —
558
+ // measured, in this repo, on a different control ([[react-event-currenttarget-updater]]).
559
+ const r = e.currentTarget.getBoundingClientRect();
560
+ setBig({
561
+ key: rowKey,
562
+ index: i,
563
+ label,
564
+ multi: isMulti,
565
+ text: valueText(p.value),
566
+ anchor: { left: r.left, top: r.top, right: r.right, bottom: r.bottom,
567
+ width: r.width, height: r.height },
568
+ });
569
+ }}
570
+ >
571
+ {/* Drawn, never a glyph: the design constitution's "no emojis in the UI" covers
572
+ the arrow characters too, and every other mark in this product is a path in the
573
+ same 16x16 stroke vocabulary. Two corners pulling apart = "make this bigger". */}
574
+ <svg viewBox="0 0 16 16" aria-hidden className="autoc-expand-icon">
575
+ <path d="M9.5 2.5h4v4M13.5 2.5 9.6 6.4" />
576
+ <path d="M6.5 13.5h-4v-4M2.5 13.5l3.9-3.9" />
577
+ </svg>
578
+ </button>
579
+ ) : null}
580
+ {/* ⭐ ITEM 16 / D-71 — THE APPEND-PICKER, BESIDE the input and never instead of it.
581
+ Free text stays the primary control: these are the values this deployment has SEEN,
582
+ not the values that exist, so the picker is a shortcut over a superset it cannot
583
+ enumerate. Appending in code also sidesteps D-69 entirely — a picked value never has
584
+ to survive a round trip through the text box. */}
585
+ {wantsObserved && !isNullary ? (
586
+ <select
587
+ className="auto-input is-small autoc-catpick"
588
+ aria-label={`Add a value seen before, to ${label}`}
589
+ title={
590
+ cats.length
591
+ ? "Values this workspace has actually seen, most-seen first. Adds to the list."
592
+ : "No category values have been seen in your data yet — type one."
593
+ }
594
+ value=""
595
+ disabled={!cats.length}
596
+ onChange={(e) => {
597
+ if (e.target.value) appendValue(e.target.value);
598
+ }}
599
+ >
600
+ <option value="">{cats.length ? "Seen before…" : "None seen yet"}</option>
601
+ {cats.map((c) => (
602
+ // The COUNT is the honest half (D-59): a value seen once and one seen forty times
603
+ // are different bets, and an alphabetical list of bare strings hides that.
604
+ <option key={c.value} value={c.value}>
605
+ {c.value} ({c.count.toLocaleString()})
606
+ </option>
607
+ ))}
608
+ </select>
609
+ ) : null}
610
+ {isMulti && Array.isArray(p.value) && p.value.length > 1 ? (
611
+ /*
612
+ * ⭐ ITEM 16 / D-71 — HOW MANY VALUES THIS CONDITION CARRIES, and against what ceiling.
613
+ *
614
+ * The COUNT is ours and is always shown. The CEILING is the server's and is shown only
615
+ * when the server sends one (`guard.maxValues` — an open `ASK ->A:`; `guard` ships
616
+ * `{minNarrowing, maxRecords}` today).
617
+ * ⛔ NO CLIENT-SIDE `4`. D-71's exit says "how many of D-68's 4 value slots" — but A
618
+ * shipped `depth_refusal()`, which computes the depth of the shape actually emitted
619
+ * rather than budgeting fixed slots, so the number 4 describes a model that no longer
620
+ * exists. Printing it would be a client copy of a server rule that has already changed
621
+ * once, which is the specific failure this file's header refuses to commit
622
+ * ([[measure-the-real-call]]: a correct-looking answer about the wrong subject).
623
+ */
624
+ <span className="autob-cond-any" title="Any one of these is a match">
625
+ any of {p.value.length}
626
+ {discover?.guard?.maxValues ? ` of ${discover.guard.maxValues}` : ""}
627
+ </span>
628
+ ) : null}
629
+ <button
630
+ type="button"
631
+ className="autob-cond-x"
632
+ aria-label={`Remove this condition on ${label}`}
633
+ title="Remove this condition"
634
+ onClick={() => removeAt(i)}
635
+ >
636
+ <svg width="12" height="12" viewBox="0 0 16 16" fill="none" aria-hidden="true">
637
+ <path d="M4 4l8 8M12 4l-8 8" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round" />
638
+ </svg>
639
+ </button>
640
+ </div>
641
+ );
642
+ };
643
+
644
+ const fieldRow = (row: FieldRow) => {
645
+ const on = row.at.length > 0;
646
+ return (
647
+ <div className={"autob-tgl" + (on ? " is-on" : "")} key={row.name}>
648
+ <div className="autob-tgl-head">
649
+ <button
650
+ type="button"
651
+ className={"auto-step-switch autob-tgl-switch" + (on ? " is-on" : "")}
652
+ aria-pressed={on}
653
+ aria-label={`${on ? "Stop searching" : "Search"} on ${row.label}`}
654
+ title={`${on ? "Stop searching" : "Search"} on ${row.label}`}
655
+ onClick={() => toggleField(row)}
656
+ >
657
+ <span className="auto-step-switch-knob" />
658
+ </button>
659
+ <span className="autob-tgl-mark">
660
+ <FilterMark name={row.name} />
661
+ </span>
662
+ {/* ⭐ THE HUMAN LABEL (owner item 1: "wtf is id/fbid").
663
+ This used to render the vendor's own column name verbatim, and the comment
664
+ defending it was RIGHT about the reason — a refusal names the field, and a label
665
+ that reads differently makes the error unmatchable. So the label did not move
666
+ alone: `field_label()` is what the SERVER's refusals use too, which is the only
667
+ version of this change that does not trade one confusion for another. */}
668
+ <span className="autob-tgl-name" title={row.hint || undefined}>
669
+ {row.label}
670
+ </span>
671
+ {/*
672
+ ⚠ ONE CHIP, AND IT MARKS THE MINORITY. The first version chipped every
673
+ non-narrowing field "broad" AND every unpopulated one "no values seen" —
674
+ which put two chips on 18 of the 21 rows, and one of them repeated the group
675
+ heading the row was already sitting under. A mark that is on almost every row
676
+ stops being a mark. Read off the screenshot; no assertion could have said it.
677
+
678
+ So: the group label carries POPULATED (it is what the groups ARE), and the
679
+ chip carries NARROWING, inverted to mark the five that DO narrow. That is the
680
+ actionable half — the server refuses a filter with no narrowing condition, so
681
+ "these are the ones that satisfy it" is the sentence a user needs, and
682
+ "this one does not" is not.
683
+ */}
684
+ {row.narrowing === true ? (
685
+ <span
686
+ className="autob-tgl-flag is-narrow"
687
+ title="A search needs at least one condition like this."
688
+ >
689
+ narrows
690
+ </span>
691
+ ) : null}
692
+ </div>
693
+ {on ? (
694
+ <div className="autob-tgl-body">
695
+ {row.at.map((i) => condition(i, row.label, row))}
696
+ {row.hint ? <p className="autob-tgl-hint">{row.hint}</p> : null}
697
+ <button
698
+ type="button"
699
+ className="autob-cond-add"
700
+ onClick={() =>
701
+ onPreds([...preds, { name: row.name, operator: firstOperator(discover, row.name) }])
702
+ }
703
+ >
704
+ Another condition on {row.label}
705
+ </button>
706
+ </div>
707
+ ) : null}
708
+ </div>
709
+ );
710
+ };
711
+
712
+ return (
713
+ <>
714
+ <div className="auto-field">
715
+ <label htmlFor="auto-records">How many profiles to fetch</label>
716
+ <input
717
+ id="auto-records"
718
+ className="auto-input"
719
+ type="number"
720
+ min={1}
721
+ max={maxRecords}
722
+ value={recordsLimit}
723
+ onChange={(e) => onRecordsLimit(Number(e.target.value) || 1)}
724
+ />
725
+ </div>
726
+ {/* Owner item 5: "Stop at the 'A search must be at most 500 profiles a run'". The two
727
+ sentences that followed explained the vendor's billing model to somebody who wants a
728
+ list of florists. */}
729
+ <p className="auto-hint">At most {maxRecords.toLocaleString()} profiles a run.</p>
730
+
731
+ <h3>Conditions</h3>
732
+ <div className="auto-field autob-match">
733
+ <label htmlFor="auto-joinop">Match</label>
734
+ <select
735
+ id="auto-joinop"
736
+ className="auto-input"
737
+ value={joinOp}
738
+ onChange={(e) => onJoinOp(e.target.value)}
739
+ >
740
+ <option value="and">All of them</option>
741
+ <option value="or">Any of them</option>
742
+ </select>
743
+ </div>
744
+
745
+ {!rows.length ? (
746
+ <p className="auto-note">
747
+ The searchable fields have not loaded yet.
748
+ </p>
749
+ ) : null}
750
+
751
+ {withValues.length ? (
752
+ <div className="autob-tglgroup">
753
+ <p
754
+ className="autob-tglgroup-label"
755
+ title="Most accounts have this filled in."
756
+ >
757
+ Usually filled in
758
+ </p>
759
+ {withValues.map(fieldRow)}
760
+ </div>
761
+ ) : null}
762
+
763
+ {rest.length ? (
764
+ <div className="autob-tglgroup">
765
+ <p
766
+ className="autob-tglgroup-label"
767
+ title={
768
+ knownMeta
769
+ ? "Most accounts leave these blank, so a condition here can return nothing."
770
+ : "We have not measured how often these are filled in."
771
+ }
772
+ >
773
+ {knownMeta ? "Often blank" : "Not measured"}
774
+ </p>
775
+ {rest.map(fieldRow)}
776
+ </div>
777
+ ) : null}
778
+
779
+ {strays.length ? (
780
+ <div className="autob-tglgroup">
781
+ <p className="autob-tglgroup-label">Not in the searchable list</p>
782
+ {strays.map(({ p, i }) => (
783
+ <div className="autob-tgl is-on is-stray" key={`stray-${i}`}>
784
+ <div className="autob-tgl-head">
785
+ {/* A stray gets a mark too — "every filter row" includes the ones the vendor no
786
+ longer offers, and a row missing the icon its neighbours have would read as a
787
+ different KIND of thing rather than as a field that fell out of the list. It
788
+ draws the neutral fallback by construction: a stray is, by definition, not in
789
+ the vocabulary this component has arms for. */}
790
+ <span className="autob-tgl-mark">
791
+ <FilterMark name={p.name} />
792
+ </span>
793
+ <span className="autob-tgl-name">{p.name}</span>
794
+ <span className="autob-tgl-flag is-stray" title="The server will refuse this condition by name.">
795
+ not searchable
796
+ </span>
797
+ </div>
798
+ <div className="autob-tgl-body">{condition(i, p.name)}</div>
799
+ </div>
800
+ ))}
801
+ </div>
802
+ ) : null}
803
+
804
+ {/* The consequence kept, the mechanism dropped: a condition on an often-blank field
805
+ returning nothing is a thing that will happen to somebody and confuse them. Why the
806
+ search is slow when broad is our problem, not theirs. */}
807
+ <p className="auto-hint">
808
+ Conditions on often-blank fields can come back with nothing.
809
+ </p>
810
+
811
+ <h3>What it costs</h3>
812
+ <div className="auto-head-actions">
813
+ <button type="button" className="auto-btn" onClick={onEstimate}>
814
+ Estimate this search
815
+ </button>
816
+ </div>
817
+ {estimate ? (
818
+ <p className="auto-hint">
819
+ About <strong>${estimate.usd}</strong> for {estimate.records} profiles, at $
820
+ {estimate.unitUsd} each.{" "}
821
+ {/*
822
+ THE CAVEAT IS NOT OPTIONAL. The vendor never quotes a price before a run and this
823
+ deployment cannot read its own balance, so presenting this as a billed figure
824
+ would be inventing a measurement.
825
+ */}
826
+ <strong>This is an estimate</strong> — {estimate.note}.
827
+ </p>
828
+ ) : null}
829
+
830
+ {/* ⭐ WAVE 27 · OWNER ITEM 29 — THE BIG EDITOR, rendered ONCE for the whole panel.
831
+ ⛔ Not one overlay per condition row: `AnchoredOverlay` mounts a body portal and
832
+ installs document-level dismiss/focus handlers, so N of them would be N listener
833
+ stacks fighting over one Escape key. One overlay, told which row it is editing.
834
+ ⚠ `role="dialog"`, not "menu": it holds a text field, and the menu layer's arrow-key
835
+ handling would take the cursor keys away from the text being typed. */}
836
+ {big ? (
837
+ <AnchoredOverlay
838
+ anchor={big.anchor}
839
+ className="autoc-bigedit"
840
+ placement="right-start"
841
+ role="dialog"
842
+ ariaLabel={`Value for ${big.label}`}
843
+ onDismiss={() => setBig(null)}
844
+ dataKind="find-value-editor"
845
+ >
846
+ <label htmlFor="autoc-bigedit-text">{big.label}</label>
847
+ <textarea
848
+ id="autoc-bigedit-text"
849
+ className="autoc-bigedit-text"
850
+ data-overlay-autofocus
851
+ autoFocus
852
+ rows={7}
853
+ spellCheck={false}
854
+ value={big.text}
855
+ placeholder={big.multi ? "floral, flower, wedding florist, event styling" : ""}
856
+ onChange={(e) => setBig((cur) => (cur ? { ...cur, text: e.target.value } : cur))}
857
+ onKeyDown={(e) => {
858
+ // Escape abandons; Ctrl/Cmd+Enter commits. A bare Enter types a NEWLINE, because
859
+ // this is a textarea and the whole reason it exists is that the value is long.
860
+ if (e.key === "Escape") setBig(null);
861
+ if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
862
+ setAt(big.index, { value: parseValue(big.text, big.multi) });
863
+ setBig(null);
864
+ }
865
+ }}
866
+ />
867
+ {/* The same sentence the rail input's placeholder makes, said where there is room
868
+ for it: a comma list is several values, and the server turns them into an OR. */}
869
+ {big.multi ? (
870
+ <p className="auto-hint">
871
+ Separate values with commas — the search matches any of them.
872
+ </p>
873
+ ) : null}
874
+ <div className="auto-head-actions">
875
+ <button
876
+ type="button"
877
+ className="auto-btn auto-btn--primary"
878
+ onClick={() => {
879
+ setAt(big.index, { value: parseValue(big.text, big.multi) });
880
+ setBig(null);
881
+ }}
882
+ >
883
+ Done
884
+ </button>
885
+ <button type="button" className="auto-btn" onClick={() => setBig(null)}>
886
+ Cancel
887
+ </button>
888
+ </div>
889
+ </AnchoredOverlay>
890
+ ) : null}
891
+ </>
892
+ );
893
+ }
web/src/automation/AutomationSurface.tsx CHANGED
@@ -1,664 +1,664 @@
1
- // ---------------------------------------------------------------------------
2
- // automation/AutomationSurface.tsx — the Automation surface (contract C-AUTONAV).
3
- //
4
- // ⛔ THE CONTRACT THIS FILE GUARANTEES, verbatim: this module path, a DEFAULT
5
- // export, and NO PROPS. The shell mounts `<AutomationSurface />` and hands it
6
- // nothing; it fetches its own data from `/api/v1/automations`. That is the whole
7
- // interface, and it is stated here because the mount line lives in a file this
8
- // session does not own — a named export would cost a cross-session round trip.
9
- //
10
- // "Automation replaces the Views rail" (owner) is implemented the way the wave
11
- // scout established: the secondary rail is NOT a shell concept — each surface
12
- // owns its own. So this renders a sibling of `.cg-views` built from the same
13
- // `--lp-rail-w` tokens and the same fold behaviour, and the Views rail is
14
- // untouched.
15
- // ---------------------------------------------------------------------------
16
- import type { KeyboardEvent as ReactKeyboardEvent } from "react";
17
- import { useCallback, useEffect, useRef, useState } from "react";
18
-
19
- import type { AutomationOpenDetail } from "../apiContract";
20
- import { AUTOMATION_OPEN_EVENT } from "../apiContract";
21
- // ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
22
- // The owner's reason is the mark's own geometry: *"these automations are basically loops."*
23
- // C7 says consume the existing icon component and do not inline a new SVG path, and
24
- // `shell/Brand.tsx`'s header says why in more detail than a rule could: the mark's one source of
25
- // truth is generated, it has already been hand-redrawn once, and the copy silently painted LAST
26
- // WAVE'S BRAND while a comment asserted parity. So this imports the component that POINTS at the
27
- // generated artifact. [[loopable-nav-logo-toggle]] — one element paints the mark.
28
- import { Mark } from "../shell/Brand";
29
- import AutomationDetail from "./AutomationDetail";
30
- import type { Automation, AutomationList } from "./automationApi";
31
- import {
32
- AutomationError,
33
- createAutomation,
34
- listAutomations,
35
- liveStepOf,
36
- patchAutomation,
37
- } from "./automationApi";
38
-
39
- /**
40
- * The run poll's cadence (item 6). Named constants because they are a MEASURED trade-off, not a
41
- * taste: `POLL_MAX_MS` is the longest a finished run can still look live, and it is the only
42
- * cost the backoff has.
43
- */
44
- const POLL_MIN_MS = 2500;
45
- const POLL_MAX_MS = 8000;
46
-
47
- /*
48
- * ⭐ WAVE 26 · ITEM 18 / R14 — `stateTitle` STOOD HERE AND IS DELETED WITH THE DOT IT DESCRIBED.
49
- *
50
- * It composed the dot's tooltip ("Running now — <step>", "Last run failed — <summary>"), so it had
51
- * exactly one reader and no reason to outlive it. Deleting the render and keeping the composer is
52
- * how a file accumulates functions that look live and are not — and `noUnusedLocals` would have
53
- * caught this one, which is not a reason to lean on it: the NEXT such function might still have a
54
- * second caller and compile fine.
55
- *
56
- * ⛔ WHY THE TOOLTIP WAS NOT MOVED ONTO THE MARK INSTEAD. R14 is "same mark on every automation;
57
- * no status colour". Hanging "Last run failed" off a mark that is deliberately status-free just
58
- * moves the status channel into `title`/`aria-label`, where it is worse: invisible to the eye,
59
- * announced to a screen reader, and contradicting the visual. The run state has a home one click
60
- * away — the Builder's own "Last run succeeded / failed" lines, which R13 explicitly keeps.
61
- *
62
- * ⚠ `stateOf` ITSELF IS UNTOUCHED in `automationApi.ts`. `Shell.tsx:64/1095` imports it and
63
- * `liveStepOf` calls it — deleting the export to tidy up this file would break a fence I do not own.
64
- */
65
-
66
- /**
67
- * The rail's second line — ONE fact, not two.
68
- *
69
- * ⚠ It carried "Next 2026-08-04 06:00 · last 2026-08-04 00:20", and the rail is `--lp-rail-w`
70
- * wide, so it rendered as "Next 2026-08-04 06:00 · l…" — a second line whose only complete word
71
- * was "Next". Two facts that both truncate are worth less than one that fits. The forward-looking
72
- * one wins when a schedule exists (it is the question the rail answers: does this run itself?),
73
- * the last run otherwise. The full picture is one click away in the editor's run history, and the
74
- * row's `title` already says both.
75
- *
76
- * Caught by READING the screenshot: the DOM was correct and the CSS ellipsis was doing exactly
77
- * its job. [[ui-invisible-to-assertions]] — judge the pixels even when everything is green.
78
- */
79
- function railSubtitle(a: Automation): string {
80
- /*
81
- * ⭐ WAVE 24 (owner item 6, "Run once now is laggy / looks stuck") — WHILE IT IS RUNNING,
82
- * THE ONE FACT WORTH THE LINE IS WHAT IT IS DOING.
83
- *
84
- * MEASURED, not guessed (`scratchpad/perf_automations.py`, mailbox D-5): the engine has been
85
- * publishing a live step the whole time (`status.step`, `routes_automation.py:50-51`) and NO
86
- * SURFACE HAS EVER RENDERED IT. So a `discover_instagram` run parked in its legitimate
87
- * `BD_FILTER_WAIT = 120 s` vendor wait and a genuinely hung thread were pixel-identical: a
88
- * pulsing blue dot and a subtitle still reciting the schedule. "Looks stuck" was not a
89
- * performance problem — the poll costs 23 ms — it was the product declining to say.
90
- *
91
- * It takes the line rather than joining it, for the reason this function's note already
92
- * gives: the rail is `--lp-rail-w` wide and two facts that both truncate are worth less than
93
- * one that fits. The schedule is still one click away and the dot's `title` carries both.
94
- */
95
- const step = liveStepOf(a);
96
- if (step) return step;
97
- if (a.schedule?.enabled) return a.nextRunAt ? `Next ${a.nextRunAt}` : "Scheduled";
98
- /*
99
- * ⭐ WAVE 26 · ITEM 19 / R13 — `Last run 2026-08-06 13:58` IS DELETED FROM THIS LINE.
100
- *
101
- * ⛔ SCOPE, because R13 draws a line that is easy to over-read: the ruling names THIS secondary
102
- * line under the automation's name. The Builder's "Last run succeeded / failed" RESULT lines
103
- * (`AutomationBuilder.tsx:1676`, `:1679`) are a different surface and are explicitly NOT in
104
- * scope — they are the answer to "Run once now", so deleting them would leave a button with no
105
- * outcome.
106
- *
107
- * "Manual only" STAYS, and the distinction is the ruling's own: a dated run record is a
108
- * changing FACT ABOUT THE PAST, while "this one has no schedule" is a stable property of the
109
- * automation — the same question `Next …`/`Scheduled` answers for its siblings. Dropping it too
110
- * would leave a manual automation with a blank second line and nothing saying why.
111
- */
112
- return "Manual only";
113
- }
114
-
115
- /*
116
- * ⛔ W23-W5's LISTENER IS AT MODULE SCOPE, AND THAT IS THE POINT — not a stylistic choice.
117
- *
118
- * The frame's click-through sets `window.location.hash = "#/automation"` and signals on the
119
- * VERY NEXT LINE (Shell.tsx:1647-1650, and its comment is right that the order matters). But a
120
- * hash write does not mount anything synchronously: `hashchange` is delivered as a task, the
121
- * router state then updates, React renders, and only THEN does a component effect subscribe.
122
- * A listener registered in `useEffect` therefore misses every click that arrives from another
123
- * page — which is every click, since a reader looking at Alerts is by definition not already
124
- * on this surface. The event would dispatch into nothing and every gate would stay green:
125
- * exactly the wave-20 item-25 failure the contract's own note describes, reproduced one layer
126
- * down.
127
- *
128
- * This module is imported statically by the shell, so this listener exists from app start. It
129
- * LATCHES the request; the component consumes the latch when it mounts and hears live events
130
- * while it is mounted. A request nobody claims is dropped on the next one — the latch is a
131
- * one-slot mailbox, never a queue.
132
- */
133
- let pendingOpen: AutomationOpenDetail | null = null;
134
- const openSubscribers = new Set<(detail: AutomationOpenDetail) => void>();
135
-
136
- if (typeof window !== "undefined") {
137
- window.addEventListener(AUTOMATION_OPEN_EVENT, (event) => {
138
- const detail = (event as CustomEvent<AutomationOpenDetail>).detail;
139
- if (!detail?.autoId) return;
140
- if (openSubscribers.size) {
141
- for (const notify of openSubscribers) notify(detail);
142
- return;
143
- }
144
- pendingOpen = detail;
145
- });
146
- }
147
-
148
- /*
149
- * ⛔ THE `AUTOMATION_CREATE_EVENT` LISTENER STOOD HERE AND IS DELETED WITH ITS SIGNALLER
150
- * (wave 25 item 5a, ruling R8) — latch, subscriber set and all.
151
- *
152
- * Wave 24 built it to close the opposite defect: the event was declared, signalled from
153
- * `Shell.tsx`, and consumed NOWHERE, so all three "Automated database" doors navigated to
154
- * `#/automation` and then did nothing, in production, with every gate green. R8 now deletes those
155
- * three doors — creating a database and pointing an automation at it are two acts — and they were
156
- * the event's only signaller.
157
- *
158
- * ⛔ SO THIS SIDE GOES TOO, IN THE SAME CHANGE. A listener with no signaller is the same defect
159
- * read from the other end: it compiles, it costs nothing at runtime, and it reads to the next
160
- * person as a live channel. The rule that catches both is now DERIVED rather than remembered —
161
- * `verify_automation_ui.py` enumerates every `AUTOMATION_*_EVENT` in `apiContract.ts` and demands
162
- * a signal site AND a listener site for each.
163
- *
164
- * ⚠ NOTHING ABOUT CREATING AN AUTOMATION IS LOST. `createAndOpen` is untouched and still has two
165
- * doors, both on this surface: the rail's button and the empty state's. What is gone is the claim
166
- * that a database can be created BY asking for an automation.
167
- */
168
-
169
- /**
170
- * A default name for a brand-new automation, and it has to be one the server ACCEPTS.
171
- *
172
- * MEASURED against the live route rather than assumed (`scratchpad/probe_names.py`): a create
173
- * with no name is a 400 (`name the automation`), and duplicate names are ALLOWED. So the name
174
- * cannot be omitted, and a fixed literal would stack "New automation" three deep in a rail that
175
- * sorts by name with nothing to tell the rows apart. Numbering from the existing list is a
176
- * client-side convenience over a server that does not care: if two tabs race, the loser gets a
177
- * duplicate name, which is legal, visible and renameable — never an error the user has to read.
178
- */
179
- function nextAutomationName(existing: Automation[]): string {
180
- const taken = new Set(existing.map((a) => (a.name || "").trim().toLowerCase()));
181
- for (let n = existing.length + 1; ; n += 1) {
182
- const candidate = `Automation ${n}`;
183
- if (!taken.has(candidate.toLowerCase())) return candidate;
184
- }
185
- }
186
-
187
- export default function AutomationSurface() {
188
- const [data, setData] = useState<AutomationList | null>(null);
189
- const [error, setError] = useState("");
190
- const [activeId, setActiveId] = useState<string>("");
191
- const [railShut, setRailShut] = useState(false);
192
- const [busy, setBusy] = useState("");
193
- /**
194
- * ⭐ WAVE 24 / C-CREATE — a create is now a ROUND TRIP, not a local wizard, so the door has to
195
- * say it is busy. Without this a second click while the POST is in flight makes a second
196
- * automation, and the server allows duplicate names, so the user gets two rows that look
197
- * identical and no error to explain either of them.
198
- */
199
- const [creatingNow, setCreatingNow] = useState(false);
200
- /** An open request waiting for the list to arrive (see the resolver below). */
201
- const [openRequest, setOpenRequest] = useState<AutomationOpenDetail | null>(null);
202
- /* ⛔ `createRequest` LEFT WITH THE EVENT THAT SET IT (wave 25, R8) — see the tombstone above. */
203
- const listRef = useRef<HTMLDivElement | null>(null);
204
-
205
- /**
206
- * ⛔ C14 LEG 2 — THE ID THE USER LAST ASKED FOR, and it is a ref because it has to be
207
- * written SYNCHRONOUSLY, inside the click, before any promise that was already in flight
208
- * can resolve. State would not do: a resolution racing React's commit would read the
209
- * previous value, which is the exact window this guard exists to close.
210
- */
211
- const wantedId = useRef("");
212
-
213
- /**
214
- * ⭐ THE ONLY PLACE THE RAIL'S SELECTION MOVES ON PURPOSE. Every deliberate change of
215
- * automation — a rail click, a create, a delete — goes through here, so "what the user
216
- * asked for" and "what is on screen" are written in one statement and cannot drift apart.
217
- * A resolution that wants to steer the rail compares itself against `wantedId` instead.
218
- */
219
  const select = useCallback((id: string, _stage = "") => {
220
- wantedId.current = id;
221
- setActiveId(id);
222
- }, []);
223
-
224
- /**
225
- * ⛔ C14 LEG 3 (the paint half) — A STALE READ MAY NOT REPAINT.
226
- *
227
- * Every call takes the next generation; only the newest one is allowed to `setData`. The
228
- * defect this closes is not cosmetic: a poll issued BEFORE a delete resolves AFTER it, and
229
- * a list that still contains the deleted automation puts a row back in the rail that the
230
- * user has just watched disappear. Same for a toggle, and same for a create.
231
- *
232
- * ⚠ THE RETURN VALUE IS NOT GUARDED, deliberately. The caller asked a question and gets
233
- * its own answer — coupling the two would mean a create whose `load()` was overtaken by a
234
- * poll silently failed to select the automation it had just made.
235
- */
236
- const gen = useRef(0);
237
-
238
- const load = useCallback(async (abort?: AbortSignal) => {
239
- const mine = ++gen.current;
240
- try {
241
- const next = await listAutomations(abort);
242
- if (mine === gen.current) {
243
- setData(next);
244
- setError("");
245
- }
246
- return next;
247
- } catch (e) {
248
- if ((e as Error)?.name === "AbortError") return null;
249
- if (mine === gen.current)
250
- setError(
251
- e instanceof AutomationError
252
- ? e.message
253
- : "The automation service did not answer."
254
- );
255
- return null;
256
- }
257
- }, []);
258
-
259
- useEffect(() => {
260
- const ac = new AbortController();
261
- void load(ac.signal);
262
- return () => ac.abort();
263
- }, [load]);
264
-
265
- /**
266
- * ⭐⭐ WAVE 24 / C-CREATE (a) — CREATE AND OPEN. This REPLACED the three-step wizard.
267
- *
268
- * R6 is what makes it possible: `plain` is a real kind with no machine graph nodes, and it is
269
- * what `POST /automations` stores when the body names none. So "new automation" stopped being
270
- * a question ("which of three kinds?" — two of which can no longer be created at all) and
271
- * became what it says: a new automation, open, on its trigger picker. Choosing a SOURCE is now
272
- * picking the `ig_profile_match` trigger, which is where that choice belongs.
273
- *
274
- * ⛔ THE GUARD IS A REF, not the `creatingNow` state, and this file already carries the scar
275
- * that explains why (C14 leg 2): a state read inside a click closure is the value from the
276
- * last render, so two fast clicks both see `false` and both POST. Duplicate names are legal
277
- * server-side, so the user would get two identical rows and no error. The ref is written
278
- * synchronously, inside the click, before anything can await.
279
- */
280
- const creatingRef = useRef(false);
281
- const createAndOpen = useCallback(async () => {
282
- if (creatingRef.current) return;
283
- creatingRef.current = true;
284
- setCreatingNow(true);
285
- try {
286
- const res = await createAutomation({
287
- name: nextAutomationName(data?.automations || []),
288
- });
289
- const id = res?.automation?.id || "";
290
- // The create flow is the one caller allowed to name a different id (C14 leg 2) — the
291
- // automation did not exist when the click happened, so there is nothing to race with.
292
- const next = await load();
293
- if (id && next) select(id);
294
- } catch (e) {
295
- setError(
296
- e instanceof AutomationError ? e.message : "The automation was not created."
297
- );
298
- } finally {
299
- creatingRef.current = false;
300
- setCreatingNow(false);
301
- }
302
- }, [data, load, select]);
303
-
304
- // A run is a background thread on the server, so the surface has to ASK whether it
305
- // finished. Polling only while something is actually running keeps an idle surface
306
- // silent — a fixed interval would be a request every few seconds forever, for a page
307
- // whose contents change a handful of times a day.
308
- //
309
- // ⛔ C14 LEG 3 (the abort half). It used to call `load()` bare — no signal, nothing to
310
- // cancel — so a request the interval had already issued kept going after the effect that
311
- // owned it was gone. One controller per effect run, aborted with the interval, means a poll
312
- // cannot outlive the condition that justified it. The generation guard inside `load` covers
313
- // the rest: a response that survives the abort still cannot repaint over a newer one.
314
- //
315
- // ⭐ WAVE 24 (item 6, C-PERF) — THE INTERVAL BECAME A BACKOFF, and the measurement is why it is
316
- // a SMALL change rather than the big one the contract's hypothesis 2 asked for.
317
- //
318
- // MEASURED (`scratchpad/perf_automations.py`, mailbox D-5): this poll costs ~23 ms and 24 KB
319
- // for a ten-automation tenant, and the engine work the hypothesis blamed — a `graph()` rebuild
320
- // per automation — is **41 µs each**, i.e. under 4% of the request. The payload was never the
321
- // problem, so nothing here gets a cheaper endpoint.
322
- //
323
- // What the numbers DO indict is the aggregate: a discovery run legitimately blocks up to
324
- // `BD_FILTER_WAIT` = 120 s, and two independent 2.5 s polls across this file and the detail
325
- // spend ~96 requests and ~2.3 MB over that window — in the same single process the run itself
326
- // is a thread in. So the delay grows 2.5 → 5 → 8 s and stops there.
327
- //
328
- // ⚠ NOT "to learn nothing", and the distinction is the whole justification. That WAS true when
329
- // this file rendered no live step: the poll's only observable effect was a dot that had already
330
- // been pulsing for two minutes. It stopped being true in this same change — `liveStepOf` now
331
- // paints `status.step`, so a poll carries the one fact worth having. The backoff is therefore
332
- // NOT "stop asking a pointless question"; it is FEWER ROUND TRIPS FOR THE SAME INFORMATION, on
333
- // a step text that changes every few seconds at most, not every 2.5.
334
- // The cap is deliberately low: it bounds how long a FINISHED run can still look live, which is
335
- // the only thing a backoff can make worse, and 8 s of that is worth ~⅔ fewer requests.
336
- //
337
- // ⛔ THE KEY IS THE RUNNING SET, NOT A BOOLEAN, and that is what makes the reset correct: the
338
- // effect re-runs — and the delay drops back to 2.5 s — the moment a run starts or finishes, so
339
- // a user who clicks Run now gets the fast cadence again instead of inheriting the tail of the
340
- // previous run's backoff. A bare `anyRunning` boolean cannot see the second run start.
341
- // A step text changing does NOT re-key it, so the interval never thrashes.
342
- //
343
- // C14 LEG 3 (the abort half) is unchanged and still load-bearing: one controller per effect
344
- // run, aborted with the timer, so a poll cannot outlive the condition that justified it. The
345
- // generation guard inside `load` covers the rest.
346
- const runningKey = (data?.automations || [])
347
- .filter((a) => a.running)
348
- .map((a) => a.id)
349
- .sort()
350
- .join(",");
351
- useEffect(() => {
352
- if (!runningKey) return undefined;
353
- const ac = new AbortController();
354
- let delay = POLL_MIN_MS;
355
- let timer = 0;
356
- const tick = () => {
357
- void load(ac.signal);
358
- delay = Math.min(delay * 2, POLL_MAX_MS);
359
- timer = window.setTimeout(tick, delay);
360
- };
361
- timer = window.setTimeout(tick, delay);
362
- return () => {
363
- window.clearTimeout(timer);
364
- ac.abort();
365
- };
366
- }, [runningKey, load]);
367
-
368
- // ── W23-W5, the surface's half: hear the request, then answer it when we CAN ──────────
369
- useEffect(() => {
370
- const notify = (detail: AutomationOpenDetail) => setOpenRequest(detail);
371
- openSubscribers.add(notify);
372
- if (pendingOpen) {
373
- const latched = pendingOpen;
374
- pendingOpen = null;
375
- setOpenRequest(latched);
376
- }
377
- return () => {
378
- openSubscribers.delete(notify);
379
- };
380
- }, []);
381
-
382
- /* ⛔ THE C-CREATE(b) SUBSCRIBE EFFECT AND ITS RESOLVER STOOD HERE (wave 25, R8). They heard the
383
- create event and, once the list had arrived, called `createAndOpen` — the "wait for the list
384
- or every automation is named Automation 1" note lives on in `nextAutomationName`, which is
385
- still numbered off `existing`. With no signaller there is nothing to hear. */
386
-
387
- /**
388
- * ⚠ THE REQUEST OUTLIVES THE FETCH, and it has to. The reader clicks a notification from
389
- * another page, so this surface is mounting WITH AN EMPTY LIST — "select it if it is in the
390
- * list" would drop every real click and keep only the one case where the reader was already
391
- * here. So the request is held until `data` exists, and only then answered.
392
- *
393
- * An automation the reader can no longer open does NOTHING (the contract's own words): the
394
- * request is cleared either way, so a stale id cannot sit here re-firing against every
395
- * subsequent list.
396
- */
397
- useEffect(() => {
398
- if (!openRequest || !data) return;
399
- const found = (data.automations || []).some((a) => a.id === openRequest.autoId);
400
- if (found) {
401
- select(openRequest.autoId, openRequest.stageId || "");
402
- }
403
- setOpenRequest(null);
404
- }, [openRequest, data, select]);
405
-
406
- const items = data?.automations || [];
407
- const active = items.find((a) => a.id === activeId) || null;
408
-
409
- const toggleEnabled = async (a: Automation) => {
410
- setBusy(a.id);
411
- try {
412
- await patchAutomation(a.id, {
413
- schedule: { cron: a.schedule.cron, enabled: !a.schedule.enabled },
414
- });
415
- await load();
416
- } catch (e) {
417
- setError(e instanceof AutomationError ? e.message : "That change was not saved.");
418
- } finally {
419
- setBusy("");
420
- }
421
- };
422
-
423
- const onRailKey = (event: ReactKeyboardEvent<HTMLDivElement>) => {
424
- if (event.key !== "ArrowDown" && event.key !== "ArrowUp") return;
425
- const rows = Array.from(
426
- listRef.current?.querySelectorAll<HTMLButtonElement>(".auto-row-main") || []
427
- );
428
- if (!rows.length) return;
429
- event.preventDefault();
430
- const at = rows.indexOf(document.activeElement as HTMLButtonElement);
431
- const next = event.key === "ArrowDown" ? (at + 1) % rows.length
432
- : (at - 1 + rows.length) % rows.length;
433
- rows[next < 0 ? 0 : next]?.focus();
434
- };
435
-
436
- return (
437
- <div className="auto-surface">
438
- <aside
439
- className={"auto-rail" + (railShut ? " is-collapsed" : "")}
440
- aria-label="Automations"
441
- >
442
- {/* The same three-bars fold control the Views rail carries — the two rails are
443
- one idea, so they must not fold with two different affordances. */}
444
- <div className="auto-rail-top">
445
- <button
446
- type="button"
447
- className="cg-rail-toggle"
448
- aria-label={railShut ? "Expand automations" : "Minimize automations"}
449
- aria-expanded={!railShut}
450
- title={railShut ? "Expand automations" : "Minimize automations"}
451
- onClick={() => setRailShut((v) => !v)}
452
- >
453
- <svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
454
- <path
455
- d="M2.5 4.4h11M2.5 8h11M2.5 11.6h11"
456
- stroke="currentColor"
457
- strokeWidth="1.35"
458
- strokeLinecap="round"
459
- />
460
- </svg>
461
- </button>
462
- </div>
463
-
464
- <div className="auto-create">
465
- <button
466
- type="button"
467
- className="auto-create-btn"
468
- disabled={creatingNow}
469
- onClick={() => void createAndOpen()}
470
- >
471
- <svg width="14" height="14" viewBox="0 0 16 16" fill="none" aria-hidden="true">
472
- <path
473
- d="M8 3.4v9.2M3.4 8h9.2"
474
- stroke="currentColor"
475
- strokeWidth="1.5"
476
- strokeLinecap="round"
477
- />
478
- </svg>
479
- New automation
480
- </button>
481
- </div>
482
-
483
- <div className="auto-list" ref={listRef} onKeyDown={onRailKey}>
484
- {items.map((a) => {
485
- return (
486
- <div
487
- key={a.id}
488
- className={"auto-row" + (a.id === activeId ? " is-active" : "")}
489
- >
490
- <button
491
- type="button"
492
- className="auto-row-main"
493
- onClick={() => select(a.id)}
494
- >
495
- {/* ⭐ ITEM 18 / R14 — THE LOOP MARK, WHERE THE STATUS DOT WAS. Same mark on
496
- every row, no status colour: the owner's point is that an automation IS a
497
- loop, not that this one is green. The `.auto-loopmark` wrapper is what sizes
498
- it — `.lp-mark` is shell-owned CSS and outside this session's region, so the
499
- box is mine and the mark is theirs. */}
500
- <span className="auto-loopmark">
501
- <Mark size={15} />
502
- </span>
503
- <span className="auto-row-text">
504
- <span className="auto-row-name">{a.name}</span>
505
- <span className="auto-row-desc">{railSubtitle(a)}</span>
506
- </span>
507
- </button>
508
- <button
509
- type="button"
510
- className={"auto-row-switch" + (a.schedule?.enabled ? " is-on" : "")}
511
- disabled={busy === a.id}
512
- aria-pressed={!!a.schedule?.enabled}
513
- title={
514
- a.schedule?.enabled
515
- ? `Scheduled: ${a.schedule.cron}. Click to pause.`
516
- : "Not scheduled. Click to enable."
517
- }
518
- onClick={() => void toggleEnabled(a)}
519
- >
520
- <span className="auto-row-switch-knob" />
521
- </button>
522
- </div>
523
- );
524
- })}
525
- {/* ONE LINE (R13). It used to describe the kinds — "one reads a public web page…
526
- the other fills an automation column…" — which was two sentences, wrong by the
527
- time a third kind shipped, and printed inside a `--lp-rail-w` column. What the
528
- kinds are belongs to the create form, which is one click away and lists all of
529
- them from the server. */}
530
- {!items.length && !error ? (
531
- <p className="auto-rail-empty">No automations yet.</p>
532
- ) : null}
533
- </div>
534
- </aside>
535
-
536
- <section className="auto-main">
537
- {error ? (
538
- <div className="auto-banner is-error" role="alert">
539
- {error}
540
- </div>
541
- ) : null}
542
- {data && !data.storeAvailable ? (
543
- <div className="auto-banner is-warn" role="status">
544
- The tenant store is unavailable, so nothing can be saved right now.
545
- </div>
546
- ) : null}
547
-
548
- {/*
549
- ⛔ THE `creating` BRANCH IS GONE (wave 24, C-CREATE a / owner items 3 + 4).
550
- `AutomationCreate` used to render here — a three-step wizard whose first step asked
551
- which DATABASE to write into (an existing one, a new one, or one the automation itself
552
- would create) and whose second asked which of three KINDS.
553
- R6 retires that question at the root: two of the three kinds can no longer be created
554
- at all, and the third is now reached by picking a trigger. A new automation is created
555
- the moment it is asked for (`createAndOpen`) and opens on its own Builder, so there is
556
- no intermediate face left to render and no `creating` state to hold.
557
- ⭐ WAVE 25 (R8) retires the LAST of that vocabulary: the third answer was the
558
- "automated database", and it is gone from every door in the product. This button is now
559
- one of exactly two ways to make an automation, and both are on this surface.
560
- The import went with it — it is what would break `npx tsc -b` for the WHOLE client the
561
- moment session B deletes the file, which is why this deletion is sequenced first.
562
- */}
563
- {active ? (
564
- <AutomationDetail
565
- key={active.id}
566
- automation={active}
567
- /* ⛔ `kinds={data?.kinds || []}` LEFT HERE WITH THE PROP IT FED (wave 25, D-57) —
568
- both halves in one change, because either alone is a `tsc` error. */
569
- cronPresets={data?.cronPresets || []}
570
- paidReady={!!data?.paidReady}
571
- discover={data?.discover}
572
- // The trigger vocabulary (C3). Forwarded as-is — absent stays absent, so the
573
- // trigger face can tell "the server offered nothing" from "the server offered
574
- // an empty list" rather than collapsing both into a picker with no options.
575
- triggers={data?.triggers}
576
- // C4's action menu + the builder's ceilings. Forwarded as-is for the same reason
577
- // `triggers` is: absent must stay absent, so the builder can tell "the server
578
- // offered nothing" from "the server offered an empty list".
579
- catalog={data?.actionsCatalog}
580
- vocab={data?.flow}
581
- // `?? null` and never `|| {enabled:false}`: an absent tick bit is "the server did
582
- // not say", which Step 1 prints as its own sentence. Defaulting it here would
583
- // turn a missing field into a claim about production (C6 amendment #1).
584
- tick={data?.tick ?? null}
585
- /*
586
- * ⭐ W24-W1 (item 6) — WHAT THIS RUN IS DOING, on the one surface that survives in
587
- * BOTH views. REQUIRED on the far side deliberately: an optional prop that nobody
588
- * passes degrades to "the feature does not exist", which is indistinguishable from
589
- * "it was never built" — and this whole item exists because a live step text rode
590
- * the wire for three waves with nothing rendering it.
591
- * `liveStepOf` returns the step ONLY while the automation is running; a stale step
592
- * from a finished run is a worse answer than none.
593
- */
594
- liveStep={liveStepOf(active)}
595
- /*
596
- * ⛔ C14 LEG 2 — THE STALE-ID WRITE-BACK, and this is the ghost's second cause.
597
- *
598
- * The detail calls `onSaved(automation.id)` from six places (save, the trigger
599
- * picker, a schedule change, a node switch, a card move, a run). Each closure
600
- * captures the automation it was mounted for, so a save that resolves AFTER the
601
- * user has clicked a different row used to call `setActiveId(the OLD id)` — the
602
- * rail jumped back, the detail remounted, and what the user saw was the previous
603
- * automation reappearing over the one they had just opened. It looked like a
604
- * rendering bug; it was a resolution steering the selection.
605
- *
606
- * A resolution may no longer steer anything. It reloads the list — that part was
607
- * always right — and it re-asserts the selection ONLY when its id is still the one
608
- * the user asked for, which makes the write a no-op in the good case and nothing
609
- * at all in the bad one.
610
- */
611
- onSaved={async (id) => {
612
- const next = await load();
613
- if (id && next && id === wantedId.current) select(id);
614
- }}
615
- onDeleted={async () => {
616
- select("");
617
- await load();
618
- }}
619
- />
620
- ) : (
621
- /*
622
- * THE EMPTY STATE IS ONE LINE AND A BUTTON (owner ruling R13 — "never
623
- * over-explain", now a DESIGN.md law).
624
- *
625
- * It was a heading, a three-sentence paragraph and a 130-word bulleted list
626
- * describing all three kinds. Every word of it was true and none of it was READ:
627
- * an empty state is passed through, not studied, and the person looking at it has
628
- * already decided to make an automation. The kinds are described where the choice
629
- * is actually made — the create form lists them FROM THE SERVER, so that copy also
630
- * cannot go stale the way this list had (it described two kinds after a third
631
- * shipped).
632
- *
633
- * The button is here rather than only in the rail because this pane is where the
634
- * eye is; a create affordance the user has to go find is the same defect as the
635
- * paragraph, spent differently.
636
- */
637
- <div className="autob-empty">
638
- {/*
639
- ⛔ NO TITLE HERE ANY MORE (C13, owner item 2). This pane carried an `h1`
640
- reading "Automations" at 20px/600 — a THIRD title treatment on a page that
641
- also had the header's editable 16px/700 input, against every database page's
642
- single 16px/650 `shell-db-name`. The shell now wraps this branch in the same
643
- `shell-db-frame` + `DbHead` a database gets (wiring W23-W1), so the page's name
644
- is drawn once, by the one component that draws every other page's name. A
645
- stand-in restyled to match would have been a second copy of the same fact,
646
- free to drift the day the header moves.
647
- */}
648
- <p className="autob-empty-line">
649
- A job this workspace runs for you, on demand or on a schedule.
650
- </p>
651
- <button
652
- type="button"
653
- className="auto-btn is-primary"
654
- disabled={creatingNow}
655
- onClick={() => void createAndOpen()}
656
- >
657
- New automation
658
- </button>
659
- </div>
660
- )}
661
- </section>
662
- </div>
663
- );
664
- }
 
1
+ // ---------------------------------------------------------------------------
2
+ // automation/AutomationSurface.tsx — the Automation surface (contract C-AUTONAV).
3
+ //
4
+ // ⛔ THE CONTRACT THIS FILE GUARANTEES, verbatim: this module path, a DEFAULT
5
+ // export, and NO PROPS. The shell mounts `<AutomationSurface />` and hands it
6
+ // nothing; it fetches its own data from `/api/v1/automations`. That is the whole
7
+ // interface, and it is stated here because the mount line lives in a file this
8
+ // session does not own — a named export would cost a cross-session round trip.
9
+ //
10
+ // "Automation replaces the Views rail" (owner) is implemented the way the wave
11
+ // scout established: the secondary rail is NOT a shell concept — each surface
12
+ // owns its own. So this renders a sibling of `.cg-views` built from the same
13
+ // `--lp-rail-w` tokens and the same fold behaviour, and the Views rail is
14
+ // untouched.
15
+ // ---------------------------------------------------------------------------
16
+ import type { KeyboardEvent as ReactKeyboardEvent } from "react";
17
+ import { useCallback, useEffect, useRef, useState } from "react";
18
+
19
+ import type { AutomationOpenDetail } from "../apiContract";
20
+ import { AUTOMATION_OPEN_EVENT } from "../apiContract";
21
+ // ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
22
+ // The owner's reason is the mark's own geometry: *"these automations are basically loops."*
23
+ // C7 says consume the existing icon component and do not inline a new SVG path, and
24
+ // `shell/Brand.tsx`'s header says why in more detail than a rule could: the mark's one source of
25
+ // truth is generated, it has already been hand-redrawn once, and the copy silently painted LAST
26
+ // WAVE'S BRAND while a comment asserted parity. So this imports the component that POINTS at the
27
+ // generated artifact. [[loopable-nav-logo-toggle]] — one element paints the mark.
28
+ import { Mark } from "../shell/Brand";
29
+ import AutomationDetail from "./AutomationDetail";
30
+ import type { Automation, AutomationList } from "./automationApi";
31
+ import {
32
+ AutomationError,
33
+ createAutomation,
34
+ listAutomations,
35
+ liveStepOf,
36
+ patchAutomation,
37
+ } from "./automationApi";
38
+
39
+ /**
40
+ * The run poll's cadence (item 6). Named constants because they are a MEASURED trade-off, not a
41
+ * taste: `POLL_MAX_MS` is the longest a finished run can still look live, and it is the only
42
+ * cost the backoff has.
43
+ */
44
+ const POLL_MIN_MS = 2500;
45
+ const POLL_MAX_MS = 8000;
46
+
47
+ /*
48
+ * ⭐ WAVE 26 · ITEM 18 / R14 — `stateTitle` STOOD HERE AND IS DELETED WITH THE DOT IT DESCRIBED.
49
+ *
50
+ * It composed the dot's tooltip ("Running now — <step>", "Last run failed — <summary>"), so it had
51
+ * exactly one reader and no reason to outlive it. Deleting the render and keeping the composer is
52
+ * how a file accumulates functions that look live and are not — and `noUnusedLocals` would have
53
+ * caught this one, which is not a reason to lean on it: the NEXT such function might still have a
54
+ * second caller and compile fine.
55
+ *
56
+ * ⛔ WHY THE TOOLTIP WAS NOT MOVED ONTO THE MARK INSTEAD. R14 is "same mark on every automation;
57
+ * no status colour". Hanging "Last run failed" off a mark that is deliberately status-free just
58
+ * moves the status channel into `title`/`aria-label`, where it is worse: invisible to the eye,
59
+ * announced to a screen reader, and contradicting the visual. The run state has a home one click
60
+ * away — the Builder's own "Last run succeeded / failed" lines, which R13 explicitly keeps.
61
+ *
62
+ * ⚠ `stateOf` ITSELF IS UNTOUCHED in `automationApi.ts`. `Shell.tsx:64/1095` imports it and
63
+ * `liveStepOf` calls it — deleting the export to tidy up this file would break a fence I do not own.
64
+ */
65
+
66
+ /**
67
+ * The rail's second line — ONE fact, not two.
68
+ *
69
+ * ⚠ It carried "Next 2026-08-04 06:00 · last 2026-08-04 00:20", and the rail is `--lp-rail-w`
70
+ * wide, so it rendered as "Next 2026-08-04 06:00 · l…" — a second line whose only complete word
71
+ * was "Next". Two facts that both truncate are worth less than one that fits. The forward-looking
72
+ * one wins when a schedule exists (it is the question the rail answers: does this run itself?),
73
+ * the last run otherwise. The full picture is one click away in the editor's run history, and the
74
+ * row's `title` already says both.
75
+ *
76
+ * Caught by READING the screenshot: the DOM was correct and the CSS ellipsis was doing exactly
77
+ * its job. [[ui-invisible-to-assertions]] — judge the pixels even when everything is green.
78
+ */
79
+ function railSubtitle(a: Automation): string {
80
+ /*
81
+ * ⭐ WAVE 24 (owner item 6, "Run once now is laggy / looks stuck") — WHILE IT IS RUNNING,
82
+ * THE ONE FACT WORTH THE LINE IS WHAT IT IS DOING.
83
+ *
84
+ * MEASURED, not guessed (`scratchpad/perf_automations.py`, mailbox D-5): the engine has been
85
+ * publishing a live step the whole time (`status.step`, `routes_automation.py:50-51`) and NO
86
+ * SURFACE HAS EVER RENDERED IT. So a `discover_instagram` run parked in its legitimate
87
+ * `BD_FILTER_WAIT = 120 s` vendor wait and a genuinely hung thread were pixel-identical: a
88
+ * pulsing blue dot and a subtitle still reciting the schedule. "Looks stuck" was not a
89
+ * performance problem — the poll costs 23 ms — it was the product declining to say.
90
+ *
91
+ * It takes the line rather than joining it, for the reason this function's note already
92
+ * gives: the rail is `--lp-rail-w` wide and two facts that both truncate are worth less than
93
+ * one that fits. The schedule is still one click away and the dot's `title` carries both.
94
+ */
95
+ const step = liveStepOf(a);
96
+ if (step) return step;
97
+ if (a.schedule?.enabled) return a.nextRunAt ? `Next ${a.nextRunAt}` : "Scheduled";
98
+ /*
99
+ * ⭐ WAVE 26 · ITEM 19 / R13 — `Last run 2026-08-06 13:58` IS DELETED FROM THIS LINE.
100
+ *
101
+ * ⛔ SCOPE, because R13 draws a line that is easy to over-read: the ruling names THIS secondary
102
+ * line under the automation's name. The Builder's "Last run succeeded / failed" RESULT lines
103
+ * (`AutomationBuilder.tsx:1676`, `:1679`) are a different surface and are explicitly NOT in
104
+ * scope — they are the answer to "Run once now", so deleting them would leave a button with no
105
+ * outcome.
106
+ *
107
+ * "Manual only" STAYS, and the distinction is the ruling's own: a dated run record is a
108
+ * changing FACT ABOUT THE PAST, while "this one has no schedule" is a stable property of the
109
+ * automation — the same question `Next …`/`Scheduled` answers for its siblings. Dropping it too
110
+ * would leave a manual automation with a blank second line and nothing saying why.
111
+ */
112
+ return "Manual only";
113
+ }
114
+
115
+ /*
116
+ * ⛔ W23-W5's LISTENER IS AT MODULE SCOPE, AND THAT IS THE POINT — not a stylistic choice.
117
+ *
118
+ * The frame's click-through sets `window.location.hash = "#/automation"` and signals on the
119
+ * VERY NEXT LINE (Shell.tsx:1647-1650, and its comment is right that the order matters). But a
120
+ * hash write does not mount anything synchronously: `hashchange` is delivered as a task, the
121
+ * router state then updates, React renders, and only THEN does a component effect subscribe.
122
+ * A listener registered in `useEffect` therefore misses every click that arrives from another
123
+ * page — which is every click, since a reader looking at Alerts is by definition not already
124
+ * on this surface. The event would dispatch into nothing and every gate would stay green:
125
+ * exactly the wave-20 item-25 failure the contract's own note describes, reproduced one layer
126
+ * down.
127
+ *
128
+ * This module is imported statically by the shell, so this listener exists from app start. It
129
+ * LATCHES the request; the component consumes the latch when it mounts and hears live events
130
+ * while it is mounted. A request nobody claims is dropped on the next one — the latch is a
131
+ * one-slot mailbox, never a queue.
132
+ */
133
+ let pendingOpen: AutomationOpenDetail | null = null;
134
+ const openSubscribers = new Set<(detail: AutomationOpenDetail) => void>();
135
+
136
+ if (typeof window !== "undefined") {
137
+ window.addEventListener(AUTOMATION_OPEN_EVENT, (event) => {
138
+ const detail = (event as CustomEvent<AutomationOpenDetail>).detail;
139
+ if (!detail?.autoId) return;
140
+ if (openSubscribers.size) {
141
+ for (const notify of openSubscribers) notify(detail);
142
+ return;
143
+ }
144
+ pendingOpen = detail;
145
+ });
146
+ }
147
+
148
+ /*
149
+ * ⛔ THE `AUTOMATION_CREATE_EVENT` LISTENER STOOD HERE AND IS DELETED WITH ITS SIGNALLER
150
+ * (wave 25 item 5a, ruling R8) — latch, subscriber set and all.
151
+ *
152
+ * Wave 24 built it to close the opposite defect: the event was declared, signalled from
153
+ * `Shell.tsx`, and consumed NOWHERE, so all three "Automated database" doors navigated to
154
+ * `#/automation` and then did nothing, in production, with every gate green. R8 now deletes those
155
+ * three doors — creating a database and pointing an automation at it are two acts — and they were
156
+ * the event's only signaller.
157
+ *
158
+ * ⛔ SO THIS SIDE GOES TOO, IN THE SAME CHANGE. A listener with no signaller is the same defect
159
+ * read from the other end: it compiles, it costs nothing at runtime, and it reads to the next
160
+ * person as a live channel. The rule that catches both is now DERIVED rather than remembered —
161
+ * `verify_automation_ui.py` enumerates every `AUTOMATION_*_EVENT` in `apiContract.ts` and demands
162
+ * a signal site AND a listener site for each.
163
+ *
164
+ * ⚠ NOTHING ABOUT CREATING AN AUTOMATION IS LOST. `createAndOpen` is untouched and still has two
165
+ * doors, both on this surface: the rail's button and the empty state's. What is gone is the claim
166
+ * that a database can be created BY asking for an automation.
167
+ */
168
+
169
+ /**
170
+ * A default name for a brand-new automation, and it has to be one the server ACCEPTS.
171
+ *
172
+ * MEASURED against the live route rather than assumed (`scratchpad/probe_names.py`): a create
173
+ * with no name is a 400 (`name the automation`), and duplicate names are ALLOWED. So the name
174
+ * cannot be omitted, and a fixed literal would stack "New automation" three deep in a rail that
175
+ * sorts by name with nothing to tell the rows apart. Numbering from the existing list is a
176
+ * client-side convenience over a server that does not care: if two tabs race, the loser gets a
177
+ * duplicate name, which is legal, visible and renameable — never an error the user has to read.
178
+ */
179
+ function nextAutomationName(existing: Automation[]): string {
180
+ const taken = new Set(existing.map((a) => (a.name || "").trim().toLowerCase()));
181
+ for (let n = existing.length + 1; ; n += 1) {
182
+ const candidate = `Automation ${n}`;
183
+ if (!taken.has(candidate.toLowerCase())) return candidate;
184
+ }
185
+ }
186
+
187
+ export default function AutomationSurface() {
188
+ const [data, setData] = useState<AutomationList | null>(null);
189
+ const [error, setError] = useState("");
190
+ const [activeId, setActiveId] = useState<string>("");
191
+ const [railShut, setRailShut] = useState(false);
192
+ const [busy, setBusy] = useState("");
193
+ /**
194
+ * ⭐ WAVE 24 / C-CREATE — a create is now a ROUND TRIP, not a local wizard, so the door has to
195
+ * say it is busy. Without this a second click while the POST is in flight makes a second
196
+ * automation, and the server allows duplicate names, so the user gets two rows that look
197
+ * identical and no error to explain either of them.
198
+ */
199
+ const [creatingNow, setCreatingNow] = useState(false);
200
+ /** An open request waiting for the list to arrive (see the resolver below). */
201
+ const [openRequest, setOpenRequest] = useState<AutomationOpenDetail | null>(null);
202
+ /* ⛔ `createRequest` LEFT WITH THE EVENT THAT SET IT (wave 25, R8) — see the tombstone above. */
203
+ const listRef = useRef<HTMLDivElement | null>(null);
204
+
205
+ /**
206
+ * ⛔ C14 LEG 2 — THE ID THE USER LAST ASKED FOR, and it is a ref because it has to be
207
+ * written SYNCHRONOUSLY, inside the click, before any promise that was already in flight
208
+ * can resolve. State would not do: a resolution racing React's commit would read the
209
+ * previous value, which is the exact window this guard exists to close.
210
+ */
211
+ const wantedId = useRef("");
212
+
213
+ /**
214
+ * ⭐ THE ONLY PLACE THE RAIL'S SELECTION MOVES ON PURPOSE. Every deliberate change of
215
+ * automation — a rail click, a create, a delete — goes through here, so "what the user
216
+ * asked for" and "what is on screen" are written in one statement and cannot drift apart.
217
+ * A resolution that wants to steer the rail compares itself against `wantedId` instead.
218
+ */
219
  const select = useCallback((id: string, _stage = "") => {
220
+ wantedId.current = id;
221
+ setActiveId(id);
222
+ }, []);
223
+
224
+ /**
225
+ * ⛔ C14 LEG 3 (the paint half) — A STALE READ MAY NOT REPAINT.
226
+ *
227
+ * Every call takes the next generation; only the newest one is allowed to `setData`. The
228
+ * defect this closes is not cosmetic: a poll issued BEFORE a delete resolves AFTER it, and
229
+ * a list that still contains the deleted automation puts a row back in the rail that the
230
+ * user has just watched disappear. Same for a toggle, and same for a create.
231
+ *
232
+ * ⚠ THE RETURN VALUE IS NOT GUARDED, deliberately. The caller asked a question and gets
233
+ * its own answer — coupling the two would mean a create whose `load()` was overtaken by a
234
+ * poll silently failed to select the automation it had just made.
235
+ */
236
+ const gen = useRef(0);
237
+
238
+ const load = useCallback(async (abort?: AbortSignal) => {
239
+ const mine = ++gen.current;
240
+ try {
241
+ const next = await listAutomations(abort);
242
+ if (mine === gen.current) {
243
+ setData(next);
244
+ setError("");
245
+ }
246
+ return next;
247
+ } catch (e) {
248
+ if ((e as Error)?.name === "AbortError") return null;
249
+ if (mine === gen.current)
250
+ setError(
251
+ e instanceof AutomationError
252
+ ? e.message
253
+ : "The automation service did not answer."
254
+ );
255
+ return null;
256
+ }
257
+ }, []);
258
+
259
+ useEffect(() => {
260
+ const ac = new AbortController();
261
+ void load(ac.signal);
262
+ return () => ac.abort();
263
+ }, [load]);
264
+
265
+ /**
266
+ * ⭐⭐ WAVE 24 / C-CREATE (a) — CREATE AND OPEN. This REPLACED the three-step wizard.
267
+ *
268
+ * R6 is what makes it possible: `plain` is a real kind with no machine graph nodes, and it is
269
+ * what `POST /automations` stores when the body names none. So "new automation" stopped being
270
+ * a question ("which of three kinds?" — two of which can no longer be created at all) and
271
+ * became what it says: a new automation, open, on its trigger picker. Choosing a SOURCE is now
272
+ * picking the `ig_profile_match` trigger, which is where that choice belongs.
273
+ *
274
+ * ⛔ THE GUARD IS A REF, not the `creatingNow` state, and this file already carries the scar
275
+ * that explains why (C14 leg 2): a state read inside a click closure is the value from the
276
+ * last render, so two fast clicks both see `false` and both POST. Duplicate names are legal
277
+ * server-side, so the user would get two identical rows and no error. The ref is written
278
+ * synchronously, inside the click, before anything can await.
279
+ */
280
+ const creatingRef = useRef(false);
281
+ const createAndOpen = useCallback(async () => {
282
+ if (creatingRef.current) return;
283
+ creatingRef.current = true;
284
+ setCreatingNow(true);
285
+ try {
286
+ const res = await createAutomation({
287
+ name: nextAutomationName(data?.automations || []),
288
+ });
289
+ const id = res?.automation?.id || "";
290
+ // The create flow is the one caller allowed to name a different id (C14 leg 2) — the
291
+ // automation did not exist when the click happened, so there is nothing to race with.
292
+ const next = await load();
293
+ if (id && next) select(id);
294
+ } catch (e) {
295
+ setError(
296
+ e instanceof AutomationError ? e.message : "The automation was not created."
297
+ );
298
+ } finally {
299
+ creatingRef.current = false;
300
+ setCreatingNow(false);
301
+ }
302
+ }, [data, load, select]);
303
+
304
+ // A run is a background thread on the server, so the surface has to ASK whether it
305
+ // finished. Polling only while something is actually running keeps an idle surface
306
+ // silent — a fixed interval would be a request every few seconds forever, for a page
307
+ // whose contents change a handful of times a day.
308
+ //
309
+ // ⛔ C14 LEG 3 (the abort half). It used to call `load()` bare — no signal, nothing to
310
+ // cancel — so a request the interval had already issued kept going after the effect that
311
+ // owned it was gone. One controller per effect run, aborted with the interval, means a poll
312
+ // cannot outlive the condition that justified it. The generation guard inside `load` covers
313
+ // the rest: a response that survives the abort still cannot repaint over a newer one.
314
+ //
315
+ // ⭐ WAVE 24 (item 6, C-PERF) — THE INTERVAL BECAME A BACKOFF, and the measurement is why it is
316
+ // a SMALL change rather than the big one the contract's hypothesis 2 asked for.
317
+ //
318
+ // MEASURED (`scratchpad/perf_automations.py`, mailbox D-5): this poll costs ~23 ms and 24 KB
319
+ // for a ten-automation tenant, and the engine work the hypothesis blamed — a `graph()` rebuild
320
+ // per automation — is **41 µs each**, i.e. under 4% of the request. The payload was never the
321
+ // problem, so nothing here gets a cheaper endpoint.
322
+ //
323
+ // What the numbers DO indict is the aggregate: a discovery run legitimately blocks up to
324
+ // `BD_FILTER_WAIT` = 120 s, and two independent 2.5 s polls across this file and the detail
325
+ // spend ~96 requests and ~2.3 MB over that window — in the same single process the run itself
326
+ // is a thread in. So the delay grows 2.5 → 5 → 8 s and stops there.
327
+ //
328
+ // ⚠ NOT "to learn nothing", and the distinction is the whole justification. That WAS true when
329
+ // this file rendered no live step: the poll's only observable effect was a dot that had already
330
+ // been pulsing for two minutes. It stopped being true in this same change — `liveStepOf` now
331
+ // paints `status.step`, so a poll carries the one fact worth having. The backoff is therefore
332
+ // NOT "stop asking a pointless question"; it is FEWER ROUND TRIPS FOR THE SAME INFORMATION, on
333
+ // a step text that changes every few seconds at most, not every 2.5.
334
+ // The cap is deliberately low: it bounds how long a FINISHED run can still look live, which is
335
+ // the only thing a backoff can make worse, and 8 s of that is worth ~⅔ fewer requests.
336
+ //
337
+ // ⛔ THE KEY IS THE RUNNING SET, NOT A BOOLEAN, and that is what makes the reset correct: the
338
+ // effect re-runs — and the delay drops back to 2.5 s — the moment a run starts or finishes, so
339
+ // a user who clicks Run now gets the fast cadence again instead of inheriting the tail of the
340
+ // previous run's backoff. A bare `anyRunning` boolean cannot see the second run start.
341
+ // A step text changing does NOT re-key it, so the interval never thrashes.
342
+ //
343
+ // C14 LEG 3 (the abort half) is unchanged and still load-bearing: one controller per effect
344
+ // run, aborted with the timer, so a poll cannot outlive the condition that justified it. The
345
+ // generation guard inside `load` covers the rest.
346
+ const runningKey = (data?.automations || [])
347
+ .filter((a) => a.running)
348
+ .map((a) => a.id)
349
+ .sort()
350
+ .join(",");
351
+ useEffect(() => {
352
+ if (!runningKey) return undefined;
353
+ const ac = new AbortController();
354
+ let delay = POLL_MIN_MS;
355
+ let timer = 0;
356
+ const tick = () => {
357
+ void load(ac.signal);
358
+ delay = Math.min(delay * 2, POLL_MAX_MS);
359
+ timer = window.setTimeout(tick, delay);
360
+ };
361
+ timer = window.setTimeout(tick, delay);
362
+ return () => {
363
+ window.clearTimeout(timer);
364
+ ac.abort();
365
+ };
366
+ }, [runningKey, load]);
367
+
368
+ // ── W23-W5, the surface's half: hear the request, then answer it when we CAN ──────────
369
+ useEffect(() => {
370
+ const notify = (detail: AutomationOpenDetail) => setOpenRequest(detail);
371
+ openSubscribers.add(notify);
372
+ if (pendingOpen) {
373
+ const latched = pendingOpen;
374
+ pendingOpen = null;
375
+ setOpenRequest(latched);
376
+ }
377
+ return () => {
378
+ openSubscribers.delete(notify);
379
+ };
380
+ }, []);
381
+
382
+ /* ⛔ THE C-CREATE(b) SUBSCRIBE EFFECT AND ITS RESOLVER STOOD HERE (wave 25, R8). They heard the
383
+ create event and, once the list had arrived, called `createAndOpen` — the "wait for the list
384
+ or every automation is named Automation 1" note lives on in `nextAutomationName`, which is
385
+ still numbered off `existing`. With no signaller there is nothing to hear. */
386
+
387
+ /**
388
+ * ⚠ THE REQUEST OUTLIVES THE FETCH, and it has to. The reader clicks a notification from
389
+ * another page, so this surface is mounting WITH AN EMPTY LIST — "select it if it is in the
390
+ * list" would drop every real click and keep only the one case where the reader was already
391
+ * here. So the request is held until `data` exists, and only then answered.
392
+ *
393
+ * An automation the reader can no longer open does NOTHING (the contract's own words): the
394
+ * request is cleared either way, so a stale id cannot sit here re-firing against every
395
+ * subsequent list.
396
+ */
397
+ useEffect(() => {
398
+ if (!openRequest || !data) return;
399
+ const found = (data.automations || []).some((a) => a.id === openRequest.autoId);
400
+ if (found) {
401
+ select(openRequest.autoId, openRequest.stageId || "");
402
+ }
403
+ setOpenRequest(null);
404
+ }, [openRequest, data, select]);
405
+
406
+ const items = data?.automations || [];
407
+ const active = items.find((a) => a.id === activeId) || null;
408
+
409
+ const toggleEnabled = async (a: Automation) => {
410
+ setBusy(a.id);
411
+ try {
412
+ await patchAutomation(a.id, {
413
+ schedule: { cron: a.schedule.cron, enabled: !a.schedule.enabled },
414
+ });
415
+ await load();
416
+ } catch (e) {
417
+ setError(e instanceof AutomationError ? e.message : "That change was not saved.");
418
+ } finally {
419
+ setBusy("");
420
+ }
421
+ };
422
+
423
+ const onRailKey = (event: ReactKeyboardEvent<HTMLDivElement>) => {
424
+ if (event.key !== "ArrowDown" && event.key !== "ArrowUp") return;
425
+ const rows = Array.from(
426
+ listRef.current?.querySelectorAll<HTMLButtonElement>(".auto-row-main") || []
427
+ );
428
+ if (!rows.length) return;
429
+ event.preventDefault();
430
+ const at = rows.indexOf(document.activeElement as HTMLButtonElement);
431
+ const next = event.key === "ArrowDown" ? (at + 1) % rows.length
432
+ : (at - 1 + rows.length) % rows.length;
433
+ rows[next < 0 ? 0 : next]?.focus();
434
+ };
435
+
436
+ return (
437
+ <div className="auto-surface">
438
+ <aside
439
+ className={"auto-rail" + (railShut ? " is-collapsed" : "")}
440
+ aria-label="Automations"
441
+ >
442
+ {/* The same three-bars fold control the Views rail carries — the two rails are
443
+ one idea, so they must not fold with two different affordances. */}
444
+ <div className="auto-rail-top">
445
+ <button
446
+ type="button"
447
+ className="cg-rail-toggle"
448
+ aria-label={railShut ? "Expand automations" : "Minimize automations"}
449
+ aria-expanded={!railShut}
450
+ title={railShut ? "Expand automations" : "Minimize automations"}
451
+ onClick={() => setRailShut((v) => !v)}
452
+ >
453
+ <svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
454
+ <path
455
+ d="M2.5 4.4h11M2.5 8h11M2.5 11.6h11"
456
+ stroke="currentColor"
457
+ strokeWidth="1.35"
458
+ strokeLinecap="round"
459
+ />
460
+ </svg>
461
+ </button>
462
+ </div>
463
+
464
+ <div className="auto-create">
465
+ <button
466
+ type="button"
467
+ className="auto-create-btn"
468
+ disabled={creatingNow}
469
+ onClick={() => void createAndOpen()}
470
+ >
471
+ <svg width="14" height="14" viewBox="0 0 16 16" fill="none" aria-hidden="true">
472
+ <path
473
+ d="M8 3.4v9.2M3.4 8h9.2"
474
+ stroke="currentColor"
475
+ strokeWidth="1.5"
476
+ strokeLinecap="round"
477
+ />
478
+ </svg>
479
+ New automation
480
+ </button>
481
+ </div>
482
+
483
+ <div className="auto-list" ref={listRef} onKeyDown={onRailKey}>
484
+ {items.map((a) => {
485
+ return (
486
+ <div
487
+ key={a.id}
488
+ className={"auto-row" + (a.id === activeId ? " is-active" : "")}
489
+ >
490
+ <button
491
+ type="button"
492
+ className="auto-row-main"
493
+ onClick={() => select(a.id)}
494
+ >
495
+ {/* ⭐ ITEM 18 / R14 — THE LOOP MARK, WHERE THE STATUS DOT WAS. Same mark on
496
+ every row, no status colour: the owner's point is that an automation IS a
497
+ loop, not that this one is green. The `.auto-loopmark` wrapper is what sizes
498
+ it — `.lp-mark` is shell-owned CSS and outside this session's region, so the
499
+ box is mine and the mark is theirs. */}
500
+ <span className="auto-loopmark">
501
+ <Mark size={15} />
502
+ </span>
503
+ <span className="auto-row-text">
504
+ <span className="auto-row-name">{a.name}</span>
505
+ <span className="auto-row-desc">{railSubtitle(a)}</span>
506
+ </span>
507
+ </button>
508
+ <button
509
+ type="button"
510
+ className={"auto-row-switch" + (a.schedule?.enabled ? " is-on" : "")}
511
+ disabled={busy === a.id}
512
+ aria-pressed={!!a.schedule?.enabled}
513
+ title={
514
+ a.schedule?.enabled
515
+ ? `Scheduled: ${a.schedule.cron}. Click to pause.`
516
+ : "Not scheduled. Click to enable."
517
+ }
518
+ onClick={() => void toggleEnabled(a)}
519
+ >
520
+ <span className="auto-row-switch-knob" />
521
+ </button>
522
+ </div>
523
+ );
524
+ })}
525
+ {/* ONE LINE (R13). It used to describe the kinds — "one reads a public web page…
526
+ the other fills an automation column…" — which was two sentences, wrong by the
527
+ time a third kind shipped, and printed inside a `--lp-rail-w` column. What the
528
+ kinds are belongs to the create form, which is one click away and lists all of
529
+ them from the server. */}
530
+ {!items.length && !error ? (
531
+ <p className="auto-rail-empty">No automations yet.</p>
532
+ ) : null}
533
+ </div>
534
+ </aside>
535
+
536
+ <section className="auto-main">
537
+ {error ? (
538
+ <div className="auto-banner is-error" role="alert">
539
+ {error}
540
+ </div>
541
+ ) : null}
542
+ {data && !data.storeAvailable ? (
543
+ <div className="auto-banner is-warn" role="status">
544
+ The tenant store is unavailable, so nothing can be saved right now.
545
+ </div>
546
+ ) : null}
547
+
548
+ {/*
549
+ ⛔ THE `creating` BRANCH IS GONE (wave 24, C-CREATE a / owner items 3 + 4).
550
+ `AutomationCreate` used to render here — a three-step wizard whose first step asked
551
+ which DATABASE to write into (an existing one, a new one, or one the automation itself
552
+ would create) and whose second asked which of three KINDS.
553
+ R6 retires that question at the root: two of the three kinds can no longer be created
554
+ at all, and the third is now reached by picking a trigger. A new automation is created
555
+ the moment it is asked for (`createAndOpen`) and opens on its own Builder, so there is
556
+ no intermediate face left to render and no `creating` state to hold.
557
+ ⭐ WAVE 25 (R8) retires the LAST of that vocabulary: the third answer was the
558
+ "automated database", and it is gone from every door in the product. This button is now
559
+ one of exactly two ways to make an automation, and both are on this surface.
560
+ The import went with it — it is what would break `npx tsc -b` for the WHOLE client the
561
+ moment session B deletes the file, which is why this deletion is sequenced first.
562
+ */}
563
+ {active ? (
564
+ <AutomationDetail
565
+ key={active.id}
566
+ automation={active}
567
+ /* ⛔ `kinds={data?.kinds || []}` LEFT HERE WITH THE PROP IT FED (wave 25, D-57) —
568
+ both halves in one change, because either alone is a `tsc` error. */
569
+ cronPresets={data?.cronPresets || []}
570
+ paidReady={!!data?.paidReady}
571
+ discover={data?.discover}
572
+ // The trigger vocabulary (C3). Forwarded as-is — absent stays absent, so the
573
+ // trigger face can tell "the server offered nothing" from "the server offered
574
+ // an empty list" rather than collapsing both into a picker with no options.
575
+ triggers={data?.triggers}
576
+ // C4's action menu + the builder's ceilings. Forwarded as-is for the same reason
577
+ // `triggers` is: absent must stay absent, so the builder can tell "the server
578
+ // offered nothing" from "the server offered an empty list".
579
+ catalog={data?.actionsCatalog}
580
+ vocab={data?.flow}
581
+ // `?? null` and never `|| {enabled:false}`: an absent tick bit is "the server did
582
+ // not say", which Step 1 prints as its own sentence. Defaulting it here would
583
+ // turn a missing field into a claim about production (C6 amendment #1).
584
+ tick={data?.tick ?? null}
585
+ /*
586
+ * ⭐ W24-W1 (item 6) — WHAT THIS RUN IS DOING, on the one surface that survives in
587
+ * BOTH views. REQUIRED on the far side deliberately: an optional prop that nobody
588
+ * passes degrades to "the feature does not exist", which is indistinguishable from
589
+ * "it was never built" — and this whole item exists because a live step text rode
590
+ * the wire for three waves with nothing rendering it.
591
+ * `liveStepOf` returns the step ONLY while the automation is running; a stale step
592
+ * from a finished run is a worse answer than none.
593
+ */
594
+ liveStep={liveStepOf(active)}
595
+ /*
596
+ * ⛔ C14 LEG 2 — THE STALE-ID WRITE-BACK, and this is the ghost's second cause.
597
+ *
598
+ * The detail calls `onSaved(automation.id)` from six places (save, the trigger
599
+ * picker, a schedule change, a node switch, a card move, a run). Each closure
600
+ * captures the automation it was mounted for, so a save that resolves AFTER the
601
+ * user has clicked a different row used to call `setActiveId(the OLD id)` — the
602
+ * rail jumped back, the detail remounted, and what the user saw was the previous
603
+ * automation reappearing over the one they had just opened. It looked like a
604
+ * rendering bug; it was a resolution steering the selection.
605
+ *
606
+ * A resolution may no longer steer anything. It reloads the list — that part was
607
+ * always right — and it re-asserts the selection ONLY when its id is still the one
608
+ * the user asked for, which makes the write a no-op in the good case and nothing
609
+ * at all in the bad one.
610
+ */
611
+ onSaved={async (id) => {
612
+ const next = await load();
613
+ if (id && next && id === wantedId.current) select(id);
614
+ }}
615
+ onDeleted={async () => {
616
+ select("");
617
+ await load();
618
+ }}
619
+ />
620
+ ) : (
621
+ /*
622
+ * THE EMPTY STATE IS ONE LINE AND A BUTTON (owner ruling R13 — "never
623
+ * over-explain", now a DESIGN.md law).
624
+ *
625
+ * It was a heading, a three-sentence paragraph and a 130-word bulleted list
626
+ * describing all three kinds. Every word of it was true and none of it was READ:
627
+ * an empty state is passed through, not studied, and the person looking at it has
628
+ * already decided to make an automation. The kinds are described where the choice
629
+ * is actually made — the create form lists them FROM THE SERVER, so that copy also
630
+ * cannot go stale the way this list had (it described two kinds after a third
631
+ * shipped).
632
+ *
633
+ * The button is here rather than only in the rail because this pane is where the
634
+ * eye is; a create affordance the user has to go find is the same defect as the
635
+ * paragraph, spent differently.
636
+ */
637
+ <div className="autob-empty">
638
+ {/*
639
+ ⛔ NO TITLE HERE ANY MORE (C13, owner item 2). This pane carried an `h1`
640
+ reading "Automations" at 20px/600 — a THIRD title treatment on a page that
641
+ also had the header's editable 16px/700 input, against every database page's
642
+ single 16px/650 `shell-db-name`. The shell now wraps this branch in the same
643
+ `shell-db-frame` + `DbHead` a database gets (wiring W23-W1), so the page's name
644
+ is drawn once, by the one component that draws every other page's name. A
645
+ stand-in restyled to match would have been a second copy of the same fact,
646
+ free to drift the day the header moves.
647
+ */}
648
+ <p className="autob-empty-line">
649
+ A job this workspace runs for you, on demand or on a schedule.
650
+ </p>
651
+ <button
652
+ type="button"
653
+ className="auto-btn is-primary"
654
+ disabled={creatingNow}
655
+ onClick={() => void createAndOpen()}
656
+ >
657
+ New automation
658
+ </button>
659
+ </div>
660
+ )}
661
+ </section>
662
+ </div>
663
+ );
664
+ }
web/src/automation/automationApi.ts CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/automation/steps.ts CHANGED
@@ -1,422 +1,422 @@
1
- // ---------------------------------------------------------------------------
2
- // automation/steps.ts — the arithmetic behind the numbered Steps (owner ruling R9).
3
- //
4
- // Pure functions, deliberately separate from the component: both of the things in
5
- // here are the kind that go subtly wrong (a step number, a cron round-trip) and
6
- // stay wrong invisibly, so they are written where they can be read on their own.
7
- //
8
- // ⚠ NOTHING HERE INVENTS A STEP. The node list, its order and its `col` all come
9
- // from the server's `engine.graph()` — the same module that RUNS the steps — and
10
- // this file only counts them (contract C6). A client-side second idea of what the
11
- // steps are is the exact drift the split was built to prevent; the rationale is
12
- // written at AutomationCreate.tsx:1-13.
13
- // ---------------------------------------------------------------------------
14
- import type { Action, ActionCatalogRow, Branch, Cond, GraphNode, TriggerOption }
15
- from "./automationApi";
16
-
17
- /**
18
- * The branches of an If / then, TOLERATING the pre-wave-24 `{cond, actions}` shape on READ.
19
- *
20
- * ⚠ THIS MIRRORS `automation_engine.group_branches` AND THE DUPLICATION IS DELIBERATE, which is
21
- * worth defending because this file's own header forbids a second idea of the server's data. It
22
- * is a READ-SIDE TOLERANCE, not a second validator: it never decides what is legal, and every
23
- * write still goes to `clean_actions`, which returns the one canonical shape. The alternative is
24
- * strictly worse — a group stored in the old shape would render with no branches and no children,
25
- * so a live automation's actions would simply VANISH from the screen while running perfectly.
26
- * A migration the server performs on write and the client tolerates on read is the pair that has
27
- * no window; trusting `branches` alone has one, for exactly as long as a definition sits unsaved.
28
- */
29
- export function groupBranches(a: Action): Branch[] {
30
- const cfg = (a?.config || {}) as {
31
- branches?: Branch[];
32
- cond?: Cond | null;
33
- actions?: Action[];
34
- };
35
- if (Array.isArray(cfg.branches)) return cfg.branches;
36
- // The pre-wave-24 shape, read as the one branch the engine migrates it to.
37
- if (Array.isArray(cfg.actions))
38
- return [{ id: "b1", label: "A", cond: cfg.cond ?? null, actions: cfg.actions }];
39
- return [];
40
- }
41
-
42
- /**
43
- * R8's STEP NUMBERS for a whole flow, keyed by action id.
44
- *
45
- * THE RULING, and each clause is a line here:
46
- * · the TRIGGER is unnumbered — it is not in this map at all, and the card carries the word
47
- * "Trigger" instead. (This REVERSES wave 21's R9, "Step 1 is always the Trigger".)
48
- * · the first action is 1, the second 2, …
49
- * · an If / then occupies ONE number. Its branches are alternatives, not later steps, so they
50
- * consume no numbers of their own.
51
- * · a branch's children are numbered RELATIVE TO THEIR BRANCH — the fork at step 2 gives every
52
- * one of its branches a `2.1`, a `2.2`, and so on. Two branches therefore both contain a
53
- * `2.1`, which is correct rather than colliding: they are alternatives in lettered lanes and
54
- * only one of them ever runs (the engine takes the first matching branch and breaks).
55
- *
56
- * ⛔ IT DOES NOT LETTER THE BRANCHES, and that is a change from the contract text. `clean_actions`
57
- * assigns the letter server-side (`_branch_letter`, and "Otherwise" for the null-cond last leg),
58
- * preserving any label the client sends — so the letter rides on `Branch.label` and a second
59
- * lettering here would be exactly the client-copy-of-a-server-vocabulary this wave keeps deleting.
60
- * The client sends an EMPTY label and renders what comes back, which also means letters re-flow
61
- * correctly when a branch is deleted instead of going stale.
62
- */
63
- export function numberActions(actions: Action[]): Map<string, string> {
64
- const out = new Map<string, string>();
65
- const walk = (list: Action[], prefix: string) => {
66
- (list || []).forEach((a, i) => {
67
- const n = prefix ? `${prefix}.${i + 1}` : String(i + 1);
68
- out.set(a.id, n);
69
- if (a.kind === "group")
70
- for (const br of groupBranches(a)) walk(br.actions || [], n);
71
- });
72
- };
73
- walk(actions || [], "");
74
- return out;
75
- }
76
-
77
- export interface Step {
78
- node: GraphNode;
79
- /** The number the card carries. */
80
- n: number;
81
- /** True when this node shares its position with the one before it — an ALTERNATIVE, not a next. */
82
- alt: boolean;
83
- }
84
-
85
- /**
86
- * Number the server's nodes for display.
87
- *
88
- * ⭐ THE NUMBER IS `col + 1`, NOT the array index, and that is the honest one. The
89
- * `field_instagram` graph forks: `capture_paid` and `capture_anon` both sit at
90
- * `col: 3` because they are the SAME position in the flow reached two ways (the
91
- * paid rung answers, or the anonymous ladder does). Numbering them 4 and 5 would
92
- * state a sequence that never happens. They share a number and the second is
93
- * marked `alt`, so the list says "either of these" rather than "then".
94
- *
95
- * "Step 1 is always the Trigger" (R9) therefore falls out of the payload — the
96
- * trigger is the node at `col: 0` — instead of being asserted by this client. If
97
- * the engine ever emits something else first, the UI shows what the engine does.
98
- */
99
- export function numberSteps(nodes: GraphNode[]): Step[] {
100
- return (nodes || []).map((node, i) => {
101
- const col = typeof node.col === "number" ? node.col : i;
102
- const before = i > 0 ? nodes[i - 1] : null;
103
- const beforeCol = before && typeof before.col === "number" ? before.col : -1;
104
- return { node, n: col + 1, alt: i > 0 && beforeCol === col };
105
- });
106
- }
107
-
108
- /**
109
- * Move `dragId` into `dropId`'s position within one list (owner item 11).
110
- *
111
- * ⛔ IT RETURNS `null` RATHER THAN THE LIST UNCHANGED, and the distinction is what keeps a
112
- * pointless PATCH off the wire: "these two are the same card", "one of them is not in this list"
113
- * and "here is your new order" are three different answers, and collapsing the first two into
114
- * "the order you already had" would have the caller write the flow back to the server on every
115
- * aborted drag. The caller writes only when this says something happened.
116
- *
117
- * THE CARD LANDS EXACTLY WHERE THE TARGET WAS, in both directions — splice out, then splice in at
118
- * the target's ORIGINAL index. Dragging down, the target shifts up; dragging up, it shifts down.
119
- * (The tempting version — insert at the target's index *after* the removal — is off by one when
120
- * dragging downwards and drops the card one slot short of where the pointer is, which reads as
121
- * the drag not having worked.)
122
- */
123
- export function reorderList<T extends { id: string }>(
124
- list: T[],
125
- dragId: string,
126
- dropId: string
127
- ): T[] | null {
128
- if (!dragId || !dropId || dragId === dropId) return null;
129
- const src = list || [];
130
- const from = src.findIndex((x) => x.id === dragId);
131
- const to = src.findIndex((x) => x.id === dropId);
132
- if (from < 0 || to < 0) return null;
133
- const next = [...src];
134
- const [moved] = next.splice(from, 1);
135
- if (!moved) return null;
136
- next.splice(to, 0, moved);
137
- return next;
138
- }
139
-
140
- /** One Properties section under "How this fetches": a panel, and every node that opens it. */
141
- export interface PanelGroup {
142
- /** The panel key — what the caller passes to `renderNodeBody`. */
143
- panel: string;
144
- /** Every machine node whose `panel` is this one, in the server's own order. */
145
- nodes: GraphNode[];
146
- }
147
-
148
- /**
149
- * The machine steps, GROUPED BY PANEL (item 5, contract C-CFG).
150
- *
151
- * ⛔ GROUPED, NEVER MAPPED ONE-TO-ONE, and the difference is a defect rather than a nicety.
152
- * `panel` is MANY-TO-ONE over nodes, which is easy to miss because the old surface hid it: you
153
- * clicked ONE card and got ONE panel. `field_instagram` gives `capture`, `capture_paid`,
154
- * `capture_anon` AND `capture_metrics` all `panel: "capture"` (`automation_engine.py:3334-3352`),
155
- * and `discover_instagram` gives both of its nodes `panel: "find"` (`:3381`/`:3387`). So a body
156
- * rendered per NODE would print the capture prose four times over, and — the one that actually
157
- * loses work — mount the 21-toggle discovery filter TWICE against a single `preds` array, two
158
- * editors writing one piece of state where whichever blurred last silently wins.
159
- *
160
- * The detail panel has always joined on `panel` and never on node id (`AutomationDetail`'s own
161
- * note says so); this keeps that law now that the panels are no longer reached by clicking a card.
162
- *
163
- * ORDER IS THE SERVER'S — first appearance wins, so the sections read in flow order rather than
164
- * in whatever order a Map or a sort would produce.
165
- */
166
- export function groupByPanel(nodes: GraphNode[]): PanelGroup[] {
167
- const order: string[] = [];
168
- const byPanel = new Map<string, GraphNode[]>();
169
- for (const n of nodes || []) {
170
- // `panel` falls back to the id exactly as `graph()`'s own `node()` does (`panel or nid`),
171
- // so a node the engine ships without one still gets its own section instead of joining
172
- // every other panel-less node under the empty string.
173
- const key = n.panel || n.id;
174
- const seen = byPanel.get(key);
175
- if (seen) seen.push(n);
176
- else {
177
- byPanel.set(key, [n]);
178
- order.push(key);
179
- }
180
- }
181
- return order.map((panel) => ({ panel, nodes: byPanel.get(panel) || [] }));
182
- }
183
-
184
- /** How the Step-1 card is currently set: what it repeats on, and at what time. */
185
- export interface TriggerShape {
186
- /** `day` = every day · `0`..`6` = that weekday (cron numbering, 0 = Sunday) · `custom` = a cron only. */
187
- every: string;
188
- /** `HH:MM`, empty when the cron does not express a single fire time. */
189
- time: string;
190
- }
191
-
192
- const DAILY = /^(\d{1,2}) (\d{1,2}) \* \* \*$/;
193
- const WEEKLY = /^(\d{1,2}) (\d{1,2}) \* \* ([0-7])$/;
194
-
195
- function two(n: number): string {
196
- return String(n).padStart(2, "0");
197
- }
198
-
199
- /**
200
- * A stored cron → the controls that can express it.
201
- *
202
- * ⛔ ANYTHING THIS CANNOT EXPRESS COMES BACK AS `custom`, NEVER AS THE FIRST OPTION.
203
- * A `<select>` whose value matches no `<option>` renders the first one, and the next
204
- * Save then writes a schedule nobody chose — the defect this codebase has already
205
- * paid for twice ([[cg-condition-builder-items]]; the two `(not in this database)`
206
- * options in AutomationDetail exist for the same reason). A quarter-hourly weekday
207
- * cron is a legal schedule the engine runs happily; the editor's job is to leave it
208
- * alone, not to quietly turn it into "every day at 08:00".
209
- *
210
- * (Written without the literal cron string on purpose — a step-slash inside a block
211
- * comment ENDS the comment, and the compiler's 20 cascading errors say nothing about
212
- * the cron.)
213
- */
214
- export function readCron(cron: string): TriggerShape {
215
- const raw = String(cron || "").trim();
216
- const daily = DAILY.exec(raw);
217
- if (daily) {
218
- const [, m, h] = daily;
219
- if (Number(h) <= 23 && Number(m) <= 59) return { every: "day", time: `${two(Number(h))}:${two(Number(m))}` };
220
- }
221
- const weekly = WEEKLY.exec(raw);
222
- if (weekly) {
223
- const [, m, h, d] = weekly;
224
- if (Number(h) <= 23 && Number(m) <= 59) {
225
- return { every: String(Number(d) % 7), time: `${two(Number(h))}:${two(Number(m))}` };
226
- }
227
- }
228
- return { every: "custom", time: "" };
229
- }
230
-
231
- /** The controls → a cron string. `custom` keeps whatever the user typed, untouched. */
232
- export function writeCron(every: string, time: string, custom: string): string {
233
- if (every === "custom") return String(custom || "").trim();
234
- const [h, m] = String(time || "06:00").split(":");
235
- const hh = Math.min(23, Math.max(0, Number(h) || 0));
236
- const mm = Math.min(59, Math.max(0, Number(m) || 0));
237
- return every === "day" ? `${mm} ${hh} * * *` : `${mm} ${hh} * * ${Number(every) || 0}`;
238
- }
239
-
240
- /** The weekday options, in the order a week is read rather than in cron's 0-first order. */
241
- export const WEEKDAYS: { value: string; label: string }[] = [
242
- { value: "1", label: "Every Monday" },
243
- { value: "2", label: "Every Tuesday" },
244
- { value: "3", label: "Every Wednesday" },
245
- { value: "4", label: "Every Thursday" },
246
- { value: "5", label: "Every Friday" },
247
- { value: "6", label: "Every Saturday" },
248
- { value: "0", label: "Every Sunday" },
249
- ];
250
-
251
- // ── WAVE 25 · C2: THE GROUPED TRIGGER PICKER'S ARITHMETIC ───────────────────────────────────
252
- //
253
- // ⛔ WHY THIS IS HERE AND NOT IN THE COMPONENT. Both functions below are the kind that go subtly
254
- // wrong and stay wrong INVISIBLY — a group silently sorted first, a stored trigger silently
255
- // rendering as nothing — and this file exists precisely so that class can be executed under node
256
- // by `verify_steps.py` instead of eyeballed in a screenshot. It is also D-58's minimum: the
257
- // builder's largest surface has had no gate but `tsc` and eyes, and the picker's grouped and
258
- // SELECTED states are the two things a screenshot is worst at proving (the selected row looks
259
- // identical to an unselected one at a glance, which is the whole scar).
260
- //
261
- // ⚠ NOTHING HERE INVENTS A GROUP, A CAPTION OR AN ORDER — the file header's law, applied. Every
262
- // one of the three rides `TriggerOption` from `routes_automation._triggers_vocab`.
263
-
264
- /** One connector's rows inside the Connector group (Gmail / Webhooks / Scraper / TikTok). */
265
- export interface TriggerSubGroup {
266
- key: string;
267
- label: string;
268
- rows: TriggerOption[];
269
- }
270
-
271
- /** One rendered section of the picker: a caption, the rows under it, and any connector nests. */
272
- export interface TriggerGroup {
273
- /** The server's group id. `""` when the server sent none — one unlabelled section. */
274
- key: string;
275
- /** The server's caption, PRINTED verbatim. `""` renders no heading, never an invented one. */
276
- label: string;
277
- order: number;
278
- /** Rows that hang directly off the group head. */
279
- rows: TriggerOption[];
280
- /** Connector nests, in first-appearance order. Empty for Time and Database. */
281
- sub: TriggerSubGroup[];
282
- }
283
-
284
- /**
285
- * The picker's sections: CATEGORY first (Time / Database / Connector), then the trigger, with
286
- * connector rows nested under their own product name.
287
- *
288
- * ⛔ ORDER COMES FROM `groupOrder`, AND AN ABSENT ONE SORTS **LAST**. This is the
289
- * `ACTION_GROUP_ORDER` rule and it is the safe direction: an unordered group is one this client
290
- * has no server opinion about, and putting it at the TOP of the menu would present the
291
- * unclassified thing as the primary answer. Ties keep FIRST-APPEARANCE order, which is what the
292
- * old `<select>` used as its only ordering and is still the honest fallback for a server that
293
- * sends no `groupOrder` at all.
294
- *
295
- * ⛔ AND NOTHING IS DROPPED. Every option in, every option out — `planned` and `ready:false` rows
296
- * included, because a picker that hides them answers "can it run when an email arrives?" with
297
- * silence, and the question then gets asked again (R9's precedent). Faded is a wall the server
298
- * enforces at `clean_trigger`; a shorter list is a lie.
299
- */
300
- export function groupTriggers(options: TriggerOption[]): TriggerGroup[] {
301
- const out: TriggerGroup[] = [];
302
- const byKey = new Map<string, TriggerGroup>();
303
- for (const t of options) {
304
- const key = t.group || "";
305
- let g = byKey.get(key);
306
- if (!g) {
307
- g = {
308
- key,
309
- // ⚠ `groupLabel` OR NOTHING. Falling back to `t.group` would print the id ("connector")
310
- // as a heading — a client inventing the server's wording, one `||` at a time.
311
- label: t.groupLabel || "",
312
- order: typeof t.groupOrder === "number" ? t.groupOrder : Number.MAX_SAFE_INTEGER,
313
- rows: [],
314
- sub: [],
315
- };
316
- byKey.set(key, g);
317
- out.push(g);
318
- }
319
- const c = t.connector;
320
- if (c && c.key) {
321
- let s = g.sub.find((x) => x.key === c.key);
322
- if (!s) {
323
- s = { key: c.key, label: c.label || c.key, rows: [] };
324
- g.sub.push(s);
325
- }
326
- s.rows.push(t);
327
- } else {
328
- g.rows.push(t);
329
- }
330
- }
331
- // A STABLE sort by order alone: `Array.prototype.sort` is stable in every runtime this ships to
332
- // (ES2019 mandates it), so equal orders keep the first-appearance sequence above.
333
- return out.sort((a, b) => a.order - b.order);
334
- }
335
-
336
- /** One rendered section of the ACTION menu: the same shape `TriggerGroup` has, over the
337
- * catalog's rows. */
338
- export interface ActionGroup {
339
- key: string;
340
- order: number;
341
- rows: ActionCatalogRow[];
342
- sub: { key: string; label: string; rows: ActionCatalogRow[] }[];
343
- }
344
-
345
- /**
346
- * ⭐ WAVE 27 · OWNER ITEM 33 / CONTRACT C4 — the action menu, with connector rows NESTED.
347
- *
348
- * ⛔ THE SAME ARITHMETIC AS `groupTriggers`, AND THAT IS THE POINT. The trigger picker has
349
- * nested connectors since wave 24; the action menu is the identical question about a different
350
- * catalog, and C4 asks for the nest to be "driven by server data, name-for-name". So this reads
351
- * `row.connector` exactly as `groupTriggers` reads `t.connector`, sorts by `groupOrder` with an
352
- * absent one LAST, and holds no list of connector names of its own.
353
- *
354
- * ⚠ INERT UNTIL THE SERVER STAMPS THE ROWS, by construction rather than by a flag: with no
355
- * `connector` on any row, every row lands in `rows` and `sub` is empty — which renders the flat
356
- * menu that ships today. That is what lets this half land before B's, instead of two sessions
357
- * having to meet in the middle.
358
- *
359
- * ⛔ THE GROUP KEY IS ALSO ITS LABEL here, unlike triggers. The action catalog's `group` IS the
360
- * printed caption ("Web action", "Database", "Connected", "Advanced logic") — the server sends
361
- * no separate `groupLabel` for actions — so this returns `key` and the caller prints it. Adding
362
- * a `label` member that merely copied `key` would invent a second name for one string.
363
- */
364
- export function groupActions(rows: ActionCatalogRow[]): ActionGroup[] {
365
- const out: ActionGroup[] = [];
366
- const byKey = new Map<string, ActionGroup>();
367
- for (const c of rows) {
368
- const key = c.group || "";
369
- let g = byKey.get(key);
370
- if (!g) {
371
- g = {
372
- key,
373
- order: typeof c.groupOrder === "number" ? c.groupOrder : Number.MAX_SAFE_INTEGER,
374
- rows: [],
375
- sub: [],
376
- };
377
- byKey.set(key, g);
378
- out.push(g);
379
- }
380
- const con = c.connector;
381
- if (con && con.key) {
382
- let s = g.sub.find((x) => x.key === con.key);
383
- if (!s) {
384
- // The server's LABEL, or its key when it sent none — never a prettified guess.
385
- s = { key: con.key, label: con.label || con.key, rows: [] };
386
- g.sub.push(s);
387
- }
388
- s.rows.push(c);
389
- } else {
390
- g.rows.push(c);
391
- }
392
- }
393
- // Stable by order alone (ES2019 mandates a stable sort), so equal orders keep first
394
- // appearance — the same rule `groupTriggers` states one function up.
395
- return out.sort((a, b) => a.order - b.order);
396
- }
397
-
398
- /**
399
- * The option the picker must render as SELECTED — and this function is the scar, in one place.
400
- *
401
- * ⛔ A `<select>` WHOSE `value` MATCHES NO `<option>` RENDERS THE FIRST ONE. A custom listbox
402
- * fails DIFFERENTLY and worse: it renders nothing selected, looks unconfigured, and the next
403
- * patch writes the blank over a real stored key. This repo has paid for the first form twice
404
- * (`enters_view`'s view id, and the condition builder's operator) and the control it replaces
405
- * carried an explicit guard for it.
406
- *
407
- * So: a stored key the server did not offer comes back as a SYNTHETIC row saying so, never as
408
- * `null`. `null` means one thing only — nothing is picked yet — which is also what R14 gates the
409
- * Configuration section on, so conflating the two would hide a configured automation's settings.
410
- */
411
- export function selectedTrigger(
412
- options: TriggerOption[],
413
- key: string
414
- ): TriggerOption | null {
415
- if (!key) return null;
416
- const hit = options.find((t) => t.key === key);
417
- if (hit) return hit;
418
- // ⚠ `ready: false` and NO `detail`: this row is not something the reader can act on, and
419
- // inventing a description for a trigger this deployment does not offer would be the client
420
- // speaking for a server that said nothing.
421
- return { key, label: `${key} (not offered here)`, ready: false };
422
- }
 
1
+ // ---------------------------------------------------------------------------
2
+ // automation/steps.ts — the arithmetic behind the numbered Steps (owner ruling R9).
3
+ //
4
+ // Pure functions, deliberately separate from the component: both of the things in
5
+ // here are the kind that go subtly wrong (a step number, a cron round-trip) and
6
+ // stay wrong invisibly, so they are written where they can be read on their own.
7
+ //
8
+ // ⚠ NOTHING HERE INVENTS A STEP. The node list, its order and its `col` all come
9
+ // from the server's `engine.graph()` — the same module that RUNS the steps — and
10
+ // this file only counts them (contract C6). A client-side second idea of what the
11
+ // steps are is the exact drift the split was built to prevent; the rationale is
12
+ // written at AutomationCreate.tsx:1-13.
13
+ // ---------------------------------------------------------------------------
14
+ import type { Action, ActionCatalogRow, Branch, Cond, GraphNode, TriggerOption }
15
+ from "./automationApi";
16
+
17
+ /**
18
+ * The branches of an If / then, TOLERATING the pre-wave-24 `{cond, actions}` shape on READ.
19
+ *
20
+ * ⚠ THIS MIRRORS `automation_engine.group_branches` AND THE DUPLICATION IS DELIBERATE, which is
21
+ * worth defending because this file's own header forbids a second idea of the server's data. It
22
+ * is a READ-SIDE TOLERANCE, not a second validator: it never decides what is legal, and every
23
+ * write still goes to `clean_actions`, which returns the one canonical shape. The alternative is
24
+ * strictly worse — a group stored in the old shape would render with no branches and no children,
25
+ * so a live automation's actions would simply VANISH from the screen while running perfectly.
26
+ * A migration the server performs on write and the client tolerates on read is the pair that has
27
+ * no window; trusting `branches` alone has one, for exactly as long as a definition sits unsaved.
28
+ */
29
+ export function groupBranches(a: Action): Branch[] {
30
+ const cfg = (a?.config || {}) as {
31
+ branches?: Branch[];
32
+ cond?: Cond | null;
33
+ actions?: Action[];
34
+ };
35
+ if (Array.isArray(cfg.branches)) return cfg.branches;
36
+ // The pre-wave-24 shape, read as the one branch the engine migrates it to.
37
+ if (Array.isArray(cfg.actions))
38
+ return [{ id: "b1", label: "A", cond: cfg.cond ?? null, actions: cfg.actions }];
39
+ return [];
40
+ }
41
+
42
+ /**
43
+ * R8's STEP NUMBERS for a whole flow, keyed by action id.
44
+ *
45
+ * THE RULING, and each clause is a line here:
46
+ * · the TRIGGER is unnumbered — it is not in this map at all, and the card carries the word
47
+ * "Trigger" instead. (This REVERSES wave 21's R9, "Step 1 is always the Trigger".)
48
+ * · the first action is 1, the second 2, …
49
+ * · an If / then occupies ONE number. Its branches are alternatives, not later steps, so they
50
+ * consume no numbers of their own.
51
+ * · a branch's children are numbered RELATIVE TO THEIR BRANCH — the fork at step 2 gives every
52
+ * one of its branches a `2.1`, a `2.2`, and so on. Two branches therefore both contain a
53
+ * `2.1`, which is correct rather than colliding: they are alternatives in lettered lanes and
54
+ * only one of them ever runs (the engine takes the first matching branch and breaks).
55
+ *
56
+ * ⛔ IT DOES NOT LETTER THE BRANCHES, and that is a change from the contract text. `clean_actions`
57
+ * assigns the letter server-side (`_branch_letter`, and "Otherwise" for the null-cond last leg),
58
+ * preserving any label the client sends — so the letter rides on `Branch.label` and a second
59
+ * lettering here would be exactly the client-copy-of-a-server-vocabulary this wave keeps deleting.
60
+ * The client sends an EMPTY label and renders what comes back, which also means letters re-flow
61
+ * correctly when a branch is deleted instead of going stale.
62
+ */
63
+ export function numberActions(actions: Action[]): Map<string, string> {
64
+ const out = new Map<string, string>();
65
+ const walk = (list: Action[], prefix: string) => {
66
+ (list || []).forEach((a, i) => {
67
+ const n = prefix ? `${prefix}.${i + 1}` : String(i + 1);
68
+ out.set(a.id, n);
69
+ if (a.kind === "group")
70
+ for (const br of groupBranches(a)) walk(br.actions || [], n);
71
+ });
72
+ };
73
+ walk(actions || [], "");
74
+ return out;
75
+ }
76
+
77
+ export interface Step {
78
+ node: GraphNode;
79
+ /** The number the card carries. */
80
+ n: number;
81
+ /** True when this node shares its position with the one before it — an ALTERNATIVE, not a next. */
82
+ alt: boolean;
83
+ }
84
+
85
+ /**
86
+ * Number the server's nodes for display.
87
+ *
88
+ * ⭐ THE NUMBER IS `col + 1`, NOT the array index, and that is the honest one. The
89
+ * `field_instagram` graph forks: `capture_paid` and `capture_anon` both sit at
90
+ * `col: 3` because they are the SAME position in the flow reached two ways (the
91
+ * paid rung answers, or the anonymous ladder does). Numbering them 4 and 5 would
92
+ * state a sequence that never happens. They share a number and the second is
93
+ * marked `alt`, so the list says "either of these" rather than "then".
94
+ *
95
+ * "Step 1 is always the Trigger" (R9) therefore falls out of the payload — the
96
+ * trigger is the node at `col: 0` — instead of being asserted by this client. If
97
+ * the engine ever emits something else first, the UI shows what the engine does.
98
+ */
99
+ export function numberSteps(nodes: GraphNode[]): Step[] {
100
+ return (nodes || []).map((node, i) => {
101
+ const col = typeof node.col === "number" ? node.col : i;
102
+ const before = i > 0 ? nodes[i - 1] : null;
103
+ const beforeCol = before && typeof before.col === "number" ? before.col : -1;
104
+ return { node, n: col + 1, alt: i > 0 && beforeCol === col };
105
+ });
106
+ }
107
+
108
+ /**
109
+ * Move `dragId` into `dropId`'s position within one list (owner item 11).
110
+ *
111
+ * ⛔ IT RETURNS `null` RATHER THAN THE LIST UNCHANGED, and the distinction is what keeps a
112
+ * pointless PATCH off the wire: "these two are the same card", "one of them is not in this list"
113
+ * and "here is your new order" are three different answers, and collapsing the first two into
114
+ * "the order you already had" would have the caller write the flow back to the server on every
115
+ * aborted drag. The caller writes only when this says something happened.
116
+ *
117
+ * THE CARD LANDS EXACTLY WHERE THE TARGET WAS, in both directions — splice out, then splice in at
118
+ * the target's ORIGINAL index. Dragging down, the target shifts up; dragging up, it shifts down.
119
+ * (The tempting version — insert at the target's index *after* the removal — is off by one when
120
+ * dragging downwards and drops the card one slot short of where the pointer is, which reads as
121
+ * the drag not having worked.)
122
+ */
123
+ export function reorderList<T extends { id: string }>(
124
+ list: T[],
125
+ dragId: string,
126
+ dropId: string
127
+ ): T[] | null {
128
+ if (!dragId || !dropId || dragId === dropId) return null;
129
+ const src = list || [];
130
+ const from = src.findIndex((x) => x.id === dragId);
131
+ const to = src.findIndex((x) => x.id === dropId);
132
+ if (from < 0 || to < 0) return null;
133
+ const next = [...src];
134
+ const [moved] = next.splice(from, 1);
135
+ if (!moved) return null;
136
+ next.splice(to, 0, moved);
137
+ return next;
138
+ }
139
+
140
+ /** One Properties section under "How this fetches": a panel, and every node that opens it. */
141
+ export interface PanelGroup {
142
+ /** The panel key — what the caller passes to `renderNodeBody`. */
143
+ panel: string;
144
+ /** Every machine node whose `panel` is this one, in the server's own order. */
145
+ nodes: GraphNode[];
146
+ }
147
+
148
+ /**
149
+ * The machine steps, GROUPED BY PANEL (item 5, contract C-CFG).
150
+ *
151
+ * ⛔ GROUPED, NEVER MAPPED ONE-TO-ONE, and the difference is a defect rather than a nicety.
152
+ * `panel` is MANY-TO-ONE over nodes, which is easy to miss because the old surface hid it: you
153
+ * clicked ONE card and got ONE panel. `field_instagram` gives `capture`, `capture_paid`,
154
+ * `capture_anon` AND `capture_metrics` all `panel: "capture"` (`automation_engine.py:3334-3352`),
155
+ * and `discover_instagram` gives both of its nodes `panel: "find"` (`:3381`/`:3387`). So a body
156
+ * rendered per NODE would print the capture prose four times over, and — the one that actually
157
+ * loses work — mount the 21-toggle discovery filter TWICE against a single `preds` array, two
158
+ * editors writing one piece of state where whichever blurred last silently wins.
159
+ *
160
+ * The detail panel has always joined on `panel` and never on node id (`AutomationDetail`'s own
161
+ * note says so); this keeps that law now that the panels are no longer reached by clicking a card.
162
+ *
163
+ * ORDER IS THE SERVER'S — first appearance wins, so the sections read in flow order rather than
164
+ * in whatever order a Map or a sort would produce.
165
+ */
166
+ export function groupByPanel(nodes: GraphNode[]): PanelGroup[] {
167
+ const order: string[] = [];
168
+ const byPanel = new Map<string, GraphNode[]>();
169
+ for (const n of nodes || []) {
170
+ // `panel` falls back to the id exactly as `graph()`'s own `node()` does (`panel or nid`),
171
+ // so a node the engine ships without one still gets its own section instead of joining
172
+ // every other panel-less node under the empty string.
173
+ const key = n.panel || n.id;
174
+ const seen = byPanel.get(key);
175
+ if (seen) seen.push(n);
176
+ else {
177
+ byPanel.set(key, [n]);
178
+ order.push(key);
179
+ }
180
+ }
181
+ return order.map((panel) => ({ panel, nodes: byPanel.get(panel) || [] }));
182
+ }
183
+
184
+ /** How the Step-1 card is currently set: what it repeats on, and at what time. */
185
+ export interface TriggerShape {
186
+ /** `day` = every day · `0`..`6` = that weekday (cron numbering, 0 = Sunday) · `custom` = a cron only. */
187
+ every: string;
188
+ /** `HH:MM`, empty when the cron does not express a single fire time. */
189
+ time: string;
190
+ }
191
+
192
+ const DAILY = /^(\d{1,2}) (\d{1,2}) \* \* \*$/;
193
+ const WEEKLY = /^(\d{1,2}) (\d{1,2}) \* \* ([0-7])$/;
194
+
195
+ function two(n: number): string {
196
+ return String(n).padStart(2, "0");
197
+ }
198
+
199
+ /**
200
+ * A stored cron → the controls that can express it.
201
+ *
202
+ * ⛔ ANYTHING THIS CANNOT EXPRESS COMES BACK AS `custom`, NEVER AS THE FIRST OPTION.
203
+ * A `<select>` whose value matches no `<option>` renders the first one, and the next
204
+ * Save then writes a schedule nobody chose — the defect this codebase has already
205
+ * paid for twice ([[cg-condition-builder-items]]; the two `(not in this database)`
206
+ * options in AutomationDetail exist for the same reason). A quarter-hourly weekday
207
+ * cron is a legal schedule the engine runs happily; the editor's job is to leave it
208
+ * alone, not to quietly turn it into "every day at 08:00".
209
+ *
210
+ * (Written without the literal cron string on purpose — a step-slash inside a block
211
+ * comment ENDS the comment, and the compiler's 20 cascading errors say nothing about
212
+ * the cron.)
213
+ */
214
+ export function readCron(cron: string): TriggerShape {
215
+ const raw = String(cron || "").trim();
216
+ const daily = DAILY.exec(raw);
217
+ if (daily) {
218
+ const [, m, h] = daily;
219
+ if (Number(h) <= 23 && Number(m) <= 59) return { every: "day", time: `${two(Number(h))}:${two(Number(m))}` };
220
+ }
221
+ const weekly = WEEKLY.exec(raw);
222
+ if (weekly) {
223
+ const [, m, h, d] = weekly;
224
+ if (Number(h) <= 23 && Number(m) <= 59) {
225
+ return { every: String(Number(d) % 7), time: `${two(Number(h))}:${two(Number(m))}` };
226
+ }
227
+ }
228
+ return { every: "custom", time: "" };
229
+ }
230
+
231
+ /** The controls → a cron string. `custom` keeps whatever the user typed, untouched. */
232
+ export function writeCron(every: string, time: string, custom: string): string {
233
+ if (every === "custom") return String(custom || "").trim();
234
+ const [h, m] = String(time || "06:00").split(":");
235
+ const hh = Math.min(23, Math.max(0, Number(h) || 0));
236
+ const mm = Math.min(59, Math.max(0, Number(m) || 0));
237
+ return every === "day" ? `${mm} ${hh} * * *` : `${mm} ${hh} * * ${Number(every) || 0}`;
238
+ }
239
+
240
+ /** The weekday options, in the order a week is read rather than in cron's 0-first order. */
241
+ export const WEEKDAYS: { value: string; label: string }[] = [
242
+ { value: "1", label: "Every Monday" },
243
+ { value: "2", label: "Every Tuesday" },
244
+ { value: "3", label: "Every Wednesday" },
245
+ { value: "4", label: "Every Thursday" },
246
+ { value: "5", label: "Every Friday" },
247
+ { value: "6", label: "Every Saturday" },
248
+ { value: "0", label: "Every Sunday" },
249
+ ];
250
+
251
+ // ── WAVE 25 · C2: THE GROUPED TRIGGER PICKER'S ARITHMETIC ───────────────────────────────────
252
+ //
253
+ // ⛔ WHY THIS IS HERE AND NOT IN THE COMPONENT. Both functions below are the kind that go subtly
254
+ // wrong and stay wrong INVISIBLY — a group silently sorted first, a stored trigger silently
255
+ // rendering as nothing — and this file exists precisely so that class can be executed under node
256
+ // by `verify_steps.py` instead of eyeballed in a screenshot. It is also D-58's minimum: the
257
+ // builder's largest surface has had no gate but `tsc` and eyes, and the picker's grouped and
258
+ // SELECTED states are the two things a screenshot is worst at proving (the selected row looks
259
+ // identical to an unselected one at a glance, which is the whole scar).
260
+ //
261
+ // ⚠ NOTHING HERE INVENTS A GROUP, A CAPTION OR AN ORDER — the file header's law, applied. Every
262
+ // one of the three rides `TriggerOption` from `routes_automation._triggers_vocab`.
263
+
264
+ /** One connector's rows inside the Connector group (Gmail / Webhooks / Scraper / TikTok). */
265
+ export interface TriggerSubGroup {
266
+ key: string;
267
+ label: string;
268
+ rows: TriggerOption[];
269
+ }
270
+
271
+ /** One rendered section of the picker: a caption, the rows under it, and any connector nests. */
272
+ export interface TriggerGroup {
273
+ /** The server's group id. `""` when the server sent none — one unlabelled section. */
274
+ key: string;
275
+ /** The server's caption, PRINTED verbatim. `""` renders no heading, never an invented one. */
276
+ label: string;
277
+ order: number;
278
+ /** Rows that hang directly off the group head. */
279
+ rows: TriggerOption[];
280
+ /** Connector nests, in first-appearance order. Empty for Time and Database. */
281
+ sub: TriggerSubGroup[];
282
+ }
283
+
284
+ /**
285
+ * The picker's sections: CATEGORY first (Time / Database / Connector), then the trigger, with
286
+ * connector rows nested under their own product name.
287
+ *
288
+ * ⛔ ORDER COMES FROM `groupOrder`, AND AN ABSENT ONE SORTS **LAST**. This is the
289
+ * `ACTION_GROUP_ORDER` rule and it is the safe direction: an unordered group is one this client
290
+ * has no server opinion about, and putting it at the TOP of the menu would present the
291
+ * unclassified thing as the primary answer. Ties keep FIRST-APPEARANCE order, which is what the
292
+ * old `<select>` used as its only ordering and is still the honest fallback for a server that
293
+ * sends no `groupOrder` at all.
294
+ *
295
+ * ⛔ AND NOTHING IS DROPPED. Every option in, every option out — `planned` and `ready:false` rows
296
+ * included, because a picker that hides them answers "can it run when an email arrives?" with
297
+ * silence, and the question then gets asked again (R9's precedent). Faded is a wall the server
298
+ * enforces at `clean_trigger`; a shorter list is a lie.
299
+ */
300
+ export function groupTriggers(options: TriggerOption[]): TriggerGroup[] {
301
+ const out: TriggerGroup[] = [];
302
+ const byKey = new Map<string, TriggerGroup>();
303
+ for (const t of options) {
304
+ const key = t.group || "";
305
+ let g = byKey.get(key);
306
+ if (!g) {
307
+ g = {
308
+ key,
309
+ // ⚠ `groupLabel` OR NOTHING. Falling back to `t.group` would print the id ("connector")
310
+ // as a heading — a client inventing the server's wording, one `||` at a time.
311
+ label: t.groupLabel || "",
312
+ order: typeof t.groupOrder === "number" ? t.groupOrder : Number.MAX_SAFE_INTEGER,
313
+ rows: [],
314
+ sub: [],
315
+ };
316
+ byKey.set(key, g);
317
+ out.push(g);
318
+ }
319
+ const c = t.connector;
320
+ if (c && c.key) {
321
+ let s = g.sub.find((x) => x.key === c.key);
322
+ if (!s) {
323
+ s = { key: c.key, label: c.label || c.key, rows: [] };
324
+ g.sub.push(s);
325
+ }
326
+ s.rows.push(t);
327
+ } else {
328
+ g.rows.push(t);
329
+ }
330
+ }
331
+ // A STABLE sort by order alone: `Array.prototype.sort` is stable in every runtime this ships to
332
+ // (ES2019 mandates it), so equal orders keep the first-appearance sequence above.
333
+ return out.sort((a, b) => a.order - b.order);
334
+ }
335
+
336
+ /** One rendered section of the ACTION menu: the same shape `TriggerGroup` has, over the
337
+ * catalog's rows. */
338
+ export interface ActionGroup {
339
+ key: string;
340
+ order: number;
341
+ rows: ActionCatalogRow[];
342
+ sub: { key: string; label: string; rows: ActionCatalogRow[] }[];
343
+ }
344
+
345
+ /**
346
+ * ⭐ WAVE 27 · OWNER ITEM 33 / CONTRACT C4 — the action menu, with connector rows NESTED.
347
+ *
348
+ * ⛔ THE SAME ARITHMETIC AS `groupTriggers`, AND THAT IS THE POINT. The trigger picker has
349
+ * nested connectors since wave 24; the action menu is the identical question about a different
350
+ * catalog, and C4 asks for the nest to be "driven by server data, name-for-name". So this reads
351
+ * `row.connector` exactly as `groupTriggers` reads `t.connector`, sorts by `groupOrder` with an
352
+ * absent one LAST, and holds no list of connector names of its own.
353
+ *
354
+ * ⚠ INERT UNTIL THE SERVER STAMPS THE ROWS, by construction rather than by a flag: with no
355
+ * `connector` on any row, every row lands in `rows` and `sub` is empty — which renders the flat
356
+ * menu that ships today. That is what lets this half land before B's, instead of two sessions
357
+ * having to meet in the middle.
358
+ *
359
+ * ⛔ THE GROUP KEY IS ALSO ITS LABEL here, unlike triggers. The action catalog's `group` IS the
360
+ * printed caption ("Web action", "Database", "Connected", "Advanced logic") — the server sends
361
+ * no separate `groupLabel` for actions — so this returns `key` and the caller prints it. Adding
362
+ * a `label` member that merely copied `key` would invent a second name for one string.
363
+ */
364
+ export function groupActions(rows: ActionCatalogRow[]): ActionGroup[] {
365
+ const out: ActionGroup[] = [];
366
+ const byKey = new Map<string, ActionGroup>();
367
+ for (const c of rows) {
368
+ const key = c.group || "";
369
+ let g = byKey.get(key);
370
+ if (!g) {
371
+ g = {
372
+ key,
373
+ order: typeof c.groupOrder === "number" ? c.groupOrder : Number.MAX_SAFE_INTEGER,
374
+ rows: [],
375
+ sub: [],
376
+ };
377
+ byKey.set(key, g);
378
+ out.push(g);
379
+ }
380
+ const con = c.connector;
381
+ if (con && con.key) {
382
+ let s = g.sub.find((x) => x.key === con.key);
383
+ if (!s) {
384
+ // The server's LABEL, or its key when it sent none — never a prettified guess.
385
+ s = { key: con.key, label: con.label || con.key, rows: [] };
386
+ g.sub.push(s);
387
+ }
388
+ s.rows.push(c);
389
+ } else {
390
+ g.rows.push(c);
391
+ }
392
+ }
393
+ // Stable by order alone (ES2019 mandates a stable sort), so equal orders keep first
394
+ // appearance — the same rule `groupTriggers` states one function up.
395
+ return out.sort((a, b) => a.order - b.order);
396
+ }
397
+
398
+ /**
399
+ * The option the picker must render as SELECTED — and this function is the scar, in one place.
400
+ *
401
+ * ⛔ A `<select>` WHOSE `value` MATCHES NO `<option>` RENDERS THE FIRST ONE. A custom listbox
402
+ * fails DIFFERENTLY and worse: it renders nothing selected, looks unconfigured, and the next
403
+ * patch writes the blank over a real stored key. This repo has paid for the first form twice
404
+ * (`enters_view`'s view id, and the condition builder's operator) and the control it replaces
405
+ * carried an explicit guard for it.
406
+ *
407
+ * So: a stored key the server did not offer comes back as a SYNTHETIC row saying so, never as
408
+ * `null`. `null` means one thing only — nothing is picked yet — which is also what R14 gates the
409
+ * Configuration section on, so conflating the two would hide a configured automation's settings.
410
+ */
411
+ export function selectedTrigger(
412
+ options: TriggerOption[],
413
+ key: string
414
+ ): TriggerOption | null {
415
+ if (!key) return null;
416
+ const hit = options.find((t) => t.key === key);
417
+ if (hit) return hit;
418
+ // ⚠ `ready: false` and NO `detail`: this row is not something the reader can act on, and
419
+ // inventing a description for a trigger this deployment does not offer would be the client
420
+ // speaking for a server that said nothing.
421
+ return { key, label: `${key} (not offered here)`, ready: false };
422
+ }
web/src/customer-grid/ColumnMenu.tsx CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/customer-grid/CustomerGrid.tsx CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/customer-grid/MapView.tsx CHANGED
@@ -1,946 +1,946 @@
1
- // ---------------------------------------------------------------------------
2
- // customer-grid / MapView.tsx
3
- // "Map" is a VIEW beside Grid/List/Calendar/Kanban, not a module: it projects
4
- // the SAME pipeline output the grid paints, so the toolbar count, filters and
5
- // sorts mean exactly what they mean on the grid; only the projection changes.
6
- //
7
- // Wave 8 grows it up (I2/I3/I5/I6, contracts C2/C3/C7):
8
- // I2 wheel/pinch zoom, drag to pan, and a box-select that feeds the grid's
9
- // OWN selection — so the existing "N selected · Add to cohort" bar serves
10
- // the map with no second cohort picker to keep in sync.
11
- // I3 colour pins by any single-select field (`display.colorField`) + legend.
12
- // I5 size pins by any numeric/measure field (`display.sizeField`) + legend.
13
- // I6 a less templatic picture: water wash, land fill, thin C1 strokes,
14
- // graticule, hover halos.
15
- //
16
- // C7 CALL — NO map library. C7 allows maplibre-gl "with a self-hosted
17
- // vector/raster-free style (NO external tile fetches ... if real tiles need a
18
- // network fetch, that is a blocker: fall back)". A basemap IS a network fetch,
19
- // and self-hosting tiles means shipping them inside a single-file embed. A
20
- // tileless maplibre would render exactly the geometry below for +230 KB gz on a
21
- // 251 KB budget. Declined on the contract's own terms; see the mailbox.
22
- //
23
- // Honesty rules (rule 8b), carried and extended:
24
- // - rows WITHOUT coordinates are never silently hidden — a chip says
25
- // "N matching records have no location";
26
- // - a pin per row at its geocoded position, no clustering and no jitter (two
27
- // customers at one address overlap rather than being drawn where they are
28
- // not);
29
- // - the colour legend STATES it when a field has more choices than the
30
- // palette has entries, rather than recycling a colour onto a second meaning;
31
- // - a row with no value under the size field gets the smallest dot and the
32
- // legend says so — it is never dropped from the map.
33
- // ---------------------------------------------------------------------------
34
-
35
- import { useCallback, useEffect, useMemo, useRef, useState } from "react";
36
- import type { KeyboardEvent as ReactKeyboardEvent, PointerEvent as ReactPointerEvent } from "react";
37
- import type { Field, Row } from "./types";
38
- import { formatDisplay } from "./cells";
39
- import { LAND_PATH, LAKE_PATHS } from "./mapGeometry";
40
- import {
41
- BASEMAP_DETAIL_K,
42
- bubbleRadius,
43
- cardBox,
44
- dashPattern,
45
- fitView,
46
- fromScreen,
47
- googleDirectionsUrl,
48
- googleMapsUrl,
49
- googleRouteUrl,
50
- googleZoomForK,
51
- graticuleOpacity,
52
- hairline,
53
- haversineKm,
54
- isPlottable,
55
- normRect,
56
- planRoute,
57
- project,
58
- toScreen,
59
- unproject,
60
- WORLD,
61
- zoomAt,
62
- zoomLimits,
63
- } from "./mapProjection";
64
- import type { GeoStop, Pt, View } from "./mapProjection";
65
-
66
- const VIEW_W = 1000;
67
- const VIEW_H = 620;
68
- const R_MIN = 3.2;
69
- const R_MAX = 15;
70
- const R_PLAIN = 4.5;
71
- const R_NULL = 2.6;
72
-
73
- /**
74
- * The pin palette: C1's cycle (blue -> green -> yellow -> red) then its
75
- * 20%-darkened variants, exactly as the contract specifies for series colour.
76
- * `fill` is the pastel; `line` is a darker companion, because a C1 pastel
77
- * measures 1.4-1.9:1 on white and a fill alone would be a smudge (the same
78
- * measurement that drove the I7c amendment).
79
- */
80
- const SERIES: { fill: string; line: string }[] = [
81
- { fill: "#9DBFF2", line: "#5F7FB0" },
82
- { fill: "#A5D8B4", line: "#5E9C74" },
83
- { fill: "#F5D989", line: "#B39A46" },
84
- { fill: "#F0A8A0", line: "#B76D65" },
85
- { fill: "#7E99C2", line: "#4A6790" },
86
- { fill: "#84AD91", line: "#4C7A5C" },
87
- { fill: "#C4AE6E", line: "#8C7838" },
88
- { fill: "#C0867F", line: "#8A5049" },
89
- ];
90
- /** Anything past the palette, plus the blank bucket. Neutral ON PURPOSE: a
91
- * ninth colour that repeats the first would make two meanings look identical. */
92
- const OVERFLOW = { fill: "#D7DBE3", line: "#8A909C" };
93
- const DEFAULT_PIN = { fill: "#9DBFF2", line: "#4F6079" };
94
-
95
- interface MapPoint {
96
- pid: number;
97
- title: string;
98
- p: Pt;
99
- /** The ORIGINAL coordinate, kept beside the projected one so the Google
100
- * hand-off links to the exact geocode we plotted rather than to a round-trip
101
- * through the projection. */
102
- lat: number;
103
- lon: number;
104
- colorKey: string | null;
105
- size: number | null;
106
- /** The RAW cell values behind the two encodings. The hover card formats these
107
- * with `formatDisplay`, the grid's own formatter — `colorKey` is trimmed for
108
- * bucketing and `size` is coerced for the bubble scale, so neither is what a
109
- * human should be shown. */
110
- colorVal: Row[keyof Row];
111
- sizeVal: Row[keyof Row];
112
- }
113
-
114
- function coord(v: Row[keyof Row]): number | null {
115
- if (v == null || v === "") return null;
116
- const n = typeof v === "number" ? v : Number(v);
117
- return Number.isFinite(n) ? n : null;
118
- }
119
-
120
- function numOrNull(v: Row[keyof Row]): number | null {
121
- if (v == null || v === "") return null;
122
- const n = typeof v === "number" ? v : Number(v);
123
- return Number.isFinite(n) ? n : null;
124
- }
125
-
126
- export function MapView({
127
- rows,
128
- field,
129
- colorField,
130
- sizeField,
131
- selectedPids,
132
- onSelectPids,
133
- onOpen,
134
- }: {
135
- /** DISTINCT data rows from the full pipeline, overlay edits layered — the
136
- * calendar/kanban contract, verbatim. */
137
- rows: Row[];
138
- /** The locked identity column — pin tooltips and aria labels. */
139
- field: Field;
140
- /** I3 — the view's `display.colorField`, already resolved to a real field. */
141
- colorField?: Field;
142
- /** I5 — the view's `display.sizeField`, already resolved to a real field. */
143
- sizeField?: Field;
144
- /** The grid's selection, shared: pins render selected, and the existing
145
- * selection bar is what offers "Add to cohort" (C3). */
146
- selectedPids: ReadonlySet<number>;
147
- onSelectPids: (pids: number[], mode: "replace" | "add") => void;
148
- onOpen: (pid: number) => void;
149
- }) {
150
- const [hoverPid, setHoverPid] = useState<number | null>(null);
151
- // I18-R — the route planner. Off until asked for: a route drawn over a
152
- // selection nobody asked to route is just clutter.
153
- const [routeOn, setRouteOn] = useState(false);
154
- const [roundTrip, setRoundTrip] = useState(false);
155
- const [routeStartPid, setRouteStartPid] = useState<number | null>(null);
156
- const [view, setView] = useState<View | null>(null);
157
- const [drag, setDrag] = useState<{ x0: number; y0: number; x1: number; y1: number } | null>(null);
158
- /**
159
- * ⚠ The svg element lives in STATE, not in a ref, and that is load-bearing.
160
- *
161
- * `view` starts null, so the FIRST render returns the empty state and there is
162
- * no <svg> in the tree at all. A `useRef` would still be null when the wheel
163
- * effect below first ran, and by the time the svg actually mounted the
164
- * effect's deps (`localPoint`, `kMin`, `kMax`) were all unchanged — so React
165
- * would never re-run it and the wheel listener would never be attached. Wheel
166
- * zoom would be silently dead on the ordinary path.
167
- *
168
- * Holding the element in state makes attachment a consequence of MOUNTING
169
- * rather than of a dependency happening to change, so the hazard cannot come
170
- * back. No unit test can see this: the gate is pure TS under node, and a
171
- * screenshot of a static page has no React in it.
172
- */
173
- const [svgEl, setSvgEl] = useState<SVGSVGElement | null>(null);
174
- const panRef = useRef<{ x: number; y: number; tx: number; ty: number } | null>(null);
175
- /** A finished drag must not also read as a click on the pin underneath — the
176
- * kanban card's lesson (viewModes.tsx), same fix. */
177
- const movedRef = useRef(false);
178
- /** Once the user has zoomed or panned, a data change must NOT yank the view
179
- * back. Before that, refitting on new data is the helpful behaviour. */
180
- const touchedRef = useRef(false);
181
-
182
- const points = useMemo(() => {
183
- const out: MapPoint[] = [];
184
- for (const r of rows) {
185
- const lat = coord(r.lat);
186
- const lon = coord(r.lon);
187
- if (lat == null || lon == null) continue;
188
- if (Math.abs(lat) > 90 || Math.abs(lon) > 180) continue;
189
- out.push({
190
- pid: r.pid,
191
- title: String(r[field.key] ?? ""),
192
- p: project(lon, lat),
193
- lat,
194
- lon,
195
- colorKey: colorField ? String(r[colorField.key] ?? "").trim() : null,
196
- size: sizeField ? numOrNull(r[sizeField.key]) : null,
197
- colorVal: colorField ? r[colorField.key] : null,
198
- sizeVal: sizeField ? r[sizeField.key] : null,
199
- });
200
- }
201
- return out;
202
- }, [rows, field.key, colorField, sizeField]);
203
- const noCoords = rows.length - points.length;
204
-
205
- // The fit is the INITIAL view, not the projection (see mapProjection.ts).
206
- const fit = useMemo(
207
- () => fitView(points.map((p) => p.p), VIEW_W, VIEW_H),
208
- [points]
209
- );
210
- useEffect(() => {
211
- if (!fit) return;
212
- if (!touchedRef.current || view == null) setView(fit);
213
- // `view` is deliberately absent from the deps: this effect exists to seed
214
- // and re-fit, and re-running it on every pan would fight the user for the
215
- // camera.
216
- // eslint-disable-next-line react-hooks/exhaustive-deps
217
- }, [fit]);
218
-
219
- const { kMin, kMax } = useMemo(
220
- () => (fit ? zoomLimits(fit.k, VIEW_H, BASEMAP_DETAIL_K) : { kMin: 1, kMax: 1 }),
221
- [fit]
222
- );
223
-
224
- /** Colour buckets, in first-seen order so the legend is stable. */
225
- const colorBuckets = useMemo(() => {
226
- if (!colorField) return null;
227
- const order: string[] = [];
228
- const counts = new Map<string, number>();
229
- for (const p of points) {
230
- const k = p.colorKey ?? "";
231
- if (!counts.has(k)) {
232
- counts.set(k, 0);
233
- order.push(k);
234
- }
235
- counts.set(k, (counts.get(k) ?? 0) + 1);
236
- }
237
- const named = order.filter((k) => k !== "");
238
- const swatch = new Map<string, { fill: string; line: string }>();
239
- named.forEach((k, i) => swatch.set(k, i < SERIES.length ? SERIES[i] : OVERFLOW));
240
- return {
241
- order,
242
- counts,
243
- swatch,
244
- overflow: Math.max(0, named.length - SERIES.length),
245
- };
246
- }, [colorField, points]);
247
-
248
- /** The size field's observed range across MAPPED points (not the whole table:
249
- * the legend must describe the picture actually on screen). */
250
- const sizeRange = useMemo(() => {
251
- if (!sizeField) return null;
252
- let min = Infinity;
253
- let max = -Infinity;
254
- let missing = 0;
255
- for (const p of points) {
256
- if (p.size == null) {
257
- missing += 1;
258
- continue;
259
- }
260
- min = Math.min(min, p.size);
261
- max = Math.max(max, p.size);
262
- }
263
- if (!Number.isFinite(min)) return { min: 0, max: 0, missing, none: true };
264
- return { min, max, missing, none: false };
265
- }, [sizeField, points]);
266
-
267
- /** A coarse pointer means a phone or tablet, where Google's free URL takes 3
268
- * waypoints rather than 9. A media query, not user-agent sniffing. */
269
- const coarsePointer = useMemo(
270
- () =>
271
- typeof window !== "undefined" &&
272
- typeof window.matchMedia === "function" &&
273
- window.matchMedia("(pointer: coarse)").matches,
274
- []
275
- );
276
-
277
- /** Selected pins that can actually be routed, in a STABLE order (by pid) —
278
- * a Set's iteration order must not be what decides a route. */
279
- const routable = useMemo(
280
- () =>
281
- selectedPids.size < 2
282
- ? []
283
- : points
284
- .filter((p) => selectedPids.has(p.pid) && isPlottable(p.lat, p.lon))
285
- .sort((a, b) => a.pid - b.pid),
286
- [points, selectedPids]
287
- );
288
- /** Selected records with no usable coordinate. Counted and shown, never
289
- * folded silently into the stop total ([[no-unverifiable-aggregates]]). */
290
- const unroutable = selectedPids.size - routable.length;
291
-
292
- const plan = useMemo(() => {
293
- if (!routeOn || routable.length < 2) return null;
294
- const stops: GeoStop[] = routable.map((p) => ({ lat: p.lat, lon: p.lon }));
295
- // Default origin: the WESTERNMOST stop. Deterministic, stable while the user
296
- // pans, and sayable out loud — unlike "whatever ended up at index 0". The
297
- // picker below overrides it.
298
- let start = 0;
299
- for (let i = 1; i < routable.length; i++)
300
- if (routable[i].lon < routable[start].lon) start = i;
301
- if (routeStartPid != null) {
302
- const i = routable.findIndex((p) => p.pid === routeStartPid);
303
- if (i >= 0) start = i;
304
- }
305
- const { order, km } = planRoute(stops, haversineKm, { start, roundTrip });
306
- return {
307
- ordered: order.map((i) => routable[i]),
308
- km,
309
- link: googleRouteUrl(order.map((i) => stops[i]), { roundTrip, coarsePointer }),
310
- };
311
- }, [routeOn, routable, roundTrip, routeStartPid, coarsePointer]);
312
-
313
- const paint = useCallback(
314
- (p: MapPoint) => {
315
- if (!colorBuckets) return DEFAULT_PIN;
316
- const k = p.colorKey ?? "";
317
- if (k === "") return OVERFLOW;
318
- return colorBuckets.swatch.get(k) ?? OVERFLOW;
319
- },
320
- [colorBuckets]
321
- );
322
-
323
- const radius = useCallback(
324
- (p: MapPoint) => {
325
- if (!sizeField || !sizeRange || sizeRange.none) return R_PLAIN;
326
- return bubbleRadius(p.size, sizeRange.min, sizeRange.max, R_MIN, R_MAX, R_NULL);
327
- },
328
- [sizeField, sizeRange]
329
- );
330
-
331
- /** Client coords -> the SVG's own user-space coords (the viewBox scales). */
332
- const localPoint = useCallback(
333
- (clientX: number, clientY: number): Pt => {
334
- if (!svgEl) return { x: 0, y: 0 };
335
- const r = svgEl.getBoundingClientRect();
336
- return {
337
- x: ((clientX - r.left) / r.width) * VIEW_W,
338
- y: ((clientY - r.top) / r.height) * VIEW_H,
339
- };
340
- },
341
- [svgEl]
342
- );
343
-
344
- /**
345
- * ⚠ Wheel zoom MUST be a native, non-passive listener. React registers
346
- * `wheel` PASSIVELY at the root, so `e.preventDefault()` inside an `onWheel`
347
- * prop is a silent no-op: the page scrolls out from under the map while you
348
- * zoom, which reads as "the zoom is broken". Nothing in a screenshot shows
349
- * this, and no assertion on the rendered DOM can see it either.
350
- *
351
- * The zoom maths itself is `zoomAt` — the function the gate already proves
352
- * holds the point under the cursor still. Wave 8 re-derived that formula
353
- * inline here, so the tested copy and the shipped copy were two copies.
354
- */
355
- useEffect(() => {
356
- if (!svgEl) return;
357
- const onWheelNative = (e: WheelEvent) => {
358
- e.preventDefault();
359
- const { x, y } = localPoint(e.clientX, e.clientY);
360
- touchedRef.current = true;
361
- setView((v) => (v ? zoomAt(v, Math.exp(-e.deltaY * 0.0016), x, y, kMin, kMax) : v));
362
- };
363
- svgEl.addEventListener("wheel", onWheelNative, { passive: false });
364
- return () => svgEl.removeEventListener("wheel", onWheelNative);
365
- }, [svgEl, localPoint, kMin, kMax]);
366
-
367
- /** Zoom about the viewport centre — the button and keyboard gesture, where
368
- * there is no cursor to hold still. */
369
- const zoomBy = useCallback(
370
- (factor: number) => {
371
- touchedRef.current = true;
372
- setView((v) => (v ? zoomAt(v, factor, VIEW_W / 2, VIEW_H / 2, kMin, kMax) : v));
373
- },
374
- [kMin, kMax]
375
- );
376
-
377
- const doFit = useCallback(() => {
378
- touchedRef.current = false;
379
- setView(fit);
380
- }, [fit]);
381
-
382
- const panBy = useCallback((dx: number, dy: number) => {
383
- touchedRef.current = true;
384
- setView((v) => (v ? { ...v, tx: v.tx + dx, ty: v.ty + dy } : v));
385
- }, []);
386
-
387
- /** The camera had NO keyboard path at all before wave 9 — scroll wheel only,
388
- * which is unusable without a mouse and unreachable for anyone driving the
389
- * page from the keyboard. */
390
- const onFrameKeyDown = useCallback(
391
- (e: ReactKeyboardEvent<HTMLDivElement>) => {
392
- const step = e.shiftKey ? 160 : 60;
393
- switch (e.key) {
394
- case "ArrowLeft": panBy(step, 0); break;
395
- case "ArrowRight": panBy(-step, 0); break;
396
- case "ArrowUp": panBy(0, step); break;
397
- case "ArrowDown": panBy(0, -step); break;
398
- case "+": case "=": zoomBy(1.6); break;
399
- case "-": case "_": zoomBy(1 / 1.6); break;
400
- case "0": doFit(); break;
401
- // Esc abandons a marquee mid-drag. It falls THROUGH when there is no
402
- // drag, so it keeps closing whatever the host has open.
403
- case "Escape": if (!drag) return; setDrag(null); break;
404
- default: return;
405
- }
406
- e.preventDefault();
407
- },
408
- [panBy, zoomBy, doFit, drag]
409
- );
410
-
411
- const onPointerDown = useCallback(
412
- (e: ReactPointerEvent<SVGSVGElement>) => {
413
- if (e.button !== 0 || !view) return;
414
- const { x, y } = localPoint(e.clientX, e.clientY);
415
- movedRef.current = false;
416
- e.currentTarget.setPointerCapture(e.pointerId);
417
- // Shift (or Ctrl/Cmd) turns the drag into a SELECTION rectangle; a plain
418
- // drag pans. Both gestures are on the same button because a map that
419
- // needs a mode toggle to select is a map people never select on.
420
- if (e.shiftKey || e.ctrlKey || e.metaKey) setDrag({ x0: x, y0: y, x1: x, y1: y });
421
- else panRef.current = { x, y, tx: view.tx, ty: view.ty };
422
- },
423
- [view, localPoint]
424
- );
425
-
426
- const onPointerMove = useCallback(
427
- (e: ReactPointerEvent<SVGSVGElement>) => {
428
- const { x, y } = localPoint(e.clientX, e.clientY);
429
- if (drag) {
430
- movedRef.current = true;
431
- setDrag((d) => (d ? { ...d, x1: x, y1: y } : d));
432
- return;
433
- }
434
- const pan = panRef.current;
435
- if (!pan) return;
436
- if (Math.abs(x - pan.x) + Math.abs(y - pan.y) > 2) movedRef.current = true;
437
- touchedRef.current = true;
438
- setView((v) => (v ? { ...v, tx: pan.tx + (x - pan.x), ty: pan.ty + (y - pan.y) } : v));
439
- },
440
- [drag, localPoint]
441
- );
442
-
443
- const onPointerUp = useCallback(
444
- (e: ReactPointerEvent<SVGSVGElement>) => {
445
- if (drag && view) {
446
- const r = normRect(drag.x0, drag.y0, drag.x1, drag.y1);
447
- // A rectangle smaller than a few px is a mis-click, not a selection —
448
- // clearing the user's set on a stray shift-click would be its own bug.
449
- if (r.x1 - r.x0 > 3 && r.y1 - r.y0 > 3) {
450
- const hits: number[] = [];
451
- for (const p of points) {
452
- const s = toScreen(p.p, view);
453
- if (s.x >= r.x0 && s.x <= r.x1 && s.y >= r.y0 && s.y <= r.y1) hits.push(p.pid);
454
- }
455
- onSelectPids(hits, e.altKey ? "add" : "replace");
456
- }
457
- setDrag(null);
458
- }
459
- panRef.current = null;
460
- if (e.currentTarget.hasPointerCapture(e.pointerId))
461
- e.currentTarget.releasePointerCapture(e.pointerId);
462
- },
463
- [drag, view, points, onSelectPids]
464
- );
465
-
466
- if (!view || !fit) {
467
- return (
468
- <div className="cg-mode-empty">
469
- No records with a location to map yet.
470
- {noCoords > 0 &&
471
- ` ${noCoords.toLocaleString()} matching record${noCoords === 1 ? "" : "s"} have no location.`}
472
- </div>
473
- );
474
- }
475
-
476
- const hovered = hoverPid != null ? points.find((p) => p.pid === hoverPid) : undefined;
477
- const rect = drag ? normRect(drag.x0, drag.y0, drag.x1, drag.y1) : null;
478
- const tf = `translate(${view.tx.toFixed(2)} ${view.ty.toFixed(2)}) scale(${view.k.toFixed(6)})`;
479
- // Strokes live in the transformed group, so they are pre-divided by the zoom.
480
- // ⚠ This is the map's ONE stroke mechanism — see the note over `hairline` in
481
- // mapProjection.ts. No `.cg-map*` rule may add `vector-effect:
482
- // non-scaling-stroke` on top; that double-cancel is the wave-9 blur bug and
483
- // `scalingConflicts()` gates the stylesheet against it.
484
- const hair = (w: number) => hairline(w, view.k);
485
- const gratOpacity = graticuleOpacity(view.k);
486
- // I18 — the Google hand-off. Our vendored basemap is honest to roughly metro
487
- // scale; below that the answer is a LINK, not 270 KB gz of tile renderer plus
488
- // a hosted planet file. Nothing is fetched and no coordinate leaves the page
489
- // unless the user deliberately clicks.
490
- const centre = unproject(fromScreen({ x: VIEW_W / 2, y: VIEW_H / 2 }, view));
491
- const areaUrl = googleMapsUrl(centre.lat, centre.lon, googleZoomForK(view.k));
492
- // A single selected pin gets its own exact hand-off. Selection is persistent,
493
- // unlike hover — and the hover card must stay pointer-events:none, so a link
494
- // could never live in it without becoming a click trap.
495
- const solo = selectedPids.size === 1 ? points.find((p) => selectedPids.has(p.pid)) : undefined;
496
- const sizeLegend = sizeRange && !sizeRange.none && sizeRange.max > sizeRange.min
497
- ? [sizeRange.min, (sizeRange.min + sizeRange.max) / 2, sizeRange.max]
498
- : null;
499
-
500
- return (
501
- <div className="cg-mapview">
502
- <div className="cg-map-bar">
503
- {/* The count is "N of M", never a bare N: a pin can only be drawn for a
504
- row the host geocoded, and a lone "1,402" silently redefines the
505
- toolbar's 1,550 ([[no-unverifiable-aggregates]]). */}
506
- <span className="cg-cal-note cg-map-count">
507
- <strong>{points.length.toLocaleString()}</strong> of{" "}
508
- {rows.length.toLocaleString()} mapped · pinned at each customer's address
509
- </span>
510
- {noCoords > 0 && (
511
- <span
512
- className="cg-cal-nodate"
513
- title={`${noCoords.toLocaleString()} matching record${noCoords === 1 ? "" : "s"} have no geocoded location and are not on the map. They remain in the Grid and List views.`}
514
- >
515
- {noCoords.toLocaleString()} matching record{noCoords === 1 ? " has" : "s have"} no location
516
- </span>
517
- )}
518
- {/* One selected pin -> the exact geocode, handed off to Google. By
519
- lat/lon and never by name: a name search can resolve somewhere else,
520
- and then this link and our pin disagree about where a customer is.
521
- rel="noopener noreferrer" strips the Referer, so Google never learns
522
- which tenant or deployment the click came from. */}
523
- {solo && (
524
- <span className="cg-map-go">
525
- <a
526
- href={googleMapsUrl(solo.lat, solo.lon)}
527
- target="_blank"
528
- rel="noopener noreferrer"
529
- title={`Open ${solo.title} in Google Maps at ${solo.lat.toFixed(5)}, ${solo.lon.toFixed(5)} (new tab)`}
530
- >
531
- {solo.title || "Selected pin"} in Google Maps
532
- </a>
533
- <a
534
- href={googleDirectionsUrl(solo.lat, solo.lon)}
535
- target="_blank"
536
- rel="noopener noreferrer"
537
- title={`Directions to ${solo.title} (new tab)`}
538
- >
539
- Directions
540
- </a>
541
- </span>
542
- )}
543
- {/* I18-R — the route planner. Appears only with a multi-pin selection, so
544
- it is mutually exclusive with the single-pin links above and the bar
545
- never carries both. */}
546
- {selectedPids.size >= 2 && (
547
- <span className="cg-map-route">
548
- {!routeOn ? (
549
- <button
550
- type="button"
551
- className="cg-map-route-go"
552
- onClick={() => setRouteOn(true)}
553
- disabled={routable.length < 2}
554
- title={
555
- routable.length < 2
556
- ? "At least two selected records need a location to plan a route"
557
- : "Order these stops into a route"
558
- }
559
- >
560
- Plan route ({routable.length.toLocaleString()} stops)
561
- </button>
562
- ) : (
563
- plan && (
564
- <>
565
- <span className="cg-map-route-sum">
566
- <strong>{plan.ordered.length.toLocaleString()}</strong> stops ·{" "}
567
- {Math.round(plan.km).toLocaleString()} km
568
- {/* ⚠ Never call this a driving distance, and never derive a
569
- time from it: it is the sum of straight lines. Saying so
570
- is the difference between a useful estimate and a lie. */}
571
- <span className="cg-map-route-note"> straight-line, not driving distance</span>
572
- </span>
573
- <label className="cg-map-route-opt">
574
- Start
575
- {/* ⚠ `value` is always set — a <select> without one renders its
576
- FIRST option regardless of state. It mirrors the ACTUAL
577
- origin, so the westernmost default shows itself too. */}
578
- <select
579
- className="cg-map-route-start"
580
- value={String(plan.ordered[0].pid)}
581
- onChange={(e) => setRouteStartPid(Number(e.target.value))}
582
- >
583
- {routable.map((p) => (
584
- <option key={p.pid} value={p.pid}>
585
- {p.title || `#${p.pid}`}
586
- </option>
587
- ))}
588
- </select>
589
- </label>
590
- <label className="cg-map-route-opt">
591
- <input
592
- type="checkbox"
593
- checked={roundTrip}
594
- onChange={(e) => setRoundTrip(e.target.checked)}
595
- />
596
- Return to start
597
- </label>
598
- {plan.link && (
599
- <a
600
- href={plan.link.url}
601
- target="_blank"
602
- rel="noopener noreferrer"
603
- title="Open this route in Google Maps for driving directions (new tab)"
604
- >
605
- Open route in Google Maps
606
- {/* The free URL takes 9 waypoints on desktop and 3 on a
607
- phone. When the route is longer, SAY which part rides. */}
608
- {plan.link.used < plan.ordered.length &&
609
- ` (first ${plan.link.used} of ${plan.ordered.length})`}
610
- </a>
611
- )}
612
- <button
613
- type="button"
614
- className="cg-map-route-go"
615
- onClick={() => {
616
- setRouteOn(false);
617
- setRouteStartPid(null);
618
- }}
619
- >
620
- Clear
621
- </button>
622
- </>
623
- )
624
- )}
625
- {unroutable > 0 && (
626
- <span className="cg-cal-nodate">
627
- {unroutable.toLocaleString()} selected record{unroutable === 1 ? " has" : "s have"} no
628
- location and cannot be routed
629
- </span>
630
- )}
631
- </span>
632
- )}
633
- <span className="cg-map-hint">
634
- Scroll to zoom · drag to pan · shift-drag to select
635
- </span>
636
- </div>
637
- <div
638
- className="cg-map-frame"
639
- tabIndex={0}
640
- role="group"
641
- aria-label="Map canvas. Arrow keys pan, plus and minus zoom, zero fits to data."
642
- onKeyDown={onFrameKeyDown}
643
- >
644
- <svg
645
- ref={setSvgEl}
646
- viewBox={`0 0 ${VIEW_W} ${VIEW_H}`}
647
- preserveAspectRatio="xMidYMid meet"
648
- className={"cg-map-svg" + (drag ? " is-selecting" : "")}
649
- role="img"
650
- aria-label={`Map of ${points.length.toLocaleString()} customers`}
651
- onPointerDown={onPointerDown}
652
- onPointerMove={onPointerMove}
653
- onPointerUp={onPointerUp}
654
- onPointerCancel={onPointerUp}
655
- >
656
- <rect className="cg-map-water" x={0} y={0} width={VIEW_W} height={VIEW_H} />
657
- <g transform={tf}>
658
- {/* Graticule every 10 degrees. It earns its place zoomed OUT, where
659
- it is the only thing giving scale; once the real state borders
660
- arrive it would be a second line system fighting the first, so it
661
- fades away before they take over. Skipped entirely at zero
662
- opacity — ~36 invisible lines are still 36 nodes to lay out. */}
663
- {gratOpacity > 0.01 && (
664
- <g className="cg-map-grat" strokeWidth={hair(0.6)} opacity={gratOpacity}>
665
- {Array.from({ length: 17 }, (_, i) => {
666
- const y = project(0, -80 + i * 10).y;
667
- return <line key={`p${i}`} x1={0} y1={y} x2={WORLD} y2={y} />;
668
- })}
669
- {Array.from({ length: 19 }, (_, i) => {
670
- const x = project(-180 + i * 20, 0).x;
671
- return <line key={`m${i}`} x1={x} y1={0} x2={x} y2={WORLD} />;
672
- })}
673
- </g>
674
- )}
675
- {/* One path, every US state ring. Filled AND stroked, so interior
676
- state borders come free from the same geometry — no second pass
677
- and no chance of the borders disagreeing with the coastline.
678
- 0.9 rather than wave 8's 1.1: that weight was tuned for a single
679
- lone coastline, and it reads heavy once ~169 rings share it. */}
680
- <path className="cg-map-land" d={LAND_PATH} strokeWidth={hair(0.9)} />
681
- {LAKE_PATHS.map((d, i) => (
682
- <path key={i} className="cg-map-lake" d={d} strokeWidth={hair(0.8)} />
683
- ))}
684
- {/* The planned path, UNDER the pins so it never hides a stop. Inside
685
- the zoomed group, so it pans and scales with the geography; the
686
- stroke is pre-divided by k like every other line here. */}
687
- {plan && plan.ordered.length > 1 && (
688
- <polyline
689
- className="cg-map-route-line"
690
- strokeWidth={hair(1.8)}
691
- // ⚠ The dash MUST be pre-divided by the zoom too — a CSS
692
- // dasharray is in user units and would scale with the group,
693
- // turning the route into a few disconnected strokes.
694
- strokeDasharray={dashPattern(5, 4, view.k)}
695
- points={
696
- plan.ordered.map((p) => `${p.p.x},${p.p.y}`).join(" ") +
697
- (roundTrip ? ` ${plan.ordered[0].p.x},${plan.ordered[0].p.y}` : "")
698
- }
699
- />
700
- )}
701
- {points.map((p) => {
702
- const c = paint(p);
703
- const on = selectedPids.has(p.pid);
704
- const r = radius(p) / view.k;
705
- return (
706
- <circle
707
- key={p.pid}
708
- className={
709
- "cg-map-pin" +
710
- (hoverPid === p.pid ? " is-hover" : "") +
711
- (on ? " is-selected" : "")
712
- }
713
- cx={p.p.x}
714
- cy={p.p.y}
715
- r={hoverPid === p.pid ? r * 1.28 : r}
716
- fill={c.fill}
717
- stroke={on ? "#202433" : c.line}
718
- strokeWidth={hair(on ? 2 : 1)}
719
- role="button"
720
- tabIndex={0}
721
- aria-label={`Open ${p.title}`}
722
- onMouseEnter={() => setHoverPid(p.pid)}
723
- onMouseLeave={() => setHoverPid((h) => (h === p.pid ? null : h))}
724
- onFocus={() => setHoverPid(p.pid)}
725
- onBlur={() => setHoverPid((h) => (h === p.pid ? null : h))}
726
- onClick={() => {
727
- if (movedRef.current) return; // a finished pan/box is not a click
728
- onOpen(p.pid);
729
- }}
730
- onKeyDown={(e) => {
731
- if (e.key !== "Enter" && e.key !== " ") return;
732
- e.preventDefault();
733
- onOpen(p.pid);
734
- }}
735
- >
736
- <title>{p.title}</title>
737
- </circle>
738
- );
739
- })}
740
- </g>
741
- {/* I18 — the hover card. Wave 8 painted the title alone; a map whose
742
- pins carry a colour and a size encoding should say what they ARE.
743
- ⚠ Every value goes through `formatDisplay`, the SAME formatter the
744
- grid cells use, so a currency, a percentage or a date can never
745
- read one way on the map and another way in the table.
746
- Drawn in SVG screen space rather than as an HTML overlay: the
747
- viewBox letterboxes under preserveAspectRatio, so an HTML card
748
- would need the rendered scale re-derived, and this needs no
749
- conversion at all. pointer-events stay off — a card that can
750
- swallow the next click is a scar this codebase already carries. */}
751
- {/* Stop numbers, in SCREEN space so they stay legible at every zoom.
752
- pointer-events off: a badge sitting over a pin must not steal the
753
- click that opens the record — re-rooting the route is the "Start"
754
- picker's job, where it is visible and reversible. */}
755
- {plan &&
756
- plan.ordered.map((p, i) => {
757
- const s = toScreen(p.p, view);
758
- return (
759
- <g
760
- className="cg-map-stopno"
761
- key={p.pid}
762
- transform={`translate(${s.x.toFixed(2)} ${(s.y - 13).toFixed(2)})`}
763
- >
764
- <circle r={7.5} />
765
- <text textAnchor="middle" dy="3.4">{i + 1}</text>
766
- </g>
767
- );
768
- })}
769
- {hovered && (() => {
770
- const s = toScreen(hovered.p, view);
771
- const lines: string[] = [];
772
- if (colorField)
773
- lines.push(`${colorField.label}: ${formatDisplay(colorField, hovered.colorVal) || "—"}`);
774
- if (sizeField)
775
- lines.push(`${sizeField.label}: ${formatDisplay(sizeField, hovered.sizeVal) || "—"}`);
776
- const title = hovered.title || "(untitled)";
777
- // Inter's average advance at 11.5px. An estimate, deliberately
778
- // generous: too wide is a slightly roomy card, too narrow is text
779
- // spilling past its own background.
780
- const w = Math.max(title.length, ...lines.map((l) => l.length)) * 6.2 + 20;
781
- const h = 21 + lines.length * 14;
782
- const b = cardBox(s.x, s.y, w, h, VIEW_W, VIEW_H);
783
- return (
784
- <g className="cg-map-card" aria-hidden="true">
785
- <rect x={b.x} y={b.y} width={w} height={h} rx={5} />
786
- <text className="cg-map-card-t" x={b.x + 10} y={b.y + 15}>{title}</text>
787
- {lines.map((l, i) => (
788
- <text className="cg-map-card-l" key={i} x={b.x + 10} y={b.y + 15 + 14 * (i + 1)}>
789
- {l}
790
- </text>
791
- ))}
792
- </g>
793
- );
794
- })()}
795
- {rect && (
796
- <rect
797
- className="cg-map-marquee"
798
- x={rect.x0}
799
- y={rect.y0}
800
- width={rect.x1 - rect.x0}
801
- height={rect.y1 - rect.y0}
802
- />
803
- )}
804
- </svg>
805
-
806
- {/* --- I18 — the camera controls. Zoom in / zoom out / fit to data, the
807
- affordance every map has, replacing wave 8's link-button-in-a-text-bar.
808
- OUTSIDE the <svg> deliberately: a mousedown on a button inside it would
809
- begin a pan. Icons are SVG strokes — no emoji, no glyph font. The zoom
810
- buttons DISABLE at the limits, which is also how the honesty cap on
811
- zoom-in makes itself visible instead of just feeling stuck. --- */}
812
- <div className="cg-map-ctl" role="group" aria-label="Map camera">
813
- <button
814
- type="button"
815
- className="cg-map-ctl-b"
816
- // A greyed button with no reason reads as broken, not as honest —
817
- // and filtering to a single metro can push the FITTED zoom past the
818
- // cap, so this can be disabled the instant the view opens. Say why,
819
- // and point at the thing that does go further.
820
- title={
821
- view.k >= kMax * (1 - 1e-9)
822
- ? "Zoom in — at the limit. The basemap's detail ends here; use Open in Google Maps for street level."
823
- : "Zoom in (+)"
824
- }
825
- aria-label="Zoom in"
826
- disabled={view.k >= kMax * (1 - 1e-9)}
827
- onClick={() => zoomBy(1.6)}
828
- >
829
- <svg viewBox="0 0 16 16" aria-hidden="true"><path d="M8 3.6v8.8M3.6 8h8.8" /></svg>
830
- </button>
831
- <button
832
- type="button"
833
- className="cg-map-ctl-b"
834
- title="Zoom out (−)"
835
- aria-label="Zoom out"
836
- disabled={view.k <= kMin * (1 + 1e-9)}
837
- onClick={() => zoomBy(1 / 1.6)}
838
- >
839
- <svg viewBox="0 0 16 16" aria-hidden="true"><path d="M3.6 8h8.8" /></svg>
840
- </button>
841
- <button
842
- type="button"
843
- className="cg-map-ctl-b"
844
- title="Fit to data (0)"
845
- aria-label="Fit to data"
846
- onClick={doFit}
847
- >
848
- <svg viewBox="0 0 16 16" aria-hidden="true">
849
- <path d="M2.6 5.8V2.6h3.2M10.2 2.6h3.2v3.2M13.4 10.2v3.2h-3.2M5.8 13.4H2.6v-3.2" />
850
- <path d="M6.6 8h2.8M8 6.6v2.8" />
851
- </svg>
852
- </button>
853
- {/* The street-level hand-off, aimed at whatever is on screen right
854
- now. This is what makes the zoom cap honest rather than merely
855
- restrictive: the map stops where its geometry stops, and points at
856
- something that does not. */}
857
- <a
858
- className="cg-map-ctl-b"
859
- href={areaUrl}
860
- target="_blank"
861
- rel="noopener noreferrer"
862
- title="Open this area in Google Maps (new tab)"
863
- aria-label="Open this area in Google Maps"
864
- >
865
- <svg viewBox="0 0 16 16" aria-hidden="true">
866
- <path d="M7 3.8H4.3a.9.9 0 0 0-.9.9v6.9a.9.9 0 0 0 .9.9h6.9a.9.9 0 0 0 .9-.9V9.2" />
867
- <path d="M8.6 3.4h4v4M12.6 3.4 7.4 8.6" />
868
- </svg>
869
- </a>
870
- </div>
871
-
872
- {/* --- legends (I3/I5). Floated over the map, never in the flow. --- */}
873
- {(colorBuckets || sizeLegend) && (
874
- <div className="cg-map-legends">
875
- {colorField && colorBuckets && (
876
- <div className="cg-map-legend">
877
- <div className="cg-map-legend-t">{colorField.label}</div>
878
- {colorBuckets.order.slice(0, SERIES.length + 1).map((k) => {
879
- const c = k === "" ? OVERFLOW : colorBuckets.swatch.get(k) ?? OVERFLOW;
880
- return (
881
- <div key={k || "(blank)"} className="cg-map-legend-row">
882
- <span
883
- className="cg-map-swatch"
884
- style={{ background: c.fill, borderColor: c.line }}
885
- />
886
- <span className="cg-map-legend-k">{k === "" ? "(blank)" : k}</span>
887
- <span className="cg-map-legend-n">
888
- {(colorBuckets.counts.get(k) ?? 0).toLocaleString()}
889
- </span>
890
- </div>
891
- );
892
- })}
893
- {colorBuckets.overflow > 0 && (
894
- <div className="cg-map-legend-note">
895
- {colorBuckets.overflow.toLocaleString()} further value
896
- {colorBuckets.overflow === 1 ? " is" : "s are"} drawn grey — the palette
897
- holds {SERIES.length} colours and reusing one would make two values look
898
- like the same value.
899
- </div>
900
- )}
901
- </div>
902
- )}
903
- {sizeField && sizeRange && (
904
- <div className="cg-map-legend">
905
- <div className="cg-map-legend-t">{sizeField.label}</div>
906
- {sizeLegend ? (
907
- <div className="cg-map-sizerow">
908
- {sizeLegend.map((v, i) => {
909
- const r = bubbleRadius(v, sizeRange.min, sizeRange.max, R_MIN, R_MAX, R_NULL);
910
- return (
911
- <span key={i} className="cg-map-sizeitem">
912
- <svg width={R_MAX * 2 + 2} height={R_MAX * 2 + 2} aria-hidden>
913
- <circle
914
- cx={R_MAX + 1}
915
- cy={R_MAX + 1}
916
- r={r}
917
- fill={DEFAULT_PIN.fill}
918
- stroke={DEFAULT_PIN.line}
919
- />
920
- </svg>
921
- <span className="cg-map-legend-k">{formatDisplay(sizeField, v)}</span>
922
- </span>
923
- );
924
- })}
925
- </div>
926
- ) : (
927
- <div className="cg-map-legend-note">
928
- Every mapped record has the same {sizeField.label.toLowerCase()}, so the
929
- bubbles cannot differ in size.
930
- </div>
931
- )}
932
- {sizeRange.missing > 0 && (
933
- <div className="cg-map-legend-note">
934
- {sizeRange.missing.toLocaleString()} mapped record
935
- {sizeRange.missing === 1 ? " has" : "s have"} no value — drawn at the
936
- smallest dot, never removed from the map.
937
- </div>
938
- )}
939
- </div>
940
- )}
941
- </div>
942
- )}
943
- </div>
944
- </div>
945
- );
946
- }
 
1
+ // ---------------------------------------------------------------------------
2
+ // customer-grid / MapView.tsx
3
+ // "Map" is a VIEW beside Grid/List/Calendar/Kanban, not a module: it projects
4
+ // the SAME pipeline output the grid paints, so the toolbar count, filters and
5
+ // sorts mean exactly what they mean on the grid; only the projection changes.
6
+ //
7
+ // Wave 8 grows it up (I2/I3/I5/I6, contracts C2/C3/C7):
8
+ // I2 wheel/pinch zoom, drag to pan, and a box-select that feeds the grid's
9
+ // OWN selection — so the existing "N selected · Add to cohort" bar serves
10
+ // the map with no second cohort picker to keep in sync.
11
+ // I3 colour pins by any single-select field (`display.colorField`) + legend.
12
+ // I5 size pins by any numeric/measure field (`display.sizeField`) + legend.
13
+ // I6 a less templatic picture: water wash, land fill, thin C1 strokes,
14
+ // graticule, hover halos.
15
+ //
16
+ // C7 CALL — NO map library. C7 allows maplibre-gl "with a self-hosted
17
+ // vector/raster-free style (NO external tile fetches ... if real tiles need a
18
+ // network fetch, that is a blocker: fall back)". A basemap IS a network fetch,
19
+ // and self-hosting tiles means shipping them inside a single-file embed. A
20
+ // tileless maplibre would render exactly the geometry below for +230 KB gz on a
21
+ // 251 KB budget. Declined on the contract's own terms; see the mailbox.
22
+ //
23
+ // Honesty rules (rule 8b), carried and extended:
24
+ // - rows WITHOUT coordinates are never silently hidden — a chip says
25
+ // "N matching records have no location";
26
+ // - a pin per row at its geocoded position, no clustering and no jitter (two
27
+ // customers at one address overlap rather than being drawn where they are
28
+ // not);
29
+ // - the colour legend STATES it when a field has more choices than the
30
+ // palette has entries, rather than recycling a colour onto a second meaning;
31
+ // - a row with no value under the size field gets the smallest dot and the
32
+ // legend says so — it is never dropped from the map.
33
+ // ---------------------------------------------------------------------------
34
+
35
+ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
36
+ import type { KeyboardEvent as ReactKeyboardEvent, PointerEvent as ReactPointerEvent } from "react";
37
+ import type { Field, Row } from "./types";
38
+ import { formatDisplay } from "./cells";
39
+ import { LAND_PATH, LAKE_PATHS } from "./mapGeometry";
40
+ import {
41
+ BASEMAP_DETAIL_K,
42
+ bubbleRadius,
43
+ cardBox,
44
+ dashPattern,
45
+ fitView,
46
+ fromScreen,
47
+ googleDirectionsUrl,
48
+ googleMapsUrl,
49
+ googleRouteUrl,
50
+ googleZoomForK,
51
+ graticuleOpacity,
52
+ hairline,
53
+ haversineKm,
54
+ isPlottable,
55
+ normRect,
56
+ planRoute,
57
+ project,
58
+ toScreen,
59
+ unproject,
60
+ WORLD,
61
+ zoomAt,
62
+ zoomLimits,
63
+ } from "./mapProjection";
64
+ import type { GeoStop, Pt, View } from "./mapProjection";
65
+
66
+ const VIEW_W = 1000;
67
+ const VIEW_H = 620;
68
+ const R_MIN = 3.2;
69
+ const R_MAX = 15;
70
+ const R_PLAIN = 4.5;
71
+ const R_NULL = 2.6;
72
+
73
+ /**
74
+ * The pin palette: C1's cycle (blue -> green -> yellow -> red) then its
75
+ * 20%-darkened variants, exactly as the contract specifies for series colour.
76
+ * `fill` is the pastel; `line` is a darker companion, because a C1 pastel
77
+ * measures 1.4-1.9:1 on white and a fill alone would be a smudge (the same
78
+ * measurement that drove the I7c amendment).
79
+ */
80
+ const SERIES: { fill: string; line: string }[] = [
81
+ { fill: "#9DBFF2", line: "#5F7FB0" },
82
+ { fill: "#A5D8B4", line: "#5E9C74" },
83
+ { fill: "#F5D989", line: "#B39A46" },
84
+ { fill: "#F0A8A0", line: "#B76D65" },
85
+ { fill: "#7E99C2", line: "#4A6790" },
86
+ { fill: "#84AD91", line: "#4C7A5C" },
87
+ { fill: "#C4AE6E", line: "#8C7838" },
88
+ { fill: "#C0867F", line: "#8A5049" },
89
+ ];
90
+ /** Anything past the palette, plus the blank bucket. Neutral ON PURPOSE: a
91
+ * ninth colour that repeats the first would make two meanings look identical. */
92
+ const OVERFLOW = { fill: "#D7DBE3", line: "#8A909C" };
93
+ const DEFAULT_PIN = { fill: "#9DBFF2", line: "#4F6079" };
94
+
95
+ interface MapPoint {
96
+ pid: number;
97
+ title: string;
98
+ p: Pt;
99
+ /** The ORIGINAL coordinate, kept beside the projected one so the Google
100
+ * hand-off links to the exact geocode we plotted rather than to a round-trip
101
+ * through the projection. */
102
+ lat: number;
103
+ lon: number;
104
+ colorKey: string | null;
105
+ size: number | null;
106
+ /** The RAW cell values behind the two encodings. The hover card formats these
107
+ * with `formatDisplay`, the grid's own formatter — `colorKey` is trimmed for
108
+ * bucketing and `size` is coerced for the bubble scale, so neither is what a
109
+ * human should be shown. */
110
+ colorVal: Row[keyof Row];
111
+ sizeVal: Row[keyof Row];
112
+ }
113
+
114
+ function coord(v: Row[keyof Row]): number | null {
115
+ if (v == null || v === "") return null;
116
+ const n = typeof v === "number" ? v : Number(v);
117
+ return Number.isFinite(n) ? n : null;
118
+ }
119
+
120
+ function numOrNull(v: Row[keyof Row]): number | null {
121
+ if (v == null || v === "") return null;
122
+ const n = typeof v === "number" ? v : Number(v);
123
+ return Number.isFinite(n) ? n : null;
124
+ }
125
+
126
+ export function MapView({
127
+ rows,
128
+ field,
129
+ colorField,
130
+ sizeField,
131
+ selectedPids,
132
+ onSelectPids,
133
+ onOpen,
134
+ }: {
135
+ /** DISTINCT data rows from the full pipeline, overlay edits layered — the
136
+ * calendar/kanban contract, verbatim. */
137
+ rows: Row[];
138
+ /** The locked identity column — pin tooltips and aria labels. */
139
+ field: Field;
140
+ /** I3 — the view's `display.colorField`, already resolved to a real field. */
141
+ colorField?: Field;
142
+ /** I5 — the view's `display.sizeField`, already resolved to a real field. */
143
+ sizeField?: Field;
144
+ /** The grid's selection, shared: pins render selected, and the existing
145
+ * selection bar is what offers "Add to cohort" (C3). */
146
+ selectedPids: ReadonlySet<number>;
147
+ onSelectPids: (pids: number[], mode: "replace" | "add") => void;
148
+ onOpen: (pid: number) => void;
149
+ }) {
150
+ const [hoverPid, setHoverPid] = useState<number | null>(null);
151
+ // I18-R — the route planner. Off until asked for: a route drawn over a
152
+ // selection nobody asked to route is just clutter.
153
+ const [routeOn, setRouteOn] = useState(false);
154
+ const [roundTrip, setRoundTrip] = useState(false);
155
+ const [routeStartPid, setRouteStartPid] = useState<number | null>(null);
156
+ const [view, setView] = useState<View | null>(null);
157
+ const [drag, setDrag] = useState<{ x0: number; y0: number; x1: number; y1: number } | null>(null);
158
+ /**
159
+ * ⚠ The svg element lives in STATE, not in a ref, and that is load-bearing.
160
+ *
161
+ * `view` starts null, so the FIRST render returns the empty state and there is
162
+ * no <svg> in the tree at all. A `useRef` would still be null when the wheel
163
+ * effect below first ran, and by the time the svg actually mounted the
164
+ * effect's deps (`localPoint`, `kMin`, `kMax`) were all unchanged — so React
165
+ * would never re-run it and the wheel listener would never be attached. Wheel
166
+ * zoom would be silently dead on the ordinary path.
167
+ *
168
+ * Holding the element in state makes attachment a consequence of MOUNTING
169
+ * rather than of a dependency happening to change, so the hazard cannot come
170
+ * back. No unit test can see this: the gate is pure TS under node, and a
171
+ * screenshot of a static page has no React in it.
172
+ */
173
+ const [svgEl, setSvgEl] = useState<SVGSVGElement | null>(null);
174
+ const panRef = useRef<{ x: number; y: number; tx: number; ty: number } | null>(null);
175
+ /** A finished drag must not also read as a click on the pin underneath — the
176
+ * kanban card's lesson (viewModes.tsx), same fix. */
177
+ const movedRef = useRef(false);
178
+ /** Once the user has zoomed or panned, a data change must NOT yank the view
179
+ * back. Before that, refitting on new data is the helpful behaviour. */
180
+ const touchedRef = useRef(false);
181
+
182
+ const points = useMemo(() => {
183
+ const out: MapPoint[] = [];
184
+ for (const r of rows) {
185
+ const lat = coord(r.lat);
186
+ const lon = coord(r.lon);
187
+ if (lat == null || lon == null) continue;
188
+ if (Math.abs(lat) > 90 || Math.abs(lon) > 180) continue;
189
+ out.push({
190
+ pid: r.pid,
191
+ title: String(r[field.key] ?? ""),
192
+ p: project(lon, lat),
193
+ lat,
194
+ lon,
195
+ colorKey: colorField ? String(r[colorField.key] ?? "").trim() : null,
196
+ size: sizeField ? numOrNull(r[sizeField.key]) : null,
197
+ colorVal: colorField ? r[colorField.key] : null,
198
+ sizeVal: sizeField ? r[sizeField.key] : null,
199
+ });
200
+ }
201
+ return out;
202
+ }, [rows, field.key, colorField, sizeField]);
203
+ const noCoords = rows.length - points.length;
204
+
205
+ // The fit is the INITIAL view, not the projection (see mapProjection.ts).
206
+ const fit = useMemo(
207
+ () => fitView(points.map((p) => p.p), VIEW_W, VIEW_H),
208
+ [points]
209
+ );
210
+ useEffect(() => {
211
+ if (!fit) return;
212
+ if (!touchedRef.current || view == null) setView(fit);
213
+ // `view` is deliberately absent from the deps: this effect exists to seed
214
+ // and re-fit, and re-running it on every pan would fight the user for the
215
+ // camera.
216
+ // eslint-disable-next-line react-hooks/exhaustive-deps
217
+ }, [fit]);
218
+
219
+ const { kMin, kMax } = useMemo(
220
+ () => (fit ? zoomLimits(fit.k, VIEW_H, BASEMAP_DETAIL_K) : { kMin: 1, kMax: 1 }),
221
+ [fit]
222
+ );
223
+
224
+ /** Colour buckets, in first-seen order so the legend is stable. */
225
+ const colorBuckets = useMemo(() => {
226
+ if (!colorField) return null;
227
+ const order: string[] = [];
228
+ const counts = new Map<string, number>();
229
+ for (const p of points) {
230
+ const k = p.colorKey ?? "";
231
+ if (!counts.has(k)) {
232
+ counts.set(k, 0);
233
+ order.push(k);
234
+ }
235
+ counts.set(k, (counts.get(k) ?? 0) + 1);
236
+ }
237
+ const named = order.filter((k) => k !== "");
238
+ const swatch = new Map<string, { fill: string; line: string }>();
239
+ named.forEach((k, i) => swatch.set(k, i < SERIES.length ? SERIES[i] : OVERFLOW));
240
+ return {
241
+ order,
242
+ counts,
243
+ swatch,
244
+ overflow: Math.max(0, named.length - SERIES.length),
245
+ };
246
+ }, [colorField, points]);
247
+
248
+ /** The size field's observed range across MAPPED points (not the whole table:
249
+ * the legend must describe the picture actually on screen). */
250
+ const sizeRange = useMemo(() => {
251
+ if (!sizeField) return null;
252
+ let min = Infinity;
253
+ let max = -Infinity;
254
+ let missing = 0;
255
+ for (const p of points) {
256
+ if (p.size == null) {
257
+ missing += 1;
258
+ continue;
259
+ }
260
+ min = Math.min(min, p.size);
261
+ max = Math.max(max, p.size);
262
+ }
263
+ if (!Number.isFinite(min)) return { min: 0, max: 0, missing, none: true };
264
+ return { min, max, missing, none: false };
265
+ }, [sizeField, points]);
266
+
267
+ /** A coarse pointer means a phone or tablet, where Google's free URL takes 3
268
+ * waypoints rather than 9. A media query, not user-agent sniffing. */
269
+ const coarsePointer = useMemo(
270
+ () =>
271
+ typeof window !== "undefined" &&
272
+ typeof window.matchMedia === "function" &&
273
+ window.matchMedia("(pointer: coarse)").matches,
274
+ []
275
+ );
276
+
277
+ /** Selected pins that can actually be routed, in a STABLE order (by pid) —
278
+ * a Set's iteration order must not be what decides a route. */
279
+ const routable = useMemo(
280
+ () =>
281
+ selectedPids.size < 2
282
+ ? []
283
+ : points
284
+ .filter((p) => selectedPids.has(p.pid) && isPlottable(p.lat, p.lon))
285
+ .sort((a, b) => a.pid - b.pid),
286
+ [points, selectedPids]
287
+ );
288
+ /** Selected records with no usable coordinate. Counted and shown, never
289
+ * folded silently into the stop total ([[no-unverifiable-aggregates]]). */
290
+ const unroutable = selectedPids.size - routable.length;
291
+
292
+ const plan = useMemo(() => {
293
+ if (!routeOn || routable.length < 2) return null;
294
+ const stops: GeoStop[] = routable.map((p) => ({ lat: p.lat, lon: p.lon }));
295
+ // Default origin: the WESTERNMOST stop. Deterministic, stable while the user
296
+ // pans, and sayable out loud — unlike "whatever ended up at index 0". The
297
+ // picker below overrides it.
298
+ let start = 0;
299
+ for (let i = 1; i < routable.length; i++)
300
+ if (routable[i].lon < routable[start].lon) start = i;
301
+ if (routeStartPid != null) {
302
+ const i = routable.findIndex((p) => p.pid === routeStartPid);
303
+ if (i >= 0) start = i;
304
+ }
305
+ const { order, km } = planRoute(stops, haversineKm, { start, roundTrip });
306
+ return {
307
+ ordered: order.map((i) => routable[i]),
308
+ km,
309
+ link: googleRouteUrl(order.map((i) => stops[i]), { roundTrip, coarsePointer }),
310
+ };
311
+ }, [routeOn, routable, roundTrip, routeStartPid, coarsePointer]);
312
+
313
+ const paint = useCallback(
314
+ (p: MapPoint) => {
315
+ if (!colorBuckets) return DEFAULT_PIN;
316
+ const k = p.colorKey ?? "";
317
+ if (k === "") return OVERFLOW;
318
+ return colorBuckets.swatch.get(k) ?? OVERFLOW;
319
+ },
320
+ [colorBuckets]
321
+ );
322
+
323
+ const radius = useCallback(
324
+ (p: MapPoint) => {
325
+ if (!sizeField || !sizeRange || sizeRange.none) return R_PLAIN;
326
+ return bubbleRadius(p.size, sizeRange.min, sizeRange.max, R_MIN, R_MAX, R_NULL);
327
+ },
328
+ [sizeField, sizeRange]
329
+ );
330
+
331
+ /** Client coords -> the SVG's own user-space coords (the viewBox scales). */
332
+ const localPoint = useCallback(
333
+ (clientX: number, clientY: number): Pt => {
334
+ if (!svgEl) return { x: 0, y: 0 };
335
+ const r = svgEl.getBoundingClientRect();
336
+ return {
337
+ x: ((clientX - r.left) / r.width) * VIEW_W,
338
+ y: ((clientY - r.top) / r.height) * VIEW_H,
339
+ };
340
+ },
341
+ [svgEl]
342
+ );
343
+
344
+ /**
345
+ * ⚠ Wheel zoom MUST be a native, non-passive listener. React registers
346
+ * `wheel` PASSIVELY at the root, so `e.preventDefault()` inside an `onWheel`
347
+ * prop is a silent no-op: the page scrolls out from under the map while you
348
+ * zoom, which reads as "the zoom is broken". Nothing in a screenshot shows
349
+ * this, and no assertion on the rendered DOM can see it either.
350
+ *
351
+ * The zoom maths itself is `zoomAt` — the function the gate already proves
352
+ * holds the point under the cursor still. Wave 8 re-derived that formula
353
+ * inline here, so the tested copy and the shipped copy were two copies.
354
+ */
355
+ useEffect(() => {
356
+ if (!svgEl) return;
357
+ const onWheelNative = (e: WheelEvent) => {
358
+ e.preventDefault();
359
+ const { x, y } = localPoint(e.clientX, e.clientY);
360
+ touchedRef.current = true;
361
+ setView((v) => (v ? zoomAt(v, Math.exp(-e.deltaY * 0.0016), x, y, kMin, kMax) : v));
362
+ };
363
+ svgEl.addEventListener("wheel", onWheelNative, { passive: false });
364
+ return () => svgEl.removeEventListener("wheel", onWheelNative);
365
+ }, [svgEl, localPoint, kMin, kMax]);
366
+
367
+ /** Zoom about the viewport centre — the button and keyboard gesture, where
368
+ * there is no cursor to hold still. */
369
+ const zoomBy = useCallback(
370
+ (factor: number) => {
371
+ touchedRef.current = true;
372
+ setView((v) => (v ? zoomAt(v, factor, VIEW_W / 2, VIEW_H / 2, kMin, kMax) : v));
373
+ },
374
+ [kMin, kMax]
375
+ );
376
+
377
+ const doFit = useCallback(() => {
378
+ touchedRef.current = false;
379
+ setView(fit);
380
+ }, [fit]);
381
+
382
+ const panBy = useCallback((dx: number, dy: number) => {
383
+ touchedRef.current = true;
384
+ setView((v) => (v ? { ...v, tx: v.tx + dx, ty: v.ty + dy } : v));
385
+ }, []);
386
+
387
+ /** The camera had NO keyboard path at all before wave 9 — scroll wheel only,
388
+ * which is unusable without a mouse and unreachable for anyone driving the
389
+ * page from the keyboard. */
390
+ const onFrameKeyDown = useCallback(
391
+ (e: ReactKeyboardEvent<HTMLDivElement>) => {
392
+ const step = e.shiftKey ? 160 : 60;
393
+ switch (e.key) {
394
+ case "ArrowLeft": panBy(step, 0); break;
395
+ case "ArrowRight": panBy(-step, 0); break;
396
+ case "ArrowUp": panBy(0, step); break;
397
+ case "ArrowDown": panBy(0, -step); break;
398
+ case "+": case "=": zoomBy(1.6); break;
399
+ case "-": case "_": zoomBy(1 / 1.6); break;
400
+ case "0": doFit(); break;
401
+ // Esc abandons a marquee mid-drag. It falls THROUGH when there is no
402
+ // drag, so it keeps closing whatever the host has open.
403
+ case "Escape": if (!drag) return; setDrag(null); break;
404
+ default: return;
405
+ }
406
+ e.preventDefault();
407
+ },
408
+ [panBy, zoomBy, doFit, drag]
409
+ );
410
+
411
+ const onPointerDown = useCallback(
412
+ (e: ReactPointerEvent<SVGSVGElement>) => {
413
+ if (e.button !== 0 || !view) return;
414
+ const { x, y } = localPoint(e.clientX, e.clientY);
415
+ movedRef.current = false;
416
+ e.currentTarget.setPointerCapture(e.pointerId);
417
+ // Shift (or Ctrl/Cmd) turns the drag into a SELECTION rectangle; a plain
418
+ // drag pans. Both gestures are on the same button because a map that
419
+ // needs a mode toggle to select is a map people never select on.
420
+ if (e.shiftKey || e.ctrlKey || e.metaKey) setDrag({ x0: x, y0: y, x1: x, y1: y });
421
+ else panRef.current = { x, y, tx: view.tx, ty: view.ty };
422
+ },
423
+ [view, localPoint]
424
+ );
425
+
426
+ const onPointerMove = useCallback(
427
+ (e: ReactPointerEvent<SVGSVGElement>) => {
428
+ const { x, y } = localPoint(e.clientX, e.clientY);
429
+ if (drag) {
430
+ movedRef.current = true;
431
+ setDrag((d) => (d ? { ...d, x1: x, y1: y } : d));
432
+ return;
433
+ }
434
+ const pan = panRef.current;
435
+ if (!pan) return;
436
+ if (Math.abs(x - pan.x) + Math.abs(y - pan.y) > 2) movedRef.current = true;
437
+ touchedRef.current = true;
438
+ setView((v) => (v ? { ...v, tx: pan.tx + (x - pan.x), ty: pan.ty + (y - pan.y) } : v));
439
+ },
440
+ [drag, localPoint]
441
+ );
442
+
443
+ const onPointerUp = useCallback(
444
+ (e: ReactPointerEvent<SVGSVGElement>) => {
445
+ if (drag && view) {
446
+ const r = normRect(drag.x0, drag.y0, drag.x1, drag.y1);
447
+ // A rectangle smaller than a few px is a mis-click, not a selection —
448
+ // clearing the user's set on a stray shift-click would be its own bug.
449
+ if (r.x1 - r.x0 > 3 && r.y1 - r.y0 > 3) {
450
+ const hits: number[] = [];
451
+ for (const p of points) {
452
+ const s = toScreen(p.p, view);
453
+ if (s.x >= r.x0 && s.x <= r.x1 && s.y >= r.y0 && s.y <= r.y1) hits.push(p.pid);
454
+ }
455
+ onSelectPids(hits, e.altKey ? "add" : "replace");
456
+ }
457
+ setDrag(null);
458
+ }
459
+ panRef.current = null;
460
+ if (e.currentTarget.hasPointerCapture(e.pointerId))
461
+ e.currentTarget.releasePointerCapture(e.pointerId);
462
+ },
463
+ [drag, view, points, onSelectPids]
464
+ );
465
+
466
+ if (!view || !fit) {
467
+ return (
468
+ <div className="cg-mode-empty">
469
+ No records with a location to map yet.
470
+ {noCoords > 0 &&
471
+ ` ${noCoords.toLocaleString()} matching record${noCoords === 1 ? "" : "s"} have no location.`}
472
+ </div>
473
+ );
474
+ }
475
+
476
+ const hovered = hoverPid != null ? points.find((p) => p.pid === hoverPid) : undefined;
477
+ const rect = drag ? normRect(drag.x0, drag.y0, drag.x1, drag.y1) : null;
478
+ const tf = `translate(${view.tx.toFixed(2)} ${view.ty.toFixed(2)}) scale(${view.k.toFixed(6)})`;
479
+ // Strokes live in the transformed group, so they are pre-divided by the zoom.
480
+ // ⚠ This is the map's ONE stroke mechanism — see the note over `hairline` in
481
+ // mapProjection.ts. No `.cg-map*` rule may add `vector-effect:
482
+ // non-scaling-stroke` on top; that double-cancel is the wave-9 blur bug and
483
+ // `scalingConflicts()` gates the stylesheet against it.
484
+ const hair = (w: number) => hairline(w, view.k);
485
+ const gratOpacity = graticuleOpacity(view.k);
486
+ // I18 — the Google hand-off. Our vendored basemap is honest to roughly metro
487
+ // scale; below that the answer is a LINK, not 270 KB gz of tile renderer plus
488
+ // a hosted planet file. Nothing is fetched and no coordinate leaves the page
489
+ // unless the user deliberately clicks.
490
+ const centre = unproject(fromScreen({ x: VIEW_W / 2, y: VIEW_H / 2 }, view));
491
+ const areaUrl = googleMapsUrl(centre.lat, centre.lon, googleZoomForK(view.k));
492
+ // A single selected pin gets its own exact hand-off. Selection is persistent,
493
+ // unlike hover — and the hover card must stay pointer-events:none, so a link
494
+ // could never live in it without becoming a click trap.
495
+ const solo = selectedPids.size === 1 ? points.find((p) => selectedPids.has(p.pid)) : undefined;
496
+ const sizeLegend = sizeRange && !sizeRange.none && sizeRange.max > sizeRange.min
497
+ ? [sizeRange.min, (sizeRange.min + sizeRange.max) / 2, sizeRange.max]
498
+ : null;
499
+
500
+ return (
501
+ <div className="cg-mapview">
502
+ <div className="cg-map-bar">
503
+ {/* The count is "N of M", never a bare N: a pin can only be drawn for a
504
+ row the host geocoded, and a lone "1,402" silently redefines the
505
+ toolbar's 1,550 ([[no-unverifiable-aggregates]]). */}
506
+ <span className="cg-cal-note cg-map-count">
507
+ <strong>{points.length.toLocaleString()}</strong> of{" "}
508
+ {rows.length.toLocaleString()} mapped · pinned at each customer's address
509
+ </span>
510
+ {noCoords > 0 && (
511
+ <span
512
+ className="cg-cal-nodate"
513
+ title={`${noCoords.toLocaleString()} matching record${noCoords === 1 ? "" : "s"} have no geocoded location and are not on the map. They remain in the Grid and List views.`}
514
+ >
515
+ {noCoords.toLocaleString()} matching record{noCoords === 1 ? " has" : "s have"} no location
516
+ </span>
517
+ )}
518
+ {/* One selected pin -> the exact geocode, handed off to Google. By
519
+ lat/lon and never by name: a name search can resolve somewhere else,
520
+ and then this link and our pin disagree about where a customer is.
521
+ rel="noopener noreferrer" strips the Referer, so Google never learns
522
+ which tenant or deployment the click came from. */}
523
+ {solo && (
524
+ <span className="cg-map-go">
525
+ <a
526
+ href={googleMapsUrl(solo.lat, solo.lon)}
527
+ target="_blank"
528
+ rel="noopener noreferrer"
529
+ title={`Open ${solo.title} in Google Maps at ${solo.lat.toFixed(5)}, ${solo.lon.toFixed(5)} (new tab)`}
530
+ >
531
+ {solo.title || "Selected pin"} in Google Maps
532
+ </a>
533
+ <a
534
+ href={googleDirectionsUrl(solo.lat, solo.lon)}
535
+ target="_blank"
536
+ rel="noopener noreferrer"
537
+ title={`Directions to ${solo.title} (new tab)`}
538
+ >
539
+ Directions
540
+ </a>
541
+ </span>
542
+ )}
543
+ {/* I18-R — the route planner. Appears only with a multi-pin selection, so
544
+ it is mutually exclusive with the single-pin links above and the bar
545
+ never carries both. */}
546
+ {selectedPids.size >= 2 && (
547
+ <span className="cg-map-route">
548
+ {!routeOn ? (
549
+ <button
550
+ type="button"
551
+ className="cg-map-route-go"
552
+ onClick={() => setRouteOn(true)}
553
+ disabled={routable.length < 2}
554
+ title={
555
+ routable.length < 2
556
+ ? "At least two selected records need a location to plan a route"
557
+ : "Order these stops into a route"
558
+ }
559
+ >
560
+ Plan route ({routable.length.toLocaleString()} stops)
561
+ </button>
562
+ ) : (
563
+ plan && (
564
+ <>
565
+ <span className="cg-map-route-sum">
566
+ <strong>{plan.ordered.length.toLocaleString()}</strong> stops ·{" "}
567
+ {Math.round(plan.km).toLocaleString()} km
568
+ {/* ⚠ Never call this a driving distance, and never derive a
569
+ time from it: it is the sum of straight lines. Saying so
570
+ is the difference between a useful estimate and a lie. */}
571
+ <span className="cg-map-route-note"> straight-line, not driving distance</span>
572
+ </span>
573
+ <label className="cg-map-route-opt">
574
+ Start
575
+ {/* ⚠ `value` is always set — a <select> without one renders its
576
+ FIRST option regardless of state. It mirrors the ACTUAL
577
+ origin, so the westernmost default shows itself too. */}
578
+ <select
579
+ className="cg-map-route-start"
580
+ value={String(plan.ordered[0].pid)}
581
+ onChange={(e) => setRouteStartPid(Number(e.target.value))}
582
+ >
583
+ {routable.map((p) => (
584
+ <option key={p.pid} value={p.pid}>
585
+ {p.title || `#${p.pid}`}
586
+ </option>
587
+ ))}
588
+ </select>
589
+ </label>
590
+ <label className="cg-map-route-opt">
591
+ <input
592
+ type="checkbox"
593
+ checked={roundTrip}
594
+ onChange={(e) => setRoundTrip(e.target.checked)}
595
+ />
596
+ Return to start
597
+ </label>
598
+ {plan.link && (
599
+ <a
600
+ href={plan.link.url}
601
+ target="_blank"
602
+ rel="noopener noreferrer"
603
+ title="Open this route in Google Maps for driving directions (new tab)"
604
+ >
605
+ Open route in Google Maps
606
+ {/* The free URL takes 9 waypoints on desktop and 3 on a
607
+ phone. When the route is longer, SAY which part rides. */}
608
+ {plan.link.used < plan.ordered.length &&
609
+ ` (first ${plan.link.used} of ${plan.ordered.length})`}
610
+ </a>
611
+ )}
612
+ <button
613
+ type="button"
614
+ className="cg-map-route-go"
615
+ onClick={() => {
616
+ setRouteOn(false);
617
+ setRouteStartPid(null);
618
+ }}
619
+ >
620
+ Clear
621
+ </button>
622
+ </>
623
+ )
624
+ )}
625
+ {unroutable > 0 && (
626
+ <span className="cg-cal-nodate">
627
+ {unroutable.toLocaleString()} selected record{unroutable === 1 ? " has" : "s have"} no
628
+ location and cannot be routed
629
+ </span>
630
+ )}
631
+ </span>
632
+ )}
633
+ <span className="cg-map-hint">
634
+ Scroll to zoom · drag to pan · shift-drag to select
635
+ </span>
636
+ </div>
637
+ <div
638
+ className="cg-map-frame"
639
+ tabIndex={0}
640
+ role="group"
641
+ aria-label="Map canvas. Arrow keys pan, plus and minus zoom, zero fits to data."
642
+ onKeyDown={onFrameKeyDown}
643
+ >
644
+ <svg
645
+ ref={setSvgEl}
646
+ viewBox={`0 0 ${VIEW_W} ${VIEW_H}`}
647
+ preserveAspectRatio="xMidYMid meet"
648
+ className={"cg-map-svg" + (drag ? " is-selecting" : "")}
649
+ role="img"
650
+ aria-label={`Map of ${points.length.toLocaleString()} customers`}
651
+ onPointerDown={onPointerDown}
652
+ onPointerMove={onPointerMove}
653
+ onPointerUp={onPointerUp}
654
+ onPointerCancel={onPointerUp}
655
+ >
656
+ <rect className="cg-map-water" x={0} y={0} width={VIEW_W} height={VIEW_H} />
657
+ <g transform={tf}>
658
+ {/* Graticule every 10 degrees. It earns its place zoomed OUT, where
659
+ it is the only thing giving scale; once the real state borders
660
+ arrive it would be a second line system fighting the first, so it
661
+ fades away before they take over. Skipped entirely at zero
662
+ opacity — ~36 invisible lines are still 36 nodes to lay out. */}
663
+ {gratOpacity > 0.01 && (
664
+ <g className="cg-map-grat" strokeWidth={hair(0.6)} opacity={gratOpacity}>
665
+ {Array.from({ length: 17 }, (_, i) => {
666
+ const y = project(0, -80 + i * 10).y;
667
+ return <line key={`p${i}`} x1={0} y1={y} x2={WORLD} y2={y} />;
668
+ })}
669
+ {Array.from({ length: 19 }, (_, i) => {
670
+ const x = project(-180 + i * 20, 0).x;
671
+ return <line key={`m${i}`} x1={x} y1={0} x2={x} y2={WORLD} />;
672
+ })}
673
+ </g>
674
+ )}
675
+ {/* One path, every US state ring. Filled AND stroked, so interior
676
+ state borders come free from the same geometry — no second pass
677
+ and no chance of the borders disagreeing with the coastline.
678
+ 0.9 rather than wave 8's 1.1: that weight was tuned for a single
679
+ lone coastline, and it reads heavy once ~169 rings share it. */}
680
+ <path className="cg-map-land" d={LAND_PATH} strokeWidth={hair(0.9)} />
681
+ {LAKE_PATHS.map((d, i) => (
682
+ <path key={i} className="cg-map-lake" d={d} strokeWidth={hair(0.8)} />
683
+ ))}
684
+ {/* The planned path, UNDER the pins so it never hides a stop. Inside
685
+ the zoomed group, so it pans and scales with the geography; the
686
+ stroke is pre-divided by k like every other line here. */}
687
+ {plan && plan.ordered.length > 1 && (
688
+ <polyline
689
+ className="cg-map-route-line"
690
+ strokeWidth={hair(1.8)}
691
+ // ⚠ The dash MUST be pre-divided by the zoom too — a CSS
692
+ // dasharray is in user units and would scale with the group,
693
+ // turning the route into a few disconnected strokes.
694
+ strokeDasharray={dashPattern(5, 4, view.k)}
695
+ points={
696
+ plan.ordered.map((p) => `${p.p.x},${p.p.y}`).join(" ") +
697
+ (roundTrip ? ` ${plan.ordered[0].p.x},${plan.ordered[0].p.y}` : "")
698
+ }
699
+ />
700
+ )}
701
+ {points.map((p) => {
702
+ const c = paint(p);
703
+ const on = selectedPids.has(p.pid);
704
+ const r = radius(p) / view.k;
705
+ return (
706
+ <circle
707
+ key={p.pid}
708
+ className={
709
+ "cg-map-pin" +
710
+ (hoverPid === p.pid ? " is-hover" : "") +
711
+ (on ? " is-selected" : "")
712
+ }
713
+ cx={p.p.x}
714
+ cy={p.p.y}
715
+ r={hoverPid === p.pid ? r * 1.28 : r}
716
+ fill={c.fill}
717
+ stroke={on ? "#202433" : c.line}
718
+ strokeWidth={hair(on ? 2 : 1)}
719
+ role="button"
720
+ tabIndex={0}
721
+ aria-label={`Open ${p.title}`}
722
+ onMouseEnter={() => setHoverPid(p.pid)}
723
+ onMouseLeave={() => setHoverPid((h) => (h === p.pid ? null : h))}
724
+ onFocus={() => setHoverPid(p.pid)}
725
+ onBlur={() => setHoverPid((h) => (h === p.pid ? null : h))}
726
+ onClick={() => {
727
+ if (movedRef.current) return; // a finished pan/box is not a click
728
+ onOpen(p.pid);
729
+ }}
730
+ onKeyDown={(e) => {
731
+ if (e.key !== "Enter" && e.key !== " ") return;
732
+ e.preventDefault();
733
+ onOpen(p.pid);
734
+ }}
735
+ >
736
+ <title>{p.title}</title>
737
+ </circle>
738
+ );
739
+ })}
740
+ </g>
741
+ {/* I18 — the hover card. Wave 8 painted the title alone; a map whose
742
+ pins carry a colour and a size encoding should say what they ARE.
743
+ ⚠ Every value goes through `formatDisplay`, the SAME formatter the
744
+ grid cells use, so a currency, a percentage or a date can never
745
+ read one way on the map and another way in the table.
746
+ Drawn in SVG screen space rather than as an HTML overlay: the
747
+ viewBox letterboxes under preserveAspectRatio, so an HTML card
748
+ would need the rendered scale re-derived, and this needs no
749
+ conversion at all. pointer-events stay off — a card that can
750
+ swallow the next click is a scar this codebase already carries. */}
751
+ {/* Stop numbers, in SCREEN space so they stay legible at every zoom.
752
+ pointer-events off: a badge sitting over a pin must not steal the
753
+ click that opens the record — re-rooting the route is the "Start"
754
+ picker's job, where it is visible and reversible. */}
755
+ {plan &&
756
+ plan.ordered.map((p, i) => {
757
+ const s = toScreen(p.p, view);
758
+ return (
759
+ <g
760
+ className="cg-map-stopno"
761
+ key={p.pid}
762
+ transform={`translate(${s.x.toFixed(2)} ${(s.y - 13).toFixed(2)})`}
763
+ >
764
+ <circle r={7.5} />
765
+ <text textAnchor="middle" dy="3.4">{i + 1}</text>
766
+ </g>
767
+ );
768
+ })}
769
+ {hovered && (() => {
770
+ const s = toScreen(hovered.p, view);
771
+ const lines: string[] = [];
772
+ if (colorField)
773
+ lines.push(`${colorField.label}: ${formatDisplay(colorField, hovered.colorVal) || "—"}`);
774
+ if (sizeField)
775
+ lines.push(`${sizeField.label}: ${formatDisplay(sizeField, hovered.sizeVal) || "—"}`);
776
+ const title = hovered.title || "(untitled)";
777
+ // Inter's average advance at 11.5px. An estimate, deliberately
778
+ // generous: too wide is a slightly roomy card, too narrow is text
779
+ // spilling past its own background.
780
+ const w = Math.max(title.length, ...lines.map((l) => l.length)) * 6.2 + 20;
781
+ const h = 21 + lines.length * 14;
782
+ const b = cardBox(s.x, s.y, w, h, VIEW_W, VIEW_H);
783
+ return (
784
+ <g className="cg-map-card" aria-hidden="true">
785
+ <rect x={b.x} y={b.y} width={w} height={h} rx={5} />
786
+ <text className="cg-map-card-t" x={b.x + 10} y={b.y + 15}>{title}</text>
787
+ {lines.map((l, i) => (
788
+ <text className="cg-map-card-l" key={i} x={b.x + 10} y={b.y + 15 + 14 * (i + 1)}>
789
+ {l}
790
+ </text>
791
+ ))}
792
+ </g>
793
+ );
794
+ })()}
795
+ {rect && (
796
+ <rect
797
+ className="cg-map-marquee"
798
+ x={rect.x0}
799
+ y={rect.y0}
800
+ width={rect.x1 - rect.x0}
801
+ height={rect.y1 - rect.y0}
802
+ />
803
+ )}
804
+ </svg>
805
+
806
+ {/* --- I18 — the camera controls. Zoom in / zoom out / fit to data, the
807
+ affordance every map has, replacing wave 8's link-button-in-a-text-bar.
808
+ OUTSIDE the <svg> deliberately: a mousedown on a button inside it would
809
+ begin a pan. Icons are SVG strokes — no emoji, no glyph font. The zoom
810
+ buttons DISABLE at the limits, which is also how the honesty cap on
811
+ zoom-in makes itself visible instead of just feeling stuck. --- */}
812
+ <div className="cg-map-ctl" role="group" aria-label="Map camera">
813
+ <button
814
+ type="button"
815
+ className="cg-map-ctl-b"
816
+ // A greyed button with no reason reads as broken, not as honest —
817
+ // and filtering to a single metro can push the FITTED zoom past the
818
+ // cap, so this can be disabled the instant the view opens. Say why,
819
+ // and point at the thing that does go further.
820
+ title={
821
+ view.k >= kMax * (1 - 1e-9)
822
+ ? "Zoom in — at the limit. The basemap's detail ends here; use Open in Google Maps for street level."
823
+ : "Zoom in (+)"
824
+ }
825
+ aria-label="Zoom in"
826
+ disabled={view.k >= kMax * (1 - 1e-9)}
827
+ onClick={() => zoomBy(1.6)}
828
+ >
829
+ <svg viewBox="0 0 16 16" aria-hidden="true"><path d="M8 3.6v8.8M3.6 8h8.8" /></svg>
830
+ </button>
831
+ <button
832
+ type="button"
833
+ className="cg-map-ctl-b"
834
+ title="Zoom out (−)"
835
+ aria-label="Zoom out"
836
+ disabled={view.k <= kMin * (1 + 1e-9)}
837
+ onClick={() => zoomBy(1 / 1.6)}
838
+ >
839
+ <svg viewBox="0 0 16 16" aria-hidden="true"><path d="M3.6 8h8.8" /></svg>
840
+ </button>
841
+ <button
842
+ type="button"
843
+ className="cg-map-ctl-b"
844
+ title="Fit to data (0)"
845
+ aria-label="Fit to data"
846
+ onClick={doFit}
847
+ >
848
+ <svg viewBox="0 0 16 16" aria-hidden="true">
849
+ <path d="M2.6 5.8V2.6h3.2M10.2 2.6h3.2v3.2M13.4 10.2v3.2h-3.2M5.8 13.4H2.6v-3.2" />
850
+ <path d="M6.6 8h2.8M8 6.6v2.8" />
851
+ </svg>
852
+ </button>
853
+ {/* The street-level hand-off, aimed at whatever is on screen right
854
+ now. This is what makes the zoom cap honest rather than merely
855
+ restrictive: the map stops where its geometry stops, and points at
856
+ something that does not. */}
857
+ <a
858
+ className="cg-map-ctl-b"
859
+ href={areaUrl}
860
+ target="_blank"
861
+ rel="noopener noreferrer"
862
+ title="Open this area in Google Maps (new tab)"
863
+ aria-label="Open this area in Google Maps"
864
+ >
865
+ <svg viewBox="0 0 16 16" aria-hidden="true">
866
+ <path d="M7 3.8H4.3a.9.9 0 0 0-.9.9v6.9a.9.9 0 0 0 .9.9h6.9a.9.9 0 0 0 .9-.9V9.2" />
867
+ <path d="M8.6 3.4h4v4M12.6 3.4 7.4 8.6" />
868
+ </svg>
869
+ </a>
870
+ </div>
871
+
872
+ {/* --- legends (I3/I5). Floated over the map, never in the flow. --- */}
873
+ {(colorBuckets || sizeLegend) && (
874
+ <div className="cg-map-legends">
875
+ {colorField && colorBuckets && (
876
+ <div className="cg-map-legend">
877
+ <div className="cg-map-legend-t">{colorField.label}</div>
878
+ {colorBuckets.order.slice(0, SERIES.length + 1).map((k) => {
879
+ const c = k === "" ? OVERFLOW : colorBuckets.swatch.get(k) ?? OVERFLOW;
880
+ return (
881
+ <div key={k || "(blank)"} className="cg-map-legend-row">
882
+ <span
883
+ className="cg-map-swatch"
884
+ style={{ background: c.fill, borderColor: c.line }}
885
+ />
886
+ <span className="cg-map-legend-k">{k === "" ? "(blank)" : k}</span>
887
+ <span className="cg-map-legend-n">
888
+ {(colorBuckets.counts.get(k) ?? 0).toLocaleString()}
889
+ </span>
890
+ </div>
891
+ );
892
+ })}
893
+ {colorBuckets.overflow > 0 && (
894
+ <div className="cg-map-legend-note">
895
+ {colorBuckets.overflow.toLocaleString()} further value
896
+ {colorBuckets.overflow === 1 ? " is" : "s are"} drawn grey — the palette
897
+ holds {SERIES.length} colours and reusing one would make two values look
898
+ like the same value.
899
+ </div>
900
+ )}
901
+ </div>
902
+ )}
903
+ {sizeField && sizeRange && (
904
+ <div className="cg-map-legend">
905
+ <div className="cg-map-legend-t">{sizeField.label}</div>
906
+ {sizeLegend ? (
907
+ <div className="cg-map-sizerow">
908
+ {sizeLegend.map((v, i) => {
909
+ const r = bubbleRadius(v, sizeRange.min, sizeRange.max, R_MIN, R_MAX, R_NULL);
910
+ return (
911
+ <span key={i} className="cg-map-sizeitem">
912
+ <svg width={R_MAX * 2 + 2} height={R_MAX * 2 + 2} aria-hidden>
913
+ <circle
914
+ cx={R_MAX + 1}
915
+ cy={R_MAX + 1}
916
+ r={r}
917
+ fill={DEFAULT_PIN.fill}
918
+ stroke={DEFAULT_PIN.line}
919
+ />
920
+ </svg>
921
+ <span className="cg-map-legend-k">{formatDisplay(sizeField, v)}</span>
922
+ </span>
923
+ );
924
+ })}
925
+ </div>
926
+ ) : (
927
+ <div className="cg-map-legend-note">
928
+ Every mapped record has the same {sizeField.label.toLowerCase()}, so the
929
+ bubbles cannot differ in size.
930
+ </div>
931
+ )}
932
+ {sizeRange.missing > 0 && (
933
+ <div className="cg-map-legend-note">
934
+ {sizeRange.missing.toLocaleString()} mapped record
935
+ {sizeRange.missing === 1 ? " has" : "s have"} no value — drawn at the
936
+ smallest dot, never removed from the map.
937
+ </div>
938
+ )}
939
+ </div>
940
+ )}
941
+ </div>
942
+ )}
943
+ </div>
944
+ </div>
945
+ );
946
+ }
web/src/customer-grid/OverlaySurface.tsx CHANGED
@@ -1,355 +1,355 @@
1
- import {
2
- createContext,
3
- useCallback,
4
- useContext,
5
- useEffect,
6
- useId,
7
- useLayoutEffect,
8
- useMemo,
9
- useRef,
10
- useState,
11
- } from "react";
12
- import type {
13
- AriaRole,
14
- CSSProperties,
15
- KeyboardEvent as ReactKeyboardEvent,
16
- ReactNode,
17
- RefObject,
18
- } from "react";
19
- import { createPortal } from "react-dom";
20
- import { computeOverlayPosition } from "./overlayPlacement";
21
- import type { Placement as OverlayPlacement } from "./overlayPlacement";
22
-
23
- export interface AnchorRect {
24
- left: number;
25
- top: number;
26
- right: number;
27
- bottom: number;
28
- width: number;
29
- height: number;
30
- }
31
-
32
- type Anchor = HTMLElement | AnchorRect;
33
- /**
34
- * Wave-9 I14 adds `right-start`: the panel sits BESIDE the anchor (top edges aligned), not
35
- * under it — the owner's "flyout to the RIGHT". It is a real placement rather than a margin
36
- * hack because the views rail is a 188px column at the left edge of an iframe, and a panel
37
- * absolutely-positioned inside `.cg-views` would be clipped by the rail it escapes; every
38
- * other overlay here is already body-level and fixed for the same reason.
39
- *
40
- * The placement ARITHMETIC moved to overlayPlacement.ts with that change, so the collision
41
- * maths is gated rather than eyeballed (verify_overlay.py).
42
- */
43
- type Placement = OverlayPlacement;
44
-
45
- interface OverlayStack {
46
- register: (id: string) => () => void;
47
- isTop: (id: string) => boolean;
48
- }
49
-
50
- const OverlayStackContext = createContext<OverlayStack | null>(null);
51
-
52
- const FOCUSABLE = [
53
- "[data-overlay-autofocus]",
54
- "button:not([disabled])",
55
- "input:not([disabled])",
56
- "select:not([disabled])",
57
- "textarea:not([disabled])",
58
- "[href]",
59
- '[tabindex]:not([tabindex="-1"])',
60
- ].join(",");
61
-
62
- export function OverlayProvider({ children }: { children: ReactNode }) {
63
- const stack = useRef<string[]>([]);
64
- const api = useMemo<OverlayStack>(
65
- () => ({
66
- register: (id) => {
67
- stack.current = [...stack.current.filter((item) => item !== id), id];
68
- return () => {
69
- stack.current = stack.current.filter((item) => item !== id);
70
- };
71
- },
72
- isTop: (id) => stack.current.at(-1) === id,
73
- }),
74
- []
75
- );
76
- return (
77
- <OverlayStackContext.Provider value={api}>
78
- {children}
79
- </OverlayStackContext.Provider>
80
- );
81
- }
82
-
83
- /**
84
- * ⚠ NULL IS A REACHABLE VALUE HERE, and it must not be fatal.
85
- *
86
- * Half the call sites pass `someRef.current`, which is legitimately null on the render
87
- * before the ref attaches, and `Anchor` does not include null — so every one of them was
88
- * one ordering accident away from `"getBoundingClientRect" in null`, a TypeError thrown
89
- * during RENDER, which unmounts the whole tree. That is not a hypothetical: it is what
90
- * `_qa_live_rail.py` reproduced on the shipped build when the saved-view menu's anchor
91
- * came back null (2026-08-04).
92
- *
93
- * A missing anchor is a positioning problem, not a reason to lose the application. The
94
- * panel degrades to a zero-rect at the viewport origin — visible, dismissible, obviously
95
- * wrong — while the root causes stay fixable at their own call sites.
96
- */
97
- const NO_RECT: AnchorRect = { left: 0, top: 0, right: 0, bottom: 0, width: 0, height: 0 };
98
-
99
- function anchorRect(anchor: Anchor | null | undefined): AnchorRect {
100
- if (!anchor) return NO_RECT;
101
- if ("getBoundingClientRect" in anchor) {
102
- const rect = anchor.getBoundingClientRect();
103
- return {
104
- left: rect.left,
105
- top: rect.top,
106
- right: rect.right,
107
- bottom: rect.bottom,
108
- width: rect.width,
109
- height: rect.height,
110
- };
111
- }
112
- return anchor;
113
- }
114
-
115
- function samePosition(a: CSSProperties, b: CSSProperties): boolean {
116
- return (
117
- a.left === b.left &&
118
- a.top === b.top &&
119
- a.maxWidth === b.maxWidth &&
120
- a.maxHeight === b.maxHeight &&
121
- a.visibility === b.visibility
122
- );
123
- }
124
-
125
- interface OverlayLayerOptions {
126
- panelRef: RefObject<HTMLElement | null>;
127
- onDismiss: () => void;
128
- dismissOnOutside?: boolean;
129
- initialFocus?: "first" | "none" | string;
130
- restoreFocus?: boolean;
131
- trapFocus?: boolean;
132
- outsideElements?: Array<HTMLElement | null>;
133
- }
134
-
135
- /** Shared dismissal/focus contract for anchored menus and fixed drawers. */
136
- // oxlint-disable-next-line react/only-export-components -- shares the private overlay stack context.
137
- export function useOverlayLayer({
138
- panelRef,
139
- onDismiss,
140
- dismissOnOutside = true,
141
- initialFocus = "first",
142
- restoreFocus = true,
143
- trapFocus = false,
144
- outsideElements = [],
145
- }: OverlayLayerOptions): void {
146
- const id = useId();
147
- const stack = useContext(OverlayStackContext);
148
- const dismissRef = useRef(onDismiss);
149
- const outsideRef = useRef(outsideElements);
150
- dismissRef.current = onDismiss;
151
- outsideRef.current = outsideElements;
152
-
153
- useEffect(() => stack?.register(id), [id, stack]);
154
-
155
- useEffect(() => {
156
- const onPointerDown = (event: PointerEvent) => {
157
- if (!dismissOnOutside || (stack && !stack.isTop(id))) return;
158
- const target = event.target as Node | null;
159
- if (!target || panelRef.current?.contains(target)) return;
160
- if (outsideRef.current.some((element) => element?.contains(target))) return;
161
- if (
162
- document.activeElement instanceof HTMLElement &&
163
- panelRef.current?.contains(document.activeElement)
164
- )
165
- document.activeElement.blur();
166
- dismissRef.current();
167
- };
168
- const onKeyDown = (event: KeyboardEvent) => {
169
- if (stack && !stack.isTop(id)) return;
170
- if (event.key === "Escape") {
171
- event.preventDefault();
172
- event.stopPropagation();
173
- if (
174
- document.activeElement instanceof HTMLElement &&
175
- panelRef.current?.contains(document.activeElement)
176
- )
177
- document.activeElement.blur();
178
- dismissRef.current();
179
- return;
180
- }
181
- if (event.key !== "Tab" || !trapFocus) return;
182
- const focusable = Array.from(
183
- panelRef.current?.querySelectorAll<HTMLElement>(FOCUSABLE) ?? []
184
- ).filter((element) => element.getClientRects().length > 0);
185
- if (!focusable.length) {
186
- event.preventDefault();
187
- panelRef.current?.focus();
188
- return;
189
- }
190
- const first = focusable[0];
191
- const last = focusable.at(-1)!;
192
- const active = document.activeElement;
193
- if (event.shiftKey && (active === first || !panelRef.current?.contains(active))) {
194
- event.preventDefault();
195
- last.focus();
196
- } else if (!event.shiftKey && active === last) {
197
- event.preventDefault();
198
- first.focus();
199
- }
200
- };
201
- document.addEventListener("pointerdown", onPointerDown, true);
202
- document.addEventListener("keydown", onKeyDown, true);
203
- return () => {
204
- document.removeEventListener("pointerdown", onPointerDown, true);
205
- document.removeEventListener("keydown", onKeyDown, true);
206
- };
207
- }, [dismissOnOutside, id, panelRef, stack, trapFocus]);
208
-
209
- useLayoutEffect(() => {
210
- const previous =
211
- document.activeElement instanceof HTMLElement ? document.activeElement : null;
212
- const frame = window.requestAnimationFrame(() => {
213
- if (initialFocus === "none") return;
214
- const selector = initialFocus === "first" ? FOCUSABLE : initialFocus;
215
- const target = panelRef.current?.querySelector<HTMLElement>(selector);
216
- target?.focus({ preventScroll: true });
217
- });
218
- return () => {
219
- window.cancelAnimationFrame(frame);
220
- if (restoreFocus && previous?.isConnected) previous.focus({ preventScroll: true });
221
- };
222
- }, [initialFocus, panelRef, restoreFocus]);
223
- }
224
-
225
- export function BodyPortal({ children }: { children: ReactNode }) {
226
- return createPortal(children, document.body);
227
- }
228
-
229
- interface AnchoredOverlayProps {
230
- /** ⚠ Nullable BY DECLARATION as of 2026-08-04. Several call sites pass `ref.current`,
231
- * which is null before the ref attaches, and the old non-null type made that a
232
- * render-time TypeError instead of a type error. See `anchorRect`. */
233
- anchor: Anchor | null | undefined;
234
- className: string;
235
- children: ReactNode;
236
- onDismiss: () => void;
237
- placement?: Placement;
238
- role?: AriaRole;
239
- ariaLabel?: string;
240
- id?: string;
241
- initialFocus?: "first" | "none" | string;
242
- restoreFocus?: boolean;
243
- dismissOnOutside?: boolean;
244
- onKeyDown?: (event: ReactKeyboardEvent<HTMLElement>) => void;
245
- dataKind?: string;
246
- }
247
-
248
- /** Body-level fixed overlay with iframe-viewport collision handling. */
249
- export function AnchoredOverlay({
250
- anchor,
251
- className,
252
- children,
253
- onDismiss,
254
- placement = "bottom-start",
255
- role,
256
- ariaLabel,
257
- id,
258
- initialFocus = "first",
259
- restoreFocus = true,
260
- dismissOnOutside = true,
261
- onKeyDown,
262
- dataKind,
263
- }: AnchoredOverlayProps) {
264
- const panelRef = useRef<HTMLDivElement>(null);
265
- const [style, setStyle] = useState<CSSProperties>({
266
- position: "fixed",
267
- left: 0,
268
- top: 0,
269
- visibility: "hidden",
270
- });
271
- const anchorElement = anchor && "getBoundingClientRect" in anchor ? anchor : null;
272
-
273
- useOverlayLayer({
274
- panelRef,
275
- onDismiss,
276
- dismissOnOutside,
277
- initialFocus,
278
- restoreFocus,
279
- outsideElements: [anchorElement],
280
- });
281
-
282
- const updatePosition = useCallback(() => {
283
- const panel = panelRef.current;
284
- if (!panel) return;
285
- const target = anchorRect(anchor);
286
- const viewport = window.visualViewport;
287
- const viewportLeft = viewport?.offsetLeft ?? 0;
288
- const viewportTop = viewport?.offsetTop ?? 0;
289
- const viewportWidth = viewport?.width ?? window.innerWidth;
290
- const viewportHeight = viewport?.height ?? window.innerHeight;
291
- const measured = panel.getBoundingClientRect();
292
- // The arithmetic lives in overlayPlacement.ts so a gate can run it under node. This
293
- // reads the DOM, decides nothing.
294
- const placed = computeOverlayPosition({
295
- placement,
296
- target,
297
- panel: {
298
- width: Math.max(measured.width, panel.scrollWidth),
299
- height: Math.max(measured.height, panel.scrollHeight),
300
- },
301
- viewport: {
302
- left: viewportLeft,
303
- top: viewportTop,
304
- width: viewportWidth,
305
- height: viewportHeight,
306
- },
307
- });
308
- const next: CSSProperties = {
309
- position: "fixed",
310
- left: placed.left,
311
- top: placed.top,
312
- maxWidth: placed.maxWidth,
313
- maxHeight: placed.maxHeight,
314
- visibility: "visible",
315
- };
316
- setStyle((current) => (samePosition(current, next) ? current : next));
317
- }, [anchor, placement]);
318
-
319
- useLayoutEffect(() => {
320
- updatePosition();
321
- const frame = window.requestAnimationFrame(updatePosition);
322
- const observer = new ResizeObserver(updatePosition);
323
- if (panelRef.current) observer.observe(panelRef.current);
324
- if (anchorElement) observer.observe(anchorElement);
325
- window.addEventListener("resize", updatePosition);
326
- window.addEventListener("scroll", updatePosition, true);
327
- window.visualViewport?.addEventListener("resize", updatePosition);
328
- window.visualViewport?.addEventListener("scroll", updatePosition);
329
- return () => {
330
- window.cancelAnimationFrame(frame);
331
- observer.disconnect();
332
- window.removeEventListener("resize", updatePosition);
333
- window.removeEventListener("scroll", updatePosition, true);
334
- window.visualViewport?.removeEventListener("resize", updatePosition);
335
- window.visualViewport?.removeEventListener("scroll", updatePosition);
336
- };
337
- }, [anchorElement, updatePosition]);
338
-
339
- return (
340
- <BodyPortal>
341
- <div
342
- ref={panelRef}
343
- id={id}
344
- className={className}
345
- style={style}
346
- role={role}
347
- aria-label={ariaLabel}
348
- data-overlay-kind={dataKind}
349
- onKeyDown={onKeyDown}
350
- >
351
- {children}
352
- </div>
353
- </BodyPortal>
354
- );
355
- }
 
1
+ import {
2
+ createContext,
3
+ useCallback,
4
+ useContext,
5
+ useEffect,
6
+ useId,
7
+ useLayoutEffect,
8
+ useMemo,
9
+ useRef,
10
+ useState,
11
+ } from "react";
12
+ import type {
13
+ AriaRole,
14
+ CSSProperties,
15
+ KeyboardEvent as ReactKeyboardEvent,
16
+ ReactNode,
17
+ RefObject,
18
+ } from "react";
19
+ import { createPortal } from "react-dom";
20
+ import { computeOverlayPosition } from "./overlayPlacement";
21
+ import type { Placement as OverlayPlacement } from "./overlayPlacement";
22
+
23
+ export interface AnchorRect {
24
+ left: number;
25
+ top: number;
26
+ right: number;
27
+ bottom: number;
28
+ width: number;
29
+ height: number;
30
+ }
31
+
32
+ type Anchor = HTMLElement | AnchorRect;
33
+ /**
34
+ * Wave-9 I14 adds `right-start`: the panel sits BESIDE the anchor (top edges aligned), not
35
+ * under it — the owner's "flyout to the RIGHT". It is a real placement rather than a margin
36
+ * hack because the views rail is a 188px column at the left edge of an iframe, and a panel
37
+ * absolutely-positioned inside `.cg-views` would be clipped by the rail it escapes; every
38
+ * other overlay here is already body-level and fixed for the same reason.
39
+ *
40
+ * The placement ARITHMETIC moved to overlayPlacement.ts with that change, so the collision
41
+ * maths is gated rather than eyeballed (verify_overlay.py).
42
+ */
43
+ type Placement = OverlayPlacement;
44
+
45
+ interface OverlayStack {
46
+ register: (id: string) => () => void;
47
+ isTop: (id: string) => boolean;
48
+ }
49
+
50
+ const OverlayStackContext = createContext<OverlayStack | null>(null);
51
+
52
+ const FOCUSABLE = [
53
+ "[data-overlay-autofocus]",
54
+ "button:not([disabled])",
55
+ "input:not([disabled])",
56
+ "select:not([disabled])",
57
+ "textarea:not([disabled])",
58
+ "[href]",
59
+ '[tabindex]:not([tabindex="-1"])',
60
+ ].join(",");
61
+
62
+ export function OverlayProvider({ children }: { children: ReactNode }) {
63
+ const stack = useRef<string[]>([]);
64
+ const api = useMemo<OverlayStack>(
65
+ () => ({
66
+ register: (id) => {
67
+ stack.current = [...stack.current.filter((item) => item !== id), id];
68
+ return () => {
69
+ stack.current = stack.current.filter((item) => item !== id);
70
+ };
71
+ },
72
+ isTop: (id) => stack.current.at(-1) === id,
73
+ }),
74
+ []
75
+ );
76
+ return (
77
+ <OverlayStackContext.Provider value={api}>
78
+ {children}
79
+ </OverlayStackContext.Provider>
80
+ );
81
+ }
82
+
83
+ /**
84
+ * ⚠ NULL IS A REACHABLE VALUE HERE, and it must not be fatal.
85
+ *
86
+ * Half the call sites pass `someRef.current`, which is legitimately null on the render
87
+ * before the ref attaches, and `Anchor` does not include null — so every one of them was
88
+ * one ordering accident away from `"getBoundingClientRect" in null`, a TypeError thrown
89
+ * during RENDER, which unmounts the whole tree. That is not a hypothetical: it is what
90
+ * `_qa_live_rail.py` reproduced on the shipped build when the saved-view menu's anchor
91
+ * came back null (2026-08-04).
92
+ *
93
+ * A missing anchor is a positioning problem, not a reason to lose the application. The
94
+ * panel degrades to a zero-rect at the viewport origin — visible, dismissible, obviously
95
+ * wrong — while the root causes stay fixable at their own call sites.
96
+ */
97
+ const NO_RECT: AnchorRect = { left: 0, top: 0, right: 0, bottom: 0, width: 0, height: 0 };
98
+
99
+ function anchorRect(anchor: Anchor | null | undefined): AnchorRect {
100
+ if (!anchor) return NO_RECT;
101
+ if ("getBoundingClientRect" in anchor) {
102
+ const rect = anchor.getBoundingClientRect();
103
+ return {
104
+ left: rect.left,
105
+ top: rect.top,
106
+ right: rect.right,
107
+ bottom: rect.bottom,
108
+ width: rect.width,
109
+ height: rect.height,
110
+ };
111
+ }
112
+ return anchor;
113
+ }
114
+
115
+ function samePosition(a: CSSProperties, b: CSSProperties): boolean {
116
+ return (
117
+ a.left === b.left &&
118
+ a.top === b.top &&
119
+ a.maxWidth === b.maxWidth &&
120
+ a.maxHeight === b.maxHeight &&
121
+ a.visibility === b.visibility
122
+ );
123
+ }
124
+
125
+ interface OverlayLayerOptions {
126
+ panelRef: RefObject<HTMLElement | null>;
127
+ onDismiss: () => void;
128
+ dismissOnOutside?: boolean;
129
+ initialFocus?: "first" | "none" | string;
130
+ restoreFocus?: boolean;
131
+ trapFocus?: boolean;
132
+ outsideElements?: Array<HTMLElement | null>;
133
+ }
134
+
135
+ /** Shared dismissal/focus contract for anchored menus and fixed drawers. */
136
+ // oxlint-disable-next-line react/only-export-components -- shares the private overlay stack context.
137
+ export function useOverlayLayer({
138
+ panelRef,
139
+ onDismiss,
140
+ dismissOnOutside = true,
141
+ initialFocus = "first",
142
+ restoreFocus = true,
143
+ trapFocus = false,
144
+ outsideElements = [],
145
+ }: OverlayLayerOptions): void {
146
+ const id = useId();
147
+ const stack = useContext(OverlayStackContext);
148
+ const dismissRef = useRef(onDismiss);
149
+ const outsideRef = useRef(outsideElements);
150
+ dismissRef.current = onDismiss;
151
+ outsideRef.current = outsideElements;
152
+
153
+ useEffect(() => stack?.register(id), [id, stack]);
154
+
155
+ useEffect(() => {
156
+ const onPointerDown = (event: PointerEvent) => {
157
+ if (!dismissOnOutside || (stack && !stack.isTop(id))) return;
158
+ const target = event.target as Node | null;
159
+ if (!target || panelRef.current?.contains(target)) return;
160
+ if (outsideRef.current.some((element) => element?.contains(target))) return;
161
+ if (
162
+ document.activeElement instanceof HTMLElement &&
163
+ panelRef.current?.contains(document.activeElement)
164
+ )
165
+ document.activeElement.blur();
166
+ dismissRef.current();
167
+ };
168
+ const onKeyDown = (event: KeyboardEvent) => {
169
+ if (stack && !stack.isTop(id)) return;
170
+ if (event.key === "Escape") {
171
+ event.preventDefault();
172
+ event.stopPropagation();
173
+ if (
174
+ document.activeElement instanceof HTMLElement &&
175
+ panelRef.current?.contains(document.activeElement)
176
+ )
177
+ document.activeElement.blur();
178
+ dismissRef.current();
179
+ return;
180
+ }
181
+ if (event.key !== "Tab" || !trapFocus) return;
182
+ const focusable = Array.from(
183
+ panelRef.current?.querySelectorAll<HTMLElement>(FOCUSABLE) ?? []
184
+ ).filter((element) => element.getClientRects().length > 0);
185
+ if (!focusable.length) {
186
+ event.preventDefault();
187
+ panelRef.current?.focus();
188
+ return;
189
+ }
190
+ const first = focusable[0];
191
+ const last = focusable.at(-1)!;
192
+ const active = document.activeElement;
193
+ if (event.shiftKey && (active === first || !panelRef.current?.contains(active))) {
194
+ event.preventDefault();
195
+ last.focus();
196
+ } else if (!event.shiftKey && active === last) {
197
+ event.preventDefault();
198
+ first.focus();
199
+ }
200
+ };
201
+ document.addEventListener("pointerdown", onPointerDown, true);
202
+ document.addEventListener("keydown", onKeyDown, true);
203
+ return () => {
204
+ document.removeEventListener("pointerdown", onPointerDown, true);
205
+ document.removeEventListener("keydown", onKeyDown, true);
206
+ };
207
+ }, [dismissOnOutside, id, panelRef, stack, trapFocus]);
208
+
209
+ useLayoutEffect(() => {
210
+ const previous =
211
+ document.activeElement instanceof HTMLElement ? document.activeElement : null;
212
+ const frame = window.requestAnimationFrame(() => {
213
+ if (initialFocus === "none") return;
214
+ const selector = initialFocus === "first" ? FOCUSABLE : initialFocus;
215
+ const target = panelRef.current?.querySelector<HTMLElement>(selector);
216
+ target?.focus({ preventScroll: true });
217
+ });
218
+ return () => {
219
+ window.cancelAnimationFrame(frame);
220
+ if (restoreFocus && previous?.isConnected) previous.focus({ preventScroll: true });
221
+ };
222
+ }, [initialFocus, panelRef, restoreFocus]);
223
+ }
224
+
225
+ export function BodyPortal({ children }: { children: ReactNode }) {
226
+ return createPortal(children, document.body);
227
+ }
228
+
229
+ interface AnchoredOverlayProps {
230
+ /** ⚠ Nullable BY DECLARATION as of 2026-08-04. Several call sites pass `ref.current`,
231
+ * which is null before the ref attaches, and the old non-null type made that a
232
+ * render-time TypeError instead of a type error. See `anchorRect`. */
233
+ anchor: Anchor | null | undefined;
234
+ className: string;
235
+ children: ReactNode;
236
+ onDismiss: () => void;
237
+ placement?: Placement;
238
+ role?: AriaRole;
239
+ ariaLabel?: string;
240
+ id?: string;
241
+ initialFocus?: "first" | "none" | string;
242
+ restoreFocus?: boolean;
243
+ dismissOnOutside?: boolean;
244
+ onKeyDown?: (event: ReactKeyboardEvent<HTMLElement>) => void;
245
+ dataKind?: string;
246
+ }
247
+
248
+ /** Body-level fixed overlay with iframe-viewport collision handling. */
249
+ export function AnchoredOverlay({
250
+ anchor,
251
+ className,
252
+ children,
253
+ onDismiss,
254
+ placement = "bottom-start",
255
+ role,
256
+ ariaLabel,
257
+ id,
258
+ initialFocus = "first",
259
+ restoreFocus = true,
260
+ dismissOnOutside = true,
261
+ onKeyDown,
262
+ dataKind,
263
+ }: AnchoredOverlayProps) {
264
+ const panelRef = useRef<HTMLDivElement>(null);
265
+ const [style, setStyle] = useState<CSSProperties>({
266
+ position: "fixed",
267
+ left: 0,
268
+ top: 0,
269
+ visibility: "hidden",
270
+ });
271
+ const anchorElement = anchor && "getBoundingClientRect" in anchor ? anchor : null;
272
+
273
+ useOverlayLayer({
274
+ panelRef,
275
+ onDismiss,
276
+ dismissOnOutside,
277
+ initialFocus,
278
+ restoreFocus,
279
+ outsideElements: [anchorElement],
280
+ });
281
+
282
+ const updatePosition = useCallback(() => {
283
+ const panel = panelRef.current;
284
+ if (!panel) return;
285
+ const target = anchorRect(anchor);
286
+ const viewport = window.visualViewport;
287
+ const viewportLeft = viewport?.offsetLeft ?? 0;
288
+ const viewportTop = viewport?.offsetTop ?? 0;
289
+ const viewportWidth = viewport?.width ?? window.innerWidth;
290
+ const viewportHeight = viewport?.height ?? window.innerHeight;
291
+ const measured = panel.getBoundingClientRect();
292
+ // The arithmetic lives in overlayPlacement.ts so a gate can run it under node. This
293
+ // reads the DOM, decides nothing.
294
+ const placed = computeOverlayPosition({
295
+ placement,
296
+ target,
297
+ panel: {
298
+ width: Math.max(measured.width, panel.scrollWidth),
299
+ height: Math.max(measured.height, panel.scrollHeight),
300
+ },
301
+ viewport: {
302
+ left: viewportLeft,
303
+ top: viewportTop,
304
+ width: viewportWidth,
305
+ height: viewportHeight,
306
+ },
307
+ });
308
+ const next: CSSProperties = {
309
+ position: "fixed",
310
+ left: placed.left,
311
+ top: placed.top,
312
+ maxWidth: placed.maxWidth,
313
+ maxHeight: placed.maxHeight,
314
+ visibility: "visible",
315
+ };
316
+ setStyle((current) => (samePosition(current, next) ? current : next));
317
+ }, [anchor, placement]);
318
+
319
+ useLayoutEffect(() => {
320
+ updatePosition();
321
+ const frame = window.requestAnimationFrame(updatePosition);
322
+ const observer = new ResizeObserver(updatePosition);
323
+ if (panelRef.current) observer.observe(panelRef.current);
324
+ if (anchorElement) observer.observe(anchorElement);
325
+ window.addEventListener("resize", updatePosition);
326
+ window.addEventListener("scroll", updatePosition, true);
327
+ window.visualViewport?.addEventListener("resize", updatePosition);
328
+ window.visualViewport?.addEventListener("scroll", updatePosition);
329
+ return () => {
330
+ window.cancelAnimationFrame(frame);
331
+ observer.disconnect();
332
+ window.removeEventListener("resize", updatePosition);
333
+ window.removeEventListener("scroll", updatePosition, true);
334
+ window.visualViewport?.removeEventListener("resize", updatePosition);
335
+ window.visualViewport?.removeEventListener("scroll", updatePosition);
336
+ };
337
+ }, [anchorElement, updatePosition]);
338
+
339
+ return (
340
+ <BodyPortal>
341
+ <div
342
+ ref={panelRef}
343
+ id={id}
344
+ className={className}
345
+ style={style}
346
+ role={role}
347
+ aria-label={ariaLabel}
348
+ data-overlay-kind={dataKind}
349
+ onKeyDown={onKeyDown}
350
+ >
351
+ {children}
352
+ </div>
353
+ </BodyPortal>
354
+ );
355
+ }
web/src/customer-grid/RecordDetail.tsx CHANGED
The diff for this file is too large to render. See raw diff