fsanyoto commited on
Commit
cf17b22
Β·
verified Β·
1 Parent(s): 003d5fa

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes. Β  See raw diff
Files changed (50) hide show
  1. RELEASES.json +7 -1
  2. VERSION +1 -1
  3. api/ai_review.py +58 -23
  4. api/automation_engine.py +83 -0
  5. api/connectors_tt.py +568 -568
  6. api/main.py +19 -0
  7. api/odoo_relational.py +0 -0
  8. api/providers.py +321 -4
  9. api/routes_admin.py +0 -0
  10. api/routes_agent_harness.py +455 -0
  11. api/routes_alerts.py +668 -668
  12. api/routes_automation.py +279 -8
  13. api/routes_grid.py +81 -0
  14. api/routes_keychain.py +0 -0
  15. api/routes_nav.py +0 -0
  16. api/routes_oauth.py +114 -114
  17. api/routes_odoo_tables.py +0 -0
  18. api/routes_publish.py +0 -0
  19. api/routes_query.py +0 -0
  20. api/routes_records.py +211 -211
  21. api/routes_script_views.py +329 -0
  22. api/routes_shares.py +393 -393
  23. api/routes_slack.py +28 -12
  24. api/routes_statements.py +31 -2
  25. api/routes_tables.py +0 -0
  26. api/routes_web_agent.py +5 -4
  27. platform/aios_grid.py +0 -0
  28. platform/core/perm_scope.py +349 -0
  29. platform/core/perms.py +31 -18
  30. platform/core/script_sandbox.py +519 -0
  31. platform/core/shares.py +19 -11
  32. platform/core/store.py +0 -0
  33. platform/core/table_store.py +746 -618
  34. platform/core/user_tables.py +0 -0
  35. platform/harness/meta_store.py +521 -521
  36. platform/model/topics/odoo_accounts.yml +60 -60
  37. platform/model/topics/odoo_agents.yml +59 -59
  38. platform/model/topics/odoo_bills.yml +86 -86
  39. platform/model/topics/odoo_customers.yml +213 -213
  40. platform/model/topics/odoo_invoices.yml +98 -98
  41. platform/model/topics/odoo_orders.yml +87 -87
  42. platform/model/topics/odoo_products.yml +147 -147
  43. platform/model/topics/odoo_vendors.yml +103 -103
  44. platform/modules/collections_send.py +7 -3
  45. platform/modules/product_data.py +804 -804
  46. platform/modules/products.py +737 -737
  47. web/public/sample_customers.json +558 -558
  48. web/src/alerts/alertsModel.ts +418 -418
  49. web/src/assistant/AssistantPage.tsx +11 -1
  50. web/src/automation/AutomationBuilder.tsx +557 -16
RELEASES.json CHANGED
@@ -1,6 +1,12 @@
1
  {
2
- "current": "55ced50",
3
  "releases": [
 
 
 
 
 
 
4
  {
5
  "version": "v28",
6
  "sha": "2793e4b",
 
1
  {
2
+ "current": "c9ad659 (c9ad659)",
3
  "releases": [
4
+ {
5
+ "version": "v29",
6
+ "sha": "55ced50",
7
+ "date": "2026-08-18",
8
+ "subject": "v29: the build staging has been running since 2026-08-18 (55ced50)."
9
+ },
10
  {
11
  "version": "v28",
12
  "sha": "2793e4b",
VERSION CHANGED
@@ -1 +1 @@
1
- 55ced50
 
1
+ c9ad659 (c9ad659)
api/ai_review.py CHANGED
@@ -56,7 +56,9 @@ PROVIDERS = [
56
  # labels it already has in front of it.
57
  {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic",
58
  "url": "https://api.anthropic.com/v1/messages",
59
- "model": "claude-haiku-4-5"},
 
 
60
  ]
61
  ANTHROPIC_VERSION = "2023-06-01"
62
  TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20)
@@ -277,13 +279,23 @@ MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engi
277
  MAX_PROMPT_CHARS = 2000
278
 
279
 
280
- def flow_providers():
281
- """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off."""
282
- pin = (os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower()
 
 
 
 
 
 
 
283
  by_name = {p["name"]: p for p in PROVIDERS}
284
- order = [pin] if pin else list(FLOW_PROVIDER_ORDER)
285
- return [by_name[n] for n in order
286
  if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
 
 
 
 
287
 
288
 
289
  def flow_schema(kinds, trigger_keys, table_keys):
@@ -421,7 +433,7 @@ def _flow_from_tool_call(obj):
421
 
422
 
423
  def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None,
424
- st=None, user=""):
425
  """A sentence -> `(draft, refusal_sentence, provider)`. β›” NOTHING IS SAVED HERE.
426
 
427
  Exactly one of `draft` and `refusal_sentence` is truthy β€” the same contract
@@ -456,7 +468,7 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou
456
  draft, refusal = _flow_from_tool_call(chat(messages, tools))
457
  return draft, refusal, "injected"
458
 
459
- provs = flow_providers()
460
  if not provs:
461
  # β›” SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
462
  # never built [[flag-shipped-without-its-writer]].
@@ -464,30 +476,53 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou
464
  "be drafted from a description yet"), None
465
  tmo = float(timeout or TIMEOUT_SECONDS)
466
  problems = []
 
467
  for p in provs:
468
- if p["shape"] != "openai":
469
- # Anthropic's tool transport differs; it stays on the ladder for `decide()` and is
470
- # SKIPPED here rather than half-supported. Named in `problems` so it is not invisible.
471
- problems.append(f"{p['name']}: tool calls are not wired for this shape")
472
- continue
 
 
 
 
473
  try:
474
- r = requests.post(p["url"], timeout=tmo,
475
- headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
476
- "Content-Type": "application/json"},
477
- json={"model": p["model"], "messages": messages, "tools": tools,
478
- "tool_choice": "required", "temperature": 0.1,
479
- "max_tokens": 1500})
 
 
 
 
 
 
 
480
  except Exception as e: # noqa: BLE001
481
  problems.append(f"{p['name']}: {type(e).__name__}")
482
  continue
483
  if r.status_code != 200:
484
- problems.append(f"{p['name']}: HTTP {r.status_code}")
 
 
 
 
 
485
  continue
486
  try:
487
  body = r.json()
488
- calls = (((body.get("choices") or [{}])[0].get("message") or {})
489
- .get("tool_calls") or [])
490
- args = json.loads(calls[0]["function"]["arguments"]) if calls else None
 
 
 
 
 
 
491
  except Exception as e: # noqa: BLE001
492
  problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
493
  continue
 
56
  # labels it already has in front of it.
57
  {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic",
58
  "url": "https://api.anthropic.com/v1/messages",
59
+ # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above,
60
+ # and a deployment whose side rungs are out of credit can raise the tier without a release.
61
+ "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"},
62
  ]
63
  ANTHROPIC_VERSION = "2023-06-01"
64
  TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20)
 
279
  MAX_PROMPT_CHARS = 2000
280
 
281
 
282
+ def flow_providers(pin=None):
283
+ """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.
284
+
285
+ ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and
286
+ the draft door used to read `prompt` off the body and nothing else β€” so the key the client sent
287
+ was accepted and dropped, and the picker was a control over nothing.
288
+ ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the
289
+ ladder stopped offering must not turn every later draft into an error; the caller is told which
290
+ rung actually answered, which is the honest half.
291
+ """
292
  by_name = {p["name"]: p for p in PROVIDERS}
293
+ live = [n for n in FLOW_PROVIDER_ORDER
 
294
  if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
295
+ wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower()
296
+ if wanted and wanted in live:
297
+ return [by_name[wanted]]
298
+ return [by_name[n] for n in live]
299
 
300
 
301
  def flow_schema(kinds, trigger_keys, table_keys):
 
433
 
434
 
435
  def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None,
436
+ st=None, user="", model=None):
437
  """A sentence -> `(draft, refusal_sentence, provider)`. β›” NOTHING IS SAVED HERE.
438
 
439
  Exactly one of `draft` and `refusal_sentence` is truthy β€” the same contract
 
468
  draft, refusal = _flow_from_tool_call(chat(messages, tools))
469
  return draft, refusal, "injected"
470
 
471
+ provs = flow_providers(model)
472
  if not provs:
473
  # β›” SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
474
  # never built [[flag-shipped-without-its-writer]].
 
476
  "be drafted from a description yet"), None
477
  tmo = float(timeout or TIMEOUT_SECONDS)
478
  problems = []
479
+ import providers as _prov
480
  for p in provs:
481
+ # ⭐⭐ W36-T35 / R4 β€” THE OWNER QUOTED THIS LINE BACK AT US. It used to read
482
+ # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`,
483
+ # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the
484
+ # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."*
485
+ # It is wired now, through the ONE wire in `providers`, and it matters more than it looks:
486
+ # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter
487
+ # 402), so without this branch the drafter cannot answer at all.
488
+ # ⚠ NO `effort` ON THIS DOOR β€” `output_config.effort` errors on Haiku 4.5, the tier this
489
+ # ladder runs. The parameter is the caller's to send, which is why the wire takes it.
490
  try:
491
+ if p["shape"] == "anthropic":
492
+ req = _prov.anthropic_request(
493
+ model=p["model"], key=os.environ[p["env"]].strip(), system=None,
494
+ messages=messages, tools=tools, max_tokens=1500, tool_choice="required")
495
+ r = requests.post(req["url"], timeout=tmo,
496
+ headers=req["headers"], json=req["json"])
497
+ else:
498
+ r = requests.post(p["url"], timeout=tmo,
499
+ headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
500
+ "Content-Type": "application/json"},
501
+ json={"model": p["model"], "messages": messages, "tools": tools,
502
+ "tool_choice": "required", "temperature": 0.1,
503
+ "max_tokens": 1500})
504
  except Exception as e: # noqa: BLE001
505
  problems.append(f"{p['name']}: {type(e).__name__}")
506
  continue
507
  if r.status_code != 200:
508
+ # β›” A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off
509
+ # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and
510
+ # the memo makes the next turn SKIP the empty account instead of paying for it again.
511
+ if _prov.is_credit_failure(r.status_code, r.text):
512
+ _prov.mark_no_credit(p["name"])
513
+ problems.append(_prov.refusal_sentence(p["name"], r.status_code, r.text))
514
  continue
515
  try:
516
  body = r.json()
517
+ if p["shape"] == "anthropic":
518
+ _text, args, _refused = _prov.anthropic_read(body)
519
+ if _refused:
520
+ problems.append(f"{p['name']}: {_refused}")
521
+ continue
522
+ else:
523
+ calls = (((body.get("choices") or [{}])[0].get("message") or {})
524
+ .get("tool_calls") or [])
525
+ args = json.loads(calls[0]["function"]["arguments"]) if calls else None
526
  except Exception as e: # noqa: BLE001
527
  problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
528
  continue
api/automation_engine.py CHANGED
@@ -9219,6 +9219,36 @@ ACTION_CATALOG = [
9219
  "ready": True,
9220
  "detail": "Assemble this month's statements and park them for review. Nothing is sent until "
9221
  "somebody opens the batch and clicks Send"},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9222
  {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
9223
  "detail": "Needs the Slack connector"},
9224
  # ── 4. ADVANCED LOGIC ────────────────────────────────────────────────────────────────────
@@ -9306,6 +9336,33 @@ def _tenant_may_use(kind, rt):
9306
  return True if gate is None else bool(rt is not None and gate(rt))
9307
 
9308
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9309
  def action_catalog(rt=None):
9310
  """The catalog as the wire carries it β€” a copy, because a caller that mutated the module
9311
  constant would change every later reader's answer.
@@ -10831,6 +10888,32 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print
10831
  "and nothing was sent.")
10832
  counts["webBlocked"] += 1
10833
  continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10834
  elif kind == "ai_agent":
10835
  # ⭐⭐ W33-T56 (owner item 7, ruling R3) β€” THE FUZZY STEP, AT RUN TIME.
10836
  #
 
9219
  "ready": True,
9220
  "detail": "Assemble this month's statements and park them for review. Nothing is sent until "
9221
  "somebody opens the batch and clicks Send"},
9222
+ # ⭐⭐ WAVE 36 Β· W36-T39 β€” D-277 CLOSED THE WAY THAT ROW ITSELF RECOMMENDED: *"an
9223
+ # `ACTION_CATALOG` row with `menu: false` β€” one line, reusing the door R18 built this same wave
9224
+ # to keep the five web kinds readable but unofferable. (a) looks right; it is E's file."*
9225
+ #
9226
+ # β›” THE DEFECT WAS A KIND IN NO CATALOG ON EITHER SIDE. `routes_automation._field_agent_rows`
9227
+ # emits `flow.actions[0].kind == "ai_enrich"` for every AI-enrichment column in the tenant, and
9228
+ # that string appeared ZERO times here and ZERO times in `aios-web/web/src/automation/`. The
9229
+ # builder resolves a stored step's caption with `catalog.find(c => c.kind === a.kind)` and falls
9230
+ # back to `a.kind`, so a field agent opened in the Agents module showed a RAW TOKEN.
9231
+ #
9232
+ # ⚠ `menu: False`, NEVER `ready: False`. `ready: False` renders as "coming soon" β€” a promise β€”
9233
+ # and `clean_actions` refuses the kind outright, which would 400 the synthetic row the moment
9234
+ # anything validated it. `menu: False` is the exact shape R18 built: withheld from the picker,
9235
+ # still resolvable as a caption, still valid.
9236
+ # β›” AND IT IS NOT ADDABLE BY HAND ON PURPOSE. An enrichment belongs to a COLUMN; the automation
9237
+ # canvas is not where one is created, which is why `patch_automation` already refuses a
9238
+ # `field:` id with "change its prompt, model or schedule on the column itself".
9239
+ # ⭐⭐ AND A THIRD INSTANCE, FOUND BY W36-T39's OWN GATE ON ITS FIRST RUN. `_odoo_sync_row`
9240
+ # emits `flow.actions[0].kind == "odoo_sync"` for the connector's synthetic schedule agent, and
9241
+ # that kind was in no catalog either β€” the same raw token on the same screen as D-277, one row
9242
+ # down. Two known instances were enough to justify the check; the check then produced a third
9243
+ # nobody had booked, which is the difference between a gate and a regression test.
9244
+ {"kind": "odoo_sync", "label": "Sync from Odoo", "group": "Connected",
9245
+ "ready": True, "menu": False,
9246
+ "detail": "Pull the connected Odoo databases on a schedule. Configured on the connector, not "
9247
+ "here"},
9248
+ {"kind": "ai_enrich", "label": "Enrich this column with AI", "group": "Connected",
9249
+ "ready": True, "menu": False,
9250
+ "detail": "Fill an AI column for the records this flow walks. Configured on the column, not "
9251
+ "here"},
9252
  {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
9253
  "detail": "Needs the Slack connector"},
9254
  # ── 4. ADVANCED LOGIC ────────────────────────────────────────────────────────────────────
 
9336
  return True if gate is None else bool(rt is not None and gate(rt))
9337
 
9338
 
9339
+ def catalog_kinds():
9340
+ """Every action kind this module knows about β€” the LABEL vocabulary.
9341
+
9342
+ ⭐ D-277's WHOLE LESSON IN ONE SENTENCE: the client resolves a stored step's caption out of the
9343
+ catalog and falls back to the raw kind token, so a kind the server can EMIT and the catalog
9344
+ does not carry is a token on somebody's screen. This is the set that must cover every kind any
9345
+ server path can put into a `flow.actions` entry, whether or not a person may add it.
9346
+ """
9347
+ return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG)
9348
+
9349
+
9350
+ def configurable_kinds():
9351
+ """The kinds a PERSON may add from the picker, and must therefore be able to configure.
9352
+
9353
+ ⭐⭐ W36-T39 β€” THE CONTRACT BETWEEN THE TWO TREES, DERIVED AND NEVER LISTED. `ready` alone is
9354
+ the wrong set: the five `web_*` kinds are ready and `menu: False` (R18), and `ai_enrich` is
9355
+ ready and `menu: False` (D-277) β€” all six are real, runnable, captioned, and unofferable. What
9356
+ a client must be able to CONFIGURE is exactly what a person can ADD.
9357
+
9358
+ β›” DERIVED FROM THE CATALOG, so a new row joins the contract by existing. A hand-kept list
9359
+ would be a third copy of the vocabulary, and the two copies this ticket exists to reconcile
9360
+ were already one too many.
9361
+ """
9362
+ return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG
9363
+ if row.get("ready") and row.get("menu", True) is not False)
9364
+
9365
+
9366
  def action_catalog(rt=None):
9367
  """The catalog as the wire carries it β€” a copy, because a caller that mutated the module
9368
  constant would change every later reader's answer.
 
10888
  "and nothing was sent.")
10889
  counts["webBlocked"] += 1
10890
  continue
10891
+ elif kind == "odoo_sync":
10892
+ # Same refusal, same reason as the arm below: a catalog row with no arm is walked,
10893
+ # counted, and reports success having done nothing. The connector owns this sync.
10894
+ if "odoo_sync_not_run_here" not in web_notes:
10895
+ web_notes.append("odoo_sync_not_run_here")
10896
+ log("[aios-auto] odoo_sync: the Odoo pull runs on the connector's own "
10897
+ "schedule, not from this canvas. Nothing was done")
10898
+ counts["webBlocked"] += 1
10899
+ continue
10900
+ elif kind == "ai_enrich":
10901
+ # β›”β›” A REFUSAL ARM, AND IT EXISTS FOR THE REASON THE `send_statement` ARM ABOVE
10902
+ # STATES: `_walk` has no terminal `else`, so a catalog row whose arm is missing is
10903
+ # walked, COUNTED, reports the run `ok` and writes nothing β€” a step somebody
10904
+ # configured that succeeds at doing nothing. Adding the row (D-277) without this
10905
+ # arm would have opened exactly that window.
10906
+ # ⚠ AND THE REFUSAL IS THE TRUTH, not a stub. An AI column is filled by
10907
+ # `ai_enrich`, driven from the column's own editor; the synthetic agent row this
10908
+ # kind appears in is DERIVED from that column and is never stored, so nothing
10909
+ # reaches here through the ordinary path. If something ever does, it must say so
10910
+ # rather than report success.
10911
+ if "ai_enrich_not_run_here" not in web_notes:
10912
+ web_notes.append("ai_enrich_not_run_here")
10913
+ log("[aios-auto] ai_enrich: an AI column is filled from the column's own "
10914
+ "editor, not from this canvas. Nothing was done")
10915
+ counts["webBlocked"] += 1
10916
+ continue
10917
  elif kind == "ai_agent":
10918
  # ⭐⭐ W33-T56 (owner item 7, ruling R3) β€” THE FUZZY STEP, AT RUN TIME.
10919
  #
api/connectors_tt.py CHANGED
@@ -1,568 +1,568 @@
1
- """connectors_tt.py β€” the TIKTOK connector (wave 29 Β· item 7 Β· DEBT D-9 Β· rulings R1 + R2).
2
-
3
- Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
4
- automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
5
- exists at all rather than another thousand lines inside the engine.
6
-
7
- β›” **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema β€” 40 profile / 43
8
- post / 17 comment fields, each with the vendor's own type, description and `pii` flag β€” was read
9
- live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
10
- `.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
11
- close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
12
- name below reads through a candidate list it is because the vendor has two names for one fact
13
- (`biography`/`signature`, `region`/`country`), never because the name is uncertain.
14
-
15
- β›” **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned β€”
16
- public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
17
- vendor key is a key to a SUPPLIER.
18
-
19
- ⭐ **THE TRANSPORT IS `connectors_bd.py` β€” SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE
20
- OTHER PLATFORM'S CONNECTOR** (WAVE 30 Β· T09, DEBT D-128). `bd_call`, `bd_scrape` and
21
- `bd_filter_start` take the dataset id as a PARAMETER β€” they are Bright Data's wire, not
22
- Instagram's β€” and re-implementing them here would be a second copy of the deferral handling, the
23
- truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
24
- Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from
25
- `connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier.
26
- ⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because
27
- the sentence above is the kind that quietly stops being true.
28
-
29
- ⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
30
- 1. the real ROW shape β€” `/metadata` describes a DATASET, and Instagram's rows carry undeclared
31
- envelope keys (`timestamp`, `input`) that no metadata call mentions;
32
- 2. that a declared field POPULATES β€” Bright Data's Instagram Reels *declares* `views: number`
33
- and delivers an account-grain wrong number (Β§4e). **Declared is not delivered**, and the one
34
- TikTok claim that matters most (`play_count`) is exactly a declaration.
35
- """
36
- from __future__ import annotations
37
-
38
- import automation_engine as engine
39
- # ⚠ T09 β€” FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an
40
- # import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were
41
- # imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever
42
- # read them off this module (the engine imports them from the transport itself), so they were four
43
- # lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form.
44
- from connectors_bd import (
45
- _bd_first_url,
46
- _bd_flag,
47
- _bd_list,
48
- _bd_source_payload,
49
- _first,
50
- _ig_int,
51
- bd_scrape,
52
- )
53
-
54
- # ---------------------------------------------------------------------------------------------
55
- # THE DATASETS
56
- # ---------------------------------------------------------------------------------------------
57
- # ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT β€” the same finding Instagram produced. `GET
58
- # /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
59
- # full field list, and the two DISCOVERY halves answer **404 for our key**:
60
- # `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
61
- # β‡’ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
62
- # does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
63
- TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
64
- TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
65
- TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
66
-
67
- #: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
68
- #: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
69
- TT_POST_TYPE_VIDEO = "video"
70
- TT_POST_TYPE_CONTENT = "content"
71
-
72
- #: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
73
- #: beside the dataset ids because it is the same class of vendor fact.
74
- #: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
75
- #: string from `_PROFILE_RULES` (contract C2, E's half) β€” this exists for the connector's own
76
- #: batch calls, and the gate asserts the two agree rather than trusting that they do.
77
- TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
78
-
79
-
80
- def tt_profile_url(handle):
81
- """`nurilab` β†’ `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
82
- h = str(handle or "").strip().lstrip("@")
83
- return TT_PROFILE_URL.format(handle=h) if h else ""
84
-
85
-
86
- # ---------------------------------------------------------------------------------------------
87
- # THE FIELD MAPS
88
- # ---------------------------------------------------------------------------------------------
89
- # Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
90
- # in `automation_engine`. The rules they all obey, stated once:
91
- #
92
- # * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
93
- # so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
94
- # returns `None` (never 0) when nothing matches, which is what makes that possible.
95
- # * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
96
- # rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
97
- # * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
98
- # vendor's side is preserved rather than silently discarded while our column model catches up.
99
- # * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
100
- # (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
101
- # ride in `source_payload` where they make no claim.
102
-
103
-
104
- def _tt_str(node, *names):
105
- """The first non-empty string among `names`, or None when the vendor sent nothing.
106
-
107
- β›” `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest β€”
108
- an empty string written into a cell claims "we looked and it is empty".
109
- """
110
- v = _first(node, *names)
111
- if v is None:
112
- return None
113
- s = str(v).strip()
114
- return s or None
115
-
116
-
117
- def _tt_pct(node, *names):
118
- """A vendor 0–1 engagement fraction β†’ our stored 0–100 percentage, or None.
119
-
120
- β›” THE Γ—100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
121
- the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` β€”
122
- measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
123
- """
124
- v = _first(node, *names)
125
- if v is None:
126
- return None
127
- out = engine._pct100(v)
128
- return out or None
129
-
130
-
131
- def _tt_day(node, *names):
132
- """A vendor stamp β†’ `YYYY-MM-DD`, or None. Our `date` columns store a day."""
133
- v = _first(node, *names)
134
- if v is None:
135
- return None
136
- return engine._day(v) or None
137
-
138
-
139
- def _drop_blanks(row):
140
- """The one place a mapped row loses its `None`s β€” see the BLANK MEANS NOT READ rule above."""
141
- return {k: v for k, v in row.items() if v is not None and v != ""}
142
-
143
-
144
- def normalize_profile(node, handle=""):
145
- """A TikTok Profiles row β†’ the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
146
-
147
- ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
148
- * `biography` is PRIMARY and `signature` the FALLBACK β€” the probe measured `signature`
149
- populated on 85% of rows and they carry the same text;
150
- * `region` is PRIMARY and `country` the FALLBACK β€” `region` is the one with a documented
151
- two-letter-ISO description, `country` has no description at all.
152
- ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
153
- `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
154
- only count of its kind the dataset offers.
155
- """
156
- node = node if isinstance(node, dict) else {}
157
- account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
158
- return _drop_blanks({
159
- "platform": engine.PLATFORM_TIKTOK,
160
- "handle": account,
161
- "full_name": _tt_str(node, "nickname"),
162
- "tt_id": _tt_str(node, "id"),
163
- "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
164
- "bio": _tt_str(node, "biography", "signature"),
165
- "external_url": _bd_first_url(_first(node, "bio_link")),
166
- "verified": _bd_flag(node, "is_verified"),
167
- "is_private": _bd_flag(node, "is_private"),
168
- # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
169
- # on their own description. It is the closest thing TikTok has to Instagram's
170
- # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent β€” so
171
- # the approximation only ever fills a cell the vendor actually answered.
172
- "is_business": _bd_flag(node, "is_commerce_user"),
173
- "followers": _ig_int(_first(node, "followers")),
174
- "following": _ig_int(_first(node, "following")),
175
- "posts_count": _ig_int(_first(node, "videos_count")),
176
- # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
177
- # nulls). It is not a post-level number and it is not our `likes` column, which is why it
178
- # is stored under a different name.
179
- "likes_received": _ig_int(_first(node, "likes")),
180
- "avg_engagement": _tt_pct(node, "awg_engagement_rate"),
181
- "like_engagement": _tt_pct(node, "like_engagement_rate"),
182
- "comment_engagement": _tt_pct(node, "comment_engagement_rate"),
183
- "country_code": _tt_str(node, "region", "country"),
184
- "region": _tt_str(node, "region"),
185
- "predicted_lang": _tt_str(node, "predicted_lang"),
186
- # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP β€” `create_time` on a profile is when the ACCOUNT
187
- # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
188
- # which is why the append law dates a snapshot by when WE read it.
189
- "account_created_at": _tt_day(node, "create_time"),
190
- "source_payload": _bd_source_payload(node),
191
- })
192
-
193
-
194
- def tt_post_type(node):
195
- """A TikTok post row β†’ one of OUR three type options, or None.
196
-
197
- β›” THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
198
- `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
199
- vendor's token would fail `_clean_field`'s option check on the way in and would put an
200
- untranslated API word in front of a user on the way out.
201
-
202
- So: `video` is `video`, and `content` β€” TikTok's photo-mode post β€” is `image`, EXCEPT when the
203
- row carries more than one `carousel_images` entry, which is what a carousel IS on either
204
- network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the
205
- token cannot express it, so inferring `carousel` from the word would be inventing a fact.
206
- ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
207
- that is exactly what would make the wrong default invisible).
208
- """
209
- raw = str((node or {}).get("post_type") or "").strip().lower()
210
- if raw == TT_POST_TYPE_VIDEO:
211
- return "video"
212
- if raw != TT_POST_TYPE_CONTENT:
213
- return None
214
- images = (node or {}).get("carousel_images")
215
- return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
216
-
217
-
218
- def normalize_post(node):
219
- """A TikTok Posts row β†’ the `ut_tt_posts` cell shape. None when it carries no identity.
220
-
221
- β›” THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
222
- Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
223
- fact β€” `play_count` IS the count TikTok displays under a video β€” so it maps to `views` and
224
- `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
225
- manufacture a second measurement that a rollup could average or double-count, which is a worse
226
- outcome than the missing column it would paper over.
227
-
228
- ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
229
- the probe measured them as the same shape. That equality is what lets the comments→posts link
230
- join with NO normaliser, which the Instagram side never had.
231
- ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
232
- ⚠ `commerce_info` is a business/commerce LOCATION per its own description β€” cities and
233
- countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
234
- no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
235
- """
236
- node = node if isinstance(node, dict) else {}
237
- shortcode = _tt_str(node, "shortcode", "post_id")
238
- if not shortcode:
239
- return None
240
- return _drop_blanks({
241
- "platform": engine.PLATFORM_TIKTOK,
242
- "shortcode": shortcode,
243
- # β›”β›” `account_id`, NOT `profile_username` β€” MEASURED on a real Posts row 2026-08-12.
244
- # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the
245
- # @handle (`"d1na_th"`) β€” the same field `normalize_profile` already reads for `handle`, so
246
- # one name means one thing across both corpora. Reading the display name silently broke the
247
- # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched
248
- # NOTHING, so a person could not filter posts by creator and a rollup would count zero.
249
- # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a
250
- # different fact, and filling a join key with it is worse than leaving it blank β€” a blank
251
- # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]].
252
- # The URL is the honest second source: it carries the handle by construction.
253
- "influencer_key": (_tt_str(node, "account_id")
254
- or tt_handle(_tt_str(node, "profile_url", "url") or "") or None),
255
- "posted_at": _tt_day(node, "create_time"),
256
- "type": tt_post_type(node),
257
- "caption": _tt_str(node, "description"),
258
- "url": _tt_str(node, "url"),
259
- "hashtags": _bd_list(node, "hashtags"),
260
- "tagged_location": _tt_str(node, "commerce_info"),
261
- "views": _ig_int(_first(node, "play_count")),
262
- "likes": _ig_int(_first(node, "digg_count")),
263
- "comments": _ig_int(_first(node, "comment_count")),
264
- "shares": _ig_int(_first(node, "num_share_count")),
265
- "saves": _ig_int(_first(node, "collect_count")),
266
- "video_duration": _ig_int(_first(node, "video_duration")),
267
- "source_payload": _bd_source_payload(node),
268
- })
269
-
270
-
271
- def normalize_comment(node):
272
- """A TikTok Comments row β†’ the `ut_tt_comments` cell shape. None without a comment id.
273
-
274
- ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
275
- `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
276
- `source_payload`. Reading the array's length instead would be a second, disagreeing answer to
277
- a question the vendor already answers β€” and it would disagree, because a page of replies is not
278
- all of them.
279
- ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
280
- text and needs defensive parsing. It still goes through `_tt_day` β€” one date path, so a vendor
281
- that changes its mind cannot change ours.
282
- ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
283
- PII) stay in `source_payload` and are promoted to no column, which is the same posture the
284
- Instagram comment schema takes for the same D-24 reason.
285
- """
286
- node = node if isinstance(node, dict) else {}
287
- comment_key = _tt_str(node, "comment_id")
288
- if not comment_key:
289
- return None
290
- return _drop_blanks({
291
- "platform": engine.PLATFORM_TIKTOK,
292
- "comment_key": comment_key,
293
- "shortcode": _tt_str(node, "post_id"),
294
- # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
295
- # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
296
- # primary first, so a row carrying the rich form is not silently served the plain one.
297
- # β›” The commenter's identity is deliberately NOT promoted β€” see `TT_COMMENT_FIELDS`.
298
- "text": _tt_str(node, "comment_text", "comment_text_only"),
299
- "commented_at": _tt_day(node, "date_created"),
300
- "likes": _ig_int(_first(node, "num_likes")),
301
- "replies": _ig_int(_first(node, "num_replies")),
302
- "source_payload": _bd_source_payload(node),
303
- })
304
-
305
-
306
- #: ⭐ The three maps, addressable by name β€” so a gate (and the runners in T04-T06) can walk them
307
- #: rather than naming three functions, and so adding a fourth dataset is one entry.
308
- TT_NORMALIZERS = {
309
- "tt_profile": normalize_profile,
310
- "tt_post_metrics": normalize_post,
311
- "tt_comments": normalize_comment,
312
- }
313
-
314
-
315
- # ---------------------------------------------------------------------------------------------
316
- # THE FETCH β€” wave 30 Β· W30-T08 (carrying wave-29's dropped T05)
317
- # ---------------------------------------------------------------------------------------------
318
-
319
- def tt_handle(url):
320
- """A TikTok profile URL **or** a bare handle β†’ the handle. `''` when it is neither.
321
-
322
- Deliberately permissive about the input and strict about the output, because the two callers
323
- hand it different things: an automation stores whatever a person typed in the profile column
324
- (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean
325
- `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot
326
- resolve to two different handles.
327
- """
328
- s = str(url or "").strip()
329
- if not s:
330
- return ""
331
- if "tiktok.com" in s.lower():
332
- # Everything after the first `@`, up to the next path segment or query.
333
- tail = s.split("@", 1)[1] if "@" in s else ""
334
- s = tail.split("/")[0].split("?")[0].split("#")[0]
335
- s = s.strip().lstrip("@").strip()
336
- # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores.
337
- return s if s and all(c.isalnum() or c in "._" for c in s) else ""
338
-
339
-
340
- def tt_post_urls(node, limit=0):
341
- """⭐ WAVE 30 Β· T10 β€” the profile row's own post permalinks, newest-first as the vendor sends.
342
-
343
- β›” THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
344
- we have already bought, so the posts read is a scrape of links we hold, never a search for them.
345
- The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**,
346
- so a design that reached for either would not merely be dearer, it would not work.
347
-
348
- β›”β›” CORRECTED 2026-08-12 β€” THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an
349
- array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe
350
- read `/datasets/{id}/metadata` β€” a DATASET description β€” and the phrase quoted was the field's
351
- `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`**
352
- (measured below), so the reader built against the quoted sentence found nothing, forever, in
353
- silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration
354
- of X"* are different claims, and prose cannot be told apart by a reader downstream β€” which is
355
- why the correction names the method, not just the value.
356
-
357
- ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*,
358
- and preferring whichever happened to be longer is how one creator's window silently differs
359
- from another's.
360
- ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S β€” `config.maxPosts`,
361
- validated 1..12 β€” and it is applied here rather than after the scrape so an unwanted post is
362
- never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window,
363
- not a number this function may invent.
364
- """
365
- raw = (node or {}).get("top_videos")
366
- out = []
367
- for item in raw if isinstance(raw, list) else []:
368
- # β›”β›” MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
369
- # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a
370
- # live handle returns **19 DICTS**, keyed
371
- # `video_url Β· video_id Β· playcount Β· diggcount Β· commentcount Β· share_count Β·
372
- # favorites_count Β· create_date Β· cover_image`.
373
- # The docstring above cites the $0 probe as having "measured" permalinks β€” it had not, and
374
- # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says
375
- # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field
376
- # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's
377
- # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was
378
- # the first. [[reachable-is-not-the-same-as-built]]
379
- # β‡’ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY
380
- # list on every real profile β€” post capture could not have worked for anybody, and the
381
- # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture
382
- # written from a schema tests the schema.
383
- # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept
384
- # because it costs nothing and is what a future corpus revision would most likely use. The
385
- # bare-string branch stays for the same reason β€” this widens what is ACCEPTED and invents
386
- # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as
387
- # before, rather than to a URL built out of a guess.
388
- # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring
389
- # whichever array happened to be longer is how one creator's window silently differs from
390
- # another's, and that reasoning is unchanged by this correction.
391
- if isinstance(item, dict):
392
- u = str(item.get("video_url") or item.get("url") or "").strip()
393
- else:
394
- u = str(item or "").strip()
395
- if u.lower().startswith("http") and u not in out:
396
- out.append(u)
397
- return out[:limit] if limit and limit > 0 else out
398
-
399
-
400
- def pull_posts_tt(post_urls, log=print, deferred=None):
401
- """The TikTok Posts dataset for a list of permalinks β†’ `(rows, note)`, already normalised.
402
-
403
- ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side
404
- measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still
405
- running at 67 minutes. Nothing here loops per URL.
406
- """
407
- urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
408
- if not urls:
409
- return [], ""
410
- rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred)
411
- if note:
412
- log(f"[aios-tt] posts: {note}")
413
- return [], note
414
- out = [r for r in (normalize_post(n) for n in rows) if r]
415
- return out, ""
416
-
417
-
418
- def pull_comments_tt(post_urls, log=print, deferred=None):
419
- """The TikTok Comments dataset for a list of POST permalinks β†’ `(rows, note)`, normalised.
420
-
421
- β›” THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on
422
- both networks: a comments scrape ingests identifiable third parties who never entered anybody's
423
- list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes
424
- none of them to a column; this function adds no new exposure, it just has to be asked for.
425
- """
426
- urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
427
- if not urls:
428
- return [], ""
429
- rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred)
430
- if note:
431
- log(f"[aios-tt] comments: {note}")
432
- return [], note
433
- out = [r for r in (normalize_comment(n) for n in rows) if r]
434
- return out, ""
435
-
436
-
437
- #: ⭐⭐ WAVE 30 Β· D-156 β€” THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY.
438
- #: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral
439
- #: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's
440
- #: metric queue β€” the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse
441
- #: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can.
442
- TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS)
443
-
444
-
445
- def _media_deferrals(deferred):
446
- """The POSTS/COMMENTS entries of a `bd_scrape` deferral list β€” never the profile's.
447
-
448
- ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the
449
- handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one
450
- kind here, so the id already carries everything a mapper choice depends on β€” and importing
451
- Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines).
452
- """
453
- out = []
454
- for d in deferred or []:
455
- if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS:
456
- out.append(dict(d))
457
- return out
458
-
459
-
460
- def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
461
- max_posts=0, post_metrics=False, comment_metrics=False):
462
- """ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
463
-
464
- `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
465
- so the engine's enrich branch treats every network identically and no caller learns a new
466
- shape.
467
-
468
- ⭐ WAVE 30 Β· T10 β€” POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's
469
- do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` β€”
470
- no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics`
471
- then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction
472
- rather than by a rule: their input IS a post permalink, so asking for comments with post capture
473
- off is a request with no subject, and it returns none instead of quietly buying posts nobody
474
- asked for.
475
-
476
- β›” `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than
477
- pessimistic. The Instagram contract reads `ok` only when identity AND media both landed
478
- (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower
479
- count and no posts must not paint green over a posts table that did not grow"*). So: posts not
480
- ASKED for β†’ `partial`, saying so; posts asked for and landed β†’ `ok`; asked for and none came β†’
481
- `partial` with the vendor's reason. The state answers "did this pull deliver what it went for",
482
- never "did the function finish".
483
-
484
- ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the
485
- paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider,
486
- with its own note explaining that a multi-provider chain is a promise something walks it and
487
- that nothing walks Instagram's second name today either. So a refusal here is final, and it
488
- says so instead of implying a retry somewhere.
489
- """
490
- handle = tt_handle(url)
491
- if not handle:
492
- return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "",
493
- "note": f"{url!r} is not a TikTok profile URL or handle"}
494
-
495
- # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from
496
- # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not
497
- # happen; a miss falls through to the single-URL call below.
498
- cached = prefetch.get(handle) if isinstance(prefetch, dict) else None
499
- _deferred = []
500
- if isinstance(cached, dict) and cached:
501
- rows, note = [cached], ""
502
- else:
503
- rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred)
504
-
505
- node = rows[0] if rows else {}
506
- profile = normalize_profile(node, handle) if node else {}
507
- # β›” THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile`
508
- # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` β€” truthy, and
509
- # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly
510
- # this reason, and answering "0 followers" instead is the failure it exists to prevent.
511
- unreadable = profile.get("followers") is None and profile.get("following") is None
512
- if note or unreadable:
513
- # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still
514
- # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the
515
- # same record. That was live on the Instagram profile path until 2026-08-09 β€” measured on
516
- # nurilab as two runs, two fresh snapshots, both abandoned β€” and it is not being
517
- # reintroduced here by omission.
518
- if isinstance(pending_profile, list):
519
- for d in _deferred:
520
- pending_profile.append({**d, "kind": "profile", "influencer": handle})
521
- why = note or ("the scrape answered, but no follower/following counts were readable in it "
522
- "(the field names may have moved - see tiktok-capture.md)")
523
- return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
524
- "deferredProfile": [d.get("snapshotId") for d in _deferred],
525
- "note": why}
526
-
527
- # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------
528
- if not post_metrics:
529
- return {"state": "partial", "profile": profile, "posts": [], "comments": [],
530
- "via": "brightdata",
531
- "note": note or "profile read; post capture is off for this step"}
532
- urls = tt_post_urls(node, limit=max_posts)
533
- if not urls:
534
- # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
535
- # saying so is what stops the next run paying to be told the same thing.
536
- return {"state": "partial", "profile": profile, "posts": [], "comments": [],
537
- "via": "brightdata",
538
- "note": note or "profile read; this account's row carried no post links"}
539
- posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred)
540
- comments, c_note = ([], "")
541
- if comment_metrics and posts:
542
- # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought β€”
543
- # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused
544
- # is not silently asked about again one rung later.
545
- comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")],
546
- log=log, deferred=_deferred)
547
- # ⭐⭐ WAVE 30 Β· D-156 β€” THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is
548
- # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to
549
- # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE
550
- # deferral is impossible β€” the profile branch above returns `blocked` on any note β€” so **every
551
- # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose
552
- # collector writes preset profile cells onto somebody's record from post rows. The engine keeps
553
- # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`).
554
- # β‡’ So this returns them under their OWN key, filtered by dataset identity
555
- # (`_media_deferrals`), and the engine files them in the metric queue with the handle it
556
- # already holds. Returning rather than appending also keeps the queue's vocabulary out of a
557
- # connector: this module knows which CORPUS deferred, never what the engine calls it.
558
- # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that
559
- # matters most β€” that is exactly the run where the vendor took too long, so a caller reading
560
- # the ids only from the success path would lose every batch it actually paid for.
561
- deferred_media = _media_deferrals(_deferred)
562
- if not posts:
563
- return {"state": "partial", "profile": profile, "posts": [], "comments": [],
564
- "via": "brightdata", "deferredMedia": deferred_media,
565
- "note": p_note or note or "profile read; the post source returned nothing"}
566
- return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
567
- "via": "brightdata", "deferredMedia": deferred_media,
568
- "note": c_note or note or ""}
 
1
+ """connectors_tt.py β€” the TIKTOK connector (wave 29 Β· item 7 Β· DEBT D-9 Β· rulings R1 + R2).
2
+
3
+ Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
4
+ automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
5
+ exists at all rather than another thousand lines inside the engine.
6
+
7
+ β›” **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema β€” 40 profile / 43
8
+ post / 17 comment fields, each with the vendor's own type, description and `pii` flag β€” was read
9
+ live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
10
+ `.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
11
+ close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
12
+ name below reads through a candidate list it is because the vendor has two names for one fact
13
+ (`biography`/`signature`, `region`/`country`), never because the name is uncertain.
14
+
15
+ β›” **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned β€”
16
+ public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
17
+ vendor key is a key to a SUPPLIER.
18
+
19
+ ⭐ **THE TRANSPORT IS `connectors_bd.py` β€” SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE
20
+ OTHER PLATFORM'S CONNECTOR** (WAVE 30 Β· T09, DEBT D-128). `bd_call`, `bd_scrape` and
21
+ `bd_filter_start` take the dataset id as a PARAMETER β€” they are Bright Data's wire, not
22
+ Instagram's β€” and re-implementing them here would be a second copy of the deferral handling, the
23
+ truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
24
+ Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from
25
+ `connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier.
26
+ ⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because
27
+ the sentence above is the kind that quietly stops being true.
28
+
29
+ ⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
30
+ 1. the real ROW shape β€” `/metadata` describes a DATASET, and Instagram's rows carry undeclared
31
+ envelope keys (`timestamp`, `input`) that no metadata call mentions;
32
+ 2. that a declared field POPULATES β€” Bright Data's Instagram Reels *declares* `views: number`
33
+ and delivers an account-grain wrong number (Β§4e). **Declared is not delivered**, and the one
34
+ TikTok claim that matters most (`play_count`) is exactly a declaration.
35
+ """
36
+ from __future__ import annotations
37
+
38
+ import automation_engine as engine
39
+ # ⚠ T09 β€” FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an
40
+ # import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were
41
+ # imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever
42
+ # read them off this module (the engine imports them from the transport itself), so they were four
43
+ # lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form.
44
+ from connectors_bd import (
45
+ _bd_first_url,
46
+ _bd_flag,
47
+ _bd_list,
48
+ _bd_source_payload,
49
+ _first,
50
+ _ig_int,
51
+ bd_scrape,
52
+ )
53
+
54
+ # ---------------------------------------------------------------------------------------------
55
+ # THE DATASETS
56
+ # ---------------------------------------------------------------------------------------------
57
+ # ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT β€” the same finding Instagram produced. `GET
58
+ # /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
59
+ # full field list, and the two DISCOVERY halves answer **404 for our key**:
60
+ # `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
61
+ # β‡’ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
62
+ # does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
63
+ TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
64
+ TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
65
+ TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
66
+
67
+ #: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
68
+ #: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
69
+ TT_POST_TYPE_VIDEO = "video"
70
+ TT_POST_TYPE_CONTENT = "content"
71
+
72
+ #: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
73
+ #: beside the dataset ids because it is the same class of vendor fact.
74
+ #: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
75
+ #: string from `_PROFILE_RULES` (contract C2, E's half) β€” this exists for the connector's own
76
+ #: batch calls, and the gate asserts the two agree rather than trusting that they do.
77
+ TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
78
+
79
+
80
+ def tt_profile_url(handle):
81
+ """`nurilab` β†’ `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
82
+ h = str(handle or "").strip().lstrip("@")
83
+ return TT_PROFILE_URL.format(handle=h) if h else ""
84
+
85
+
86
+ # ---------------------------------------------------------------------------------------------
87
+ # THE FIELD MAPS
88
+ # ---------------------------------------------------------------------------------------------
89
+ # Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
90
+ # in `automation_engine`. The rules they all obey, stated once:
91
+ #
92
+ # * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
93
+ # so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
94
+ # returns `None` (never 0) when nothing matches, which is what makes that possible.
95
+ # * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
96
+ # rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
97
+ # * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
98
+ # vendor's side is preserved rather than silently discarded while our column model catches up.
99
+ # * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
100
+ # (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
101
+ # ride in `source_payload` where they make no claim.
102
+
103
+
104
+ def _tt_str(node, *names):
105
+ """The first non-empty string among `names`, or None when the vendor sent nothing.
106
+
107
+ β›” `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest β€”
108
+ an empty string written into a cell claims "we looked and it is empty".
109
+ """
110
+ v = _first(node, *names)
111
+ if v is None:
112
+ return None
113
+ s = str(v).strip()
114
+ return s or None
115
+
116
+
117
+ def _tt_pct(node, *names):
118
+ """A vendor 0–1 engagement fraction β†’ our stored 0–100 percentage, or None.
119
+
120
+ β›” THE Γ—100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
121
+ the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` β€”
122
+ measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
123
+ """
124
+ v = _first(node, *names)
125
+ if v is None:
126
+ return None
127
+ out = engine._pct100(v)
128
+ return out or None
129
+
130
+
131
+ def _tt_day(node, *names):
132
+ """A vendor stamp β†’ `YYYY-MM-DD`, or None. Our `date` columns store a day."""
133
+ v = _first(node, *names)
134
+ if v is None:
135
+ return None
136
+ return engine._day(v) or None
137
+
138
+
139
+ def _drop_blanks(row):
140
+ """The one place a mapped row loses its `None`s β€” see the BLANK MEANS NOT READ rule above."""
141
+ return {k: v for k, v in row.items() if v is not None and v != ""}
142
+
143
+
144
+ def normalize_profile(node, handle=""):
145
+ """A TikTok Profiles row β†’ the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
146
+
147
+ ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
148
+ * `biography` is PRIMARY and `signature` the FALLBACK β€” the probe measured `signature`
149
+ populated on 85% of rows and they carry the same text;
150
+ * `region` is PRIMARY and `country` the FALLBACK β€” `region` is the one with a documented
151
+ two-letter-ISO description, `country` has no description at all.
152
+ ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
153
+ `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
154
+ only count of its kind the dataset offers.
155
+ """
156
+ node = node if isinstance(node, dict) else {}
157
+ account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
158
+ return _drop_blanks({
159
+ "platform": engine.PLATFORM_TIKTOK,
160
+ "handle": account,
161
+ "full_name": _tt_str(node, "nickname"),
162
+ "tt_id": _tt_str(node, "id"),
163
+ "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
164
+ "bio": _tt_str(node, "biography", "signature"),
165
+ "external_url": _bd_first_url(_first(node, "bio_link")),
166
+ "verified": _bd_flag(node, "is_verified"),
167
+ "is_private": _bd_flag(node, "is_private"),
168
+ # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
169
+ # on their own description. It is the closest thing TikTok has to Instagram's
170
+ # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent β€” so
171
+ # the approximation only ever fills a cell the vendor actually answered.
172
+ "is_business": _bd_flag(node, "is_commerce_user"),
173
+ "followers": _ig_int(_first(node, "followers")),
174
+ "following": _ig_int(_first(node, "following")),
175
+ "posts_count": _ig_int(_first(node, "videos_count")),
176
+ # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
177
+ # nulls). It is not a post-level number and it is not our `likes` column, which is why it
178
+ # is stored under a different name.
179
+ "likes_received": _ig_int(_first(node, "likes")),
180
+ "avg_engagement": _tt_pct(node, "awg_engagement_rate"),
181
+ "like_engagement": _tt_pct(node, "like_engagement_rate"),
182
+ "comment_engagement": _tt_pct(node, "comment_engagement_rate"),
183
+ "country_code": _tt_str(node, "region", "country"),
184
+ "region": _tt_str(node, "region"),
185
+ "predicted_lang": _tt_str(node, "predicted_lang"),
186
+ # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP β€” `create_time` on a profile is when the ACCOUNT
187
+ # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
188
+ # which is why the append law dates a snapshot by when WE read it.
189
+ "account_created_at": _tt_day(node, "create_time"),
190
+ "source_payload": _bd_source_payload(node),
191
+ })
192
+
193
+
194
+ def tt_post_type(node):
195
+ """A TikTok post row β†’ one of OUR three type options, or None.
196
+
197
+ β›” THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
198
+ `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
199
+ vendor's token would fail `_clean_field`'s option check on the way in and would put an
200
+ untranslated API word in front of a user on the way out.
201
+
202
+ So: `video` is `video`, and `content` β€” TikTok's photo-mode post β€” is `image`, EXCEPT when the
203
+ row carries more than one `carousel_images` entry, which is what a carousel IS on either
204
+ network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the
205
+ token cannot express it, so inferring `carousel` from the word would be inventing a fact.
206
+ ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
207
+ that is exactly what would make the wrong default invisible).
208
+ """
209
+ raw = str((node or {}).get("post_type") or "").strip().lower()
210
+ if raw == TT_POST_TYPE_VIDEO:
211
+ return "video"
212
+ if raw != TT_POST_TYPE_CONTENT:
213
+ return None
214
+ images = (node or {}).get("carousel_images")
215
+ return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
216
+
217
+
218
+ def normalize_post(node):
219
+ """A TikTok Posts row β†’ the `ut_tt_posts` cell shape. None when it carries no identity.
220
+
221
+ β›” THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
222
+ Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
223
+ fact β€” `play_count` IS the count TikTok displays under a video β€” so it maps to `views` and
224
+ `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
225
+ manufacture a second measurement that a rollup could average or double-count, which is a worse
226
+ outcome than the missing column it would paper over.
227
+
228
+ ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
229
+ the probe measured them as the same shape. That equality is what lets the comments→posts link
230
+ join with NO normaliser, which the Instagram side never had.
231
+ ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
232
+ ⚠ `commerce_info` is a business/commerce LOCATION per its own description β€” cities and
233
+ countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
234
+ no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
235
+ """
236
+ node = node if isinstance(node, dict) else {}
237
+ shortcode = _tt_str(node, "shortcode", "post_id")
238
+ if not shortcode:
239
+ return None
240
+ return _drop_blanks({
241
+ "platform": engine.PLATFORM_TIKTOK,
242
+ "shortcode": shortcode,
243
+ # β›”β›” `account_id`, NOT `profile_username` β€” MEASURED on a real Posts row 2026-08-12.
244
+ # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the
245
+ # @handle (`"d1na_th"`) β€” the same field `normalize_profile` already reads for `handle`, so
246
+ # one name means one thing across both corpora. Reading the display name silently broke the
247
+ # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched
248
+ # NOTHING, so a person could not filter posts by creator and a rollup would count zero.
249
+ # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a
250
+ # different fact, and filling a join key with it is worse than leaving it blank β€” a blank
251
+ # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]].
252
+ # The URL is the honest second source: it carries the handle by construction.
253
+ "influencer_key": (_tt_str(node, "account_id")
254
+ or tt_handle(_tt_str(node, "profile_url", "url") or "") or None),
255
+ "posted_at": _tt_day(node, "create_time"),
256
+ "type": tt_post_type(node),
257
+ "caption": _tt_str(node, "description"),
258
+ "url": _tt_str(node, "url"),
259
+ "hashtags": _bd_list(node, "hashtags"),
260
+ "tagged_location": _tt_str(node, "commerce_info"),
261
+ "views": _ig_int(_first(node, "play_count")),
262
+ "likes": _ig_int(_first(node, "digg_count")),
263
+ "comments": _ig_int(_first(node, "comment_count")),
264
+ "shares": _ig_int(_first(node, "num_share_count")),
265
+ "saves": _ig_int(_first(node, "collect_count")),
266
+ "video_duration": _ig_int(_first(node, "video_duration")),
267
+ "source_payload": _bd_source_payload(node),
268
+ })
269
+
270
+
271
+ def normalize_comment(node):
272
+ """A TikTok Comments row β†’ the `ut_tt_comments` cell shape. None without a comment id.
273
+
274
+ ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
275
+ `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
276
+ `source_payload`. Reading the array's length instead would be a second, disagreeing answer to
277
+ a question the vendor already answers β€” and it would disagree, because a page of replies is not
278
+ all of them.
279
+ ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
280
+ text and needs defensive parsing. It still goes through `_tt_day` β€” one date path, so a vendor
281
+ that changes its mind cannot change ours.
282
+ ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
283
+ PII) stay in `source_payload` and are promoted to no column, which is the same posture the
284
+ Instagram comment schema takes for the same D-24 reason.
285
+ """
286
+ node = node if isinstance(node, dict) else {}
287
+ comment_key = _tt_str(node, "comment_id")
288
+ if not comment_key:
289
+ return None
290
+ return _drop_blanks({
291
+ "platform": engine.PLATFORM_TIKTOK,
292
+ "comment_key": comment_key,
293
+ "shortcode": _tt_str(node, "post_id"),
294
+ # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
295
+ # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
296
+ # primary first, so a row carrying the rich form is not silently served the plain one.
297
+ # β›” The commenter's identity is deliberately NOT promoted β€” see `TT_COMMENT_FIELDS`.
298
+ "text": _tt_str(node, "comment_text", "comment_text_only"),
299
+ "commented_at": _tt_day(node, "date_created"),
300
+ "likes": _ig_int(_first(node, "num_likes")),
301
+ "replies": _ig_int(_first(node, "num_replies")),
302
+ "source_payload": _bd_source_payload(node),
303
+ })
304
+
305
+
306
+ #: ⭐ The three maps, addressable by name β€” so a gate (and the runners in T04-T06) can walk them
307
+ #: rather than naming three functions, and so adding a fourth dataset is one entry.
308
+ TT_NORMALIZERS = {
309
+ "tt_profile": normalize_profile,
310
+ "tt_post_metrics": normalize_post,
311
+ "tt_comments": normalize_comment,
312
+ }
313
+
314
+
315
+ # ---------------------------------------------------------------------------------------------
316
+ # THE FETCH β€” wave 30 Β· W30-T08 (carrying wave-29's dropped T05)
317
+ # ---------------------------------------------------------------------------------------------
318
+
319
+ def tt_handle(url):
320
+ """A TikTok profile URL **or** a bare handle β†’ the handle. `''` when it is neither.
321
+
322
+ Deliberately permissive about the input and strict about the output, because the two callers
323
+ hand it different things: an automation stores whatever a person typed in the profile column
324
+ (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean
325
+ `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot
326
+ resolve to two different handles.
327
+ """
328
+ s = str(url or "").strip()
329
+ if not s:
330
+ return ""
331
+ if "tiktok.com" in s.lower():
332
+ # Everything after the first `@`, up to the next path segment or query.
333
+ tail = s.split("@", 1)[1] if "@" in s else ""
334
+ s = tail.split("/")[0].split("?")[0].split("#")[0]
335
+ s = s.strip().lstrip("@").strip()
336
+ # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores.
337
+ return s if s and all(c.isalnum() or c in "._" for c in s) else ""
338
+
339
+
340
+ def tt_post_urls(node, limit=0):
341
+ """⭐ WAVE 30 Β· T10 β€” the profile row's own post permalinks, newest-first as the vendor sends.
342
+
343
+ β›” THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
344
+ we have already bought, so the posts read is a scrape of links we hold, never a search for them.
345
+ The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**,
346
+ so a design that reached for either would not merely be dearer, it would not work.
347
+
348
+ β›”β›” CORRECTED 2026-08-12 β€” THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an
349
+ array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe
350
+ read `/datasets/{id}/metadata` β€” a DATASET description β€” and the phrase quoted was the field's
351
+ `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`**
352
+ (measured below), so the reader built against the quoted sentence found nothing, forever, in
353
+ silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration
354
+ of X"* are different claims, and prose cannot be told apart by a reader downstream β€” which is
355
+ why the correction names the method, not just the value.
356
+
357
+ ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*,
358
+ and preferring whichever happened to be longer is how one creator's window silently differs
359
+ from another's.
360
+ ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S β€” `config.maxPosts`,
361
+ validated 1..12 β€” and it is applied here rather than after the scrape so an unwanted post is
362
+ never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window,
363
+ not a number this function may invent.
364
+ """
365
+ raw = (node or {}).get("top_videos")
366
+ out = []
367
+ for item in raw if isinstance(raw, list) else []:
368
+ # β›”β›” MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
369
+ # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a
370
+ # live handle returns **19 DICTS**, keyed
371
+ # `video_url Β· video_id Β· playcount Β· diggcount Β· commentcount Β· share_count Β·
372
+ # favorites_count Β· create_date Β· cover_image`.
373
+ # The docstring above cites the $0 probe as having "measured" permalinks β€” it had not, and
374
+ # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says
375
+ # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field
376
+ # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's
377
+ # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was
378
+ # the first. [[reachable-is-not-the-same-as-built]]
379
+ # β‡’ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY
380
+ # list on every real profile β€” post capture could not have worked for anybody, and the
381
+ # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture
382
+ # written from a schema tests the schema.
383
+ # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept
384
+ # because it costs nothing and is what a future corpus revision would most likely use. The
385
+ # bare-string branch stays for the same reason β€” this widens what is ACCEPTED and invents
386
+ # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as
387
+ # before, rather than to a URL built out of a guess.
388
+ # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring
389
+ # whichever array happened to be longer is how one creator's window silently differs from
390
+ # another's, and that reasoning is unchanged by this correction.
391
+ if isinstance(item, dict):
392
+ u = str(item.get("video_url") or item.get("url") or "").strip()
393
+ else:
394
+ u = str(item or "").strip()
395
+ if u.lower().startswith("http") and u not in out:
396
+ out.append(u)
397
+ return out[:limit] if limit and limit > 0 else out
398
+
399
+
400
+ def pull_posts_tt(post_urls, log=print, deferred=None):
401
+ """The TikTok Posts dataset for a list of permalinks β†’ `(rows, note)`, already normalised.
402
+
403
+ ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side
404
+ measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still
405
+ running at 67 minutes. Nothing here loops per URL.
406
+ """
407
+ urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
408
+ if not urls:
409
+ return [], ""
410
+ rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred)
411
+ if note:
412
+ log(f"[aios-tt] posts: {note}")
413
+ return [], note
414
+ out = [r for r in (normalize_post(n) for n in rows) if r]
415
+ return out, ""
416
+
417
+
418
+ def pull_comments_tt(post_urls, log=print, deferred=None):
419
+ """The TikTok Comments dataset for a list of POST permalinks β†’ `(rows, note)`, normalised.
420
+
421
+ β›” THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on
422
+ both networks: a comments scrape ingests identifiable third parties who never entered anybody's
423
+ list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes
424
+ none of them to a column; this function adds no new exposure, it just has to be asked for.
425
+ """
426
+ urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
427
+ if not urls:
428
+ return [], ""
429
+ rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred)
430
+ if note:
431
+ log(f"[aios-tt] comments: {note}")
432
+ return [], note
433
+ out = [r for r in (normalize_comment(n) for n in rows) if r]
434
+ return out, ""
435
+
436
+
437
+ #: ⭐⭐ WAVE 30 Β· D-156 β€” THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY.
438
+ #: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral
439
+ #: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's
440
+ #: metric queue β€” the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse
441
+ #: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can.
442
+ TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS)
443
+
444
+
445
+ def _media_deferrals(deferred):
446
+ """The POSTS/COMMENTS entries of a `bd_scrape` deferral list β€” never the profile's.
447
+
448
+ ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the
449
+ handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one
450
+ kind here, so the id already carries everything a mapper choice depends on β€” and importing
451
+ Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines).
452
+ """
453
+ out = []
454
+ for d in deferred or []:
455
+ if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS:
456
+ out.append(dict(d))
457
+ return out
458
+
459
+
460
+ def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
461
+ max_posts=0, post_metrics=False, comment_metrics=False):
462
+ """ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
463
+
464
+ `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
465
+ so the engine's enrich branch treats every network identically and no caller learns a new
466
+ shape.
467
+
468
+ ⭐ WAVE 30 Β· T10 β€” POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's
469
+ do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` β€”
470
+ no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics`
471
+ then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction
472
+ rather than by a rule: their input IS a post permalink, so asking for comments with post capture
473
+ off is a request with no subject, and it returns none instead of quietly buying posts nobody
474
+ asked for.
475
+
476
+ β›” `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than
477
+ pessimistic. The Instagram contract reads `ok` only when identity AND media both landed
478
+ (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower
479
+ count and no posts must not paint green over a posts table that did not grow"*). So: posts not
480
+ ASKED for β†’ `partial`, saying so; posts asked for and landed β†’ `ok`; asked for and none came β†’
481
+ `partial` with the vendor's reason. The state answers "did this pull deliver what it went for",
482
+ never "did the function finish".
483
+
484
+ ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the
485
+ paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider,
486
+ with its own note explaining that a multi-provider chain is a promise something walks it and
487
+ that nothing walks Instagram's second name today either. So a refusal here is final, and it
488
+ says so instead of implying a retry somewhere.
489
+ """
490
+ handle = tt_handle(url)
491
+ if not handle:
492
+ return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "",
493
+ "note": f"{url!r} is not a TikTok profile URL or handle"}
494
+
495
+ # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from
496
+ # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not
497
+ # happen; a miss falls through to the single-URL call below.
498
+ cached = prefetch.get(handle) if isinstance(prefetch, dict) else None
499
+ _deferred = []
500
+ if isinstance(cached, dict) and cached:
501
+ rows, note = [cached], ""
502
+ else:
503
+ rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred)
504
+
505
+ node = rows[0] if rows else {}
506
+ profile = normalize_profile(node, handle) if node else {}
507
+ # β›” THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile`
508
+ # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` β€” truthy, and
509
+ # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly
510
+ # this reason, and answering "0 followers" instead is the failure it exists to prevent.
511
+ unreadable = profile.get("followers") is None and profile.get("following") is None
512
+ if note or unreadable:
513
+ # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still
514
+ # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the
515
+ # same record. That was live on the Instagram profile path until 2026-08-09 β€” measured on
516
+ # nurilab as two runs, two fresh snapshots, both abandoned β€” and it is not being
517
+ # reintroduced here by omission.
518
+ if isinstance(pending_profile, list):
519
+ for d in _deferred:
520
+ pending_profile.append({**d, "kind": "profile", "influencer": handle})
521
+ why = note or ("the scrape answered, but no follower/following counts were readable in it "
522
+ "(the field names may have moved - see tiktok-capture.md)")
523
+ return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
524
+ "deferredProfile": [d.get("snapshotId") for d in _deferred],
525
+ "note": why}
526
+
527
+ # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------
528
+ if not post_metrics:
529
+ return {"state": "partial", "profile": profile, "posts": [], "comments": [],
530
+ "via": "brightdata",
531
+ "note": note or "profile read; post capture is off for this step"}
532
+ urls = tt_post_urls(node, limit=max_posts)
533
+ if not urls:
534
+ # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
535
+ # saying so is what stops the next run paying to be told the same thing.
536
+ return {"state": "partial", "profile": profile, "posts": [], "comments": [],
537
+ "via": "brightdata",
538
+ "note": note or "profile read; this account's row carried no post links"}
539
+ posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred)
540
+ comments, c_note = ([], "")
541
+ if comment_metrics and posts:
542
+ # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought β€”
543
+ # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused
544
+ # is not silently asked about again one rung later.
545
+ comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")],
546
+ log=log, deferred=_deferred)
547
+ # ⭐⭐ WAVE 30 Β· D-156 β€” THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is
548
+ # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to
549
+ # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE
550
+ # deferral is impossible β€” the profile branch above returns `blocked` on any note β€” so **every
551
+ # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose
552
+ # collector writes preset profile cells onto somebody's record from post rows. The engine keeps
553
+ # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`).
554
+ # β‡’ So this returns them under their OWN key, filtered by dataset identity
555
+ # (`_media_deferrals`), and the engine files them in the metric queue with the handle it
556
+ # already holds. Returning rather than appending also keeps the queue's vocabulary out of a
557
+ # connector: this module knows which CORPUS deferred, never what the engine calls it.
558
+ # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that
559
+ # matters most β€” that is exactly the run where the vendor took too long, so a caller reading
560
+ # the ids only from the success path would lose every batch it actually paid for.
561
+ deferred_media = _media_deferrals(_deferred)
562
+ if not posts:
563
+ return {"state": "partial", "profile": profile, "posts": [], "comments": [],
564
+ "via": "brightdata", "deferredMedia": deferred_media,
565
+ "note": p_note or note or "profile read; the post source returned nothing"}
566
+ return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
567
+ "via": "brightdata", "deferredMedia": deferred_media,
568
+ "note": c_note or note or ""}
api/main.py CHANGED
@@ -85,6 +85,8 @@ import routes_slack # noqa: E402 (wave 33 R4/C2 β€” Manage agent + the Slack d
85
  import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 β€” the star; E's router, E's line)
86
  import routes_usage # noqa: E402 (wave 35 R9/C7 β€” the AI usage meter; E's router, E's line)
87
  import routes_feedback # noqa: E402 (wave 35 R8/C6 β€” feedback to the operator plane; E's router)
 
 
88
  from core import grid_events # noqa: E402
89
  # D-315 / D-305 (2026-08-18) β€” the two store REFUSALS get their own app-level handlers below.
90
  # ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
@@ -376,6 +378,23 @@ app.include_router(routes_usage.router) # R9 / C7 β€” GET /usage, the one
376
  # POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does
377
  # not widen anything a tenant admin can reach β€” `verify_api` proves that by having one try.
378
  app.include_router(routes_feedback.router) # R8 / C6 β€” feedback to the operator plane
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
379
 
380
 
381
  # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
 
85
  import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 β€” the star; E's router, E's line)
86
  import routes_usage # noqa: E402 (wave 35 R9/C7 β€” the AI usage meter; E's router, E's line)
87
  import routes_feedback # noqa: E402 (wave 35 R8/C6 β€” feedback to the operator plane; E's router)
88
+ import routes_agent_harness # noqa: E402 (wave 36 R8/C4 β€” the agent harness store; E's router)
89
+ import routes_script_views # noqa: E402 (wave 36 R3/R10/C3 β€” script Views; E's router)
90
  from core import grid_events # noqa: E402
91
  # D-315 / D-305 (2026-08-18) β€” the two store REFUSALS get their own app-level handlers below.
92
  # ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
 
378
  # POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does
379
  # not widen anything a tenant admin can reach β€” `verify_api` proves that by having one try.
380
  app.include_router(routes_feedback.router) # R8 / C6 β€” feedback to the operator plane
381
+ # ⭐⭐ WAVE 36 (R8 / C4) β€” THE AGENT HARNESS FILE STORE, mounted in the SAME change that created
382
+ # `routes_agent_harness.py`. Eighth consecutive wave in which this block is the artefact the
383
+ # protocol nearly loses; `verify_web_agent::section_w36_harness` asserts this path in
384
+ # `app.openapi()["paths"]` and CALLS the route, because mounted is not callable (D-107).
385
+ # ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the
386
+ # end of this file, or a GET answers 404 and a PUT answers 405 while every gate stays green.
387
+ # β›” ITS PATHS SIT UNDER `/agents/{id}/...`, WHICH `routes_slack` ALSO SERVES β€” and that is safe
388
+ # rather than lucky: a FastAPI path parameter never spans a `/`, so `/agents/{agent_id}` cannot
389
+ # match `/agents/x/harness`. The two routers share a prefix and no route.
390
+ app.include_router(routes_agent_harness.router) # R8 / C4 β€” versioned agent harness files
391
+ # ⭐⭐ WAVE 36 (R3 / R10 / C3) β€” THE SCRIPT VIEW, owner item 6. Mounted in the SAME change that
392
+ # created `routes_script_views.py`; `verify_script_views.py` asserts both of its paths in
393
+ # `app.openapi()["paths"]` AND calls them, and its NC comments this line out.
394
+ # β›” ITS RUN DOOR SPAWNS A SUBPROCESS AND IS A PLAIN `def`, so FastAPI runs it in the threadpool.
395
+ # Mounting it does not put a ten-second wait anywhere near the event loop; the router's own header
396
+ # says why that is not a style choice.
397
+ app.include_router(routes_script_views.router) # R3 / R10 / C3 β€” code-script database Views
398
 
399
 
400
  # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
api/odoo_relational.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/providers.py CHANGED
@@ -124,7 +124,7 @@ PROVIDERS: dict[str, Provider] = {
124
  "likes + comments + shares + saves, and play_count inline (DECLARED "
125
  "no-empties-or-zeros, UNPROVEN until one live pull)"),
126
  "tt_comments": Capability(True, _COST_BRIGHTDATA,
127
- "separate paid dataset, opt-in β€” 17 fields"),
128
  }),
129
  "apify": Provider(
130
  key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
@@ -132,7 +132,7 @@ PROVIDERS: dict[str, Provider] = {
132
  # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
133
  # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
134
  "ig_post_views": Capability(True, _COST_APIFY,
135
- "videoPlayCount β€” matches Instagram's displayed views"),
136
  "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"),
137
  "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"),
138
  }),
@@ -240,12 +240,12 @@ def run(capability, work, satisfied=None, log=None):
240
  secs = round(time.time() - started, 2)
241
  if note:
242
  attempts.append(Attempt(provider.key, False, note, 0, secs))
243
- log(f" {provider.label}: {note} β€” falling through")
244
  continue
245
  if not ok(result):
246
  attempts.append(Attempt(provider.key, False, "answered without the field asked for",
247
  0, secs))
248
- log(f" {provider.label}: answered, but not with what was asked for β€” falling through")
249
  last = result if last is None else last
250
  continue
251
  attempts.append(Attempt(provider.key, True, "", _count(result), secs))
@@ -282,6 +282,323 @@ def wire():
282
  }
283
 
284
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
285
  # =============================================================================================
286
  # THE CANONICAL SCHEMA β€” owner ruling 2026-08-08:
287
  # *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set
 
124
  "likes + comments + shares + saves, and play_count inline (DECLARED "
125
  "no-empties-or-zeros, UNPROVEN until one live pull)"),
126
  "tt_comments": Capability(True, _COST_BRIGHTDATA,
127
+ "separate paid dataset, opt-in: 17 fields"),
128
  }),
129
  "apify": Provider(
130
  key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
 
132
  # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
133
  # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
134
  "ig_post_views": Capability(True, _COST_APIFY,
135
+ "videoPlayCount, matches Instagram's displayed views"),
136
  "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"),
137
  "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"),
138
  }),
 
240
  secs = round(time.time() - started, 2)
241
  if note:
242
  attempts.append(Attempt(provider.key, False, note, 0, secs))
243
+ log(f" {provider.label}: {note}, falling through")
244
  continue
245
  if not ok(result):
246
  attempts.append(Attempt(provider.key, False, "answered without the field asked for",
247
  0, secs))
248
+ log(f" {provider.label}: answered, but not with what was asked for, falling through")
249
  last = result if last is None else last
250
  continue
251
  attempts.append(Attempt(provider.key, True, "", _count(result), secs))
 
282
  }
283
 
284
 
285
+
286
+
287
+ # ═══════════════════ WAVE 36 Β· W36-T35 (ruling R4, contract C5) β€” THE LLM LADDER ════════════════
288
+ #
289
+ # Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the
290
+ # assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras:
291
+ # HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this
292
+ # shape'."*
293
+ #
294
+ # ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the
295
+ # tool-calling path that always works β€” a WIRE, in `routes_query`. (2) A provider with no credit is
296
+ # SKIPPED rather than tried β€” a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a
297
+ # screen β€” a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read.
298
+ #
299
+ # ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is
300
+ # `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is
301
+ # billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half
302
+ # the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the
303
+ # thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both
304
+ # sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does.
305
+ #
306
+ # β›” THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that
307
+ # recovers a tool call out of a provider's 400 β€” the evidence that guessing at capability failed.
308
+ # A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling
309
+ # turn at all, so the guess never has to be made.
310
+
311
+ #: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT:
312
+ #: the balance can be topped up at any moment, so this expires rather than latching. Fifteen
313
+ #: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short
314
+ #: enough that a top-up is picked up without a restart.
315
+ CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900)
316
+
317
+ #: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO β€” it is a
318
+ #: latency optimisation, not a billing record. A second container learns the same thing from its
319
+ #: own first 402, and neither one can be wrong for longer than the cooldown.
320
+ _NO_CREDIT: dict[str, float] = {}
321
+
322
+
323
+ @dataclass(frozen=True)
324
+ class LlmProvider:
325
+ """One chat-completions endpoint, and what it is DECLARED able to do."""
326
+ name: str
327
+ label: str
328
+ env: str
329
+ url: str
330
+ model: str
331
+ #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API,
332
+ #: which is a different body, a different auth header and a different result shape.
333
+ wire: str
334
+ caps: dict = field(default_factory=dict)
335
+
336
+ def configured(self) -> bool:
337
+ return bool((os.environ.get(self.env) or "").strip())
338
+
339
+ def can(self, capability: str) -> bool:
340
+ cap = self.caps.get(capability)
341
+ return bool(cap and cap.capable and self.configured())
342
+
343
+
344
+ #: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put
345
+ #: cerebras first *"because this path needs tool calling and cerebras carries this account's
346
+ #: tool-capable model"* β€” R4 replaces that premise: Anthropic is the tool-calling path that always
347
+ #: works, and the others are the cheap seats it falls through to.
348
+ LLM_PROVIDERS: dict[str, LlmProvider] = {
349
+ "anthropic": LlmProvider(
350
+ name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY",
351
+ url="https://api.anthropic.com/v1/messages",
352
+ # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with
353
+ # `AIOS_ANTHROPIC_MODEL` β€” the id is read at call time, so a cheaper tier
354
+ # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release.
355
+ model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5",
356
+ wire="anthropic",
357
+ caps={
358
+ # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a
359
+ # typed `tool_use` content block carrying parsed `input`. There is nothing to recover
360
+ # out of a 400 and no regex in the path β€” which is exactly what `_FAILED_GEN` exists to
361
+ # apologise for on the other wire.
362
+ "llm_tool_calling": Capability(True, 0.0,
363
+ "typed tool_use content block; no text recovery path"),
364
+ "llm_chat": Capability(True, 0.0, "Messages API"),
365
+ "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"),
366
+ }),
367
+ "cerebras": LlmProvider(
368
+ name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY",
369
+ url="https://api.cerebras.ai/v1/chat/completions",
370
+ model="gpt-oss-120b", wire="openai",
371
+ caps={
372
+ "llm_tool_calling": Capability(True, 0.0,
373
+ "DECLARED, not measured: this account's tool-capable "
374
+ "model per the wave-32 ladder note"),
375
+ "llm_chat": Capability(True, 0.0, "OpenAI-compatible"),
376
+ "llm_json_mode": Capability(True, 0.0, "response_format json_object"),
377
+ }),
378
+ "groq": LlmProvider(
379
+ name="groq", label="Groq", env="GROQ_API_KEY",
380
+ url="https://api.groq.com/openai/v1/chat/completions",
381
+ model="llama-3.3-70b-versatile", wire="openai",
382
+ caps={
383
+ "llm_tool_calling": Capability(True, 0.0,
384
+ "DECLARED, not measured. ⚠ `routes_query._FAILED_GEN` "
385
+ "exists because SOME provider on this wire answers 400 "
386
+ "with the call in `failed_generation`; the repo never "
387
+ "recorded which. Flip this to False the day it is"),
388
+ "llm_chat": Capability(True, 0.0, "OpenAI-compatible"),
389
+ "llm_json_mode": Capability(True, 0.0, "response_format json_object"),
390
+ }),
391
+ "openrouter": LlmProvider(
392
+ name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY",
393
+ url="https://openrouter.ai/api/v1/chat/completions",
394
+ model="openai/gpt-4o-mini", wire="openai",
395
+ caps={
396
+ "llm_tool_calling": Capability(True, 0.0, "DECLARED, not measured"),
397
+ "llm_chat": Capability(True, 0.0, "OpenAI-compatible"),
398
+ "llm_json_mode": Capability(True, 0.0, "response_format json_object"),
399
+ }),
400
+ }
401
+
402
+ LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter")
403
+
404
+
405
+ def mark_no_credit(name, seconds=None):
406
+ """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4).
407
+
408
+ β›” THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for
409
+ that to live: a stateless ladder re-tries the empty account on every single turn, pays its
410
+ round trip, and shows the reader a longer error each time. This is that memo.
411
+ """
412
+ _NO_CREDIT[str(name)] = time.time() + float(
413
+ CREDIT_COOLDOWN_S if seconds is None else seconds)
414
+ return _NO_CREDIT[str(name)]
415
+
416
+
417
+ def no_credit(name):
418
+ """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed."""
419
+ until = _NO_CREDIT.get(str(name))
420
+ if not until:
421
+ return False
422
+ if time.time() >= until:
423
+ _NO_CREDIT.pop(str(name), None)
424
+ return False
425
+ return True
426
+
427
+
428
+ def clear_credit_memo(name=None):
429
+ """Forget one memo, or all of them. For a gate, and for an operator after a top-up."""
430
+ if name is None:
431
+ _NO_CREDIT.clear()
432
+ else:
433
+ _NO_CREDIT.pop(str(name), None)
434
+
435
+
436
+ def llm_chain(capability="llm_tool_calling"):
437
+ """The provider order for `capability` β€” declaration first, credit memo second.
438
+
439
+ Three filters, in this order, and each removes a DIFFERENT kind of row:
440
+ 1. `can()` β€” declared capable AND configured. An incapable row is never offered, so a
441
+ turn cannot be spent discovering it (the `chain()` rule, one layer up).
442
+ 2. `no_credit()` β€” R4's skip. A provider that told us its balance is empty is passed over
443
+ until the memo expires.
444
+ 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment
445
+ can be moved off a vendor without a release β€” the same clause `AIOS_PROVIDER_ORDER`
446
+ carries for the scraping side.
447
+ """
448
+ raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip()
449
+ names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER)
450
+ return [LLM_PROVIDERS[n] for n in names
451
+ if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)]
452
+
453
+
454
+ #: What an HTTP status MEANS, in words a person can act on. β›”β›” R4's THIRD CLAUSE LIVES HERE AND
455
+ #: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner
456
+ #: quoted it back at us twice. Every sentence names the VENDOR and the ACTION.
457
+ _STATUS_WORDS = {
458
+ 401: "{label} would not accept our key",
459
+ 403: "{label} would not accept our key",
460
+ 402: "{label} is out of credit",
461
+ 404: "{label} does not offer the model we asked it for",
462
+ 408: "{label} took too long",
463
+ 413: "the question was too long for {label}",
464
+ 429: "{label} is rate limiting us right now",
465
+ }
466
+
467
+ #: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent
468
+ #: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the
469
+ #: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and
470
+ #: it only ever decides whether to SKIP a provider, never whether to trust one.
471
+ _CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit",
472
+ "quota exceeded", "billing", "payment required", "add credits")
473
+
474
+
475
+ def is_credit_failure(status, body=""):
476
+ """Did this response mean "the account is empty"? Status first, then the vendor's own words."""
477
+ if int(status or 0) == 402:
478
+ return True
479
+ if int(status or 0) not in (400, 403, 429):
480
+ return False
481
+ return any(word in str(body or "").lower() for word in _CREDIT_WORDS)
482
+
483
+
484
+ def refusal_sentence(name, status, body=""):
485
+ """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace.
486
+
487
+ ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key
488
+ prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a
489
+ credit problem", and what reaches the caller is this module's own wording.
490
+ """
491
+ label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label
492
+ if is_credit_failure(status, body):
493
+ return f"{label} is out of credit"
494
+ code = int(status or 0)
495
+ if code in _STATUS_WORDS:
496
+ return _STATUS_WORDS[code].format(label=label)
497
+ if 500 <= code <= 599:
498
+ return f"{label} is having trouble at their end"
499
+ return f"{label} did not answer"
500
+
501
+
502
+ # ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════
503
+ #
504
+ # β›”β›” TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API.
505
+ # `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation
506
+ # drafter) both need it, and the owner quoted an error from EACH of them in one breath:
507
+ # *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool
508
+ # calls are not wired for this shape"*. Two implementations of one wire is
509
+ # [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the
510
+ # ladder that declares the rung.
511
+ #
512
+ # FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own:
513
+ # 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message
514
+ # 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function`
515
+ # 3. `temperature` and friends are REMOVED on the current model family
516
+ # 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string
517
+ #
518
+ # ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on
519
+ # Haiku 4.5, so it is a PARAMETER here and the caller decides β€” the drafter runs on haiku and omits
520
+ # it, the assistant runs on the Opus tier and sends it.
521
+
522
+ #: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model.
523
+ ANTHROPIC_VERSION = "2023-06-01"
524
+
525
+
526
+ def anthropic_request(*, model, key, system, messages, tools, max_tokens,
527
+ tool_choice="auto", effort=None):
528
+ """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing.
529
+
530
+ `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted
531
+ out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list,
532
+ re-addressed rather than re-derived, so a schema change happens in one place.
533
+ """
534
+ system_text = "\n\n".join(str(m.get("content") or "") for m in messages
535
+ if m.get("role") == "system")
536
+ if system:
537
+ system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system)
538
+ turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"),
539
+ "content": str(m.get("content") or "")}
540
+ for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()]
541
+ body = {
542
+ "model": str(model),
543
+ "max_tokens": int(max_tokens),
544
+ "system": system_text,
545
+ "messages": turns,
546
+ "tools": [{"name": t["function"]["name"],
547
+ "description": t["function"]["description"],
548
+ "input_schema": t["function"]["parameters"]} for t in (tools or [])],
549
+ "tool_choice": {"type": "any" if tool_choice in ("required", "any") else "auto"},
550
+ }
551
+ if effort:
552
+ body["output_config"] = {"effort": str(effort)}
553
+ return {"url": LLM_PROVIDERS["anthropic"].url,
554
+ "headers": {"x-api-key": str(key),
555
+ "anthropic-version": ANTHROPIC_VERSION,
556
+ "content-type": "application/json"},
557
+ "json": body}
558
+
559
+
560
+ def anthropic_read(body):
561
+ """`(text, tool_input, refusal)` out of a Messages API answer.
562
+
563
+ β›” `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with
564
+ `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]`
565
+ unconditionally breaks on exactly the turn a person most needs explained.
566
+ ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict β€” no `json.loads`, and
567
+ no regex recovering a call out of a 400, which is what the other wire needs.
568
+ """
569
+ body = body if isinstance(body, dict) else {}
570
+ if str(body.get("stop_reason") or "") == "refusal":
571
+ return "", None, "the assistant declined to answer that one"
572
+ blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)]
573
+ text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip()
574
+ calls = [b for b in blocks if b.get("type") == "tool_use"]
575
+ got = calls[0].get("input") if calls else None
576
+ return text, (got if isinstance(got, dict) else None), None
577
+
578
+
579
+ def llm_status(capability="llm_tool_calling"):
580
+ """Per provider: configured, declared-capable, in cooldown, and WHY β€” contract C5's payload.
581
+
582
+ ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read
583
+ THIS, so a model offered in one place cannot be missing from the other, and neither can offer a
584
+ provider the ladder would refuse to call [[permitted-is-not-answerable]].
585
+ """
586
+ rows = []
587
+ for name in LLM_DEFAULT_ORDER:
588
+ p = LLM_PROVIDERS[name]
589
+ cap = p.caps.get(capability)
590
+ rows.append({
591
+ "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire,
592
+ "configured": p.configured(),
593
+ "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable),
594
+ "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable),
595
+ "capable": bool(cap and cap.capable),
596
+ "outOfCredit": no_credit(name),
597
+ "note": (cap.note if cap else ""),
598
+ })
599
+ return rows
600
+
601
+
602
  # =============================================================================================
603
  # THE CANONICAL SCHEMA β€” owner ruling 2026-08-08:
604
  # *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set
api/routes_admin.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_agent_harness.py ADDED
@@ -0,0 +1,455 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """routes_agent_harness.py β€” CONTRACT C4: the agent's HARNESS, kept as versioned files.
2
+
3
+ Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file
4
+ pertaining to the agent skills, router etc. So we build a user can build a custom harness for each
5
+ agent through the chat interface."*
6
+
7
+ GET /api/v1/agents/{id}/harness the file LIST (no bodies)
8
+ GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions
9
+ GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim
10
+ PUT /api/v1/agents/{id}/harness write a NEW version of one file
11
+ DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history
12
+
13
+ ⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write
14
+ appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the
15
+ history is a record of what happened rather than of what somebody last wanted it to look like.
16
+
17
+ β›” **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8,
18
+ `routes_automation`); a harness file is not. Do not copy this file's posture over there or that
19
+ one's over here β€” the two rulings differ on purpose and they sit one screen apart in the product.
20
+
21
+ ⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every
22
+ write through this router is made BY a signed-in administrator, so `author` is stamped from the
23
+ session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by
24
+ a person or by the agent in the chat β€” a claim the client is entitled to make, because both are
25
+ permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side
26
+ door the automation engine uses when the agent writes with no session at all; that one stamps the
27
+ agent's own id as the author, which is the only case where `author` is not a username.
28
+
29
+ β›” **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of
30
+ one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in
31
+ `trimmed` on every payload that mentions the file, so a reader can see that the history is partial
32
+ rather than infer that the file was only ever saved twice. This is the tenant document, which is
33
+ already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is
34
+ a store-sized leak with a UI in front of it.
35
+ """
36
+ from datetime import datetime, timezone
37
+
38
+ from fastapi import APIRouter, Body, Depends
39
+
40
+ from deps import Session, err, require_session
41
+
42
+ router = APIRouter(prefix="/api/v1")
43
+
44
+ #: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides
45
+ #: `runtime.store_key`'s prefix and never lands in tenant #0's namespace β€” the same rule
46
+ #: `routes_slack.AGENTS_KEY` follows for the agent records these hang off.
47
+ HARNESS_KEY = "agent_harness"
48
+
49
+ #: One body. Generous for a skill or a router file and far under the point where a single write
50
+ #: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a
51
+ #: truncation: a silently truncated skill file is a harness that does not do what its text says.
52
+ MAX_BODY_BYTES = 128 * 1024
53
+
54
+ #: Files per agent. A refusal, not a trim β€” creating the 65th file is a different act from saving
55
+ #: the 101st version of one, and only the second can be a routine consequence of ordinary editing.
56
+ MAX_FILES = 64
57
+
58
+ #: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header).
59
+ MAX_VERSIONS = 100
60
+
61
+ MAX_PATH = 200
62
+
63
+ #: What a path may contain. β›” THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK β€”
64
+ #: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is
65
+ #: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a
66
+ #: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a
67
+ #: list of forbidden sequences: an allow-list cannot be walked around by a spelling.
68
+ _PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/")
69
+
70
+ AUTHOR_KINDS = ("user", "agent")
71
+
72
+
73
+ def _now():
74
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
75
+
76
+
77
+ def _all(rt):
78
+ """`{agent_id: {path: record}}` for one tenant. `{}` on any failure β€” an unreadable bucket must
79
+ degrade to "this agent has no harness files", never to a 500 on the pane that lists them."""
80
+ try:
81
+ found = rt.get(HARNESS_KEY) or {}
82
+ except Exception: # noqa: BLE001
83
+ return {}
84
+ return found if isinstance(found, dict) else {}
85
+
86
+
87
+ def _files(rt, agent_id):
88
+ found = _all(rt).get(str(agent_id))
89
+ return found if isinstance(found, dict) else {}
90
+
91
+
92
+ #: What kind of principal an id names. `None` = this tenant has no agent with that id.
93
+ #:
94
+ #: β›”β›” THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY
95
+ #: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under
96
+ #: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface β€”
97
+ #: `Shell.tsx` mounts `<Lazily surface="Agents"><AutomationSurface /></Lazily>`, and
98
+ #: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds
99
+ #: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that
100
+ #: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival
101
+ #: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on
102
+ #: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO
103
+ #: occurrences of Canvas, Properties, Run history or panelTabs.
104
+ #:
105
+ #: ⭐ ONE STORE, BOTH PRINCIPALS β€” never a second bucket keyed by surface. The owner's "agent
106
+ #: skills, router" files in two places is the parallel code path item 13 exists to refuse.
107
+ AGENT_SLACK = "slack"
108
+ AGENT_AUTOMATION = "automation"
109
+
110
+
111
+ def agent_kind(rt, agent_id):
112
+ """Which registry holds this id β€” `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`.
113
+
114
+ ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at
115
+ read time from a column definition or a connector schedule; they have no stored home, so a
116
+ harness file hung off one is an orphan the moment the column changes. `patch_automation`
117
+ already refuses those ids for the neighbouring reason, and this refuses them by simply not
118
+ finding them β€” `all_definitions` holds stored automations only.
119
+ """
120
+ import routes_slack
121
+ aid = str(agent_id or "")
122
+ if isinstance(routes_slack._agents(rt).get(aid), dict):
123
+ return AGENT_SLACK
124
+ import automation_engine as engine
125
+ try:
126
+ known = engine.all_definitions(rt) or {}
127
+ except Exception: # noqa: BLE001
128
+ return None
129
+ return AGENT_AUTOMATION if aid in known else None
130
+
131
+
132
+ def _known_agent(rt, agent_id):
133
+ """Does this tenant have an agent with this id, in EITHER registry?"""
134
+ return agent_kind(rt, agent_id) is not None
135
+
136
+
137
+ def agent_wall(session, agent_id):
138
+ """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies.
139
+
140
+ β›”β›” ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH β€” it is the refusal to invent a
141
+ THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door
142
+ in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in
143
+ `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the
144
+ agent it hangs off, so it is reached by whoever may already configure that agent. Picking one
145
+ of the two walls for both would either lock the Agents module's own users out of a tab the
146
+ owner asked for, or hand the Slack permission wall to anyone with an automation grant.
147
+ """
148
+ kind = agent_kind(session.runtime, agent_id)
149
+ if kind is None:
150
+ raise err(404, "no_agent", "there is no agent with that id in this workspace")
151
+ if kind == AGENT_SLACK:
152
+ import core.perms as perms
153
+ if not perms.is_admin(session.user):
154
+ raise err(403, "forbidden", "administrators only")
155
+ else:
156
+ session.require("automation")
157
+ return kind
158
+
159
+
160
+ def normalize_path(raw):
161
+ """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable.
162
+
163
+ β›”β›” ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There
164
+ are two ways into this store β€” the HTTP route (which must answer 400) and `record_version()`
165
+ (which must raise `ValueError`, having no response to put a status into). Written twice, the
166
+ two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one
167
+ copy is invisible because the other still refuses, so nothing goes red and the wall is now
168
+ half there. Written once, a change to the rule is felt at both doors and by the gate.
169
+ """
170
+ path = str(raw or "").strip().strip("/")
171
+ if not path or len(path) > MAX_PATH:
172
+ return None
173
+ if set(path) - _PATH_OK or ".." in path or "//" in path:
174
+ return None
175
+ return path
176
+
177
+
178
+ def _clean_path(raw):
179
+ """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason."""
180
+ path = normalize_path(raw)
181
+ if path is None:
182
+ if not str(raw or "").strip().strip("/"):
183
+ raise err(400, "no_path", "a harness file needs a path, for example skills/router.md")
184
+ if len(str(raw)) > MAX_PATH:
185
+ raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters")
186
+ raise err(400, "bad_path",
187
+ "a harness path may use letters, digits, dot, dash, underscore and / only")
188
+ return path
189
+
190
+
191
+ def _blank(path, author, author_kind):
192
+ return {"path": path, "versions": [], "trimmed": 0,
193
+ "created": _now(), "createdBy": author, "createdKind": author_kind}
194
+
195
+
196
+ def _append(record, body, author, author_kind, restored_from=None):
197
+ """Append ONE version to a file record, in place, and report what was trimmed.
198
+
199
+ The version NUMBER is monotonic and survives trimming β€” it counts writes, not stored entries.
200
+ A version list whose numbers restart at 1 after a trim would make two different bodies share a
201
+ name, and `restoredFrom` would then point at whichever one happened to be in the window.
202
+ """
203
+ # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback
204
+ # the live document and may run it more than once; appending in place would then stack two
205
+ # copies of the same version into the history on a retry.
206
+ prior = record.get("versions") if isinstance(record.get("versions"), list) else []
207
+ last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0)
208
+ entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind,
209
+ "created": _now(), "bytes": len(body.encode("utf-8"))}
210
+ if restored_from:
211
+ entry["restoredFrom"] = int(restored_from)
212
+ versions = [*prior, entry]
213
+ dropped = max(0, len(versions) - MAX_VERSIONS)
214
+ if dropped:
215
+ versions = versions[dropped:]
216
+ record["versions"] = versions
217
+ record["trimmed"] = int(record.get("trimmed") or 0) + dropped
218
+ return entry
219
+
220
+
221
+ def _head(record):
222
+ """The newest version of a file record, or `None` for a record with no versions at all."""
223
+ versions = record.get("versions") if isinstance(record.get("versions"), list) else []
224
+ return versions[-1] if versions else None
225
+
226
+
227
+ def _row(record):
228
+ """One file, as the LIST door reports it: everything except the bodies.
229
+
230
+ ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would
231
+ ship the whole harness on every pane render; the Harness tab lists first and opens one file
232
+ second, which is exactly the shape this answers.
233
+ """
234
+ head = _head(record) or {}
235
+ versions = record.get("versions") if isinstance(record.get("versions"), list) else []
236
+ return {"path": record.get("path") or "",
237
+ "version": int(head.get("version") or 0),
238
+ "bytes": int(head.get("bytes") or 0),
239
+ "author": head.get("author") or record.get("createdBy") or "",
240
+ "authorKind": head.get("authorKind") or record.get("createdKind") or "user",
241
+ "updated": head.get("created") or record.get("created") or "",
242
+ "created": record.get("created") or "",
243
+ "versions": len(versions),
244
+ "trimmed": int(record.get("trimmed") or 0)}
245
+
246
+
247
+ def _version_rows(record):
248
+ """The history of one file, newest first, WITHOUT the bodies.
249
+
250
+ A body per version is what makes a diff possible, and it is also what makes this payload big:
251
+ 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing
252
+ (`?path=…&version=N`), which is two round trips for a diff and none for a history list.
253
+ """
254
+ versions = record.get("versions") if isinstance(record.get("versions"), list) else []
255
+ out = []
256
+ for entry in reversed(versions):
257
+ if not isinstance(entry, dict):
258
+ continue
259
+ row = {"version": int(entry.get("version") or 0),
260
+ "author": entry.get("author") or "",
261
+ "authorKind": entry.get("authorKind") or "user",
262
+ "created": entry.get("created") or "",
263
+ "bytes": int(entry.get("bytes") or 0)}
264
+ if entry.get("restoredFrom"):
265
+ row["restoredFrom"] = int(entry["restoredFrom"])
266
+ out.append(row)
267
+ return out
268
+
269
+
270
+ def _limits():
271
+ """The caps, IN the payload, so a client can say "this file is full" before a write fails.
272
+
273
+ ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the
274
+ one cap that acts without refusing; these report the three that refuse.
275
+ """
276
+ return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES,
277
+ "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH}
278
+
279
+
280
+ # ── the write door the SERVER uses (no session) ────────────────────────────────────────────────
281
+ def record_version(runtime, agent_id, path, body, author="", author_kind="agent",
282
+ restored_from=None):
283
+ """Write one version from INSIDE the server β€” the agent's own half of R8.
284
+
285
+ ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent"
286
+ true. An agent acting inside an automation run holds no session and no cookie; if its only way
287
+ to write were the HTTP door it would have to borrow a person's identity, and the authorship
288
+ column would then be a record of who was logged in rather than of what wrote the file.
289
+
290
+ Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a
291
+ `ValueError`, because a server-side caller has no HTTP response to put a 404 into.
292
+ """
293
+ agent_id = str(agent_id or "")
294
+ if not _known_agent(runtime, agent_id):
295
+ raise ValueError(f"no agent {agent_id!r} in this workspace")
296
+ path = normalize_path(path)
297
+ if path is None:
298
+ raise ValueError("that is not an acceptable harness path")
299
+ body = str(body or "")
300
+ if len(body.encode("utf-8")) > MAX_BODY_BYTES:
301
+ raise ValueError("harness body is over the size limit")
302
+ kind = author_kind if author_kind in AUTHOR_KINDS else "agent"
303
+ author = str(author or agent_id)
304
+ # β›” THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a
305
+ # store `update` callback propagates out of a half-run read-modify-write, and the one thing a
306
+ # refusal must never do is leave the caller unsure whether the write happened.
307
+ existing = _files(runtime, agent_id)
308
+ if path not in existing and len(existing) >= MAX_FILES:
309
+ raise ValueError(f"this agent already has {MAX_FILES} harness files")
310
+ appended = {}
311
+
312
+ def _set(cur):
313
+ cur = dict(cur or {})
314
+ files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {}
315
+ record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind)
316
+ appended.clear()
317
+ appended.update(_append(record, body, author, kind, restored_from))
318
+ files[path] = record
319
+ cur[agent_id] = files
320
+ return cur
321
+
322
+ runtime.update(HARNESS_KEY, _set, flush="sync")
323
+ return appended
324
+
325
+
326
+ # ── the routes ────────────────────────────────────────────────────────────────────────────────
327
+ @router.get("/agents/{agent_id}/harness")
328
+ def get_harness(agent_id: str, path: str = "", version: int = 0,
329
+ session: Session = Depends(require_session)):
330
+ """The list, one file, or one older body β€” decided by the query string (C4).
331
+
332
+ ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than
333
+ consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to
334
+ editing a permission than to editing a document.
335
+ """
336
+ agent_id = str(agent_id or "")
337
+ agent_wall(session, agent_id)
338
+ files = _files(session.runtime, agent_id)
339
+ if not path:
340
+ rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)]
341
+ return {"agent": agent_id, "files": rows, "limits": _limits()}
342
+
343
+ wanted = _clean_path(path)
344
+ record = files.get(wanted)
345
+ if not isinstance(record, dict):
346
+ raise err(404, "no_file", f"this agent has no harness file at {wanted}")
347
+ versions = record.get("versions") if isinstance(record.get("versions"), list) else []
348
+ if version:
349
+ for entry in versions:
350
+ if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version):
351
+ return {"agent": agent_id, **_row(record), "body": entry.get("body") or "",
352
+ "atVersion": int(version), "history": _version_rows(record),
353
+ "limits": _limits()}
354
+ # β›” A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked
355
+ # for something that existed and no longer does, and telling it apart from a bad number is
356
+ # the difference between "roll back to v3" failing loudly and failing as if v3 never was.
357
+ trimmed = int(record.get("trimmed") or 0)
358
+ if trimmed and int(version) <= trimmed:
359
+ raise err(410, "version_trimmed",
360
+ f"version {int(version)} is older than the {MAX_VERSIONS} versions kept "
361
+ f"for this file, and its body is gone")
362
+ raise err(404, "no_version", f"this file has no version {int(version)}")
363
+ head = _head(record) or {}
364
+ return {"agent": agent_id, **_row(record), "body": head.get("body") or "",
365
+ "atVersion": int(head.get("version") or 0), "history": _version_rows(record),
366
+ "limits": _limits()}
367
+
368
+
369
+ @router.put("/agents/{agent_id}/harness")
370
+ def put_harness(agent_id: str, body: dict = Body(default=None),
371
+ session: Session = Depends(require_session)):
372
+ """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4).
373
+
374
+ β›” THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a
375
+ UI affordance; it is this function refusing to have a code path that replaces a body. A
376
+ roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that
377
+ somebody went back rather than pretending the intervening versions never happened.
378
+ """
379
+ agent_id = str(agent_id or "")
380
+ agent_wall(session, agent_id)
381
+ body = body if isinstance(body, dict) else {}
382
+ path = _clean_path(body.get("path"))
383
+ text = body.get("body")
384
+ if not isinstance(text, str):
385
+ raise err(400, "no_body", "a harness file needs a body, even an empty one")
386
+ if len(text.encode("utf-8")) > MAX_BODY_BYTES:
387
+ raise err(413, "body_too_long",
388
+ f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger")
389
+
390
+ # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a
391
+ # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the
392
+ # session and is never read off the request. Nothing is bought by forging the first β€” both
393
+ # principals may write (R8) β€” and everything would be bought by forging the second.
394
+ kind = str(body.get("authorKind") or "user").strip().lower()
395
+ if kind not in AUTHOR_KINDS:
396
+ raise err(400, "bad_author_kind", "authorKind is either user or agent")
397
+ restored = body.get("restoredFrom")
398
+ try:
399
+ restored = int(restored) if restored else None
400
+ except (TypeError, ValueError):
401
+ raise err(400, "bad_version", "restoredFrom must be a version number")
402
+
403
+ files = _files(session.runtime, agent_id)
404
+ if path not in files and len(files) >= MAX_FILES:
405
+ raise err(409, "too_many_files",
406
+ f"this agent already has {MAX_FILES} harness files; delete one to add another")
407
+ try:
408
+ record_version(session.runtime, agent_id, path, text,
409
+ author=session.uname, author_kind=kind, restored_from=restored)
410
+ except ValueError as exc:
411
+ raise err(400, "refused", str(exc))
412
+
413
+ fresh = _files(session.runtime, agent_id).get(path)
414
+ if not isinstance(fresh, dict) or not _head(fresh):
415
+ # The store took the write and did not record it. A 200 here would tell an administrator
416
+ # their skill file was saved when it was not β€” the shape `routes_slack` refuses too.
417
+ raise err(503, "store_unavailable", "the harness file was NOT saved")
418
+ head = _head(fresh)
419
+ return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "",
420
+ "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh),
421
+ "limits": _limits()}
422
+
423
+
424
+ @router.delete("/agents/{agent_id}/harness")
425
+ def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)):
426
+ """Drop one harness file AND its history.
427
+
428
+ ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete β€” losing
429
+ versions as a side effect of an edit. Deleting a file is a person deciding the file should not
430
+ exist, which is a different act with a different button, and a store with no way to remove a
431
+ file is one where a typo'd path is permanent.
432
+ """
433
+ agent_id = str(agent_id or "")
434
+ agent_wall(session, agent_id)
435
+ wanted = _clean_path(path)
436
+ if wanted not in _files(session.runtime, agent_id):
437
+ raise err(404, "no_file", f"this agent has no harness file at {wanted}")
438
+
439
+ def _set(cur):
440
+ cur = dict(cur or {})
441
+ files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {}
442
+ files.pop(wanted, None)
443
+ # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a
444
+ # bucket accumulates a row for every agent anybody ever opened the tab on.
445
+ if files:
446
+ cur[agent_id] = files
447
+ else:
448
+ cur.pop(agent_id, None)
449
+ return cur
450
+
451
+ session.runtime.update(HARNESS_KEY, _set, flush="sync")
452
+ return {"agent": agent_id, "deleted": wanted,
453
+ "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items())
454
+ if isinstance(rec, dict)],
455
+ "limits": _limits()}
api/routes_alerts.py CHANGED
@@ -1,668 +1,668 @@
1
- """routes_alerts.py β€” the Alerts module (wave 20, owner item 25, contract C-ALERT).
2
-
3
- GET /api/v1/alerts -> {alerts:[...]}
4
- POST /api/v1/alerts <- {viewId, topic, label?}
5
- DELETE /api/v1/alerts/{alert_id}
6
- POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
7
- GET /api/v1/notifications -> {unread, items:[...]}
8
- POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
9
-
10
- The semantics β€” an alert is a view plus a remembered matched set, a notification is a NEW
11
- ENTRANT, and the first evaluation seeds silently β€” live in `core.alerts` with the reasoning.
12
- This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
13
-
14
- ⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
15
- for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
16
- full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
17
- book, and the notification would name customers that user may not see β€” a permission leak wearing
18
- a notification's clothes. The owner's own scope is the only correct basis for their alert.
19
-
20
- ⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
21
- `routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
22
- alert can see is by construction a row its owner could open. Re-implementing the filter here
23
- would be a second definition of "matches", and those two would drift.
24
- """
25
- import re
26
-
27
- from fastapi import APIRouter, Body, Depends
28
-
29
- import core.alerts as alerts
30
- from deps import Session, err, require_session
31
-
32
- router = APIRouter(prefix="/api/v1")
33
-
34
- #: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
35
- _TOPICS = ("customer", "product")
36
-
37
- # ── ⭐⭐ WAVE 32 Β· T20 Β· CONTRACT C3 β€” THE INBOX SHAPE, DERIVED ON READ ────────────────────────
38
- #
39
- # `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there).
40
- #
41
- # β›” DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three
42
- # keys onto the record at write time would give them to notifications minted AFTER the deploy and
43
- # to nothing else β€” every notification already sitting in every tenant's inbox would open nothing,
44
- # and the feature would be correct in the source and absent from the product
45
- # ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a
46
- # notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is
47
- # another lane's file this wave β€” but that is the convenience, not the reason.
48
- #
49
- # ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them
50
- # apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets
51
- # `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding
52
- # here means the client branches on ONE field instead of re-deriving the same split.
53
- #
54
- # β›” **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape β€” `kind='automation_review'`
55
- # + `autoId`, a card arriving at a review stage β€” and its producer `notify_review` was deleted by
56
- # W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`)
57
- # records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch,
58
- # its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review
59
- # branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real
60
- # caller"* β€” the residue is zero, so the branch goes. It is not carried into the Inbox: a stored
61
- # review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary
62
- # `alert` with no target, i.e. an honest unclickable row, which is correct β€” the board it pointed
63
- # at was deleted two waves ago.
64
-
65
- #: C3's `kind` vocabulary. Plain strings on the wire β€” the client must never union over them
66
- #: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row).
67
- NOTIF_KIND_ALERT = "alert"
68
- NOTIF_KIND_AUTOMATION = "automation"
69
- NOTIF_KIND_SHARE = "share"
70
-
71
- #: C3's `target.module` vocabulary, and the automation sub-selection.
72
- TARGET_MODULE_DATABASE = "database"
73
- TARGET_MODULE_AUTOMATION = "automation"
74
- TARGET_TAB_RUNS = "runs"
75
-
76
- #: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with
77
- #: no producer is a string that reads as a feature β€” the reason this constant is named here and
78
- #: cited from `routes_shares` rather than typed twice).
79
- SHARE_TOPIC = "share"
80
-
81
- #: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`.
82
- AUTOMATION_TOPIC = "automation"
83
-
84
- _UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z")
85
-
86
-
87
- def route_for_topic(topic):
88
- """A grid SCOPE key -> the registry route that renders it, or None.
89
-
90
- β›” THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED
91
- (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only
92
- pair that differ β€” the registry names the surface (`customer_data`) while the grid names the
93
- scope (`customer`) β€” so a topic passed through as a route sends every click to a page that
94
- does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT
95
- rather than plausible, because an absent target renders as a row that does not pretend to be
96
- clickable, and a wrong one renders as a click that silently goes nowhere.
97
- """
98
- t = str(topic or "").strip()
99
- if t == "customer":
100
- return "customer_data"
101
- if t == "product":
102
- return "product_data"
103
- if _UT_TOPIC.match(t):
104
- return t
105
- return None
106
-
107
-
108
- def _refusal_code(exc):
109
- """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`.
110
-
111
- ⭐ W32-T22. Four refusals travel up the assembly chain β€” `unknown_table` (404), `forbidden`
112
- (403), `window_required` (409) and `store_not_ready` (503) β€” and each already names its own
113
- cause. Anything that reduces all four to one word is throwing away the only information the
114
- reader could have acted on. Returns `""` for a plain exception, so a caller can tell
115
- "refused, and here is why" apart from "broke, and we do not know why".
116
- """
117
- detail = getattr(exc, "detail", None)
118
- if isinstance(detail, dict):
119
- inner = detail.get("error")
120
- if isinstance(inner, dict):
121
- return str(inner.get("code") or "")
122
- return ""
123
-
124
-
125
- def notification_view(item):
126
- """One STORED notification -> the shape the Inbox renders. PURE, and total.
127
-
128
- Never raises and never drops a row: an item it cannot classify comes back as an `alert` with
129
- no `target`, which the client renders as an unclickable row rather than hiding. An inbox that
130
- silently omits what it does not understand is the one failure a reader cannot detect.
131
- """
132
- if not isinstance(item, dict):
133
- return item
134
- topic = str(item.get("topic") or "").strip()
135
- alert_id = str(item.get("alertId") or "").strip()
136
-
137
- # β›” THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic
138
- # says `automation` but whose producer key never arrived (a truncated payload, a server
139
- # mid-deploy) would otherwise be handed a target naming NOTHING β€” a click that appears to work
140
- # and silently does not, which is this repo's most-repeated failure shape. Failing the test
141
- # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None.
142
- if topic == AUTOMATION_TOPIC and alert_id:
143
- kind = NOTIF_KIND_AUTOMATION
144
- target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS}
145
- elif topic == SHARE_TOPIC and alert_id:
146
- # ⭐ W32-T28: the sharer writes `key=<the ROUTE to open>` and, for a shared VIEW,
147
- # `row_id=<the view to select>`.
148
- #
149
- # β›” `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it
150
- # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the
151
- # target read `{module: "database", id: "view_42"}` β€” an instruction to open a database
152
- # called `view_42`. It looked right in the payload and would have opened nothing. The
153
- # producer resolves the object to its topic and hands over the route; this branch only
154
- # shapes what it is given.
155
- kind = NOTIF_KIND_SHARE
156
- row_id = str(item.get("rowId") or "").strip()
157
- target = {"module": TARGET_MODULE_DATABASE, "id": alert_id,
158
- **({"tab": row_id} if row_id else {})}
159
- else:
160
- kind = NOTIF_KIND_ALERT
161
- route = route_for_topic(topic)
162
- view_id = str(item.get("viewId") or "").strip()
163
- target = None if route is None else (
164
- {"module": TARGET_MODULE_DATABASE, "id": route,
165
- **({"tab": view_id} if view_id else {})})
166
-
167
- # The email split: `subject` is the HEADER (what this is about β€” the alert, the automation,
168
- # the database), `label` stays the BODY (what happened β€” the record that entered, the run
169
- # summary). They were one field, which is why a notification read as a sentence with no
170
- # sender and the pane could not be laid out like mail.
171
- subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip()
172
- # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it
173
- # is D-101's dead one; leaving it through would give the client two vocabularies for one
174
- # question, which is the defect this wave's item 6 is about in a different file.
175
- # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") β€” THE SENDER, WHICH DID NOT EXIST.
176
- #
177
- # β›” A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was
178
- # occupied by `kindLabel(n.kind)` β€” the literals "Alert" / "Automation" / "Shared with you" β€”
179
- # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in
180
- # the model or in the markup. Mail has a from. This is it.
181
- #
182
- # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE β€” and the exception is the point.
183
- # An alert firing and an automation landing rows have no person behind them; their honest
184
- # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a
185
- # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and
186
- # this reads it back. β›” It is NOT parsed out of the body prose ("<name> shared this with
187
- # you") β€” a sender recovered by regexing a sentence breaks the first time the sentence is
188
- # reworded, and it would break silently, in the header.
189
- #
190
- # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an
191
- # empty from column reads as a broken inbox rather than as an old notification.
192
- actor = str(item.get("actor") or "").strip()
193
- if kind == NOTIF_KIND_SHARE:
194
- sender = actor or "A teammate"
195
- elif kind == NOTIF_KIND_AUTOMATION:
196
- # β›” "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's
197
- # `senderOf` already falls back to `AGENTS_MODULE_LABEL` β€” but `if (sent) return sent`
198
- # runs FIRST, so this server literal won and every actor-less automation notification
199
- # showed the retired module name in the inbox's From column.
200
- sender = actor or "Agents"
201
- else:
202
- sender = actor or "Alerts"
203
- out = {**item, "read": bool(item.get("read")), "kind": kind,
204
- "subject": subject or "Notification", "sender": sender}
205
- if target is not None:
206
- out["target"] = target
207
- return out
208
-
209
-
210
- def inbox_view(box):
211
- """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}.
212
-
213
- ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a
214
- recount here would make the badge a function of whatever this page happened to include, which
215
- is the exact defect `alertsModel.parseInbox`'s own header records from the other side.
216
- """
217
- if not isinstance(box, dict):
218
- return box
219
- items = box.get("items")
220
- if not isinstance(items, list):
221
- return box
222
- # ⭐⭐ W33-T28 / D-208 β€” THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client
223
- # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`.
224
- #
225
- # β›” THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and
226
- # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH
227
- # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?"
228
- # against a browser clock would re-introduce the drift the offset exists to remove β€” a reader a
229
- # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands
230
- # now come from the same machine.
231
- # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree β€”
232
- # the note two lines up records what happened last time only one of them was enriched.
233
- return {**box, "now": alerts._now_iso(),
234
- "items": [notification_view(n) for n in items]}
235
-
236
-
237
- def _view_by_id(g, view_id):
238
- """One saved view out of an assembly, by id. `None` when there is no such view.
239
-
240
- β›”β›” W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") β€” THIS
241
- FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND
242
- `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes
243
- it straight out and both `ut_assembly` and `grid_assembly` return it unchanged β€” so `.get` on
244
- it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the
245
- `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.**
246
-
247
- β›” AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In
248
- `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare
249
- FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our
250
- side"* β€” the exact sentence the owner reported (D-107's shape, again: an attribute error above
251
- the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical
252
- line is swallowed by `/notifications`' `except Exception: continue`, so **every stored
253
- view-alert was silently dropped from the Inbox** and nobody had anything to report at all.
254
- One expression, one loud symptom and one silent one.
255
-
256
- ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site
257
- be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]].
258
-
259
- ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was
260
- keyed `{id: view}` β€” which is precisely why the gate was green while production raised on
261
- every call. The fixture is moving to the production shape in this same change, and tolerating
262
- both here means a caller that legitimately holds one cannot resurrect the bug.
263
- """
264
- want = str(view_id or "")
265
- if not want:
266
- return None
267
- views = (g or {}).get("views")
268
- if isinstance(views, dict):
269
- found = views.get(want)
270
- return found if isinstance(found, dict) else None
271
- if not isinstance(views, list):
272
- return None
273
- for v in views:
274
- if isinstance(v, dict) and str(v.get("id") or "") == want:
275
- return v
276
- return None
277
-
278
-
279
- def _topic_or_400(raw):
280
- topic = str(raw or "").strip().lower()
281
- if topic.startswith("ut_") or topic in _TOPICS:
282
- return topic
283
- raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
284
-
285
-
286
- def _owner_session(session: Session, owner: str):
287
- """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
288
-
289
- ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields β€” so an
290
- owner session is built by swapping the `user` RECORD and letting both derive themselves. An
291
- earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
292
- first write hook of the wave; the properties are the single definition of who a session is,
293
- and going around them is how a session with an admin flag and a non-admin record exists.
294
-
295
- Returns None when the owner is gone or deactivated β€” their alerts then stop evaluating rather
296
- than evaluating as somebody else, which is the fail-closed direction.
297
- """
298
- import core.users as users
299
-
300
- if str(owner) == str(session.uname):
301
- return session
302
- rec = (users.registry() or {}).get(str(owner))
303
- if not isinstance(rec, dict) or not rec.get("active", True):
304
- return None
305
- # `_public` is THE definition of what a session may know about its own account (never a hash
306
- # or a salt) β€” the same one `routes_auth` uses. Building the dict by hand here would be a
307
- # second definition, and the one that leaks is always the copy.
308
- return Session(tenant=session.tenant, user=users._public(str(owner), rec),
309
- claims=session.claims, runtime=session.runtime)
310
-
311
-
312
- def _evaluate(session: Session, rec: dict, assemblies=None):
313
- """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
314
-
315
- ⭐⭐ W31-T24 β€” `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole
316
- of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each
317
- one built a FULL assembly β€” the pool, the workspace, `rows_from_pool` over every row. Two
318
- alerts on one view built that table twice; ten built it ten times. Nothing dedupes them,
319
- because each `_evaluate` was a closed call.
320
- ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module
321
- note β€” evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak
322
- wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and
323
- must never share an entry. Getting that key wrong is the one way this optimisation could leak.
324
- ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want:
325
- they evaluate ONE alert and a memo for a single call is pure overhead.
326
- """
327
- import aios_grid
328
- from harness import filter_eval
329
-
330
- owner_sess = _owner_session(session, rec.get("owner"))
331
- if owner_sess is None:
332
- return {"skipped": "owner_unavailable"}
333
- topic = str(rec.get("topic") or "")
334
- memo_key = (topic, str(owner_sess.uname))
335
- g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None
336
- if g is None:
337
- try:
338
- if topic.startswith("ut_"):
339
- from routes_tables import ut_assembly
340
- # β›” `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up.
341
- # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the
342
- # one-shot field-name correction acks for every `ut_` topic that has an alert β€”
343
- # taking them from the `/workspace` refresh that exists to show them to the person
344
- # who made the edit. The customer branch below has always passed False; this one
345
- # inherited a default nobody re-read. An inbox poll must never consume a one-shot.
346
- g = ut_assembly(owner_sess, topic,
347
- storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}",
348
- consume_corrections=False)
349
- else:
350
- from routes_customers import grid_assembly
351
- g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
352
- except Exception as e: # noqa: BLE001
353
- # ⭐ W32-T22 β€” SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not
354
- # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it
355
- # keeps a blanket catch β€” but it now reports the refusal's OWN code where there is
356
- # one. `type(e).__name__` said `HTTPException` for four different causes, and
357
- # `lastError` is the only place a user ever learns why an alert stopped firing.
358
- #
359
- # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an
360
- # evaluation genuinely needs the rows to run the filter over. So an alert on a
361
- # read-through grid is created (T22) and then skips at evaluation with
362
- # `window_required` naming why β€” which is D-184's remaining half, and it is a
363
- # SENTENCE now rather than silence.
364
- return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__}
365
- if isinstance(assemblies, dict):
366
- assemblies[memo_key] = g
367
-
368
- view = _view_by_id(g, rec.get("viewId"))
369
- if not isinstance(view, dict):
370
- # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
371
- # deleting the alert: an alert that silently vanishes is indistinguishable from one that
372
- # never fires, and the user cannot debug what is not there.
373
- return {"skipped": "view_missing"}
374
-
375
- # The SAME row build the grid and `/customers` use β€” `rows_from_pool` is what puts derived
376
- # and overlay values on a row. Evaluating a filter against raw pool dicts would silently
377
- # never match any condition on a user-created or measure column.
378
- #
379
- # β›”β›” AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND
380
- # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths.
381
- #
382
- # `routes_tables.table_rows` β€” the grid the person is looking at β€” merges the DEFINITION rows
383
- # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner
384
- # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it:
385
- # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell
386
- # evaluated as BLANK. Measured on one assembly, same view, same rows:
387
- # rows_src state='unpaid' / 'paid'
388
- # _evaluate saw state='' / '' ⇐ every base cell blank
389
- # the GRID saw state='unpaid' / 'paid'
390
- # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty`
391
- # matched EVERYTHING while the view showed none. **The alert did not merely miss rows β€” it
392
- # inverted.** End to end: a row whose value arrived by import, automation, paste or the create
393
- # door never fired; only a value typed as a hand EDIT did.
394
- #
395
- # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit;
396
- # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`);
397
- # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns
398
- # `skipped: window_required`).
399
- # β›” ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it
400
- # would let a stale definition value shadow an edit the user has just made β€” the same defect
401
- # `table_rows`' own note records, arriving from the other side.
402
- _ov = (g.get("ws") or {}).get("overlays") or {}
403
- _merged = {}
404
- for _r in g["rows_src"]:
405
- _pid = str(_r.get("pid"))
406
- _cells = {k: v for k, v in _r.items() if k != "pid"}
407
- _o = _ov.get(_pid)
408
- if isinstance(_o, dict):
409
- _cells.update(_o)
410
- _merged[_pid] = _cells
411
- rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged,
412
- derived=g.get("derived"))
413
- config = view.get("config") or view
414
- ctx = filter_eval.EvalCtx(
415
- cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
416
- for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
417
- measure_sets=g.get("measure_sets") or {},
418
- today=g.get("today"))
419
- pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
420
- member_pids=config.get("memberPids"))
421
- labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
422
- return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
423
- labels=labels, partial=False, st=session.runtime)
424
-
425
-
426
- @router.get("/alerts")
427
- def list_alerts(session: Session = Depends(require_session)):
428
- return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
429
- st=session.runtime)}
430
-
431
-
432
- @router.post("/alerts")
433
- def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
434
- body = body or {}
435
- view_id = str(body.get("viewId") or "").strip()
436
- if not view_id:
437
- raise err(400, "bad_view", "an alert needs the id of the view it watches")
438
- topic = _topic_or_400(body.get("topic"))
439
- _require_filtered_view(session, topic, view_id)
440
- import uuid
441
- aid = f"al_{uuid.uuid4().hex[:12]}"
442
- rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
443
- label=body.get("label") or "", st=session.runtime)
444
- # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
445
- # Deferring this to the first write hook would mean the next edit announces the whole view.
446
- outcome = _evaluate(session, rec)
447
- return {"alert": {**rec, "seeded": True}, "first": outcome}
448
-
449
-
450
- def _require_filtered_view(session: Session, topic: str, view_id: str):
451
- """400 unless `view_id` exists on `topic` AND actually narrows something.
452
-
453
- β›” AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
454
- that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
455
- (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
456
- entrant again β€” there is nothing left to enter. The owner's words are *"when a Record gets
457
- into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
458
- discovered by never being notified.
459
-
460
- `is_rule_active` is the SAME activeness predicate the engine and the column tints use β€” a
461
- half-typed rule is not a filter, and this must agree with what actually narrows or it would
462
- accept a view whose one rule the engine then ignores.
463
-
464
- ⭐⭐ WAVE 32 Β· T22 (owner item 17) β€” THIS FUNCTION WAS THE ERROR. Two defects, stacked, and
465
- the second one hid the first.
466
-
467
- (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is
468
- `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a
469
- read-through grid built the whole pool β€” and `scoped_pool` refuses that with
470
- `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False`
471
- (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`,
472
- runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half,
473
- closed** β€” an alert on a read-through grid can now be made at all.
474
- (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException`
475
- is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and
476
- `503 store_not_ready` β€” four refusals that each say what is wrong β€” were all replaced by
477
- *"the table is unavailable β€” try again in a moment"*. β›” AND THAT SENTENCE NEVER REACHED
478
- A USER EITHER: `alertsApi.errorMessage` discards the text of any status β‰₯ 500 by design
479
- (a 5xx body is the server's internals), substituting *"Something went wrong on our
480
- side."* β€” which is the owner's screenshot, word for word. A knowable cause returned as a
481
- 5xx is invisible by construction, so re-wording the 503 could never have fixed this.
482
- ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is
483
- honest. What it may no longer do is catch a refusal that already knows its own name.
484
- """
485
- from fastapi import HTTPException
486
-
487
- from harness import filter_eval
488
-
489
- try:
490
- if topic.startswith("ut_"):
491
- from routes_tables import ut_assembly
492
- # ⚠ `consume_corrections=False` β€” the customer branch has always passed it and this
493
- # one inherited a default nobody re-read. Creating an alert must not eat the one-shot
494
- # field-name correction acks belonging to the `/workspace` refresh that exists to show
495
- # them to the person who made the edit. Same defect `_evaluate`'s header records.
496
- g = ut_assembly(session, topic,
497
- storage_key=f"{session.tenant}:{topic}:{session.uname}",
498
- consume_corrections=False, with_rows=False)
499
- else:
500
- from routes_customers import grid_assembly
501
- g = grid_assembly(session, scope=topic, consume_corrections=False)
502
- except HTTPException:
503
- raise # it already names its own cause
504
- except Exception as e: # noqa: BLE001
505
- # Genuinely unexpected. Still a 503, and now it carries the exception TYPE β€” without it,
506
- # the one path that reaches this branch is also the one path with nothing to debug from.
507
- raise err(503, "unavailable",
508
- f"the table could not be read ({type(e).__name__}) β€” try again in a moment")
509
- view = _view_by_id(g, view_id)
510
- if not isinstance(view, dict):
511
- raise err(404, "no_view", "that view does not exist on this table")
512
- nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
513
-
514
- # β›”β›” WAVE 33 Β· T29 β€” **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT
515
- # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS
516
- # owner item 17" β€” the owner's *"Something went wrong"*. It was not, and it could not have
517
- # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines
518
- # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the
519
- # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix.
520
- #
521
- # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the
522
- # fix was right, and the three reasons it hid are the most transferable thing in this file.
523
- # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each
524
- # naming themselves as the origin of the same screenshot are mutually exclusive, and the next
525
- # reader believes whichever they meet first β€” which is why this correction sits above rather
526
- # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead
527
- # of the working tree. [[grep-output-is-not-source]]
528
- #
529
- # β›”β›” WAVE 32 Β· T22 β€” **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly,
530
- # that this was owner item 17 β€” see the correction directly above).
531
- #
532
- # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other
533
- # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF
534
- # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`.
535
- #
536
- # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the
537
- # view HAS a condition β€” and a view with a condition is the only kind an alert is allowed on.
538
- # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever
539
- # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that
540
- # worked was the refusal path: "Alert me about new records" had never once created an alert
541
- # on a filtered view.** β›” And the raise lands OUTSIDE the `try` above, so it was not even the
542
- # 503 β€” it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went
543
- # wrong on our side. Try again in a moment."*, the owner's screenshot word for word.
544
- #
545
- # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check
546
- # arity until the line RUNS, and this line runs only on the success path of a feature whose
547
- # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct,
548
- # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal
549
- # (`no_filter` reaches the user) and the transport β€” never a creation. A gate can be green,
550
- # thorough and honest about everything except the one path the feature exists for.
551
- #
552
- # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a
553
- # column up in; building a second dict here would be a second answer to one question, which
554
- # is this wave's other headline defect in a different file. Its leading underscore is a real
555
- # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around.
556
- columns = filter_eval._columns_map(g.get("fields") or [])
557
-
558
- def _any_active(ns):
559
- for n in ns or ():
560
- if isinstance(n, dict) and isinstance(n.get("children"), list):
561
- if _any_active(n["children"]):
562
- return True
563
- elif filter_eval.is_rule_active(n, columns):
564
- return True
565
- return False
566
-
567
- if not _any_active(nodes):
568
- raise err(400, "no_filter",
569
- "this view has no active filter, so no record can ever ENTER it β€” add a "
570
- "condition to the view first, then create the alert")
571
-
572
-
573
- @router.delete("/alerts/{alert_id}")
574
- def delete_alert(alert_id: str, session: Session = Depends(require_session)):
575
- rec = next((r for r in alerts.list_alerts(st=session.runtime)
576
- if str(r.get("id")) == str(alert_id)), None)
577
- if rec is None:
578
- raise err(404, "no_alert", "that alert does not exist")
579
- if str(rec.get("owner")) != str(session.uname) and not session.admin:
580
- raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
581
- alerts.delete(alert_id, st=session.runtime)
582
- return {"ok": True}
583
-
584
-
585
- @router.post("/alerts/{alert_id}/run")
586
- def run_alert(alert_id: str, session: Session = Depends(require_session)):
587
- rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
588
- st=session.runtime)
589
- if str(r.get("id")) == str(alert_id)), None)
590
- if rec is None:
591
- raise err(404, "no_alert", "that alert does not exist")
592
- return _evaluate(session, rec)
593
-
594
-
595
- @router.get("/notifications")
596
- def notifications(session: Session = Depends(require_session)):
597
- """The inbox β€” RE-EVALUATED on read, which is a deliberate design choice.
598
-
599
- ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
600
- plan was a push hook: the automation engine calls `after_write` when it lands rows. But
601
- `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
602
- evaluated as its OWNER (see `_evaluate`) β€” so a push hook would have to mint a session inside
603
- a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
604
- construction that leaks scope.
605
-
606
- Pulling on read has none of that: the caller IS a session, the assemblies are already
607
- scope-cached, and the user cannot observe the difference β€” an inbox is only ever read by
608
- someone opening it. The cost is that a notification is minted when you LOOK rather than when
609
- the row landed, so the `at` stamp is detection time, not arrival time.
610
-
611
- `after_write` stays exported for the day the engine can hand over a real identity.
612
-
613
- ⭐⭐ W31-T24 β€” ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
614
- β›” MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
615
- **20 ms in-process and 3,280 ms live** on tenant #0 β€” but tenant #0 has **ZERO alerts**
616
- (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
617
- re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
618
- body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
619
- grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic Γ— owner), which
620
- is the difference between "fine" and "three seconds per alert" the day somebody uses the
621
- feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
622
- this does: every alert is still evaluated, against the same rows, in the same order.
623
- """
624
- assemblies = {}
625
- for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
626
- try:
627
- _evaluate(session, rec, assemblies=assemblies)
628
- except Exception: # noqa: BLE001
629
- continue # one bad alert must not empty the pane
630
- # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
631
- # this wave carries a `target` too. See `notification_view`'s header for why it is derived.
632
- return inbox_view(alerts.inbox(session.uname, st=session.runtime))
633
-
634
-
635
- @router.post("/notifications/read")
636
- def read_notifications(body: dict = Body(default=None),
637
- session: Session = Depends(require_session)):
638
- body = body or {}
639
- ids = body.get("ids")
640
- if ids is not None and not isinstance(ids, list):
641
- raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
642
- # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox
643
- # module re-renders from it β€” an un-enriched answer here would strip `target` off every row
644
- # the moment somebody marked one read, i.e. the feature would work until first use.
645
- return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
646
- st=session.runtime))
647
-
648
-
649
- def after_write(session: Session, topic_key: str):
650
- """THE WRITE HOOK β€” call after a write that could change what a view matches.
651
-
652
- Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
653
- upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
654
- edit that triggered it.
655
-
656
- ⭐ W31-T24 β€” it shares `/notifications`' memo shape for the same reason: a write that changes
657
- one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
658
- ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
659
- push hook was resolved the other way). Booked rather than wired: minting a session inside the
660
- engine's worker thread is the ad-hoc identity construction this file exists to avoid.
661
- """
662
- try:
663
- assemblies = {}
664
- return alerts.after_write(topic_key, st=session.runtime,
665
- runner=lambda rec: _evaluate(session, rec,
666
- assemblies=assemblies))
667
- except Exception: # noqa: BLE001
668
- return {"evaluated": 0}
 
1
+ """routes_alerts.py β€” the Alerts module (wave 20, owner item 25, contract C-ALERT).
2
+
3
+ GET /api/v1/alerts -> {alerts:[...]}
4
+ POST /api/v1/alerts <- {viewId, topic, label?}
5
+ DELETE /api/v1/alerts/{alert_id}
6
+ POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
7
+ GET /api/v1/notifications -> {unread, items:[...]}
8
+ POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
9
+
10
+ The semantics β€” an alert is a view plus a remembered matched set, a notification is a NEW
11
+ ENTRANT, and the first evaluation seeds silently β€” live in `core.alerts` with the reasoning.
12
+ This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
13
+
14
+ ⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
15
+ for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
16
+ full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
17
+ book, and the notification would name customers that user may not see β€” a permission leak wearing
18
+ a notification's clothes. The owner's own scope is the only correct basis for their alert.
19
+
20
+ ⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
21
+ `routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
22
+ alert can see is by construction a row its owner could open. Re-implementing the filter here
23
+ would be a second definition of "matches", and those two would drift.
24
+ """
25
+ import re
26
+
27
+ from fastapi import APIRouter, Body, Depends
28
+
29
+ import core.alerts as alerts
30
+ from deps import Session, err, require_session
31
+
32
+ router = APIRouter(prefix="/api/v1")
33
+
34
+ #: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
35
+ _TOPICS = ("customer", "product")
36
+
37
+ # ── ⭐⭐ WAVE 32 Β· T20 Β· CONTRACT C3 β€” THE INBOX SHAPE, DERIVED ON READ ────────────────────────
38
+ #
39
+ # `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there).
40
+ #
41
+ # β›” DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three
42
+ # keys onto the record at write time would give them to notifications minted AFTER the deploy and
43
+ # to nothing else β€” every notification already sitting in every tenant's inbox would open nothing,
44
+ # and the feature would be correct in the source and absent from the product
45
+ # ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a
46
+ # notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is
47
+ # another lane's file this wave β€” but that is the convenience, not the reason.
48
+ #
49
+ # ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them
50
+ # apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets
51
+ # `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding
52
+ # here means the client branches on ONE field instead of re-deriving the same split.
53
+ #
54
+ # β›” **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape β€” `kind='automation_review'`
55
+ # + `autoId`, a card arriving at a review stage β€” and its producer `notify_review` was deleted by
56
+ # W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`)
57
+ # records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch,
58
+ # its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review
59
+ # branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real
60
+ # caller"* β€” the residue is zero, so the branch goes. It is not carried into the Inbox: a stored
61
+ # review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary
62
+ # `alert` with no target, i.e. an honest unclickable row, which is correct β€” the board it pointed
63
+ # at was deleted two waves ago.
64
+
65
+ #: C3's `kind` vocabulary. Plain strings on the wire β€” the client must never union over them
66
+ #: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row).
67
+ NOTIF_KIND_ALERT = "alert"
68
+ NOTIF_KIND_AUTOMATION = "automation"
69
+ NOTIF_KIND_SHARE = "share"
70
+
71
+ #: C3's `target.module` vocabulary, and the automation sub-selection.
72
+ TARGET_MODULE_DATABASE = "database"
73
+ TARGET_MODULE_AUTOMATION = "automation"
74
+ TARGET_TAB_RUNS = "runs"
75
+
76
+ #: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with
77
+ #: no producer is a string that reads as a feature β€” the reason this constant is named here and
78
+ #: cited from `routes_shares` rather than typed twice).
79
+ SHARE_TOPIC = "share"
80
+
81
+ #: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`.
82
+ AUTOMATION_TOPIC = "automation"
83
+
84
+ _UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z")
85
+
86
+
87
+ def route_for_topic(topic):
88
+ """A grid SCOPE key -> the registry route that renders it, or None.
89
+
90
+ β›” THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED
91
+ (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only
92
+ pair that differ β€” the registry names the surface (`customer_data`) while the grid names the
93
+ scope (`customer`) β€” so a topic passed through as a route sends every click to a page that
94
+ does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT
95
+ rather than plausible, because an absent target renders as a row that does not pretend to be
96
+ clickable, and a wrong one renders as a click that silently goes nowhere.
97
+ """
98
+ t = str(topic or "").strip()
99
+ if t == "customer":
100
+ return "customer_data"
101
+ if t == "product":
102
+ return "product_data"
103
+ if _UT_TOPIC.match(t):
104
+ return t
105
+ return None
106
+
107
+
108
+ def _refusal_code(exc):
109
+ """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`.
110
+
111
+ ⭐ W32-T22. Four refusals travel up the assembly chain β€” `unknown_table` (404), `forbidden`
112
+ (403), `window_required` (409) and `store_not_ready` (503) β€” and each already names its own
113
+ cause. Anything that reduces all four to one word is throwing away the only information the
114
+ reader could have acted on. Returns `""` for a plain exception, so a caller can tell
115
+ "refused, and here is why" apart from "broke, and we do not know why".
116
+ """
117
+ detail = getattr(exc, "detail", None)
118
+ if isinstance(detail, dict):
119
+ inner = detail.get("error")
120
+ if isinstance(inner, dict):
121
+ return str(inner.get("code") or "")
122
+ return ""
123
+
124
+
125
+ def notification_view(item):
126
+ """One STORED notification -> the shape the Inbox renders. PURE, and total.
127
+
128
+ Never raises and never drops a row: an item it cannot classify comes back as an `alert` with
129
+ no `target`, which the client renders as an unclickable row rather than hiding. An inbox that
130
+ silently omits what it does not understand is the one failure a reader cannot detect.
131
+ """
132
+ if not isinstance(item, dict):
133
+ return item
134
+ topic = str(item.get("topic") or "").strip()
135
+ alert_id = str(item.get("alertId") or "").strip()
136
+
137
+ # β›” THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic
138
+ # says `automation` but whose producer key never arrived (a truncated payload, a server
139
+ # mid-deploy) would otherwise be handed a target naming NOTHING β€” a click that appears to work
140
+ # and silently does not, which is this repo's most-repeated failure shape. Failing the test
141
+ # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None.
142
+ if topic == AUTOMATION_TOPIC and alert_id:
143
+ kind = NOTIF_KIND_AUTOMATION
144
+ target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS}
145
+ elif topic == SHARE_TOPIC and alert_id:
146
+ # ⭐ W32-T28: the sharer writes `key=<the ROUTE to open>` and, for a shared VIEW,
147
+ # `row_id=<the view to select>`.
148
+ #
149
+ # β›” `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it
150
+ # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the
151
+ # target read `{module: "database", id: "view_42"}` β€” an instruction to open a database
152
+ # called `view_42`. It looked right in the payload and would have opened nothing. The
153
+ # producer resolves the object to its topic and hands over the route; this branch only
154
+ # shapes what it is given.
155
+ kind = NOTIF_KIND_SHARE
156
+ row_id = str(item.get("rowId") or "").strip()
157
+ target = {"module": TARGET_MODULE_DATABASE, "id": alert_id,
158
+ **({"tab": row_id} if row_id else {})}
159
+ else:
160
+ kind = NOTIF_KIND_ALERT
161
+ route = route_for_topic(topic)
162
+ view_id = str(item.get("viewId") or "").strip()
163
+ target = None if route is None else (
164
+ {"module": TARGET_MODULE_DATABASE, "id": route,
165
+ **({"tab": view_id} if view_id else {})})
166
+
167
+ # The email split: `subject` is the HEADER (what this is about β€” the alert, the automation,
168
+ # the database), `label` stays the BODY (what happened β€” the record that entered, the run
169
+ # summary). They were one field, which is why a notification read as a sentence with no
170
+ # sender and the pane could not be laid out like mail.
171
+ subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip()
172
+ # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it
173
+ # is D-101's dead one; leaving it through would give the client two vocabularies for one
174
+ # question, which is the defect this wave's item 6 is about in a different file.
175
+ # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") β€” THE SENDER, WHICH DID NOT EXIST.
176
+ #
177
+ # β›” A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was
178
+ # occupied by `kindLabel(n.kind)` β€” the literals "Alert" / "Automation" / "Shared with you" β€”
179
+ # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in
180
+ # the model or in the markup. Mail has a from. This is it.
181
+ #
182
+ # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE β€” and the exception is the point.
183
+ # An alert firing and an automation landing rows have no person behind them; their honest
184
+ # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a
185
+ # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and
186
+ # this reads it back. β›” It is NOT parsed out of the body prose ("<name> shared this with
187
+ # you") β€” a sender recovered by regexing a sentence breaks the first time the sentence is
188
+ # reworded, and it would break silently, in the header.
189
+ #
190
+ # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an
191
+ # empty from column reads as a broken inbox rather than as an old notification.
192
+ actor = str(item.get("actor") or "").strip()
193
+ if kind == NOTIF_KIND_SHARE:
194
+ sender = actor or "A teammate"
195
+ elif kind == NOTIF_KIND_AUTOMATION:
196
+ # β›” "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's
197
+ # `senderOf` already falls back to `AGENTS_MODULE_LABEL` β€” but `if (sent) return sent`
198
+ # runs FIRST, so this server literal won and every actor-less automation notification
199
+ # showed the retired module name in the inbox's From column.
200
+ sender = actor or "Agents"
201
+ else:
202
+ sender = actor or "Alerts"
203
+ out = {**item, "read": bool(item.get("read")), "kind": kind,
204
+ "subject": subject or "Notification", "sender": sender}
205
+ if target is not None:
206
+ out["target"] = target
207
+ return out
208
+
209
+
210
+ def inbox_view(box):
211
+ """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}.
212
+
213
+ ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a
214
+ recount here would make the badge a function of whatever this page happened to include, which
215
+ is the exact defect `alertsModel.parseInbox`'s own header records from the other side.
216
+ """
217
+ if not isinstance(box, dict):
218
+ return box
219
+ items = box.get("items")
220
+ if not isinstance(items, list):
221
+ return box
222
+ # ⭐⭐ W33-T28 / D-208 β€” THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client
223
+ # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`.
224
+ #
225
+ # β›” THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and
226
+ # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH
227
+ # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?"
228
+ # against a browser clock would re-introduce the drift the offset exists to remove β€” a reader a
229
+ # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands
230
+ # now come from the same machine.
231
+ # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree β€”
232
+ # the note two lines up records what happened last time only one of them was enriched.
233
+ return {**box, "now": alerts._now_iso(),
234
+ "items": [notification_view(n) for n in items]}
235
+
236
+
237
+ def _view_by_id(g, view_id):
238
+ """One saved view out of an assembly, by id. `None` when there is no such view.
239
+
240
+ β›”β›” W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") β€” THIS
241
+ FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND
242
+ `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes
243
+ it straight out and both `ut_assembly` and `grid_assembly` return it unchanged β€” so `.get` on
244
+ it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the
245
+ `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.**
246
+
247
+ β›” AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In
248
+ `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare
249
+ FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our
250
+ side"* β€” the exact sentence the owner reported (D-107's shape, again: an attribute error above
251
+ the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical
252
+ line is swallowed by `/notifications`' `except Exception: continue`, so **every stored
253
+ view-alert was silently dropped from the Inbox** and nobody had anything to report at all.
254
+ One expression, one loud symptom and one silent one.
255
+
256
+ ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site
257
+ be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]].
258
+
259
+ ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was
260
+ keyed `{id: view}` β€” which is precisely why the gate was green while production raised on
261
+ every call. The fixture is moving to the production shape in this same change, and tolerating
262
+ both here means a caller that legitimately holds one cannot resurrect the bug.
263
+ """
264
+ want = str(view_id or "")
265
+ if not want:
266
+ return None
267
+ views = (g or {}).get("views")
268
+ if isinstance(views, dict):
269
+ found = views.get(want)
270
+ return found if isinstance(found, dict) else None
271
+ if not isinstance(views, list):
272
+ return None
273
+ for v in views:
274
+ if isinstance(v, dict) and str(v.get("id") or "") == want:
275
+ return v
276
+ return None
277
+
278
+
279
+ def _topic_or_400(raw):
280
+ topic = str(raw or "").strip().lower()
281
+ if topic.startswith("ut_") or topic in _TOPICS:
282
+ return topic
283
+ raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
284
+
285
+
286
+ def _owner_session(session: Session, owner: str):
287
+ """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
288
+
289
+ ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields β€” so an
290
+ owner session is built by swapping the `user` RECORD and letting both derive themselves. An
291
+ earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
292
+ first write hook of the wave; the properties are the single definition of who a session is,
293
+ and going around them is how a session with an admin flag and a non-admin record exists.
294
+
295
+ Returns None when the owner is gone or deactivated β€” their alerts then stop evaluating rather
296
+ than evaluating as somebody else, which is the fail-closed direction.
297
+ """
298
+ import core.users as users
299
+
300
+ if str(owner) == str(session.uname):
301
+ return session
302
+ rec = (users.registry() or {}).get(str(owner))
303
+ if not isinstance(rec, dict) or not rec.get("active", True):
304
+ return None
305
+ # `_public` is THE definition of what a session may know about its own account (never a hash
306
+ # or a salt) β€” the same one `routes_auth` uses. Building the dict by hand here would be a
307
+ # second definition, and the one that leaks is always the copy.
308
+ return Session(tenant=session.tenant, user=users._public(str(owner), rec),
309
+ claims=session.claims, runtime=session.runtime)
310
+
311
+
312
+ def _evaluate(session: Session, rec: dict, assemblies=None):
313
+ """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
314
+
315
+ ⭐⭐ W31-T24 β€” `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole
316
+ of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each
317
+ one built a FULL assembly β€” the pool, the workspace, `rows_from_pool` over every row. Two
318
+ alerts on one view built that table twice; ten built it ten times. Nothing dedupes them,
319
+ because each `_evaluate` was a closed call.
320
+ ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module
321
+ note β€” evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak
322
+ wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and
323
+ must never share an entry. Getting that key wrong is the one way this optimisation could leak.
324
+ ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want:
325
+ they evaluate ONE alert and a memo for a single call is pure overhead.
326
+ """
327
+ import aios_grid
328
+ from harness import filter_eval
329
+
330
+ owner_sess = _owner_session(session, rec.get("owner"))
331
+ if owner_sess is None:
332
+ return {"skipped": "owner_unavailable"}
333
+ topic = str(rec.get("topic") or "")
334
+ memo_key = (topic, str(owner_sess.uname))
335
+ g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None
336
+ if g is None:
337
+ try:
338
+ if topic.startswith("ut_"):
339
+ from routes_tables import ut_assembly
340
+ # β›” `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up.
341
+ # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the
342
+ # one-shot field-name correction acks for every `ut_` topic that has an alert β€”
343
+ # taking them from the `/workspace` refresh that exists to show them to the person
344
+ # who made the edit. The customer branch below has always passed False; this one
345
+ # inherited a default nobody re-read. An inbox poll must never consume a one-shot.
346
+ g = ut_assembly(owner_sess, topic,
347
+ storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}",
348
+ consume_corrections=False)
349
+ else:
350
+ from routes_customers import grid_assembly
351
+ g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
352
+ except Exception as e: # noqa: BLE001
353
+ # ⭐ W32-T22 β€” SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not
354
+ # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it
355
+ # keeps a blanket catch β€” but it now reports the refusal's OWN code where there is
356
+ # one. `type(e).__name__` said `HTTPException` for four different causes, and
357
+ # `lastError` is the only place a user ever learns why an alert stopped firing.
358
+ #
359
+ # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an
360
+ # evaluation genuinely needs the rows to run the filter over. So an alert on a
361
+ # read-through grid is created (T22) and then skips at evaluation with
362
+ # `window_required` naming why β€” which is D-184's remaining half, and it is a
363
+ # SENTENCE now rather than silence.
364
+ return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__}
365
+ if isinstance(assemblies, dict):
366
+ assemblies[memo_key] = g
367
+
368
+ view = _view_by_id(g, rec.get("viewId"))
369
+ if not isinstance(view, dict):
370
+ # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
371
+ # deleting the alert: an alert that silently vanishes is indistinguishable from one that
372
+ # never fires, and the user cannot debug what is not there.
373
+ return {"skipped": "view_missing"}
374
+
375
+ # The SAME row build the grid and `/customers` use β€” `rows_from_pool` is what puts derived
376
+ # and overlay values on a row. Evaluating a filter against raw pool dicts would silently
377
+ # never match any condition on a user-created or measure column.
378
+ #
379
+ # β›”β›” AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND
380
+ # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths.
381
+ #
382
+ # `routes_tables.table_rows` β€” the grid the person is looking at β€” merges the DEFINITION rows
383
+ # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner
384
+ # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it:
385
+ # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell
386
+ # evaluated as BLANK. Measured on one assembly, same view, same rows:
387
+ # rows_src state='unpaid' / 'paid'
388
+ # _evaluate saw state='' / '' ⇐ every base cell blank
389
+ # the GRID saw state='unpaid' / 'paid'
390
+ # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty`
391
+ # matched EVERYTHING while the view showed none. **The alert did not merely miss rows β€” it
392
+ # inverted.** End to end: a row whose value arrived by import, automation, paste or the create
393
+ # door never fired; only a value typed as a hand EDIT did.
394
+ #
395
+ # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit;
396
+ # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`);
397
+ # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns
398
+ # `skipped: window_required`).
399
+ # β›” ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it
400
+ # would let a stale definition value shadow an edit the user has just made β€” the same defect
401
+ # `table_rows`' own note records, arriving from the other side.
402
+ _ov = (g.get("ws") or {}).get("overlays") or {}
403
+ _merged = {}
404
+ for _r in g["rows_src"]:
405
+ _pid = str(_r.get("pid"))
406
+ _cells = {k: v for k, v in _r.items() if k != "pid"}
407
+ _o = _ov.get(_pid)
408
+ if isinstance(_o, dict):
409
+ _cells.update(_o)
410
+ _merged[_pid] = _cells
411
+ rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged,
412
+ derived=g.get("derived"))
413
+ config = view.get("config") or view
414
+ ctx = filter_eval.EvalCtx(
415
+ cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
416
+ for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
417
+ measure_sets=g.get("measure_sets") or {},
418
+ today=g.get("today"))
419
+ pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
420
+ member_pids=config.get("memberPids"))
421
+ labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
422
+ return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
423
+ labels=labels, partial=False, st=session.runtime)
424
+
425
+
426
+ @router.get("/alerts")
427
+ def list_alerts(session: Session = Depends(require_session)):
428
+ return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
429
+ st=session.runtime)}
430
+
431
+
432
+ @router.post("/alerts")
433
+ def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
434
+ body = body or {}
435
+ view_id = str(body.get("viewId") or "").strip()
436
+ if not view_id:
437
+ raise err(400, "bad_view", "an alert needs the id of the view it watches")
438
+ topic = _topic_or_400(body.get("topic"))
439
+ _require_filtered_view(session, topic, view_id)
440
+ import uuid
441
+ aid = f"al_{uuid.uuid4().hex[:12]}"
442
+ rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
443
+ label=body.get("label") or "", st=session.runtime)
444
+ # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
445
+ # Deferring this to the first write hook would mean the next edit announces the whole view.
446
+ outcome = _evaluate(session, rec)
447
+ return {"alert": {**rec, "seeded": True}, "first": outcome}
448
+
449
+
450
+ def _require_filtered_view(session: Session, topic: str, view_id: str):
451
+ """400 unless `view_id` exists on `topic` AND actually narrows something.
452
+
453
+ β›” AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
454
+ that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
455
+ (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
456
+ entrant again β€” there is nothing left to enter. The owner's words are *"when a Record gets
457
+ into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
458
+ discovered by never being notified.
459
+
460
+ `is_rule_active` is the SAME activeness predicate the engine and the column tints use β€” a
461
+ half-typed rule is not a filter, and this must agree with what actually narrows or it would
462
+ accept a view whose one rule the engine then ignores.
463
+
464
+ ⭐⭐ WAVE 32 Β· T22 (owner item 17) β€” THIS FUNCTION WAS THE ERROR. Two defects, stacked, and
465
+ the second one hid the first.
466
+
467
+ (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is
468
+ `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a
469
+ read-through grid built the whole pool β€” and `scoped_pool` refuses that with
470
+ `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False`
471
+ (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`,
472
+ runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half,
473
+ closed** β€” an alert on a read-through grid can now be made at all.
474
+ (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException`
475
+ is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and
476
+ `503 store_not_ready` β€” four refusals that each say what is wrong β€” were all replaced by
477
+ *"the table is unavailable β€” try again in a moment"*. β›” AND THAT SENTENCE NEVER REACHED
478
+ A USER EITHER: `alertsApi.errorMessage` discards the text of any status β‰₯ 500 by design
479
+ (a 5xx body is the server's internals), substituting *"Something went wrong on our
480
+ side."* β€” which is the owner's screenshot, word for word. A knowable cause returned as a
481
+ 5xx is invisible by construction, so re-wording the 503 could never have fixed this.
482
+ ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is
483
+ honest. What it may no longer do is catch a refusal that already knows its own name.
484
+ """
485
+ from fastapi import HTTPException
486
+
487
+ from harness import filter_eval
488
+
489
+ try:
490
+ if topic.startswith("ut_"):
491
+ from routes_tables import ut_assembly
492
+ # ⚠ `consume_corrections=False` β€” the customer branch has always passed it and this
493
+ # one inherited a default nobody re-read. Creating an alert must not eat the one-shot
494
+ # field-name correction acks belonging to the `/workspace` refresh that exists to show
495
+ # them to the person who made the edit. Same defect `_evaluate`'s header records.
496
+ g = ut_assembly(session, topic,
497
+ storage_key=f"{session.tenant}:{topic}:{session.uname}",
498
+ consume_corrections=False, with_rows=False)
499
+ else:
500
+ from routes_customers import grid_assembly
501
+ g = grid_assembly(session, scope=topic, consume_corrections=False)
502
+ except HTTPException:
503
+ raise # it already names its own cause
504
+ except Exception as e: # noqa: BLE001
505
+ # Genuinely unexpected. Still a 503, and now it carries the exception TYPE β€” without it,
506
+ # the one path that reaches this branch is also the one path with nothing to debug from.
507
+ raise err(503, "unavailable",
508
+ f"the table could not be read ({type(e).__name__}) β€” try again in a moment")
509
+ view = _view_by_id(g, view_id)
510
+ if not isinstance(view, dict):
511
+ raise err(404, "no_view", "that view does not exist on this table")
512
+ nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
513
+
514
+ # β›”β›” WAVE 33 Β· T29 β€” **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT
515
+ # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS
516
+ # owner item 17" β€” the owner's *"Something went wrong"*. It was not, and it could not have
517
+ # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines
518
+ # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the
519
+ # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix.
520
+ #
521
+ # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the
522
+ # fix was right, and the three reasons it hid are the most transferable thing in this file.
523
+ # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each
524
+ # naming themselves as the origin of the same screenshot are mutually exclusive, and the next
525
+ # reader believes whichever they meet first β€” which is why this correction sits above rather
526
+ # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead
527
+ # of the working tree. [[grep-output-is-not-source]]
528
+ #
529
+ # β›”β›” WAVE 32 Β· T22 β€” **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly,
530
+ # that this was owner item 17 β€” see the correction directly above).
531
+ #
532
+ # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other
533
+ # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF
534
+ # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`.
535
+ #
536
+ # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the
537
+ # view HAS a condition β€” and a view with a condition is the only kind an alert is allowed on.
538
+ # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever
539
+ # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that
540
+ # worked was the refusal path: "Alert me about new records" had never once created an alert
541
+ # on a filtered view.** β›” And the raise lands OUTSIDE the `try` above, so it was not even the
542
+ # 503 β€” it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went
543
+ # wrong on our side. Try again in a moment."*, the owner's screenshot word for word.
544
+ #
545
+ # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check
546
+ # arity until the line RUNS, and this line runs only on the success path of a feature whose
547
+ # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct,
548
+ # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal
549
+ # (`no_filter` reaches the user) and the transport β€” never a creation. A gate can be green,
550
+ # thorough and honest about everything except the one path the feature exists for.
551
+ #
552
+ # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a
553
+ # column up in; building a second dict here would be a second answer to one question, which
554
+ # is this wave's other headline defect in a different file. Its leading underscore is a real
555
+ # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around.
556
+ columns = filter_eval._columns_map(g.get("fields") or [])
557
+
558
+ def _any_active(ns):
559
+ for n in ns or ():
560
+ if isinstance(n, dict) and isinstance(n.get("children"), list):
561
+ if _any_active(n["children"]):
562
+ return True
563
+ elif filter_eval.is_rule_active(n, columns):
564
+ return True
565
+ return False
566
+
567
+ if not _any_active(nodes):
568
+ raise err(400, "no_filter",
569
+ "this view has no active filter, so no record can ever ENTER it β€” add a "
570
+ "condition to the view first, then create the alert")
571
+
572
+
573
+ @router.delete("/alerts/{alert_id}")
574
+ def delete_alert(alert_id: str, session: Session = Depends(require_session)):
575
+ rec = next((r for r in alerts.list_alerts(st=session.runtime)
576
+ if str(r.get("id")) == str(alert_id)), None)
577
+ if rec is None:
578
+ raise err(404, "no_alert", "that alert does not exist")
579
+ if str(rec.get("owner")) != str(session.uname) and not session.admin:
580
+ raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
581
+ alerts.delete(alert_id, st=session.runtime)
582
+ return {"ok": True}
583
+
584
+
585
+ @router.post("/alerts/{alert_id}/run")
586
+ def run_alert(alert_id: str, session: Session = Depends(require_session)):
587
+ rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
588
+ st=session.runtime)
589
+ if str(r.get("id")) == str(alert_id)), None)
590
+ if rec is None:
591
+ raise err(404, "no_alert", "that alert does not exist")
592
+ return _evaluate(session, rec)
593
+
594
+
595
+ @router.get("/notifications")
596
+ def notifications(session: Session = Depends(require_session)):
597
+ """The inbox β€” RE-EVALUATED on read, which is a deliberate design choice.
598
+
599
+ ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
600
+ plan was a push hook: the automation engine calls `after_write` when it lands rows. But
601
+ `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
602
+ evaluated as its OWNER (see `_evaluate`) β€” so a push hook would have to mint a session inside
603
+ a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
604
+ construction that leaks scope.
605
+
606
+ Pulling on read has none of that: the caller IS a session, the assemblies are already
607
+ scope-cached, and the user cannot observe the difference β€” an inbox is only ever read by
608
+ someone opening it. The cost is that a notification is minted when you LOOK rather than when
609
+ the row landed, so the `at` stamp is detection time, not arrival time.
610
+
611
+ `after_write` stays exported for the day the engine can hand over a real identity.
612
+
613
+ ⭐⭐ W31-T24 β€” ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
614
+ β›” MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
615
+ **20 ms in-process and 3,280 ms live** on tenant #0 β€” but tenant #0 has **ZERO alerts**
616
+ (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
617
+ re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
618
+ body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
619
+ grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic Γ— owner), which
620
+ is the difference between "fine" and "three seconds per alert" the day somebody uses the
621
+ feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
622
+ this does: every alert is still evaluated, against the same rows, in the same order.
623
+ """
624
+ assemblies = {}
625
+ for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
626
+ try:
627
+ _evaluate(session, rec, assemblies=assemblies)
628
+ except Exception: # noqa: BLE001
629
+ continue # one bad alert must not empty the pane
630
+ # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
631
+ # this wave carries a `target` too. See `notification_view`'s header for why it is derived.
632
+ return inbox_view(alerts.inbox(session.uname, st=session.runtime))
633
+
634
+
635
+ @router.post("/notifications/read")
636
+ def read_notifications(body: dict = Body(default=None),
637
+ session: Session = Depends(require_session)):
638
+ body = body or {}
639
+ ids = body.get("ids")
640
+ if ids is not None and not isinstance(ids, list):
641
+ raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
642
+ # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox
643
+ # module re-renders from it β€” an un-enriched answer here would strip `target` off every row
644
+ # the moment somebody marked one read, i.e. the feature would work until first use.
645
+ return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
646
+ st=session.runtime))
647
+
648
+
649
+ def after_write(session: Session, topic_key: str):
650
+ """THE WRITE HOOK β€” call after a write that could change what a view matches.
651
+
652
+ Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
653
+ upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
654
+ edit that triggered it.
655
+
656
+ ⭐ W31-T24 β€” it shares `/notifications`' memo shape for the same reason: a write that changes
657
+ one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
658
+ ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
659
+ push hook was resolved the other way). Booked rather than wired: minting a session inside the
660
+ engine's worker thread is the ad-hoc identity construction this file exists to avoid.
661
+ """
662
+ try:
663
+ assemblies = {}
664
+ return alerts.after_write(topic_key, st=session.runtime,
665
+ runner=lambda rec: _evaluate(session, rec,
666
+ assemblies=assemblies))
667
+ except Exception: # noqa: BLE001
668
+ return {"evaluated": 0}
api/routes_automation.py CHANGED
@@ -14,6 +14,7 @@ trigger becomes a public one β€” the same class of mistake as an empty-200 permi
14
  import os
15
  import re
16
  import time
 
17
 
18
  from fastapi import APIRouter, Body, Depends, Header, Request
19
 
@@ -53,7 +54,7 @@ MODULE = "automation"
53
  _GATE = module_gate(MODULE)
54
 
55
 
56
- def _wire(defn, tenant):
57
  """One automation, as the client reads it. `running` is PROCESS state, never store state β€”
58
  see the engine header on why a persisted 'running' is a permanent lock."""
59
  live = engine.running(tenant, defn.get("id"))
@@ -141,6 +142,16 @@ def _wire(defn, tenant):
141
  # back. What needs the target database's schema (an enrich binding resolved by the profile
142
  # FLAG) stays a run-time refusal β€” see `ACTION_REQUIRED`'s note.
143
  "unconfigured": engine.unconfigured_actions(defn),
 
 
 
 
 
 
 
 
 
 
144
  }
145
 
146
 
@@ -243,7 +254,7 @@ def _field_agent_rows(session):
243
  # β€” the exact way `awaitingResults` shipped inert for a whole wave. Passing the
244
  # synthetic DEFINITION through the same function makes them identical by
245
  # construction; only `system` is stamped afterwards, because no stored row has it.
246
- row = _wire(defn_syn, session.tenant)
247
  row["system"] = SYSTEM_FIELD_AGENT
248
  out.append(row)
249
  return out
@@ -334,7 +345,7 @@ def _odoo_sync_row(session, detail=False):
334
  "config": {"connector": "odoo", "every": every,
335
  "frozen": frozen}}]},
336
  }
337
- row = _wire(defn, session.tenant)
338
  row["system"] = SYSTEM_ODOO_SYNC
339
  # β›” THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence`
340
  # speaks the automation vocabulary β€” it builds "When <cron phrase>, run N actions" out of
@@ -745,7 +756,7 @@ def list_automations(session: Session = Depends(_GATE)):
745
  pass
746
  _STATEMENTS_SEEDED.add(session.tenant)
747
  defs = engine.all_definitions(session.runtime)
748
- items = [_wire(d, session.tenant) for _, d in
749
  sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())]
750
  # ⭐⭐ WAVE 34 Β· CONTRACT C3 (W34-T48) β€” field agents join the list as SYNTHETIC rows.
751
  # ⚠ MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be
@@ -1133,6 +1144,17 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends
1133
  prompt = str((body or {}).get("prompt") or "").strip()
1134
  if not prompt:
1135
  raise err(400, "no_prompt", "type what you want the automation to do")
 
 
 
 
 
 
 
 
 
 
 
1136
  # ⚠ THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open`
1137
  # per table, so the model is shown exactly the databases this caller may already see and cannot
1138
  # name one they were not granted β€” the permission wall re-used, never a second one built beside
@@ -1170,6 +1192,7 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends
1170
  # sites, this one has a real person behind it: somebody typed the sentence, so `user` is
1171
  # the caller rather than the automation's owner.
1172
  st=session.runtime, user=getattr(session, "uname", "") or "",
 
1173
  chat=_DRAFT_CHAT[0])
1174
  if refusal or not draft:
1175
  raise err(400, "draft_refused", refusal or "no automation could be drafted from that")
@@ -1263,6 +1286,244 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends
1263
  "provider": provider, "dropped": dropped, "notes": _notes, "saved": False}
1264
 
1265
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1266
  @router.get("/automations/{auto_id}")
1267
  def get_automation(auto_id: str, session: Session = Depends(_GATE)):
1268
  """One automation, by id.
@@ -1295,7 +1556,7 @@ def get_automation(auto_id: str, session: Session = Depends(_GATE)):
1295
  defn = engine.all_definitions(session.runtime).get(str(auto_id))
1296
  if defn is None:
1297
  raise err(404, "unknown_automation", "no automation with that id")
1298
- return {"automation": _wire(defn, session.tenant)}
1299
 
1300
 
1301
  @router.post("/automations")
@@ -1317,7 +1578,7 @@ def create_automation(body: dict = Body(default=None), session: Session = Depend
1317
  notes=_notes)
1318
  if error:
1319
  raise err(400, "invalid_automation", error)
1320
- return {"automation": _wire(defn, session.tenant), "notes": _notes}
1321
 
1322
 
1323
  @router.patch("/automations/{auto_id}")
@@ -1339,6 +1600,10 @@ def patch_automation(auto_id: str, body: dict = Body(default=None),
1339
  return _patch_odoo_sync(session, body or {})
1340
  # D-75, same channel as `create` above β€” an EDIT is where this matters most, because the
1341
  # person has just typed the thing that gets dropped.
 
 
 
 
1342
  _notes = []
1343
  defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname,
1344
  notes=_notes)
@@ -1346,7 +1611,10 @@ def patch_automation(auto_id: str, body: dict = Body(default=None),
1346
  raise err(400 if error != "no such automation" else 404,
1347
  "invalid_automation" if error != "no such automation" else "unknown_automation",
1348
  error)
1349
- return {"automation": _wire(defn, session.tenant), "notes": _notes}
 
 
 
1350
 
1351
 
1352
  @router.delete("/automations/{auto_id}")
@@ -1384,6 +1652,9 @@ def delete_automation(auto_id: str, session: Session = Depends(_GATE)):
1384
  if _sys:
1385
  raise err(409, "system_agent", engine.system_agent_refusal(_sys))
1386
  engine.remove(session.runtime, auto_id)
 
 
 
1387
  return {"deleted": str(auto_id)}
1388
 
1389
 
@@ -1439,7 +1710,7 @@ def toggle_automation_node(auto_id: str, node_id: str, session: Session = Depend
1439
  raise err(404 if error == "no such automation" else 400,
1440
  "unknown_automation" if error == "no such automation" else "node_not_toggleable",
1441
  error)
1442
- return {"automation": _wire(defn, session.tenant)}
1443
 
1444
 
1445
  @router.post("/automations/{auto_id}/hook/{token}")
 
14
  import os
15
  import re
16
  import time
17
+ from datetime import datetime, timezone
18
 
19
  from fastapi import APIRouter, Body, Depends, Header, Request
20
 
 
54
  _GATE = module_gate(MODULE)
55
 
56
 
57
+ def _wire(defn, tenant, rt=None):
58
  """One automation, as the client reads it. `running` is PROCESS state, never store state β€”
59
  see the engine header on why a persisted 'running' is a permanent lock."""
60
  live = engine.running(tenant, defn.get("id"))
 
142
  # back. What needs the target database's schema (an enrich binding resolved by the profile
143
  # FLAG) stays a run-time refusal β€” see `ACTION_REQUIRED`'s note.
144
  "unconfigured": engine.unconfigured_actions(defn),
145
+ # ⭐⭐ WAVE 36 Β· W36-T38 (owner item 8 / R9) β€” WHICH ACTIONS AN AGENT BUILT.
146
+ # `{action_id: {agent, agentName, created, by}}`, empty for an ordinary automation.
147
+ # β›” THE CONFIG IS NOT HIDDEN AND MUST NOT BE. R9 is a WRITE rule: the client uses this to
148
+ # draw a lock and explain who owns the step, never to withhold what the step does β€” an
149
+ # automation nobody can audit is worse than one nobody can edit.
150
+ # ⚠ `rt=None` YIELDS `{}` RATHER THAN OMITTING THE KEY, so a caller that forgets the
151
+ # runtime produces "nothing is agent-owned" (a visible, wrong-but-safe answer) instead of
152
+ # an absent prop the client silently reads as undefined
153
+ # [[flag-shipped-without-its-writer]].
154
+ "agentActions": (_agent_marks(rt).get(str(defn.get("id") or "")) or {}) if rt else {},
155
  }
156
 
157
 
 
254
  # β€” the exact way `awaitingResults` shipped inert for a whole wave. Passing the
255
  # synthetic DEFINITION through the same function makes them identical by
256
  # construction; only `system` is stamped afterwards, because no stored row has it.
257
+ row = _wire(defn_syn, session.tenant, rt=session.runtime)
258
  row["system"] = SYSTEM_FIELD_AGENT
259
  out.append(row)
260
  return out
 
345
  "config": {"connector": "odoo", "every": every,
346
  "frozen": frozen}}]},
347
  }
348
+ row = _wire(defn, session.tenant, rt=session.runtime)
349
  row["system"] = SYSTEM_ODOO_SYNC
350
  # β›” THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence`
351
  # speaks the automation vocabulary β€” it builds "When <cron phrase>, run N actions" out of
 
756
  pass
757
  _STATEMENTS_SEEDED.add(session.tenant)
758
  defs = engine.all_definitions(session.runtime)
759
+ items = [_wire(d, session.tenant, rt=session.runtime) for _, d in
760
  sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())]
761
  # ⭐⭐ WAVE 34 Β· CONTRACT C3 (W34-T48) β€” field agents join the list as SYNTHETIC rows.
762
  # ⚠ MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be
 
1144
  prompt = str((body or {}).get("prompt") or "").strip()
1145
  if not prompt:
1146
  raise err(400, "no_prompt", "type what you want the automation to do")
1147
+ # ⭐⭐ ASK D-18 (2026-08-18) β€” THE KEY THE CLIENT SENDS IS NOW READ. This door took `prompt` off
1148
+ # the body and nothing else, so the Agent chat's model toggle (W36-T34) was a control that
1149
+ # configured nothing: the value was accepted and dropped, which is the shape three tickets in
1150
+ # this wave already tripped over. D held T34 rather than mark a picker BUILT while its value
1151
+ # stopped at the browser, and was right to.
1152
+ # ⚠ UNKNOWN OR UNCONFIGURED FALLS BACK TO THE LADDER, never refuses (D-18's asked-for posture,
1153
+ # and C5's everywhere else): a model the ladder stops offering must not turn every later draft
1154
+ # into an error. The response already names the rung that ANSWERED, which is the honest half.
1155
+ _model = str((body or {}).get("model") or "").strip().lower()
1156
+ if _model in ("", "auto"):
1157
+ _model = None
1158
  # ⚠ THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open`
1159
  # per table, so the model is shown exactly the databases this caller may already see and cannot
1160
  # name one they were not granted β€” the permission wall re-used, never a second one built beside
 
1192
  # sites, this one has a real person behind it: somebody typed the sentence, so `user` is
1193
  # the caller rather than the automation's owner.
1194
  st=session.runtime, user=getattr(session, "uname", "") or "",
1195
+ model=_model,
1196
  chat=_DRAFT_CHAT[0])
1197
  if refusal or not draft:
1198
  raise err(400, "draft_refused", refusal or "no automation could be drafted from that")
 
1286
  "provider": provider, "dropped": dropped, "notes": _notes, "saved": False}
1287
 
1288
 
1289
+ # ══════════════════ WAVE 36 Β· W36-T38 (owner item 8, ruling R9) β€” AGENT-AUTHORED ACTIONS ══════
1290
+ #
1291
+ # Owner, verbatim (2026-08-18): *"Create the ability for an automation agent to create any
1292
+ # 'Action' under the Canvas, so its configuration can only be touched by the agent. Be it a tool a
1293
+ # script etc. We need to really guardrail the reach of this script."*
1294
+ #
1295
+ # ⭐⭐ R9 IS A **WRITE** RULE, NOT A VISIBILITY RULE, and the ticket says so in as many words: the
1296
+ # user MAY read an agent-authored action's configuration and MAY delete it; what they may not do
1297
+ # is hand-edit it. Hiding a config from the person whose workspace it runs in is a different
1298
+ # product, and a worse one β€” nobody can audit what they cannot see.
1299
+ #
1300
+ # β›” AND IT IS ENFORCED AT THE DOOR, NEVER IN A `disabled` ATTRIBUTE. `routes_automation` already
1301
+ # carries that lesson for a different control; a client-side lock is a suggestion, and the whole
1302
+ # point of R9 is that the agent's configuration stays coherent with what the agent believes it
1303
+ # built.
1304
+ #
1305
+ # ⚠ WHY THE MARK IS A SEPARATE BUCKET RATHER THAN A KEY ON THE ACTION. `clean_actions` builds every
1306
+ # action KEY BY KEY from an allowlist and drops anything it does not recognise (D-75) β€” so a
1307
+ # marker stored inside the action would be silently erased on the next save, and the wall would
1308
+ # quietly stop existing with every gate still green. This is an ANNOTATION layer keyed by
1309
+ # `(automation id, action id)`; the configuration itself stays where it always was, in the
1310
+ # automation, with exactly one writer.
1311
+ AGENT_ACTIONS_KEY = "automation_agent_actions"
1312
+
1313
+ #: Ids this door mints. ⚠ It must match `clean_actions`' `act_[a-z0-9_]{1,32}` or the engine
1314
+ #: re-mints it and the annotation points at an action that no longer carries that id.
1315
+ _AGENT_ACTION_PREFIX = "act_ag"
1316
+
1317
+
1318
+ def _agent_marks(rt):
1319
+ """`{auto_id: {action_id: mark}}` for one tenant. `{}` on any failure β€” an unreadable
1320
+ annotation must degrade to "nothing is agent-owned", never to a 500 on the automations rail.
1321
+
1322
+ β›” AND "DEGRADE TO NOTHING IS OWNED" IS THE SAFE DIRECTION HERE, which is worth stating because
1323
+ it usually is not. The wall this feeds protects the AGENT's coherence, not the tenant's data: a
1324
+ lost mark lets a person edit a config they own anyway, in their own workspace, on a step they
1325
+ could always have deleted outright. A wall that failed CLOSED would instead make an automation
1326
+ permanently unsavable because one annotation read timed out.
1327
+ """
1328
+ try:
1329
+ found = rt.get(AGENT_ACTIONS_KEY) or {}
1330
+ except Exception: # noqa: BLE001
1331
+ return {}
1332
+ return found if isinstance(found, dict) else {}
1333
+
1334
+
1335
+ def _flat_actions(actions, out=None, depth=0):
1336
+ """Every action in a flow, INCLUDING the ones nested inside a group's arms.
1337
+
1338
+ ⚠ NESTED ACTIONS ARE THE ONES THIS MUST NOT MISS. `unconfigured_actions` walks them for the
1339
+ same reason: a step inside an If/then branch is exactly the one a person cannot see, and a
1340
+ wall that only looked at the top level would leave the agent's own nested step editable.
1341
+ """
1342
+ out = [] if out is None else out
1343
+ if depth > 6 or not isinstance(actions, list):
1344
+ return out
1345
+ for action in actions:
1346
+ if not isinstance(action, dict):
1347
+ continue
1348
+ out.append(action)
1349
+ for key in ("then", "else", "actions"):
1350
+ _flat_actions(action.get(key), out, depth + 1)
1351
+ return out
1352
+
1353
+
1354
+ def _flow_actions(defn):
1355
+ return _flat_actions(((defn or {}).get("flow") or {}).get("actions") or [])
1356
+
1357
+
1358
+ def _same_action(left, right, rt):
1359
+ """Do these two actions carry the SAME kind and configuration?
1360
+
1361
+ β›” COMPARED AFTER `clean_actions`, ON BOTH SIDES, and that is the difference between a wall and
1362
+ a nuisance. The stored action has already been through the cleaner; a client round-trip has
1363
+ not, so it may carry a key order, a blank string or a dropped condition that means nothing.
1364
+ Comparing raw shapes would refuse an edit that changes nothing, and a wall that fires on a
1365
+ no-op is one an operator learns to route around [[one-question-two-normalizers]].
1366
+ """
1367
+ def _clean(action):
1368
+ cleaned, error = engine.clean_actions([dict(action or {}, id="act_1")], rt=rt)
1369
+ if error or not cleaned:
1370
+ return None
1371
+ one = dict(cleaned[0])
1372
+ one.pop("id", None)
1373
+ return one
1374
+
1375
+ a, b = _clean(left), _clean(right)
1376
+ return a is not None and a == b
1377
+
1378
+
1379
+ def _agent_owned_guard(session, auto_id, body):
1380
+ """R9 AT THE DOOR: a user may not change an agent-authored action's kind or configuration.
1381
+
1382
+ Three outcomes, and the middle one is the ruling:
1383
+ * the action is ABSENT from the incoming flow -> a DELETE, and R9 allows it
1384
+ * the action is present and CHANGED -> **409**, naming the action and the agent
1385
+ * the action is present and identical -> nothing happens, so an ordinary save of a
1386
+ flow that merely CONTAINS an agent action
1387
+ is not refused
1388
+ """
1389
+ marks = _agent_marks(session.runtime).get(str(auto_id)) or {}
1390
+ if not isinstance(marks, dict) or not marks:
1391
+ return
1392
+ incoming = (body or {}).get("flow")
1393
+ if not isinstance(incoming, dict) or "actions" not in incoming:
1394
+ return
1395
+ stored = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {}
1396
+ was = {str(a.get("id") or ""): a for a in _flow_actions(stored)}
1397
+ now = {str(a.get("id") or ""): a
1398
+ for a in _flat_actions(incoming.get("actions") or [])}
1399
+ for action_id, mark in marks.items():
1400
+ action_id = str(action_id)
1401
+ if action_id not in now or action_id not in was:
1402
+ continue # removed, or never stored: not an EDIT
1403
+ if _same_action(was[action_id], now[action_id], session.runtime):
1404
+ continue
1405
+ who = str((mark or {}).get("agentName") or (mark or {}).get("agent") or "an agent")
1406
+ raise err(409, "agent_owned",
1407
+ f"this step was built by {who} and only {who} can change how it is set up. "
1408
+ f"You can read it, and you can delete it, but it cannot be edited by hand")
1409
+
1410
+
1411
+ def _prune_marks(session, auto_id):
1412
+ """Drop annotations whose action is gone, and the whole entry when the automation is.
1413
+
1414
+ ⚠ CALLED AFTER EVERY WRITE, because the alternative is an annotation bucket that only ever
1415
+ grows: a mark on a deleted action would keep refusing an id nobody can see, and a mark on a
1416
+ deleted automation would sit in the tenant document forever.
1417
+ """
1418
+ auto_id = str(auto_id)
1419
+
1420
+ def _set(cur):
1421
+ cur = dict(cur or {})
1422
+ marks = cur.get(auto_id)
1423
+ if not isinstance(marks, dict):
1424
+ cur.pop(auto_id, None)
1425
+ return cur
1426
+ defn = (engine.all_definitions(session.runtime) or {}).get(auto_id)
1427
+ if defn is None:
1428
+ cur.pop(auto_id, None)
1429
+ return cur
1430
+ alive = {str(a.get("id") or "") for a in _flow_actions(defn)}
1431
+ kept = {k: v for k, v in marks.items() if str(k) in alive}
1432
+ if kept:
1433
+ cur[auto_id] = kept
1434
+ else:
1435
+ cur.pop(auto_id, None)
1436
+ return cur
1437
+
1438
+ session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync")
1439
+
1440
+
1441
+ @router.post("/automations/{auto_id}/agent-actions")
1442
+ def agent_author_action(auto_id: str, body: dict = Body(default=None),
1443
+ session: Session = Depends(_GATE)):
1444
+ """THE AGENT'S DOOR: add or replace one Action under the Canvas, marked agent-owned (R9).
1445
+
1446
+ {agent: "<agent id>", action: {kind, config, when?, id?}}
1447
+ -> {automation, actionId, agent}
1448
+
1449
+ β›” THE ACTION GOES THROUGH `clean_actions` LIKE EVERY OTHER ONE, and that is what "guardrail
1450
+ the reach of this script" means in code: an agent cannot invent a config key, cannot name a
1451
+ kind that is not in the catalog, and cannot reach a kind this tenant is not entitled to. The
1452
+ agent gets a different WALL on editing, never a wider vocabulary.
1453
+
1454
+ β›” AND IT CANNOT AUTHOR A KIND THAT IS NOT BUILT. `run_script` is `ready: False` in the
1455
+ catalog, so `clean_actions` refuses it here exactly as it refuses it for a person β€” see this
1456
+ lane's mailbox for why the script ARM is booked rather than half-built: a per-row script
1457
+ contract and a client card are both missing, and a step that reports success and does nothing
1458
+ is the failure this repo has already paid for.
1459
+
1460
+ ⚠ THE AGENT ID IS CHECKED AGAINST THE TENANT'S OWN AGENTS. A mark naming an agent that does
1461
+ not exist would refuse every future edit with a sentence naming nobody.
1462
+ """
1463
+ import routes_slack # noqa: PLC0415
1464
+
1465
+ body = body if isinstance(body, dict) else {}
1466
+ agent_id = str(body.get("agent") or "").strip()
1467
+ agent = routes_slack._agents(session.runtime).get(agent_id)
1468
+ if not isinstance(agent, dict):
1469
+ raise err(404, "no_agent", "there is no agent with that id in this workspace")
1470
+ action = body.get("action")
1471
+ if not isinstance(action, dict) or not str(action.get("kind") or "").strip():
1472
+ raise err(400, "no_action", "an action needs a kind")
1473
+
1474
+ defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id))
1475
+ if defn is None:
1476
+ raise err(404, "unknown_automation", "no automation with that id")
1477
+ if str(defn.get("system") or "").strip():
1478
+ raise err(409, "system_agent", engine.system_agent_refusal(str(defn["system"])))
1479
+
1480
+ existing = list(((defn.get("flow") or {}).get("actions") or []))
1481
+ taken = {str(a.get("id") or "") for a in _flat_actions(existing)}
1482
+ action_id = str(action.get("id") or "").strip()
1483
+ if not action_id or action_id not in taken:
1484
+ n = 1
1485
+ while f"{_AGENT_ACTION_PREFIX}{n}" in taken:
1486
+ n += 1
1487
+ action_id = f"{_AGENT_ACTION_PREFIX}{n}"
1488
+ fresh = dict(action, id=action_id)
1489
+
1490
+ # ⚠ VALIDATED BEFORE ANYTHING IS WRITTEN, so a refusal leaves the automation exactly as it was.
1491
+ checked, error = engine.clean_actions([fresh], rt=session.runtime)
1492
+ if error or not checked:
1493
+ raise err(400, "invalid_action", error or "that action could not be built")
1494
+
1495
+ replaced = False
1496
+ for i, one in enumerate(existing):
1497
+ if isinstance(one, dict) and str(one.get("id") or "") == action_id:
1498
+ existing[i], replaced = fresh, True
1499
+ break
1500
+ if not replaced:
1501
+ existing.append(fresh)
1502
+
1503
+ notes = []
1504
+ updated, error = engine.patch(session.runtime, str(auto_id),
1505
+ {"flow": {**(defn.get("flow") or {}), "actions": existing}},
1506
+ username=session.uname, notes=notes)
1507
+ if error:
1508
+ raise err(400, "invalid_automation", error)
1509
+
1510
+ def _set(cur):
1511
+ cur = dict(cur or {})
1512
+ marks = dict(cur.get(str(auto_id)) or {})
1513
+ marks[action_id] = {"agent": agent_id,
1514
+ "agentName": str(agent.get("label") or agent.get("channelName")
1515
+ or agent_id),
1516
+ "created": datetime.now(timezone.utc).isoformat(timespec="seconds"),
1517
+ "by": session.uname}
1518
+ cur[str(auto_id)] = marks
1519
+ return cur
1520
+
1521
+ session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync")
1522
+ _prune_marks(session, auto_id)
1523
+ return {"automation": _wire(updated, session.tenant, rt=session.runtime),
1524
+ "actionId": action_id, "agent": agent_id, "notes": notes}
1525
+
1526
+
1527
  @router.get("/automations/{auto_id}")
1528
  def get_automation(auto_id: str, session: Session = Depends(_GATE)):
1529
  """One automation, by id.
 
1556
  defn = engine.all_definitions(session.runtime).get(str(auto_id))
1557
  if defn is None:
1558
  raise err(404, "unknown_automation", "no automation with that id")
1559
+ return {"automation": _wire(defn, session.tenant, rt=session.runtime)}
1560
 
1561
 
1562
  @router.post("/automations")
 
1578
  notes=_notes)
1579
  if error:
1580
  raise err(400, "invalid_automation", error)
1581
+ return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes}
1582
 
1583
 
1584
  @router.patch("/automations/{auto_id}")
 
1600
  return _patch_odoo_sync(session, body or {})
1601
  # D-75, same channel as `create` above β€” an EDIT is where this matters most, because the
1602
  # person has just typed the thing that gets dropped.
1603
+ # β›”β›” W36-T38 / R9 β€” THE AGENT-OWNED WALL, AT THE DOOR, BEFORE ANY WRITE. A `disabled`
1604
+ # attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the
1605
+ # step and permits an ordinary save of a flow that merely contains one.
1606
+ _agent_owned_guard(session, auto_id, body or {})
1607
  _notes = []
1608
  defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname,
1609
  notes=_notes)
 
1611
  raise err(400 if error != "no such automation" else 404,
1612
  "invalid_automation" if error != "no such automation" else "unknown_automation",
1613
  error)
1614
+ # An action the user removed takes its annotation with it, or the mark outlives its subject
1615
+ # and refuses an id nobody can see.
1616
+ _prune_marks(session, auto_id)
1617
+ return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes}
1618
 
1619
 
1620
  @router.delete("/automations/{auto_id}")
 
1652
  if _sys:
1653
  raise err(409, "system_agent", engine.system_agent_refusal(_sys))
1654
  engine.remove(session.runtime, auto_id)
1655
+ # R9's other half: the user MAY delete an agent-authored step, and deleting the whole
1656
+ # automation takes its annotations with it rather than stranding them in the document.
1657
+ _prune_marks(session, auto_id)
1658
  return {"deleted": str(auto_id)}
1659
 
1660
 
 
1710
  raise err(404 if error == "no such automation" else 400,
1711
  "unknown_automation" if error == "no such automation" else "node_not_toggleable",
1712
  error)
1713
+ return {"automation": _wire(defn, session.tenant, rt=session.runtime)}
1714
 
1715
 
1716
  @router.post("/automations/{auto_id}/hook/{token}")
api/routes_grid.py CHANGED
@@ -1000,3 +1000,84 @@ def grid_events_route(body: dict = Body(default=None),
1000
  # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh
1001
  # data that was never in it.
1002
  return out
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1000
  # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh
1001
  # data that was never in it.
1002
  return out
1003
+
1004
+
1005
+ # ── CONTRACT C1 (W36-T20): THE REGISTRY TOPICS' ROW READERS ───────────────────────────────────
1006
+ # ⭐⭐ R6 β€” *"EVERY database gets the same permission logic, always."* `core.perm_scope.scoped_table`
1007
+ # is the ONE door to any database's rows, and it cannot import a topic's pool builder: `core` never
1008
+ # imports up (`platform/ARCHITECTURE.md`) and these pools are built by `modules/` + `aios_grid`
1009
+ # behind this layer's per-tenant cache. So the app layer DECLARES its readers, exactly as
1010
+ # `routes_odoo_tables` declares connected tables to `user_tables.register_connected`.
1011
+ #
1012
+ # β›” REGISTERED HERE RATHER THAN IN `routes_customers`/`routes_products` because those two files
1013
+ # are outside wave 36's lane-C fence. The readers themselves are three lines each and call the
1014
+ # SAME `_pool_for` + `derive_pool_scope` pair those routes call, so there is no second pool and no
1015
+ # second scope derivation β€” only a second CALLER of the one that exists.
1016
+ #
1017
+ # ⚠ AND THE TOPIC ROUTES STILL HAVE THEIR OWN DOOR TODAY. Contract C1 says `apply_row_scope` +
1018
+ # `visible_fields` "move behind" `scoped_table`; moving `routes_customers.grid_assembly` and
1019
+ # `routes_products.scoped_pool` is booked as a PENDING row (mailbox/C.md, C-1) rather than done
1020
+ # here, because neither file is in this fence. What ships now is the arm the wave is load-bearing
1021
+ # on β€” every `ut_*` database, plus E's sandbox β€” and a topic arm that is REAL rather than stubbed,
1022
+ # so `scoped_table`'s topic leg is exercised by the product instead of only by a gate.
1023
+ def _topic_rt(st, module):
1024
+ """The tenant runtime a topic pool must be built against, or a REPORTED refusal.
1025
+
1026
+ β›” A topic pool is per TENANT (`rt.pool_cache`), so `st=None` cannot be resolved to "the
1027
+ default" without picking a tenant at random β€” which on this box is tenant #0's PRODUCTION
1028
+ data. Standing rule 1's second sentence: say why, and say what to do instead.
1029
+ """
1030
+ if st is None:
1031
+ import core.perm_scope as perm_scope
1032
+ raise perm_scope.Unresolvable(
1033
+ subject="rows", effect="unreadable",
1034
+ cause=f"'{module}' is a registry topic whose pool is built per tenant and no tenant "
1035
+ f"runtime was passed",
1036
+ recommendation="pass the session's runtime as `st=`. A topic pool cannot be "
1037
+ "resolved without knowing which tenant is asking")
1038
+ return st
1039
+
1040
+
1041
+ def _customer_rows(table_key, user, st):
1042
+ """`(fields, rows)` for the customer topic β€” the SAME derivation `_team_agent` uses."""
1043
+ import aios_grid
1044
+ import core.perm_scope as perm_scope
1045
+ from routes_customers import _pool_for
1046
+
1047
+ rt = _topic_rt(st, table_key)
1048
+ team_id, agent = perm_scope.derive_pool_scope(user, table_key)
1049
+ return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent)
1050
+
1051
+
1052
+ def _product_rows(table_key, user, st):
1053
+ """`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a
1054
+ BU-pinned reader gets that BU's field contract rather than the consolidated one."""
1055
+ import core.perm_scope as perm_scope
1056
+ from routes_products import _pool_for, pd_fields
1057
+
1058
+ rt = _topic_rt(st, table_key)
1059
+ team_id, _agent = perm_scope.derive_pool_scope(user, table_key)
1060
+ return pd_fields(consolidated=team_id is None), _pool_for(rt, team_id)
1061
+
1062
+
1063
+ def _register_topic_rows():
1064
+ """Declare both topic readers to C1. Called at import; returns the registered key set.
1065
+
1066
+ ⚠ THE KEYS ARE LITERALS AND THE ROUTE IMPORTS ARE INSIDE THE READERS, on purpose: this runs at
1067
+ module import, and `from routes_products import MODULE` here would pull a sibling router in
1068
+ before its own imports have settled. Every other cross-router reference in this file is lazy
1069
+ for the same reason. The literals are held to their sources by `verify_scopes`, so they cannot
1070
+ drift into naming a topic that does not exist.
1071
+ """
1072
+ import core.perm_scope as perm_scope
1073
+
1074
+ perm_scope.register_rows(_customer_rows, MODULE)
1075
+ return perm_scope.register_rows(_product_rows, _PRODUCT_MODULE)
1076
+
1077
+
1078
+ #: ⚠ `_`-prefixed, because three functions in this file already bind the name `PRODUCT_MODULE`
1079
+ #: LOCALLY from `routes_products`. A module-level twin of that spelling would read as the same
1080
+ #: thing and be a different one β€” [[constant-two-features-share]] waiting to happen.
1081
+ _PRODUCT_MODULE = "product_data"
1082
+
1083
+ _C1_ROW_SOURCES = _register_topic_rows()
api/routes_keychain.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_nav.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_oauth.py CHANGED
@@ -1,114 +1,114 @@
1
- """routes_oauth.py β€” the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12).
2
-
3
- Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions,
4
- shapes and status codes here; every decision that could be wrong lives in the module a gate
5
- can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry,
6
- so the day a second provider lands here is the day nothing in this file changes.
7
-
8
- MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session
9
- `main.py`, and `routes_automation` is already included there β€” so this router rides inside it
10
- (`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that
11
- alters no path.
12
-
13
- ⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent
14
- screen** β€” it is a top-level navigation the client reaches by `<a href>`, never JSON. The
15
- callback 302s BACK to the return path the `state` carried (relative-only, sanitised by
16
- `oauth_connect.safe_next`), so the user lands where they left β€” connected or not, whatever
17
- went wrong rides in the query string; a dead-end error page where the app used to be reads as
18
- "the product broke", not "the connect failed".
19
- """
20
- import os
21
-
22
- from fastapi import APIRouter, Depends, Request
23
- from fastapi.responses import RedirectResponse
24
-
25
- import oauth_connect
26
- from deps import Session, err, require_session
27
-
28
- router = APIRouter(prefix="/oauth")
29
-
30
-
31
- def _redirect_uri(request: Request, provider: str) -> str:
32
- """The redirect URI this deployment registers at the provider β€” env-pinned when the
33
- container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
34
- the request. MUST match a console-registered URI verbatim, so it is computed in exactly
35
- one place.
36
-
37
- ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
38
- the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
39
- the request-derived fallback below is effectively dev-only.
40
- β›” THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
41
- this returns is where the provider sends the user BACK, and the session cookie is host-only
42
- (`aios_session.py:114-117`, no `domain=`) β€” so a callback base that disagrees with the host
43
- the user actually browsed plants the session on the wrong hostname and they return logged
44
- out. Moving the app to a new hostname means moving this value AND re-registering the
45
- resulting URI in the provider console; one without the other fails closed.
46
- Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
47
- base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
48
- if not base:
49
- base = f"{request.url.scheme}://{request.url.netloc}"
50
- return f"{base}/api/v1/oauth/{provider}/callback"
51
-
52
-
53
- @router.get("/status")
54
- def oauth_status(session: Session = Depends(require_session)):
55
- """C5's status shape for the SESSION user, one entry per registry provider:
56
- `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's
57
- `ready` reads through."""
58
- return oauth_connect.status(session.runtime, session.uname)
59
-
60
-
61
- def _offered_or_404(provider: str):
62
- """β›”β›” W32-T14 / OWNER ITEM 12 / R6 β€” A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR.
63
-
64
- The owner pasted the failure this replaces: clicking Connect on the Google card answered
65
- `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error β€” it is that the flow is
66
- not offered at all, so the honest status is the one for a URL that does not exist. `404`
67
- rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not
68
- coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr).
69
-
70
- ⚠ Every door in this router goes through it, START included, because the JSON the owner saw
71
- came from the start route and a guard on one leg is a guard on one leg.
72
- """
73
- if not oauth_connect.offered(provider):
74
- raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
75
-
76
-
77
- @router.get("/{provider}/start")
78
- def oauth_start(provider: str, request: Request, next: str = "",
79
- session: Session = Depends(require_session)):
80
- """302 to the provider's consent screen (A3 β€” a navigation, never JSON). `?next=` is the
81
- RELATIVE path the callback returns the browser to; it rides inside the single-use state,
82
- sanitised, so the round trip cannot be steered off-origin."""
83
- _offered_or_404(provider)
84
- url, problem = oauth_connect.start(provider, session.uname,
85
- _redirect_uri(request, provider), next_path=next)
86
- if problem:
87
- raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem)
88
- return RedirectResponse(url, status_code=302)
89
-
90
-
91
- @router.get("/{provider}/callback")
92
- def oauth_callback(provider: str, request: Request,
93
- session: Session = Depends(require_session),
94
- state: str = "", code: str = "", error: str = ""):
95
- """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends
96
- the browser back to the state's return path β€” connected or not (see module header)."""
97
- _offered_or_404(provider)
98
- if error:
99
- home = "/#/"
100
- return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302)
101
- email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code)
102
- sep = "&" if "?" in home else "?"
103
- if problem:
104
- return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302)
105
- return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302)
106
-
107
-
108
- @router.post("/{provider}/disconnect")
109
- def oauth_disconnect(provider: str, session: Session = Depends(require_session)):
110
- _offered_or_404(provider)
111
- if oauth_connect.provider_def(provider) is None:
112
- raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
113
- oauth_connect.disconnect(session.runtime, session.uname, provider)
114
- return {"disconnected": provider}
 
1
+ """routes_oauth.py β€” the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12).
2
+
3
+ Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions,
4
+ shapes and status codes here; every decision that could be wrong lives in the module a gate
5
+ can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry,
6
+ so the day a second provider lands here is the day nothing in this file changes.
7
+
8
+ MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session
9
+ `main.py`, and `routes_automation` is already included there β€” so this router rides inside it
10
+ (`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that
11
+ alters no path.
12
+
13
+ ⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent
14
+ screen** β€” it is a top-level navigation the client reaches by `<a href>`, never JSON. The
15
+ callback 302s BACK to the return path the `state` carried (relative-only, sanitised by
16
+ `oauth_connect.safe_next`), so the user lands where they left β€” connected or not, whatever
17
+ went wrong rides in the query string; a dead-end error page where the app used to be reads as
18
+ "the product broke", not "the connect failed".
19
+ """
20
+ import os
21
+
22
+ from fastapi import APIRouter, Depends, Request
23
+ from fastapi.responses import RedirectResponse
24
+
25
+ import oauth_connect
26
+ from deps import Session, err, require_session
27
+
28
+ router = APIRouter(prefix="/oauth")
29
+
30
+
31
+ def _redirect_uri(request: Request, provider: str) -> str:
32
+ """The redirect URI this deployment registers at the provider β€” env-pinned when the
33
+ container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
34
+ the request. MUST match a console-registered URI verbatim, so it is computed in exactly
35
+ one place.
36
+
37
+ ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
38
+ the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
39
+ the request-derived fallback below is effectively dev-only.
40
+ β›” THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
41
+ this returns is where the provider sends the user BACK, and the session cookie is host-only
42
+ (`aios_session.py:114-117`, no `domain=`) β€” so a callback base that disagrees with the host
43
+ the user actually browsed plants the session on the wrong hostname and they return logged
44
+ out. Moving the app to a new hostname means moving this value AND re-registering the
45
+ resulting URI in the provider console; one without the other fails closed.
46
+ Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
47
+ base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
48
+ if not base:
49
+ base = f"{request.url.scheme}://{request.url.netloc}"
50
+ return f"{base}/api/v1/oauth/{provider}/callback"
51
+
52
+
53
+ @router.get("/status")
54
+ def oauth_status(session: Session = Depends(require_session)):
55
+ """C5's status shape for the SESSION user, one entry per registry provider:
56
+ `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's
57
+ `ready` reads through."""
58
+ return oauth_connect.status(session.runtime, session.uname)
59
+
60
+
61
+ def _offered_or_404(provider: str):
62
+ """β›”β›” W32-T14 / OWNER ITEM 12 / R6 β€” A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR.
63
+
64
+ The owner pasted the failure this replaces: clicking Connect on the Google card answered
65
+ `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error β€” it is that the flow is
66
+ not offered at all, so the honest status is the one for a URL that does not exist. `404`
67
+ rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not
68
+ coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr).
69
+
70
+ ⚠ Every door in this router goes through it, START included, because the JSON the owner saw
71
+ came from the start route and a guard on one leg is a guard on one leg.
72
+ """
73
+ if not oauth_connect.offered(provider):
74
+ raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
75
+
76
+
77
+ @router.get("/{provider}/start")
78
+ def oauth_start(provider: str, request: Request, next: str = "",
79
+ session: Session = Depends(require_session)):
80
+ """302 to the provider's consent screen (A3 β€” a navigation, never JSON). `?next=` is the
81
+ RELATIVE path the callback returns the browser to; it rides inside the single-use state,
82
+ sanitised, so the round trip cannot be steered off-origin."""
83
+ _offered_or_404(provider)
84
+ url, problem = oauth_connect.start(provider, session.uname,
85
+ _redirect_uri(request, provider), next_path=next)
86
+ if problem:
87
+ raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem)
88
+ return RedirectResponse(url, status_code=302)
89
+
90
+
91
+ @router.get("/{provider}/callback")
92
+ def oauth_callback(provider: str, request: Request,
93
+ session: Session = Depends(require_session),
94
+ state: str = "", code: str = "", error: str = ""):
95
+ """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends
96
+ the browser back to the state's return path β€” connected or not (see module header)."""
97
+ _offered_or_404(provider)
98
+ if error:
99
+ home = "/#/"
100
+ return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302)
101
+ email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code)
102
+ sep = "&" if "?" in home else "?"
103
+ if problem:
104
+ return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302)
105
+ return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302)
106
+
107
+
108
+ @router.post("/{provider}/disconnect")
109
+ def oauth_disconnect(provider: str, session: Session = Depends(require_session)):
110
+ _offered_or_404(provider)
111
+ if oauth_connect.provider_def(provider) is None:
112
+ raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
113
+ oauth_connect.disconnect(session.runtime, session.uname, provider)
114
+ return {"disconnected": provider}
api/routes_odoo_tables.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_publish.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_query.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_records.py CHANGED
@@ -1,211 +1,211 @@
1
- """Record detail routes: durable comments, scoped to the caller's book β€” ON EVERY DATABASE.
2
-
3
- ⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a
4
- customer-shaped wall bolted to the module import line: `_in_book` asked
5
- `routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and
6
- typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the
7
- panel answered "that customer is not in your book" β€” the owner's report. The dangerous half is
8
- the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment
9
- is filed against somebody's customer where the whole team can read it.
10
-
11
- THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the
12
- events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall
13
- per scope β€” the GRANT and the ROW SET β€” by asking that topic's own route, never by re-deriving
14
- one here:
15
-
16
- customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant
17
- product `routes_products.scoped_pool` behind the `product_data` grant
18
- ut_<slug> `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall
19
- (404 unknown / 403 not yours β€” a user table has no module grant).
20
- ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` β€” the pool builds every
21
- ROW to derive a pid set this module discards, and RAISES 409 on a
22
- read-through grid past one window, which is why the record drawer painted
23
- an error page on `ut_odoo_gl_lines`.
24
-
25
- ⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a
26
- section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning
27
- another session's gate mid-wave β€” the WALL is the query parameter, not the word.
28
-
29
- ⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every
30
- shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a
31
- 400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for
32
- the same reason: a typo served as `customer` answers a question nobody asked).
33
- """
34
- from fastapi import APIRouter, Body, Depends, Query
35
-
36
- from deps import Session, err, require_session
37
-
38
- router = APIRouter(prefix="/api/v1")
39
-
40
- #: The customer topic's two names β€” one book, two surfaces (the Cohort page is the customer table
41
- #: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`.
42
- _CUSTOMER_SCOPES = ("", "customer", "cohort")
43
-
44
-
45
- def _scope_or_400(raw):
46
- scope = str(raw or "customer").strip().lower()
47
- if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
48
- return "customer" if scope in _CUSTOMER_SCOPES else scope
49
- raise err(400, "bad_scope",
50
- "scope must be customer, cohort, product or a ut_ database β€” refusing to guess")
51
-
52
-
53
- def _pool_or_refuse(session: Session, scope: str):
54
- """The pids this session may attach comments to ON THIS DATABASE β€” grant wall included.
55
-
56
- Returns **`(pids, unbounded)`** β€” a 2-tuple on EVERY branch. Raises the topic's own 403/404/503,
57
- so a caller who may not open the surface never learns anything about the row they asked about.
58
-
59
- β›” `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past
60
- one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns
61
- `frozenset()` for both β€” see the branch below.
62
- ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that
63
- do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py`
64
- that REPLACES this function with its own lambda. A gate's test double is a caller
65
- ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a
66
- bare frozenset and the section died on `ValueError: too many values to unpack` β€” no tally, no
67
- failing name. Change the shape here and that double changes with it.
68
- """
69
- if scope == "product":
70
- from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
71
-
72
- session.require(PRODUCT_MODULE)
73
- pids, _team, _rows, _fields = scoped_pool(session)
74
- # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes
75
- # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised
76
- # a `TypeError` β€” a 500 on every product comment β€” while both other branches worked. A
77
- # return shape is a contract even when the function is private.
78
- return (pids, False)
79
- if scope.startswith("ut_"):
80
- # No module grant exists for a user table β€” `_defn_or_refuse` inside `scoped_pids` IS
81
- # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the
82
- # rows routes do.
83
- #
84
- # ⭐⭐ W33-T03 / D-183 β€” `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG.
85
- # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on
86
- # a read-through grid larger than one window it RAISES `409 window_required`. So opening the
87
- # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page β€” for a panel
88
- # that renders comments about ONE row it already has. `scoped_pids` answers the identical
89
- # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes
90
- # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace`
91
- # used to become openable on those two grids.
92
- from routes_tables import scoped_pids
93
-
94
- limits = []
95
- pids, _fields, _defn = scoped_pids(session, scope, limits=limits)
96
- # β›” AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is
97
- # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a
98
- # database with no rows and for a read-through grid too big to enumerate β€” it distinguishes
99
- # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move
100
- # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which
101
- # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]).
102
- # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database
103
- # THE TENANT IS THE UNIT β€” `scoped_pool` itself carries "no per-row owner filter; the
104
- # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside
105
- # `scoped_pids` and answered 404/403. The pid set was only ever an existence check.
106
- return (pids, bool(limits))
107
- from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids
108
-
109
- session.require(CUSTOMER_MODULE)
110
- return (frozenset(allowed_pids(session)), False)
111
-
112
-
113
- def _in_book(pid, session, scope):
114
- pids, unbounded = _pool_or_refuse(session, scope)
115
- if unbounded:
116
- # The table wall passed and the row set is larger than this process will enumerate. Said
117
- # out loud rather than silently admitting: R6's second sentence is that a limit which
118
- # cannot be removed gets REPORTED, and this is the one place the report has no envelope to
119
- # ride in.
120
- print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) β€” "
121
- f"admitting on the table wall alone, per D-72")
122
- return
123
- if pid not in pids:
124
- # 403, not 404: the record may exist, but this session may not inspect it.
125
- raise err(403, "out_of_scope", "that record is not in your book")
126
-
127
-
128
- def _unavailable():
129
- return err(
130
- 503,
131
- "store_unavailable",
132
- "record comments are temporarily unavailable β€” no change was saved",
133
- )
134
-
135
-
136
- # ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments β€” comments hang off a RECORD
137
- # in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the
138
- # wall was always the query param). The old path stays as an ALIAS because the shipped client
139
- # still calls it; verify_api pins the canonical path AND that the alias answers, so removing
140
- # the alias later is a decision, never an accident.
141
- @router.get("/records/{pid}/comments")
142
- @router.get("/customers/{pid}/comments")
143
- def comments(pid: int, scope: str = Query(default="customer"),
144
- session: Session = Depends(require_session)):
145
- from core import record_comments
146
-
147
- scope = _scope_or_400(scope)
148
- _in_book(pid, session, scope)
149
- try:
150
- rows = record_comments.list_comments(session.runtime, pid, scope=scope)
151
- except record_comments.CommentsUnavailable:
152
- raise _unavailable()
153
- return {"comments": rows}
154
-
155
-
156
- @router.post("/records/{pid}/comments", status_code=201)
157
- @router.post("/customers/{pid}/comments", status_code=201)
158
- def create_comment(
159
- pid: int,
160
- body: dict = Body(default=None),
161
- scope: str = Query(default="customer"),
162
- session: Session = Depends(require_session),
163
- ):
164
- from core import record_comments
165
-
166
- scope = _scope_or_400(scope)
167
- _in_book(pid, session, scope)
168
- try:
169
- comment = record_comments.add_comment(
170
- session.runtime,
171
- pid,
172
- (body or {}).get("body"),
173
- session.uname,
174
- session.user.get("name") or session.uname,
175
- scope=scope,
176
- )
177
- except ValueError as exc:
178
- raise err(400, "bad_comment", str(exc))
179
- except record_comments.CommentsUnavailable:
180
- raise _unavailable()
181
- return {"comment": comment}
182
-
183
-
184
- @router.delete("/records/{pid}/comments/{comment_id}")
185
- @router.delete("/customers/{pid}/comments/{comment_id}")
186
- def remove_comment(
187
- pid: int,
188
- comment_id: str,
189
- scope: str = Query(default="customer"),
190
- session: Session = Depends(require_session),
191
- ):
192
- from core import record_comments
193
-
194
- scope = _scope_or_400(scope)
195
- _in_book(pid, session, scope)
196
- try:
197
- deleted = record_comments.delete_comment(
198
- session.runtime,
199
- pid,
200
- comment_id,
201
- session.uname,
202
- admin=session.admin,
203
- scope=scope,
204
- )
205
- except record_comments.CommentForbidden:
206
- raise err(403, "comment_forbidden", "only the author may delete this comment")
207
- except record_comments.CommentsUnavailable:
208
- raise _unavailable()
209
- if not deleted:
210
- raise err(404, "comment_not_found", "that comment no longer exists")
211
- return {"ok": True, "id": comment_id}
 
1
+ """Record detail routes: durable comments, scoped to the caller's book β€” ON EVERY DATABASE.
2
+
3
+ ⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a
4
+ customer-shaped wall bolted to the module import line: `_in_book` asked
5
+ `routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and
6
+ typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the
7
+ panel answered "that customer is not in your book" β€” the owner's report. The dangerous half is
8
+ the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment
9
+ is filed against somebody's customer where the whole team can read it.
10
+
11
+ THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the
12
+ events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall
13
+ per scope β€” the GRANT and the ROW SET β€” by asking that topic's own route, never by re-deriving
14
+ one here:
15
+
16
+ customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant
17
+ product `routes_products.scoped_pool` behind the `product_data` grant
18
+ ut_<slug> `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall
19
+ (404 unknown / 403 not yours β€” a user table has no module grant).
20
+ ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` β€” the pool builds every
21
+ ROW to derive a pid set this module discards, and RAISES 409 on a
22
+ read-through grid past one window, which is why the record drawer painted
23
+ an error page on `ut_odoo_gl_lines`.
24
+
25
+ ⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a
26
+ section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning
27
+ another session's gate mid-wave β€” the WALL is the query parameter, not the word.
28
+
29
+ ⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every
30
+ shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a
31
+ 400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for
32
+ the same reason: a typo served as `customer` answers a question nobody asked).
33
+ """
34
+ from fastapi import APIRouter, Body, Depends, Query
35
+
36
+ from deps import Session, err, require_session
37
+
38
+ router = APIRouter(prefix="/api/v1")
39
+
40
+ #: The customer topic's two names β€” one book, two surfaces (the Cohort page is the customer table
41
+ #: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`.
42
+ _CUSTOMER_SCOPES = ("", "customer", "cohort")
43
+
44
+
45
+ def _scope_or_400(raw):
46
+ scope = str(raw or "customer").strip().lower()
47
+ if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
48
+ return "customer" if scope in _CUSTOMER_SCOPES else scope
49
+ raise err(400, "bad_scope",
50
+ "scope must be customer, cohort, product or a ut_ database β€” refusing to guess")
51
+
52
+
53
+ def _pool_or_refuse(session: Session, scope: str):
54
+ """The pids this session may attach comments to ON THIS DATABASE β€” grant wall included.
55
+
56
+ Returns **`(pids, unbounded)`** β€” a 2-tuple on EVERY branch. Raises the topic's own 403/404/503,
57
+ so a caller who may not open the surface never learns anything about the row they asked about.
58
+
59
+ β›” `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past
60
+ one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns
61
+ `frozenset()` for both β€” see the branch below.
62
+ ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that
63
+ do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py`
64
+ that REPLACES this function with its own lambda. A gate's test double is a caller
65
+ ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a
66
+ bare frozenset and the section died on `ValueError: too many values to unpack` β€” no tally, no
67
+ failing name. Change the shape here and that double changes with it.
68
+ """
69
+ if scope == "product":
70
+ from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
71
+
72
+ session.require(PRODUCT_MODULE)
73
+ pids, _team, _rows, _fields = scoped_pool(session)
74
+ # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes
75
+ # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised
76
+ # a `TypeError` β€” a 500 on every product comment β€” while both other branches worked. A
77
+ # return shape is a contract even when the function is private.
78
+ return (pids, False)
79
+ if scope.startswith("ut_"):
80
+ # No module grant exists for a user table β€” `_defn_or_refuse` inside `scoped_pids` IS
81
+ # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the
82
+ # rows routes do.
83
+ #
84
+ # ⭐⭐ W33-T03 / D-183 β€” `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG.
85
+ # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on
86
+ # a read-through grid larger than one window it RAISES `409 window_required`. So opening the
87
+ # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page β€” for a panel
88
+ # that renders comments about ONE row it already has. `scoped_pids` answers the identical
89
+ # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes
90
+ # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace`
91
+ # used to become openable on those two grids.
92
+ from routes_tables import scoped_pids
93
+
94
+ limits = []
95
+ pids, _fields, _defn = scoped_pids(session, scope, limits=limits)
96
+ # β›” AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is
97
+ # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a
98
+ # database with no rows and for a read-through grid too big to enumerate β€” it distinguishes
99
+ # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move
100
+ # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which
101
+ # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]).
102
+ # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database
103
+ # THE TENANT IS THE UNIT β€” `scoped_pool` itself carries "no per-row owner filter; the
104
+ # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside
105
+ # `scoped_pids` and answered 404/403. The pid set was only ever an existence check.
106
+ return (pids, bool(limits))
107
+ from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids
108
+
109
+ session.require(CUSTOMER_MODULE)
110
+ return (frozenset(allowed_pids(session)), False)
111
+
112
+
113
+ def _in_book(pid, session, scope):
114
+ pids, unbounded = _pool_or_refuse(session, scope)
115
+ if unbounded:
116
+ # The table wall passed and the row set is larger than this process will enumerate. Said
117
+ # out loud rather than silently admitting: R6's second sentence is that a limit which
118
+ # cannot be removed gets REPORTED, and this is the one place the report has no envelope to
119
+ # ride in.
120
+ print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) β€” "
121
+ f"admitting on the table wall alone, per D-72")
122
+ return
123
+ if pid not in pids:
124
+ # 403, not 404: the record may exist, but this session may not inspect it.
125
+ raise err(403, "out_of_scope", "that record is not in your book")
126
+
127
+
128
+ def _unavailable():
129
+ return err(
130
+ 503,
131
+ "store_unavailable",
132
+ "record comments are temporarily unavailable β€” no change was saved",
133
+ )
134
+
135
+
136
+ # ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments β€” comments hang off a RECORD
137
+ # in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the
138
+ # wall was always the query param). The old path stays as an ALIAS because the shipped client
139
+ # still calls it; verify_api pins the canonical path AND that the alias answers, so removing
140
+ # the alias later is a decision, never an accident.
141
+ @router.get("/records/{pid}/comments")
142
+ @router.get("/customers/{pid}/comments")
143
+ def comments(pid: int, scope: str = Query(default="customer"),
144
+ session: Session = Depends(require_session)):
145
+ from core import record_comments
146
+
147
+ scope = _scope_or_400(scope)
148
+ _in_book(pid, session, scope)
149
+ try:
150
+ rows = record_comments.list_comments(session.runtime, pid, scope=scope)
151
+ except record_comments.CommentsUnavailable:
152
+ raise _unavailable()
153
+ return {"comments": rows}
154
+
155
+
156
+ @router.post("/records/{pid}/comments", status_code=201)
157
+ @router.post("/customers/{pid}/comments", status_code=201)
158
+ def create_comment(
159
+ pid: int,
160
+ body: dict = Body(default=None),
161
+ scope: str = Query(default="customer"),
162
+ session: Session = Depends(require_session),
163
+ ):
164
+ from core import record_comments
165
+
166
+ scope = _scope_or_400(scope)
167
+ _in_book(pid, session, scope)
168
+ try:
169
+ comment = record_comments.add_comment(
170
+ session.runtime,
171
+ pid,
172
+ (body or {}).get("body"),
173
+ session.uname,
174
+ session.user.get("name") or session.uname,
175
+ scope=scope,
176
+ )
177
+ except ValueError as exc:
178
+ raise err(400, "bad_comment", str(exc))
179
+ except record_comments.CommentsUnavailable:
180
+ raise _unavailable()
181
+ return {"comment": comment}
182
+
183
+
184
+ @router.delete("/records/{pid}/comments/{comment_id}")
185
+ @router.delete("/customers/{pid}/comments/{comment_id}")
186
+ def remove_comment(
187
+ pid: int,
188
+ comment_id: str,
189
+ scope: str = Query(default="customer"),
190
+ session: Session = Depends(require_session),
191
+ ):
192
+ from core import record_comments
193
+
194
+ scope = _scope_or_400(scope)
195
+ _in_book(pid, session, scope)
196
+ try:
197
+ deleted = record_comments.delete_comment(
198
+ session.runtime,
199
+ pid,
200
+ comment_id,
201
+ session.uname,
202
+ admin=session.admin,
203
+ scope=scope,
204
+ )
205
+ except record_comments.CommentForbidden:
206
+ raise err(403, "comment_forbidden", "only the author may delete this comment")
207
+ except record_comments.CommentsUnavailable:
208
+ raise _unavailable()
209
+ if not deleted:
210
+ raise err(404, "comment_not_found", "that comment no longer exists")
211
+ return {"ok": True, "id": comment_id}
api/routes_script_views.py ADDED
@@ -0,0 +1,329 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """routes_script_views.py β€” CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10).
2
+
3
+ Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user
4
+ can build whatever they want through the Agent chat interface. be able to create any dashboard
5
+ they want. User should have the ability to see the code AND the dashboard output of course… Limit
6
+ the code script View per database… Any agent can add into more AI script, so we can see different
7
+ versions or different things the AI code for us."*
8
+
9
+ GET /api/v1/script-views?database=K the views bound to ONE database
10
+ POST /api/v1/script-views create one {database, name?, source}
11
+ GET /api/v1/script-views/{id} one view, its source and its history
12
+ PUT /api/v1/script-views/{id} a NEW VERSION of the source
13
+ DELETE /api/v1/script-views/{id} drop it
14
+ POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms}
15
+
16
+ ⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may
17
+ read ONLY the database it lives in, and a script that names another database is REFUSED with a
18
+ message naming both. *Not capped*: there is **no limit on how many script views a database may
19
+ carry**, because that is how an agent offers three attempts and the owner picks one. So nothing
20
+ below counts views. What IS bounded is what makes them big β€” one source is capped, and one view's
21
+ edit history is capped and REPORTS what it dropped.
22
+
23
+ β›” **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed
24
+ `session.user`, so a script written by an administrator and opened by a scoped analyst reads the
25
+ ANALYST's rows. The author decides what the code does; the reader decides what it can see.
26
+ ⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script
27
+ that exfiltrates anything, because the only thing a script can return is a render spec drawn on
28
+ the screen of the person who ran it. There is no network, no file and no second reader.
29
+
30
+ β›” **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a
31
+ coroutine that would block the event loop for every other request in the container. FastAPI runs a
32
+ plain `def` in the threadpool, which is what makes one slow script one slow REQUEST.
33
+ """
34
+ import threading
35
+ from datetime import datetime, timezone
36
+
37
+ from fastapi import APIRouter, Body, Depends
38
+
39
+ from deps import Session, err, require_session
40
+
41
+ router = APIRouter(prefix="/api/v1")
42
+
43
+ #: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`.
44
+ VIEWS_KEY = "script_views"
45
+
46
+ MAX_NAME = 80
47
+ MAX_SOURCE_BYTES = 128 * 1024
48
+
49
+ #: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) β€” a cap on how far
50
+ #: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader
51
+ #: can see the history is partial instead of concluding the view was only ever saved twice.
52
+ MAX_HISTORY = 40
53
+
54
+ #: β›” HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal
55
+ #: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this,
56
+ #: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what
57
+ #: gives up. A 429 that says so is honest; an unbounded fork is not.
58
+ MAX_CONCURRENT_RUNS = 4
59
+ _RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS)
60
+
61
+
62
+ def _now():
63
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
64
+
65
+
66
+ def _all(rt):
67
+ """`{id: record}` for one tenant. `{}` on any failure β€” an unreadable bucket must degrade to
68
+ "this database has no script views", never to a 500 on the view rail."""
69
+ try:
70
+ found = rt.get(VIEWS_KEY) or {}
71
+ except Exception: # noqa: BLE001
72
+ return {}
73
+ return found if isinstance(found, dict) else {}
74
+
75
+
76
+ def _database_ok(session, database):
77
+ """Does this database EXIST, and may this caller read it? Answered by C1, never by a list.
78
+
79
+ β›” `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True
80
+ for an ADMIN on any key at all, including one no database answers to. So a create validated
81
+ with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing
82
+ can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it
83
+ RAISES `UnknownTable`, which is exactly the question being asked.
84
+ """
85
+ import core.perm_scope as perm_scope
86
+ try:
87
+ perm_scope.scoped_fields(session.user, database, st=session.runtime)
88
+ except perm_scope.UnknownTable:
89
+ raise err(404, "no_database", f"there is no database '{database}' in this workspace")
90
+ except perm_scope.Denied:
91
+ raise err(403, "forbidden", f"your account may not read '{database}'")
92
+ except perm_scope.Unresolvable as exc:
93
+ # The database is real and cannot be served under this call's constraints. Standing rule
94
+ # 1's second sentence: report the cause and the recommendation, never a bare refusal.
95
+ raise err(409, "unresolvable", str(exc)) from None
96
+
97
+
98
+ def _clean_source(raw):
99
+ source = str(raw or "")
100
+ if not source.strip():
101
+ raise err(400, "no_source", "a script view needs some code")
102
+ if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES:
103
+ raise err(413, "source_too_long",
104
+ f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code")
105
+ return source
106
+
107
+
108
+ def _row(rec, *, source=False):
109
+ """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names."""
110
+ out = {"id": rec.get("id") or "", "database": rec.get("database") or "",
111
+ "name": rec.get("name") or "", "author": rec.get("author") or "",
112
+ "version": int(rec.get("version") or 1),
113
+ "created": rec.get("created") or "", "updated": rec.get("updated") or "",
114
+ "versions": len(rec.get("history") or []) + 1,
115
+ "trimmed": int(rec.get("trimmed") or 0)}
116
+ if source:
117
+ out["source"] = rec.get("source") or ""
118
+ out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "",
119
+ "created": h.get("created") or "",
120
+ "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))}
121
+ for h in reversed(rec.get("history") or []) if isinstance(h, dict)]
122
+ return out
123
+
124
+
125
+ def _limits():
126
+ return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME,
127
+ "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS,
128
+ # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong.
129
+ "maxViewsPerDatabase": None}
130
+
131
+
132
+ def _put(session, view_id, mutate):
133
+ """Read-modify-write ONE view, synchronously β€” the client re-reads the rail immediately."""
134
+ def _set(cur):
135
+ cur = dict(cur or {})
136
+ nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None)
137
+ if nxt is None:
138
+ cur.pop(view_id, None)
139
+ else:
140
+ cur[view_id] = nxt
141
+ return cur
142
+
143
+ session.runtime.update(VIEWS_KEY, _set, flush="sync")
144
+
145
+
146
+ def _mine_or_admin(session, rec):
147
+ """Who may EDIT or DELETE a view: its author, or an administrator.
148
+
149
+ ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider β€” anybody who may read the database
150
+ may run any view on it, under their OWN scope. That is the whole of "so we can see different
151
+ versions or different things the AI code for us": a colleague's attempt is worth nothing if
152
+ only its author can open it.
153
+ """
154
+ if session.admin or str(rec.get("author") or "") == session.uname:
155
+ return
156
+ raise err(403, "forbidden", "only the author or an administrator can change this script view")
157
+
158
+
159
+ # ── the routes ────────────────────────────────────────────────────────────────────────────────
160
+ @router.get("/script-views")
161
+ def list_script_views(database: str = "", session: Session = Depends(require_session)):
162
+ """Every script view bound to ONE database, newest first. `database` is required."""
163
+ key = str(database or "").strip()
164
+ if not key:
165
+ raise err(400, "no_database", "name the database whose script views you want")
166
+ _database_ok(session, key)
167
+ rows = [_row(rec) for rec in _all(session.runtime).values()
168
+ if isinstance(rec, dict) and str(rec.get("database") or "") == key]
169
+ rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True)
170
+ return {"database": key, "views": rows, "limits": _limits()}
171
+
172
+
173
+ @router.post("/script-views")
174
+ def create_script_view(body: dict = Body(default=None),
175
+ session: Session = Depends(require_session)):
176
+ """Create one. β›” THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time.
177
+
178
+ A script view that cannot be run is a broken feature the reader discovers by pressing a button,
179
+ and the agent that wrote it is long gone by then. `check_source` is pure and costs no process,
180
+ so the refusal arrives while the author still has the code in front of them.
181
+ """
182
+ import secrets # noqa: PLC0415
183
+ import core.script_sandbox as sandbox # noqa: PLC0415
184
+
185
+ body = body if isinstance(body, dict) else {}
186
+ database = str(body.get("database") or "").strip()
187
+ if not database:
188
+ raise err(400, "no_database", "a script view is bound to one database")
189
+ _database_ok(session, database)
190
+ source = _clean_source(body.get("source"))
191
+ refusal = sandbox.check_source(source)
192
+ if refusal is not None:
193
+ raise err(400, refusal.code, refusal.message)
194
+
195
+ view_id = "sv_" + secrets.token_urlsafe(9)
196
+ rec = {"id": view_id, "database": database,
197
+ "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME],
198
+ "source": source, "author": session.uname, "version": 1,
199
+ "created": _now(), "updated": _now(), "history": [], "trimmed": 0}
200
+ _put(session, view_id, lambda _prior: rec)
201
+ fresh = _all(session.runtime).get(view_id)
202
+ if not isinstance(fresh, dict):
203
+ # The store took the write and did not record it. A 200 here would tell the author their
204
+ # script was saved when it was not.
205
+ raise err(503, "store_unavailable", "the script view was NOT created")
206
+ return {"view": _row(fresh, source=True), "limits": _limits()}
207
+
208
+
209
+ @router.get("/script-views/{view_id}")
210
+ def get_script_view(view_id: str, session: Session = Depends(require_session)):
211
+ """One view WITH its source and its edit history. Owner item 6's *"see the code"* half."""
212
+ rec = _all(session.runtime).get(str(view_id))
213
+ if not isinstance(rec, dict):
214
+ raise err(404, "no_view", "there is no script view with that id")
215
+ _database_ok(session, str(rec.get("database") or ""))
216
+ return {"view": _row(rec, source=True), "limits": _limits()}
217
+
218
+
219
+ @router.put("/script-views/{view_id}")
220
+ def update_script_view(view_id: str, body: dict = Body(default=None),
221
+ session: Session = Depends(require_session)):
222
+ """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place."""
223
+ import core.script_sandbox as sandbox # noqa: PLC0415
224
+
225
+ view_id = str(view_id)
226
+ rec = _all(session.runtime).get(view_id)
227
+ if not isinstance(rec, dict):
228
+ raise err(404, "no_view", "there is no script view with that id")
229
+ _mine_or_admin(session, rec)
230
+ body = body if isinstance(body, dict) else {}
231
+ source = _clean_source(body.get("source"))
232
+ refusal = sandbox.check_source(source)
233
+ if refusal is not None:
234
+ raise err(400, refusal.code, refusal.message)
235
+
236
+ def _mutate(prior):
237
+ prior = dict(prior or rec)
238
+ history = list(prior.get("history") or [])
239
+ history.append({"version": int(prior.get("version") or 1),
240
+ "source": prior.get("source") or "",
241
+ "author": prior.get("author") or "", "created": prior.get("updated") or ""})
242
+ dropped = max(0, len(history) - MAX_HISTORY)
243
+ prior["history"] = history[dropped:] if dropped else history
244
+ prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped
245
+ prior["source"] = source
246
+ prior["version"] = int(prior.get("version") or 1) + 1
247
+ prior["updated"] = _now()
248
+ if body.get("name"):
249
+ prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME]
250
+ return prior
251
+
252
+ _put(session, view_id, _mutate)
253
+ fresh = _all(session.runtime).get(view_id)
254
+ if not isinstance(fresh, dict):
255
+ raise err(503, "store_unavailable", "the new version was NOT saved")
256
+ return {"view": _row(fresh, source=True), "limits": _limits()}
257
+
258
+
259
+ @router.delete("/script-views/{view_id}")
260
+ def delete_script_view(view_id: str, session: Session = Depends(require_session)):
261
+ view_id = str(view_id)
262
+ rec = _all(session.runtime).get(view_id)
263
+ if not isinstance(rec, dict):
264
+ raise err(404, "no_view", "there is no script view with that id")
265
+ _mine_or_admin(session, rec)
266
+ _put(session, view_id, lambda _prior: None)
267
+ return {"deleted": view_id}
268
+
269
+
270
+ @router.post("/script-views/{view_id}/run")
271
+ def run_script_view(view_id: str, body: dict = Body(default=None),
272
+ session: Session = Depends(require_session)):
273
+ """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`.
274
+
275
+ β›” `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser
276
+ executes β€” the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key
277
+ before this function ever sees it, so a renderer cannot be talked into running something by a
278
+ script that was itself perfectly well behaved.
279
+
280
+ β›” AND IT IS PLAIN `def`, NOT `async def` β€” see this module's header. A ten-second subprocess
281
+ wait on the event loop is a ten-second outage for the whole container.
282
+ """
283
+ import core.script_sandbox as sandbox # noqa: PLC0415
284
+
285
+ rec = _all(session.runtime).get(str(view_id))
286
+ if not isinstance(rec, dict):
287
+ raise err(404, "no_view", "there is no script view with that id")
288
+ database = str(rec.get("database") or "")
289
+ # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it.
290
+ # It is checked exactly as a stored one is, because "unsaved" is not a permission.
291
+ draft = (body or {}).get("source") if isinstance(body, dict) else None
292
+ source = _clean_source(draft) if draft else str(rec.get("source") or "")
293
+ # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and
294
+ # asking twice builds a registry topic's pool twice. The three refusals are translated below
295
+ # instead, which is the same wall reached through the same door.
296
+
297
+ if not _RUN_SLOTS.acquire(blocking=False):
298
+ raise err(429, "busy",
299
+ f"{MAX_CONCURRENT_RUNS} script views are already running on this server. "
300
+ f"Try again in a moment")
301
+ try:
302
+ out = sandbox.run_view(session.user, database, source, st=session.runtime)
303
+ finally:
304
+ _RUN_SLOTS.release()
305
+
306
+ # β›” C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database"
307
+ # answered 200 would be a permission decision the client has to go looking for, and every
308
+ # other door in this app answers 403 for it. Everything BELOW this line is a well-formed
309
+ # request whose ANSWER is that the script did not produce a view β€” that is a 200 with
310
+ # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason.
311
+ if out.get("code") == "unknown_table":
312
+ raise err(404, "no_database", out.get("message") or f"there is no database '{database}'")
313
+ if out.get("code") == "denied":
314
+ raise err(403, "forbidden", out.get("message") or "your account may not read that database")
315
+ if out.get("code") == "unresolvable":
316
+ refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served")
317
+ refusal.detail["error"]["limit"] = out.get("limit") or {}
318
+ raise refusal
319
+
320
+ answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"),
321
+ "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")),
322
+ "ms": int(out.get("ms") or 0), "code": out.get("code") or "",
323
+ # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits
324
+ # were applied; on a Windows host the memory and CPU ones were not, and a screen
325
+ # that claims an enforcement which did not happen is the failure the rule is about.
326
+ "caps": out.get("caps") or {}}
327
+ if not answer["ok"]:
328
+ answer["error"] = out.get("message") or "the script view did not produce a view"
329
+ return answer
api/routes_shares.py CHANGED
@@ -1,393 +1,393 @@
1
- """routes_shares.py β€” the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
2
-
3
- GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
4
- PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
5
- GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
6
-
7
- `kind` ∈ view | folder | database. Roles are `view` | `edit` β€” the same two words the view rail
8
- already speaks, now extended to folders and databases so there is ONE vocabulary in the UI
9
- (R10: "the same picker views use").
10
-
11
- β›” **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
12
- `shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
13
- CONTENT and may not change who else can reach it β€” otherwise anyone you shared a view with could
14
- widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor
15
- for non-administrators; that is a courtesy, and this check is the wall.
16
-
17
- ⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL β€” ON A GOVERNED MODULE.** `*` ("everyone") means
18
- every account that can already open the surface: `require_session` plus the topic's own gate run
19
- first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field
20
- closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that
21
- could already reach the data ([[aios-permissioning]]).
22
-
23
- β›”β›” **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED
24
- MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")`
25
- and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be
26
- DECLARED for a `ut_*` database** β€” `routes_tables.py` makes zero `perm_scope` calls and passes
27
- `hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded
28
- by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING β€” every row, every
29
- column β€” and an `*` database grant admits every account in the tenant to all of it.
30
- That is a real capability, deliberately kept; what was wrong was a docstring promising a second
31
- wall that does not exist for this kind. Scoping user tables is booked, not done
32
- (`waves/wave32/sharing-audit.md` S-8).
33
-
34
- ⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS
35
- registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared`
36
- β€” the view's own `permissions` β€” is what actually decides who may OPEN a view.** A grant here
37
- whose object is invisible under that one is a row in a list that opens a refusal, which is what
38
- made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but
39
- the two vocabularies are still two.
40
- """
41
- from fastapi import APIRouter, Body, Depends
42
-
43
- import core.shares as shares
44
- import core.users as users
45
- from deps import Session, err, require_session
46
- # ⭐ W32-T28 (C3) β€” the SHARE notification's topic word, imported from the module that CLASSIFIES
47
- # it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
48
- # reader agreeing about one string is the whole difference between an Inbox row that opens the
49
- # shared database and one that is quietly unclickable.
50
- from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
51
-
52
- router = APIRouter(prefix="/api/v1")
53
-
54
-
55
- def _kind_or_400(raw):
56
- try:
57
- return shares._check_kind(raw)
58
- except ValueError as e:
59
- raise err(400, "bad_kind", str(e))
60
-
61
-
62
- # ── ⭐⭐ WAVE 32 Β· T26 (owner item 18, ruling R12) β€” THE WALL THIS FILE SAID IT HAD ─────────────
63
- #
64
- # `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all
65
- # means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free
66
- # strings off the URL and the only dependency was `require_session`, so any signed-in account
67
- # could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`,
68
- # an object with no grant record skipped the check entirely and the caller was stamped OWNER β€”
69
- # sticky, so **the real creator was then refused on their own view, permanently.** Driven, not
70
- # argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript.
71
- #
72
- # ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own
73
- # "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the
74
- # victim next opens the dialog.
75
-
76
- #: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share
77
- #: route is not told which topic β€” so resolving one means asking each.
78
- _BUILTIN_TOPICS = ("customer", "product")
79
-
80
-
81
- def _topics(session):
82
- """Every topic whose workspace could hold a view or folder for this tenant.
83
-
84
- ⚠ `all_defs`, never `all_tables` β€” the latter is the whole 28.6 MB row payload (~703 ms on
85
- tenant #0) to answer a question about KEYS (D-185).
86
- """
87
- try:
88
- import core.user_tables as ut
89
- return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {}))
90
- except Exception: # noqa: BLE001
91
- return _BUILTIN_TOPICS
92
-
93
-
94
- def _owns_object(session, kind, oid):
95
- """May this caller CLAIM an object that has no grant record yet β€” i.e. do they own it?
96
-
97
- β›” THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking
98
- each topic's workspace in turn, which is N store reads; making every share call pay that
99
- would put a loop on a route the manage-access dialog opens. The dangerous path is the one
100
- where a caller is about to be stamped OWNER of something nobody owns β€” so the resolution runs
101
- exactly there, and the common path (a record exists, `may_administer` decides) is untouched.
102
- """
103
- if session.admin:
104
- return True
105
- if kind == "database":
106
- # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already
107
- # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
108
- # here would be the second definition this wave keeps finding.
109
- try:
110
- import core.user_tables as ut
111
- return bool(ut.may_open(oid, session.uname, is_admin=session.admin,
112
- st=session.runtime))
113
- except Exception: # noqa: BLE001
114
- return False
115
- try:
116
- import core.table_store as table_store
117
- except Exception: # noqa: BLE001
118
- return False
119
- for topic in _topics(session):
120
- try:
121
- ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
122
- hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
123
- except Exception: # noqa: BLE001
124
- continue
125
- if hit:
126
- # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the
127
- # person whose personal stratum holds it β€” anybody else reaching this line is
128
- # exactly the case S-1 describes.
129
- return str(hit[0]) == str(session.uname)
130
- return False
131
-
132
-
133
- def _can_see_object(session, kind, oid):
134
- """May this caller READ an object's grant list β€” i.e. can they reach the object at all?
135
-
136
- β›”β›” THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A
137
- REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the
138
- ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches
139
- PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with
140
- `permissions.edit = "collaborative"` and no grant record yet β€” a view bob **can open and edit
141
- in the grid** β€” answered `404` when bob opened its manage-access dialog. Measured before
142
- fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said
143
- `404 no_object`.
144
- ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared",
145
- and the wall consulted the grant registry (system A) plus stratum ownership, never the view's
146
- `permissions` (system B) β€” which is the one that actually decides who may OPEN it.
147
- ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not
148
- gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too β€”
149
- 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a
150
- legitimate collaborator their view does not exist.
151
-
152
- β›” THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its
153
- owner β€” that is S-1, and widening this predicate onto `put_share` would re-open it.
154
- """
155
- if _owns_object(session, kind, oid):
156
- return True
157
- if kind != "view":
158
- # A folder carries no per-object visibility flag of its own, and a database's `may_open`
159
- # (inside `_owns_object`) already admits grantees. Nothing wider to ask.
160
- return False
161
- try:
162
- import core.table_store as table_store
163
- for topic in _topics(session):
164
- hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid)
165
- if hit:
166
- return bool(table_store._may_see(hit[1] if len(hit) > 1 else {},
167
- session.uname, is_admin=session.admin))
168
- except Exception: # noqa: BLE001
169
- return False
170
- return False
171
-
172
-
173
- def _entries_or_400(session, entries):
174
- """Validate a grant list against the tenant's REAL, ACTIVE accounts β€” and refuse BY NAME.
175
-
176
- β›” `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly:
177
- a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE
178
- of a string and the role word β€” never that the user EXISTS, is ACTIVE, or is in this tenant**,
179
- so a typo'd name is stored, reported as a successful save, and never reaches anybody. The
180
- sharer believes the person has access. That is item 18's plain reading.
181
- ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker
182
- (`_people`). One route, two populations, and the write door was the permissive one.
183
- ⚠ `*` (everyone) is not a user and is admitted deliberately β€” it is R10's vocabulary for
184
- "every account that can already open the surface".
185
- """
186
- known = {p["username"].strip().lower() for p in _people(session.tenant)}
187
- unknown = []
188
- for e in entries or ():
189
- if not isinstance(e, dict):
190
- continue
191
- user = str(e.get("user") or "").strip().lower()
192
- if user and user != shares.EVERYONE and user not in known:
193
- unknown.append(user)
194
- if unknown:
195
- raise err(400, "unknown_people",
196
- "no active account in this workspace is named "
197
- + ", ".join(sorted(set(unknown)))
198
- + " β€” nothing was shared. Pick people from the list rather than typing a name.")
199
-
200
-
201
- @router.get("/share/mine")
202
- def my_shares(session: Session = Depends(require_session)):
203
- """Everything shared WITH me, by kind β€” the "Shared with me" rail section (R10).
204
-
205
- Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves
206
- in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL
207
- that is not malformed at all.
208
- """
209
- return shares.shared_with(session.uname, st=session.runtime)
210
-
211
-
212
- @router.get("/share/{kind}/{oid}")
213
- def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
214
- kind = _kind_or_400(kind)
215
- rec = shares.grants(kind, oid, st=session.runtime)
216
- role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
217
- may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
218
- st=session.runtime)
219
- # ⭐ W32-T26 (audit S-3) β€” A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
220
- # who owns it, everyone it is granted to, and the tenant's whole username↔name directory β€”
221
- # to any signed-in session, about objects it cannot open. Now a caller with no role on an
222
- # object must prove they can reach it, and gets a 404 otherwise: the same answer a
223
- # non-existent id gives, so the route cannot be used to probe which ids are real.
224
- # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a
225
- # caller who has no relationship with the object at all.
226
- if role is None and not _can_see_object(session, kind, oid):
227
- raise err(404, "no_object", "no such item, or it is not shared with this account")
228
- return {
229
- **rec,
230
- "role": role,
231
- "mayAdminister": may_admin,
232
- # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
233
- # them. `assignable_people` serves bare display names because `user`-kind CELLS store
234
- # display names β€” that list's shape cannot change without migrating cell values β€” so
235
- # this route serves objects of its own. Existing grants that were written as lowercased
236
- # display names are normalised by the wave-21 cleanup script.
237
- # ⭐ W32-T26 (audit S-3) β€” the roster is the EDITOR's data, so it rides only for a caller
238
- # who may open the editor. A read-only grantee gets the grant list (their fair question is
239
- # "who else has this?") and not a directory of every account in the workspace.
240
- "people": _people(session.tenant) if may_admin else [],
241
- }
242
-
243
-
244
- def _people(tenant):
245
- """[{username, name}] for this tenant β€” same population as `assignable_people`, with the
246
- BINDING identity alongside the display one."""
247
- try:
248
- reg = users.registry() or {}
249
- except Exception:
250
- return []
251
- want = str(tenant or '').strip().lower()
252
- out = []
253
- for uname, u in reg.items():
254
- if not isinstance(u, dict) or u.get('active') is False:
255
- continue
256
- if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
257
- continue
258
- out.append({"username": str(uname), "name": str(u.get('name') or uname)})
259
- return sorted(out, key=lambda p: p["name"].lower())
260
-
261
-
262
- @router.put("/share/{kind}/{oid}")
263
- def put_share(kind: str, oid: str, body: dict = Body(default=None),
264
- session: Session = Depends(require_session)):
265
- kind = _kind_or_400(kind)
266
- body = body or {}
267
- rec = shares.grants(kind, oid, st=session.runtime)
268
- # An object with NO grant record yet has no owner β€” the first person to share it claims it.
269
- # That is safe because reaching this route at all means passing the surface's own wall, and
270
- # the alternative (refusing until somebody seeds an owner) would make a brand-new folder
271
- # unshareable by the person who just made it.
272
- if rec["owner"]:
273
- if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
274
- st=session.runtime):
275
- raise err(403, "not_owner",
276
- "only the owner of this item (or an administrator) can change who it is "
277
- "shared with")
278
- # β›”β›” W32-T26 (audit S-1) β€” THE CLAIM NOW HAS A PRECONDITION. An object with no grant record
279
- # is still claimed by the first person to share it β€” that rule is right, and refusing until
280
- # somebody seeds an owner would make a brand-new folder unshareable by the person who just
281
- # made it. What was missing is the half the old comment ASSERTED and the code never did: the
282
- # claimant has to be able to reach the object. Without this, any signed-in account could
283
- # stamp itself owner of an id it had never seen and lock the real creator out for good.
284
- elif not _owns_object(session, kind, oid):
285
- raise err(404, "no_object", "no such item, or it is not shared with this account")
286
- entries = body.get("entries")
287
- if not isinstance(entries, list):
288
- raise err(400, "bad_entries",
289
- "entries must be a list of {user, role} β€” send [] to un-share, which is how "
290
- "revoking is expressed")
291
- _entries_or_400(session, entries)
292
- out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
293
- st=session.runtime)
294
- _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
295
- return out
296
-
297
-
298
- def _notify_new_grantees(session, kind, oid, before, after):
299
- """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) β€” tell the RECEIVER, in their Inbox.
300
-
301
- Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was
302
- silent: the grant landed in a rail section the receiver had to notice on their own, which is
303
- why "I shared it with you" and "I never saw it" were both true.
304
-
305
- β›” WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read
306
- because an alert is a live question about rows; a share is an EVENT that happened once, and
307
- polling for it would mean re-deriving "was this new?" on every inbox open β€” the diff below
308
- only exists here, at the moment the set changes.
309
-
310
- ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence),
311
- so every save re-sends everyone who was already there. Diffing against `before` is what stops
312
- a rename or a role change from ringing the bell for people whose access did not change.
313
- ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in
314
- the tenant on a single click is a broadcast nobody asked for. The rail still shows it.
315
- ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it β€” the
316
- grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`.
317
- """
318
- try:
319
- was = {e.get("user") for e in (before or ()) if isinstance(e, dict)}
320
- fresh = [str(e.get("user")) for e in (after or ())
321
- if isinstance(e, dict) and e.get("user") not in was
322
- and e.get("user") != shares.EVERYONE]
323
- if not fresh:
324
- return
325
- import core.alerts as alerts
326
-
327
- label, route, view_id = _object_ref(session, kind, oid)
328
- if not route:
329
- # β›” NO ROUTE, NO NOTIFICATION β€” the receiver would get a row that opens nothing, and
330
- # `notification_view` would have to invent a target. Silence is the honest answer
331
- # here; the rail still shows the grant under "Shared with me".
332
- return
333
- sharer = str(session.user.get("name") or session.uname)
334
- for user in fresh:
335
- # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must
336
- # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes
337
- # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED
338
- # from there rather than typed again β€” one vocabulary, one owner.
339
- # ⭐⭐ W33-T28 (`ASK C-14`, answered) β€” `actor` IS THE SENDER, AND IT IS THE ONLY WAY
340
- # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and
341
- # are honestly named by their machine; a SHARE has a real person, and only this call
342
- # site knows who. β›” It is passed as its OWN field rather than recovered from the
343
- # `detail` prose below: a sender parsed out of "<name> shared this with you" breaks
344
- # the first time the sentence is reworded, silently, in the header
345
- # [[grep-output-is-not-source]]. The prose stays as the body; this is the From.
346
- alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id,
347
- detail=f"{sharer} shared this with you", actor=sharer,
348
- st=session.runtime)
349
- except Exception: # noqa: BLE001
350
- return
351
-
352
-
353
- def _object_ref(session, kind, oid):
354
- """`(label, route, view_id)` β€” what to CALL the shared thing, and where it OPENS.
355
-
356
- β›” THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT
357
- WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced
358
- `target: {module: "database", id: "view_42"}` β€” an instruction to open a database named
359
- `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not
360
- addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to
361
- travel. Caught by looking at the notification the driver actually produced, not by reading
362
- the code back.
363
-
364
- ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the
365
- receiver nothing they can act on, and the id is already in the target.
366
- ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than
367
- a row that opens nowhere.
368
- """
369
- try:
370
- if kind == "database":
371
- import core.user_tables as ut
372
- defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
373
- # A user table IS its own route key in both vocabularies (`route_for_topic`).
374
- return (str(defn.get("label") or "").strip() or "A database", str(oid), "")
375
- import core.table_store as table_store
376
- from routes_alerts import route_for_topic
377
- for topic in _topics(session):
378
- ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
379
- hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
380
- if not hit:
381
- continue
382
- route = route_for_topic(topic)
383
- if not route:
384
- break
385
- row = hit[1] if len(hit) > 1 else {}
386
- name = str((row or {}).get("name") or "").strip()
387
- # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens
388
- # the grid and stops there rather than naming a view the receiver did not get.
389
- return (name or ("A view" if kind == "view" else "A folder"),
390
- route, str(oid) if kind == "view" else "")
391
- except Exception: # noqa: BLE001
392
- pass
393
- return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "")
 
1
+ """routes_shares.py β€” the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
2
+
3
+ GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
4
+ PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
5
+ GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
6
+
7
+ `kind` ∈ view | folder | database. Roles are `view` | `edit` β€” the same two words the view rail
8
+ already speaks, now extended to folders and databases so there is ONE vocabulary in the UI
9
+ (R10: "the same picker views use").
10
+
11
+ β›” **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
12
+ `shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
13
+ CONTENT and may not change who else can reach it β€” otherwise anyone you shared a view with could
14
+ widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor
15
+ for non-administrators; that is a courtesy, and this check is the wall.
16
+
17
+ ⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL β€” ON A GOVERNED MODULE.** `*` ("everyone") means
18
+ every account that can already open the surface: `require_session` plus the topic's own gate run
19
+ first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field
20
+ closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that
21
+ could already reach the data ([[aios-permissioning]]).
22
+
23
+ β›”β›” **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED
24
+ MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")`
25
+ and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be
26
+ DECLARED for a `ut_*` database** β€” `routes_tables.py` makes zero `perm_scope` calls and passes
27
+ `hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded
28
+ by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING β€” every row, every
29
+ column β€” and an `*` database grant admits every account in the tenant to all of it.
30
+ That is a real capability, deliberately kept; what was wrong was a docstring promising a second
31
+ wall that does not exist for this kind. Scoping user tables is booked, not done
32
+ (`waves/wave32/sharing-audit.md` S-8).
33
+
34
+ ⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS
35
+ registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared`
36
+ β€” the view's own `permissions` β€” is what actually decides who may OPEN a view.** A grant here
37
+ whose object is invisible under that one is a row in a list that opens a refusal, which is what
38
+ made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but
39
+ the two vocabularies are still two.
40
+ """
41
+ from fastapi import APIRouter, Body, Depends
42
+
43
+ import core.shares as shares
44
+ import core.users as users
45
+ from deps import Session, err, require_session
46
+ # ⭐ W32-T28 (C3) β€” the SHARE notification's topic word, imported from the module that CLASSIFIES
47
+ # it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
48
+ # reader agreeing about one string is the whole difference between an Inbox row that opens the
49
+ # shared database and one that is quietly unclickable.
50
+ from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
51
+
52
+ router = APIRouter(prefix="/api/v1")
53
+
54
+
55
+ def _kind_or_400(raw):
56
+ try:
57
+ return shares._check_kind(raw)
58
+ except ValueError as e:
59
+ raise err(400, "bad_kind", str(e))
60
+
61
+
62
+ # ── ⭐⭐ WAVE 32 Β· T26 (owner item 18, ruling R12) β€” THE WALL THIS FILE SAID IT HAD ─────────────
63
+ #
64
+ # `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all
65
+ # means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free
66
+ # strings off the URL and the only dependency was `require_session`, so any signed-in account
67
+ # could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`,
68
+ # an object with no grant record skipped the check entirely and the caller was stamped OWNER β€”
69
+ # sticky, so **the real creator was then refused on their own view, permanently.** Driven, not
70
+ # argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript.
71
+ #
72
+ # ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own
73
+ # "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the
74
+ # victim next opens the dialog.
75
+
76
+ #: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share
77
+ #: route is not told which topic β€” so resolving one means asking each.
78
+ _BUILTIN_TOPICS = ("customer", "product")
79
+
80
+
81
+ def _topics(session):
82
+ """Every topic whose workspace could hold a view or folder for this tenant.
83
+
84
+ ⚠ `all_defs`, never `all_tables` β€” the latter is the whole 28.6 MB row payload (~703 ms on
85
+ tenant #0) to answer a question about KEYS (D-185).
86
+ """
87
+ try:
88
+ import core.user_tables as ut
89
+ return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {}))
90
+ except Exception: # noqa: BLE001
91
+ return _BUILTIN_TOPICS
92
+
93
+
94
+ def _owns_object(session, kind, oid):
95
+ """May this caller CLAIM an object that has no grant record yet β€” i.e. do they own it?
96
+
97
+ β›” THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking
98
+ each topic's workspace in turn, which is N store reads; making every share call pay that
99
+ would put a loop on a route the manage-access dialog opens. The dangerous path is the one
100
+ where a caller is about to be stamped OWNER of something nobody owns β€” so the resolution runs
101
+ exactly there, and the common path (a record exists, `may_administer` decides) is untouched.
102
+ """
103
+ if session.admin:
104
+ return True
105
+ if kind == "database":
106
+ # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already
107
+ # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
108
+ # here would be the second definition this wave keeps finding.
109
+ try:
110
+ import core.user_tables as ut
111
+ return bool(ut.may_open(oid, session.uname, is_admin=session.admin,
112
+ st=session.runtime))
113
+ except Exception: # noqa: BLE001
114
+ return False
115
+ try:
116
+ import core.table_store as table_store
117
+ except Exception: # noqa: BLE001
118
+ return False
119
+ for topic in _topics(session):
120
+ try:
121
+ ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
122
+ hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
123
+ except Exception: # noqa: BLE001
124
+ continue
125
+ if hit:
126
+ # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the
127
+ # person whose personal stratum holds it β€” anybody else reaching this line is
128
+ # exactly the case S-1 describes.
129
+ return str(hit[0]) == str(session.uname)
130
+ return False
131
+
132
+
133
+ def _can_see_object(session, kind, oid):
134
+ """May this caller READ an object's grant list β€” i.e. can they reach the object at all?
135
+
136
+ β›”β›” THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A
137
+ REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the
138
+ ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches
139
+ PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with
140
+ `permissions.edit = "collaborative"` and no grant record yet β€” a view bob **can open and edit
141
+ in the grid** β€” answered `404` when bob opened its manage-access dialog. Measured before
142
+ fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said
143
+ `404 no_object`.
144
+ ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared",
145
+ and the wall consulted the grant registry (system A) plus stratum ownership, never the view's
146
+ `permissions` (system B) β€” which is the one that actually decides who may OPEN it.
147
+ ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not
148
+ gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too β€”
149
+ 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a
150
+ legitimate collaborator their view does not exist.
151
+
152
+ β›” THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its
153
+ owner β€” that is S-1, and widening this predicate onto `put_share` would re-open it.
154
+ """
155
+ if _owns_object(session, kind, oid):
156
+ return True
157
+ if kind != "view":
158
+ # A folder carries no per-object visibility flag of its own, and a database's `may_open`
159
+ # (inside `_owns_object`) already admits grantees. Nothing wider to ask.
160
+ return False
161
+ try:
162
+ import core.table_store as table_store
163
+ for topic in _topics(session):
164
+ hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid)
165
+ if hit:
166
+ return bool(table_store._may_see(hit[1] if len(hit) > 1 else {},
167
+ session.uname, is_admin=session.admin))
168
+ except Exception: # noqa: BLE001
169
+ return False
170
+ return False
171
+
172
+
173
+ def _entries_or_400(session, entries):
174
+ """Validate a grant list against the tenant's REAL, ACTIVE accounts β€” and refuse BY NAME.
175
+
176
+ β›” `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly:
177
+ a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE
178
+ of a string and the role word β€” never that the user EXISTS, is ACTIVE, or is in this tenant**,
179
+ so a typo'd name is stored, reported as a successful save, and never reaches anybody. The
180
+ sharer believes the person has access. That is item 18's plain reading.
181
+ ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker
182
+ (`_people`). One route, two populations, and the write door was the permissive one.
183
+ ⚠ `*` (everyone) is not a user and is admitted deliberately β€” it is R10's vocabulary for
184
+ "every account that can already open the surface".
185
+ """
186
+ known = {p["username"].strip().lower() for p in _people(session.tenant)}
187
+ unknown = []
188
+ for e in entries or ():
189
+ if not isinstance(e, dict):
190
+ continue
191
+ user = str(e.get("user") or "").strip().lower()
192
+ if user and user != shares.EVERYONE and user not in known:
193
+ unknown.append(user)
194
+ if unknown:
195
+ raise err(400, "unknown_people",
196
+ "no active account in this workspace is named "
197
+ + ", ".join(sorted(set(unknown)))
198
+ + " β€” nothing was shared. Pick people from the list rather than typing a name.")
199
+
200
+
201
+ @router.get("/share/mine")
202
+ def my_shares(session: Session = Depends(require_session)):
203
+ """Everything shared WITH me, by kind β€” the "Shared with me" rail section (R10).
204
+
205
+ Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves
206
+ in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL
207
+ that is not malformed at all.
208
+ """
209
+ return shares.shared_with(session.uname, st=session.runtime)
210
+
211
+
212
+ @router.get("/share/{kind}/{oid}")
213
+ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
214
+ kind = _kind_or_400(kind)
215
+ rec = shares.grants(kind, oid, st=session.runtime)
216
+ role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
217
+ may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
218
+ st=session.runtime)
219
+ # ⭐ W32-T26 (audit S-3) β€” A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
220
+ # who owns it, everyone it is granted to, and the tenant's whole username↔name directory β€”
221
+ # to any signed-in session, about objects it cannot open. Now a caller with no role on an
222
+ # object must prove they can reach it, and gets a 404 otherwise: the same answer a
223
+ # non-existent id gives, so the route cannot be used to probe which ids are real.
224
+ # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a
225
+ # caller who has no relationship with the object at all.
226
+ if role is None and not _can_see_object(session, kind, oid):
227
+ raise err(404, "no_object", "no such item, or it is not shared with this account")
228
+ return {
229
+ **rec,
230
+ "role": role,
231
+ "mayAdminister": may_admin,
232
+ # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
233
+ # them. `assignable_people` serves bare display names because `user`-kind CELLS store
234
+ # display names β€” that list's shape cannot change without migrating cell values β€” so
235
+ # this route serves objects of its own. Existing grants that were written as lowercased
236
+ # display names are normalised by the wave-21 cleanup script.
237
+ # ⭐ W32-T26 (audit S-3) β€” the roster is the EDITOR's data, so it rides only for a caller
238
+ # who may open the editor. A read-only grantee gets the grant list (their fair question is
239
+ # "who else has this?") and not a directory of every account in the workspace.
240
+ "people": _people(session.tenant) if may_admin else [],
241
+ }
242
+
243
+
244
+ def _people(tenant):
245
+ """[{username, name}] for this tenant β€” same population as `assignable_people`, with the
246
+ BINDING identity alongside the display one."""
247
+ try:
248
+ reg = users.registry() or {}
249
+ except Exception:
250
+ return []
251
+ want = str(tenant or '').strip().lower()
252
+ out = []
253
+ for uname, u in reg.items():
254
+ if not isinstance(u, dict) or u.get('active') is False:
255
+ continue
256
+ if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
257
+ continue
258
+ out.append({"username": str(uname), "name": str(u.get('name') or uname)})
259
+ return sorted(out, key=lambda p: p["name"].lower())
260
+
261
+
262
+ @router.put("/share/{kind}/{oid}")
263
+ def put_share(kind: str, oid: str, body: dict = Body(default=None),
264
+ session: Session = Depends(require_session)):
265
+ kind = _kind_or_400(kind)
266
+ body = body or {}
267
+ rec = shares.grants(kind, oid, st=session.runtime)
268
+ # An object with NO grant record yet has no owner β€” the first person to share it claims it.
269
+ # That is safe because reaching this route at all means passing the surface's own wall, and
270
+ # the alternative (refusing until somebody seeds an owner) would make a brand-new folder
271
+ # unshareable by the person who just made it.
272
+ if rec["owner"]:
273
+ if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
274
+ st=session.runtime):
275
+ raise err(403, "not_owner",
276
+ "only the owner of this item (or an administrator) can change who it is "
277
+ "shared with")
278
+ # β›”β›” W32-T26 (audit S-1) β€” THE CLAIM NOW HAS A PRECONDITION. An object with no grant record
279
+ # is still claimed by the first person to share it β€” that rule is right, and refusing until
280
+ # somebody seeds an owner would make a brand-new folder unshareable by the person who just
281
+ # made it. What was missing is the half the old comment ASSERTED and the code never did: the
282
+ # claimant has to be able to reach the object. Without this, any signed-in account could
283
+ # stamp itself owner of an id it had never seen and lock the real creator out for good.
284
+ elif not _owns_object(session, kind, oid):
285
+ raise err(404, "no_object", "no such item, or it is not shared with this account")
286
+ entries = body.get("entries")
287
+ if not isinstance(entries, list):
288
+ raise err(400, "bad_entries",
289
+ "entries must be a list of {user, role} β€” send [] to un-share, which is how "
290
+ "revoking is expressed")
291
+ _entries_or_400(session, entries)
292
+ out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
293
+ st=session.runtime)
294
+ _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
295
+ return out
296
+
297
+
298
+ def _notify_new_grantees(session, kind, oid, before, after):
299
+ """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) �� tell the RECEIVER, in their Inbox.
300
+
301
+ Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was
302
+ silent: the grant landed in a rail section the receiver had to notice on their own, which is
303
+ why "I shared it with you" and "I never saw it" were both true.
304
+
305
+ β›” WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read
306
+ because an alert is a live question about rows; a share is an EVENT that happened once, and
307
+ polling for it would mean re-deriving "was this new?" on every inbox open β€” the diff below
308
+ only exists here, at the moment the set changes.
309
+
310
+ ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence),
311
+ so every save re-sends everyone who was already there. Diffing against `before` is what stops
312
+ a rename or a role change from ringing the bell for people whose access did not change.
313
+ ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in
314
+ the tenant on a single click is a broadcast nobody asked for. The rail still shows it.
315
+ ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it β€” the
316
+ grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`.
317
+ """
318
+ try:
319
+ was = {e.get("user") for e in (before or ()) if isinstance(e, dict)}
320
+ fresh = [str(e.get("user")) for e in (after or ())
321
+ if isinstance(e, dict) and e.get("user") not in was
322
+ and e.get("user") != shares.EVERYONE]
323
+ if not fresh:
324
+ return
325
+ import core.alerts as alerts
326
+
327
+ label, route, view_id = _object_ref(session, kind, oid)
328
+ if not route:
329
+ # β›” NO ROUTE, NO NOTIFICATION β€” the receiver would get a row that opens nothing, and
330
+ # `notification_view` would have to invent a target. Silence is the honest answer
331
+ # here; the rail still shows the grant under "Shared with me".
332
+ return
333
+ sharer = str(session.user.get("name") or session.uname)
334
+ for user in fresh:
335
+ # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must
336
+ # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes
337
+ # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED
338
+ # from there rather than typed again β€” one vocabulary, one owner.
339
+ # ⭐⭐ W33-T28 (`ASK C-14`, answered) β€” `actor` IS THE SENDER, AND IT IS THE ONLY WAY
340
+ # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and
341
+ # are honestly named by their machine; a SHARE has a real person, and only this call
342
+ # site knows who. β›” It is passed as its OWN field rather than recovered from the
343
+ # `detail` prose below: a sender parsed out of "<name> shared this with you" breaks
344
+ # the first time the sentence is reworded, silently, in the header
345
+ # [[grep-output-is-not-source]]. The prose stays as the body; this is the From.
346
+ alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id,
347
+ detail=f"{sharer} shared this with you", actor=sharer,
348
+ st=session.runtime)
349
+ except Exception: # noqa: BLE001
350
+ return
351
+
352
+
353
+ def _object_ref(session, kind, oid):
354
+ """`(label, route, view_id)` β€” what to CALL the shared thing, and where it OPENS.
355
+
356
+ β›” THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT
357
+ WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced
358
+ `target: {module: "database", id: "view_42"}` β€” an instruction to open a database named
359
+ `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not
360
+ addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to
361
+ travel. Caught by looking at the notification the driver actually produced, not by reading
362
+ the code back.
363
+
364
+ ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the
365
+ receiver nothing they can act on, and the id is already in the target.
366
+ ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than
367
+ a row that opens nowhere.
368
+ """
369
+ try:
370
+ if kind == "database":
371
+ import core.user_tables as ut
372
+ defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
373
+ # A user table IS its own route key in both vocabularies (`route_for_topic`).
374
+ return (str(defn.get("label") or "").strip() or "A database", str(oid), "")
375
+ import core.table_store as table_store
376
+ from routes_alerts import route_for_topic
377
+ for topic in _topics(session):
378
+ ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
379
+ hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
380
+ if not hit:
381
+ continue
382
+ route = route_for_topic(topic)
383
+ if not route:
384
+ break
385
+ row = hit[1] if len(hit) > 1 else {}
386
+ name = str((row or {}).get("name") or "").strip()
387
+ # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens
388
+ # the grid and stops there rather than naming a view the receiver did not get.
389
+ return (name or ("A view" if kind == "view" else "A folder"),
390
+ route, str(oid) if kind == "view" else "")
391
+ except Exception: # noqa: BLE001
392
+ pass
393
+ return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "")
api/routes_slack.py CHANGED
@@ -229,7 +229,10 @@ def slack_creds(rt):
229
  def _summary(agent, modules):
230
  """One sentence per agent for the list row β€” computed here, because the alternative is one
231
  round trip per row to fill one cell (the same reason `AdminUser.access` exists)."""
232
- governed = [m for m in modules if m.get("enforced")]
 
 
 
233
  perms = agent.get("perms") or {}
234
  open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access"))
235
  if not governed:
@@ -304,16 +307,26 @@ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)):
304
  if not isinstance(a, dict):
305
  raise err(404, "no_such_agent", "no agent with that id")
306
  modules = routes_admin._perm_modules(session)
307
- enforced = [m["key"] for m in modules if m.get("enforced")]
 
 
 
 
 
308
  stored = a.get("perms") or {}
309
  principal = agent_principal(a)
310
  import core.perm_scope as perm_scope
311
  perms_out = {}
312
- for k in enforced:
313
  e = stored.get(k)
 
 
 
 
 
314
  perms_out[k] = e if isinstance(e, dict) else {
315
- "access": bool(perm_scope.may_access(principal, k)), "filter": None,
316
- "hiddenFields": []}
317
  return {"id": str(agent_id), "channel": a.get("channel") or "",
318
  "channelName": a.get("channelName") or "", "label": a.get("label") or "",
319
  "active": a.get("active", True) is not False,
@@ -326,7 +339,8 @@ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)):
326
  # the "Everything (admin)" state for a principal that cannot have it.
327
  "is_admin": False,
328
  "modules": modules,
329
- "fields_by_module": {k: routes_admin._module_fields(k) for k in enforced}}
 
330
 
331
 
332
  @router.put("/agents/{agent_id}/perms")
@@ -341,14 +355,16 @@ def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None),
341
  raise err(400, "empty_patch", "no perms to save")
342
  mods = routes_admin._perm_modules(session)
343
  # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot
344
- # evaluate, refuses a hiddenFields key that names nothing, refuses an unenforced `ut_*` key,
345
- # and refuses a BU condition the pushdown cannot read. Every one of those is exactly as true
346
- # for a channel as for a person, and a second validator here is a second place for one of
347
- # them to go missing.
 
 
 
348
  cleaned = routes_admin._clean_perms(
349
  body.get("perms"),
350
- enforced_keys={m["key"] for m in mods if m.get("enforced")},
351
- listed_keys={m["key"] for m in mods}) or {}
352
 
353
  import core.perm_scope as perm_scope
354
 
 
229
  def _summary(agent, modules):
230
  """One sentence per agent for the list row β€” computed here, because the alternative is one
231
  round trip per row to fill one cell (the same reason `AdminUser.access` exists)."""
232
+ # ⭐ W36-T22 / C2 β€” every listed database is governed now; the `enforced` flag it used to
233
+ # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and
234
+ # summarised every agent as "" (no databases at all).
235
+ governed = list(modules)
236
  perms = agent.get("perms") or {}
237
  open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access"))
238
  if not governed:
 
307
  if not isinstance(a, dict):
308
  raise err(404, "no_such_agent", "no agent with that id")
309
  modules = routes_admin._perm_modules(session)
310
+ # ⭐⭐ W36-T22 / C2 β€” EVERY listed database, `ut_*` included. β›” THIS IS WHY THE FLAG'S
311
+ # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the
312
+ # key is None, so this list would have been EMPTY and the channel perms editor would have
313
+ # governed ZERO databases while looking entirely correct. A Slack channel is a principal of
314
+ # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue.
315
+ governed = [m["key"] for m in modules]
316
  stored = a.get("perms") or {}
317
  principal = agent_principal(a)
318
  import core.perm_scope as perm_scope
319
  perms_out = {}
320
+ for k in governed:
321
  e = stored.get(k)
322
+ # β›” W36-T22 β€” `may_read`, the same evaluator the read door uses, for the reason spelled
323
+ # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a
324
+ # channel agent is not a person with a share, so a `ut_*` key it has not been granted
325
+ # defaults CLOSED β€” which is the fail-closed direction for a bot, and the same answer the
326
+ # table routes would give it.
327
  perms_out[k] = e if isinstance(e, dict) else {
328
+ "access": bool(perm_scope.may_read(principal, k, st=session.runtime)),
329
+ "filter": None, "hiddenFields": []}
330
  return {"id": str(agent_id), "channel": a.get("channel") or "",
331
  "channelName": a.get("channelName") or "", "label": a.get("label") or "",
332
  "active": a.get("active", True) is not False,
 
339
  # the "Everything (admin)" state for a principal that cannot have it.
340
  "is_admin": False,
341
  "modules": modules,
342
+ "fields_by_module": {k: routes_admin._module_fields(k, session=session)
343
+ for k in governed}}
344
 
345
 
346
  @router.put("/agents/{agent_id}/perms")
 
355
  raise err(400, "empty_patch", "no perms to save")
356
  mods = routes_admin._perm_modules(session)
357
  # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot
358
+ # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the
359
+ # pushdown cannot read. Every one of those is exactly as true for a channel as for a person,
360
+ # and a second validator here is a second place for one of them to go missing.
361
+ # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal
362
+ # is DELETED β€” W36-T21 armed the wall over every database, so there is no unenforced key left
363
+ # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives
364
+ # it ([[two-gates-can-assert-opposite-things]]).
365
  cleaned = routes_admin._clean_perms(
366
  body.get("perms"),
367
+ governed_keys={m["key"] for m in mods}, session=session) or {}
 
368
 
369
  import core.perm_scope as perm_scope
370
 
api/routes_statements.py CHANGED
@@ -129,9 +129,14 @@ def preview(body: dict = Body(default=None), session: Session = Depends(_gate)):
129
  subject = (t.get("subject") or cs.DEFAULT_SUBJECT)
130
  try:
131
  subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month)
132
- except (KeyError, IndexError):
133
  # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they
134
  # can see what they typed rather than getting an opaque error.
 
 
 
 
 
135
  pass
136
  return {
137
  "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO,
@@ -153,7 +158,31 @@ def send(body: dict = Body(default=None), session: Session = Depends(_gate)):
153
  names = [str(n) for n in (body.get("customers") or []) if str(n).strip()]
154
  if not names:
155
  raise err(400, "bad_request", "name at least one customer")
156
- override = (body.get("overrideTo") or "").strip() or None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  if override and len(names) != 1:
158
  raise err(400, "bad_request", "a test send takes exactly one customer")
159
  t = body.get("templates") or {}
 
129
  subject = (t.get("subject") or cs.DEFAULT_SUBJECT)
130
  try:
131
  subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month)
132
+ except Exception: # noqa: BLE001
133
  # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they
134
  # can see what they typed rather than getting an opaque error.
135
+ # β›” W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE.
136
+ # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a
137
+ # TypeError, and every one of them is the same user mistake this branch was written for.
138
+ # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the
139
+ # one screen somebody opens to avoid mailing 200 people the wrong thing.
140
  pass
141
  return {
142
  "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO,
 
158
  names = [str(n) for n in (body.get("customers") or []) if str(n).strip()]
159
  if not names:
160
  raise err(400, "bad_request", "name at least one customer")
161
+ # β›”β›” W36-T42 / D-299 β€” A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A
162
+ # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent
163
+ # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space)
164
+ # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On
165
+ # the one route in this product that is allowed to write to Odoo, the difference between a
166
+ # rehearsal and mailing a live customer was one empty string.
167
+ #
168
+ # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo`
169
+ # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature
170
+ # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key
171
+ # being present, or `test: true`, is that statement, and an empty address beside it is the
172
+ # mistake. So both spellings of "absent" are covered: the key present and blank, and `test`
173
+ # asserted with no key at all.
174
+ #
175
+ # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND.
176
+ # `automationApi.testSendStatement` refuses an empty address before the request is built, and
177
+ # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client
178
+ # wall is bypassable by construction (anything can POST); this one is not.
179
+ declared_test = ("overrideTo" in body) or bool(body.get("test"))
180
+ override = str(body.get("overrideTo") or "").strip()
181
+ if declared_test and not override:
182
+ raise err(400, "no_override_address",
183
+ "a test send needs the address to send the test to. Without one this would "
184
+ "mail the customer's own address, which is the opposite of a test")
185
+ override = override or None
186
  if override and len(names) != 1:
187
  raise err(400, "bad_request", "a test send takes exactly one customer")
188
  t = body.get("templates") or {}
api/routes_tables.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_web_agent.py CHANGED
@@ -22,10 +22,11 @@ admin-gated, because it spends money and because D-51 Β§5's authorisation postur
22
  the tenant is authorised to use, at their instruction") is not something an ordinary member
23
  should be able to commit the tenant to.
24
 
25
- β›” THIS ROUTER IS NOT MOUNTED YET. `main.py` belongs to another lane this wave, so the one
26
- `app.include_router(routes_web_agent.router)` line is a cross-fence ask β€” and
27
- `verify_web_agent.py` FAILS until it lands, deliberately: three finished routers once shipped
28
- 404-dead behind entirely green gates, and a gate that tolerates it is how that happens twice.
 
29
  """
30
  from fastapi import APIRouter, Body, Depends
31
 
 
22
  the tenant is authorised to use, at their instruction") is not something an ordinary member
23
  should be able to commit the tenant to.
24
 
25
+ ⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and
26
+ `main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting
27
+ either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which
28
+ is the same class of stale claim as a green gate on a router nobody wired: three finished routers
29
+ once shipped 404-dead behind entirely green gates, and prose is not the control that stops it.
30
  """
31
  from fastapi import APIRouter, Body, Depends
32
 
platform/aios_grid.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/core/perm_scope.py CHANGED
@@ -122,6 +122,19 @@ def hidden_keys(user, module, fields):
122
  ⚠ This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a
123
  graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate
124
  anyway; without it a self-referential pair would spin here.
 
 
 
 
 
 
 
 
 
 
 
 
 
125
  """
126
  e = entry(user, module)
127
  if perms.is_admin(user) or not e:
@@ -137,6 +150,9 @@ def hidden_keys(user, module, fields):
137
  expr = f.get('formula')
138
  if isinstance(expr, str) and expr:
139
  refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()}
 
 
 
140
 
141
  MAX_PASSES = 12
142
  for _ in range(MAX_PASSES):
@@ -452,3 +468,336 @@ def _group_dba_team(group):
452
  if vals <= allowed:
453
  return tid
454
  return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  ⚠ This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a
123
  graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate
124
  anyway; without it a self-referential pair would spin here.
125
+
126
+ ⭐⭐ W36-T21 β€” AND A ROLLUP IS THE SAME LEAK ONE MECHANISM OVER, which matters now that this
127
+ closure runs on the `ut_*` databases rather than only on the two registry topics. A rollup
128
+ names a LINK COLUMN OF THIS TABLE (`rollup.link`) and aggregates a field on the table that
129
+ link points at β€” so `ut_odoo_customers.ar_outstanding` is *"sum `residual` over the invoices
130
+ this row links to"*. Hide `invoices` and keep `ar_outstanding` and the reader still learns
131
+ what the hidden link contains, in aggregate; the three outcomes are exactly the three the
132
+ formula argument above enumerates, and only "strip both" is coherent. Verified against the
133
+ real declarations (`odoo_relational.customer_fields`) rather than assumed: every rollup there
134
+ is either `{'link': <a link column on THIS table>, 'field': <a column on the TARGET table>}`
135
+ or a `source` topic aggregate, so `rollup.link` is the ONE same-table reference a rollup makes
136
+ and `rollup.field` is deliberately not treated as one β€” it names another database's column,
137
+ which has its own wall.
138
  """
139
  e = entry(user, module)
140
  if perms.is_admin(user) or not e:
 
150
  expr = f.get('formula')
151
  if isinstance(expr, str) and expr:
152
  refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()}
153
+ link = (f.get('rollup') or {}).get('link') if isinstance(f.get('rollup'), dict) else None
154
+ if isinstance(link, str) and link.strip():
155
+ refs.setdefault(f['key'], set()).add(link.strip())
156
 
157
  MAX_PASSES = 12
158
  for _ in range(MAX_PASSES):
 
468
  if vals <= allowed:
469
  return tid
470
  return None
471
+
472
+
473
+ # ── C1: THE ONE DOOR TO ANY DATABASE'S ROWS (wave 36, W36-T20) ────────────────────────────────
474
+ #: ⭐⭐ OWNER RULING R6, AND IT IS WHY THIS SECTION EXISTS AT ALL: *"EVERY database gets the same
475
+ #: permission logic, always"* β€” per-user field visibility AND row filtration on every database
476
+ #: carrying a unique id, whatever created it, with a NEW database inheriting it by construction
477
+ #: rather than by a list somebody maintains.
478
+ #:
479
+ #: β›” THE PRODUCT HAD TWO PERMISSION SYSTEMS AND ONLY ONE WAS ARMED. Everything above this line
480
+ #: walls the REGISTRY topics (`customer_data`, `product_data`) and is called only from the topic
481
+ #: assemblies. Every OTHER database is a `ut_*` table walled by `user_tables.may_open` alone β€”
482
+ #: creator, admin, or a `core.shares` grant β€” which is a BINARY door: you see all 31,418 rows of
483
+ #: `ut_odoo_invoices` or none of them. `perms.tenant_governable_modules`' docstring booked this
484
+ #: work in as many words (*"Arming `perm_scope` over `ut_*` … booked, not faked"*), and owner
485
+ #: item 11 is that booking coming due.
486
+ #:
487
+ #: ⚠ AND THE PREMISE THE GRILL GOT WRONG, because the fix depends on it: those databases are NOT
488
+ #: user-created. Ten of them (`ut_odoo_invoices`, `…_orders`, `…_agents`, `…_accounts`, `…_bills`,
489
+ #: `…_vendors`, `…_order_lines`, `…_gl_lines`, `…_customers`, `…_products`) are generated by the
490
+ #: KEYCHAIN connector (`aios-web/api/odoo_relational.py`). **`ut_` is a storage prefix, not a
491
+ #: statement about origin**, and a wall keyed off it was reading a naming artefact as a security
492
+ #: boundary.
493
+ #:
494
+ #: β›”β›” THE TWO QUESTIONS STAY TWO QUESTIONS. `may_open` answers *"IF you see this database"* and
495
+ #: is untouched by this section; C1 answers *"WHICH rows and fields"*. `may_read` below COMPOSES
496
+ #: them β€” it calls `may_open`, it does not reimplement it β€” because merging them is how this
497
+ #: codebase got two ideas of who owns a table once already (`user_tables.may_open`'s own wave-20
498
+ #: note). One resolver per question, asked in order.
499
+
500
+
501
+ class UnknownTable(LookupError):
502
+ """No database in this tenant answers to that key.
503
+
504
+ β›” RAISED, NEVER RETURNED AS AN EMPTY LIST (contract C1). An empty list reads as *"this
505
+ database is empty"* β€” indistinguishable from a real empty table, and the caller least able to
506
+ notice is the one that wanted rows. This repo has shipped that exact silent-empty answer
507
+ before (`user_tables.all_defs`' own correction note; [[empty-answer-vs-unfinished-answer]]).
508
+ """
509
+
510
+
511
+ class Denied(PermissionError):
512
+ """This principal may not read this database at all. The IF question, answered by `may_read`."""
513
+
514
+
515
+ class Unresolvable(RuntimeError):
516
+ """The rows exist and cannot be served under this call's constraints β€” R6's SECOND SENTENCE.
517
+
518
+ ⭐ STANDING RULE 1 IS TWO SENTENCES AND THE SECOND IS THE HALF THAT GETS DROPPED: *"if there
519
+ is lag or it can't be done, you need to explicitly tell me why and recommend a fix"*. So a
520
+ limit that genuinely cannot be removed is REPORTED with its cause and a recommendation, never
521
+ silently enforced as a short answer. Carries the same four keys
522
+ `routes_tables._PID_SCOPE_LIMIT` already puts on the wire, so a route can hand this straight
523
+ to a client without a second vocabulary ([[one-question-two-normalizers]]).
524
+ """
525
+
526
+ def __init__(self, subject, effect, cause, recommendation):
527
+ self.subject, self.effect = subject, effect
528
+ self.cause, self.recommendation = cause, recommendation
529
+ super().__init__(f"{subject}: {effect}. {cause}. {recommendation}")
530
+
531
+ def as_limit(self):
532
+ """The dict shape `routes_tables` puts in an assembly's `limits` list."""
533
+ return {"subject": self.subject, "effect": self.effect,
534
+ "cause": self.cause, "recommendation": self.recommendation}
535
+
536
+
537
+ #: Row readers DECLARED by the app layer, keyed by EXACT database key.
538
+ #: `reader(table_key, user, st) -> (fields, rows)`.
539
+ #:
540
+ #: β›” WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and
541
+ #: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache
542
+ #: (`routes_customers._pool_for`), which is two layers above this file. Same idiom `user_tables`
543
+ #: already uses for exactly this reason β€” `register_connected`, `register_read_through`,
544
+ #: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being
545
+ #: interrogated by it."*
546
+ _ROW_SOURCES = {}
547
+
548
+ #: THE reader for a read-through `ut_*` grid β€” one reader, because there is one mirror.
549
+ #: `reader(table_key, field_keys, st) -> rows`.
550
+ _MIRROR_READER = None
551
+
552
+
553
+ def register_rows(reader, *table_keys):
554
+ """Declare who reads a NAMED database's rows. Returns the registered key set.
555
+
556
+ ⚠ The return value is the registrar's own answer on purpose: a public function whose only
557
+ caller is a `verify_*.py` file is a feature no user can reach, and this repo has a gate that
558
+ says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write
559
+ door's return keeps one construction site of the set instead of two.
560
+ """
561
+ for key in table_keys:
562
+ k = str(key or '').strip()
563
+ if k:
564
+ _ROW_SOURCES[k] = reader
565
+ return frozenset(_ROW_SOURCES)
566
+
567
+
568
+ def register_mirror(reader):
569
+ """Declare THE reader for read-through `ut_*` grids (`routes_tables._read_through_rows`)."""
570
+ global _MIRROR_READER
571
+ _MIRROR_READER = reader
572
+ return _MIRROR_READER is not None
573
+
574
+
575
+ #: β›” `row_sources()` IS DELETED (W36-T24 / owner item 13), AND THE REASON IS THE ONE THIS WAVE
576
+ #: KEEPS FINDING. It returned `frozenset(_ROW_SOURCES)` under a docstring calling itself *"The ONE
577
+ #: list to read"* β€” and `register_rows` ALREADY returns exactly that, which is the same idiom
578
+ #: `user_tables.register_connected_prefix` uses and the same reason: routing the read door through
579
+ #: the write door's return keeps ONE construction site of the set. A second accessor beside it is a
580
+ #: parallel path with nothing of its own to say, and it shipped with no caller outside `verify_*.py`
581
+ #: β€” the shape that is whole, correct and unreachable ([[artifact-with-no-importer]]; reported by
582
+ #: the integrator's `web_reachability` pass, `mailbox/A.md` A-43). The registrar's return is the
583
+ #: read: `routes_grid._C1_ROW_SOURCES` is that value, held where it is registered.
584
+
585
+
586
+ def _ut():
587
+ import core.user_tables as user_tables
588
+ return user_tables
589
+
590
+
591
+ def may_read(user, table_key, st=None):
592
+ """May this principal read `table_key` AT ALL β€” the IF question, on EVERY database.
593
+
594
+ β›” COMPOSED, NOT RE-DERIVED, and the order is the whole rule:
595
+
596
+ 1. an admin reads everything (break-glass β€” `deps._user_for` hands back a hardcoded master
597
+ dict on a store outage and it will never carry a `perms` block);
598
+ 2. an EXPLICIT stored `access: false` DENIES, on any database. This is the toggle owner
599
+ item 11 asks for, and it is a **deny-only overlay**: it may revoke a database the wall
600
+ below would admit, and it may never grant one that wall refuses;
601
+ 3. a `ut_*` database defers to `user_tables.may_open` β€” creator, admin, or a `core.shares`
602
+ grant β€” UNMODIFIED. W36-T21: *"`may_open` still decides IF the database is visible."*
603
+ 4. anything else is a registry topic and defers to `may_access` above.
604
+
605
+ β›”β›” WHY ABSENCE MUST NOT DENY ON A `ut_*` KEY, which is the opposite of what leg 4 does.
606
+ `may_access` reads migrated-and-undeclared as DENY β€” correct for a topic, because
607
+ `routes_admin` writes an entry for every governable topic on every save. ⚠ NO `ut_*` ENTRY WAS
608
+ STORABLE AT ALL UNTIL W36-T22 β€” `_clean_perms` refused the key with `unenforced_module` β€” so
609
+ every record migrated before this wave carries no entry for any of them, and reading that
610
+ absence as a decision would revoke all ten keychain databases from every migrated account the
611
+ moment this arms. That is not R6, it is an outage. Leg 3 therefore asks the wall that HAS been
612
+ answering rather than the marker that has not, and it keeps being right AFTER the flag's
613
+ deletion: an admin who has never opened the editor for a database has still not decided
614
+ anything about it.
615
+
616
+ β›”β›” PASS THE **PUBLIC** RECORD, NOT THE ONE OUT OF `users.json`. Leg 3 needs a username, and a
617
+ stored record is keyed BY username in that bucket and does not carry one INSIDE it β€” only
618
+ `core.users._public(uname, rec)` puts it there, which is what `deps.Session.user` holds. Hand
619
+ this the raw record and `may_open` gets a `None` viewer and fail-closes, so every `ut_*`
620
+ database reads as DENIED for an account that can open all of them. It fails in the SAFE
621
+ direction and is silently wrong, which is the worst pair to debug β€” it cost two call sites in
622
+ one afternoon: a gate double, and `routes_admin.get_perms`' own fix for this very outage.
623
+ """
624
+ if perms.is_admin(user):
625
+ return True
626
+ e = entry(user, table_key)
627
+ if e is not None and not bool(e.get('access', True)):
628
+ return False
629
+ key = str(table_key or '')
630
+ if key.startswith(_ut().KEY_PREFIX):
631
+ return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st))
632
+ return may_access(user, table_key)
633
+
634
+
635
+ def wall_declared(user, table_key):
636
+ """Is a ROW or FIELD narrowing declared for this principal on this database?
637
+
638
+ β›” THE QUESTION A DOOR ASKS BEFORE SERVING ROWS IT CANNOT SCOPE. `perms.py` warned that a
639
+ stored `ut_*` wall would be INERT β€” *"the editor would say DENY, the table routes would keep
640
+ serving, and nothing anywhere would say so"*. A route that cannot apply C1 must therefore
641
+ REFUSE for a principal this returns True for, rather than serve the whole database. False for
642
+ an admin (they bypass the wall entirely) and for any record with no entry, so a door asking
643
+ this pays nothing and changes nothing for everybody who has no wall.
644
+ """
645
+ if perms.is_admin(user):
646
+ return False
647
+ e = entry(user, table_key)
648
+ if not e:
649
+ return False
650
+ return bool(e.get('filter')) or bool(e.get('hiddenFields'))
651
+
652
+
653
+ def row_scope_applies(user, table_key):
654
+ """Does a permanent ROW filter narrow this principal on this database?
655
+
656
+ ⚠ `wall_declared`'s narrower half, and it exists so a rows-free caller can SKIP building rows
657
+ it would only need in order to filter them. `routes_tables.scoped_pids` is that caller: its
658
+ whole point is that the pid set costs no row pass, and paying for one on every database
659
+ switch β€” for every account, walled or not β€” would undo W30-T30 to enforce a rule that applies
660
+ to almost nobody. Asked here rather than spelled out at the call site, so there is ONE
661
+ statement of when the row wall bites ([[one-question-two-normalizers]]).
662
+ """
663
+ if perms.is_admin(user):
664
+ return False
665
+ return bool((entry(user, table_key) or {}).get('filter'))
666
+
667
+
668
+ def scoped_table(user, table_key, st=None, ctx=None):
669
+ """⭐⭐ CONTRACT C1 β€” the rows of ANY database, already field-stripped and row-filtered for
670
+ `user`. Registry topic or `ut_*`; there is no third kind and no per-database branch.
671
+
672
+ rows = scoped_table(user, 'ut_odoo_invoices') # a keychain database
673
+ rows = scoped_table(user, 'customer_data') # a registry topic
674
+
675
+ `user` is a user RECORD (the dict `deps.Session.user` carries), not a username β€” the whole
676
+ wall is a pure function of that record. BOTH arguments are positional and REQUIRED: a caller
677
+ that forgets the principal must not run, because the only thing a defaulted one could mean is
678
+ "unscoped", which is the widening direction.
679
+
680
+ β›” FAIL-CLOSED, THREE WAYS, AND EACH IS A DIFFERENT EXCEPTION so a caller can answer with the
681
+ right status instead of guessing: `UnknownTable` (no such database β€” never an empty list),
682
+ `Denied` (the IF question said no), `Unresolvable` (the rows cannot be served and here is
683
+ why β€” standing rule 1's second sentence).
684
+
685
+ ⚠ NO CAP. A connected source is read THROUGH the mirror in full (standing rule 1); the only
686
+ thing that stops it is a population that exceeds one materialisation window, and that arrives
687
+ as `Unresolvable` carrying its cause and a recommendation rather than as a short answer.
688
+
689
+ ⭐ E's SCRIPT SANDBOX HOLDS NO SECOND PATH TO THE STORE (wiring W1), which is why this is
690
+ THE door rather than A door: everything a sandboxed script may read, it reads here, under the
691
+ CALLING user's scope (R5).
692
+ """
693
+ _fields, rows = _scoped(user, table_key, st=st, ctx=ctx)
694
+ return rows
695
+
696
+
697
+ def scoped_fields(user, table_key, st=None):
698
+ """The COLUMNS of any database this principal may see β€” C1's other half.
699
+
700
+ β›” IT IS NOT A CONVENIENCE, IT IS THE SECOND WIRE. `strip_row`'s own note above says it: the
701
+ field list and the row payload are two different wires, and narrowing one without the other
702
+ leaves the value sitting where anything can read it. A caller that must render a scoped table
703
+ needs both, and E cannot read a `ut_*` definition to learn its columns β€” the sandbox has no
704
+ second path to the store (W1). So both come from here, off one wall.
705
+
706
+ ⚠ On a `ut_*` database this reads the DEFINITION only β€” the projection, no rows (D-213). On a
707
+ registry topic it goes through the registered reader, which builds that topic's pool; the
708
+ pool is cached per scope on the tenant runtime, so it is a cache hit next to `scoped_table`.
709
+ """
710
+ fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False)
711
+ return fields
712
+
713
+
714
+ def _scoped(user, table_key, st=None, ctx=None, want_rows=True):
715
+ """`(fields, rows)` β€” ONE evaluator behind both public doors, so they cannot disagree."""
716
+ key = str(table_key or '').strip()
717
+ if not key:
718
+ raise UnknownTable('a database key is required. This door will not guess which database '
719
+ 'was meant')
720
+ if not may_read(user, key, st=st):
721
+ raise Denied(f"this account may not read '{key}'")
722
+ fields, rows = _read(key, user, st, want_rows)
723
+ # THE FIELD WALL β€” a TRANSITIVE closure, so hiding a column also hides every formula computed
724
+ # FROM it. Resolved ONCE and used for both wires; see `hidden_keys` for why a set difference
725
+ # is the wrong shape here.
726
+ hide = hidden_keys(user, key, fields)
727
+ if not want_rows:
728
+ return (visible_fields(fields, user, key) if hide else fields), []
729
+ # THE ROW WALL β€” `permits()`, so a permanent filter this evaluator cannot answer DENIES
730
+ # rather than being ignored. Evaluated against the UNSTRIPPED contract on purpose: a
731
+ # permanent filter may name a column the reader is not allowed to SEE, and dropping the
732
+ # predicate would widen the read rather than narrow it.
733
+ rows = apply_row_scope(rows, user, key, fields, ctx)
734
+ if hide:
735
+ fields = visible_fields(fields, user, key)
736
+ rows = [strip_row(r, hide) for r in rows]
737
+ return fields, rows
738
+
739
+
740
+ def _read(table_key, user, st, want_rows=True):
741
+ """`(fields, rows)` BEFORE the wall β€” the app layer's reader, or core's own for a `ut_*`."""
742
+ reader = _ROW_SOURCES.get(table_key)
743
+ if reader is not None:
744
+ fields, rows = reader(table_key, user, st)
745
+ return list(fields or ()), list(rows or ())
746
+ ut = _ut()
747
+ if not table_key.startswith(ut.KEY_PREFIX):
748
+ # β›” A TOPIC WITH NO REGISTERED READER IS UNKNOWN, NOT EMPTY. In a process that never
749
+ # imported the API layer this is the honest answer: nothing here can build that pool.
750
+ raise UnknownTable(f"no database named '{table_key}' in this workspace, and no reader "
751
+ f"is registered for it")
752
+ return _read_user_table(table_key, st, want_rows)
753
+
754
+
755
+ def _read_user_table(table_key, st, want_rows=True):
756
+ """core's OWN reader for a `ut_*` database. Answers with NO registrar, deliberately.
757
+
758
+ ⭐ WHY IT LIVES IN `core` RATHER THAN BEING REGISTERED LIKE THE TOPICS, and it is the same
759
+ argument that seeds `user_tables._CONNECTED_PREFIXES` rather than registering it: a cold
760
+ process β€” E's sandbox subprocess, a worker, a gate β€” that never imported an API route still
761
+ owes the right answer for `ut_odoo_invoices`. A registrar-only design would raise there, and
762
+ the sandbox is exactly such a process.
763
+
764
+ ⚠ THE WALL IS ANSWERED ON A PROJECTION AND THE ROWS ARE NOT. `lend_defs` serves definitions
765
+ without the 28.6 MB of rows (D-213), which is every read this function makes when
766
+ `want_rows` is false; a projected document RAISES on `rows` rather than answering empty, so
767
+ the materialised arm below takes the whole read explicitly.
768
+ """
769
+ ut = _ut()
770
+ lent = ut.lend_defs(st)
771
+ defn = ut.get(table_key, st=lent)
772
+ if not defn:
773
+ raise UnknownTable(f"no database named '{table_key}'")
774
+ fields = [dict(f) for f in (defn.get('fields') or [])]
775
+ if not want_rows:
776
+ return fields, []
777
+ if not ut.materialises(table_key, st=st, defn=defn):
778
+ # A read-through grid stores no rows here β€” they live in the mirror, and reading
779
+ # `defn['rows']` would find an empty dict and serve an EMPTY GRID: correct-looking,
780
+ # wrong, and silent.
781
+ if _MIRROR_READER is None:
782
+ raise Unresolvable(
783
+ subject='rows', effect='unreadable',
784
+ cause=(f"'{table_key}' is served read-through from the connector mirror and no "
785
+ f'mirror reader is registered in this process'),
786
+ recommendation=('call `perm_scope.register_mirror(...)` from the app layer before '
787
+ 'reading a read-through database, or read it through the API'))
788
+ keys = {f['key'] for f in fields if f.get('key')}
789
+ return fields, list(_MIRROR_READER(table_key, keys, st) or ())
790
+ whole = ut.get(table_key, st=st)
791
+ if whole is None:
792
+ # Deleted between the wall and here. The same refusal, not an empty table.
793
+ raise UnknownTable(f"no database named '{table_key}'")
794
+ field_keys = {f['key'] for f in fields if f.get('key')}
795
+ rows = []
796
+ for rid, row in (whole.get('rows') or {}).items():
797
+ if not str(rid).isdigit():
798
+ continue
799
+ r = {k: v for k, v in (row or {}).items() if k in field_keys}
800
+ r['pid'] = int(rid)
801
+ rows.append(r)
802
+ rows.sort(key=lambda r: r['pid'])
803
+ return fields, rows
platform/core/perms.py CHANGED
@@ -269,19 +269,29 @@ def tenant_governable_modules(runtime, topics, ut_entries=()):
269
  in; this function is the only place the tenant is applied.
270
  `ut_entries` β€” this tenant's own `ut_*` databases, in `user_tables.nav_entries` shape.
271
 
272
- Returns `[{'key', 'label', 'enforced'}]`.
273
-
274
- β›”β›” `enforced` IS THE HONEST HALF AND IT IS NOT DECORATION. `perm_scope` walls the REGISTRY
275
- topics β€” `grid_assembly` calls `may_access`, `visible_fields` and `apply_row_scope` on every
276
- read. It is never consulted for a `ut_*` database: `routes_tables`' visibility is
277
- `user_tables.may_open` (creator, admin, or a `core.shares` grant) and there is no module
278
- grant in that path at all. So a stored `ut_*` wall would be INERT β€” the editor would say
279
- DENY, the table routes would keep serving, and nothing anywhere would say so. That is the
280
- same class as the empty-dropdown defect (wave 26 item 24): a control that answers when it
281
- cannot. The database is LISTED, because the owner asked to see every database; the wall is
282
- declared unenforced, and `routes_admin::_clean_perms` REFUSES to store one against it rather
283
- than accepting a rule nobody applies. Arming `perm_scope` over `ut_*` is a change to
284
- `routes_tables`/`user_tables` (lane A's fence) β€” booked, not faked.
 
 
 
 
 
 
 
 
 
 
285
  """
286
  out = []
287
  for t in topics or ():
@@ -293,20 +303,23 @@ def tenant_governable_modules(runtime, topics, ut_entries=()):
293
  # "Odoo products") would have moved one of them and left the permission editor showing the
294
  # old name with nothing to notice.
295
  label = (t.get('label') or registry.BY_KEY.get(key, {}).get('label') or key)
296
- out.append({'key': key, 'label': label, 'enforced': True})
297
  seen = {r['key'] for r in out}
298
  for e in ut_entries or ():
299
  key = str((e or {}).get('key') or '').strip()
300
  if not key or key in seen:
301
  continue
302
  seen.add(key)
303
- out.append({'key': key, 'label': (e.get('label') or key), 'enforced': False})
304
  return out
305
 
306
 
307
- def enforced_module_keys(modules):
308
- """The subset of `governable_modules`' answer a permission block may actually be stored for."""
309
- return {m['key'] for m in (modules or ()) if m.get('enforced')}
 
 
 
310
 
311
 
312
  def landing_page(user):
 
269
  in; this function is the only place the tenant is applied.
270
  `ut_entries` β€” this tenant's own `ut_*` databases, in `user_tables.nav_entries` shape.
271
 
272
+ Returns `[{'key', 'label'}]`.
273
+
274
+ ⭐⭐ WAVE 36 (W36-T22 / CONTRACT C2 / OWNER RULING R6) β€” **`enforced` IS DELETED, NOT DEFAULTED
275
+ TO `True`.** This docstring carried the paragraph that booked the work, and it read:
276
+
277
+ *"`perm_scope` walls the REGISTRY topics … It is never consulted for a `ut_*` database …
278
+ So a stored `ut_*` wall would be INERT β€” the editor would say DENY, the table routes would
279
+ keep serving, and nothing anywhere would say so … Arming `perm_scope` over `ut_*` is a
280
+ change to `routes_tables`/`user_tables` β€” booked, not faked."*
281
+
282
+ That change is W36-T21 and it has landed: `perm_scope.scoped_table` is the ONE door to any
283
+ database's rows, `routes_tables` applies the row filter and the hidden-field closure on every
284
+ `ut_*` read, and a door that cannot apply them REFUSES rather than serving the lot. So there
285
+ is no longer a second class of database for the flag to distinguish, and **a flag that is
286
+ always true is a lie with a green gate behind it** β€” which is why C2 says DELETE rather than
287
+ default. Owner item 11, verbatim: *"EVERY database should be able to be toggleable by admin …
288
+ Everything that is a database with Unique ID, is always going to be configurable with regards
289
+ to Fields, and Filtration per User. Make this infrastructure robust. And always true when we
290
+ want to onboard more databases into our App."*
291
+
292
+ ⚠ AND THE LIST STAYS DERIVED. Topics come from the registry, `ut_*` from the tenant's own
293
+ `nav_entries` β€” so a database created after this wave is governable with no code change and no
294
+ list edit. That is the half of R6 a flag could never have delivered.
295
  """
296
  out = []
297
  for t in topics or ():
 
303
  # "Odoo products") would have moved one of them and left the permission editor showing the
304
  # old name with nothing to notice.
305
  label = (t.get('label') or registry.BY_KEY.get(key, {}).get('label') or key)
306
+ out.append({'key': key, 'label': label})
307
  seen = {r['key'] for r in out}
308
  for e in ut_entries or ():
309
  key = str((e or {}).get('key') or '').strip()
310
  if not key or key in seen:
311
  continue
312
  seen.add(key)
313
+ out.append({'key': key, 'label': (e.get('label') or key)})
314
  return out
315
 
316
 
317
+ #: β›” `enforced_module_keys` IS DELETED (W36-T22 / C2), NOT NEUTERED TO "every governable key".
318
+ #: It read `{m['key'] for m in modules if m.get('enforced')}`, so with the flag gone it would
319
+ #: answer the EMPTY SET and `_clean_perms` would refuse every module β€” and "fix" it by returning
320
+ #: everything is a function whose answer no longer depends on its argument, i.e. the always-true
321
+ #: flag wearing a different costume. Its one caller (`routes_admin._enforced_keys`) goes with it;
322
+ #: what replaces both is the module list itself, which is the only list there is now.
323
 
324
 
325
  def landing_page(user):
platform/core/script_sandbox.py ADDED
@@ -0,0 +1,519 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/script_sandbox.py β€” WAVE 36 (R5 / R10, contract C1): running a tenant's OWN Python.
2
+
3
+ Owner item 6: *"Add code script as an interface (database View) so a user can build whatever they
4
+ want through the Agent chat interface."* Item 8: *"We need to really guardrail the reach of this
5
+ script. So let's really grill this down."* R10 ruled it SERVER-SIDE PYTHON after the trade was
6
+ stated, so this file is the guardrail, and one engine serves both items.
7
+
8
+ ════════════════════════════════════════════════════════════════════════════════════════════════
9
+ β›”β›” THE ONE PARAGRAPH TO READ BEFORE CHANGING ANYTHING HERE.
10
+
11
+ In-process CPython cannot deliver two of this ticket's clauses. An AST allow-list plus a curated
12
+ namespace stops import, file, network and environment access β€” but it **cannot cap memory and
13
+ cannot interrupt a runaway loop**, because a `while True:` in the same interpreter is not a slow
14
+ request, it is the tenant's ONE FastAPI process gone. So the script runs in a **SUBPROCESS**:
15
+ `resource.setrlimit` for address space and CPU, a hard wall-clock kill from the parent, and the
16
+ allow-list inside. Neither half is sufficient; both are load-bearing.
17
+
18
+ ⭐ AND THE SUBPROCESS RECEIVES **ROWS, NEVER A STORE**. The parent calls C1's `scoped_table` under
19
+ the CALLING user's record and serialises the result; the child imports nothing from this repo and
20
+ holds no credential, no runtime and no store handle. Wiring W1 ("the sandbox has no second store
21
+ path") is then true by CONSTRUCTION rather than by discipline, and it is checkable: the child
22
+ reports its own `sys.modules`, and no `core.*` name may appear in it.
23
+
24
+ β›” NEVER A BLACKLIST. Every rule below is an ALLOW-LIST β€” a set of node types, a set of attribute
25
+ names, a dict of builtins. A blacklist of dangerous spellings is bypassable by construction, and
26
+ the bypass is usually one string method away (`"{0.__class__}".format(x)` performs its attribute
27
+ lookup inside `format`, so there is no `ast.Attribute` node to refuse).
28
+ ════════════════════════════════════════════════════════════════════════════════════════════════
29
+
30
+ The two layers, and they refuse DIFFERENT things on purpose:
31
+
32
+ 1. `check_source()` β€” a pure function over source text. Refuses a construct the language offers
33
+ and this sandbox does not: `import`, `class`, `with`, `async`, `yield`, `global`, and every
34
+ attribute name outside `ALLOWED_ATTRS`.
35
+ 2. `SANDBOX_BUILTINS` β€” the names that resolve at all. `__import__`, `open`, `eval`, `exec`,
36
+ `compile`, `getattr`, `globals`, `vars` and `type` are simply absent, so a source that gets
37
+ past layer 1 still finds nothing to call.
38
+
39
+ ⚠ THAT DUPLICATION IS DELIBERATE AND IT CHANGES HOW THE GATE MUST BE WRITTEN. `import os` is
40
+ refused twice, so a negative control that drops ONE layer sees the other refuse and reports
41
+ green β€” the shape that already cost this wave one missed control in `routes_agent_harness`. So
42
+ each layer is tested AT ITS OWN BOUNDARY: `check_source()` is called directly on source strings,
43
+ and `run()` is driven end to end. An NC drops one entry from one frozenset and the matching
44
+ boundary goes red.
45
+ """
46
+ import ast
47
+ import json
48
+ import os
49
+ import subprocess
50
+ import sys
51
+ import tempfile
52
+ import time
53
+ from pathlib import Path
54
+
55
+ #: Wall clock, enforced by the PARENT with a kill. The one cap that works on every platform.
56
+ DEFAULT_TIMEOUT_S = 10.0
57
+
58
+ #: Address space for the child (`RLIMIT_AS`). POSIX only β€” see `run()`'s `caps` report.
59
+ DEFAULT_MEMORY_BYTES = 512 * 1024 * 1024
60
+
61
+ #: CPU seconds for the child (`RLIMIT_CPU`). POSIX only. Deliberately above the wall clock: the
62
+ #: wall-clock kill is the primary control and this is the backstop for a child that stops being
63
+ #: reachable. A CPU limit BELOW the timeout would make every slow script look like a CPU refusal.
64
+ DEFAULT_CPU_SECONDS = 15
65
+
66
+ #: What the script may print, in bytes. `print` is a curated builtin writing to a capped buffer,
67
+ #: and the child's real stdout goes to DEVNULL β€” so a script cannot fill a pipe, and anything
68
+ #: that escaped far enough to write to fd 1 has nowhere for it to land.
69
+ MAX_STDOUT_BYTES = 64 * 1024
70
+
71
+ #: The serialised ROW payload handed to the child. β›” A REFUSAL, NEVER A TRUNCATION (standing rule
72
+ #: 1): a short answer from a data tool is a wrong answer that looks right. Over this, `run()`
73
+ #: returns a named limit carrying its cause and a recommendation.
74
+ MAX_PAYLOAD_BYTES = 32 * 1024 * 1024
75
+
76
+ #: The emitted spec. A render spec is a description of a picture; one larger than this is data
77
+ #: pretending to be a description.
78
+ MAX_SPEC_BYTES = 2 * 1024 * 1024
79
+
80
+ MAX_SOURCE_BYTES = 128 * 1024
81
+
82
+
83
+ # ═══════���══════════════════════════════════════════════ LAYER 1 β€” the AST allow-list ═══════════
84
+ #: Every `ast` node class a script may contain. β›” THE ABSENCES ARE THE POLICY: `Import` /
85
+ #: `ImportFrom` (no module reaches the script), `ClassDef` (a class body is a namespace with its
86
+ #: own scoping rules and buys a data script nothing), `With` (a context manager is `__enter__`
87
+ #: by another spelling), `Global` / `Nonlocal` (rebinding the sandbox's own names), and every
88
+ #: `Async*` / `Await` / `Yield` form (this engine is synchronous; a coroutine that is never
89
+ #: awaited is a silent no-op that looks like a working script).
90
+ ALLOWED_NODES = frozenset("""
91
+ Module Expr Assign AugAssign AnnAssign NamedExpr Return Pass Break Continue Delete Assert Raise
92
+ If For While Try TryStar ExceptHandler FunctionDef Lambda arguments arg keyword
93
+ BoolOp BinOp UnaryOp IfExp Dict Set List Tuple Starred Subscript Slice Compare Call Attribute Name
94
+ Constant JoinedStr FormattedValue ListComp SetComp DictComp GeneratorExp comprehension
95
+ Load Store Del
96
+ And Or Not Invert UAdd USub
97
+ Add Sub Mult Div FloorDiv Mod Pow LShift RShift BitOr BitXor BitAnd MatMult
98
+ Eq NotEq Lt LtE Gt GtE Is IsNot In NotIn
99
+ """.split())
100
+
101
+ #: Every attribute name a script may READ or CALL. β›”β›” THIS IS THE LOAD-BEARING SET, and it is
102
+ #: an allow-list of NAMES rather than a refusal of dunders, because the interesting escapes are
103
+ #: ordinary-looking: `f.__globals__` on any function reaches the runner's own module namespace,
104
+ #: `e.__traceback__.tb_frame.f_globals` reaches it from an exception handler, and `().__class__`
105
+ #: reaches `object.__subclasses__`. None of those names is here, and neither is any name this
106
+ #: sandbox has not been asked for.
107
+ #: ⚠ `format` IS ABSENT DELIBERATELY. `"{0.__class__}".format(x)` performs the attribute lookup
108
+ #: INSIDE `str.format`, where no `ast.Attribute` node exists for layer 1 to see. f-strings are
109
+ #: fine β€” `f"{x.__class__}"` compiles to a real `Attribute` node and is refused.
110
+ ALLOWED_ATTRS = frozenset("""
111
+ append extend insert pop remove clear sort reverse copy count index
112
+ keys values items get setdefault update
113
+ add discard union intersection difference issubset issuperset
114
+ join split rsplit splitlines strip lstrip rstrip lower upper title capitalize casefold
115
+ replace startswith endswith find rfind zfill ljust rjust center partition removeprefix removesuffix
116
+ isdigit isalpha isalnum isspace isupper islower isnumeric
117
+ real imag numerator denominator
118
+ """.split())
119
+
120
+
121
+ class Refused(Exception):
122
+ """A named refusal: `code` for a caller to branch on, `message` for a person to read."""
123
+
124
+ def __init__(self, code, message):
125
+ self.code, self.message = code, message
126
+ super().__init__(f"{code}: {message}")
127
+
128
+
129
+ def _attr_ok(name):
130
+ """An attribute name passes only if it is on the list AND is not private.
131
+
132
+ ⚠ THE SECOND TEST IS NOT A BLACKLIST β€” it narrows an allow-list that already excludes every
133
+ private name. It is here so that adding a name to `ALLOWED_ATTRS` cannot open a dunder by
134
+ accident, which is the one edit a future reader is most likely to make in a hurry.
135
+ """
136
+ return name in ALLOWED_ATTRS and not name.startswith("_")
137
+
138
+
139
+ def check_source(source):
140
+ """LAYER 1. Return a `Refused` for source this sandbox will not run, or `None`.
141
+
142
+ ⭐ PURE, AND THAT IS WHAT MAKES IT TESTABLE AT ITS OWN BOUNDARY. It reads no file, spawns no
143
+ process and touches no store, so a gate can hand it a hundred hostile strings for free and an
144
+ NC can drop one entry from one frozenset and watch exactly this function change its answer.
145
+ """
146
+ text = str(source or "")
147
+ if len(text.encode("utf-8", "replace")) > MAX_SOURCE_BYTES:
148
+ return Refused("source_too_long",
149
+ f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of source")
150
+ try:
151
+ tree = ast.parse(text)
152
+ except SyntaxError as exc:
153
+ return Refused("syntax", f"line {exc.lineno or 0}: {exc.msg}")
154
+
155
+ for node in ast.walk(tree):
156
+ kind = type(node).__name__
157
+ if kind not in ALLOWED_NODES:
158
+ return Refused("refused_construct",
159
+ f"line {getattr(node, 'lineno', 0)}: this sandbox does not run "
160
+ f"{_english(kind)}")
161
+ if isinstance(node, ast.Attribute) and not _attr_ok(node.attr):
162
+ return Refused("refused_attribute",
163
+ f"line {getattr(node, 'lineno', 0)}: the attribute "
164
+ f"'{node.attr}' is not available inside a script view")
165
+ # β›” A NAME may not be private either. `_` prefixed names are the runner's own, and a
166
+ # script that could bind one could shadow the machinery it runs on top of.
167
+ if isinstance(node, ast.Name) and node.id.startswith("_"):
168
+ return Refused("reserved_name",
169
+ f"line {getattr(node, 'lineno', 0)}: names starting with an "
170
+ f"underscore are reserved by the sandbox")
171
+ if isinstance(node, (ast.FunctionDef, ast.arg, ast.ExceptHandler)) and str(
172
+ getattr(node, "name", None) or getattr(node, "arg", "") or "").startswith("_"):
173
+ return Refused("reserved_name",
174
+ f"line {getattr(node, 'lineno', 0)}: names starting with an "
175
+ f"underscore are reserved by the sandbox")
176
+ if isinstance(node, ast.keyword) and str(node.arg or "").startswith("_"):
177
+ return Refused("reserved_name",
178
+ f"line {getattr(node, 'lineno', 0)}: keyword arguments starting with "
179
+ f"an underscore are reserved by the sandbox")
180
+ return None
181
+
182
+
183
+ _ENGLISH = {
184
+ "Import": "an import", "ImportFrom": "an import", "ClassDef": "a class definition",
185
+ "With": "a with block", "AsyncWith": "a with block", "AsyncFor": "an async loop",
186
+ "AsyncFunctionDef": "an async function", "Await": "await", "Yield": "yield",
187
+ "YieldFrom": "yield from", "Global": "a global statement", "Nonlocal": "a nonlocal statement",
188
+ "Match": "a match statement",
189
+ }
190
+
191
+
192
+ def _english(kind):
193
+ return _ENGLISH.get(kind, f"a {kind} expression")
194
+
195
+
196
+ # ══════════════════════════════════════════ LAYER 2 β€” the namespace, and the child program ═════
197
+ #: The builtins a script may reach, BY NAME. Everything else is a `NameError` in the child.
198
+ #: β›” THE ABSENCES, again, are the policy: `__import__` `open` `eval` `exec` `compile` `input`
199
+ #: `getattr` `setattr` `delattr` `globals` `locals` `vars` `dir` `type` `super` `object` `help`
200
+ #: `exit` `breakpoint` `memoryview` `id`. Several are harmless on their own; each one is a step
201
+ #: on a published escape, and none has ever been asked for by a script that shapes rows.
202
+ #: ⚠ THE EXCEPTION CLASSES ARE HERE BECAUSE `try:` IS, and a `try` block whose `except` clause
203
+ #: cannot name what it catches is a construct that reads as supported and is not. They are safe
204
+ #: for the same reason everything else is: `Exception.__subclasses__` needs an attribute this
205
+ #: sandbox does not allow, so a class object in the namespace is a leaf, not a doorway.
206
+ SANDBOX_BUILTIN_NAMES = (
207
+ "abs all any bool bytes callable chr dict divmod enumerate filter float frozenset hash hex "
208
+ "int isinstance issubclass iter len list map max min next oct ord pow range repr reversed "
209
+ "round set slice sorted str sum tuple zip True False None "
210
+ "Exception ValueError TypeError KeyError IndexError ZeroDivisionError ArithmeticError "
211
+ "AttributeError StopIteration OverflowError"
212
+ ).split()
213
+
214
+ #: The literal program the child runs. It is TEXT rather than a module because the child must
215
+ #: import nothing from this repo: a module would be found on `sys.path` and would drag `core`
216
+ #: with it, which is exactly the second store path W1 forbids.
217
+ #: ⚠ Every name in here is underscore-prefixed and layer 1 refuses a script from binding one, so
218
+ #: the runner's own machinery cannot be shadowed by the source it executes.
219
+ _RUNNER = r'''
220
+ import json as _json, os as _os, sys as _sys
221
+
222
+ _pay = _json.loads(open(_sys.argv[1], "r", encoding="utf-8").read())
223
+ _out = {"ok": False, "code": "not_run", "message": "the script did not run",
224
+ "stdout": "", "spec": None, "caps": {"wallClock": True, "memory": False, "cpu": False}}
225
+
226
+ # ── the caps this platform can actually apply, reported either way (standing rule 1) ──────────
227
+ try:
228
+ import resource as _res
229
+ _mem = int(_pay["memoryBytes"])
230
+ _res.setrlimit(_res.RLIMIT_AS, (_mem, _mem))
231
+ _out["caps"]["memory"] = True
232
+ _cpu = int(_pay["cpuSeconds"])
233
+ _res.setrlimit(_res.RLIMIT_CPU, (_cpu, _cpu))
234
+ _out["caps"]["cpu"] = True
235
+ except Exception:
236
+ # `resource` is POSIX only. The wall-clock kill in the parent still applies, and `caps` says
237
+ # which of the three held, never a silent partial.
238
+ pass
239
+
240
+ _printed = []
241
+ _spent = [0]
242
+ _LIMIT = int(_pay["maxStdout"])
243
+
244
+
245
+ def _print(*_a, **_k):
246
+ _text = (_k.get("sep") or " ").join(str(_x) for _x in _a) + (_k.get("end") or "\n")
247
+ _room = _LIMIT - _spent[0]
248
+ if _room > 0:
249
+ _printed.append(_text[:_room])
250
+ _spent[0] += len(_text)
251
+
252
+
253
+ class _Refusal(Exception):
254
+ """The SANDBOX refusing, as distinct from the SCRIPT failing.
255
+
256
+ Without its own class these arrive as `ValueError`, indistinguishable from a `ValueError` the
257
+ script raised itself, and the answer then says "refused" about an ordinary bug in the tenant's
258
+ own code. Two different facts, two different codes.
259
+ """
260
+
261
+
262
+ _emitted = []
263
+
264
+
265
+ def _emit(_spec):
266
+ if not isinstance(_spec, dict):
267
+ raise _Refusal("emit() takes a view spec, which is a dictionary")
268
+ if _emitted:
269
+ raise _Refusal("emit() was already called; a script view emits exactly one view")
270
+ _emitted.append(_spec)
271
+
272
+
273
+ _rows = _pay["rows"]
274
+ _fields = _pay["fields"]
275
+ _bound = _pay["table"]
276
+
277
+
278
+ def _scoped_table(_table=None):
279
+ if _table is not None and str(_table) != _bound:
280
+ raise _Refusal(
281
+ "this script view is bound to the database '" + _bound + "' and asked for '"
282
+ + str(_table) + "'. A script view reads its own database only")
283
+ return [dict(_r) for _r in _rows]
284
+
285
+
286
+ def _scoped_fields():
287
+ return [dict(_f) for _f in _fields]
288
+
289
+
290
+ _ns = {"__builtins__": {_n: __builtins__[_n] if isinstance(__builtins__, dict)
291
+ else getattr(__builtins__, _n)
292
+ for _n in _pay["builtins"]}}
293
+ _ns["__builtins__"]["print"] = _print
294
+ _ns["print"] = _print
295
+ _ns["emit"] = _emit
296
+ _ns["scoped_table"] = _scoped_table
297
+ _ns["scoped_fields"] = _scoped_fields
298
+ _ns["table"] = _bound
299
+
300
+ try:
301
+ exec(compile(_pay["source"], "<script view>", "exec"), _ns)
302
+ if not _emitted:
303
+ _out.update(ok=False, code="no_view",
304
+ message="the script finished without calling emit(spec)")
305
+ else:
306
+ _out.update(ok=True, code="", message="", spec=_emitted[0])
307
+ except _Refusal as _e:
308
+ _out.update(ok=False, code="refused", message=str(_e)[:400])
309
+ except MemoryError:
310
+ _out.update(ok=False, code="memory",
311
+ message="the script used more memory than a script view is allowed")
312
+ except NameError as _e:
313
+ _out.update(ok=False, code="refused_name",
314
+ message=str(_e)[:200] + ". A script view may use only the names the sandbox "
315
+ "provides")
316
+ except BaseException as _e:
317
+ _out.update(ok=False, code="error",
318
+ message=type(_e).__name__ + ": " + str(_e)[:400])
319
+
320
+ _out["stdout"] = "".join(_printed)
321
+ _out["truncated"] = _spent[0] > _LIMIT
322
+ # ⭐ THE PROBE: what this child actually had. The PARENT strips it unless it was asked for, so a
323
+ # production run never carries it and a gate can still prove that no `core.*` module and no
324
+ # secret-shaped environment key was ever inside this process.
325
+ # ⚠ SNAPSHOTTED AFTER `exec`, and `os` is imported at the TOP so this list does not depend on
326
+ # dict-literal evaluation order. The first draft called `__import__("os")` inside this very
327
+ # expression, so whether `os` appeared depended on which value Python built first: a probe whose
328
+ # contents move with an unrelated edit is a probe a gate cannot assert against.
329
+ _out["probe"] = {"modules": sorted(_sys.modules), "env": sorted(_os.environ)}
330
+ open(_sys.argv[2], "w", encoding="utf-8").write(_json.dumps(_out, default=str))
331
+ '''
332
+
333
+
334
+ def _child_env():
335
+ """The child's WHOLE environment. An allow-list of two keys, and neither is a credential.
336
+
337
+ β›” NOT `os.environ.copy()` MINUS SOMETHING. A subtractive environment ships every key nobody
338
+ thought to name: `HF_TOKEN`, `ODOO_PASSWORD`, `ANTHROPIC_API_KEY` and whatever the next
339
+ connector adds. The three names below are here because Python will not start on Windows
340
+ without them; on Linux this returns `{}` and the child runs with no environment at all.
341
+ """
342
+ env = {}
343
+ for name in ("SystemRoot", "SYSTEMROOT", "WINDIR"):
344
+ if os.environ.get(name):
345
+ env[name] = os.environ[name]
346
+ return env
347
+
348
+
349
+ def run(source, rows, fields, table_key, *, timeout_s=DEFAULT_TIMEOUT_S,
350
+ memory_bytes=DEFAULT_MEMORY_BYTES, cpu_seconds=DEFAULT_CPU_SECONDS, probe=False):
351
+ """Run ONE script over rows that are ALREADY scoped. Returns C3's envelope plus `caps`.
352
+
353
+ {ok, code, message, spec, stdout, truncated, ms, caps: {wallClock, memory, cpu}}
354
+
355
+ β›” THIS FUNCTION NEVER TOUCHES A STORE, AND THAT IS THE POINT: it takes rows. `run_view()`
356
+ below is the door that fetches them through C1; keeping the two apart is what lets a gate
357
+ drive the sandbox with no tenant, no runtime and no credential anywhere in the process.
358
+
359
+ ⚠ `caps` IS PART OF THE ANSWER, NOT DEBUG OUTPUT. On Windows `resource` does not exist, so
360
+ the memory and CPU limits are NOT applied and this says so. A caller that reports `ok:true`
361
+ without reading `caps` is claiming an enforcement that did not happen (standing rule 1).
362
+ """
363
+ started = time.monotonic()
364
+ refusal = check_source(source)
365
+ if refusal is not None:
366
+ return _refusal(refusal.code, refusal.message, started)
367
+
368
+ payload = {"source": str(source or ""), "rows": rows, "fields": fields,
369
+ "table": str(table_key or ""), "builtins": SANDBOX_BUILTIN_NAMES,
370
+ "maxStdout": MAX_STDOUT_BYTES, "memoryBytes": int(memory_bytes),
371
+ "cpuSeconds": int(cpu_seconds)}
372
+ try:
373
+ blob = json.dumps(payload, default=str)
374
+ except (TypeError, ValueError) as exc:
375
+ return _refusal("bad_rows", f"these rows cannot be handed to a script ({exc})", started)
376
+ if len(blob.encode("utf-8", "replace")) > MAX_PAYLOAD_BYTES:
377
+ # β›” REPORTED, NOT TRUNCATED (standing rule 1's second sentence): cause and recommendation,
378
+ # in the words the owner asked for, rather than a quietly short answer.
379
+ return _refusal(
380
+ "payload_too_large",
381
+ f"this database's rows are larger than the {MAX_PAYLOAD_BYTES // (1024 * 1024)} MB a "
382
+ f"script view can be handed at once. Narrow the view with a filter, or raise the "
383
+ f"sandbox payload limit for this deployment", started)
384
+
385
+ with tempfile.TemporaryDirectory(prefix="aios-script-") as work:
386
+ pay_path = Path(work) / "payload.json"
387
+ res_path = Path(work) / "result.json"
388
+ pay_path.write_text(blob, encoding="utf-8")
389
+ # `-I` isolates the interpreter (no PYTHON* env, no user site), `-S` skips site-packages,
390
+ # and the program arrives on STDIN so there is no file for anything to import it as.
391
+ # β›” `-X utf8` AND AN EXPLICIT `encoding` ARE NOT TIDINESS. Without them this pipe is
392
+ # encoded with the parent's locale codec, which on this Windows box is cp1252: the runner
393
+ # text below cannot be represented in it and `subprocess.run` died with
394
+ # `UnicodeEncodeError` before the child ever started. A sandbox whose behaviour depends on
395
+ # the operator's locale is a sandbox with two behaviours. `-I` implies `-E`, so
396
+ # `PYTHONUTF8` in the environment could not have carried this, it has to be a flag.
397
+ argv = [sys.executable, "-I", "-S", "-X", "utf8", "-", str(pay_path), str(res_path)]
398
+ try:
399
+ done = subprocess.run(
400
+ argv, input=_RUNNER, text=True, encoding="utf-8", errors="replace",
401
+ cwd=work, env=_child_env(),
402
+ stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout_s)
403
+ except subprocess.TimeoutExpired:
404
+ return _refusal("timeout",
405
+ f"the script ran longer than {timeout_s:g} seconds and was stopped",
406
+ started)
407
+ except OSError as exc:
408
+ return _refusal("no_sandbox",
409
+ f"a script view could not be started on this deployment ({exc})",
410
+ started)
411
+
412
+ if not res_path.is_file():
413
+ # The child died without writing an answer: an rlimit signal, an OOM kill, or a crash.
414
+ # ⚠ NAMED BY ITS RETURN CODE rather than reported as a generic failure β€” a memory kill
415
+ # and a bug in this file must not read identically to an operator.
416
+ return _refusal(*_died(done), started)
417
+ try:
418
+ out = json.loads(res_path.read_text(encoding="utf-8"))
419
+ except (OSError, ValueError) as exc:
420
+ return _refusal("unreadable", f"the script's answer could not be read ({exc})",
421
+ started)
422
+
423
+ out["ms"] = int((time.monotonic() - started) * 1000)
424
+ if out.get("ok"):
425
+ spec_error = _check_spec(out.get("spec"))
426
+ if spec_error:
427
+ out.update(ok=False, code="bad_spec", message=spec_error, spec=None)
428
+ if not probe:
429
+ out.pop("probe", None)
430
+ return out
431
+
432
+
433
+ def _died(done):
434
+ """`(code, message)` for a child that produced no answer."""
435
+ rc = done.returncode
436
+ tail = " ".join((done.stderr or "").split())[-300:]
437
+ if rc in (-9, 137):
438
+ return "memory", "the script was stopped for using too much memory"
439
+ if rc in (-24, 152):
440
+ return "timeout", "the script used more processor time than a script view is allowed"
441
+ return "crashed", f"the script view engine stopped without an answer{': ' + tail if tail else ''}"
442
+
443
+
444
+ def _refusal(code, message, started):
445
+ return {"ok": False, "code": code, "message": message, "spec": None, "stdout": "",
446
+ "truncated": False, "ms": int((time.monotonic() - started) * 1000),
447
+ "caps": {"wallClock": True, "memory": False, "cpu": False}}
448
+
449
+
450
+ def _check_spec(spec):
451
+ """C3: a spec is a DESCRIPTION the client draws. Never HTML, never a script, never a URL.
452
+
453
+ β›” THE CHECK IS ON THE KEYS, NOT ON THE STRING CONTENTS. Scanning values for `<script>` is a
454
+ blacklist and would pass `<SCR` + `IPT>`; refusing a spec that carries an `html`, `script`,
455
+ `src` or `onclick` key refuses the SHAPE that would let a renderer be talked into executing
456
+ something. The vocabulary of legal `kind`s is the ROUTE's business (W36-T37) β€” this is the
457
+ floor every caller gets whether or not the route above it remembers.
458
+ """
459
+ if not isinstance(spec, dict):
460
+ return "the script emitted something that is not a view spec"
461
+ try:
462
+ blob = json.dumps(spec)
463
+ except (TypeError, ValueError):
464
+ return "the emitted view spec is not something the client can be sent"
465
+ if len(blob.encode("utf-8", "replace")) > MAX_SPEC_BYTES:
466
+ return (f"the emitted view spec is over {MAX_SPEC_BYTES // (1024 * 1024)} MB. A view spec "
467
+ f"describes a picture; it is not where the rows go")
468
+ banned = {"html", "innerhtml", "script", "src", "srcdoc", "href", "style", "onclick", "onload"}
469
+ found = sorted(k for k in _keys_of(spec) if str(k).lower() in banned)
470
+ if found:
471
+ return (f"a view spec may not carry {', '.join(found)}. The client DRAWS a spec, so a "
472
+ f"markup or URL key would be a script by another name")
473
+ return None
474
+
475
+
476
+ def _keys_of(value, depth=0):
477
+ """Every key anywhere in a nested spec. Bounded, so a deep structure cannot spin this."""
478
+ if depth > 12:
479
+ return
480
+ if isinstance(value, dict):
481
+ for key, sub in value.items():
482
+ yield key
483
+ yield from _keys_of(sub, depth + 1)
484
+ elif isinstance(value, (list, tuple)):
485
+ for sub in value:
486
+ yield from _keys_of(sub, depth + 1)
487
+
488
+
489
+ # ══════════════════════════════════════════════ THE DOOR β€” C1 is the ONLY way to a row ═════════
490
+ def run_view(user, table_key, source, st=None, **kw):
491
+ """Fetch through C1 under `user`'s scope, then run the script over what came back (R5).
492
+
493
+ ⭐⭐ THE FETCH HAPPENS IN THE PARENT AND ONLY ROWS CROSS INTO THE CHILD. That is wiring W1
494
+ made structural: the child has no runtime to ask, no store handle to open and no credential
495
+ to use, so "a script cannot read what its caller cannot read" is not a rule anybody has to
496
+ keep β€” there is no second path for it to be broken through.
497
+
498
+ β›” C1'S THREE EXCEPTIONS ARE ANSWERED, NEVER SWALLOWED. `UnknownTable`, `Denied` and
499
+ `Unresolvable` mean three different things to a person; collapsing them into "no rows" is the
500
+ silent-empty answer C1 was written to make impossible. `Unresolvable.as_limit()` is handed
501
+ through in the words it was raised with β€” standing rule 1's second sentence, verbatim.
502
+ """
503
+ import core.perm_scope as perm_scope
504
+
505
+ key = str(table_key or "")
506
+ try:
507
+ rows = perm_scope.scoped_table(user, key, st=st)
508
+ fields = perm_scope.scoped_fields(user, key, st=st)
509
+ except perm_scope.UnknownTable as exc:
510
+ return _refusal("unknown_table", str(exc) or f"there is no database '{key}'",
511
+ time.monotonic())
512
+ except perm_scope.Denied as exc:
513
+ return _refusal("denied", str(exc) or "this account may not read that database",
514
+ time.monotonic())
515
+ except perm_scope.Unresolvable as exc:
516
+ out = _refusal("unresolvable", str(exc), time.monotonic())
517
+ out["limit"] = exc.as_limit()
518
+ return out
519
+ return run(source, rows, fields, key, **kw)
platform/core/shares.py CHANGED
@@ -57,18 +57,26 @@ KINDS = ('view', 'folder', 'database')
57
  #: the sentence holds. `require_session` plus the topic's own gate run first, and the
58
  #: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is
59
  #: merged, so a grant can only narrow-or-equal what that account could already reach.
60
- #: * `kind='database'` on a `ut_*` table β€” **THE SENTENCE IS FALSE AND THIS REGISTRY IS THE
61
- #: ONLY WALL.** `routes_admin._PERM_MODULES` is `("customer_data", "product_data")` and
62
- #: `_clean_perms` 400s anything else, so no row filter and no hidden field can even be
63
- #: DECLARED for a user table; `routes_tables.py` makes zero `perm_scope` calls and passes
64
- #: `hidden_keys=frozenset()`. A `database` grant is therefore ALL-OR-NOTHING β€” every row,
65
- #: every column β€” and an `*` database grant admits every account in the tenant to all of it.
 
66
  #:
67
- #: ⚠ THAT IS A REAL CAPABILITY, DELIBERATELY KEPT, not a hole to plug in passing. What was wrong
68
- #: was a docstring promising a second wall that does not exist for this kind; scoping user tables
69
- #: is booked (S-8), not done. `routes_shares.py`'s module docstring says the same thing at the
70
- #: OTHER door β€” the audit's fix is "say so at both", and one door saying it is how the next reader
71
- #: gets the confident half [[one-question-two-normalizers]].
 
 
 
 
 
 
 
72
  EVERYONE = '*'
73
 
74
  ROLES = ('view', 'edit')
 
57
  #: the sentence holds. `require_session` plus the topic's own gate run first, and the
58
  #: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is
59
  #: merged, so a grant can only narrow-or-equal what that account could already reach.
60
+ #: * `kind='database'` on a `ut_*` table β€” ⭐⭐ **THE SENTENCE HOLDS HERE TOO NOW, AND THAT IS
61
+ #: W36-T21 / OWNER RULING R6 (audit S-8, CLOSED).** It did not until wave 36, and the reason
62
+ #: is worth keeping: `routes_admin._clean_perms` 400'd any key outside
63
+ #: `("customer_data", "product_data")`, so no row filter and no hidden field could even be
64
+ #: DECLARED for a user table, `routes_tables.py` made zero `perm_scope` calls, and it passed
65
+ #: `hidden_keys=frozenset()`. A `database` grant was therefore ALL-OR-NOTHING β€” every row,
66
+ #: every column β€” and this registry was the only wall behind it.
67
  #:
68
+ #: ⭐ WHAT CHANGED: `perm_scope.scoped_table` (contract C1) is the ONE door to any database's
69
+ #: rows. `routes_tables` applies the permanent filter before `pids` is taken and the transitive
70
+ #: hidden-field closure after `workspace_wire`, on EVERY `ut_*` read β€” the same code that walls
71
+ #: `customer_data` β€” and a door that cannot apply them REFUSES rather than serving the lot. So a
72
+ #: `database` grant is once again bounded by a second wall: it decides WHETHER an account reaches
73
+ #: the database, and C1 decides WHICH rows and columns it then sees. An `*` grant still admits
74
+ #: every account in the tenant, and each of them still only sees what their own wall allows.
75
+ #:
76
+ #: ⚠ `routes_shares.py`'s module docstring carries the OLD sentence at the other door and is in
77
+ #: no wave-36 fence β€” the audit's own fix was "say so at both", so one of the two is now stale.
78
+ #: Booked in `mailbox/C.md` (C-14) rather than edited across a fence
79
+ #: [[two-gates-can-assert-opposite-things]].
80
  EVERYONE = '*'
81
 
82
  ROLES = ('view', 'edit')
platform/core/store.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/core/table_store.py CHANGED
@@ -1,618 +1,746 @@
1
- """The generic per-user TABLE WORKSPACE store β€” the persistence half of the table-page factory.
2
-
3
- One durable store key holds one table OBJECT's per-user Airtable-style state:
4
-
5
- {username: {'views': {view_id: SavedView},
6
- 'fields': {field_key: Field}, # notes + custom_ + measure_ strata
7
- 'overlays': {str(pid): {field_key: value}}}}
8
-
9
- `make(table_key)` returns the six operations a table page's host loop needs, closed over that
10
- key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
11
- these exact functions β€” the logic MOVED here 2026-07-27 so that duplicating the Customer table
12
- pattern to a new object is a registry row + a config, not a copy of the store plumbing
13
- (owner directive: the table-page factory).
14
-
15
- A LIST (membership/formula semantics) is deliberately a different store from a VIEW
16
- (presentation/query state) β€” see modules/customer_data.py's customer_lists key.
17
- """
18
- import core.store as store
19
-
20
- #: Wave-9 I17 β€” the SHARED bucket. Views whose permissions make them visible to anyone but
21
- #: their creator live here instead of in a personal workspace, under a key that cannot collide
22
- #: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
23
- #: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
24
- #: here, and a view shared is removed from its creator's workspace. Two homes would mean two
25
- #: divergent copies the moment either was edited.
26
- SHARED_KEY = '__shared__'
27
-
28
-
29
- def _may_see(view, viewer, is_admin=False):
30
- """Visibility for ONE shared view, fail-closed.
31
-
32
- 'collaborative' = everyone who can already open the module (the caller has gated that).
33
- 'users' = the named users, plus the creator, plus admins β€” an admin who could not
34
- see a view could not administer it either.
35
- Anything unrecognised returns False rather than defaulting open: an unreadable permission
36
- must never widen access ([[aios-permissioning]] β€” no fail-open defaults).
37
- """
38
- if not isinstance(view, dict):
39
- return False
40
- if view.get('createdBy') == viewer or is_admin:
41
- return True
42
- perms = view.get('permissions') or {}
43
- edit = perms.get('edit')
44
- if edit == 'collaborative':
45
- return True
46
- if edit == 'users':
47
- return viewer in set(perms.get('users') or ())
48
- return False # 'personal', absent, or junk
49
-
50
-
51
- def _may_edit(view, viewer, is_admin=False):
52
- """Who may WRITE a shared view. Same set as visibility today β€” the owner's item asks 'who
53
- can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
54
- separate function so they can diverge (a future read-only share) without hunting callers."""
55
- return _may_see(view, viewer, is_admin)
56
-
57
-
58
- def _may_administer(view, viewer, is_admin=False):
59
- """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
60
-
61
- Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
62
- could grant themselves sole ownership of somebody else's view, or quietly widen a
63
- users-scoped view to everyone β€” the classic privilege-escalation-by-edit hole.
64
- """
65
- if not isinstance(view, dict):
66
- return False
67
- return bool(is_admin) or view.get('createdBy') == viewer
68
-
69
-
70
- def is_shared(view):
71
- """A view belongs in the shared bucket when its permissions reach beyond its creator."""
72
- return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
73
-
74
-
75
- def source_override_is_empty(payload):
76
- """Does this stored definition of a SOURCE (non-custom) column carry any user state?
77
-
78
- A cleared note on an immutable source field returns to the canonical schema instead of
79
- leaving a meaningless override row. Custom fields remain even with an empty note β€” and so
80
- does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format
81
- override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY.
82
-
83
- β›” EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently
84
- stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days`
85
- built an override whose only content was that summary, so this rule threw the whole row away
86
- on write while the menu went on reading "Summary: Average" from the client's own optimistic
87
- copy until the next login β€” a discarded WRITE wearing the face of a failed read
88
- ([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores
89
- its definitions through another door entirely, kept it.
90
-
91
- ⚠ ONE RULE, TWO CALLERS β€” here and `grid_events`' store-less fallback. Two copies of a
92
- discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]).
93
- ⚠ A CLEARED summary still drops the row, which is the intent: with nothing else set, the
94
- column goes back to whatever the contract declares for it.
95
- """
96
- payload = payload or {}
97
- if payload.get('custom'):
98
- return False
99
- return (not str(payload.get('note') or '').strip()
100
- and not isinstance(payload.get('measure'), dict)
101
- and not isinstance(payload.get('format'), dict)
102
- and not str(payload.get('agg') or '').strip())
103
-
104
-
105
- def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
106
- """Allocate one human-facing name inside a store.update transaction.
107
-
108
- Keys/ids remain structural identity. Names compare case-insensitively after collapsing
109
- whitespace, because those variants are indistinguishable in the UI. This helper belongs
110
- in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
111
- choose the same free name before either write lands.
112
- """
113
- limit = max(1, int(max_len))
114
-
115
- def _clean(value):
116
- return ' '.join(str(value or '').split())
117
-
118
- base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
119
- taken = {_clean(value).casefold() for value in existing if _clean(value)}
120
- if base.casefold() not in taken:
121
- return base
122
- index = 2
123
- while True:
124
- suffix = f' {index}'
125
- stem = base[:max(0, limit - len(suffix))].rstrip()
126
- candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
127
- if candidate.casefold() not in taken:
128
- return candidate
129
- index += 1
130
-
131
-
132
- class TableStore:
133
- """The six store operations for one table object's workspace, closed over its store key.
134
-
135
- `st` (wave 18, C3-UT) is the STORE HANDLE β€” anything exposing `get(name)` /
136
- `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
137
- zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
138
- apply the tenant prefix / repo binding β€” which is what makes a user table created by a
139
- Nurilab admin land in Nurilab's store instead of Royal's.
140
- """
141
-
142
- def __init__(self, table_key, st=None):
143
- self.table_key = table_key
144
- self._st = st if st is not None else store
145
-
146
- @property
147
- def st(self):
148
- """The bound store handle β€” for SIBLING registries (core/shares) that must read the
149
- same tenant's buckets this workspace lives in (wave 21, C1)."""
150
- return self._st
151
-
152
- def find_view(self, view_id):
153
- """`(owner_username, view)` for a view living in ANY personal stratum, else None.
154
-
155
- ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
156
- view means locating the OWNER's record inside this topic's bucket. Personal strata
157
- only β€” the `__shared__` bucket has its own read path (`shared_views`), and serving one
158
- view from two finders is how two copies drift."""
159
- vid = str(view_id or '').strip()
160
- if not vid:
161
- return None
162
- try:
163
- data = self._st.get(self.table_key) or {}
164
- except Exception:
165
- return None
166
- for username, ws in data.items():
167
- if username == SHARED_KEY or not isinstance(ws, dict):
168
- continue
169
- v = (ws.get('views') or {}).get(vid)
170
- if isinstance(v, dict):
171
- return str(username), dict(v)
172
- return None
173
-
174
- def find_folder(self, folder_id):
175
- """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
176
- personal stratum, else None. `find_view`'s sibling, and here for the same reason.
177
-
178
- ⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
179
- `folder` and has since wave 20, but only the VIEW kind was ever projected β€” so "share
180
- this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
181
- Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
182
- owns it, and WHICH views are filed in it. Folder membership lives in the owner's
183
- `itemFolders` map (item id -> folder id), never on the view record, so the views are
184
- found by asking that map rather than by reading a list off the folder.
185
-
186
- Views only (`folders['views']`): the cohort surface has its own store and its own
187
- sharing question, and answering both here would make one function mean two things.
188
- """
189
- fid = str(folder_id or '').strip()
190
- if not fid:
191
- return None
192
- try:
193
- data = self._st.get(self.table_key) or {}
194
- except Exception:
195
- return None
196
- for username, ws in data.items():
197
- if username == SHARED_KEY or not isinstance(ws, dict):
198
- continue
199
- rows = (ws.get('folders') or {}).get('views') or []
200
- hit = next((f for f in rows
201
- if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
202
- if not hit:
203
- continue
204
- # ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) β€”
205
- # reading item ids off the top level finds the surface names instead and matches
206
- # nothing, so the projection silently returns an EMPTY folder and the feature looks
207
- # exactly as broken as it was before the fix. Caught by this change's own gate,
208
- # which is the entire argument for writing one.
209
- placed = (ws.get('itemFolders') or {}).get('views') or {}
210
- views = ws.get('views') or {}
211
- inside = {str(vid): dict(v) for vid, v in views.items()
212
- if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
213
- return str(username), dict(hit), inside
214
- return None
215
-
216
- # ---------------------------------------------------------------- read
217
- def workspace(self, username, consume_corrections=True):
218
- """One user's durable workspace: always the full three-strata shape."""
219
- try:
220
- data = self._st.get(self.table_key) or {}
221
- ws = data.get(username, {}) or {}
222
- # A collision acknowledgement is protocol state, not part of a field definition.
223
- # Consume it with the first fresh workspace payload after the correcting write,
224
- # then splice a bounded copy into that payload only. Keeping it out of `fields`
225
- # prevents an old request id surviving forever and overriding a later rename.
226
- corrections = {}
227
- if consume_corrections and ws.get('fieldCorrections'):
228
- def _take(current):
229
- current_ws = current.get(username) or {}
230
- pending = current_ws.get('fieldCorrections') or {}
231
- corrections.update({
232
- str(key)[:80]: dict(value)
233
- for key, value in pending.items()
234
- if isinstance(value, dict)
235
- })
236
- current_ws.pop('fieldCorrections', None)
237
- return current
238
-
239
- data = self._st.update(self.table_key, _take, flush='async')
240
- ws = (data or {}).get(username, {}) or {}
241
- except Exception:
242
- ws = {}
243
- corrections = {}
244
- fields = {
245
- key: dict(value) if isinstance(value, dict) else value
246
- for key, value in (ws.get('fields') or {}).items()
247
- }
248
- for key, ack in corrections.items():
249
- field = fields.get(key)
250
- accepted_label = str(ack.get('label') or '')[:120]
251
- requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
252
- correction_id = str(ack.get('labelCorrectionId') or '')[:180]
253
- # A newer field write clears/replaces the pending ack in the SAME transaction.
254
- # The label check is an extra belt against ever attaching a stale ack to a newer
255
- # definition if a future store implementation weakens that ordering.
256
- if (isinstance(field, dict) and accepted_label
257
- and str(field.get('label') or '') == accepted_label
258
- and requested_label and correction_id):
259
- field['labelCorrectedFrom'] = requested_label
260
- field['labelCorrectionId'] = correction_id
261
- out = {
262
- 'views': dict(ws.get('views') or {}),
263
- 'fields': fields,
264
- 'overlays': dict(ws.get('overlays') or {}),
265
- # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
266
- # stratum rather than a key on each item β€” see aios_grid.clean_folders for why
267
- # (a cohort lives in another store, and filing is an organising act, not part of
268
- # what a view is). Absent for every workspace saved before this wave, which is
269
- # exactly "no folders yet".
270
- 'folders': dict(ws.get('folders') or {}),
271
- 'itemFolders': dict(ws.get('itemFolders') or {}),
272
- }
273
- # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
274
- # client's localStorage copy wins when present; this is the server's answer for a
275
- # fresh profile, which used to fall all the way to the system default view.
276
- if ws.get('activeViewId'):
277
- out['activeViewId'] = str(ws['activeViewId'])
278
- # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
279
- # stratum, absent until the user first reorders β€” exactly "default order".
280
- if isinstance(ws.get('recordLayout'), dict):
281
- out['recordLayout'] = dict(ws['recordLayout'])
282
- return out
283
-
284
- # ---------------------------------------------------------------- write
285
- def _update(self, username, change):
286
- def _up(data):
287
- ws = data.setdefault(username, {})
288
- ws.setdefault('views', {})
289
- ws.setdefault('fields', {})
290
- ws.setdefault('overlays', {})
291
- ws.setdefault('folders', {})
292
- ws.setdefault('itemFolders', {})
293
- change(ws)
294
- return data
295
- # flush='async' (wave-7 W3): this is THE hot path β€” every autosaved filter tweak,
296
- # column note and typed overlay cell lands here inside the component round-trip, and
297
- # the historical synchronous hub commit cost seconds per edit. The mutation applies to
298
- # the in-process cache (read-your-writes for every subsequent render); the hub write
299
- # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
300
- return self._st.update(self.table_key, _up, flush='async')
301
-
302
- def rename_choice_values(self, username, change):
303
- """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
304
-
305
- ⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
306
- public "do anything to the workspace" method is an invitation to put write logic in
307
- callers instead of here, and every OTHER method on this class exists precisely because
308
- that logic belongs in one place. Renaming a choice is the one operation that must touch
309
- three strata AT ONCE β€” the field's `choices`, the cells in `overlays`, and the views that
310
- filter or colour by the old value β€” inside a SINGLE transaction, because a rename that
311
- updated the cells and not the filters would leave a saved view matching nothing.
312
-
313
- `change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
314
- """
315
- return self._update(username, change)
316
-
317
- def save_active_view(self, username, view_id):
318
- """Remember which view this user last opened (owner item 3, 2026-07-31).
319
-
320
- Presentation state, not authorisation: the READ side re-validates the id against what
321
- the caller may actually see, so a stale or foreign id degrades to the default view
322
- rather than granting anything. Stored per user like every other stratum.
323
- """
324
- vid = str(view_id or '').strip()[:120]
325
- if not vid or username == SHARED_KEY:
326
- return
327
-
328
- def _set(ws):
329
- ws['activeViewId'] = vid
330
- self._update(username, _set)
331
-
332
- def save_record_layout(self, username, order):
333
- """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
334
-
335
- Presentation state for ONE surface β€” the record modal. Deliberately not view config:
336
- the owner's ask is per-user, not per-view, and it must never reorder grid columns.
337
- The event handler validated keys against the live field set; the wire re-validates at
338
- serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
339
- An empty order clears the stratum back to "default order".
340
- """
341
- if username == SHARED_KEY:
342
- return
343
- clean, seen = [], set()
344
- for key in (order or [])[:200]:
345
- key = str(key or '').strip()[:80]
346
- if key and key not in seen:
347
- seen.add(key)
348
- clean.append(key)
349
-
350
- def _set(ws):
351
- if clean:
352
- ws['recordLayout'] = {'order': clean}
353
- else:
354
- ws.pop('recordLayout', None)
355
-
356
- self._update(username, _set)
357
-
358
- def save_folders(self, username, folders, item_folders):
359
- """Replace the folder stratum wholesale (wave-8 I11).
360
-
361
- Wholesale rather than per-folder because the caller has ALREADY validated the complete
362
- picture through aios_grid.clean_folders / clean_item_folders, and those two are
363
- interdependent: a placement is only legal while its folder exists, so committing them
364
- separately would leave a window where a reader sees an item filed into a folder that is
365
- not there yet. One write, one consistent state.
366
- """
367
- def _set(ws):
368
- ws['folders'] = dict(folders or {})
369
- ws['itemFolders'] = dict(item_folders or {})
370
- self._update(username, _set)
371
-
372
- def save_view_order(self, username, order):
373
- """⭐ WAVE-27 item 5 (contract C7) β€” this user's own ORDER for the views rail.
374
-
375
- Wholesale, like `save_folders` above and for the same reason: the client sends the full
376
- list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
377
- view went.
378
-
379
- β›” PER USER, and it belongs in this stratum rather than on the view records themselves.
380
- `aios_grid`'s own folder note argues it out for placements and every word applies: an
381
- arrangement is a per-user ORGANISING act, not part of what a view IS β€” so keeping it out
382
- of the view config means duplicating, sharing or exporting a view does not drag one
383
- person's rail position along with it. It also means a SHARED view can sit in a different
384
- place for each person who can see it, which is the only coherent answer once two people
385
- share one view.
386
-
387
- An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
388
- """
389
- def _set(ws):
390
- clean = []
391
- seen = set()
392
- for vid in (order or []):
393
- vid = str(vid).strip()[:120]
394
- if vid and vid not in seen:
395
- seen.add(vid)
396
- clean.append(vid)
397
- if clean:
398
- ws['viewOrder'] = clean
399
- else:
400
- ws.pop('viewOrder', None)
401
- self._update(username, _set)
402
-
403
- def shared_views(self, viewer, is_admin=False):
404
- """Every SHARED view this viewer may see, by id (wave-9 I17).
405
-
406
- Read-only and independent of the viewer's own workspace: the caller merges. Returns
407
- only what `_may_see` allows, so a caller cannot accidentally render somebody else's
408
- personal view by forgetting to filter.
409
- """
410
- try:
411
- bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
412
- except Exception:
413
- return {}
414
- return {vid: dict(v) for vid, v in bucket.items()
415
- if _may_see(v, viewer, is_admin)}
416
-
417
- def shared_view(self, view_id):
418
- """One shared view RAW β€” no visibility filter. For authorisation decisions only: a
419
- caller must know a view exists and who owns it before it can decide whether the actor
420
- may touch it. Never hand the result to a renderer without checking `_may_see`."""
421
- try:
422
- return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
423
- ).get('views', {}).get(str(view_id))
424
- except Exception:
425
- return None
426
-
427
- def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
428
- """Upsert a SavedView into its ONE home β€” personal workspace or the shared bucket.
429
-
430
- `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
431
- it lands in, the view is REMOVED from the other, so a view can never exist as two
432
- copies that diverge on the next edit.
433
-
434
- ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called β€” this layer
435
- moves data and does not know who is asking. `_cl_handle_one` is the wall.
436
- """
437
- view_id = str((view or {}).get('id') or '').strip()
438
- if not view_id:
439
- raise ValueError('view id is required')
440
- if username == SHARED_KEY:
441
- raise ValueError('reserved username')
442
- to_shared = is_shared(view) if shared is None else bool(shared)
443
- requested = dict(view)
444
- accepted = {}
445
-
446
- def _up(data):
447
- # View names are tenant-global: every personal workspace plus the shared bucket.
448
- # This deliberately includes views the actor cannot see. The only disclosed fact
449
- # is that a display name is already taken, while the categorical "no duplicate
450
- # view names" contract remains true when a personal view is later shared.
451
- names = list(reserved_names or ())
452
- for workspace in data.values():
453
- if not isinstance(workspace, dict):
454
- continue
455
- names.extend(
456
- value.get('name')
457
- for candidate_id, value in (workspace.get('views') or {}).items()
458
- if candidate_id != view_id and isinstance(value, dict)
459
- )
460
- payload = dict(requested)
461
- payload['name'] = _unique_name(payload.get('name'), names)
462
- accepted.clear()
463
- accepted.update(payload)
464
- if to_shared:
465
- bucket = data.setdefault(SHARED_KEY, {})
466
- bucket.setdefault('views', {})[view_id] = payload
467
- # it may have lived in the creator's workspace before being shared
468
- owner = data.get(payload.get('createdBy') or username) or {}
469
- (owner.get('views') or {}).pop(view_id, None)
470
- else:
471
- ws = data.setdefault(username, {})
472
- ws.setdefault('views', {})[view_id] = payload
473
- (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
474
- return data
475
-
476
- self._st.update(self.table_key, _up, flush='async')
477
- return dict(accepted)
478
-
479
- def delete_view(self, username, view_id):
480
- """Delete a custom/list view override. The system all-rows view is guarded by caller.
481
-
482
- Removes from BOTH homes: the caller has already authorised the delete, and leaving a
483
- stale copy in the other bucket would resurrect the view on the next read.
484
- """
485
- vid = str(view_id)
486
-
487
- def _up(data):
488
- (data.get(username, {}).get('views') or {}).pop(vid, None)
489
- (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
490
- return data
491
-
492
- self._st.update(self.table_key, _up, flush='async')
493
-
494
- def save_field(self, username, field, reserved_names=(), correction_id=None):
495
- """Persist a column note or a user-created (custom_/measure_) field definition."""
496
- key = str((field or {}).get('key') or '').strip()
497
- if not key:
498
- raise ValueError('field key is required')
499
- requested = dict(field)
500
- accepted = {}
501
-
502
- def _save(ws):
503
- names = list(reserved_names or ())
504
- names.extend(
505
- value.get('label')
506
- for candidate_key, value in (ws.get('fields') or {}).items()
507
- if candidate_key != key and isinstance(value, dict)
508
- )
509
- payload = dict(requested)
510
- payload.pop('labelCorrectedFrom', None)
511
- payload.pop('labelCorrectionId', None)
512
- requested_label = ' '.join(
513
- str(payload.get('label') or 'Untitled').split())[:120].rstrip()
514
- payload['label'] = _unique_name(requested_label, names)
515
- corrections = ws.setdefault('fieldCorrections', {})
516
- corrections.pop(key, None)
517
- if payload['label'] != requested_label and correction_id:
518
- corrections[key] = {
519
- 'label': payload['label'],
520
- 'labelCorrectedFrom': requested_label,
521
- 'labelCorrectionId': str(correction_id)[:180],
522
- }
523
- if not corrections:
524
- ws.pop('fieldCorrections', None)
525
- accepted.clear()
526
- accepted.update(payload)
527
- if source_override_is_empty(payload):
528
- ws['fields'].pop(key, None)
529
- else:
530
- ws['fields'][key] = payload
531
-
532
- self._update(username, _save)
533
- return dict(accepted)
534
-
535
- def duplicate_field(self, username, source_key, new_key, field,
536
- reserved_names=(), correction_id=None):
537
- """Clone a user-created field in ONE store transaction (wave-5 item 1): the new
538
- definition plus β€” for `custom_` overlay sources only β€” every stored cell value under
539
- the source key. One transaction, because a def without its values (or values without a
540
- def) is exactly the orphan state delete_field exists to prevent, in reverse.
541
- The caller validated both keys (same created stratum) and stamped the clone's
542
- createdBy; this layer only moves data."""
543
- source_key = str(source_key or '').strip()
544
- new_key = str(new_key or '').strip()
545
- if not source_key or not new_key or source_key == new_key:
546
- raise ValueError('duplicate_field needs two distinct keys')
547
- requested = dict(field)
548
- accepted = {}
549
-
550
- def _dup(ws):
551
- names = list(reserved_names or ())
552
- names.extend(
553
- value.get('label')
554
- for candidate_key, value in (ws.get('fields') or {}).items()
555
- if candidate_key != new_key and isinstance(value, dict)
556
- )
557
- payload = dict(requested)
558
- payload.pop('labelCorrectedFrom', None)
559
- payload.pop('labelCorrectionId', None)
560
- requested_label = ' '.join(
561
- str(payload.get('label') or 'Untitled').split())[:120].rstrip()
562
- payload['label'] = _unique_name(requested_label, names)
563
- corrections = ws.setdefault('fieldCorrections', {})
564
- corrections.pop(new_key, None)
565
- if payload['label'] != requested_label and correction_id:
566
- corrections[new_key] = {
567
- 'label': payload['label'],
568
- 'labelCorrectedFrom': requested_label,
569
- 'labelCorrectionId': str(correction_id)[:180],
570
- }
571
- if not corrections:
572
- ws.pop('fieldCorrections', None)
573
- accepted.clear()
574
- accepted.update(payload)
575
- ws['fields'][new_key] = payload
576
- if source_key.startswith('custom_'):
577
- for row in ws['overlays'].values():
578
- if isinstance(row, dict) and source_key in row:
579
- row[new_key] = row[source_key]
580
-
581
- self._update(username, _dup)
582
- return dict(accepted)
583
-
584
- def delete_field(self, username, key):
585
- """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
586
-
587
- Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
588
- columns β€” the caller enforces the prefix). The stored overlay VALUES for the key are
589
- scrubbed with it: a deleted column's cells must not linger as orphan data that would
590
- silently resurface if the key were ever reused. Views referencing the key self-heal on
591
- their next autosave (an unknown colId is dropped) β€” the rule every stale key rides.
592
- """
593
- key = str(key or '').strip()
594
- if not key:
595
- return
596
-
597
- def _drop(ws):
598
- ws['fields'].pop(key, None)
599
- for row in ws['overlays'].values():
600
- if isinstance(row, dict):
601
- row.pop(key, None)
602
-
603
- self._update(username, _drop)
604
-
605
- def patch_overlay(self, username, pid, updates):
606
- """Patch only the external editable stratum; never writes to the source system."""
607
- clean = dict(updates or {})
608
- if not clean:
609
- return
610
-
611
- def _patch(ws):
612
- ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
613
-
614
- self._update(username, _patch)
615
-
616
-
617
- def make(table_key, st=None):
618
- return TableStore(table_key, st=st)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The generic per-user TABLE WORKSPACE store β€” the persistence half of the table-page factory.
2
+
3
+ One durable store key holds one table OBJECT's per-user Airtable-style state:
4
+
5
+ {username: {'views': {view_id: SavedView},
6
+ 'fields': {field_key: Field}, # notes + custom_ + measure_ strata
7
+ 'overlays': {str(pid): {field_key: value}}}}
8
+
9
+ `make(table_key)` returns the six operations a table page's host loop needs, closed over that
10
+ key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
11
+ these exact functions β€” the logic MOVED here 2026-07-27 so that duplicating the Customer table
12
+ pattern to a new object is a registry row + a config, not a copy of the store plumbing
13
+ (owner directive: the table-page factory).
14
+
15
+ A LIST (membership/formula semantics) is deliberately a different store from a VIEW
16
+ (presentation/query state) β€” see modules/customer_data.py's customer_lists key.
17
+ """
18
+ import core.store as store
19
+
20
+ #: Wave-9 I17 β€” the SHARED bucket. Views whose permissions make them visible to anyone but
21
+ #: their creator live here instead of in a personal workspace, under a key that cannot collide
22
+ #: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
23
+ #: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
24
+ #: here, and a view shared is removed from its creator's workspace. Two homes would mean two
25
+ #: divergent copies the moment either was edited.
26
+ SHARED_KEY = '__shared__'
27
+
28
+ #: ⭐⭐ W36-T25 β€” THE COLUMN SUMMARY IS THE DATABASE'S, NOT ONE ACCOUNT'S.
29
+ #:
30
+ #: Owner item 1, second half, verbatim: *"When i filter Avi on my computer sum works for the Sales
31
+ #: last 365 days column but not when my colleague filter it β€” The sum of the field is not showing
32
+ #: at the bottom even when i zoom out. It only works on my screen??"* Measured while scouting it:
33
+ #: `save_field` writes the WHOLE field payload into `data[username]`, so a column summary β€” which
34
+ #: is a statement about the COLUMN, identical for every reader by construction β€” was stored once
35
+ #: per account. Set it, and the totals row exists for you and for nobody else. `computeAggs` then
36
+ #: paints no totals row at all for the colleague (`showTotals` is false when no field carries an
37
+ #: `agg`), which is exactly "not showing at the bottom".
38
+ #:
39
+ #: β›” ONLY `agg` MOVES, AND THE LINE IS NOT ARBITRARY. Width, column order, the note, the display
40
+ #: format and every `custom_`/`measure_` definition stay per user, because each of those is a
41
+ #: statement about how ONE PERSON reads the column. "Sum this column" is a statement about what
42
+ #: the column MEANS, and two accounts disagreeing about it is the defect, not a preference.
43
+ #:
44
+ #: ⚠ IT LIVES IN THE `__shared__` MEMBER OF THIS SAME BUCKET rather than in
45
+ #: `core/shared_overlay.py`'s `<key>__shared` document, and the reason is transactional: a summary
46
+ #: is written by the same `save_field` call that writes the note beside it, and the two must land
47
+ #: or fail together. `__shared__` is already this store's tenant-wide member (shared VIEWS live
48
+ #: there) and is guarded against colliding with a username by `is_shared` and by `core/users.py`'s
49
+ #: never-dunder rule, so there is no second bucket, no second flush and no second failure mode.
50
+ SHARED_FIELD_KEYS = ('agg',)
51
+
52
+
53
+ def _may_see(view, viewer, is_admin=False):
54
+ """Visibility for ONE shared view, fail-closed.
55
+
56
+ 'collaborative' = everyone who can already open the module (the caller has gated that).
57
+ 'users' = the named users, plus the creator, plus admins β€” an admin who could not
58
+ see a view could not administer it either.
59
+ Anything unrecognised returns False rather than defaulting open: an unreadable permission
60
+ must never widen access ([[aios-permissioning]] β€” no fail-open defaults).
61
+ """
62
+ if not isinstance(view, dict):
63
+ return False
64
+ if view.get('createdBy') == viewer or is_admin:
65
+ return True
66
+ perms = view.get('permissions') or {}
67
+ edit = perms.get('edit')
68
+ if edit == 'collaborative':
69
+ return True
70
+ if edit == 'users':
71
+ return viewer in set(perms.get('users') or ())
72
+ return False # 'personal', absent, or junk
73
+
74
+
75
+ def _may_edit(view, viewer, is_admin=False):
76
+ """Who may WRITE a shared view. Same set as visibility today β€” the owner's item asks 'who
77
+ can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
78
+ separate function so they can diverge (a future read-only share) without hunting callers."""
79
+ return _may_see(view, viewer, is_admin)
80
+
81
+
82
+ def _may_administer(view, viewer, is_admin=False):
83
+ """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
84
+
85
+ Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
86
+ could grant themselves sole ownership of somebody else's view, or quietly widen a
87
+ users-scoped view to everyone β€” the classic privilege-escalation-by-edit hole.
88
+ """
89
+ if not isinstance(view, dict):
90
+ return False
91
+ return bool(is_admin) or view.get('createdBy') == viewer
92
+
93
+
94
+ def is_shared(view):
95
+ """A view belongs in the shared bucket when its permissions reach beyond its creator."""
96
+ return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
97
+
98
+
99
+ def _shared_fields(data):
100
+ """The tenant-wide field stratum of one workspace document β€” `{field_key: {'agg': …}}`.
101
+
102
+ ⚠ TOTAL AND FAIL-SOFT: a document written before W36-T25 has no `__shared__` member at all,
103
+ and one written by a future version may have something else in it. Either way this answers an
104
+ empty mapping rather than raising, because the caller is a READ that must still serve the
105
+ workspace β€” a column with no summary is the state every column was in yesterday.
106
+ """
107
+ shared = (data or {}).get(SHARED_KEY)
108
+ fields = (shared or {}).get('fields') if isinstance(shared, dict) else None
109
+ return fields if isinstance(fields, dict) else {}
110
+
111
+
112
+ def split_shared(payload):
113
+ """`(per_user, shared)` β€” one field payload divided at the strata boundary.
114
+
115
+ ⭐ W36-T25. `shared` carries only `SHARED_FIELD_KEYS` that are actually SET; `per_user` is the
116
+ payload without them. Split HERE rather than at each write door so the three doors that store
117
+ a field (`save_field`, `duplicate_field`, and the delete that must clear it) cannot come apart
118
+ about where a summary lives β€” which is the whole reason this module exists rather than being
119
+ copied per topic.
120
+ """
121
+ payload = dict(payload or {})
122
+ shared = {}
123
+ for key in SHARED_FIELD_KEYS:
124
+ value = str(payload.pop(key, '') or '').strip()
125
+ if value:
126
+ shared[key] = value
127
+ return payload, shared
128
+
129
+
130
+ def source_override_is_empty(payload):
131
+ """Does this stored definition of a SOURCE (non-custom) column carry any user state?
132
+
133
+ A cleared note on an immutable source field returns to the canonical schema instead of
134
+ leaving a meaningless override row. Custom fields remain even with an empty note β€” and so
135
+ does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format
136
+ override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY.
137
+
138
+ β›” EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently
139
+ stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days`
140
+ built an override whose only content was that summary, so this rule threw the whole row away
141
+ on write while the menu went on reading "Summary: Average" from the client's own optimistic
142
+ copy until the next login β€” a discarded WRITE wearing the face of a failed read
143
+ ([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores
144
+ its definitions through another door entirely, kept it.
145
+
146
+ ⚠ ONE RULE, TWO CALLERS β€” here and `grid_events`' store-less fallback. Two copies of a
147
+ discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]).
148
+ ⚠ A CLEARED summary still drops the row, which is the intent: with nothing else set, the
149
+ column goes back to whatever the contract declares for it.
150
+ """
151
+ payload = payload or {}
152
+ if payload.get('custom'):
153
+ return False
154
+ return (not str(payload.get('note') or '').strip()
155
+ and not isinstance(payload.get('measure'), dict)
156
+ and not isinstance(payload.get('format'), dict)
157
+ and not str(payload.get('agg') or '').strip())
158
+
159
+
160
+ def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
161
+ """Allocate one human-facing name inside a store.update transaction.
162
+
163
+ Keys/ids remain structural identity. Names compare case-insensitively after collapsing
164
+ whitespace, because those variants are indistinguishable in the UI. This helper belongs
165
+ in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
166
+ choose the same free name before either write lands.
167
+ """
168
+ limit = max(1, int(max_len))
169
+
170
+ def _clean(value):
171
+ return ' '.join(str(value or '').split())
172
+
173
+ base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
174
+ taken = {_clean(value).casefold() for value in existing if _clean(value)}
175
+ if base.casefold() not in taken:
176
+ return base
177
+ index = 2
178
+ while True:
179
+ suffix = f' {index}'
180
+ stem = base[:max(0, limit - len(suffix))].rstrip()
181
+ candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
182
+ if candidate.casefold() not in taken:
183
+ return candidate
184
+ index += 1
185
+
186
+
187
+ class TableStore:
188
+ """The six store operations for one table object's workspace, closed over its store key.
189
+
190
+ `st` (wave 18, C3-UT) is the STORE HANDLE β€” anything exposing `get(name)` /
191
+ `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
192
+ zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
193
+ apply the tenant prefix / repo binding β€” which is what makes a user table created by a
194
+ Nurilab admin land in Nurilab's store instead of Royal's.
195
+ """
196
+
197
+ def __init__(self, table_key, st=None):
198
+ self.table_key = table_key
199
+ self._st = st if st is not None else store
200
+
201
+ @property
202
+ def st(self):
203
+ """The bound store handle β€” for SIBLING registries (core/shares) that must read the
204
+ same tenant's buckets this workspace lives in (wave 21, C1)."""
205
+ return self._st
206
+
207
+ def find_view(self, view_id):
208
+ """`(owner_username, view)` for a view living in ANY personal stratum, else None.
209
+
210
+ ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
211
+ view means locating the OWNER's record inside this topic's bucket. Personal strata
212
+ only β€” the `__shared__` bucket has its own read path (`shared_views`), and serving one
213
+ view from two finders is how two copies drift."""
214
+ vid = str(view_id or '').strip()
215
+ if not vid:
216
+ return None
217
+ try:
218
+ data = self._st.get(self.table_key) or {}
219
+ except Exception:
220
+ return None
221
+ for username, ws in data.items():
222
+ if username == SHARED_KEY or not isinstance(ws, dict):
223
+ continue
224
+ v = (ws.get('views') or {}).get(vid)
225
+ if isinstance(v, dict):
226
+ return str(username), dict(v)
227
+ return None
228
+
229
+ def find_folder(self, folder_id):
230
+ """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
231
+ personal stratum, else None. `find_view`'s sibling, and here for the same reason.
232
+
233
+ ⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
234
+ `folder` and has since wave 20, but only the VIEW kind was ever projected β€” so "share
235
+ this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
236
+ Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
237
+ owns it, and WHICH views are filed in it. Folder membership lives in the owner's
238
+ `itemFolders` map (item id -> folder id), never on the view record, so the views are
239
+ found by asking that map rather than by reading a list off the folder.
240
+
241
+ Views only (`folders['views']`): the cohort surface has its own store and its own
242
+ sharing question, and answering both here would make one function mean two things.
243
+ """
244
+ fid = str(folder_id or '').strip()
245
+ if not fid:
246
+ return None
247
+ try:
248
+ data = self._st.get(self.table_key) or {}
249
+ except Exception:
250
+ return None
251
+ for username, ws in data.items():
252
+ if username == SHARED_KEY or not isinstance(ws, dict):
253
+ continue
254
+ rows = (ws.get('folders') or {}).get('views') or []
255
+ hit = next((f for f in rows
256
+ if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
257
+ if not hit:
258
+ continue
259
+ # ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) β€”
260
+ # reading item ids off the top level finds the surface names instead and matches
261
+ # nothing, so the projection silently returns an EMPTY folder and the feature looks
262
+ # exactly as broken as it was before the fix. Caught by this change's own gate,
263
+ # which is the entire argument for writing one.
264
+ placed = (ws.get('itemFolders') or {}).get('views') or {}
265
+ views = ws.get('views') or {}
266
+ inside = {str(vid): dict(v) for vid, v in views.items()
267
+ if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
268
+ return str(username), dict(hit), inside
269
+ return None
270
+
271
+ # ---------------------------------------------------------------- read
272
+ def workspace(self, username, consume_corrections=True):
273
+ """One user's durable workspace: always the full three-strata shape."""
274
+ try:
275
+ data = self._st.get(self.table_key) or {}
276
+ ws = data.get(username, {}) or {}
277
+ # A collision acknowledgement is protocol state, not part of a field definition.
278
+ # Consume it with the first fresh workspace payload after the correcting write,
279
+ # then splice a bounded copy into that payload only. Keeping it out of `fields`
280
+ # prevents an old request id surviving forever and overriding a later rename.
281
+ corrections = {}
282
+ if consume_corrections and ws.get('fieldCorrections'):
283
+ def _take(current):
284
+ current_ws = current.get(username) or {}
285
+ pending = current_ws.get('fieldCorrections') or {}
286
+ corrections.update({
287
+ str(key)[:80]: dict(value)
288
+ for key, value in pending.items()
289
+ if isinstance(value, dict)
290
+ })
291
+ current_ws.pop('fieldCorrections', None)
292
+ return current
293
+
294
+ data = self._st.update(self.table_key, _take, flush='async')
295
+ ws = (data or {}).get(username, {}) or {}
296
+ except Exception:
297
+ data = {}
298
+ ws = {}
299
+ corrections = {}
300
+ fields = {
301
+ key: dict(value) if isinstance(value, dict) else value
302
+ for key, value in (ws.get('fields') or {}).items()
303
+ }
304
+ # ⭐⭐ W36-T25 β€” THE TENANT-WIDE COLUMN SUMMARY, MERGED OVER THIS USER'S STRATUM.
305
+ # β›” IT MUST BE ABLE TO CREATE AN ENTRY, not only decorate one, and that is the whole
306
+ # reason this is a merge rather than a lookup: the colleague who never touched the column
307
+ # has NO per-user record for it, so a decorate-only pass would have left them with exactly
308
+ # the blank totals row the owner reported. `aios_grid.workspace_wire` reads `meta['agg']`
309
+ # off whatever is here, base column or custom one alike.
310
+ for key, shared in _shared_fields(data).items():
311
+ agg = str((shared or {}).get('agg') or '').strip()
312
+ if not agg:
313
+ continue
314
+ entry = fields.get(key)
315
+ fields[key] = {**entry, 'agg': agg} if isinstance(entry, dict) else {'agg': agg}
316
+ for key, ack in corrections.items():
317
+ field = fields.get(key)
318
+ accepted_label = str(ack.get('label') or '')[:120]
319
+ requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
320
+ correction_id = str(ack.get('labelCorrectionId') or '')[:180]
321
+ # A newer field write clears/replaces the pending ack in the SAME transaction.
322
+ # The label check is an extra belt against ever attaching a stale ack to a newer
323
+ # definition if a future store implementation weakens that ordering.
324
+ if (isinstance(field, dict) and accepted_label
325
+ and str(field.get('label') or '') == accepted_label
326
+ and requested_label and correction_id):
327
+ field['labelCorrectedFrom'] = requested_label
328
+ field['labelCorrectionId'] = correction_id
329
+ out = {
330
+ 'views': dict(ws.get('views') or {}),
331
+ 'fields': fields,
332
+ 'overlays': dict(ws.get('overlays') or {}),
333
+ # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
334
+ # stratum rather than a key on each item β€” see aios_grid.clean_folders for why
335
+ # (a cohort lives in another store, and filing is an organising act, not part of
336
+ # what a view is). Absent for every workspace saved before this wave, which is
337
+ # exactly "no folders yet".
338
+ 'folders': dict(ws.get('folders') or {}),
339
+ 'itemFolders': dict(ws.get('itemFolders') or {}),
340
+ }
341
+ # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
342
+ # client's localStorage copy wins when present; this is the server's answer for a
343
+ # fresh profile, which used to fall all the way to the system default view.
344
+ if ws.get('activeViewId'):
345
+ out['activeViewId'] = str(ws['activeViewId'])
346
+ # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
347
+ # stratum, absent until the user first reorders β€” exactly "default order".
348
+ if isinstance(ws.get('recordLayout'), dict):
349
+ out['recordLayout'] = dict(ws['recordLayout'])
350
+ return out
351
+
352
+ # ---------------------------------------------------------------- write
353
+ def _update(self, username, change, shared=None):
354
+ """Apply `change(ws)` to this user's stratum, and `shared(shared_fields)` to the
355
+ tenant-wide one, in ONE transaction.
356
+
357
+ ⭐ W36-T25 β€” `shared` IS A SECOND CALLBACK RATHER THAN A SECOND `update`, and that is the
358
+ whole reason it exists here instead of at the caller. `save_field` writes a note (per
359
+ user) and a column summary (tenant-wide) from ONE payload; two transactions would let the
360
+ summary land while the note did not, and the store's own commit is asynchronous, so the
361
+ window is real rather than theoretical. One `update`, one flush, one failure mode.
362
+ """
363
+ def _up(data):
364
+ ws = data.setdefault(username, {})
365
+ ws.setdefault('views', {})
366
+ ws.setdefault('fields', {})
367
+ ws.setdefault('overlays', {})
368
+ ws.setdefault('folders', {})
369
+ ws.setdefault('itemFolders', {})
370
+ change(ws)
371
+ if shared is not None:
372
+ shared(data.setdefault(SHARED_KEY, {}).setdefault('fields', {}))
373
+ return data
374
+ # flush='async' (wave-7 W3): this is THE hot path β€” every autosaved filter tweak,
375
+ # column note and typed overlay cell lands here inside the component round-trip, and
376
+ # the historical synchronous hub commit cost seconds per edit. The mutation applies to
377
+ # the in-process cache (read-your-writes for every subsequent render); the hub write
378
+ # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
379
+ return self._st.update(self.table_key, _up, flush='async')
380
+
381
+ def rename_choice_values(self, username, change):
382
+ """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
383
+
384
+ ⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
385
+ public "do anything to the workspace" method is an invitation to put write logic in
386
+ callers instead of here, and every OTHER method on this class exists precisely because
387
+ that logic belongs in one place. Renaming a choice is the one operation that must touch
388
+ three strata AT ONCE β€” the field's `choices`, the cells in `overlays`, and the views that
389
+ filter or colour by the old value β€” inside a SINGLE transaction, because a rename that
390
+ updated the cells and not the filters would leave a saved view matching nothing.
391
+
392
+ `change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
393
+ """
394
+ return self._update(username, change)
395
+
396
+ def save_active_view(self, username, view_id):
397
+ """Remember which view this user last opened (owner item 3, 2026-07-31).
398
+
399
+ Presentation state, not authorisation: the READ side re-validates the id against what
400
+ the caller may actually see, so a stale or foreign id degrades to the default view
401
+ rather than granting anything. Stored per user like every other stratum.
402
+ """
403
+ vid = str(view_id or '').strip()[:120]
404
+ if not vid or username == SHARED_KEY:
405
+ return
406
+
407
+ def _set(ws):
408
+ ws['activeViewId'] = vid
409
+ self._update(username, _set)
410
+
411
+ def save_record_layout(self, username, order):
412
+ """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
413
+
414
+ Presentation state for ONE surface β€” the record modal. Deliberately not view config:
415
+ the owner's ask is per-user, not per-view, and it must never reorder grid columns.
416
+ The event handler validated keys against the live field set; the wire re-validates at
417
+ serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
418
+ An empty order clears the stratum back to "default order".
419
+ """
420
+ if username == SHARED_KEY:
421
+ return
422
+ clean, seen = [], set()
423
+ for key in (order or [])[:200]:
424
+ key = str(key or '').strip()[:80]
425
+ if key and key not in seen:
426
+ seen.add(key)
427
+ clean.append(key)
428
+
429
+ def _set(ws):
430
+ if clean:
431
+ ws['recordLayout'] = {'order': clean}
432
+ else:
433
+ ws.pop('recordLayout', None)
434
+
435
+ self._update(username, _set)
436
+
437
+ def save_folders(self, username, folders, item_folders):
438
+ """Replace the folder stratum wholesale (wave-8 I11).
439
+
440
+ Wholesale rather than per-folder because the caller has ALREADY validated the complete
441
+ picture through aios_grid.clean_folders / clean_item_folders, and those two are
442
+ interdependent: a placement is only legal while its folder exists, so committing them
443
+ separately would leave a window where a reader sees an item filed into a folder that is
444
+ not there yet. One write, one consistent state.
445
+ """
446
+ def _set(ws):
447
+ ws['folders'] = dict(folders or {})
448
+ ws['itemFolders'] = dict(item_folders or {})
449
+ self._update(username, _set)
450
+
451
+ def save_view_order(self, username, order):
452
+ """⭐ WAVE-27 item 5 (contract C7) β€” this user's own ORDER for the views rail.
453
+
454
+ Wholesale, like `save_folders` above and for the same reason: the client sends the full
455
+ list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
456
+ view went.
457
+
458
+ β›” PER USER, and it belongs in this stratum rather than on the view records themselves.
459
+ `aios_grid`'s own folder note argues it out for placements and every word applies: an
460
+ arrangement is a per-user ORGANISING act, not part of what a view IS β€” so keeping it out
461
+ of the view config means duplicating, sharing or exporting a view does not drag one
462
+ person's rail position along with it. It also means a SHARED view can sit in a different
463
+ place for each person who can see it, which is the only coherent answer once two people
464
+ share one view.
465
+
466
+ An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
467
+ """
468
+ def _set(ws):
469
+ clean = []
470
+ seen = set()
471
+ for vid in (order or []):
472
+ vid = str(vid).strip()[:120]
473
+ if vid and vid not in seen:
474
+ seen.add(vid)
475
+ clean.append(vid)
476
+ if clean:
477
+ ws['viewOrder'] = clean
478
+ else:
479
+ ws.pop('viewOrder', None)
480
+ self._update(username, _set)
481
+
482
+ def shared_views(self, viewer, is_admin=False):
483
+ """Every SHARED view this viewer may see, by id (wave-9 I17).
484
+
485
+ Read-only and independent of the viewer's own workspace: the caller merges. Returns
486
+ only what `_may_see` allows, so a caller cannot accidentally render somebody else's
487
+ personal view by forgetting to filter.
488
+ """
489
+ try:
490
+ bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
491
+ except Exception:
492
+ return {}
493
+ return {vid: dict(v) for vid, v in bucket.items()
494
+ if _may_see(v, viewer, is_admin)}
495
+
496
+ def shared_view(self, view_id):
497
+ """One shared view RAW β€” no visibility filter. For authorisation decisions only: a
498
+ caller must know a view exists and who owns it before it can decide whether the actor
499
+ may touch it. Never hand the result to a renderer without checking `_may_see`."""
500
+ try:
501
+ return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
502
+ ).get('views', {}).get(str(view_id))
503
+ except Exception:
504
+ return None
505
+
506
+ def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
507
+ """Upsert a SavedView into its ONE home β€” personal workspace or the shared bucket.
508
+
509
+ `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
510
+ it lands in, the view is REMOVED from the other, so a view can never exist as two
511
+ copies that diverge on the next edit.
512
+
513
+ ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called β€” this layer
514
+ moves data and does not know who is asking. `_cl_handle_one` is the wall.
515
+ """
516
+ view_id = str((view or {}).get('id') or '').strip()
517
+ if not view_id:
518
+ raise ValueError('view id is required')
519
+ if username == SHARED_KEY:
520
+ raise ValueError('reserved username')
521
+ to_shared = is_shared(view) if shared is None else bool(shared)
522
+ requested = dict(view)
523
+ accepted = {}
524
+
525
+ def _up(data):
526
+ # View names are tenant-global: every personal workspace plus the shared bucket.
527
+ # This deliberately includes views the actor cannot see. The only disclosed fact
528
+ # is that a display name is already taken, while the categorical "no duplicate
529
+ # view names" contract remains true when a personal view is later shared.
530
+ names = list(reserved_names or ())
531
+ for workspace in data.values():
532
+ if not isinstance(workspace, dict):
533
+ continue
534
+ names.extend(
535
+ value.get('name')
536
+ for candidate_id, value in (workspace.get('views') or {}).items()
537
+ if candidate_id != view_id and isinstance(value, dict)
538
+ )
539
+ payload = dict(requested)
540
+ payload['name'] = _unique_name(payload.get('name'), names)
541
+ accepted.clear()
542
+ accepted.update(payload)
543
+ if to_shared:
544
+ bucket = data.setdefault(SHARED_KEY, {})
545
+ bucket.setdefault('views', {})[view_id] = payload
546
+ # it may have lived in the creator's workspace before being shared
547
+ owner = data.get(payload.get('createdBy') or username) or {}
548
+ (owner.get('views') or {}).pop(view_id, None)
549
+ else:
550
+ ws = data.setdefault(username, {})
551
+ ws.setdefault('views', {})[view_id] = payload
552
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
553
+ return data
554
+
555
+ self._st.update(self.table_key, _up, flush='async')
556
+ return dict(accepted)
557
+
558
+ def delete_view(self, username, view_id):
559
+ """Delete a custom/list view override. The system all-rows view is guarded by caller.
560
+
561
+ Removes from BOTH homes: the caller has already authorised the delete, and leaving a
562
+ stale copy in the other bucket would resurrect the view on the next read.
563
+ """
564
+ vid = str(view_id)
565
+
566
+ def _up(data):
567
+ (data.get(username, {}).get('views') or {}).pop(vid, None)
568
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
569
+ return data
570
+
571
+ self._st.update(self.table_key, _up, flush='async')
572
+
573
+ def save_field(self, username, field, reserved_names=(), correction_id=None):
574
+ """Persist a column note or a user-created (custom_/measure_) field definition.
575
+
576
+ ⭐⭐ W36-T25 β€” THE COLUMN SUMMARY GOES TO THE TENANT-WIDE STRATUM, EVERYTHING ELSE STAYS
577
+ PER USER, in ONE transaction. See `SHARED_FIELD_KEYS` above for the owner's report and for
578
+ why the line falls where it does. The returned `accepted` still carries the summary: the
579
+ caller is echoing back the field it just stored, and dropping a key from that echo would
580
+ tell the client its write was refused ([[read-path-cannot-witness-write-path]]).
581
+ """
582
+ key = str((field or {}).get('key') or '').strip()
583
+ if not key:
584
+ raise ValueError('field key is required')
585
+ requested = dict(field)
586
+ accepted = {}
587
+
588
+ def _save(ws):
589
+ names = list(reserved_names or ())
590
+ names.extend(
591
+ value.get('label')
592
+ for candidate_key, value in (ws.get('fields') or {}).items()
593
+ if candidate_key != key and isinstance(value, dict)
594
+ )
595
+ payload = dict(requested)
596
+ payload.pop('labelCorrectedFrom', None)
597
+ payload.pop('labelCorrectionId', None)
598
+ requested_label = ' '.join(
599
+ str(payload.get('label') or 'Untitled').split())[:120].rstrip()
600
+ payload['label'] = _unique_name(requested_label, names)
601
+ corrections = ws.setdefault('fieldCorrections', {})
602
+ corrections.pop(key, None)
603
+ if payload['label'] != requested_label and correction_id:
604
+ corrections[key] = {
605
+ 'label': payload['label'],
606
+ 'labelCorrectedFrom': requested_label,
607
+ 'labelCorrectionId': str(correction_id)[:180],
608
+ }
609
+ if not corrections:
610
+ ws.pop('fieldCorrections', None)
611
+ accepted.clear()
612
+ accepted.update(payload)
613
+ # ⭐⭐ W36-T25 β€” SPLIT AFTER the label allocation and the correction bookkeeping, so
614
+ # both still see the whole payload, and BEFORE the per-user write.
615
+ mine, shared_now = split_shared(payload)
616
+ shared_write.clear()
617
+ shared_write.update(shared_now)
618
+ # ⚠ EMPTINESS IS JUDGED ON THE PER-USER HALF. A source column whose ONLY state was a
619
+ # summary now has no per-user state at all, and leaving an `{}` override behind is the
620
+ # meaningless row `source_override_is_empty` exists to prevent.
621
+ if source_override_is_empty(mine):
622
+ ws['fields'].pop(key, None)
623
+ else:
624
+ ws['fields'][key] = mine
625
+
626
+ shared_write = {}
627
+
628
+ def _share(shared_fields):
629
+ # β›” A CLEARED SUMMARY MUST REMOVE THE ROW, not leave an empty one: `workspace` treats
630
+ # any entry it finds as a live tenant-wide summary, so an `{'agg': ''}` husk would be
631
+ # skipped today and become a resurrection hazard the moment the read grows a second
632
+ # shared key. Absence is the only honest spelling of "nobody set one".
633
+ if shared_write:
634
+ shared_fields[key] = {**(shared_fields.get(key) or {}), **shared_write}
635
+ else:
636
+ shared_fields.pop(key, None)
637
+
638
+ self._update(username, _save, shared=_share)
639
+ return dict(accepted)
640
+
641
+ def duplicate_field(self, username, source_key, new_key, field,
642
+ reserved_names=(), correction_id=None):
643
+ """Clone a user-created field in ONE store transaction (wave-5 item 1): the new
644
+ definition plus β€” for `custom_` overlay sources only β€” every stored cell value under
645
+ the source key. One transaction, because a def without its values (or values without a
646
+ def) is exactly the orphan state delete_field exists to prevent, in reverse.
647
+ The caller validated both keys (same created stratum) and stamped the clone's
648
+ createdBy; this layer only moves data."""
649
+ source_key = str(source_key or '').strip()
650
+ new_key = str(new_key or '').strip()
651
+ if not source_key or not new_key or source_key == new_key:
652
+ raise ValueError('duplicate_field needs two distinct keys')
653
+ requested = dict(field)
654
+ accepted = {}
655
+
656
+ def _dup(ws):
657
+ names = list(reserved_names or ())
658
+ names.extend(
659
+ value.get('label')
660
+ for candidate_key, value in (ws.get('fields') or {}).items()
661
+ if candidate_key != new_key and isinstance(value, dict)
662
+ )
663
+ payload = dict(requested)
664
+ payload.pop('labelCorrectedFrom', None)
665
+ payload.pop('labelCorrectionId', None)
666
+ requested_label = ' '.join(
667
+ str(payload.get('label') or 'Untitled').split())[:120].rstrip()
668
+ payload['label'] = _unique_name(requested_label, names)
669
+ corrections = ws.setdefault('fieldCorrections', {})
670
+ corrections.pop(new_key, None)
671
+ if payload['label'] != requested_label and correction_id:
672
+ corrections[new_key] = {
673
+ 'label': payload['label'],
674
+ 'labelCorrectedFrom': requested_label,
675
+ 'labelCorrectionId': str(correction_id)[:180],
676
+ }
677
+ if not corrections:
678
+ ws.pop('fieldCorrections', None)
679
+ accepted.clear()
680
+ accepted.update(payload)
681
+ # ⭐ W36-T25: a CLONE carries the original's summary, and a summary is the database's
682
+ # (see `SHARED_FIELD_KEYS`). Storing it per user here would give the clone a different
683
+ # residency from every other column β€” one door out of three disagreeing about where a
684
+ # thing lives is how `save_field` and this function drift.
685
+ mine, shared_now = split_shared(payload)
686
+ shared_write.clear()
687
+ shared_write.update(shared_now)
688
+ ws['fields'][new_key] = mine
689
+ if source_key.startswith('custom_'):
690
+ for row in ws['overlays'].values():
691
+ if isinstance(row, dict) and source_key in row:
692
+ row[new_key] = row[source_key]
693
+
694
+ shared_write = {}
695
+
696
+ def _share(shared_fields):
697
+ if shared_write:
698
+ shared_fields[new_key] = {**(shared_fields.get(new_key) or {}), **shared_write}
699
+ else:
700
+ shared_fields.pop(new_key, None)
701
+
702
+ self._update(username, _dup, shared=_share)
703
+ return dict(accepted)
704
+
705
+ def delete_field(self, username, key):
706
+ """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
707
+
708
+ Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
709
+ columns β€” the caller enforces the prefix). The stored overlay VALUES for the key are
710
+ scrubbed with it: a deleted column's cells must not linger as orphan data that would
711
+ silently resurface if the key were ever reused. Views referencing the key self-heal on
712
+ their next autosave (an unknown colId is dropped) β€” the rule every stale key rides.
713
+
714
+ ⭐⭐ W36-T25 β€” AND THE TENANT-WIDE SUMMARY GOES WITH IT, for exactly the reason the
715
+ paragraph above gives about cells: an orphan `agg` under a deleted key is state nobody can
716
+ see and nobody can clear, and it would attach itself to the next column that happens to
717
+ take the key back. ⚠ This is the ONE stratum a per-user delete may reach across accounts,
718
+ and it is safe because the summary was never this user's to begin with β€” deleting the
719
+ COLUMN is a tenant-wide act already.
720
+ """
721
+ key = str(key or '').strip()
722
+ if not key:
723
+ return
724
+
725
+ def _drop(ws):
726
+ ws['fields'].pop(key, None)
727
+ for row in ws['overlays'].values():
728
+ if isinstance(row, dict):
729
+ row.pop(key, None)
730
+
731
+ self._update(username, _drop, shared=lambda shared_fields: shared_fields.pop(key, None))
732
+
733
+ def patch_overlay(self, username, pid, updates):
734
+ """Patch only the external editable stratum; never writes to the source system."""
735
+ clean = dict(updates or {})
736
+ if not clean:
737
+ return
738
+
739
+ def _patch(ws):
740
+ ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
741
+
742
+ self._update(username, _patch)
743
+
744
+
745
+ def make(table_key, st=None):
746
+ return TableStore(table_key, st=st)
platform/core/user_tables.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/harness/meta_store.py CHANGED
@@ -1,521 +1,521 @@
1
- """meta_store.py β€” the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
2
-
3
- python platform/harness/meta_store.py --sync pull into tenant #0's mirror
4
- python platform/harness/meta_store.py --sync --tenant gtmlab
5
- python platform/harness/meta_store.py --status what is in the mirror now
6
-
7
- β›” WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
8
- hardwired to XML-RPC β€” it does `import core.odoo as O` and speaks `search_read` β€” so no amount of
9
- spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
10
- per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
11
- delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
12
- store with two loaders, not two stores.**
13
-
14
- ⭐ THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
15
- the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
16
- `XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
17
- `Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
18
- different first hop.
19
-
20
- ⚠ EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
21
- carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
22
- Ad Account 45 Β· Campaign 35 Β· Ad Set 54 Β· Ad 36 Β· Creative 58 Β· Insights 57
23
- β›” AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
24
- came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
25
- nothing going red β€” which is exactly the "do not drop any column" instruction, broken silently.
26
- The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
27
-
28
- ⚠ `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
29
- answers `403 (#200) Requires business_management permission`, which this token does not carry. One
30
- column of 285. R6's second sentence β€” a limit that cannot be removed gets stated with its cause.
31
-
32
- β›” IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
33
- larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
34
- (W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
35
- """
36
- import argparse
37
- import json
38
- import os
39
- import sys
40
- import time
41
- import urllib.error
42
- import urllib.parse
43
- import urllib.request
44
- from pathlib import Path
45
-
46
- _HERE = Path(__file__).resolve().parent
47
- if str(_HERE.parent) not in sys.path:
48
- sys.path.insert(0, str(_HERE.parent))
49
-
50
- from harness import datastore # noqa: E402
51
-
52
- GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
53
- GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
54
-
55
- #: Numeric columns, by name. Everything else is VARCHAR β€” including ids (see the header) and
56
- #: including anything Graph returns as a nested object, which is stored as compact JSON text.
57
- _INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
58
- "inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
59
- "account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
60
- "min_daily_budget", "min_campaign_group_spend_cap"}
61
- _DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
62
- "inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
63
- "daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
64
- "daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
65
- "lifetime_spend_cap", "lifetime_imps"}
66
-
67
- #: ⭐ THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
68
- #: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
69
- #: the grid links in `meta_relational`.
70
- ACCOUNT_FIELDS = (
71
- "account_id account_status age amount_spent balance business_city business_country_code "
72
- "business_name business_state business_street business_street2 business_zip capabilities "
73
- "created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
74
- "has_migrated_permissions id io_number is_attribution_spec_system_default "
75
- "is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
76
- "is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
77
- "name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
78
- "timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
79
- ).split()
80
-
81
- CAMPAIGN_FIELDS = (
82
- "account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
83
- "campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
84
- "created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
85
- "last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
86
- "promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
87
- "special_ad_category special_ad_category_country spend_cap start_time status stop_time "
88
- "topline_id updated_time"
89
- ).split()
90
-
91
- ADSET_FIELDS = (
92
- "account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
93
- "bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
94
- "campaign_active_time campaign_attribution campaign_id configured_status created_time "
95
- "creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
96
- "effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
97
- "issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
98
- "lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
99
- "optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
100
- "rf_prediction_id source_adset source_adset_id start_time status targeting "
101
- "targeting_optimization_types time_based_ad_rotation_id_blocks "
102
- "time_based_ad_rotation_intervals updated_time use_new_app_click"
103
- ).split()
104
-
105
- AD_FIELDS = (
106
- "account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
107
- "adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
108
- "conversion_domain created_time creative demolink_hash display_sequence effective_status "
109
- "engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
110
- "preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
111
- "tracking_and_conversion_with_defaults tracking_specs updated_time"
112
- ).split()
113
-
114
- CREATIVE_FIELDS = (
115
- "account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
116
- "branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
117
- "category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
118
- "destination_set_id dynamic_ad_voice effective_authorization_category "
119
- "effective_instagram_media_id effective_object_story_id enable_direct_install "
120
- "enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
121
- "instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
122
- "link_deep_link_url link_destination_display_url link_og_id link_url "
123
- "messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
124
- "object_type object_url place_page_set_id platform_customizations playable_asset_id "
125
- "portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
126
- "template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
127
- "use_page_actor_override video_id"
128
- ).split()
129
-
130
- INSIGHT_FIELDS = (
131
- "account_currency account_id account_name action_values actions ad_id ad_name adset_id "
132
- "adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
133
- "clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
134
- "cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
135
- "date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
136
- "impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
137
- "optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
138
- "social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
139
- "video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
140
- "video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
141
- "video_play_actions video_thruplay_watched_actions website_purchase_roas"
142
- ).split()
143
-
144
- #: table -> (graph edge on the account | None, measured fields, parent column)
145
- SPECS = {
146
- "meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
147
- "meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
148
- "meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
149
- "meta_ads": ("ads", AD_FIELDS, "adset_id"),
150
- "meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
151
- }
152
-
153
- #: The daily Insights grain (R2). One row per (ad, day) β€” the id is synthesised because Insights
154
- #: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
155
- INSIGHTS_TABLE = "meta_insights_daily"
156
- INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
157
-
158
- #: β›” INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
159
- #: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
160
- #: β€” Graph's way of saying the synchronous query is too heavy (the async report-run API is the
161
- #: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
162
- #: page 25 answer 200. So the window is walked in slices with every column intact:
163
- #: 57 fields Β· ad level Β· 7d Β· limit 25 -> 200, 25 rows
164
- #: 57 fields Β· account level Β· 7d -> 200, 7 rows
165
- #: ⚠ Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over β€”
166
- #: it drops columns R2 requires, and it does so invisibly.
167
- INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
168
- INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
169
- INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
170
-
171
-
172
- def _slices(days, size):
173
- """[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
174
- from datetime import date, timedelta
175
- end = date.today()
176
- start = end - timedelta(days=max(1, days) - 1)
177
- out, cur = [], start
178
- while cur <= end:
179
- stop = min(cur + timedelta(days=max(1, size) - 1), end)
180
- out.append((cur.isoformat(), stop.isoformat()))
181
- cur = stop + timedelta(days=1)
182
- return out
183
-
184
- #: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
185
- #: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
186
- #: not a row cap on a connected source (R6 forbids that) β€” it is a runaway guard, and reaching it
187
- #: is reported as a problem, never absorbed.
188
- MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
189
- PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
190
-
191
-
192
- class MetaError(RuntimeError):
193
- """A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
194
-
195
-
196
- def token():
197
- """The Meta token from the environment or gitignored `platform/.env`; "" when absent.
198
-
199
- ⚠ Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
200
- `platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
201
- (`core` never imports up). Fifteen lines is the price of that boundary.
202
- """
203
- tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
204
- if tok:
205
- return tok.strip()
206
- env = _HERE.parent / ".env"
207
- if env.exists():
208
- for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
209
- if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
210
- _, _, v = line.partition("=")
211
- return v.strip().strip('"').strip("'")
212
- return ""
213
-
214
-
215
- def _get(path, tok, **params):
216
- params["access_token"] = tok
217
- url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
218
- req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
219
- try:
220
- with urllib.request.urlopen(req, timeout=120) as r:
221
- return json.loads(r.read().decode("utf-8", "replace"))
222
- except urllib.error.HTTPError as e:
223
- try:
224
- msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
225
- ).get("message") or ""
226
- except Exception:
227
- msg = ""
228
- raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
229
- except Exception as e:
230
- raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
231
-
232
-
233
- #: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
234
- #: why it cannot be treated as "the server is broken, give up".
235
- _TOO_MUCH = "reduce the amount of data"
236
-
237
- #: β›” THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
238
- #: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
239
- #: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" β€” the
240
- #: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
241
- #: ⚠ It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
242
- #: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
243
- #: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
244
- _RATE_LIMITED = "too many calls"
245
- _RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
246
- _RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
247
-
248
-
249
- def _walk(path, tok, log, **params):
250
- """Every page of an edge, paged by CURSOR under our own parameters.
251
-
252
- β›” IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
253
- first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
254
- you're asking for"**, the retry at limit=25 succeeds β€” and then `next` carries the ORIGINAL
255
- limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
256
- Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
257
-
258
- ⭐ It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
259
- silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
260
- every column per row.
261
-
262
- -> (rows, hit_page_cap)
263
- """
264
- out, pages = [], 0
265
- limit = int(params.pop("limit", None) or PAGE)
266
- after, waited = None, 0
267
- while True:
268
- call = dict(params, limit=limit)
269
- if after:
270
- call["after"] = after
271
- try:
272
- body = _get(path, tok, **call)
273
- except MetaError as e:
274
- if _TOO_MUCH in str(e) and limit > 5:
275
- limit = max(5, limit // 4)
276
- log(f" page too heavy for /{path} - retrying at limit={limit} "
277
- f"(a payload ceiling, not a row cap; every column is still asked for)")
278
- continue
279
- if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
280
- waited += 1
281
- log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
282
- f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
283
- time.sleep(_RATE_WAIT)
284
- continue
285
- if _RATE_LIMITED in str(e).lower():
286
- log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
287
- f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
288
- f"limit is per ad account and persists for minutes. Fix: re-run "
289
- f"`--sync --only <table>` later; the upsert is idempotent.")
290
- return out, True
291
- raise
292
- out.extend(body.get("data") or [])
293
- pages += 1
294
- after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
295
- has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
296
- if not has_next:
297
- return out, False
298
- if pages >= MAX_PAGES:
299
- log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
300
- f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
301
- f"window with META_INSIGHTS_DAYS.")
302
- return out, True
303
-
304
-
305
- def _cell(value):
306
- """One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
307
- being dropped: R2 says every field the API returns, and `targeting` is a field."""
308
- if value is None or isinstance(value, (str, int, float, bool)):
309
- return value
310
- return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
311
-
312
-
313
- def _coltype(name, rows=None):
314
- """The DuckDB type for one column β€” decided by the DATA when there is data, by the name only
315
- as a fallback.
316
-
317
- β›” THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`,
318
- `cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of
319
- action-type objects** on the Insights edge:
320
- [{"action_type":"outbound_click","value":"1.459854"}]
321
- Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string
322
- '[{...}]' to DOUBLE` β€” after five entity tables had already been written, so the sync looked
323
- like it worked and then blew up on the last table.
324
- ⭐ Same principle the field CATALOG is built on, applied one layer down: **ask the response,
325
- do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because
326
- that is what `_cell` stores; anything else falls back to the measured name hints.
327
- """
328
- if rows:
329
- seen, kinds = 0, set()
330
- for r in rows:
331
- v = r.get(name)
332
- if v is None or v == "":
333
- continue
334
- kinds.add("json" if isinstance(v, (list, dict)) else
335
- "bool" if isinstance(v, bool) else
336
- "int" if isinstance(v, int) else
337
- "float" if isinstance(v, float) else "str")
338
- seen += 1
339
- if seen >= 200:
340
- break
341
- if kinds:
342
- if "json" in kinds or "str" in kinds:
343
- return "VARCHAR" # a JSON blob, or a numeric STRING
344
- if kinds <= {"int", "bool"}:
345
- return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT")
346
- return "DOUBLE"
347
- if name in _INT:
348
- return "BIGINT"
349
- if name in _DBL:
350
- return "DOUBLE"
351
- return "VARCHAR" # ids included β€” see the header
352
-
353
-
354
- def _ensure(con, table, fields, rows=None):
355
- """Create or widen the table, with every column typed from `rows` when they are available.
356
-
357
- ⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a
358
- column in place, and this table is DERIVED data that can be re-pulled in minutes β€” so a type
359
- disagreement drops and recreates rather than limping on with a column that refuses every
360
- insert. The alternative is a mirror that is permanently unwritable for one bad guess.
361
- """
362
- want = {f: _coltype(f, rows) for f in fields}
363
- have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
364
- if have:
365
- clash = [f for f, ty in want.items() if f in have and have[f] != ty
366
- and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")]
367
- if clash:
368
- con.execute(f"DROP TABLE {table}")
369
- have = {}
370
- if not have:
371
- cols = ", ".join(f"{f} {want[f]}" for f in fields)
372
- con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
373
- if "id" not in fields else
374
- f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
375
- return
376
- for f in fields:
377
- if f not in have:
378
- con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}")
379
-
380
-
381
- def _upsert(con, table, fields, rows):
382
- """Delete-then-insert by id β€” the same idempotence `datastore._upsert` gives the Odoo half, so
383
- a re-sync updates in place and can never append a second copy of the same object."""
384
- if not rows:
385
- return 0
386
- cols = list(fields)
387
- ids = [str(r.get("id") or "") for r in rows]
388
- q = ",".join("?" for _ in ids)
389
- con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
390
- con.executemany(
391
- f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
392
- [[_cell(r.get(c)) for c in cols] for r in rows])
393
- return len(rows)
394
-
395
-
396
- def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None):
397
- """Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
398
-
399
- Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
400
- exactly as `odoo_relational.refresh` is.
401
- """
402
- tok = tok or token()
403
- # β›” THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME β€” and the difference cost a
404
- # live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s
405
- # `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already
406
- # imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path
407
- # that trips the per-ad-account rate limit and never finishes. The comment beside that call
408
- # claimed a short window the code could not deliver.
409
- # ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it
410
- # makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT.
411
- days = int(insights_days or INSIGHTS_DAYS)
412
- report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
413
- if not tok:
414
- report["problems"].append(
415
- "META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
416
- "was pulled. This is a missing CREDENTIAL, not a missing capability.")
417
- return report
418
-
419
- path = datastore.path_for(tenant_key)
420
- if Path(datastore.DB_PATH) != Path(path):
421
- datastore.use_path(path)
422
- con = datastore.connect()
423
- log(f" mirror: {Path(path).name}")
424
-
425
- accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
426
- report["accounts"] = [a.get("id") for a in accts]
427
- if not accts:
428
- report["problems"].append("the token reaches no ad accounts")
429
- return report
430
-
431
- # ── the five entity levels ────────────────────────────────────────────────────────────────
432
- for table, (edge, fields, _parent) in SPECS.items():
433
- rows, capped = [], False
434
- for acct in accts:
435
- aid = acct["id"]
436
- if edge is None:
437
- rows.append(_get(aid, tok, fields=",".join(fields)))
438
- else:
439
- got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
440
- rows.extend(got)
441
- capped = capped or hit
442
- _ensure(con, table, fields, rows)
443
- n = _upsert(con, table, fields, rows)
444
- total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
445
- report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
446
- log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
447
- + (" !! PAGE CAP HIT" if capped else ""))
448
- if capped:
449
- report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
450
-
451
- # ── the daily Insights grain ──────────────────────────────────────────────────────────────
452
- if insights:
453
- rows, capped = [], False
454
- for acct in accts:
455
- for since, until in _slices(days, INSIGHTS_SLICE):
456
- got, hit = _walk(f"{acct['id']}/insights", tok, log,
457
- fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
458
- time_increment="1", limit=INSIGHTS_PAGE,
459
- time_range=json.dumps({"since": since, "until": until}))
460
- rows.extend(got)
461
- capped = capped or hit
462
- # β›” Insights rows carry no id. The key must be STABLE across runs or every re-sync
463
- # appends a second copy of the same day β€” so it is composed from the grain itself.
464
- for r in rows:
465
- r["id"] = ":".join(str(r.get(k) or "") for k in
466
- (f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
467
- fields = ["id"] + INSIGHT_FIELDS
468
- _ensure(con, INSIGHTS_TABLE, fields, rows)
469
- n = _upsert(con, INSIGHTS_TABLE, fields, rows)
470
- total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
471
- report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
472
- log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
473
- + (" !! PAGE CAP HIT" if capped else ""))
474
-
475
- con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
476
- ["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
477
- sum(t["in_mirror"] for t in report["tables"].values()),
478
- time.strftime("%Y-%m-%d %H:%M:%S")])
479
- return report
480
-
481
-
482
- def status(tenant_key="royal-imports"):
483
- """What is in the mirror right now, per table. Never fetches."""
484
- path = datastore.path_for(tenant_key)
485
- if Path(datastore.DB_PATH) != Path(path):
486
- datastore.use_path(path)
487
- out = {}
488
- con = datastore.connect()
489
- for table in list(SPECS) + [INSIGHTS_TABLE]:
490
- try:
491
- out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
492
- except Exception:
493
- out[table] = None # table absent = never synced
494
- return out
495
-
496
-
497
- def main(argv=None):
498
- ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
499
- ap.add_argument("--sync", action="store_true")
500
- ap.add_argument("--status", action="store_true")
501
- ap.add_argument("--tenant", default="royal-imports")
502
- ap.add_argument("--no-insights", action="store_true")
503
- ap.add_argument("--insights-days", type=int, default=None,
504
- help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say")
505
- a = ap.parse_args(argv)
506
- if a.status:
507
- for k, v in status(a.tenant).items():
508
- print(f" {k:<20} {'(never synced)' if v is None else v}")
509
- return 0
510
- if not a.sync:
511
- ap.print_help()
512
- return 2
513
- rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days)
514
- for p in rep["problems"]:
515
- print(" PROBLEM:", p)
516
- print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
517
- return 1 if rep["problems"] else 0
518
-
519
-
520
- if __name__ == "__main__":
521
- sys.exit(main())
 
1
+ """meta_store.py β€” the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
2
+
3
+ python platform/harness/meta_store.py --sync pull into tenant #0's mirror
4
+ python platform/harness/meta_store.py --sync --tenant gtmlab
5
+ python platform/harness/meta_store.py --status what is in the mirror now
6
+
7
+ β›” WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
8
+ hardwired to XML-RPC β€” it does `import core.odoo as O` and speaks `search_read` β€” so no amount of
9
+ spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
10
+ per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
11
+ delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
12
+ store with two loaders, not two stores.**
13
+
14
+ ⭐ THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
15
+ the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
16
+ `XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
17
+ `Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
18
+ different first hop.
19
+
20
+ ⚠ EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
21
+ carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
22
+ Ad Account 45 Β· Campaign 35 Β· Ad Set 54 Β· Ad 36 Β· Creative 58 Β· Insights 57
23
+ β›” AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
24
+ came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
25
+ nothing going red β€” which is exactly the "do not drop any column" instruction, broken silently.
26
+ The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
27
+
28
+ ⚠ `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
29
+ answers `403 (#200) Requires business_management permission`, which this token does not carry. One
30
+ column of 285. R6's second sentence β€” a limit that cannot be removed gets stated with its cause.
31
+
32
+ β›” IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
33
+ larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
34
+ (W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
35
+ """
36
+ import argparse
37
+ import json
38
+ import os
39
+ import sys
40
+ import time
41
+ import urllib.error
42
+ import urllib.parse
43
+ import urllib.request
44
+ from pathlib import Path
45
+
46
+ _HERE = Path(__file__).resolve().parent
47
+ if str(_HERE.parent) not in sys.path:
48
+ sys.path.insert(0, str(_HERE.parent))
49
+
50
+ from harness import datastore # noqa: E402
51
+
52
+ GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
53
+ GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
54
+
55
+ #: Numeric columns, by name. Everything else is VARCHAR β€” including ids (see the header) and
56
+ #: including anything Graph returns as a nested object, which is stored as compact JSON text.
57
+ _INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
58
+ "inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
59
+ "account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
60
+ "min_daily_budget", "min_campaign_group_spend_cap"}
61
+ _DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
62
+ "inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
63
+ "daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
64
+ "daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
65
+ "lifetime_spend_cap", "lifetime_imps"}
66
+
67
+ #: ⭐ THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
68
+ #: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
69
+ #: the grid links in `meta_relational`.
70
+ ACCOUNT_FIELDS = (
71
+ "account_id account_status age amount_spent balance business_city business_country_code "
72
+ "business_name business_state business_street business_street2 business_zip capabilities "
73
+ "created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
74
+ "has_migrated_permissions id io_number is_attribution_spec_system_default "
75
+ "is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
76
+ "is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
77
+ "name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
78
+ "timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
79
+ ).split()
80
+
81
+ CAMPAIGN_FIELDS = (
82
+ "account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
83
+ "campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
84
+ "created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
85
+ "last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
86
+ "promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
87
+ "special_ad_category special_ad_category_country spend_cap start_time status stop_time "
88
+ "topline_id updated_time"
89
+ ).split()
90
+
91
+ ADSET_FIELDS = (
92
+ "account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
93
+ "bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
94
+ "campaign_active_time campaign_attribution campaign_id configured_status created_time "
95
+ "creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
96
+ "effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
97
+ "issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
98
+ "lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
99
+ "optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
100
+ "rf_prediction_id source_adset source_adset_id start_time status targeting "
101
+ "targeting_optimization_types time_based_ad_rotation_id_blocks "
102
+ "time_based_ad_rotation_intervals updated_time use_new_app_click"
103
+ ).split()
104
+
105
+ AD_FIELDS = (
106
+ "account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
107
+ "adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
108
+ "conversion_domain created_time creative demolink_hash display_sequence effective_status "
109
+ "engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
110
+ "preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
111
+ "tracking_and_conversion_with_defaults tracking_specs updated_time"
112
+ ).split()
113
+
114
+ CREATIVE_FIELDS = (
115
+ "account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
116
+ "branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
117
+ "category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
118
+ "destination_set_id dynamic_ad_voice effective_authorization_category "
119
+ "effective_instagram_media_id effective_object_story_id enable_direct_install "
120
+ "enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
121
+ "instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
122
+ "link_deep_link_url link_destination_display_url link_og_id link_url "
123
+ "messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
124
+ "object_type object_url place_page_set_id platform_customizations playable_asset_id "
125
+ "portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
126
+ "template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
127
+ "use_page_actor_override video_id"
128
+ ).split()
129
+
130
+ INSIGHT_FIELDS = (
131
+ "account_currency account_id account_name action_values actions ad_id ad_name adset_id "
132
+ "adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
133
+ "clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
134
+ "cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
135
+ "date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
136
+ "impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
137
+ "optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
138
+ "social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
139
+ "video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
140
+ "video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
141
+ "video_play_actions video_thruplay_watched_actions website_purchase_roas"
142
+ ).split()
143
+
144
+ #: table -> (graph edge on the account | None, measured fields, parent column)
145
+ SPECS = {
146
+ "meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
147
+ "meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
148
+ "meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
149
+ "meta_ads": ("ads", AD_FIELDS, "adset_id"),
150
+ "meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
151
+ }
152
+
153
+ #: The daily Insights grain (R2). One row per (ad, day) β€” the id is synthesised because Insights
154
+ #: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
155
+ INSIGHTS_TABLE = "meta_insights_daily"
156
+ INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
157
+
158
+ #: β›” INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
159
+ #: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
160
+ #: β€” Graph's way of saying the synchronous query is too heavy (the async report-run API is the
161
+ #: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
162
+ #: page 25 answer 200. So the window is walked in slices with every column intact:
163
+ #: 57 fields Β· ad level Β· 7d Β· limit 25 -> 200, 25 rows
164
+ #: 57 fields Β· account level Β· 7d -> 200, 7 rows
165
+ #: ⚠ Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over β€”
166
+ #: it drops columns R2 requires, and it does so invisibly.
167
+ INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
168
+ INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
169
+ INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
170
+
171
+
172
+ def _slices(days, size):
173
+ """[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
174
+ from datetime import date, timedelta
175
+ end = date.today()
176
+ start = end - timedelta(days=max(1, days) - 1)
177
+ out, cur = [], start
178
+ while cur <= end:
179
+ stop = min(cur + timedelta(days=max(1, size) - 1), end)
180
+ out.append((cur.isoformat(), stop.isoformat()))
181
+ cur = stop + timedelta(days=1)
182
+ return out
183
+
184
+ #: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
185
+ #: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
186
+ #: not a row cap on a connected source (R6 forbids that) β€” it is a runaway guard, and reaching it
187
+ #: is reported as a problem, never absorbed.
188
+ MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
189
+ PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
190
+
191
+
192
+ class MetaError(RuntimeError):
193
+ """A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
194
+
195
+
196
+ def token():
197
+ """The Meta token from the environment or gitignored `platform/.env`; "" when absent.
198
+
199
+ ⚠ Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
200
+ `platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
201
+ (`core` never imports up). Fifteen lines is the price of that boundary.
202
+ """
203
+ tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
204
+ if tok:
205
+ return tok.strip()
206
+ env = _HERE.parent / ".env"
207
+ if env.exists():
208
+ for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
209
+ if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
210
+ _, _, v = line.partition("=")
211
+ return v.strip().strip('"').strip("'")
212
+ return ""
213
+
214
+
215
+ def _get(path, tok, **params):
216
+ params["access_token"] = tok
217
+ url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
218
+ req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
219
+ try:
220
+ with urllib.request.urlopen(req, timeout=120) as r:
221
+ return json.loads(r.read().decode("utf-8", "replace"))
222
+ except urllib.error.HTTPError as e:
223
+ try:
224
+ msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
225
+ ).get("message") or ""
226
+ except Exception:
227
+ msg = ""
228
+ raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
229
+ except Exception as e:
230
+ raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
231
+
232
+
233
+ #: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
234
+ #: why it cannot be treated as "the server is broken, give up".
235
+ _TOO_MUCH = "reduce the amount of data"
236
+
237
+ #: β›” THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
238
+ #: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
239
+ #: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" β€” the
240
+ #: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
241
+ #: ⚠ It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
242
+ #: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
243
+ #: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
244
+ _RATE_LIMITED = "too many calls"
245
+ _RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
246
+ _RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
247
+
248
+
249
+ def _walk(path, tok, log, **params):
250
+ """Every page of an edge, paged by CURSOR under our own parameters.
251
+
252
+ β›” IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
253
+ first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
254
+ you're asking for"**, the retry at limit=25 succeeds β€” and then `next` carries the ORIGINAL
255
+ limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
256
+ Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
257
+
258
+ ⭐ It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
259
+ silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
260
+ every column per row.
261
+
262
+ -> (rows, hit_page_cap)
263
+ """
264
+ out, pages = [], 0
265
+ limit = int(params.pop("limit", None) or PAGE)
266
+ after, waited = None, 0
267
+ while True:
268
+ call = dict(params, limit=limit)
269
+ if after:
270
+ call["after"] = after
271
+ try:
272
+ body = _get(path, tok, **call)
273
+ except MetaError as e:
274
+ if _TOO_MUCH in str(e) and limit > 5:
275
+ limit = max(5, limit // 4)
276
+ log(f" page too heavy for /{path} - retrying at limit={limit} "
277
+ f"(a payload ceiling, not a row cap; every column is still asked for)")
278
+ continue
279
+ if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
280
+ waited += 1
281
+ log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
282
+ f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
283
+ time.sleep(_RATE_WAIT)
284
+ continue
285
+ if _RATE_LIMITED in str(e).lower():
286
+ log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
287
+ f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
288
+ f"limit is per ad account and persists for minutes. Fix: re-run "
289
+ f"`--sync --only <table>` later; the upsert is idempotent.")
290
+ return out, True
291
+ raise
292
+ out.extend(body.get("data") or [])
293
+ pages += 1
294
+ after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
295
+ has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
296
+ if not has_next:
297
+ return out, False
298
+ if pages >= MAX_PAGES:
299
+ log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
300
+ f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
301
+ f"window with META_INSIGHTS_DAYS.")
302
+ return out, True
303
+
304
+
305
+ def _cell(value):
306
+ """One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
307
+ being dropped: R2 says every field the API returns, and `targeting` is a field."""
308
+ if value is None or isinstance(value, (str, int, float, bool)):
309
+ return value
310
+ return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
311
+
312
+
313
+ def _coltype(name, rows=None):
314
+ """The DuckDB type for one column β€” decided by the DATA when there is data, by the name only
315
+ as a fallback.
316
+
317
+ β›” THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`,
318
+ `cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of
319
+ action-type objects** on the Insights edge:
320
+ [{"action_type":"outbound_click","value":"1.459854"}]
321
+ Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string
322
+ '[{...}]' to DOUBLE` β€” after five entity tables had already been written, so the sync looked
323
+ like it worked and then blew up on the last table.
324
+ ⭐ Same principle the field CATALOG is built on, applied one layer down: **ask the response,
325
+ do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because
326
+ that is what `_cell` stores; anything else falls back to the measured name hints.
327
+ """
328
+ if rows:
329
+ seen, kinds = 0, set()
330
+ for r in rows:
331
+ v = r.get(name)
332
+ if v is None or v == "":
333
+ continue
334
+ kinds.add("json" if isinstance(v, (list, dict)) else
335
+ "bool" if isinstance(v, bool) else
336
+ "int" if isinstance(v, int) else
337
+ "float" if isinstance(v, float) else "str")
338
+ seen += 1
339
+ if seen >= 200:
340
+ break
341
+ if kinds:
342
+ if "json" in kinds or "str" in kinds:
343
+ return "VARCHAR" # a JSON blob, or a numeric STRING
344
+ if kinds <= {"int", "bool"}:
345
+ return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT")
346
+ return "DOUBLE"
347
+ if name in _INT:
348
+ return "BIGINT"
349
+ if name in _DBL:
350
+ return "DOUBLE"
351
+ return "VARCHAR" # ids included β€” see the header
352
+
353
+
354
+ def _ensure(con, table, fields, rows=None):
355
+ """Create or widen the table, with every column typed from `rows` when they are available.
356
+
357
+ ⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a
358
+ column in place, and this table is DERIVED data that can be re-pulled in minutes β€” so a type
359
+ disagreement drops and recreates rather than limping on with a column that refuses every
360
+ insert. The alternative is a mirror that is permanently unwritable for one bad guess.
361
+ """
362
+ want = {f: _coltype(f, rows) for f in fields}
363
+ have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
364
+ if have:
365
+ clash = [f for f, ty in want.items() if f in have and have[f] != ty
366
+ and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")]
367
+ if clash:
368
+ con.execute(f"DROP TABLE {table}")
369
+ have = {}
370
+ if not have:
371
+ cols = ", ".join(f"{f} {want[f]}" for f in fields)
372
+ con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
373
+ if "id" not in fields else
374
+ f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
375
+ return
376
+ for f in fields:
377
+ if f not in have:
378
+ con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}")
379
+
380
+
381
+ def _upsert(con, table, fields, rows):
382
+ """Delete-then-insert by id β€” the same idempotence `datastore._upsert` gives the Odoo half, so
383
+ a re-sync updates in place and can never append a second copy of the same object."""
384
+ if not rows:
385
+ return 0
386
+ cols = list(fields)
387
+ ids = [str(r.get("id") or "") for r in rows]
388
+ q = ",".join("?" for _ in ids)
389
+ con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
390
+ con.executemany(
391
+ f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
392
+ [[_cell(r.get(c)) for c in cols] for r in rows])
393
+ return len(rows)
394
+
395
+
396
+ def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None):
397
+ """Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
398
+
399
+ Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
400
+ exactly as `odoo_relational.refresh` is.
401
+ """
402
+ tok = tok or token()
403
+ # β›” THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME β€” and the difference cost a
404
+ # live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s
405
+ # `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already
406
+ # imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path
407
+ # that trips the per-ad-account rate limit and never finishes. The comment beside that call
408
+ # claimed a short window the code could not deliver.
409
+ # ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it
410
+ # makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT.
411
+ days = int(insights_days or INSIGHTS_DAYS)
412
+ report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
413
+ if not tok:
414
+ report["problems"].append(
415
+ "META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
416
+ "was pulled. This is a missing CREDENTIAL, not a missing capability.")
417
+ return report
418
+
419
+ path = datastore.path_for(tenant_key)
420
+ if Path(datastore.DB_PATH) != Path(path):
421
+ datastore.use_path(path)
422
+ con = datastore.connect()
423
+ log(f" mirror: {Path(path).name}")
424
+
425
+ accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
426
+ report["accounts"] = [a.get("id") for a in accts]
427
+ if not accts:
428
+ report["problems"].append("the token reaches no ad accounts")
429
+ return report
430
+
431
+ # ── the five entity levels ────────────────────────────────────────────────────────────────
432
+ for table, (edge, fields, _parent) in SPECS.items():
433
+ rows, capped = [], False
434
+ for acct in accts:
435
+ aid = acct["id"]
436
+ if edge is None:
437
+ rows.append(_get(aid, tok, fields=",".join(fields)))
438
+ else:
439
+ got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
440
+ rows.extend(got)
441
+ capped = capped or hit
442
+ _ensure(con, table, fields, rows)
443
+ n = _upsert(con, table, fields, rows)
444
+ total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
445
+ report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
446
+ log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
447
+ + (" !! PAGE CAP HIT" if capped else ""))
448
+ if capped:
449
+ report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
450
+
451
+ # ── the daily Insights grain ──────────────────────────────────────────────────────────────
452
+ if insights:
453
+ rows, capped = [], False
454
+ for acct in accts:
455
+ for since, until in _slices(days, INSIGHTS_SLICE):
456
+ got, hit = _walk(f"{acct['id']}/insights", tok, log,
457
+ fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
458
+ time_increment="1", limit=INSIGHTS_PAGE,
459
+ time_range=json.dumps({"since": since, "until": until}))
460
+ rows.extend(got)
461
+ capped = capped or hit
462
+ # β›” Insights rows carry no id. The key must be STABLE across runs or every re-sync
463
+ # appends a second copy of the same day β€” so it is composed from the grain itself.
464
+ for r in rows:
465
+ r["id"] = ":".join(str(r.get(k) or "") for k in
466
+ (f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
467
+ fields = ["id"] + INSIGHT_FIELDS
468
+ _ensure(con, INSIGHTS_TABLE, fields, rows)
469
+ n = _upsert(con, INSIGHTS_TABLE, fields, rows)
470
+ total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
471
+ report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
472
+ log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
473
+ + (" !! PAGE CAP HIT" if capped else ""))
474
+
475
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
476
+ ["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
477
+ sum(t["in_mirror"] for t in report["tables"].values()),
478
+ time.strftime("%Y-%m-%d %H:%M:%S")])
479
+ return report
480
+
481
+
482
+ def status(tenant_key="royal-imports"):
483
+ """What is in the mirror right now, per table. Never fetches."""
484
+ path = datastore.path_for(tenant_key)
485
+ if Path(datastore.DB_PATH) != Path(path):
486
+ datastore.use_path(path)
487
+ out = {}
488
+ con = datastore.connect()
489
+ for table in list(SPECS) + [INSIGHTS_TABLE]:
490
+ try:
491
+ out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
492
+ except Exception:
493
+ out[table] = None # table absent = never synced
494
+ return out
495
+
496
+
497
+ def main(argv=None):
498
+ ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
499
+ ap.add_argument("--sync", action="store_true")
500
+ ap.add_argument("--status", action="store_true")
501
+ ap.add_argument("--tenant", default="royal-imports")
502
+ ap.add_argument("--no-insights", action="store_true")
503
+ ap.add_argument("--insights-days", type=int, default=None,
504
+ help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say")
505
+ a = ap.parse_args(argv)
506
+ if a.status:
507
+ for k, v in status(a.tenant).items():
508
+ print(f" {k:<20} {'(never synced)' if v is None else v}")
509
+ return 0
510
+ if not a.sync:
511
+ ap.print_help()
512
+ return 2
513
+ rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days)
514
+ for p in rep["problems"]:
515
+ print(" PROBLEM:", p)
516
+ print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
517
+ return 1 if rep["problems"] else 0
518
+
519
+
520
+ if __name__ == "__main__":
521
+ sys.exit(main())
platform/model/topics/odoo_accounts.yml CHANGED
@@ -1,60 +1,60 @@
1
- # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_accounts
9
- label: "Odoo GL accounts"
10
- entity: account.account
11
- # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_accounts
13
- subject: "odoo:account.account"
14
- grain: "one row per GL account in the chart of accounts"
15
- scope:
16
- population: "every account.account record (192 measured)"
17
- store:
18
- table: account_account
19
- alias: a
20
- # NO date_col β€” a registry is not a dated event stream. Stated rather than
21
- # omitted, so its absence reads as a fact and not as an unfinished file.
22
- dims:
23
- account_type: {label: "Account type"}
24
-
25
- # key / label / type / kind, derived from the grid contract. `kind` says where the
26
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
27
- # from another topic; a `link` points at another database.
28
- fields:
29
- - key: account_code
30
- label: "Code"
31
- type: text
32
- kind: data
33
- - key: account_name
34
- label: "Account"
35
- type: text
36
- kind: data
37
- - key: odoo_id
38
- label: "Odoo ID"
39
- type: int
40
- kind: data
41
- means: "The `account.account` id. Also this row's id."
42
- - key: account_type
43
- label: "Type"
44
- type: select
45
- kind: data
46
- - key: is_expense
47
- label: "Expense account"
48
- type: checkbox
49
- kind: data
50
- means: "Ticked for the expense family - the same predicate the semantic layer's gl_lines topic uses, so this column and that topic cannot disagree."
51
- - key: refreshed
52
- label: "Refreshed"
53
- type: date
54
- kind: data
55
- means: "When this row was last reconciled against Odoo."
56
-
57
- ai_context: >
58
- The chart of accounts β€” the registry gl_lines posts against.
59
- Use it to resolve an account code to a name or a type.
60
- ⚠ `is_expense` is a DERIVED boolean standing in for Odoo's five-value `internal_group` (asset/liability/equity/income/expense), which the mirror does not yet carry.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_accounts
9
+ label: "Odoo GL accounts"
10
+ entity: account.account
11
+ # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_accounts
13
+ subject: "odoo:account.account"
14
+ grain: "one row per GL account in the chart of accounts"
15
+ scope:
16
+ population: "every account.account record (192 measured)"
17
+ store:
18
+ table: account_account
19
+ alias: a
20
+ # NO date_col β€” a registry is not a dated event stream. Stated rather than
21
+ # omitted, so its absence reads as a fact and not as an unfinished file.
22
+ dims:
23
+ account_type: {label: "Account type"}
24
+
25
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
26
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
27
+ # from another topic; a `link` points at another database.
28
+ fields:
29
+ - key: account_code
30
+ label: "Code"
31
+ type: text
32
+ kind: data
33
+ - key: account_name
34
+ label: "Account"
35
+ type: text
36
+ kind: data
37
+ - key: odoo_id
38
+ label: "Odoo ID"
39
+ type: int
40
+ kind: data
41
+ means: "The `account.account` id. Also this row's id."
42
+ - key: account_type
43
+ label: "Type"
44
+ type: select
45
+ kind: data
46
+ - key: is_expense
47
+ label: "Expense account"
48
+ type: checkbox
49
+ kind: data
50
+ means: "Ticked for the expense family - the same predicate the semantic layer's gl_lines topic uses, so this column and that topic cannot disagree."
51
+ - key: refreshed
52
+ label: "Refreshed"
53
+ type: date
54
+ kind: data
55
+ means: "When this row was last reconciled against Odoo."
56
+
57
+ ai_context: >
58
+ The chart of accounts β€” the registry gl_lines posts against.
59
+ Use it to resolve an account code to a name or a type.
60
+ ⚠ `is_expense` is a DERIVED boolean standing in for Odoo's five-value `internal_group` (asset/liability/equity/income/expense), which the mirror does not yet carry.
platform/model/topics/odoo_agents.yml CHANGED
@@ -1,59 +1,59 @@
1
- # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_agents
9
- label: "Odoo agents"
10
- entity: res.partner
11
- # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_agents
13
- subject: "odoo:res.partner.agent"
14
- grain: "one row per sales agent"
15
- scope:
16
- population: "a UNION of two disagreeing sources β€” partners carrying commission lines, and partners flagged res_partner.agent = TRUE"
17
- why_union: "measured 2026-08-09: 16 carry commission lines, 17 carry the flag, and the union is 19 β€” either source alone silently drops real agents"
18
- store:
19
- table: res_partner
20
- alias: p
21
- # NO date_col β€” a registry is not a dated event stream. Stated rather than
22
- # omitted, so its absence reads as a fact and not as an unfinished file.
23
-
24
- # key / label / type / kind, derived from the grid contract. `kind` says where the
25
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
26
- # from another topic; a `link` points at another database.
27
- fields:
28
- - key: agent
29
- label: "Agent"
30
- type: text
31
- kind: data
32
- - key: odoo_id
33
- label: "Odoo ID"
34
- type: int
35
- kind: data
36
- means: "The `res.partner` id. Also this row's id."
37
- - key: agent_id
38
- label: "Odoo agent id"
39
- type: int
40
- kind: data
41
- - key: flagged
42
- label: "Flagged in Odoo"
43
- type: checkbox
44
- kind: data
45
- means: "Ticked = `res.partner.agent` is set. Unticked agents were found by their commission lines instead - both are real, which is why this table is the union of the two."
46
- - key: commissioned
47
- label: "Has commission lines"
48
- type: checkbox
49
- kind: data
50
- - key: refreshed
51
- label: "Refreshed"
52
- type: date
53
- kind: data
54
- means: "When this row was last reconciled against Odoo."
55
-
56
- ai_context: >
57
- The sales-agent registry.
58
- `flagged` and `commissioned` are the two SOURCES, kept as separate columns rather than merged, because they disagree and the disagreement is information.
59
- For an agent's BOOK use sales_lines/commission_lines with the agent dim.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_agents
9
+ label: "Odoo agents"
10
+ entity: res.partner
11
+ # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_agents
13
+ subject: "odoo:res.partner.agent"
14
+ grain: "one row per sales agent"
15
+ scope:
16
+ population: "a UNION of two disagreeing sources β€” partners carrying commission lines, and partners flagged res_partner.agent = TRUE"
17
+ why_union: "measured 2026-08-09: 16 carry commission lines, 17 carry the flag, and the union is 19 β€” either source alone silently drops real agents"
18
+ store:
19
+ table: res_partner
20
+ alias: p
21
+ # NO date_col β€” a registry is not a dated event stream. Stated rather than
22
+ # omitted, so its absence reads as a fact and not as an unfinished file.
23
+
24
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
25
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
26
+ # from another topic; a `link` points at another database.
27
+ fields:
28
+ - key: agent
29
+ label: "Agent"
30
+ type: text
31
+ kind: data
32
+ - key: odoo_id
33
+ label: "Odoo ID"
34
+ type: int
35
+ kind: data
36
+ means: "The `res.partner` id. Also this row's id."
37
+ - key: agent_id
38
+ label: "Odoo agent id"
39
+ type: int
40
+ kind: data
41
+ - key: flagged
42
+ label: "Flagged in Odoo"
43
+ type: checkbox
44
+ kind: data
45
+ means: "Ticked = `res.partner.agent` is set. Unticked agents were found by their commission lines instead - both are real, which is why this table is the union of the two."
46
+ - key: commissioned
47
+ label: "Has commission lines"
48
+ type: checkbox
49
+ kind: data
50
+ - key: refreshed
51
+ label: "Refreshed"
52
+ type: date
53
+ kind: data
54
+ means: "When this row was last reconciled against Odoo."
55
+
56
+ ai_context: >
57
+ The sales-agent registry.
58
+ `flagged` and `commissioned` are the two SOURCES, kept as separate columns rather than merged, because they disagree and the disagreement is information.
59
+ For an agent's BOOK use sales_lines/commission_lines with the agent dim.
platform/model/topics/odoo_bills.yml CHANGED
@@ -1,86 +1,86 @@
1
- # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_bills
9
- label: "Odoo vendor bills"
10
- entity: account.move
11
- # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_bills
13
- subject: "odoo:account.move.vendor"
14
- grain: "one row per posted VENDOR bill or vendor credit note (document grain)"
15
- scope:
16
- population: "move_type in (in_invoice, in_refund) AND state = posted"
17
- signs: "for payables the residual is negative on the Odoo side; take the absolute value for an AP figure"
18
- store:
19
- table: account_move
20
- alias: m
21
- date_col: "m.invoice_date"
22
- dims:
23
- vendor: {label: "Vendor"}
24
- payment_state: {label: "Payment state"}
25
- move_type: {label: "Document type"}
26
-
27
- # key / label / type / kind, derived from the grid contract. `kind` says where the
28
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
- # from another topic; a `link` points at another database.
30
- fields:
31
- - key: bill_no
32
- label: "Bill"
33
- type: text
34
- kind: data
35
- - key: odoo_id
36
- label: "Odoo ID"
37
- type: int
38
- kind: data
39
- means: "The `account.move` id. Also this row's id."
40
- - key: vendor
41
- label: "Vendor"
42
- type: text
43
- kind: data
44
- - key: vendor_id
45
- label: "Odoo vendor id"
46
- type: int
47
- kind: data
48
- - key: invoice_date
49
- label: "Bill date"
50
- type: date
51
- kind: data
52
- - key: due_date
53
- label: "Due date"
54
- type: date
55
- kind: data
56
- - key: amount_untaxed
57
- label: "Billed $"
58
- type: currency
59
- kind: data
60
- - key: residual
61
- label: "Outstanding $"
62
- type: currency
63
- kind: data
64
- - key: payment_state
65
- label: "Payment state"
66
- type: select
67
- kind: data
68
- - key: move_type
69
- label: "Document"
70
- type: select
71
- kind: data
72
- - key: vendor_link
73
- label: "Vendor record"
74
- type: link
75
- kind: link
76
- to_grid: ut_odoo_vendors
77
- - key: refreshed
78
- label: "Refreshed"
79
- type: date
80
- kind: data
81
- means: "When this row was last reconciled against Odoo."
82
-
83
- ai_context: >
84
- Vendor bills at DOCUMENT grain β€” the payables side of account_move.
85
- Use it for AP questions.
86
- ⚠ The census found `ref` (the VENDOR's own invoice number), `journal_id` and `currency_id` populated in Odoo and absent from the mirror, so they cannot be answered from here yet.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_bills
9
+ label: "Odoo vendor bills"
10
+ entity: account.move
11
+ # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_bills
13
+ subject: "odoo:account.move.vendor"
14
+ grain: "one row per posted VENDOR bill or vendor credit note (document grain)"
15
+ scope:
16
+ population: "move_type in (in_invoice, in_refund) AND state = posted"
17
+ signs: "for payables the residual is negative on the Odoo side; take the absolute value for an AP figure"
18
+ store:
19
+ table: account_move
20
+ alias: m
21
+ date_col: "m.invoice_date"
22
+ dims:
23
+ vendor: {label: "Vendor"}
24
+ payment_state: {label: "Payment state"}
25
+ move_type: {label: "Document type"}
26
+
27
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
28
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
+ # from another topic; a `link` points at another database.
30
+ fields:
31
+ - key: bill_no
32
+ label: "Bill"
33
+ type: text
34
+ kind: data
35
+ - key: odoo_id
36
+ label: "Odoo ID"
37
+ type: int
38
+ kind: data
39
+ means: "The `account.move` id. Also this row's id."
40
+ - key: vendor
41
+ label: "Vendor"
42
+ type: text
43
+ kind: data
44
+ - key: vendor_id
45
+ label: "Odoo vendor id"
46
+ type: int
47
+ kind: data
48
+ - key: invoice_date
49
+ label: "Bill date"
50
+ type: date
51
+ kind: data
52
+ - key: due_date
53
+ label: "Due date"
54
+ type: date
55
+ kind: data
56
+ - key: amount_untaxed
57
+ label: "Billed $"
58
+ type: currency
59
+ kind: data
60
+ - key: residual
61
+ label: "Outstanding $"
62
+ type: currency
63
+ kind: data
64
+ - key: payment_state
65
+ label: "Payment state"
66
+ type: select
67
+ kind: data
68
+ - key: move_type
69
+ label: "Document"
70
+ type: select
71
+ kind: data
72
+ - key: vendor_link
73
+ label: "Vendor record"
74
+ type: link
75
+ kind: link
76
+ to_grid: ut_odoo_vendors
77
+ - key: refreshed
78
+ label: "Refreshed"
79
+ type: date
80
+ kind: data
81
+ means: "When this row was last reconciled against Odoo."
82
+
83
+ ai_context: >
84
+ Vendor bills at DOCUMENT grain β€” the payables side of account_move.
85
+ Use it for AP questions.
86
+ ⚠ The census found `ref` (the VENDOR's own invoice number), `journal_id` and `currency_id` populated in Odoo and absent from the mirror, so they cannot be answered from here yet.
platform/model/topics/odoo_customers.yml CHANGED
@@ -1,213 +1,213 @@
1
- # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_customers
9
- label: "Odoo customers"
10
- entity: res.partner
11
- # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
- grid: customer_data
13
- subject: "odoo:res.partner"
14
- grain: "one row per customer in tenant #0's scoped book β€” a REGISTRY, not a dated event stream. Identity is the Odoo `res.partner` id (the row's `pid` IS that id)."
15
- scope:
16
- population: "the customer book `modules/customer_data.pool()` builds; the grid's own contract states it as one row per customer who ordered in the last 24 months"
17
- identity: "the Odoo res.partner id β€” exposed as the `partner_id` column (W33-T43) and used as the row pid, so there is ONE id per row and no second one"
18
- merged: "W33-T44 retired `ut_odoo_customers`, which presented this same subject. This topic describes the SURVIVING database, `customer_data`, which keeps its store bucket so no saved view, grant, cohort or formula moved"
19
- no_date: "a registry has no date dimension; ask sales_lines or customer_invoices for anything time-windowed about a customer"
20
- store:
21
- table: res_partner
22
- alias: p
23
- # NO date_col β€” a registry is not a dated event stream. Stated rather than
24
- # omitted, so its absence reads as a fact and not as an unfinished file.
25
- dims:
26
- agent: {label: "Sales agent"}
27
- state: {label: "State"}
28
- country: {label: "Country"}
29
-
30
- # key / label / type / kind, derived from the grid contract. `kind` says where the
31
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
32
- # from another topic; a `link` points at another database.
33
- fields:
34
- - key: customer
35
- label: "Customer"
36
- type: text
37
- kind: data
38
- means: "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
39
- - key: partner_id
40
- label: "Odoo ID"
41
- type: int
42
- kind: data
43
- means: "The Odoo res.partner id β€” this row's identity, and the key every Odoo document joins on. W33-T43 / owner item 12: 'One unique ID per database always.' It is DERIVED rather than read off the pool row because a customer row's pid IS the partner id, so the value is already on every row and a second copy in the pool would be a second source for one fact."
44
- - key: odoo_status
45
- label: "Odoo record"
46
- type: status
47
- kind: data
48
- means: "Whether this customer still exists in Odoo. Archived means deleted there."
49
- - key: agent
50
- label: "Agent"
51
- type: text
52
- kind: data
53
- means: "The sales agent who owns this account."
54
- - key: dba
55
- label: "DBA"
56
- type: select
57
- kind: data
58
- means: "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
59
- - key: salesperson
60
- label: "Salesperson"
61
- type: text
62
- kind: data
63
- means: "Who keyed in most of this customer's orders β€” not the Agent, who owns the account."
64
- - key: street
65
- label: "Street"
66
- type: text
67
- kind: data
68
- means: "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
69
- - key: street2
70
- label: "Street 2"
71
- type: text
72
- kind: data
73
- means: "Second address line (suite, unit, floor) on the customer's Odoo address."
74
- - key: city
75
- label: "City"
76
- type: text
77
- kind: data
78
- means: "City on the customer's Odoo address."
79
- - key: state
80
- label: "State"
81
- type: text
82
- kind: data
83
- means: "State or province on the customer's Odoo address."
84
- - key: country
85
- label: "Country"
86
- type: text
87
- kind: data
88
- means: "Country on the customer's Odoo address."
89
- - key: zip
90
- label: "ZIP"
91
- type: text
92
- kind: data
93
- means: "Postal code on the customer's Odoo address."
94
- - key: customer_since
95
- label: "Customer since"
96
- type: date
97
- kind: data
98
- means: "When this customer was first set up in Odoo."
99
- - key: tags
100
- label: "Tags"
101
- type: text
102
- kind: data
103
- means: "Odoo labels on this customer, comma-separated."
104
- - key: pricelist
105
- label: "Price list"
106
- type: text
107
- kind: data
108
- means: "The price list this customer buys on."
109
- - key: payment_terms
110
- label: "Payment terms"
111
- type: text
112
- kind: data
113
- means: "Payment terms on this customer's account β€” Net 30, for example."
114
- - key: last_order
115
- label: "Last order"
116
- type: date
117
- kind: data
118
- means: "Date of the most recent confirmed order."
119
- - key: overdue_days
120
- label: "Overdue days"
121
- type: int
122
- kind: data
123
- means: "How many days late this customer is running against their own usual ordering rhythm."
124
- - key: est_missed
125
- label: "Est. missed $"
126
- type: currency
127
- kind: data
128
- means: "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
129
- - key: ar_open
130
- label: "AR current $"
131
- type: currency
132
- kind: data
133
- means: "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
134
- - key: ar_overdue
135
- label: "AR overdue $"
136
- type: currency
137
- kind: data
138
- means: "Invoiced money past due β€” same basis as the Collections page."
139
- - key: ar_outstanding
140
- label: "AR outstanding $"
141
- type: currency
142
- kind: data
143
- means: "Total invoiced money owed right now: AR current $ plus AR overdue $."
144
- - key: ar_exposure
145
- label: "Credit exposure $"
146
- type: currency
147
- kind: data
148
- means: "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
149
- - key: ar_aged_1_30
150
- label: "1-30 days $"
151
- type: currency
152
- kind: data
153
- means: "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
154
- - key: ar_aged_31_60
155
- label: "31-60 days $"
156
- type: currency
157
- kind: data
158
- means: "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
159
- - key: ar_aged_61_90
160
- label: "61-90 days $"
161
- type: currency
162
- kind: data
163
- means: "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
164
- - key: ar_aged_90_plus
165
- label: "90+ days $"
166
- type: currency
167
- kind: data
168
- means: "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
169
- - key: days_to_pay
170
- label: "Days to pay"
171
- type: int
172
- kind: data
173
- means: "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
174
- - key: top_category
175
- label: "Top category"
176
- type: text
177
- kind: data
178
- means: "The category this customer spent the most on in the last 12 months."
179
- - key: top_category_pct
180
- label: "Top category %"
181
- type: pct
182
- kind: data
183
- means: "Share of last-12-months spend that went to the top category."
184
- - key: sku_count
185
- label: "SKUs bought"
186
- type: int
187
- kind: data
188
- means: "Distinct products bought in the last 12 months."
189
- - key: top_sku
190
- label: "Top SKU"
191
- type: text
192
- kind: data
193
- means: "The product this customer spent the most on in the last 12 months."
194
- - key: days_since
195
- label: "Days since order"
196
- type: int
197
- kind: data
198
- means: "Days since the last confirmed order."
199
- - key: typical_gap_days
200
- label: "Typical gap days"
201
- type: int
202
- kind: data
203
- means: "Days this customer usually goes between orders, from their own history."
204
- - key: notes
205
- label: "Notes"
206
- type: text
207
- kind: data
208
- means: "Your notes on this customer. Saved in this app only, visible only to you."
209
-
210
- ai_context: >
211
- The customer REGISTRY as a person sees it in the app β€” the database labelled 'Odoo customers'.
212
- Use it to answer 'who is this customer', 'which customers exist', 'which agent owns them', 'where are they', and for the AR columns it carries (ar_open, ar_overdue, days_to_pay), which are reconciled by modules/ar.py.
213
- β›” For anything WINDOWED or at line grain go to sales_lines / customer_invoices / receivables β€” those topics own the time dimension and this one has none.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_customers
9
+ label: "Odoo customers"
10
+ entity: res.partner
11
+ # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
+ grid: customer_data
13
+ subject: "odoo:res.partner"
14
+ grain: "one row per customer in tenant #0's scoped book β€” a REGISTRY, not a dated event stream. Identity is the Odoo `res.partner` id (the row's `pid` IS that id)."
15
+ scope:
16
+ population: "the customer book `modules/customer_data.pool()` builds; the grid's own contract states it as one row per customer who ordered in the last 24 months"
17
+ identity: "the Odoo res.partner id β€” exposed as the `partner_id` column (W33-T43) and used as the row pid, so there is ONE id per row and no second one"
18
+ merged: "W33-T44 retired `ut_odoo_customers`, which presented this same subject. This topic describes the SURVIVING database, `customer_data`, which keeps its store bucket so no saved view, grant, cohort or formula moved"
19
+ no_date: "a registry has no date dimension; ask sales_lines or customer_invoices for anything time-windowed about a customer"
20
+ store:
21
+ table: res_partner
22
+ alias: p
23
+ # NO date_col β€” a registry is not a dated event stream. Stated rather than
24
+ # omitted, so its absence reads as a fact and not as an unfinished file.
25
+ dims:
26
+ agent: {label: "Sales agent"}
27
+ state: {label: "State"}
28
+ country: {label: "Country"}
29
+
30
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
31
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
32
+ # from another topic; a `link` points at another database.
33
+ fields:
34
+ - key: customer
35
+ label: "Customer"
36
+ type: text
37
+ kind: data
38
+ means: "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
39
+ - key: partner_id
40
+ label: "Odoo ID"
41
+ type: int
42
+ kind: data
43
+ means: "The Odoo res.partner id β€” this row's identity, and the key every Odoo document joins on. W33-T43 / owner item 12: 'One unique ID per database always.' It is DERIVED rather than read off the pool row because a customer row's pid IS the partner id, so the value is already on every row and a second copy in the pool would be a second source for one fact."
44
+ - key: odoo_status
45
+ label: "Odoo record"
46
+ type: status
47
+ kind: data
48
+ means: "Whether this customer still exists in Odoo. Archived means deleted there."
49
+ - key: agent
50
+ label: "Agent"
51
+ type: text
52
+ kind: data
53
+ means: "The sales agent who owns this account."
54
+ - key: dba
55
+ label: "DBA"
56
+ type: select
57
+ kind: data
58
+ means: "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
59
+ - key: salesperson
60
+ label: "Salesperson"
61
+ type: text
62
+ kind: data
63
+ means: "Who keyed in most of this customer's orders β€” not the Agent, who owns the account."
64
+ - key: street
65
+ label: "Street"
66
+ type: text
67
+ kind: data
68
+ means: "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
69
+ - key: street2
70
+ label: "Street 2"
71
+ type: text
72
+ kind: data
73
+ means: "Second address line (suite, unit, floor) on the customer's Odoo address."
74
+ - key: city
75
+ label: "City"
76
+ type: text
77
+ kind: data
78
+ means: "City on the customer's Odoo address."
79
+ - key: state
80
+ label: "State"
81
+ type: text
82
+ kind: data
83
+ means: "State or province on the customer's Odoo address."
84
+ - key: country
85
+ label: "Country"
86
+ type: text
87
+ kind: data
88
+ means: "Country on the customer's Odoo address."
89
+ - key: zip
90
+ label: "ZIP"
91
+ type: text
92
+ kind: data
93
+ means: "Postal code on the customer's Odoo address."
94
+ - key: customer_since
95
+ label: "Customer since"
96
+ type: date
97
+ kind: data
98
+ means: "When this customer was first set up in Odoo."
99
+ - key: tags
100
+ label: "Tags"
101
+ type: text
102
+ kind: data
103
+ means: "Odoo labels on this customer, comma-separated."
104
+ - key: pricelist
105
+ label: "Price list"
106
+ type: text
107
+ kind: data
108
+ means: "The price list this customer buys on."
109
+ - key: payment_terms
110
+ label: "Payment terms"
111
+ type: text
112
+ kind: data
113
+ means: "Payment terms on this customer's account β€” Net 30, for example."
114
+ - key: last_order
115
+ label: "Last order"
116
+ type: date
117
+ kind: data
118
+ means: "Date of the most recent confirmed order."
119
+ - key: overdue_days
120
+ label: "Overdue days"
121
+ type: int
122
+ kind: data
123
+ means: "How many days late this customer is running against their own usual ordering rhythm."
124
+ - key: est_missed
125
+ label: "Est. missed $"
126
+ type: currency
127
+ kind: data
128
+ means: "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
129
+ - key: ar_open
130
+ label: "AR current $"
131
+ type: currency
132
+ kind: data
133
+ means: "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
134
+ - key: ar_overdue
135
+ label: "AR overdue $"
136
+ type: currency
137
+ kind: data
138
+ means: "Invoiced money past due β€” same basis as the Collections page."
139
+ - key: ar_outstanding
140
+ label: "AR outstanding $"
141
+ type: currency
142
+ kind: data
143
+ means: "Total invoiced money owed right now: AR current $ plus AR overdue $."
144
+ - key: ar_exposure
145
+ label: "Credit exposure $"
146
+ type: currency
147
+ kind: data
148
+ means: "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
149
+ - key: ar_aged_1_30
150
+ label: "1-30 days $"
151
+ type: currency
152
+ kind: data
153
+ means: "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
154
+ - key: ar_aged_31_60
155
+ label: "31-60 days $"
156
+ type: currency
157
+ kind: data
158
+ means: "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
159
+ - key: ar_aged_61_90
160
+ label: "61-90 days $"
161
+ type: currency
162
+ kind: data
163
+ means: "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
164
+ - key: ar_aged_90_plus
165
+ label: "90+ days $"
166
+ type: currency
167
+ kind: data
168
+ means: "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
169
+ - key: days_to_pay
170
+ label: "Days to pay"
171
+ type: int
172
+ kind: data
173
+ means: "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
174
+ - key: top_category
175
+ label: "Top category"
176
+ type: text
177
+ kind: data
178
+ means: "The category this customer spent the most on in the last 12 months."
179
+ - key: top_category_pct
180
+ label: "Top category %"
181
+ type: pct
182
+ kind: data
183
+ means: "Share of last-12-months spend that went to the top category."
184
+ - key: sku_count
185
+ label: "SKUs bought"
186
+ type: int
187
+ kind: data
188
+ means: "Distinct products bought in the last 12 months."
189
+ - key: top_sku
190
+ label: "Top SKU"
191
+ type: text
192
+ kind: data
193
+ means: "The product this customer spent the most on in the last 12 months."
194
+ - key: days_since
195
+ label: "Days since order"
196
+ type: int
197
+ kind: data
198
+ means: "Days since the last confirmed order."
199
+ - key: typical_gap_days
200
+ label: "Typical gap days"
201
+ type: int
202
+ kind: data
203
+ means: "Days this customer usually goes between orders, from their own history."
204
+ - key: notes
205
+ label: "Notes"
206
+ type: text
207
+ kind: data
208
+ means: "Your notes on this customer. Saved in this app only, visible only to you."
209
+
210
+ ai_context: >
211
+ The customer REGISTRY as a person sees it in the app β€” the database labelled 'Odoo customers'.
212
+ Use it to answer 'who is this customer', 'which customers exist', 'which agent owns them', 'where are they', and for the AR columns it carries (ar_open, ar_overdue, days_to_pay), which are reconciled by modules/ar.py.
213
+ β›” For anything WINDOWED or at line grain go to sales_lines / customer_invoices / receivables β€” those topics own the time dimension and this one has none.
platform/model/topics/odoo_invoices.yml CHANGED
@@ -1,98 +1,98 @@
1
- # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_invoices
9
- label: "Odoo invoices"
10
- entity: account.move
11
- # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_invoices
13
- subject: "odoo:account.move.customer"
14
- grain: "one row per posted CUSTOMER invoice or credit note (document grain)"
15
- scope:
16
- population: "move_type in (out_invoice, out_refund) AND state = posted"
17
- signs: "amount_residual_signed is POSITIVE for an invoice and NEGATIVE for a credit note, so summing it nets the credit notes correctly"
18
- store:
19
- table: account_move
20
- alias: m
21
- date_col: "m.invoice_date"
22
- dims:
23
- customer: {label: "Customer"}
24
- payment_state: {label: "Payment state"}
25
- move_type: {label: "Document type"}
26
-
27
- # key / label / type / kind, derived from the grid contract. `kind` says where the
28
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
- # from another topic; a `link` points at another database.
30
- fields:
31
- - key: invoice_no
32
- label: "Invoice"
33
- type: text
34
- kind: data
35
- - key: odoo_id
36
- label: "Odoo ID"
37
- type: int
38
- kind: data
39
- means: "The `account.move` id. Also this row's id."
40
- - key: customer
41
- label: "Customer"
42
- type: text
43
- kind: data
44
- - key: partner_id
45
- label: "Odoo partner id"
46
- type: int
47
- kind: data
48
- - key: invoice_date
49
- label: "Invoice date"
50
- type: date
51
- kind: data
52
- - key: due_date
53
- label: "Due date"
54
- type: date
55
- kind: data
56
- - key: residual
57
- label: "Outstanding $"
58
- type: currency
59
- kind: data
60
- means: "Odoo's signed residual. Exactly 0 on every settled document, which is why AR rollups need no filter."
61
- - key: amount_untaxed
62
- label: "Invoiced $"
63
- type: currency
64
- kind: data
65
- - key: payment_state
66
- label: "Payment state"
67
- type: select
68
- kind: data
69
- - key: move_type
70
- label: "Document"
71
- type: select
72
- kind: data
73
- - key: wholesale_scope
74
- label: "In wholesale scope"
75
- type: checkbox
76
- kind: data
77
- means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
78
- - key: origin_order
79
- label: "Source order"
80
- type: text
81
- kind: data
82
- means: "Odoo's `invoice_origin` - usually the order name, sometimes blank."
83
- - key: order_link
84
- label: "Order record"
85
- type: link
86
- kind: link
87
- to_grid: ut_odoo_orders
88
- - key: refreshed
89
- label: "Refreshed"
90
- type: date
91
- kind: data
92
- means: "When this row was last reconciled against Odoo."
93
-
94
- ai_context: >
95
- Customer invoices at DOCUMENT grain β€” one row per invoice, not per line.
96
- Use it for AR questions: what is open, what is overdue, what was invoiced.
97
- For anything at product or line grain use invoice_lines; for the reconciled receivables view use receivables.
98
- `origin_order` is the single stored key that joins an invoice back to the order it was raised from.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_invoices
9
+ label: "Odoo invoices"
10
+ entity: account.move
11
+ # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_invoices
13
+ subject: "odoo:account.move.customer"
14
+ grain: "one row per posted CUSTOMER invoice or credit note (document grain)"
15
+ scope:
16
+ population: "move_type in (out_invoice, out_refund) AND state = posted"
17
+ signs: "amount_residual_signed is POSITIVE for an invoice and NEGATIVE for a credit note, so summing it nets the credit notes correctly"
18
+ store:
19
+ table: account_move
20
+ alias: m
21
+ date_col: "m.invoice_date"
22
+ dims:
23
+ customer: {label: "Customer"}
24
+ payment_state: {label: "Payment state"}
25
+ move_type: {label: "Document type"}
26
+
27
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
28
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
+ # from another topic; a `link` points at another database.
30
+ fields:
31
+ - key: invoice_no
32
+ label: "Invoice"
33
+ type: text
34
+ kind: data
35
+ - key: odoo_id
36
+ label: "Odoo ID"
37
+ type: int
38
+ kind: data
39
+ means: "The `account.move` id. Also this row's id."
40
+ - key: customer
41
+ label: "Customer"
42
+ type: text
43
+ kind: data
44
+ - key: partner_id
45
+ label: "Odoo partner id"
46
+ type: int
47
+ kind: data
48
+ - key: invoice_date
49
+ label: "Invoice date"
50
+ type: date
51
+ kind: data
52
+ - key: due_date
53
+ label: "Due date"
54
+ type: date
55
+ kind: data
56
+ - key: residual
57
+ label: "Outstanding $"
58
+ type: currency
59
+ kind: data
60
+ means: "Odoo's signed residual. Exactly 0 on every settled document, which is why AR rollups need no filter."
61
+ - key: amount_untaxed
62
+ label: "Invoiced $"
63
+ type: currency
64
+ kind: data
65
+ - key: payment_state
66
+ label: "Payment state"
67
+ type: select
68
+ kind: data
69
+ - key: move_type
70
+ label: "Document"
71
+ type: select
72
+ kind: data
73
+ - key: wholesale_scope
74
+ label: "In wholesale scope"
75
+ type: checkbox
76
+ kind: data
77
+ means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
78
+ - key: origin_order
79
+ label: "Source order"
80
+ type: text
81
+ kind: data
82
+ means: "Odoo's `invoice_origin` - usually the order name, sometimes blank."
83
+ - key: order_link
84
+ label: "Order record"
85
+ type: link
86
+ kind: link
87
+ to_grid: ut_odoo_orders
88
+ - key: refreshed
89
+ label: "Refreshed"
90
+ type: date
91
+ kind: data
92
+ means: "When this row was last reconciled against Odoo."
93
+
94
+ ai_context: >
95
+ Customer invoices at DOCUMENT grain β€” one row per invoice, not per line.
96
+ Use it for AR questions: what is open, what is overdue, what was invoiced.
97
+ For anything at product or line grain use invoice_lines; for the reconciled receivables view use receivables.
98
+ `origin_order` is the single stored key that joins an invoice back to the order it was raised from.
platform/model/topics/odoo_orders.yml CHANGED
@@ -1,87 +1,87 @@
1
- # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_orders
9
- label: "Odoo orders"
10
- entity: sale.order
11
- # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_orders
13
- subject: "odoo:sale.order"
14
- grain: "one row per confirmed sales order (document grain)"
15
- scope:
16
- population: "state in (sale, done) β€” quotations and cancellations excluded"
17
- vs_sales_orders: "⚠ DISTINCT from the `sales_orders` topic: that one is the wholesale-scoped analytical view (teams 5/6, house accounts excluded); THIS one is the registry a person opens as a grid, and carries the row set the app shows"
18
- store:
19
- table: sale_order
20
- alias: o
21
- date_col: "o.date_order"
22
- dims:
23
- customer: {label: "Customer"}
24
- team: {label: "Business unit"}
25
- state: {label: "State"}
26
- invoice_status: {label: "Invoice status"}
27
-
28
- # key / label / type / kind, derived from the grid contract. `kind` says where the
29
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
30
- # from another topic; a `link` points at another database.
31
- fields:
32
- - key: order_no
33
- label: "Order"
34
- type: text
35
- kind: data
36
- - key: odoo_id
37
- label: "Odoo ID"
38
- type: int
39
- kind: data
40
- means: "The `sale.order` id. Also this row's id."
41
- - key: customer
42
- label: "Customer"
43
- type: text
44
- kind: data
45
- - key: partner_id
46
- label: "Odoo partner id"
47
- type: int
48
- kind: data
49
- - key: order_date
50
- label: "Order date"
51
- type: date
52
- kind: data
53
- - key: amount_untaxed
54
- label: "Order $"
55
- type: currency
56
- kind: data
57
- - key: team
58
- label: "Business unit"
59
- type: select
60
- kind: data
61
- - key: state
62
- label: "State"
63
- type: select
64
- kind: data
65
- - key: invoice_status
66
- label: "Invoice status"
67
- type: select
68
- kind: data
69
- - key: wholesale_scope
70
- label: "In wholesale scope"
71
- type: checkbox
72
- kind: data
73
- means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
74
- - key: invoices
75
- label: "Invoices"
76
- type: link
77
- kind: link
78
- to_grid: ut_odoo_invoices
79
- - key: refreshed
80
- label: "Refreshed"
81
- type: date
82
- kind: data
83
- means: "When this row was last reconciled against Odoo."
84
-
85
- ai_context: >
86
- Sales orders at DOCUMENT grain, as the app's grid shows them.
87
- β›” If you are asked for revenue, prefer `sales_lines` (line grain, wholesale scope) or `sales_orders` β€” this topic exists to describe the ORDER RECORD a person can open, and its scope is the grid's, not the analytical model's.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_orders
9
+ label: "Odoo orders"
10
+ entity: sale.order
11
+ # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_orders
13
+ subject: "odoo:sale.order"
14
+ grain: "one row per confirmed sales order (document grain)"
15
+ scope:
16
+ population: "state in (sale, done) β€” quotations and cancellations excluded"
17
+ vs_sales_orders: "⚠ DISTINCT from the `sales_orders` topic: that one is the wholesale-scoped analytical view (teams 5/6, house accounts excluded); THIS one is the registry a person opens as a grid, and carries the row set the app shows"
18
+ store:
19
+ table: sale_order
20
+ alias: o
21
+ date_col: "o.date_order"
22
+ dims:
23
+ customer: {label: "Customer"}
24
+ team: {label: "Business unit"}
25
+ state: {label: "State"}
26
+ invoice_status: {label: "Invoice status"}
27
+
28
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
29
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
30
+ # from another topic; a `link` points at another database.
31
+ fields:
32
+ - key: order_no
33
+ label: "Order"
34
+ type: text
35
+ kind: data
36
+ - key: odoo_id
37
+ label: "Odoo ID"
38
+ type: int
39
+ kind: data
40
+ means: "The `sale.order` id. Also this row's id."
41
+ - key: customer
42
+ label: "Customer"
43
+ type: text
44
+ kind: data
45
+ - key: partner_id
46
+ label: "Odoo partner id"
47
+ type: int
48
+ kind: data
49
+ - key: order_date
50
+ label: "Order date"
51
+ type: date
52
+ kind: data
53
+ - key: amount_untaxed
54
+ label: "Order $"
55
+ type: currency
56
+ kind: data
57
+ - key: team
58
+ label: "Business unit"
59
+ type: select
60
+ kind: data
61
+ - key: state
62
+ label: "State"
63
+ type: select
64
+ kind: data
65
+ - key: invoice_status
66
+ label: "Invoice status"
67
+ type: select
68
+ kind: data
69
+ - key: wholesale_scope
70
+ label: "In wholesale scope"
71
+ type: checkbox
72
+ kind: data
73
+ means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
74
+ - key: invoices
75
+ label: "Invoices"
76
+ type: link
77
+ kind: link
78
+ to_grid: ut_odoo_invoices
79
+ - key: refreshed
80
+ label: "Refreshed"
81
+ type: date
82
+ kind: data
83
+ means: "When this row was last reconciled against Odoo."
84
+
85
+ ai_context: >
86
+ Sales orders at DOCUMENT grain, as the app's grid shows them.
87
+ β›” If you are asked for revenue, prefer `sales_lines` (line grain, wholesale scope) or `sales_orders` β€” this topic exists to describe the ORDER RECORD a person can open, and its scope is the grid's, not the analytical model's.
platform/model/topics/odoo_products.yml CHANGED
@@ -1,147 +1,147 @@
1
- # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_products
9
- label: "Odoo products"
10
- entity: product.product
11
- # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
- grid: product_data
13
- subject: "odoo:product.product"
14
- grain: "one row per SKU in the active catalogue β€” a CATALOGUE, not a dated event stream"
15
- scope:
16
- population: "every ACTIVE product, sold or not (measured 5,862). Deliberately not the sales-window universe: that lands at 2,717 and hides ~2,550 SKUs that have never sold in wholesale scope"
17
- identity: "the SKU code (`default_code`). ⚠ 33 active products carry NO code and are keyed `pid:<odoo product id>` instead, so every active product has exactly one row and none is dropped"
18
- merged: "W33-T44 retired `ut_odoo_products`, which presented this same subject on Odoo's product_id. This topic describes the SURVIVING database, `product_data`"
19
- no_date: "a catalogue has no date dimension; ask sales_lines for movement"
20
- store:
21
- table: product_product
22
- alias: pp
23
- # NO date_col β€” a registry is not a dated event stream. Stated rather than
24
- # omitted, so its absence reads as a fact and not as an unfinished file.
25
- dims:
26
- category: {label: "Category"}
27
- supplier: {label: "Supplier"}
28
-
29
- # key / label / type / kind, derived from the grid contract. `kind` says where the
30
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
31
- # from another topic; a `link` points at another database.
32
- fields:
33
- - key: code
34
- label: "SKU"
35
- type: text
36
- kind: data
37
- means: "The SKU code β€” the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
38
- - key: product
39
- label: "Product"
40
- type: text
41
- kind: data
42
- means: "Product name as it appears in Odoo."
43
- - key: category
44
- label: "Category"
45
- type: select
46
- kind: data
47
- means: "Product category; '(uncategorized)' when Odoo carries none."
48
- - key: supplier
49
- label: "Supplier"
50
- type: text
51
- kind: data
52
- means: "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet."
53
- - key: origin_country
54
- label: "Country"
55
- type: text
56
- kind: data
57
- means: "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet."
58
- - key: lead_days
59
- label: "Lead time (days)"
60
- type: int
61
- kind: data
62
- means: "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone."
63
- - key: first_cost
64
- label: "First cost"
65
- type: currency
66
- kind: data
67
- means: "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone."
68
- - key: price_fisch
69
- label: "Fisch price"
70
- type: currency
71
- kind: data
72
- means: "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
73
- - key: price_royal_1
74
- label: "Royal 1 price"
75
- type: currency
76
- kind: data
77
- means: "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
78
- - key: price_royal_2
79
- label: "Royal 2 price"
80
- type: currency
81
- kind: data
82
- means: "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
83
- - key: rev_ytd
84
- label: "Revenue YTD"
85
- type: currency
86
- kind: data
87
- means: "Year-to-date revenue for this SKU, BU-scoped when the caller is."
88
- - key: rev_ly
89
- label: "Revenue LY"
90
- type: currency
91
- kind: data
92
- means: "Same period last year β€” seasonal wholesale compares like for like."
93
- - key: yoy_pct
94
- label: "YoY %"
95
- type: pct
96
- kind: data
97
- means: "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
98
- - key: qty_ytd
99
- label: "Units YTD"
100
- type: int
101
- kind: data
102
- means: "Units sold year to date."
103
- - key: orders_ytd
104
- label: "Orders YTD"
105
- type: int
106
- kind: data
107
- means: "Distinct orders containing this SKU, year to date."
108
- - key: on_hand
109
- label: "On hand"
110
- type: int
111
- kind: data
112
- means: "Units in stock. CONSOLIDATED β€” one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
113
- - key: unit_cost
114
- label: "Unit cost"
115
- type: currency
116
- kind: data
117
- means: "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
118
- - key: inv_value
119
- label: "Stock value"
120
- type: currency
121
- kind: data
122
- means: "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
123
- - key: qty_ltm
124
- label: "Units LTM"
125
- type: int
126
- kind: data
127
- means: "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
128
- - key: dos
129
- label: "Days of supply"
130
- type: int
131
- kind: data
132
- means: "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
133
- - key: cover_gap_d
134
- label: "Cover gap (days)"
135
- type: int
136
- kind: data
137
- means: "Days of supply minus lead time. Negative means it runs out before a reorder lands."
138
- - key: stock_bucket
139
- label: "Stock status"
140
- type: select
141
- kind: data
142
- means: "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
143
-
144
- ai_context: >
145
- The product CATALOGUE β€” the database labelled 'Odoo products'.
146
- Use it for 'what do we sell', 'what does it cost' (first_cost/unit_cost), 'who supplies it', 'what is on hand' and the stock-coverage columns.
147
- β›” Revenue and unit columns here are WINDOWED SNAPSHOTS (rev_ytd, qty_ltm); for any other window, or for line grain, go to sales_lines.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_products
9
+ label: "Odoo products"
10
+ entity: product.product
11
+ # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
+ grid: product_data
13
+ subject: "odoo:product.product"
14
+ grain: "one row per SKU in the active catalogue β€” a CATALOGUE, not a dated event stream"
15
+ scope:
16
+ population: "every ACTIVE product, sold or not (measured 5,862). Deliberately not the sales-window universe: that lands at 2,717 and hides ~2,550 SKUs that have never sold in wholesale scope"
17
+ identity: "the SKU code (`default_code`). ⚠ 33 active products carry NO code and are keyed `pid:<odoo product id>` instead, so every active product has exactly one row and none is dropped"
18
+ merged: "W33-T44 retired `ut_odoo_products`, which presented this same subject on Odoo's product_id. This topic describes the SURVIVING database, `product_data`"
19
+ no_date: "a catalogue has no date dimension; ask sales_lines for movement"
20
+ store:
21
+ table: product_product
22
+ alias: pp
23
+ # NO date_col β€” a registry is not a dated event stream. Stated rather than
24
+ # omitted, so its absence reads as a fact and not as an unfinished file.
25
+ dims:
26
+ category: {label: "Category"}
27
+ supplier: {label: "Supplier"}
28
+
29
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
30
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
31
+ # from another topic; a `link` points at another database.
32
+ fields:
33
+ - key: code
34
+ label: "SKU"
35
+ type: text
36
+ kind: data
37
+ means: "The SKU code β€” the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
38
+ - key: product
39
+ label: "Product"
40
+ type: text
41
+ kind: data
42
+ means: "Product name as it appears in Odoo."
43
+ - key: category
44
+ label: "Category"
45
+ type: select
46
+ kind: data
47
+ means: "Product category; '(uncategorized)' when Odoo carries none."
48
+ - key: supplier
49
+ label: "Supplier"
50
+ type: text
51
+ kind: data
52
+ means: "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet."
53
+ - key: origin_country
54
+ label: "Country"
55
+ type: text
56
+ kind: data
57
+ means: "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet."
58
+ - key: lead_days
59
+ label: "Lead time (days)"
60
+ type: int
61
+ kind: data
62
+ means: "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone."
63
+ - key: first_cost
64
+ label: "First cost"
65
+ type: currency
66
+ kind: data
67
+ means: "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone."
68
+ - key: price_fisch
69
+ label: "Fisch price"
70
+ type: currency
71
+ kind: data
72
+ means: "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
73
+ - key: price_royal_1
74
+ label: "Royal 1 price"
75
+ type: currency
76
+ kind: data
77
+ means: "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
78
+ - key: price_royal_2
79
+ label: "Royal 2 price"
80
+ type: currency
81
+ kind: data
82
+ means: "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
83
+ - key: rev_ytd
84
+ label: "Revenue YTD"
85
+ type: currency
86
+ kind: data
87
+ means: "Year-to-date revenue for this SKU, BU-scoped when the caller is."
88
+ - key: rev_ly
89
+ label: "Revenue LY"
90
+ type: currency
91
+ kind: data
92
+ means: "Same period last year β€” seasonal wholesale compares like for like."
93
+ - key: yoy_pct
94
+ label: "YoY %"
95
+ type: pct
96
+ kind: data
97
+ means: "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
98
+ - key: qty_ytd
99
+ label: "Units YTD"
100
+ type: int
101
+ kind: data
102
+ means: "Units sold year to date."
103
+ - key: orders_ytd
104
+ label: "Orders YTD"
105
+ type: int
106
+ kind: data
107
+ means: "Distinct orders containing this SKU, year to date."
108
+ - key: on_hand
109
+ label: "On hand"
110
+ type: int
111
+ kind: data
112
+ means: "Units in stock. CONSOLIDATED β€” one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
113
+ - key: unit_cost
114
+ label: "Unit cost"
115
+ type: currency
116
+ kind: data
117
+ means: "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
118
+ - key: inv_value
119
+ label: "Stock value"
120
+ type: currency
121
+ kind: data
122
+ means: "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
123
+ - key: qty_ltm
124
+ label: "Units LTM"
125
+ type: int
126
+ kind: data
127
+ means: "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
128
+ - key: dos
129
+ label: "Days of supply"
130
+ type: int
131
+ kind: data
132
+ means: "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
133
+ - key: cover_gap_d
134
+ label: "Cover gap (days)"
135
+ type: int
136
+ kind: data
137
+ means: "Days of supply minus lead time. Negative means it runs out before a reorder lands."
138
+ - key: stock_bucket
139
+ label: "Stock status"
140
+ type: select
141
+ kind: data
142
+ means: "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
143
+
144
+ ai_context: >
145
+ The product CATALOGUE β€” the database labelled 'Odoo products'.
146
+ Use it for 'what do we sell', 'what does it cost' (first_cost/unit_cost), 'who supplies it', 'what is on hand' and the stock-coverage columns.
147
+ β›” Revenue and unit columns here are WINDOWED SNAPSHOTS (rev_ytd, qty_ltm); for any other window, or for line grain, go to sales_lines.
platform/model/topics/odoo_vendors.yml CHANGED
@@ -1,103 +1,103 @@
1
- # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_vendors
9
- label: "Odoo vendors"
10
- entity: res.partner
11
- # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_vendors
13
- subject: "odoo:res.partner.vendor"
14
- grain: "one row per partner we have posted a vendor bill to"
15
- scope:
16
- population: "DERIVED FROM THE BILLS β€” a partner with no posted vendor bill has no payable history to show, so the bill is the population"
17
- asymmetry: "deliberately unlike odoo_customers, which is NOT derived from its documents; there is no second document universe for vendors"
18
- overlap: "measured: only 9 of the vendors also appear in the customer population, so a vendor is not a customer row"
19
- store:
20
- table: res_partner
21
- alias: p
22
- # NO date_col β€” a registry is not a dated event stream. Stated rather than
23
- # omitted, so its absence reads as a fact and not as an unfinished file.
24
- dims:
25
- country: {label: "Country"}
26
-
27
- # key / label / type / kind, derived from the grid contract. `kind` says where the
28
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
- # from another topic; a `link` points at another database.
30
- fields:
31
- - key: vendor
32
- label: "Vendor"
33
- type: text
34
- kind: data
35
- - key: odoo_id
36
- label: "Odoo ID"
37
- type: int
38
- kind: data
39
- means: "The `res.partner` id. Also this row's id."
40
- - key: vendor_id
41
- label: "Odoo vendor id"
42
- type: int
43
- kind: data
44
- - key: country
45
- label: "Country"
46
- type: text
47
- kind: data
48
- - key: email
49
- label: "Email"
50
- type: text
51
- kind: data
52
- - key: phone
53
- label: "Phone"
54
- type: text
55
- kind: data
56
- - key: mobile
57
- label: "Mobile"
58
- type: text
59
- kind: data
60
- - key: vat
61
- label: "Tax ID"
62
- type: text
63
- kind: data
64
- means: "Odoo `vat` β€” the vendor's tax/VAT registration number."
65
- - key: vendor_ref
66
- label: "Vendor reference"
67
- type: text
68
- kind: data
69
- means: "Odoo `res.partner.ref` β€” our internal reference for this vendor."
70
- - key: website
71
- label: "Website"
72
- type: url
73
- kind: data
74
- - key: street
75
- label: "Street"
76
- type: text
77
- kind: data
78
- - key: street2
79
- label: "Street 2"
80
- type: text
81
- kind: data
82
- - key: city
83
- label: "City"
84
- type: text
85
- kind: data
86
- - key: zip
87
- label: "ZIP"
88
- type: text
89
- kind: data
90
- - key: bills
91
- label: "Bills"
92
- type: link
93
- kind: link
94
- to_grid: ut_odoo_bills
95
- - key: refreshed
96
- label: "Refreshed"
97
- type: date
98
- kind: data
99
- means: "When this row was last reconciled against Odoo."
100
-
101
- ai_context: >
102
- The vendor/supplier registry, with contact and tax identity (W33-T48 widened it from 5 columns to 15 after a census found 76 populated fields on the underlying partner)
103
- ⚠ The contact columns are SPARSE by nature β€” measured 65/393 with an email, 13/393 with a tax id β€” so 'blank' means Odoo has no value, never that the sync failed.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) β€” an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # β›” THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_vendors
9
+ label: "Odoo vendors"
10
+ entity: res.partner
11
+ # The database this topic DESCRIBES β€” the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_vendors
13
+ subject: "odoo:res.partner.vendor"
14
+ grain: "one row per partner we have posted a vendor bill to"
15
+ scope:
16
+ population: "DERIVED FROM THE BILLS β€” a partner with no posted vendor bill has no payable history to show, so the bill is the population"
17
+ asymmetry: "deliberately unlike odoo_customers, which is NOT derived from its documents; there is no second document universe for vendors"
18
+ overlap: "measured: only 9 of the vendors also appear in the customer population, so a vendor is not a customer row"
19
+ store:
20
+ table: res_partner
21
+ alias: p
22
+ # NO date_col β€” a registry is not a dated event stream. Stated rather than
23
+ # omitted, so its absence reads as a fact and not as an unfinished file.
24
+ dims:
25
+ country: {label: "Country"}
26
+
27
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
28
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
+ # from another topic; a `link` points at another database.
30
+ fields:
31
+ - key: vendor
32
+ label: "Vendor"
33
+ type: text
34
+ kind: data
35
+ - key: odoo_id
36
+ label: "Odoo ID"
37
+ type: int
38
+ kind: data
39
+ means: "The `res.partner` id. Also this row's id."
40
+ - key: vendor_id
41
+ label: "Odoo vendor id"
42
+ type: int
43
+ kind: data
44
+ - key: country
45
+ label: "Country"
46
+ type: text
47
+ kind: data
48
+ - key: email
49
+ label: "Email"
50
+ type: text
51
+ kind: data
52
+ - key: phone
53
+ label: "Phone"
54
+ type: text
55
+ kind: data
56
+ - key: mobile
57
+ label: "Mobile"
58
+ type: text
59
+ kind: data
60
+ - key: vat
61
+ label: "Tax ID"
62
+ type: text
63
+ kind: data
64
+ means: "Odoo `vat` β€” the vendor's tax/VAT registration number."
65
+ - key: vendor_ref
66
+ label: "Vendor reference"
67
+ type: text
68
+ kind: data
69
+ means: "Odoo `res.partner.ref` β€” our internal reference for this vendor."
70
+ - key: website
71
+ label: "Website"
72
+ type: url
73
+ kind: data
74
+ - key: street
75
+ label: "Street"
76
+ type: text
77
+ kind: data
78
+ - key: street2
79
+ label: "Street 2"
80
+ type: text
81
+ kind: data
82
+ - key: city
83
+ label: "City"
84
+ type: text
85
+ kind: data
86
+ - key: zip
87
+ label: "ZIP"
88
+ type: text
89
+ kind: data
90
+ - key: bills
91
+ label: "Bills"
92
+ type: link
93
+ kind: link
94
+ to_grid: ut_odoo_bills
95
+ - key: refreshed
96
+ label: "Refreshed"
97
+ type: date
98
+ kind: data
99
+ means: "When this row was last reconciled against Odoo."
100
+
101
+ ai_context: >
102
+ The vendor/supplier registry, with contact and tax identity (W33-T48 widened it from 5 columns to 15 after a census found 76 populated fields on the underlying partner)
103
+ ⚠ The contact columns are SPARSE by nature β€” measured 65/393 with an email, 13/393 with a tax id β€” so 'blank' means Odoo has no value, never that the sync failed.
platform/modules/collections_send.py CHANGED
@@ -180,16 +180,20 @@ def load_collection_list(odoo):
180
 
181
 
182
  # --------------------------------------------------------------- statement email
183
- DEFAULT_SUBJECT = 'Statement of Account β€” {company} β€” {month}'
 
 
 
 
184
  DEFAULT_INTRO = (
185
  'Dear {customer},<br><br>'
186
  'Please find below your current statement of account with {company}. '
187
  'According to our records, the following invoices remain open:'
188
  )
189
  DEFAULT_FOOTER = (
190
- 'If you have already sent payment, please disregard this notice β€” and thank you. '
191
  'For any questions about an invoice, simply reply to this email.<br><br>'
192
- 'Thank you for your business,<br>{company} β€” Accounts Receivable'
193
  )
194
 
195
 
 
180
 
181
 
182
  # --------------------------------------------------------------- statement email
183
+ # β›” STANDING RULE 2, AND THIS IS THE ONE STRING IN THE PRODUCT THAT LEAVES THE BUILDING.
184
+ # Every other finding `web_prose` reports is copy on a screen somebody here opens; this is the
185
+ # SUBJECT LINE of mail queued to a real debtor, over the single sanctioned Odoo write. It read
186
+ # 'Statement of Account (em dash) {company} (em dash) {month}' until W36-T42.
187
+ DEFAULT_SUBJECT = 'Statement of Account from {company}, {month}'
188
  DEFAULT_INTRO = (
189
  'Dear {customer},<br><br>'
190
  'Please find below your current statement of account with {company}. '
191
  'According to our records, the following invoices remain open:'
192
  )
193
  DEFAULT_FOOTER = (
194
+ 'If you have already sent payment, please disregard this notice, and thank you. '
195
  'For any questions about an invoice, simply reply to this email.<br><br>'
196
+ 'Thank you for your business,<br>{company}<br>Accounts Receivable'
197
  )
198
 
199
 
platform/modules/product_data.py CHANGED
@@ -1,804 +1,804 @@
1
- """modules/product_data.py β€” the PRODUCT table's pool (wave 15 item 9/10, contract C-TOPIC).
2
-
3
- The second object on the table-page factory: same grid, same engine, same permission wall β€” the
4
- only difference is the field schema and the identity. `modules/customer_data.pool()` is the
5
- template and this deliberately mirrors its signature and its row shape.
6
-
7
- β›” THREE DECISIONS THIS FILE HAD TO MAKE, EACH RECORDED BECAUSE A LATER READER WILL WONDER.
8
-
9
- 1. **THE IDENTITY IS A SKU CODE, WHICH IS A STRING, AND THE GRID WANTS AN INTEGER `pid`.**
10
- Cohort membership, `allowed_pids`, the measure channel and `rows_from_pool` all key on an
11
- integer. So each row carries BOTH: `pid` (a stable CRC32 of the code, so the same SKU gets the
12
- same id on every pull and across processes β€” never an enumeration index, which would reshuffle
13
- whenever the catalogue changes) and `code`, the real business key, as a visible column.
14
- `_assert_no_pid_collision` fails the BUILD rather than the read: two SKUs sharing a pid would
15
- silently merge in every downstream set operation, and a loud build failure is the only version
16
- of that anyone would notice.
17
-
18
- 2. **THE SCOPE RULE β€” inventory columns are CONSOLIDATED and are therefore OMITTED for a
19
- BU-scoped caller.** `products.directory(t, team_id)` is brand-shaped; `inventory.sku_inventory`
20
- is explicitly NOT (its own docstring: "on-hand stock is one physical warehouse, not
21
- brand-tagged"). Joining them for a Fisch-only user would put BU-shaped revenue beside
22
- company-wide stock IN THE SAME ROW β€” the mixed-scope value defect wave 15's amendment 3 exists
23
- for, arriving through a different door. The honest options were "omit" or "label the columns
24
- company-wide"; omit is the fail-closed one, and a column that is absent asks a question,
25
- whereas a column that is silently company-wide answers one wrongly.
26
-
27
- 3. **WHAT IS NOT HERE, NAMED RATHER THAN QUIETLY MISSING.** R4 lists ~20 fields. Vendor and
28
- COUNTRY are not in Odoo at all β€” they live in the inventory WORKBOOK
29
- ([[odoo-vendor-country-origin]]) and need a loader this module deliberately does not invent.
30
- Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost
31
- allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it
32
- does not pretend to cover columns that are absent.
33
-
34
- 4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12,
35
- 2026-08-11). It was `for r in products.directory(...)` β€” and `directory()`'s row set IS the
36
- union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid
37
- showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise:
38
-
39
- Β· β›” THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date
40
- window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics
41
- that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because
42
- ~2,550 active SKUs have never sold in wholesale scope at all.
43
- · ⭐ REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the
44
- revenue universe carries measured numbers INCLUDING a real 0.0 β€” it sold last year and
45
- not this one, and that zero is a measurement. A row that appears in NO revenue read
46
- carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the
47
- client renders as an empty cell. Blank is an admission; zero is a measurement.
48
- · ⚠ A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where
49
- that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW
50
- UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product`
51
- carries no team, so a catalogue cannot be BU-shaped at all β€” which is exactly why the two
52
- sources are JOINED rather than merged.
53
- """
54
- import json
55
- import zlib
56
- from pathlib import Path
57
-
58
- import core.odoo as O
59
- import core.periods as P
60
- import core.shared_overlay as shared_overlay
61
- import core.table_store as table_store
62
- import modules.products as products
63
-
64
- #: POOL-ROW KEYS that exist ONLY on a consolidated pull. See decision 2.
65
- #: `cover_gap_d` / `buy_now` (wave 17) join it because both are computed FROM `dos`, and a buy
66
- #: signal built on a stock number the caller cannot see would be a recommendation nobody could
67
- #: check ([[no-unverifiable-aggregates]]).
68
- #:
69
- #: ⚠ ROW KEYS, NOT FIELDS, and the distinction started mattering on 2026-08-03. Two readers:
70
- #: `verify_perm_scope` asserts the shape of a POOL ROW against this, and `routes_products`
71
- #: narrows the FIELD list with it. Since owner item 5 `buy_now` is a row key with no Field β€”
72
- #: computed as `validate()`'s oracle, projected onto no wire β€” so it belongs here for the first
73
- #: reader and is inert for the second. Removing it would stop the scope gate proving that a
74
- #: BU-scoped pull withholds it.
75
- #:
76
- #: ⭐⭐ OWNER RULING 2026-08-11 β€” THIS SET IS NOW EMPTY, AND THAT IS THE POINT. Verbatim:
77
- #: *"Just scope any inventory with Sales from Fisch, leave the rest."* Withholding the whole
78
- #: inventory block from a BU-scoped reader meant a Fisch salesperson could not see whether
79
- #: anything was IN STOCK, which is the first question they ask. The ruling splits the block by
80
- #: what a business unit can actually shape:
81
- #:
82
- #: * `on_hand` / `unit_cost` / `inv_value` β€” ONE physical warehouse, no Fisch shelf and no
83
- #: Royal shelf. Served UNSCOPED to everybody, identical in both pulls. ("leave the rest")
84
- #: * `qty_ltm` / `dos` / `stock_bucket` / `cover_gap_d` / `buy_now` β€” all SALES-derived, so a
85
- #: BU-scoped pull recomputes them from THAT unit's LTM units. ("scope any inventory with
86
- #: Sales from Fisch")
87
- #:
88
- #: This deliberately relaxes decision 2's "never a company-wide value beside a BU-shaped one":
89
- #: stock is not a value a BU can own, and a column that is absent for a Fisch reader asks a
90
- #: question they cannot answer anywhere else in the product. `verify_perm_scope` no longer
91
- #: asserts the columns are WITHHELD β€” it asserts the split above, which is a stronger claim and
92
- #: cannot pass on an empty scoped payload (the shape the old rule and an outage share).
93
- CONSOLIDATED_ONLY = ()
94
-
95
- #: β›” POOL-ROW KEYS THAT DELIBERATELY HAVE NO FIELD β€” they must never reach a browser.
96
- #:
97
- #: `rows_from_pool` projects strictly through the field contract, so "no Field" already means
98
- #: "no cell on the wire". This tuple makes that a CHECKED fact rather than a consequence nobody
99
- #: is watching: `verify_perm_scope` asserts every member is absent from BOTH product contracts,
100
- #: and that every OTHER `CONSOLIDATED_ONLY` key is present in the consolidated one.
101
- #:
102
- #: Written because the two tuples silently diverged the moment owner item 5 retired `buy_now`'s
103
- #: Field while keeping its computation, and the gate β€” which was asserting a FIELD property from
104
- #: a ROW-key list β€” went red with no way to tell a deliberate divergence from a dropped column.
105
- UNSHIPPED_ROW_KEYS = ("buy_now",)
106
-
107
- #: Wave 17 (owner item 13, ruling R3) β€” the SUPPLIER MASTER, from the curated mastersheet map
108
- #: `procurement_suppliers.json` (3,963 SKUs; supplier on 3,624, lead time on 3,563). NOT Odoo:
109
- #: the owner confirmed this data came from the Inventory System Mastersheet, which is why
110
- #: `modules/product_data`'s header said Vendor/Country were "NOT HERE, named rather than
111
- #: quietly missing" and needed "a loader this module deliberately does not invent". This is
112
- #: that loader.
113
- #:
114
- #: β›” THESE ARE CONTRACT COLUMNS, NOT USER-CREATED FIELDS, AND THE REASON IS A PERMISSION FACT.
115
- #: A user-created field and its values live in the PER-USER strata (`table_store.workspace`
116
- #: reads `store.get(key)[username]`); only VIEWS are shared. So a shared "Buy list" view that
117
- #: filtered on a user-created column would, for every OTHER account, name a column that does not
118
- #: exist β€” and an unknown column is an INACTIVE condition in the tri-state engine, which IGNORES
119
- #: it and therefore WIDENS. The buy list would silently show the whole catalogue to everyone but
120
- #: its author. Contract columns are identical for every reader, so the view means one thing.
121
- #:
122
- #: ⭐⭐ 2026-08-12 (W30-T36) β€” THE OWNER'S ASK IS NOW DELIVERED, AND THE PARAGRAPH ABOVE IS WHY IT
123
- #: TOOK THREE WAVES. Owner: *"turn this Excel sheet into a User created Field that we can edit."*
124
- #: It was parked because a per-user column silently WIDENS a shared view β€” not because editing was
125
- #: hard. Wave 29's `core/shared_overlay.py` (whose header quotes this very comment) is the stratum
126
- #: that removes the objection: **one value per (row, column) for the whole tenant**, so the column
127
- #: still means ONE thing to every reader and a shared view still filters honestly.
128
- #: β‡’ The four columns below are now `source: "overlay"` + `shared: true` in the canonical
129
- #: contract, their values live in `<TABLE_KEY>__shared`, and the master map is what SEEDS an
130
- #: unedited cell rather than what freezes it. See `SHARED_KEYS` and `_ProductTableStore`.
131
- _SUPPLIER_MAP_PATH = Path(__file__).resolve().parent.parent / "procurement_suppliers.json"
132
- _SUPPLIER_CACHE = {}
133
-
134
-
135
- def supplier_master():
136
- """`{code: {supplier, lead_days, origin_country, first_cost}}`, read once per process.
137
-
138
- Degrades to `{}` when the file is unreadable, matching `_inventory_by_code`: a product table
139
- that will not render because a master map is missing is a worse failure than one with blank
140
- supplier columns.
141
- """
142
- if _SUPPLIER_CACHE:
143
- return _SUPPLIER_CACHE
144
- try:
145
- raw = json.loads(_SUPPLIER_MAP_PATH.read_text(encoding="utf-8"))
146
- except Exception:
147
- return {}
148
- for code, meta in (raw or {}).items():
149
- if not isinstance(meta, dict):
150
- continue
151
- lead = meta.get("lead")
152
- _SUPPLIER_CACHE[str(code)] = {
153
- "supplier": (meta.get("vendor") or "") or None,
154
- "lead_days": int(lead) if isinstance(lead, (int, float)) else None,
155
- "origin_country": (meta.get("country") or "") or None,
156
- "first_cost": meta.get("first_cost"),
157
- }
158
- return _SUPPLIER_CACHE
159
-
160
- #: Wave 16 C-TOPIC β€” the PRODUCT table's OWN workspace bucket. β›” Never the customer one:
161
- #: product pids are CRC32 hashes of SKU codes and customer pids are Odoo partner ids, so in a
162
- #: SHARED overlay bucket a hash collision would silently write a product note onto somebody's
163
- #: customer (or the reverse). Separate store keys make that structurally impossible, which is
164
- #: the whole reason the table-page factory exists ("a new table object gets its own
165
- #: table_store.make('<its>_table_workspace')").
166
- TABLE_KEY = 'product_table_workspace'
167
-
168
- _GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json'
169
- _SHARED_KEYS = None
170
-
171
-
172
- def SHARED_KEYS():
173
- """The product columns whose values are TENANT-WIDE β€” derived from the canonical contract's
174
- own `shared: true`, never typed out a second time.
175
-
176
- β›” IT DELIBERATELY DOES NOT SWALLOW A READ FAILURE. Degrading to `()` would send a shared
177
- write into the per-user stratum with nothing going wrong anywhere β€” the widening defect
178
- reappearing silently, which is the one outcome this whole mechanism exists to prevent. If the
179
- canonical contract is unreadable the product grid cannot render at all (`pd_fields` parses the
180
- same file with no guard), so a raise here costs nothing that was still working.
181
- """
182
- global _SHARED_KEYS
183
- if _SHARED_KEYS is None:
184
- doc = json.loads(_GRID_FIELDS_PATH.read_text(encoding='utf-8'))
185
- _SHARED_KEYS = tuple(f['key'] for f in (doc.get('product_data') or {}).get('fields') or []
186
- if f.get('shared'))
187
- return _SHARED_KEYS
188
-
189
-
190
- class _ProductTableStore(table_store.TableStore):
191
- """The product workspace, with the SHARED columns routed to the tenant-wide stratum.
192
-
193
- ⭐⭐ THIS SUBCLASS IS THE WHOLE OF W30-T36's WRITE PATH, AND THE REASON IT LIVES HERE RATHER
194
- THAN AT A ROUTE IS THAT **THE BROWSER NEVER CALLS `PATCH /products/{pid}`** β€” measured, zero
195
- call sites in `aios-web/web/src`. A cell edit travels `POST /grid/events` β†’ `grid_events.
196
- handle_one` β†’ `_tops(ctx).patch_overlay(...)`, and `_tops` returns `ctx.table`, which
197
- `routes_grid._ctx` sets to `TABLE_OPS` for the product scope. So this object IS the seam both
198
- doors pass through; intercepting at either route would have left the other one writing a
199
- per-user value that only its author could see.
200
-
201
- ⚠ `st=self.st`, NEVER the module default. The shared stratum must resolve to the SAME store
202
- handle as the per-user one it sits beside β€” `_tops`' own comment explains that a split, where
203
- one side is tenant-scoped and the other is not, is worse than a stated residency error because
204
- a user's value would vanish the moment they saved it. Reading `self.st` means both strata move
205
- together the day that singleton gains a tenant handle.
206
- """
207
-
208
- def patch_overlay(self, username, pid, updates):
209
- clean = dict(updates or {})
210
- if not clean:
211
- return
212
- keys = set(SHARED_KEYS())
213
- shared = {k: v for k, v in clean.items() if k in keys}
214
- personal = {k: v for k, v in clean.items() if k not in keys}
215
- if shared:
216
- shared_overlay.put_cells(TABLE_KEY, pid, shared, st=self.st)
217
- if personal:
218
- super().patch_overlay(username, pid, personal)
219
-
220
-
221
- TABLE_OPS = _ProductTableStore(TABLE_KEY)
222
-
223
-
224
- def shared_cells(pids, st=None):
225
- """`{"<pid>": {key: value}}` for the SHARED columns of the rows named by `pids`.
226
-
227
- ⚠ `pids` is required and positional all the way down β€” `shared_overlay.cells` refuses to serve
228
- "everything" by design, and the caller here always holds an already-scoped pool.
229
- """
230
- return shared_overlay.cells(TABLE_KEY, pids, st=st if st is not None else TABLE_OPS.st)
231
-
232
-
233
- def sku_pid(code):
234
- """A stable integer id for a SKU code. CRC32, masked to 31 bits so it is always positive and
235
- always JSON-safe. Stable across processes and pulls, which an enumeration index is not."""
236
- return zlib.crc32(str(code).encode("utf-8")) & 0x7FFFFFFF
237
-
238
-
239
- def _assert_no_pid_collision(rows):
240
- """Two SKUs sharing a pid would MERGE in every set operation downstream β€” cohort membership,
241
- allowed_pids, the measure channel β€” and nothing would report it. Fail the build instead."""
242
- seen = {}
243
- for r in rows:
244
- prior = seen.get(r["pid"])
245
- if prior is not None and prior != r["code"]:
246
- raise ValueError(
247
- f"product_data: pid collision β€” {prior!r} and {r['code']!r} both hash to "
248
- f"{r['pid']}. Downstream set operations would merge them silently; widen the id "
249
- f"before shipping this catalogue.")
250
- seen[r["pid"]] = r["code"]
251
-
252
-
253
- def _inventory_by_code(t):
254
- """`{code: {...}}` from the inventory module, or `{}` if it cannot be read.
255
-
256
- Degrades to empty rather than raising, matching `customer_data._pool_build`'s treatment of
257
- its own slow families: a product table that will not render because inventory is momentarily
258
- unreachable is a worse failure than one with blank stock columns.
259
- """
260
- try:
261
- import modules.inventory as inventory
262
- return inventory.sku_inventory(t=t) or {}
263
- except Exception:
264
- return {}
265
-
266
-
267
- def _bu_ltm_share(t, team_id):
268
- """`{code: 0.0..1.0}` β€” this unit's SHARE of the SKU's last-twelve-months units.
269
-
270
- ⭐ OWNER 2026-08-11: *"scope any inventory with Sales from Fisch"*. The velocity half of the
271
- inventory block has to be re-shaped by a BU fact, and this is that fact.
272
-
273
- β›” A SHARE, NOT THE UNIT COUNT ITSELF β€” AND THE REASON IS A MEASURED IMPOSSIBILITY. My first
274
- version returned `products._sku_rev(...)['qty']` and used it directly as the scoped `qty_ltm`,
275
- leaving the consolidated column on `inventory.sku_inventory`'s own figure. Two readers of one
276
- question ([[one-question-two-normalizers]]): they disagree, and on 5 SKUs of 5,871 the live
277
- check found **Fisch's LTM units EXCEEDING the company's** β€” a subset larger than its superset,
278
- which no reader could explain and no reconciliation could survive.
279
-
280
- Both halves of the ratio come from the SAME read here, so the share is in [0, 1] by
281
- construction and the scoped figure can never exceed the consolidated one. It also leaves the
282
- consolidated column exactly as it was β€” the Inventory page and the Product grid still agree
283
- about company-wide units, which is what "leave the rest" asked for.
284
-
285
- `all_qty == 0` implies `bu_qty == 0` (same reader), so a share of 0 is the honest answer for
286
- "this unit never sold it": `_bucket` turns that into 'No recent sales', not zero cover.
287
-
288
- Degrades to `{}` on any failure, matching `_inventory_by_code` β€” and a missing code then takes
289
- the `_DEFAULT_SHARE` below rather than a silent 0.
290
- """
291
- try:
292
- f, to = P.ltm(t)
293
- allq = {code: (r.get('qty') or 0.0)
294
- for code, r in (products._sku_rev(f, to, None) or {}).items()}
295
- buq = {code: (r.get('qty') or 0.0)
296
- for code, r in (products._sku_rev(f, to, team_id) or {}).items()}
297
- except Exception:
298
- return {}
299
- out = {}
300
- for code, total in allq.items():
301
- out[code] = min(1.0, max(0.0, (buq.get(code, 0.0) / total))) if total > 0 else 0.0
302
- return out
303
-
304
-
305
- #: What a SKU absent from the LTM sales read is worth to a business unit. ZERO β€” it did not sell
306
- #: in anybody's book over the window, so no unit can claim its velocity. Named rather than
307
- #: inlined so the choice is visible: the alternative (1.0, "assume it is all ours") would print
308
- #: company-wide cover on a BU grid, which is the mixed-scope defect this whole rule avoids.
309
- _DEFAULT_SHARE = 0.0
310
-
311
-
312
- def _rescope_inventory(e, share):
313
- """`(qty_ltm, dos, bucket)` recomputed for ONE business unit's sales rate.
314
-
315
- The formulas are `inventory.sku_inventory`'s, applied to a BU-shaped numerator β€” NOT a second
316
- idea of what days-of-supply means. `_bucket` is imported from there for the same reason: two
317
- copies of a threshold table is how the Product grid and the Inventory page start disagreeing
318
- about which SKUs are dead.
319
-
320
- ⚠ `on_hand` is whatever the warehouse holds, unscoped β€” so a Fisch reader's `dos` answers
321
- "how long does ALL our stock last at Fisch's rate", which is the question a Fisch salesperson
322
- actually has. It is deliberately NOT a pro-rated share of the shelf: there is no such shelf,
323
- and inventing one would put a number on screen that no Odoo query could reproduce.
324
- """
325
- on_hand = e.get("on_hand")
326
- if on_hand is None:
327
- return None, None, None # no inventory row for this SKU: blank, never zero
328
- qty = float(e.get("qty_ltm") or 0.0) * float(share or 0.0)
329
- daily = qty / 365.0
330
- if daily > 0:
331
- dos_raw = on_hand / daily
332
- else:
333
- dos_raw = float('inf') if on_hand > 0 else 0.0
334
- try:
335
- import modules.inventory as inventory
336
- bucket = inventory._bucket(dos_raw, on_hand, qty)
337
- except Exception:
338
- bucket = None
339
- return qty, (None if dos_raw == float('inf') else round(float(dos_raw), 0)), bucket
340
-
341
-
342
- def _catalogue_by_code():
343
- """`{code: {'product', 'category'}}` β€” the CATALOGUE universe this pool is built from.
344
-
345
- β›” UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point.
346
- Inventory degrades to `{}` because a product table with blank stock columns beats one that
347
- will not render. The CATALOGUE is not a column β€” it is the ROW SET. A catalogue read that
348
- failed quietly would drop the grid straight back to the sold-only 2,717 with every gate
349
- green and nothing on screen saying so, which is the defect this seam exists to end
350
- ([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise
351
- into a 503 that names the cause, so the loud failure has somewhere honest to land.
352
-
353
- It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the
354
- same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs
355
- to build a pool without Odoo.
356
- """
357
- return products.catalogue()
358
-
359
-
360
- def _pricelist_by_code():
361
- """`({code: {price_*: price}}, report)` from `products.pricelist_by_code`, or `({}, …)`.
362
-
363
- A SEAM for the same two reasons `_inventory_by_code` is one: it degrades rather than raises
364
- (these are columns, not the row set), and `verify_perm_scope`'s section H stubs it to build a
365
- pool without Odoo. β›” Unstubbed there, section H would reach live Odoo through the back door
366
- and the whole file would stop being runnable offline.
367
- """
368
- try:
369
- return products.pricelist_by_code()
370
- except Exception as e:
371
- return {}, {"error": f"{type(e).__name__}: {str(e)[:200]}"}
372
-
373
-
374
- def pool(team_id=None, t=None):
375
- """One row per ACTIVE product β€” the PRODUCT analogue of `customer_data.pool`.
376
-
377
- CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 β€” see decision 4 in the module header). The row set
378
- is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs
379
- that sold in its window and contributes NO rows of its own.
380
-
381
- `team_id` shapes the revenue columns exactly as it does for customers (`products.directory`
382
- passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
383
- driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
384
- catalogue has no team.
385
- """
386
- t = t or P.today()
387
- consolidated = team_id is None
388
- # ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
389
- # itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
390
- inv = _inventory_by_code(t)
391
- bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
392
- sup = supplier_master()
393
- prices, _price_report = _pricelist_by_code()
394
- cat = _catalogue_by_code()
395
- # The LEFT side of the join, indexed by the same code key. β›” A code here that the catalogue
396
- # does not carry belongs to a product no ACTIVE record claims β€” archived, and R12 keeps those
397
- # out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue
398
- # that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY).
399
- rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
400
-
401
- rows = []
402
- for code, meta in cat.items():
403
- r = rev.get(code)
404
- row = {
405
- "pid": sku_pid(code),
406
- "code": code,
407
- # ⭐ ONE source for the name and the category, the CATALOGUE β€” not the sale line's
408
- # m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active
409
- # record's `display_name` is the current truth, and it is the same string for every
410
- # SKU that is not re-SKUed. `directory()` derives the category identically.
411
- "product": meta.get("product") or code,
412
- "category": meta.get("category") or "(uncategorized)",
413
- # ⭐⭐ W33-T43 (R2 / amendment A2) β€” ODOO'S OWN PRODUCT ID, beside the hashed `pid`.
414
- # R2 retires `ut_odoo_products` onto this key and keeps every data column it had; this
415
- # is that column. β›” NOT DERIVABLE DOWNSTREAM: `pid` is `crc32(default_code)` here,
416
- # unlike a customer row whose `pid` IS the partner id, so `aios_grid.py` cannot recover
417
- # it and it has to be carried from `products.catalogue()`.
418
- # ⚠ `None`, never 0, when the catalogue somehow has no id β€” 0 is a real Odoo id.
419
- "product_id": meta.get("id"),
420
- # ⭐ R12 β€” BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev`
421
- # keeps its measured numbers including a real 0.0 (it sold last year, not this one).
422
- "rev_ytd": r.get("rev_ytd", 0.0) if r else None,
423
- "rev_ly": r.get("rev_ly", 0.0) if r else None,
424
- "yoy_pct": r.get("yoy_pct") if r else None,
425
- "qty_ytd": r.get("qty_ytd", 0.0) if r else None,
426
- "orders_ytd": r.get("orders_ytd", 0) if r else None,
427
- }
428
- # ⭐ WAVE 30 W30-T34 β€” the PRICELIST stratum, one column per declared list. Catalogue
429
- # data like the supplier block, so it rides every pull, scoped or not: a price book is
430
- # not a thing a business unit owns a slice of, and a Fisch reader who cannot see the
431
- # Fisch price is exactly who this column is for.
432
- #
433
- # β›” `None`, NEVER 0 AND NEVER A FALLBACK, for the 1,883 SKUs no list prices. That is
434
- # W29-T52's own negative control: the single `3_global` rule computes over `list_price`,
435
- # which is 1.00 on 5,817 of 5,871 products, so the fallback is not a cheaper answer β€”
436
- # it is a wrong one wearing a currency sign.
437
- p = prices.get(code) or {}
438
- for _col, _name in products.PRICELIST_COLUMNS:
439
- row[_col] = p.get(_col)
440
- # Wave 17 R3 β€” the supplier master, on every pull (it is catalogue data, not stock).
441
- s = sup.get(code) or {}
442
- row.update({
443
- "supplier": s.get("supplier") or "(none)",
444
- "lead_days": s.get("lead_days"),
445
- "origin_country": s.get("origin_country") or "(none)",
446
- "first_cost": s.get("first_cost"),
447
- })
448
- e = inv.get(code) or {}
449
- if consolidated:
450
- qty_ltm, dos, bucket = e.get("qty_ltm"), e.get("dos"), e.get("bucket")
451
- else:
452
- qty_ltm, dos, bucket = _rescope_inventory(e, bu_share.get(code, _DEFAULT_SHARE))
453
- row.update({
454
- # UNSCOPED on purpose (owner 2026-08-11): one warehouse, no per-brand shelf.
455
- "on_hand": e.get("on_hand"),
456
- "unit_cost": e.get("unit_cost"),
457
- "inv_value": e.get("inv_value"),
458
- # SCOPED: these three are functions of how fast THIS unit sells the SKU.
459
- "qty_ltm": qty_ltm,
460
- "dos": dos,
461
- "stock_bucket": bucket,
462
- })
463
- # THE BUY TRIGGER (R3), ported from `modules/procurement`'s lead-time-cover rule:
464
- # buy when the shelf runs out before a reorder could land.
465
- #
466
- # ⚠ BLANK, NOT "OK", WHEN EITHER INPUT IS MISSING. 339 SKUs have no supplier and 400
467
- # no lead time; `dos` is null for anything that never sells through. "We don't know"
468
- # and "you're fine" are different sentences, and only one of them is safe to print
469
- # beside a purchasing decision.
470
- #
471
- # β›” `buy_now` SHIPS NOWHERE ANY MORE, and is computed anyway. OWNER 2026-08-03: it
472
- # is not a preset field β€” it is a formula over the two columns beside it, and the
473
- # platform has a formula field type for exactly that. Its Field is gone from
474
- # `aios_grid_fields.json`, and `rows_from_pool` projects rows STRICTLY through that
475
- # contract, so no Field means no cell on the wire. Nothing renders this key.
476
- #
477
- # It stays computed because deleting it would delete the PROOF. `validate()` below
478
- # reconciles the formula's predicate against this one, row for row, which is the only
479
- # thing that makes "the same exact figures" a claim rather than an assertion β€” and
480
- # this repo has the scar already (wave 17: archiving `ar` "would have DELETED the
481
- # proof"). Two comparisons per SKU is what that costs.
482
- #
483
- # ⭐ READS THE SCOPED `dos`, NOT `e['dos']`. On a Fisch pull the cover gap must answer
484
- # "does the shelf outlast a reorder AT FISCH'S RATE" β€” reading the consolidated figure
485
- # here would print a buy signal computed from both units' velocity beside a days-of-supply
486
- # computed from one, and the two columns would disagree on the same row.
487
- lead = s.get("lead_days")
488
- if isinstance(dos, (int, float)) and isinstance(lead, (int, float)) and lead > 0:
489
- row["cover_gap_d"] = int(round(dos - lead))
490
- row["buy_now"] = "Buy now" if dos < lead else "OK"
491
- else:
492
- row["cover_gap_d"] = None
493
- row["buy_now"] = None
494
- rows.append(row)
495
-
496
- # `or 0.0` reads a BLANK as zero FOR SORTING ONLY β€” never for the cell. A never-sold SKU
497
- # settles among the zero-revenue ones at the bottom, which is where it belongs; the stored
498
- # value stays None so nothing downstream can mistake "we never saw a sale" for "we measured
499
- # nothing sold".
500
- rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
501
- _assert_no_pid_collision(rows)
502
- return rows
503
-
504
-
505
- #: ⭐ THE BUY SIGNAL, AS THE FORMULA FIELD EVALUATES IT (owner, 2026-08-03).
506
- #:
507
- #: The owner's ruling was that "Buy signal" is not a preset field β€” it is a formula over two
508
- #: columns the product table already carries, and the platform has a formula field type for it.
509
- #: The formula, verbatim, is what `_seed_wave17.BUY_SIGNAL_FORMULA` creates and what the JSON
510
- #: contract's `_product_removed_buy_now` note records:
511
- #:
512
- #: IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
513
- #:
514
- #: This function is a PORT of how `customer-grid/formulaEngine.ts` evaluates that tree, not a
515
- #: restatement of the business rule β€” that is the whole point, because the two could drift and
516
- #: `validate()` is where the drift must show. Three engine behaviours it reproduces exactly:
517
- #:
518
- #: Β· a `ref` to a missing/non-numeric cell is None (`case "ref"` returns null for anything
519
- #: that is not a finite number or a string);
520
- #: Β· `cmp` returns BLANK unless BOTH sides read as numbers β€” it never coerces a blank to 0,
521
- #: which is the difference between this and a filter engine's `toNum(null) === 0`;
522
- #: Β· `IF` with a non-boolean condition returns BLANK ("no truthiness"), so a blank comparison
523
- #: propagates out as a blank cell rather than taking the false branch.
524
- BUY_SIGNAL_FORMULA = 'IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")'
525
-
526
-
527
- def _buy_signal_formula(row):
528
- """Evaluate `BUY_SIGNAL_FORMULA` over one pool row -> 'Buy now' | 'OK' | '' (blank)."""
529
- def num(v):
530
- # The engine's `ref` + `asNumber`: booleans are not values here, and a non-finite
531
- # number is null. `isinstance(True, int)` is True in Python, so bool is excluded first.
532
- if isinstance(v, bool) or not isinstance(v, (int, float)):
533
- return None
534
- return v if v == v and v not in (float('inf'), float('-inf')) else None
535
-
536
- lead, dos = num(row.get("lead_days")), num(row.get("dos"))
537
- if lead is None: # `{lead_days} > 0` is blank -> IF(blank, …) is blank
538
- return ""
539
- if not lead > 0:
540
- return "" # the formula's own else-branch
541
- if dos is None: # `{dos} < {lead_days}` is blank -> IF(blank, …) is blank
542
- return ""
543
- return "Buy now" if dos < lead else "OK"
544
-
545
-
546
- def validate(team_id=None, t=None):
547
- """Reconcile what SHIPS to an independent aggregate β€” the platform's own rule that a number
548
- which does not tie to Odoo does not ship.
549
-
550
- β›”β›” THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING
551
- (wave 29, contract C8). It reconciled Ξ£ per-SKU YTD revenue against `products._sku_rev` β€” the
552
- same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could
553
- not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave.
554
- No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either.
555
- That is the leg below, and it is the control that would have caught it
556
- ([[no-unverifiable-aggregates]]).
557
-
558
- ⚠ The revenue leg was NOT deleted with it β€” it answers a different question ("did the join
559
- lose money?") and is still the right shape for that one. What changed is that it now has to
560
- account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived
561
- products out of the grid; the total is decomposed rather than compared loosely, so a growing
562
- "outside the grid" figure shows up as a number rather than as slack in a tolerance.
563
-
564
- Still deliberately NOT covered, and said rather than implied: the inventory columns are a
565
- different module's reconciliation, and they are absent on a scoped pull anyway.
566
- """
567
- t = t or P.today()
568
- rows = pool(team_id=team_id, t=t)
569
-
570
- # ⭐⭐ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active
571
- # products, asked fresh β€” not a `len()` over anything this module or `products.catalogue()`
572
- # built. NC: drop one row from `pool()` and this goes red; that is the whole point of it.
573
- n_active = products.catalogue_count()
574
- checks = [{
575
- "check": "the pool holds one row per ACTIVE product (R12) β€” row count == an INDEPENDENT "
576
- "Odoo search_count('product.product', active=True)",
577
- "ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active,
578
- }]
579
-
580
- yf, yt = P.ytd(t)
581
- sku_rev = products._sku_rev(yf, yt, team_id) or {}
582
- in_pool = {r["code"] for r in rows}
583
- outside = {c: v for c, v in sku_rev.items() if c not in in_pool}
584
- ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2)
585
- theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2)
586
- dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2)
587
- checks.append({
588
- "check": "Ξ£ per-SKU YTD revenue in the grid + Ξ£ YTD revenue of codes no ACTIVE product "
589
- "carries == modules.products._sku_rev over the same window",
590
- "ours": round(ours + dropped, 2), "theirs": theirs,
591
- "ok": abs((ours + dropped) - theirs) < 0.01,
592
- "detail": {"in_the_grid": ours, "outside_the_grid": dropped,
593
- "codes_outside": len(outside)},
594
- })
595
- # β›” AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is
596
- # trivially true when checked against the dict that did the dropping β€” that is the
597
- # self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE
598
- # product carries means `products.catalogue()` missed a row that has revenue, which is the
599
- # 2,717 defect returning in a smaller costume.
600
- # ⚠ `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the
601
- # catalogue under its own `pid:N` key and therefore cannot be in `outside` at all.
602
- coded_outside = sorted(c for c in outside if not str(c).startswith("pid:"))
603
- still_active = (O.get_odoo().search_count(
604
- 'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)])
605
- if coded_outside else 0)
606
- checks.append({
607
- "check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived "
608
- "stay out) β€” asked of Odoo, never of the catalogue that did the dropping",
609
- "ours": still_active, "theirs": 0, "ok": still_active == 0,
610
- "detail": {"codes": coded_outside[:10], "n_codes": len(outside)},
611
- })
612
- # ── ⭐⭐ WAVE 30 W30-T34 β€” THE PRICELIST COLUMNS, AGAINST ORACLES THAT CANNOT SEE OUR JOIN ──
613
- #
614
- # Three legs, because the column can fail in three different ways and only one of them is a
615
- # count. `products.pricelist_by_code` builds a `{code: {col: price}}` map by expanding each
616
- # rule over the products it names, IN PYTHON β€” that expansion is the fragile part, so every
617
- # leg below re-asks ODOO instead of re-reading the map.
618
- price_report = _pricelist_by_code()[1]
619
- _pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
620
- _pl_id = {}
621
- for _p in _pl_rows:
622
- _pl_id.setdefault(str(_p.get('name') or '').strip(), _p['id'])
623
- _today = P.today().isoformat()
624
-
625
- _targets_memo = {}
626
-
627
- def _rule_targets(list_name):
628
- """(variant_ids, template_ids) a pricelist prices today β€” read FRESH from Odoo.
629
-
630
- Memoised for the LIFE OF THIS CALL only: legs 1 and 2 both need all three lists, and
631
- without this `validate()` makes six identical round trips instead of three (measured:
632
- ~48s of the run). Deliberately NOT an `lru_cache` β€” an oracle that survives the process
633
- is an oracle reading yesterday's Odoo.
634
- """
635
- if list_name in _targets_memo:
636
- return _targets_memo[list_name]
637
- var, tmpl = set(), set()
638
- plid = _pl_id.get(list_name)
639
- if plid is None:
640
- return var, tmpl
641
- for r in O.search_read(
642
- 'product.pricelist.item',
643
- [('pricelist_id', '=', plid), ('compute_price', '=', 'fixed'),
644
- ('applied_on', 'in', ['0_product_variant', '1_product'])],
645
- ['product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
646
- 'date_start', 'date_end']):
647
- ds, de = str(r.get('date_start') or '')[:10], str(r.get('date_end') or '')[:10]
648
- if (ds and ds > _today) or (de and de < _today):
649
- continue
650
- fp = r.get('fixed_price')
651
- if not isinstance(fp, (int, float)) or fp <= 0:
652
- continue
653
- if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
654
- var.add(O.m2o_id(r['product_id']))
655
- elif r.get('product_tmpl_id'):
656
- tmpl.add(O.m2o_id(r['product_tmpl_id']))
657
- _targets_memo[list_name] = (var, tmpl)
658
- return var, tmpl
659
-
660
- # LEG 1 β€” COVERAGE PER LIST. Ours: cells we filled. Theirs: a bare Odoo `search_count` of
661
- # ACTIVE products a fresh read of that list's rules reaches. The rule set is shared (it IS
662
- # the data) but the EXPANSION is not, and the expansion is what breaks.
663
- for _col, _name in products.PRICELIST_COLUMNS:
664
- _var, _tmpl = _rule_targets(_name)
665
- _dom = [('active', '=', True), '|', ('id', 'in', sorted(_var)),
666
- ('product_tmpl_id', 'in', sorted(_tmpl))]
667
- theirs = O.get_odoo().search_count('product.product', _dom) if (_var or _tmpl) else 0
668
- ours = sum(1 for r in rows if isinstance(r.get(_col), (int, float)))
669
- checks.append({
670
- "check": f"{_name} pricelist: SKUs priced in the grid == an INDEPENDENT Odoo count "
671
- f"of ACTIVE products its date-valid fixed rules reach",
672
- "ours": ours, "theirs": theirs, "ok": ours == theirs,
673
- "detail": {"column": _col, "variant_rules": len(_var), "template_rules": len(_tmpl)},
674
- })
675
-
676
- # LEG 2 β€” β›” THE TICKET'S OWN NEGATIVE CONTROL, AS A LEG. A SKU with no specific item must
677
- # render BLANK. Asked of ODOO, never of the map that produced the blank β€” "everything I left
678
- # empty was genuinely unpriced" is trivially true when checked against the dict that emptied
679
- # it, which is the self-sealing shape the population leg above exists to end.
680
- _unpriced = [r for r in rows
681
- if not any(isinstance(r.get(c), (int, float))
682
- for c, _n in products.PRICELIST_COLUMNS)]
683
- _codes = {r["code"] for r in _unpriced}
684
- _stray = 0
685
- if _codes:
686
- # Resolve those codes back to Odoo ids and ask whether ANY declared list prices them.
687
- _ids, _tmpls = set(), set()
688
- for _p in O.search_read('product.product',
689
- [('active', '=', True),
690
- ('default_code', 'in', sorted(c for c in _codes
691
- if not c.startswith("pid:")))],
692
- ['id', 'product_tmpl_id']):
693
- _ids.add(_p['id'])
694
- _tmpls.add(O.m2o_id(_p.get('product_tmpl_id')))
695
- for _col, _name in products.PRICELIST_COLUMNS:
696
- _v, _t = _rule_targets(_name)
697
- _stray += len((_v & _ids) | ({t for t in _tmpls if t in _t}))
698
- checks.append({
699
- "check": "every SKU rendering a BLANK price is genuinely unpriced on all three declared "
700
- "lists (W29-T52's NC: no fallback dressed as a price) β€” asked of Odoo",
701
- "ours": _stray, "theirs": 0, "ok": _stray == 0,
702
- "detail": {"blank_skus": len(_unpriced), "priced_skus": len(rows) - len(_unpriced)},
703
- })
704
-
705
- # LEG 3 β€” ⭐ R6's SECOND SENTENCE, AS A NUMBER. *"If there is lag or it can't be done, you
706
- # need to explicitly tell me why and recommend a fix."* Everything the reader cannot see is
707
- # counted here rather than dropped. The ASSERTION is the one thing that must never be true β€”
708
- # a price cell that is present and not a positive number, i.e. a "free" SKU β€” while the
709
- # rest rides `detail` so it is reported without reddening an honest day's data.
710
- _bad = [r["code"] for r in rows
711
- for c, _n in products.PRICELIST_COLUMNS
712
- if r.get(c) is not None and not (isinstance(r.get(c), (int, float)) and r[c] > 0)]
713
- # β›” MEASURED 2026-08-12 AND NOT FIXABLE FROM THIS FENCE: `aios_grid.rows_from_pool` sends
714
- # every non-text field through `_round(v)` = `round(v)` with no ndigits, i.e. to a whole
715
- # dollar. 52.1% of fixed prices carry cents and the median relative error is 5.26%, so this
716
- # column ties to the cent HERE and ships rounded. Reported, never silently enforced.
717
- _cents = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
718
- if isinstance(r.get(c), (int, float)) and abs(r[c] - round(r[c])) > 1e-9)
719
- _to_zero = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
720
- if isinstance(r.get(c), (int, float)) and round(r[c]) == 0)
721
- checks.append({
722
- "check": "no price cell is present-but-not-a-positive-number (a 0 would read as FREE; "
723
- "an unset rule must be blank), and what the reader cannot see is COUNTED",
724
- "ours": len(_bad), "theirs": 0, "ok": not _bad,
725
- "detail": {"bad_cells": _bad[:10], "reader_report": price_report,
726
- "wire_rounding_loses_cents_on": _cents,
727
- "wire_rounding_to_zero_dollars": _to_zero},
728
- })
729
-
730
- # WAVE 17 R3 β€” the BUY SIGNAL must be a total, exact partition of the catalogue, and every
731
- # member of it must be re-derivable from the two columns beside it. A signal somebody buys
732
- # stock on cannot be "mostly right": the failure that matters is a row that says OK because
733
- # an input was missing, so the blank leg is checked as hard as the other two.
734
- if team_id is None: # consolidated only β€” the inputs exist only there
735
- buy = [r for r in rows if r.get("buy_now") == "Buy now"]
736
- ok_rows = [r for r in rows if r.get("buy_now") == "OK"]
737
- blank = [r for r in rows if r.get("buy_now") is None]
738
- mis = sum(1 for r in buy if not (r["dos"] < r["lead_days"]))
739
- mis += sum(1 for r in ok_rows if not (r["dos"] >= r["lead_days"]))
740
- # A blank must be UNKNOWN β€” never a row we could have answered and quietly did not.
741
- mis += sum(1 for r in blank
742
- if isinstance(r.get("dos"), (int, float))
743
- and isinstance(r.get("lead_days"), (int, float)) and r["lead_days"] > 0)
744
- checks.append({
745
- "check": "Buy signal partitions the catalogue (buy + ok + unknown == rows, none "
746
- "misclassified)",
747
- "ours": len(buy) + len(ok_rows) + len(blank) - mis, "theirs": len(rows),
748
- "ok": mis == 0 and len(buy) + len(ok_rows) + len(blank) == len(rows),
749
- "detail": {"buy_now": len(buy), "ok": len(ok_rows), "unknown": len(blank),
750
- "misclassified": mis},
751
- })
752
- # ⭐ OWNER 2026-08-03 β€” "use the formula fields to come up to the same EXACT figures".
753
- #
754
- # This is that sentence, as a check. The retired preset column and the formula that
755
- # replaces it must agree on every SKU, in all three states, or the replacement is not a
756
- # replacement. `_buy_signal_formula` is a line-by-line port of the client formula
757
- # engine's evaluation of
758
- #
759
- # IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
760
- #
761
- # including the part that is easy to get wrong: a comparison against a BLANK is blank,
762
- # never a coerced 0. (The client engine's `cmp` returns null unless both sides read as
763
- # numbers, and `IF` refuses a non-boolean condition β€” so a missing `dos` yields "" and
764
- # not "Buy now". A filter engine would have said `0 < 30` and swept in every SKU that
765
- # never sells through; the formula engine does not, and this check is what holds it.)
766
- #
767
- # ⚠ It compares SETS OF SKUs, not counts. Two different partitions can share a shape.
768
- disagree = sorted(r["code"] for r in rows
769
- if (r.get("buy_now") or "") != _buy_signal_formula(r))
770
- checks.append({
771
- "check": 'Buy signal as a FORMULA field == the retired preset column, per SKU '
772
- '(IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), ""))',
773
- "ours": len(rows) - len(disagree), "theirs": len(rows),
774
- "ok": not disagree,
775
- "detail": {"disagreeing_skus": disagree[:10], "n_disagree": len(disagree)},
776
- })
777
- # The other half of "the same figures": the SHARED Buy list view can no longer filter on
778
- # the retired column, and its replacement conditions must select the same SKUs. They are
779
- # `dos isNotEmpty AND lead_days isNotEmpty AND lead_days > 0 AND dos < lead_days`
780
- # (_seed_wave17.views), so this reproduces exactly that conjunction.
781
- #
782
- # β›” THE NEGATIVE CONTROL IS WHY THIS IS NOT `cover_gap_d < 0`, which reads like the
783
- # obvious filter and is WRONG: `cover_gap_d` is `int(round(dos - lead))`, so a genuine
784
- # gap of -0.4 days rounds to 0 and that SKU drops off a buy list it belongs on.
785
- view_rows = {r["code"] for r in rows
786
- if isinstance(r.get("dos"), (int, float))
787
- and isinstance(r.get("lead_days"), (int, float))
788
- and r["lead_days"] > 0 and r["dos"] < r["lead_days"]}
789
- signal_rows = {r["code"] for r in buy}
790
- rounding_would_miss = sorted(
791
- c for c in signal_rows
792
- if next((r for r in rows if r["code"] == c), {}).get("cover_gap_d") == 0)
793
- checks.append({
794
- "check": "Buy list view conditions (dos/lead_days, no retired column) select "
795
- "exactly the Buy-now SKUs",
796
- "ours": len(view_rows), "theirs": len(signal_rows),
797
- "ok": view_rows == signal_rows,
798
- "detail": {"only_in_view": sorted(view_rows - signal_rows)[:10],
799
- "only_in_signal": sorted(signal_rows - view_rows)[:10],
800
- # Reported, not asserted: how many SKUs a `cover_gap_d < 0` filter would
801
- # have silently dropped. 0 today does not make that filter correct.
802
- "cover_gap_rounds_to_zero": len(rounding_would_miss)},
803
- })
804
- return checks
 
1
+ """modules/product_data.py β€” the PRODUCT table's pool (wave 15 item 9/10, contract C-TOPIC).
2
+
3
+ The second object on the table-page factory: same grid, same engine, same permission wall β€” the
4
+ only difference is the field schema and the identity. `modules/customer_data.pool()` is the
5
+ template and this deliberately mirrors its signature and its row shape.
6
+
7
+ β›” THREE DECISIONS THIS FILE HAD TO MAKE, EACH RECORDED BECAUSE A LATER READER WILL WONDER.
8
+
9
+ 1. **THE IDENTITY IS A SKU CODE, WHICH IS A STRING, AND THE GRID WANTS AN INTEGER `pid`.**
10
+ Cohort membership, `allowed_pids`, the measure channel and `rows_from_pool` all key on an
11
+ integer. So each row carries BOTH: `pid` (a stable CRC32 of the code, so the same SKU gets the
12
+ same id on every pull and across processes β€” never an enumeration index, which would reshuffle
13
+ whenever the catalogue changes) and `code`, the real business key, as a visible column.
14
+ `_assert_no_pid_collision` fails the BUILD rather than the read: two SKUs sharing a pid would
15
+ silently merge in every downstream set operation, and a loud build failure is the only version
16
+ of that anyone would notice.
17
+
18
+ 2. **THE SCOPE RULE β€” inventory columns are CONSOLIDATED and are therefore OMITTED for a
19
+ BU-scoped caller.** `products.directory(t, team_id)` is brand-shaped; `inventory.sku_inventory`
20
+ is explicitly NOT (its own docstring: "on-hand stock is one physical warehouse, not
21
+ brand-tagged"). Joining them for a Fisch-only user would put BU-shaped revenue beside
22
+ company-wide stock IN THE SAME ROW β€” the mixed-scope value defect wave 15's amendment 3 exists
23
+ for, arriving through a different door. The honest options were "omit" or "label the columns
24
+ company-wide"; omit is the fail-closed one, and a column that is absent asks a question,
25
+ whereas a column that is silently company-wide answers one wrongly.
26
+
27
+ 3. **WHAT IS NOT HERE, NAMED RATHER THAN QUIETLY MISSING.** R4 lists ~20 fields. Vendor and
28
+ COUNTRY are not in Odoo at all β€” they live in the inventory WORKBOOK
29
+ ([[odoo-vendor-country-origin]]) and need a loader this module deliberately does not invent.
30
+ Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost
31
+ allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it
32
+ does not pretend to cover columns that are absent.
33
+
34
+ 4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12,
35
+ 2026-08-11). It was `for r in products.directory(...)` β€” and `directory()`'s row set IS the
36
+ union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid
37
+ showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise:
38
+
39
+ Β· β›” THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date
40
+ window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics
41
+ that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because
42
+ ~2,550 active SKUs have never sold in wholesale scope at all.
43
+ · ⭐ REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the
44
+ revenue universe carries measured numbers INCLUDING a real 0.0 β€” it sold last year and
45
+ not this one, and that zero is a measurement. A row that appears in NO revenue read
46
+ carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the
47
+ client renders as an empty cell. Blank is an admission; zero is a measurement.
48
+ · ⚠ A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where
49
+ that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW
50
+ UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product`
51
+ carries no team, so a catalogue cannot be BU-shaped at all β€” which is exactly why the two
52
+ sources are JOINED rather than merged.
53
+ """
54
+ import json
55
+ import zlib
56
+ from pathlib import Path
57
+
58
+ import core.odoo as O
59
+ import core.periods as P
60
+ import core.shared_overlay as shared_overlay
61
+ import core.table_store as table_store
62
+ import modules.products as products
63
+
64
+ #: POOL-ROW KEYS that exist ONLY on a consolidated pull. See decision 2.
65
+ #: `cover_gap_d` / `buy_now` (wave 17) join it because both are computed FROM `dos`, and a buy
66
+ #: signal built on a stock number the caller cannot see would be a recommendation nobody could
67
+ #: check ([[no-unverifiable-aggregates]]).
68
+ #:
69
+ #: ⚠ ROW KEYS, NOT FIELDS, and the distinction started mattering on 2026-08-03. Two readers:
70
+ #: `verify_perm_scope` asserts the shape of a POOL ROW against this, and `routes_products`
71
+ #: narrows the FIELD list with it. Since owner item 5 `buy_now` is a row key with no Field β€”
72
+ #: computed as `validate()`'s oracle, projected onto no wire β€” so it belongs here for the first
73
+ #: reader and is inert for the second. Removing it would stop the scope gate proving that a
74
+ #: BU-scoped pull withholds it.
75
+ #:
76
+ #: ⭐⭐ OWNER RULING 2026-08-11 β€” THIS SET IS NOW EMPTY, AND THAT IS THE POINT. Verbatim:
77
+ #: *"Just scope any inventory with Sales from Fisch, leave the rest."* Withholding the whole
78
+ #: inventory block from a BU-scoped reader meant a Fisch salesperson could not see whether
79
+ #: anything was IN STOCK, which is the first question they ask. The ruling splits the block by
80
+ #: what a business unit can actually shape:
81
+ #:
82
+ #: * `on_hand` / `unit_cost` / `inv_value` β€” ONE physical warehouse, no Fisch shelf and no
83
+ #: Royal shelf. Served UNSCOPED to everybody, identical in both pulls. ("leave the rest")
84
+ #: * `qty_ltm` / `dos` / `stock_bucket` / `cover_gap_d` / `buy_now` β€” all SALES-derived, so a
85
+ #: BU-scoped pull recomputes them from THAT unit's LTM units. ("scope any inventory with
86
+ #: Sales from Fisch")
87
+ #:
88
+ #: This deliberately relaxes decision 2's "never a company-wide value beside a BU-shaped one":
89
+ #: stock is not a value a BU can own, and a column that is absent for a Fisch reader asks a
90
+ #: question they cannot answer anywhere else in the product. `verify_perm_scope` no longer
91
+ #: asserts the columns are WITHHELD β€” it asserts the split above, which is a stronger claim and
92
+ #: cannot pass on an empty scoped payload (the shape the old rule and an outage share).
93
+ CONSOLIDATED_ONLY = ()
94
+
95
+ #: β›” POOL-ROW KEYS THAT DELIBERATELY HAVE NO FIELD β€” they must never reach a browser.
96
+ #:
97
+ #: `rows_from_pool` projects strictly through the field contract, so "no Field" already means
98
+ #: "no cell on the wire". This tuple makes that a CHECKED fact rather than a consequence nobody
99
+ #: is watching: `verify_perm_scope` asserts every member is absent from BOTH product contracts,
100
+ #: and that every OTHER `CONSOLIDATED_ONLY` key is present in the consolidated one.
101
+ #:
102
+ #: Written because the two tuples silently diverged the moment owner item 5 retired `buy_now`'s
103
+ #: Field while keeping its computation, and the gate β€” which was asserting a FIELD property from
104
+ #: a ROW-key list β€” went red with no way to tell a deliberate divergence from a dropped column.
105
+ UNSHIPPED_ROW_KEYS = ("buy_now",)
106
+
107
+ #: Wave 17 (owner item 13, ruling R3) β€” the SUPPLIER MASTER, from the curated mastersheet map
108
+ #: `procurement_suppliers.json` (3,963 SKUs; supplier on 3,624, lead time on 3,563). NOT Odoo:
109
+ #: the owner confirmed this data came from the Inventory System Mastersheet, which is why
110
+ #: `modules/product_data`'s header said Vendor/Country were "NOT HERE, named rather than
111
+ #: quietly missing" and needed "a loader this module deliberately does not invent". This is
112
+ #: that loader.
113
+ #:
114
+ #: β›” THESE ARE CONTRACT COLUMNS, NOT USER-CREATED FIELDS, AND THE REASON IS A PERMISSION FACT.
115
+ #: A user-created field and its values live in the PER-USER strata (`table_store.workspace`
116
+ #: reads `store.get(key)[username]`); only VIEWS are shared. So a shared "Buy list" view that
117
+ #: filtered on a user-created column would, for every OTHER account, name a column that does not
118
+ #: exist β€” and an unknown column is an INACTIVE condition in the tri-state engine, which IGNORES
119
+ #: it and therefore WIDENS. The buy list would silently show the whole catalogue to everyone but
120
+ #: its author. Contract columns are identical for every reader, so the view means one thing.
121
+ #:
122
+ #: ⭐⭐ 2026-08-12 (W30-T36) β€” THE OWNER'S ASK IS NOW DELIVERED, AND THE PARAGRAPH ABOVE IS WHY IT
123
+ #: TOOK THREE WAVES. Owner: *"turn this Excel sheet into a User created Field that we can edit."*
124
+ #: It was parked because a per-user column silently WIDENS a shared view β€” not because editing was
125
+ #: hard. Wave 29's `core/shared_overlay.py` (whose header quotes this very comment) is the stratum
126
+ #: that removes the objection: **one value per (row, column) for the whole tenant**, so the column
127
+ #: still means ONE thing to every reader and a shared view still filters honestly.
128
+ #: β‡’ The four columns below are now `source: "overlay"` + `shared: true` in the canonical
129
+ #: contract, their values live in `<TABLE_KEY>__shared`, and the master map is what SEEDS an
130
+ #: unedited cell rather than what freezes it. See `SHARED_KEYS` and `_ProductTableStore`.
131
+ _SUPPLIER_MAP_PATH = Path(__file__).resolve().parent.parent / "procurement_suppliers.json"
132
+ _SUPPLIER_CACHE = {}
133
+
134
+
135
+ def supplier_master():
136
+ """`{code: {supplier, lead_days, origin_country, first_cost}}`, read once per process.
137
+
138
+ Degrades to `{}` when the file is unreadable, matching `_inventory_by_code`: a product table
139
+ that will not render because a master map is missing is a worse failure than one with blank
140
+ supplier columns.
141
+ """
142
+ if _SUPPLIER_CACHE:
143
+ return _SUPPLIER_CACHE
144
+ try:
145
+ raw = json.loads(_SUPPLIER_MAP_PATH.read_text(encoding="utf-8"))
146
+ except Exception:
147
+ return {}
148
+ for code, meta in (raw or {}).items():
149
+ if not isinstance(meta, dict):
150
+ continue
151
+ lead = meta.get("lead")
152
+ _SUPPLIER_CACHE[str(code)] = {
153
+ "supplier": (meta.get("vendor") or "") or None,
154
+ "lead_days": int(lead) if isinstance(lead, (int, float)) else None,
155
+ "origin_country": (meta.get("country") or "") or None,
156
+ "first_cost": meta.get("first_cost"),
157
+ }
158
+ return _SUPPLIER_CACHE
159
+
160
+ #: Wave 16 C-TOPIC β€” the PRODUCT table's OWN workspace bucket. β›” Never the customer one:
161
+ #: product pids are CRC32 hashes of SKU codes and customer pids are Odoo partner ids, so in a
162
+ #: SHARED overlay bucket a hash collision would silently write a product note onto somebody's
163
+ #: customer (or the reverse). Separate store keys make that structurally impossible, which is
164
+ #: the whole reason the table-page factory exists ("a new table object gets its own
165
+ #: table_store.make('<its>_table_workspace')").
166
+ TABLE_KEY = 'product_table_workspace'
167
+
168
+ _GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json'
169
+ _SHARED_KEYS = None
170
+
171
+
172
+ def SHARED_KEYS():
173
+ """The product columns whose values are TENANT-WIDE β€” derived from the canonical contract's
174
+ own `shared: true`, never typed out a second time.
175
+
176
+ β›” IT DELIBERATELY DOES NOT SWALLOW A READ FAILURE. Degrading to `()` would send a shared
177
+ write into the per-user stratum with nothing going wrong anywhere β€” the widening defect
178
+ reappearing silently, which is the one outcome this whole mechanism exists to prevent. If the
179
+ canonical contract is unreadable the product grid cannot render at all (`pd_fields` parses the
180
+ same file with no guard), so a raise here costs nothing that was still working.
181
+ """
182
+ global _SHARED_KEYS
183
+ if _SHARED_KEYS is None:
184
+ doc = json.loads(_GRID_FIELDS_PATH.read_text(encoding='utf-8'))
185
+ _SHARED_KEYS = tuple(f['key'] for f in (doc.get('product_data') or {}).get('fields') or []
186
+ if f.get('shared'))
187
+ return _SHARED_KEYS
188
+
189
+
190
+ class _ProductTableStore(table_store.TableStore):
191
+ """The product workspace, with the SHARED columns routed to the tenant-wide stratum.
192
+
193
+ ⭐⭐ THIS SUBCLASS IS THE WHOLE OF W30-T36's WRITE PATH, AND THE REASON IT LIVES HERE RATHER
194
+ THAN AT A ROUTE IS THAT **THE BROWSER NEVER CALLS `PATCH /products/{pid}`** β€” measured, zero
195
+ call sites in `aios-web/web/src`. A cell edit travels `POST /grid/events` β†’ `grid_events.
196
+ handle_one` β†’ `_tops(ctx).patch_overlay(...)`, and `_tops` returns `ctx.table`, which
197
+ `routes_grid._ctx` sets to `TABLE_OPS` for the product scope. So this object IS the seam both
198
+ doors pass through; intercepting at either route would have left the other one writing a
199
+ per-user value that only its author could see.
200
+
201
+ ⚠ `st=self.st`, NEVER the module default. The shared stratum must resolve to the SAME store
202
+ handle as the per-user one it sits beside β€” `_tops`' own comment explains that a split, where
203
+ one side is tenant-scoped and the other is not, is worse than a stated residency error because
204
+ a user's value would vanish the moment they saved it. Reading `self.st` means both strata move
205
+ together the day that singleton gains a tenant handle.
206
+ """
207
+
208
+ def patch_overlay(self, username, pid, updates):
209
+ clean = dict(updates or {})
210
+ if not clean:
211
+ return
212
+ keys = set(SHARED_KEYS())
213
+ shared = {k: v for k, v in clean.items() if k in keys}
214
+ personal = {k: v for k, v in clean.items() if k not in keys}
215
+ if shared:
216
+ shared_overlay.put_cells(TABLE_KEY, pid, shared, st=self.st)
217
+ if personal:
218
+ super().patch_overlay(username, pid, personal)
219
+
220
+
221
+ TABLE_OPS = _ProductTableStore(TABLE_KEY)
222
+
223
+
224
+ def shared_cells(pids, st=None):
225
+ """`{"<pid>": {key: value}}` for the SHARED columns of the rows named by `pids`.
226
+
227
+ ⚠ `pids` is required and positional all the way down β€” `shared_overlay.cells` refuses to serve
228
+ "everything" by design, and the caller here always holds an already-scoped pool.
229
+ """
230
+ return shared_overlay.cells(TABLE_KEY, pids, st=st if st is not None else TABLE_OPS.st)
231
+
232
+
233
+ def sku_pid(code):
234
+ """A stable integer id for a SKU code. CRC32, masked to 31 bits so it is always positive and
235
+ always JSON-safe. Stable across processes and pulls, which an enumeration index is not."""
236
+ return zlib.crc32(str(code).encode("utf-8")) & 0x7FFFFFFF
237
+
238
+
239
+ def _assert_no_pid_collision(rows):
240
+ """Two SKUs sharing a pid would MERGE in every set operation downstream β€” cohort membership,
241
+ allowed_pids, the measure channel β€” and nothing would report it. Fail the build instead."""
242
+ seen = {}
243
+ for r in rows:
244
+ prior = seen.get(r["pid"])
245
+ if prior is not None and prior != r["code"]:
246
+ raise ValueError(
247
+ f"product_data: pid collision β€” {prior!r} and {r['code']!r} both hash to "
248
+ f"{r['pid']}. Downstream set operations would merge them silently; widen the id "
249
+ f"before shipping this catalogue.")
250
+ seen[r["pid"]] = r["code"]
251
+
252
+
253
+ def _inventory_by_code(t):
254
+ """`{code: {...}}` from the inventory module, or `{}` if it cannot be read.
255
+
256
+ Degrades to empty rather than raising, matching `customer_data._pool_build`'s treatment of
257
+ its own slow families: a product table that will not render because inventory is momentarily
258
+ unreachable is a worse failure than one with blank stock columns.
259
+ """
260
+ try:
261
+ import modules.inventory as inventory
262
+ return inventory.sku_inventory(t=t) or {}
263
+ except Exception:
264
+ return {}
265
+
266
+
267
+ def _bu_ltm_share(t, team_id):
268
+ """`{code: 0.0..1.0}` β€” this unit's SHARE of the SKU's last-twelve-months units.
269
+
270
+ ⭐ OWNER 2026-08-11: *"scope any inventory with Sales from Fisch"*. The velocity half of the
271
+ inventory block has to be re-shaped by a BU fact, and this is that fact.
272
+
273
+ β›” A SHARE, NOT THE UNIT COUNT ITSELF β€” AND THE REASON IS A MEASURED IMPOSSIBILITY. My first
274
+ version returned `products._sku_rev(...)['qty']` and used it directly as the scoped `qty_ltm`,
275
+ leaving the consolidated column on `inventory.sku_inventory`'s own figure. Two readers of one
276
+ question ([[one-question-two-normalizers]]): they disagree, and on 5 SKUs of 5,871 the live
277
+ check found **Fisch's LTM units EXCEEDING the company's** β€” a subset larger than its superset,
278
+ which no reader could explain and no reconciliation could survive.
279
+
280
+ Both halves of the ratio come from the SAME read here, so the share is in [0, 1] by
281
+ construction and the scoped figure can never exceed the consolidated one. It also leaves the
282
+ consolidated column exactly as it was β€” the Inventory page and the Product grid still agree
283
+ about company-wide units, which is what "leave the rest" asked for.
284
+
285
+ `all_qty == 0` implies `bu_qty == 0` (same reader), so a share of 0 is the honest answer for
286
+ "this unit never sold it": `_bucket` turns that into 'No recent sales', not zero cover.
287
+
288
+ Degrades to `{}` on any failure, matching `_inventory_by_code` β€” and a missing code then takes
289
+ the `_DEFAULT_SHARE` below rather than a silent 0.
290
+ """
291
+ try:
292
+ f, to = P.ltm(t)
293
+ allq = {code: (r.get('qty') or 0.0)
294
+ for code, r in (products._sku_rev(f, to, None) or {}).items()}
295
+ buq = {code: (r.get('qty') or 0.0)
296
+ for code, r in (products._sku_rev(f, to, team_id) or {}).items()}
297
+ except Exception:
298
+ return {}
299
+ out = {}
300
+ for code, total in allq.items():
301
+ out[code] = min(1.0, max(0.0, (buq.get(code, 0.0) / total))) if total > 0 else 0.0
302
+ return out
303
+
304
+
305
+ #: What a SKU absent from the LTM sales read is worth to a business unit. ZERO β€” it did not sell
306
+ #: in anybody's book over the window, so no unit can claim its velocity. Named rather than
307
+ #: inlined so the choice is visible: the alternative (1.0, "assume it is all ours") would print
308
+ #: company-wide cover on a BU grid, which is the mixed-scope defect this whole rule avoids.
309
+ _DEFAULT_SHARE = 0.0
310
+
311
+
312
+ def _rescope_inventory(e, share):
313
+ """`(qty_ltm, dos, bucket)` recomputed for ONE business unit's sales rate.
314
+
315
+ The formulas are `inventory.sku_inventory`'s, applied to a BU-shaped numerator β€” NOT a second
316
+ idea of what days-of-supply means. `_bucket` is imported from there for the same reason: two
317
+ copies of a threshold table is how the Product grid and the Inventory page start disagreeing
318
+ about which SKUs are dead.
319
+
320
+ ⚠ `on_hand` is whatever the warehouse holds, unscoped β€” so a Fisch reader's `dos` answers
321
+ "how long does ALL our stock last at Fisch's rate", which is the question a Fisch salesperson
322
+ actually has. It is deliberately NOT a pro-rated share of the shelf: there is no such shelf,
323
+ and inventing one would put a number on screen that no Odoo query could reproduce.
324
+ """
325
+ on_hand = e.get("on_hand")
326
+ if on_hand is None:
327
+ return None, None, None # no inventory row for this SKU: blank, never zero
328
+ qty = float(e.get("qty_ltm") or 0.0) * float(share or 0.0)
329
+ daily = qty / 365.0
330
+ if daily > 0:
331
+ dos_raw = on_hand / daily
332
+ else:
333
+ dos_raw = float('inf') if on_hand > 0 else 0.0
334
+ try:
335
+ import modules.inventory as inventory
336
+ bucket = inventory._bucket(dos_raw, on_hand, qty)
337
+ except Exception:
338
+ bucket = None
339
+ return qty, (None if dos_raw == float('inf') else round(float(dos_raw), 0)), bucket
340
+
341
+
342
+ def _catalogue_by_code():
343
+ """`{code: {'product', 'category'}}` β€” the CATALOGUE universe this pool is built from.
344
+
345
+ β›” UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point.
346
+ Inventory degrades to `{}` because a product table with blank stock columns beats one that
347
+ will not render. The CATALOGUE is not a column β€” it is the ROW SET. A catalogue read that
348
+ failed quietly would drop the grid straight back to the sold-only 2,717 with every gate
349
+ green and nothing on screen saying so, which is the defect this seam exists to end
350
+ ([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise
351
+ into a 503 that names the cause, so the loud failure has somewhere honest to land.
352
+
353
+ It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the
354
+ same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs
355
+ to build a pool without Odoo.
356
+ """
357
+ return products.catalogue()
358
+
359
+
360
+ def _pricelist_by_code():
361
+ """`({code: {price_*: price}}, report)` from `products.pricelist_by_code`, or `({}, …)`.
362
+
363
+ A SEAM for the same two reasons `_inventory_by_code` is one: it degrades rather than raises
364
+ (these are columns, not the row set), and `verify_perm_scope`'s section H stubs it to build a
365
+ pool without Odoo. β›” Unstubbed there, section H would reach live Odoo through the back door
366
+ and the whole file would stop being runnable offline.
367
+ """
368
+ try:
369
+ return products.pricelist_by_code()
370
+ except Exception as e:
371
+ return {}, {"error": f"{type(e).__name__}: {str(e)[:200]}"}
372
+
373
+
374
+ def pool(team_id=None, t=None):
375
+ """One row per ACTIVE product β€” the PRODUCT analogue of `customer_data.pool`.
376
+
377
+ CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 β€” see decision 4 in the module header). The row set
378
+ is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs
379
+ that sold in its window and contributes NO rows of its own.
380
+
381
+ `team_id` shapes the revenue columns exactly as it does for customers (`products.directory`
382
+ passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
383
+ driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
384
+ catalogue has no team.
385
+ """
386
+ t = t or P.today()
387
+ consolidated = team_id is None
388
+ # ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
389
+ # itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
390
+ inv = _inventory_by_code(t)
391
+ bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
392
+ sup = supplier_master()
393
+ prices, _price_report = _pricelist_by_code()
394
+ cat = _catalogue_by_code()
395
+ # The LEFT side of the join, indexed by the same code key. β›” A code here that the catalogue
396
+ # does not carry belongs to a product no ACTIVE record claims β€” archived, and R12 keeps those
397
+ # out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue
398
+ # that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY).
399
+ rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
400
+
401
+ rows = []
402
+ for code, meta in cat.items():
403
+ r = rev.get(code)
404
+ row = {
405
+ "pid": sku_pid(code),
406
+ "code": code,
407
+ # ⭐ ONE source for the name and the category, the CATALOGUE β€” not the sale line's
408
+ # m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active
409
+ # record's `display_name` is the current truth, and it is the same string for every
410
+ # SKU that is not re-SKUed. `directory()` derives the category identically.
411
+ "product": meta.get("product") or code,
412
+ "category": meta.get("category") or "(uncategorized)",
413
+ # ⭐⭐ W33-T43 (R2 / amendment A2) β€” ODOO'S OWN PRODUCT ID, beside the hashed `pid`.
414
+ # R2 retires `ut_odoo_products` onto this key and keeps every data column it had; this
415
+ # is that column. β›” NOT DERIVABLE DOWNSTREAM: `pid` is `crc32(default_code)` here,
416
+ # unlike a customer row whose `pid` IS the partner id, so `aios_grid.py` cannot recover
417
+ # it and it has to be carried from `products.catalogue()`.
418
+ # ⚠ `None`, never 0, when the catalogue somehow has no id β€” 0 is a real Odoo id.
419
+ "product_id": meta.get("id"),
420
+ # ⭐ R12 β€” BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev`
421
+ # keeps its measured numbers including a real 0.0 (it sold last year, not this one).
422
+ "rev_ytd": r.get("rev_ytd", 0.0) if r else None,
423
+ "rev_ly": r.get("rev_ly", 0.0) if r else None,
424
+ "yoy_pct": r.get("yoy_pct") if r else None,
425
+ "qty_ytd": r.get("qty_ytd", 0.0) if r else None,
426
+ "orders_ytd": r.get("orders_ytd", 0) if r else None,
427
+ }
428
+ # ⭐ WAVE 30 W30-T34 β€” the PRICELIST stratum, one column per declared list. Catalogue
429
+ # data like the supplier block, so it rides every pull, scoped or not: a price book is
430
+ # not a thing a business unit owns a slice of, and a Fisch reader who cannot see the
431
+ # Fisch price is exactly who this column is for.
432
+ #
433
+ # β›” `None`, NEVER 0 AND NEVER A FALLBACK, for the 1,883 SKUs no list prices. That is
434
+ # W29-T52's own negative control: the single `3_global` rule computes over `list_price`,
435
+ # which is 1.00 on 5,817 of 5,871 products, so the fallback is not a cheaper answer β€”
436
+ # it is a wrong one wearing a currency sign.
437
+ p = prices.get(code) or {}
438
+ for _col, _name in products.PRICELIST_COLUMNS:
439
+ row[_col] = p.get(_col)
440
+ # Wave 17 R3 β€” the supplier master, on every pull (it is catalogue data, not stock).
441
+ s = sup.get(code) or {}
442
+ row.update({
443
+ "supplier": s.get("supplier") or "(none)",
444
+ "lead_days": s.get("lead_days"),
445
+ "origin_country": s.get("origin_country") or "(none)",
446
+ "first_cost": s.get("first_cost"),
447
+ })
448
+ e = inv.get(code) or {}
449
+ if consolidated:
450
+ qty_ltm, dos, bucket = e.get("qty_ltm"), e.get("dos"), e.get("bucket")
451
+ else:
452
+ qty_ltm, dos, bucket = _rescope_inventory(e, bu_share.get(code, _DEFAULT_SHARE))
453
+ row.update({
454
+ # UNSCOPED on purpose (owner 2026-08-11): one warehouse, no per-brand shelf.
455
+ "on_hand": e.get("on_hand"),
456
+ "unit_cost": e.get("unit_cost"),
457
+ "inv_value": e.get("inv_value"),
458
+ # SCOPED: these three are functions of how fast THIS unit sells the SKU.
459
+ "qty_ltm": qty_ltm,
460
+ "dos": dos,
461
+ "stock_bucket": bucket,
462
+ })
463
+ # THE BUY TRIGGER (R3), ported from `modules/procurement`'s lead-time-cover rule:
464
+ # buy when the shelf runs out before a reorder could land.
465
+ #
466
+ # ⚠ BLANK, NOT "OK", WHEN EITHER INPUT IS MISSING. 339 SKUs have no supplier and 400
467
+ # no lead time; `dos` is null for anything that never sells through. "We don't know"
468
+ # and "you're fine" are different sentences, and only one of them is safe to print
469
+ # beside a purchasing decision.
470
+ #
471
+ # β›” `buy_now` SHIPS NOWHERE ANY MORE, and is computed anyway. OWNER 2026-08-03: it
472
+ # is not a preset field β€” it is a formula over the two columns beside it, and the
473
+ # platform has a formula field type for exactly that. Its Field is gone from
474
+ # `aios_grid_fields.json`, and `rows_from_pool` projects rows STRICTLY through that
475
+ # contract, so no Field means no cell on the wire. Nothing renders this key.
476
+ #
477
+ # It stays computed because deleting it would delete the PROOF. `validate()` below
478
+ # reconciles the formula's predicate against this one, row for row, which is the only
479
+ # thing that makes "the same exact figures" a claim rather than an assertion β€” and
480
+ # this repo has the scar already (wave 17: archiving `ar` "would have DELETED the
481
+ # proof"). Two comparisons per SKU is what that costs.
482
+ #
483
+ # ⭐ READS THE SCOPED `dos`, NOT `e['dos']`. On a Fisch pull the cover gap must answer
484
+ # "does the shelf outlast a reorder AT FISCH'S RATE" β€” reading the consolidated figure
485
+ # here would print a buy signal computed from both units' velocity beside a days-of-supply
486
+ # computed from one, and the two columns would disagree on the same row.
487
+ lead = s.get("lead_days")
488
+ if isinstance(dos, (int, float)) and isinstance(lead, (int, float)) and lead > 0:
489
+ row["cover_gap_d"] = int(round(dos - lead))
490
+ row["buy_now"] = "Buy now" if dos < lead else "OK"
491
+ else:
492
+ row["cover_gap_d"] = None
493
+ row["buy_now"] = None
494
+ rows.append(row)
495
+
496
+ # `or 0.0` reads a BLANK as zero FOR SORTING ONLY β€” never for the cell. A never-sold SKU
497
+ # settles among the zero-revenue ones at the bottom, which is where it belongs; the stored
498
+ # value stays None so nothing downstream can mistake "we never saw a sale" for "we measured
499
+ # nothing sold".
500
+ rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
501
+ _assert_no_pid_collision(rows)
502
+ return rows
503
+
504
+
505
+ #: ⭐ THE BUY SIGNAL, AS THE FORMULA FIELD EVALUATES IT (owner, 2026-08-03).
506
+ #:
507
+ #: The owner's ruling was that "Buy signal" is not a preset field β€” it is a formula over two
508
+ #: columns the product table already carries, and the platform has a formula field type for it.
509
+ #: The formula, verbatim, is what `_seed_wave17.BUY_SIGNAL_FORMULA` creates and what the JSON
510
+ #: contract's `_product_removed_buy_now` note records:
511
+ #:
512
+ #: IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
513
+ #:
514
+ #: This function is a PORT of how `customer-grid/formulaEngine.ts` evaluates that tree, not a
515
+ #: restatement of the business rule β€” that is the whole point, because the two could drift and
516
+ #: `validate()` is where the drift must show. Three engine behaviours it reproduces exactly:
517
+ #:
518
+ #: Β· a `ref` to a missing/non-numeric cell is None (`case "ref"` returns null for anything
519
+ #: that is not a finite number or a string);
520
+ #: Β· `cmp` returns BLANK unless BOTH sides read as numbers β€” it never coerces a blank to 0,
521
+ #: which is the difference between this and a filter engine's `toNum(null) === 0`;
522
+ #: Β· `IF` with a non-boolean condition returns BLANK ("no truthiness"), so a blank comparison
523
+ #: propagates out as a blank cell rather than taking the false branch.
524
+ BUY_SIGNAL_FORMULA = 'IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")'
525
+
526
+
527
+ def _buy_signal_formula(row):
528
+ """Evaluate `BUY_SIGNAL_FORMULA` over one pool row -> 'Buy now' | 'OK' | '' (blank)."""
529
+ def num(v):
530
+ # The engine's `ref` + `asNumber`: booleans are not values here, and a non-finite
531
+ # number is null. `isinstance(True, int)` is True in Python, so bool is excluded first.
532
+ if isinstance(v, bool) or not isinstance(v, (int, float)):
533
+ return None
534
+ return v if v == v and v not in (float('inf'), float('-inf')) else None
535
+
536
+ lead, dos = num(row.get("lead_days")), num(row.get("dos"))
537
+ if lead is None: # `{lead_days} > 0` is blank -> IF(blank, …) is blank
538
+ return ""
539
+ if not lead > 0:
540
+ return "" # the formula's own else-branch
541
+ if dos is None: # `{dos} < {lead_days}` is blank -> IF(blank, …) is blank
542
+ return ""
543
+ return "Buy now" if dos < lead else "OK"
544
+
545
+
546
+ def validate(team_id=None, t=None):
547
+ """Reconcile what SHIPS to an independent aggregate β€” the platform's own rule that a number
548
+ which does not tie to Odoo does not ship.
549
+
550
+ β›”β›” THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING
551
+ (wave 29, contract C8). It reconciled Ξ£ per-SKU YTD revenue against `products._sku_rev` β€” the
552
+ same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could
553
+ not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave.
554
+ No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either.
555
+ That is the leg below, and it is the control that would have caught it
556
+ ([[no-unverifiable-aggregates]]).
557
+
558
+ ⚠ The revenue leg was NOT deleted with it β€” it answers a different question ("did the join
559
+ lose money?") and is still the right shape for that one. What changed is that it now has to
560
+ account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived
561
+ products out of the grid; the total is decomposed rather than compared loosely, so a growing
562
+ "outside the grid" figure shows up as a number rather than as slack in a tolerance.
563
+
564
+ Still deliberately NOT covered, and said rather than implied: the inventory columns are a
565
+ different module's reconciliation, and they are absent on a scoped pull anyway.
566
+ """
567
+ t = t or P.today()
568
+ rows = pool(team_id=team_id, t=t)
569
+
570
+ # ⭐⭐ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active
571
+ # products, asked fresh β€” not a `len()` over anything this module or `products.catalogue()`
572
+ # built. NC: drop one row from `pool()` and this goes red; that is the whole point of it.
573
+ n_active = products.catalogue_count()
574
+ checks = [{
575
+ "check": "the pool holds one row per ACTIVE product (R12) β€” row count == an INDEPENDENT "
576
+ "Odoo search_count('product.product', active=True)",
577
+ "ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active,
578
+ }]
579
+
580
+ yf, yt = P.ytd(t)
581
+ sku_rev = products._sku_rev(yf, yt, team_id) or {}
582
+ in_pool = {r["code"] for r in rows}
583
+ outside = {c: v for c, v in sku_rev.items() if c not in in_pool}
584
+ ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2)
585
+ theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2)
586
+ dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2)
587
+ checks.append({
588
+ "check": "Ξ£ per-SKU YTD revenue in the grid + Ξ£ YTD revenue of codes no ACTIVE product "
589
+ "carries == modules.products._sku_rev over the same window",
590
+ "ours": round(ours + dropped, 2), "theirs": theirs,
591
+ "ok": abs((ours + dropped) - theirs) < 0.01,
592
+ "detail": {"in_the_grid": ours, "outside_the_grid": dropped,
593
+ "codes_outside": len(outside)},
594
+ })
595
+ # β›” AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is
596
+ # trivially true when checked against the dict that did the dropping β€” that is the
597
+ # self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE
598
+ # product carries means `products.catalogue()` missed a row that has revenue, which is the
599
+ # 2,717 defect returning in a smaller costume.
600
+ # ⚠ `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the
601
+ # catalogue under its own `pid:N` key and therefore cannot be in `outside` at all.
602
+ coded_outside = sorted(c for c in outside if not str(c).startswith("pid:"))
603
+ still_active = (O.get_odoo().search_count(
604
+ 'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)])
605
+ if coded_outside else 0)
606
+ checks.append({
607
+ "check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived "
608
+ "stay out) β€” asked of Odoo, never of the catalogue that did the dropping",
609
+ "ours": still_active, "theirs": 0, "ok": still_active == 0,
610
+ "detail": {"codes": coded_outside[:10], "n_codes": len(outside)},
611
+ })
612
+ # ── ⭐⭐ WAVE 30 W30-T34 β€” THE PRICELIST COLUMNS, AGAINST ORACLES THAT CANNOT SEE OUR JOIN ──
613
+ #
614
+ # Three legs, because the column can fail in three different ways and only one of them is a
615
+ # count. `products.pricelist_by_code` builds a `{code: {col: price}}` map by expanding each
616
+ # rule over the products it names, IN PYTHON β€” that expansion is the fragile part, so every
617
+ # leg below re-asks ODOO instead of re-reading the map.
618
+ price_report = _pricelist_by_code()[1]
619
+ _pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
620
+ _pl_id = {}
621
+ for _p in _pl_rows:
622
+ _pl_id.setdefault(str(_p.get('name') or '').strip(), _p['id'])
623
+ _today = P.today().isoformat()
624
+
625
+ _targets_memo = {}
626
+
627
+ def _rule_targets(list_name):
628
+ """(variant_ids, template_ids) a pricelist prices today β€” read FRESH from Odoo.
629
+
630
+ Memoised for the LIFE OF THIS CALL only: legs 1 and 2 both need all three lists, and
631
+ without this `validate()` makes six identical round trips instead of three (measured:
632
+ ~48s of the run). Deliberately NOT an `lru_cache` β€” an oracle that survives the process
633
+ is an oracle reading yesterday's Odoo.
634
+ """
635
+ if list_name in _targets_memo:
636
+ return _targets_memo[list_name]
637
+ var, tmpl = set(), set()
638
+ plid = _pl_id.get(list_name)
639
+ if plid is None:
640
+ return var, tmpl
641
+ for r in O.search_read(
642
+ 'product.pricelist.item',
643
+ [('pricelist_id', '=', plid), ('compute_price', '=', 'fixed'),
644
+ ('applied_on', 'in', ['0_product_variant', '1_product'])],
645
+ ['product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
646
+ 'date_start', 'date_end']):
647
+ ds, de = str(r.get('date_start') or '')[:10], str(r.get('date_end') or '')[:10]
648
+ if (ds and ds > _today) or (de and de < _today):
649
+ continue
650
+ fp = r.get('fixed_price')
651
+ if not isinstance(fp, (int, float)) or fp <= 0:
652
+ continue
653
+ if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
654
+ var.add(O.m2o_id(r['product_id']))
655
+ elif r.get('product_tmpl_id'):
656
+ tmpl.add(O.m2o_id(r['product_tmpl_id']))
657
+ _targets_memo[list_name] = (var, tmpl)
658
+ return var, tmpl
659
+
660
+ # LEG 1 β€” COVERAGE PER LIST. Ours: cells we filled. Theirs: a bare Odoo `search_count` of
661
+ # ACTIVE products a fresh read of that list's rules reaches. The rule set is shared (it IS
662
+ # the data) but the EXPANSION is not, and the expansion is what breaks.
663
+ for _col, _name in products.PRICELIST_COLUMNS:
664
+ _var, _tmpl = _rule_targets(_name)
665
+ _dom = [('active', '=', True), '|', ('id', 'in', sorted(_var)),
666
+ ('product_tmpl_id', 'in', sorted(_tmpl))]
667
+ theirs = O.get_odoo().search_count('product.product', _dom) if (_var or _tmpl) else 0
668
+ ours = sum(1 for r in rows if isinstance(r.get(_col), (int, float)))
669
+ checks.append({
670
+ "check": f"{_name} pricelist: SKUs priced in the grid == an INDEPENDENT Odoo count "
671
+ f"of ACTIVE products its date-valid fixed rules reach",
672
+ "ours": ours, "theirs": theirs, "ok": ours == theirs,
673
+ "detail": {"column": _col, "variant_rules": len(_var), "template_rules": len(_tmpl)},
674
+ })
675
+
676
+ # LEG 2 β€” β›” THE TICKET'S OWN NEGATIVE CONTROL, AS A LEG. A SKU with no specific item must
677
+ # render BLANK. Asked of ODOO, never of the map that produced the blank β€” "everything I left
678
+ # empty was genuinely unpriced" is trivially true when checked against the dict that emptied
679
+ # it, which is the self-sealing shape the population leg above exists to end.
680
+ _unpriced = [r for r in rows
681
+ if not any(isinstance(r.get(c), (int, float))
682
+ for c, _n in products.PRICELIST_COLUMNS)]
683
+ _codes = {r["code"] for r in _unpriced}
684
+ _stray = 0
685
+ if _codes:
686
+ # Resolve those codes back to Odoo ids and ask whether ANY declared list prices them.
687
+ _ids, _tmpls = set(), set()
688
+ for _p in O.search_read('product.product',
689
+ [('active', '=', True),
690
+ ('default_code', 'in', sorted(c for c in _codes
691
+ if not c.startswith("pid:")))],
692
+ ['id', 'product_tmpl_id']):
693
+ _ids.add(_p['id'])
694
+ _tmpls.add(O.m2o_id(_p.get('product_tmpl_id')))
695
+ for _col, _name in products.PRICELIST_COLUMNS:
696
+ _v, _t = _rule_targets(_name)
697
+ _stray += len((_v & _ids) | ({t for t in _tmpls if t in _t}))
698
+ checks.append({
699
+ "check": "every SKU rendering a BLANK price is genuinely unpriced on all three declared "
700
+ "lists (W29-T52's NC: no fallback dressed as a price) β€” asked of Odoo",
701
+ "ours": _stray, "theirs": 0, "ok": _stray == 0,
702
+ "detail": {"blank_skus": len(_unpriced), "priced_skus": len(rows) - len(_unpriced)},
703
+ })
704
+
705
+ # LEG 3 β€” ⭐ R6's SECOND SENTENCE, AS A NUMBER. *"If there is lag or it can't be done, you
706
+ # need to explicitly tell me why and recommend a fix."* Everything the reader cannot see is
707
+ # counted here rather than dropped. The ASSERTION is the one thing that must never be true β€”
708
+ # a price cell that is present and not a positive number, i.e. a "free" SKU β€” while the
709
+ # rest rides `detail` so it is reported without reddening an honest day's data.
710
+ _bad = [r["code"] for r in rows
711
+ for c, _n in products.PRICELIST_COLUMNS
712
+ if r.get(c) is not None and not (isinstance(r.get(c), (int, float)) and r[c] > 0)]
713
+ # β›” MEASURED 2026-08-12 AND NOT FIXABLE FROM THIS FENCE: `aios_grid.rows_from_pool` sends
714
+ # every non-text field through `_round(v)` = `round(v)` with no ndigits, i.e. to a whole
715
+ # dollar. 52.1% of fixed prices carry cents and the median relative error is 5.26%, so this
716
+ # column ties to the cent HERE and ships rounded. Reported, never silently enforced.
717
+ _cents = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
718
+ if isinstance(r.get(c), (int, float)) and abs(r[c] - round(r[c])) > 1e-9)
719
+ _to_zero = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
720
+ if isinstance(r.get(c), (int, float)) and round(r[c]) == 0)
721
+ checks.append({
722
+ "check": "no price cell is present-but-not-a-positive-number (a 0 would read as FREE; "
723
+ "an unset rule must be blank), and what the reader cannot see is COUNTED",
724
+ "ours": len(_bad), "theirs": 0, "ok": not _bad,
725
+ "detail": {"bad_cells": _bad[:10], "reader_report": price_report,
726
+ "wire_rounding_loses_cents_on": _cents,
727
+ "wire_rounding_to_zero_dollars": _to_zero},
728
+ })
729
+
730
+ # WAVE 17 R3 β€” the BUY SIGNAL must be a total, exact partition of the catalogue, and every
731
+ # member of it must be re-derivable from the two columns beside it. A signal somebody buys
732
+ # stock on cannot be "mostly right": the failure that matters is a row that says OK because
733
+ # an input was missing, so the blank leg is checked as hard as the other two.
734
+ if team_id is None: # consolidated only β€” the inputs exist only there
735
+ buy = [r for r in rows if r.get("buy_now") == "Buy now"]
736
+ ok_rows = [r for r in rows if r.get("buy_now") == "OK"]
737
+ blank = [r for r in rows if r.get("buy_now") is None]
738
+ mis = sum(1 for r in buy if not (r["dos"] < r["lead_days"]))
739
+ mis += sum(1 for r in ok_rows if not (r["dos"] >= r["lead_days"]))
740
+ # A blank must be UNKNOWN β€” never a row we could have answered and quietly did not.
741
+ mis += sum(1 for r in blank
742
+ if isinstance(r.get("dos"), (int, float))
743
+ and isinstance(r.get("lead_days"), (int, float)) and r["lead_days"] > 0)
744
+ checks.append({
745
+ "check": "Buy signal partitions the catalogue (buy + ok + unknown == rows, none "
746
+ "misclassified)",
747
+ "ours": len(buy) + len(ok_rows) + len(blank) - mis, "theirs": len(rows),
748
+ "ok": mis == 0 and len(buy) + len(ok_rows) + len(blank) == len(rows),
749
+ "detail": {"buy_now": len(buy), "ok": len(ok_rows), "unknown": len(blank),
750
+ "misclassified": mis},
751
+ })
752
+ # ⭐ OWNER 2026-08-03 β€” "use the formula fields to come up to the same EXACT figures".
753
+ #
754
+ # This is that sentence, as a check. The retired preset column and the formula that
755
+ # replaces it must agree on every SKU, in all three states, or the replacement is not a
756
+ # replacement. `_buy_signal_formula` is a line-by-line port of the client formula
757
+ # engine's evaluation of
758
+ #
759
+ # IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
760
+ #
761
+ # including the part that is easy to get wrong: a comparison against a BLANK is blank,
762
+ # never a coerced 0. (The client engine's `cmp` returns null unless both sides read as
763
+ # numbers, and `IF` refuses a non-boolean condition β€” so a missing `dos` yields "" and
764
+ # not "Buy now". A filter engine would have said `0 < 30` and swept in every SKU that
765
+ # never sells through; the formula engine does not, and this check is what holds it.)
766
+ #
767
+ # ⚠ It compares SETS OF SKUs, not counts. Two different partitions can share a shape.
768
+ disagree = sorted(r["code"] for r in rows
769
+ if (r.get("buy_now") or "") != _buy_signal_formula(r))
770
+ checks.append({
771
+ "check": 'Buy signal as a FORMULA field == the retired preset column, per SKU '
772
+ '(IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), ""))',
773
+ "ours": len(rows) - len(disagree), "theirs": len(rows),
774
+ "ok": not disagree,
775
+ "detail": {"disagreeing_skus": disagree[:10], "n_disagree": len(disagree)},
776
+ })
777
+ # The other half of "the same figures": the SHARED Buy list view can no longer filter on
778
+ # the retired column, and its replacement conditions must select the same SKUs. They are
779
+ # `dos isNotEmpty AND lead_days isNotEmpty AND lead_days > 0 AND dos < lead_days`
780
+ # (_seed_wave17.views), so this reproduces exactly that conjunction.
781
+ #
782
+ # β›” THE NEGATIVE CONTROL IS WHY THIS IS NOT `cover_gap_d < 0`, which reads like the
783
+ # obvious filter and is WRONG: `cover_gap_d` is `int(round(dos - lead))`, so a genuine
784
+ # gap of -0.4 days rounds to 0 and that SKU drops off a buy list it belongs on.
785
+ view_rows = {r["code"] for r in rows
786
+ if isinstance(r.get("dos"), (int, float))
787
+ and isinstance(r.get("lead_days"), (int, float))
788
+ and r["lead_days"] > 0 and r["dos"] < r["lead_days"]}
789
+ signal_rows = {r["code"] for r in buy}
790
+ rounding_would_miss = sorted(
791
+ c for c in signal_rows
792
+ if next((r for r in rows if r["code"] == c), {}).get("cover_gap_d") == 0)
793
+ checks.append({
794
+ "check": "Buy list view conditions (dos/lead_days, no retired column) select "
795
+ "exactly the Buy-now SKUs",
796
+ "ours": len(view_rows), "theirs": len(signal_rows),
797
+ "ok": view_rows == signal_rows,
798
+ "detail": {"only_in_view": sorted(view_rows - signal_rows)[:10],
799
+ "only_in_signal": sorted(signal_rows - view_rows)[:10],
800
+ # Reported, not asserted: how many SKUs a `cover_gap_d < 0` filter would
801
+ # have silently dropped. 0 today does not make that filter correct.
802
+ "cover_gap_rounds_to_zero": len(rounding_would_miss)},
803
+ })
804
+ return checks
platform/modules/products.py CHANGED
@@ -1,737 +1,737 @@
1
- """Products / SKU module β€” SKU health: YoY movers (risers & decliners), zombie SKUs
2
- (catalog rot β€” sellable, formerly selling, now dead), new winners, coverage collapse
3
- (SKUs losing customer breadth β€” the FFS-recovery early-warning signal), and velocity leaders.
4
-
5
- Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here.
6
- Basket / co-purchase (MBA) is intentionally deferred β€” the existing client app computes it
7
- runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the
8
- project guardrails keep off the local PC. (See BACKLOG.)
9
-
10
- All line-level (sale.order.line), RI+FFS scope, excluded accounts removed β€” reusing sale_line_domain.
11
- Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/
12
- window), so it's its own function the app calls lazily and caches.
13
- """
14
- import sys
15
- import datetime as dt
16
- from pathlib import Path
17
- sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
18
- from functools import lru_cache
19
- import core.odoo as O
20
- import core.periods as P
21
- import modules.sales as sales_mod
22
-
23
- # SKU-health views are about real products β€” exclude service/delivery pseudo-SKUs
24
- # (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby
25
- # on the dot-path does not). Validation reconciles on this same non-service universe.
26
- _NO_SVC = [('product_id.type', '!=', 'service')]
27
-
28
-
29
- @lru_cache(maxsize=1)
30
- def _code_map():
31
- """product_id β†’ SKU code. Multiple product records can share a default_code (re-SKUing
32
- /duplicates); keying by code merges them so a re-coded item doesn't read as a fake
33
- decliner + fake riser. Products without a code fall back to a per-id key. Archived
34
- products are INCLUDED β€” re-SKUing typically archives the old record and creates a new
35
- one under the same code, and the old record still carries last-year sales."""
36
- prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'],
37
- limit=50000)
38
- return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
39
- for p in prods}
40
-
41
-
42
- def _sku_rev(date_from, date_to, team_id=None):
43
- """{sku_code: {'name','rev','qty','orders'}} over a window (services excluded,
44
- duplicate product records merged by SKU code)."""
45
- g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
46
- ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
47
- codes = _code_map()
48
- out = {}
49
- for r in g:
50
- pid = O.m2o_id(r.get('product_id'))
51
- if not pid:
52
- continue
53
- key = codes.get(pid, f"pid:{pid}")
54
- e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')),
55
- 'rev': 0.0, 'qty': 0.0, 'orders': 0})
56
- e['rev'] += r.get('price_subtotal') or 0.0
57
- e['qty'] += r.get('product_uom_qty') or 0.0
58
- e['orders'] += r.get('__count') or 0
59
- return out
60
-
61
-
62
- def yoy_movers(t=None, limit=20, team_id=None):
63
- """Top SKU risers and decliners by YTD-vs-same-period-LY revenue change."""
64
- t = t or P.today()
65
- yf, yt = P.ytd(t)
66
- lf, lt = P.ytd_last_year(t)
67
- this = _sku_rev(yf, yt, team_id)
68
- last = _sku_rev(lf, lt, team_id)
69
- rows = []
70
- for pid in set(this) | set(last):
71
- tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')})
72
- lr = last.get(pid, {'rev': 0.0})
73
- name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or ''
74
- rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
75
- 'change': tr['rev'] - lr['rev']})
76
- risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit]
77
- decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit]
78
- return {'risers': risers, 'decliners': decliners}
79
-
80
-
81
- def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None):
82
- """Catalog rot: SKUs that sold materially last year but are ~dead this year."""
83
- t = t or P.today()
84
- yf, yt = P.ytd(t)
85
- lf, lt = P.ytd_last_year(t)
86
- this = _sku_rev(yf, yt, team_id)
87
- last = _sku_rev(lf, lt, team_id)
88
- rows = []
89
- for pid, lr in last.items():
90
- if lr['rev'] < min_prior:
91
- continue
92
- tr = this.get(pid, {'rev': 0.0})
93
- if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior β†’ not a zombie
94
- continue
95
- rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'],
96
- 'lost': lr['rev'] - tr['rev']})
97
- rows.sort(key=lambda x: -x['lost'])
98
- return rows[:limit]
99
-
100
-
101
- def new_winners(t=None, limit=20, min_this=1500.0, team_id=None):
102
- """SKUs that barely sold last year but are selling well this year."""
103
- t = t or P.today()
104
- yf, yt = P.ytd(t)
105
- lf, lt = P.ytd_last_year(t)
106
- this = _sku_rev(yf, yt, team_id)
107
- last = _sku_rev(lf, lt, team_id)
108
- rows = []
109
- for pid, tr in this.items():
110
- if tr['rev'] < min_this:
111
- continue
112
- lr = last.get(pid, {'rev': 0.0})
113
- if lr['rev'] > 0.05 * tr['rev']:
114
- continue
115
- rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
116
- 'gained': tr['rev'] - lr['rev']})
117
- rows.sort(key=lambda x: -x['gained'])
118
- return rows[:limit]
119
-
120
-
121
- def velocity_leaders(t=None, limit=25, team_id=None):
122
- """Top SKUs by LTM unit velocity (units/month) and revenue."""
123
- t = t or P.today()
124
- lf, lt = P.ltm(t)
125
- sku = _sku_rev(lf, lt, team_id)
126
- rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0,
127
- 'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()]
128
- rows.sort(key=lambda x: -x['units_ltm'])
129
- return rows[:limit]
130
-
131
-
132
- def _code_category():
133
- """code -> category name. Built from _code_map (pid->code) + the sales category map
134
- (pid->category); the first record carrying a code sets that code's category."""
135
- codes = _code_map()
136
- cats = sales_mod._product_cat()
137
- out = {}
138
- for pid, code in codes.items():
139
- if code not in out and pid in cats:
140
- out[code] = cats[pid]
141
- return out
142
-
143
-
144
- def directory(t=None, team_id=None):
145
- """Every SKU that sold this YTD or the same period last year, with the fields the SKU
146
- drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the
147
- customer directory β€” the full list (filtering happens in the UI), sorted by YTD revenue."""
148
- t = t or P.today()
149
- yf, yt = P.ytd(t)
150
- lf, lt = P.ytd_last_year(t)
151
- this = _sku_rev(yf, yt, team_id)
152
- last = _sku_rev(lf, lt, team_id)
153
- code_cat = _code_category()
154
- rows = []
155
- for code in set(this) | set(last):
156
- tr = this.get(code, {})
157
- lr = last.get(code, {})
158
- rev_ytd = tr.get('rev', 0.0)
159
- rev_ly = lr.get('rev', 0.0)
160
- rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code,
161
- 'category': code_cat.get(code, '(uncategorized)'),
162
- 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly,
163
- 'yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
164
- 'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)})
165
- rows.sort(key=lambda r: -r['rev_ytd'])
166
- return rows
167
-
168
-
169
- def catalogue():
170
- """`{sku_code: {'product', 'category', 'id'}}` β€” EVERY ACTIVE product, sold or not.
171
-
172
- β›” THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s
173
- row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU
174
- that never sold cannot exist in it. That is CORRECT for its own callers β€” `yoy_movers`,
175
- `zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe β€”
176
- and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of
177
- 5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in
178
- the consumer, never a window removed from `directory()`: removing the window alone lands at
179
- 3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all.
180
-
181
- ⚠ NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one
182
- catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than
183
- merged β€” a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold.
184
-
185
- Keyed exactly like `_code_map()` β€” the `default_code`, or a `pid:N` fallback for the 33 active
186
- records that carry none β€” so the join against `directory()` is code-for-code with no
187
- normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields
188
- off a sale line, so a never-sold row wears the same format as a sold one.
189
-
190
- β›” RAISES on a truncated read rather than returning a short catalogue. A silently short pull
191
- would put the grid back at a plausible wrong number with every gate green β€” the exact failure
192
- this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard
193
- `modules/backorders.py:161` already uses).
194
- """
195
- dom = [('active', '=', True)]
196
- prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'],
197
- limit=50000)
198
- n = O.get_odoo().search_count('product.product', dom)
199
- if len(prods) != n:
200
- raise ValueError(
201
- f"products.catalogue: the product pull is TRUNCATED β€” read {len(prods)} rows against "
202
- f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
203
- f"nothing reporting it; raise the limit before shipping this.")
204
- code_cat = _code_category()
205
- out = {}
206
- for p in prods:
207
- code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
208
- # First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero
209
- # active products share a code, so this branch is a guard, not a merge policy.
210
- # ⭐⭐ W33-T43 (R2 / amendment A2) β€” `id` RIDES THE ROW. Odoo's `product.product` id is
211
- # already in hand (the `pid:{id}` fallback above uses it and then threw it away), and it is
212
- # the ONE column `ut_odoo_products` had that `product_data` lacked. R2 merges onto the
213
- # legacy key and keeps every data column, so retiring the twin without this would lose a
214
- # real identifier rather than a navigational link.
215
- # ⚠ A product's grid pid is `crc32(default_code)`, NOT the Odoo id β€” unlike a CUSTOMER row,
216
- # whose pid IS the partner id. That asymmetry is exactly why this cannot be derived
217
- # downstream in `aios_grid.py` and has to be carried from the source read.
218
- out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code,
219
- 'category': code_cat.get(code, '(uncategorized)'),
220
- 'id': p['id']})
221
- return out
222
-
223
-
224
- #: ⭐⭐ WAVE 30 / W30-T34 (the carried W29-T52) β€” THE PRICELIST STRATUM, PER LIST.
225
- #:
226
- #: MEASURED LIVE 2026-08-12, and every one of these numbers shaped the design rather than
227
- #: decorating it:
228
- #:
229
- #: Β· **Five pricelists, three of them material.** Date-valid fixed rules: Fisch 5,139,
230
- #: Royal 1 2,702, Royal 2 2,605, Public Pricelist 17, Giftware Deals 1. At SKU level that
231
- #: is Fisch 3,971 Β· Royal 1 1,920 Β· Royal 2 1,857 Β· Public 6 Β· Giftware 0.
232
- #: Β· **3,988 of 5,871 active products carry a usable fixed rule; 1,883 carry NONE.**
233
- #: Β· **1,853 SKUs are priced on all three lists AND THE LISTS DISAGREE** β€” median relative
234
- #: spread 16%, max 82%. That is why this is three columns rather than one "the price" or a
235
- #: low/high pair: neither can tell a Fisch rep what Fisch sells the SKU for, which is the
236
- #: only question the column exists to answer.
237
- #: Β· β›” **`list_price` is 1.00 on 5,817 of the 5,871** β€” so the single `3_global` rule
238
- #: computed over it is meaningless, exactly as W29-T52 said. **This reader never falls
239
- #: through to it**, and that is the ticket's own negative control: a SKU with no specific
240
- #: item renders BLANK, never a fallback dressed as a price.
241
- #:
242
- #: ⚠ RESOLVED BY NAME, NOT BY ID. A pricelist renamed or deleted in Odoo makes its column go
243
- #: blank and makes `report["lists_missing"]` name it β€” a hard-coded id would keep pointing at
244
- #: whatever inherited it and mislabel every cell silently.
245
- PRICELIST_COLUMNS = (
246
- ("price_fisch", "Fisch"),
247
- ("price_royal_1", "Royal 1"),
248
- ("price_royal_2", "Royal 2"),
249
- )
250
-
251
-
252
- def pricelist_by_code():
253
- """`({code: {column_key: price}}, report)` β€” the date-valid FIXED base-tier price for each
254
- declared pricelist, keyed EXACTLY as `catalogue()` keys its rows.
255
-
256
- β›” THE KEYING IS LOAD-BEARING, not a detail. `catalogue()` keys on `default_code` with a
257
- `pid:{id}` fallback for the ~33 active records that carry none. Keying this map any other
258
- way would leave those SKUs permanently blank while an oracle counting active products
259
- counted them β€” a red gate on a working build, or worse, a silent hole nobody counts.
260
-
261
- **The base tier, deliberately.** `pricecomp._tier_for` picks the highest `min_quantity` at
262
- or below an order's quantity, because it is pricing a LINE that has one. A catalogue column
263
- has no quantity in hand, so it takes the LOWEST `min_quantity` β€” the price at qty 1. Rules
264
- above that break are a bulk price, and `report["qty_break_only"]` counts the SKUs whose only
265
- rule sits on one (MEASURED: 10 rules of 10,464 carry a break at all).
266
-
267
- **Variant rules beat template rules**, matching `pricecomp._tier_for`: a `0_product_variant`
268
- rule is the more specific statement about this exact SKU.
269
-
270
- ⚠ DEGRADES TO `({}, report)` on a read failure, matching `_inventory_by_code` rather than
271
- `catalogue()`: these are COLUMNS, and a product grid that will not render because pricing is
272
- momentarily unreachable is a worse failure than one with blank price columns. The blanks are
273
- not silent β€” `product_data.validate()`'s coverage leg reconciles each column against a fresh
274
- Odoo count and goes red at zero.
275
-
276
- ⭐ **The report exists because R6's second sentence is law** (*"if it can't be done, you need
277
- to explicitly tell me why and recommend a fix"*). Everything this reader CANNOT see is
278
- counted rather than dropped: rules on pricelists the contract does not declare, non-fixed
279
- (`formula`/`percent`) rules, the `3_global` fallback, and prices that only exist above a
280
- quantity break.
281
- """
282
- # ⚠ THE COUNTERS ARE DISJOINT AND ORDER-DEPENDENT, and saying so is the difference between a
283
- # report and a misleading one. A rule is classified ONCE, by the first reason it is skipped:
284
- # undeclared list β†’ out of date β†’ not fixed β†’ global β†’ zero price. So `rules_not_fixed: 0`
285
- # means "no formula rule on a list we declare", NOT "this Odoo has no formula rules"
286
- # (MEASURED 2026-08-12: it has exactly one, and it sits on Public Pricelist, which the
287
- # contract does not declare β€” so it lands in `rules_undeclared_list`).
288
- report = {"lists_missing": [], "rules_total": 0, "rules_undeclared_list": 0,
289
- "rules_not_fixed": 0, "rules_global": 0, "rules_out_of_date": 0,
290
- "rules_zero_price": 0, "qty_break_only": 0}
291
- try:
292
- pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
293
- by_name = {}
294
- for p in pl_rows:
295
- by_name.setdefault(str(p.get('name') or '').strip(), p['id'])
296
- wanted = {}
297
- for col, name in PRICELIST_COLUMNS:
298
- pid = by_name.get(name)
299
- if pid is None:
300
- report["lists_missing"].append(name)
301
- else:
302
- wanted[pid] = col
303
-
304
- # ⭐⭐ FILTERED SERVER-SIDE, AND THE REASON IS A MEASUREMENT, NOT A STYLE PREFERENCE.
305
- # Reading all 10,464 rules with 9 fields and sorting them in Python costs **30.1s** on
306
- # this connection; the same rules under a server-side domain with 6 fields cost
307
- # **13.6s** (measured 2026-08-12, back to back). `pool()` runs this on a scope's
308
- # first-ever build, so that 16.5s is 16.5s of somebody's page load.
309
- #
310
- # ⚠ The domain reproduces the Python predicate EXACTLY, and the null legs are the part
311
- # that is easy to get wrong: an absent `date_start` is `False`, not a past date, so
312
- # `('date_start','<=',today)` ALONE would drop every open-ended rule β€” which is almost
313
- # all of them.
314
- today = P.today().isoformat()
315
- _declared = sorted(wanted)
316
- _kinds = ['0_product_variant', '1_product']
317
- _live = [('pricelist_id', 'in', _declared), ('compute_price', '=', 'fixed'),
318
- ('applied_on', 'in', _kinds),
319
- '|', ('date_start', '=', False), ('date_start', '<=', today),
320
- '|', ('date_end', '=', False), ('date_end', '>=', today),
321
- ('fixed_price', '>', 0)]
322
- rules = O.search_read(
323
- 'product.pricelist.item', _live,
324
- ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
325
- 'min_quantity']) if _declared else []
326
-
327
- # ⭐ R6's SECOND SENTENCE, PAID FOR WITH `search_count` RATHER THAN A WIDER READ. Six
328
- # counts cost ~1.2s together; the rules they count would cost 16s to read. What this
329
- # reader cannot see is still REPORTED β€” it is just no longer transferred.
330
- def _n(extra):
331
- try:
332
- return O.get_odoo().search_count('product.pricelist.item', extra)
333
- except Exception:
334
- return -1 # -1 reads as "not measured", never as zero
335
- _dated = ['|', ('date_start', '=', False), ('date_start', '<=', today),
336
- '|', ('date_end', '=', False), ('date_end', '>=', today)]
337
- report["rules_total"] = _n([])
338
- report["rules_undeclared_list"] = _n([('pricelist_id', 'not in', _declared)]) \
339
- if _declared else report["rules_total"]
340
- if _declared:
341
- _on = [('pricelist_id', 'in', _declared)]
342
- # `percent_price` and formula rules are read NOWHERE in this repo. A rule we cannot
343
- # price is one the operator is TOLD about, never one that quietly becomes a blank.
344
- report["rules_not_fixed"] = _n(_on + [('compute_price', '!=', 'fixed')])
345
- report["rules_global"] = _n(_on + [('compute_price', '=', 'fixed'),
346
- ('applied_on', 'not in', _kinds)])
347
- report["rules_out_of_date"] = (
348
- _n(_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)])
349
- - _n(_on + [('compute_price', '=', 'fixed'),
350
- ('applied_on', 'in', _kinds)] + _dated))
351
- # A zero price is not "free" β€” it is an unset rule. Blank says so; 0 does not.
352
- report["rules_zero_price"] = _n(
353
- _on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)]
354
- + _dated + [('fixed_price', '<=', 0)])
355
-
356
- by_var, by_tmpl = {}, {}
357
- for r in rules:
358
- col = wanted[O.m2o_id(r.get('pricelist_id'))]
359
- if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
360
- by_var.setdefault((col, O.m2o_id(r['product_id'])), []).append(r)
361
- elif r.get('product_tmpl_id'):
362
- by_tmpl.setdefault((col, O.m2o_id(r['product_tmpl_id'])), []).append(r)
363
-
364
- dom = [('active', '=', True)]
365
- prods = O.search_read('product.product', dom,
366
- ['id', 'default_code', 'product_tmpl_id'], limit=50000)
367
- n = O.get_odoo().search_count('product.product', dom)
368
- if len(prods) != n:
369
- # Same guard, same reason as `catalogue()`: a short pull renders as a plausible
370
- # smaller set of priced SKUs with nothing reporting it.
371
- raise ValueError(
372
- f"products.pricelist_by_code: the product pull is TRUNCATED β€” read {len(prods)} "
373
- f"rows against a search_count of {n}.")
374
- except Exception as e:
375
- report["error"] = f"{type(e).__name__}: {str(e)[:200]}"
376
- return {}, report
377
-
378
- out = {}
379
- for p in prods:
380
- code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
381
- tmpl = O.m2o_id(p.get('product_tmpl_id'))
382
- cells = {}
383
- for col, _name in PRICELIST_COLUMNS:
384
- cands = by_var.get((col, p['id'])) or by_tmpl.get((col, tmpl))
385
- if not cands:
386
- continue
387
- base = min(cands, key=lambda r: r.get('min_quantity') or 0.0)
388
- if (base.get('min_quantity') or 0.0) > 1.0:
389
- report["qty_break_only"] += 1
390
- cells[col] = base.get('fixed_price')
391
- if cells:
392
- out.setdefault(code, {}).update(cells)
393
- return out, report
394
-
395
-
396
- def catalogue_count():
397
- """The INDEPENDENT population oracle: Odoo's own count of active products.
398
-
399
- Deliberately a bare `search_count` and not a `len()` over anything this module built β€” the
400
- 2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides
401
- of its reconciliation from `_sku_rev`, so the oracle could never see a missing row.
402
- """
403
- return O.get_odoo().search_count('product.product', [('active', '=', True)])
404
-
405
-
406
- def categories(t=None, team_id=None):
407
- """Sorted distinct category names present in the SKU directory (for the drill-down filter)."""
408
- return sorted({r['category'] for r in directory(t, team_id)})
409
-
410
-
411
- def _coverage(date_from, date_to, team_id=None):
412
- """{sku_code: distinct_customer_count} via 2-level read_group (slow-ish), merged by code.
413
- (A customer buying two records sharing a code can count twice; duplicates are rare, and
414
- code-level avoids the bigger error of a re-SKUed product reading as full coverage loss.)"""
415
- g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
416
- ['__count'], ['product_id', 'order_partner_id'], lazy=False)
417
- codes = _code_map()
418
- cov = {}
419
- names = {}
420
- for r in g:
421
- pid = O.m2o_id(r.get('product_id'))
422
- if not pid:
423
- continue
424
- key = codes.get(pid, f"pid:{pid}")
425
- cov[key] = cov.get(key, 0) + 1
426
- names.setdefault(key, O.m2o_name(r.get('product_id')))
427
- return cov, names
428
-
429
-
430
- def coverage_collapse(t=None, limit=25, min_prior_custs=8, team_id=None):
431
- """SKUs that lost the most customer breadth YoY β€” early warning a SKU is dying even if
432
- revenue hasn't fully cratered yet."""
433
- t = t or P.today()
434
- yf, yt = P.ytd(t)
435
- lf, lt = P.ytd_last_year(t)
436
- cov_t, names_t = _coverage(yf, yt, team_id)
437
- cov_l, names_l = _coverage(lf, lt, team_id)
438
- rows = []
439
- for pid, lc in cov_l.items():
440
- if lc < min_prior_custs:
441
- continue
442
- tc = cov_t.get(pid, 0)
443
- drop = lc - tc
444
- if drop <= 0:
445
- continue
446
- rows.append({'code': pid, 'product': names_l.get(pid) or names_t.get(pid) or '',
447
- 'custs_ly': lc, 'custs_ytd': tc, 'lost_custs': drop,
448
- 'pct_drop': drop / lc * 100})
449
- rows.sort(key=lambda x: (-x['lost_custs'], -x['pct_drop']))
450
- return rows[:limit]
451
-
452
-
453
- # ====================================================================== SKU DRAWER (mirror of customer drawer)
454
- def _sku_product_ids(code):
455
- """All product.product ids sharing this SKU code (merged variants / archived records)."""
456
- return [pid for pid, c in _code_map().items() if c == code]
457
-
458
-
459
- def _sku_dom(pids, date_from, date_to, team_id=None):
460
- return O.sale_line_domain(date_from, date_to, team_id, extra=[('product_id', 'in', pids)] + _NO_SVC)
461
-
462
-
463
- def _sku_name_category(pids):
464
- rows = O.search_read('product.product', [('id', 'in', pids)], ['name', 'categ_id'])
465
- name = rows[0]['name'] if rows else '(unknown)'
466
- catmap = sales_mod._product_cat()
467
- main = next((catmap.get(p) for p in pids if catmap.get(p)), '(uncategorized)')
468
- return name, main
469
-
470
-
471
- def _sku_buyers(pids, date_from, date_to, team_id=None):
472
- """{partner_id: {'name','rev','qty'}} for buyers of this SKU over a window."""
473
- g = O.read_group('sale.order.line', _sku_dom(pids, date_from, date_to, team_id),
474
- ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
475
- out = {}
476
- for r in g:
477
- pid = O.m2o_id(r.get('order_partner_id'))
478
- if pid:
479
- out[pid] = {'name': O.m2o_name(r.get('order_partner_id')),
480
- 'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0}
481
- return out
482
-
483
-
484
- def sku_detail(code, t=None, team_id=None, n_months=13, allsku=None):
485
- """KPIs (rev/qty/buyers YoY, GM%), monthly trend, rank & % of BU for one SKU code.
486
- Pass `allsku` (a cached _sku_rev YTD map) to skip the ~all-SKU rank read."""
487
- t = t or P.today()
488
- pids = _sku_product_ids(code)
489
- if not pids:
490
- return None
491
- yf, yt = P.ytd(t)
492
- lf, lt = P.ytd_last_year(t)
493
- mf, mt = P.ltm(t)
494
- name, category = _sku_name_category(pids)
495
-
496
- def s(df, dtt, field='price_subtotal'):
497
- return O.sum_field('sale.order.line', _sku_dom(pids, df, dtt, team_id), field)
498
- rev_ytd, rev_ly = s(yf, yt), s(lf, lt)
499
- qty_ytd, qty_ly = s(yf, yt, 'product_uom_qty'), s(lf, lt, 'product_uom_qty')
500
- buyers_ytd = len(_sku_buyers(pids, yf, yt, team_id))
501
- buyers_ly = len(_sku_buyers(pids, lf, lt, team_id))
502
- g = O.read_group('sale.order.line', _sku_dom(pids, mf, mt, team_id),
503
- ['price_subtotal:sum', 'margin:sum'], [], lazy=False)
504
- line_rev = (g[0].get('price_subtotal') if g else 0) or 0.0
505
- margin = (g[0].get('margin') if g else 0) or 0.0
506
-
507
- # Monthly trend: bin lines by their order's month in Python (dot-path month groupby is rejected
508
- # on sale.order.line), in 2 reads instead of 26 point queries.
509
- range_start = dt.date(t.year - 2, t.month, 1).isoformat()
510
- lines = O.search_read('sale.order.line', _sku_dom(pids, range_start, t.isoformat(), team_id),
511
- ['price_subtotal', 'order_id'])
512
- oids = list({O.m2o_id(line['order_id']) for line in lines if line.get('order_id')})
513
- odate = {}
514
- for i in range(0, len(oids), 1000):
515
- for o in O.search_read('sale.order', [('id', 'in', oids[i:i + 1000])], ['date_order']):
516
- if o.get('date_order'):
517
- odate[o['id']] = str(o['date_order'])[:7]
518
- mrev = {}
519
- for line in lines:
520
- ym = odate.get(O.m2o_id(line.get('order_id')))
521
- if ym:
522
- mrev[ym] = mrev.get(ym, 0.0) + (line.get('price_subtotal') or 0.0)
523
- monthly = []
524
- for ym, start, end in P.month_starts(n_months, t):
525
- y, m = int(ym[:4]) - 1, int(ym[5:7])
526
- monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0),
527
- 'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)})
528
-
529
- allsku = allsku if allsku is not None else _sku_rev(yf, yt, team_id)
530
- total = sum(v['rev'] for v in allsku.values()) or 1.0
531
- rank = next((i + 1 for i, (c, _v) in enumerate(sorted(allsku.items(), key=lambda kv: -kv[1]['rev']))
532
- if c == code), None)
533
- return {
534
- 'code': code, 'name': name, 'category': category,
535
- 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'rev_yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
536
- 'qty_ytd': qty_ytd, 'qty_ly': qty_ly, 'qty_yoy_pct': P.yoy_pct(qty_ytd, qty_ly),
537
- 'buyers_ytd': buyers_ytd, 'buyers_ly': buyers_ly, 'buyers_delta': buyers_ytd - buyers_ly,
538
- 'gm_pct': (margin / line_rev * 100) if line_rev else 0.0, 'gm_dollars': margin,
539
- 'rank': rank, 'n_skus': len(allsku), 'pct_of_bu': rev_ytd / total * 100,
540
- 'monthly': monthly,
541
- }
542
-
543
-
544
- def sku_buyer_bridge(code, t=None, team_id=None, top=12):
545
- """Who drives the SKU's YoY: retained / new / churned buyers + a $-ranked churned call list."""
546
- t = t or P.today()
547
- pids = _sku_product_ids(code)
548
- yf, yt = P.ytd(t)
549
- lf, lt = P.ytd_last_year(t)
550
- this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
551
- tset, lset = set(this), set(last)
552
- retained, new, churned = tset & lset, tset - lset, lset - tset
553
- churned_list = sorted([{'pid': p, 'customer': last[p]['name'], 'ly_rev': last[p]['rev']}
554
- for p in churned], key=lambda x: -x['ly_rev'])[:top]
555
- return {'retained': {'n': len(retained), 'rev': sum(this[p]['rev'] for p in retained)},
556
- 'new': {'n': len(new), 'rev': sum(this[p]['rev'] for p in new)},
557
- 'churned': {'n': len(churned), 'rev': sum(last[p]['rev'] for p in churned)},
558
- 'buyer_retention_pct': (len(retained) / len(lset) * 100) if lset else 0.0,
559
- 'churned_buyers': churned_list, 'buyers_this': len(tset), 'buyers_last': len(lset),
560
- 'this_total': sum(v['rev'] for v in this.values())}
561
-
562
-
563
- def sku_concentration(code, t=None, team_id=None):
564
- """Buyer-concentration risk: top-1/top-3 share, Herfindahl index, effective buyer count (LTM)."""
565
- t = t or P.today()
566
- mf, mt = P.ltm(t)
567
- buyers = _sku_buyers(_sku_product_ids(code), mf, mt, team_id)
568
- revs = sorted([v['rev'] for v in buyers.values()], reverse=True)
569
- total = sum(revs) or 1.0
570
- hhi = sum((r / total) ** 2 for r in revs)
571
- return {'n_buyers': len(revs), 'top1_pct': (revs[0] / total * 100) if revs else 0.0,
572
- 'top3_pct': (sum(revs[:3]) / total * 100) if revs else 0.0,
573
- 'hhi': hhi, 'eff_buyers': (1 / hhi) if hhi else 0.0}
574
-
575
-
576
- def sku_price_dispersion(code, t=None, team_id=None, cap=60):
577
- """Realized $/unit per buyer (LTM) vs the volume-weighted average; recoverable $ on below-VWAP
578
- accounts. list_price is unreliable, so the dispersion among actual buyers is the margin lever."""
579
- t = t or P.today()
580
- mf, mt = P.ltm(t)
581
- g = O.read_group('sale.order.line', _sku_dom(_sku_product_ids(code), mf, mt, team_id),
582
- ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
583
- rows, tot_rev, tot_qty = [], 0.0, 0.0
584
- for r in g:
585
- pid = O.m2o_id(r.get('order_partner_id'))
586
- rev = r.get('price_subtotal') or 0.0
587
- qty = r.get('product_uom_qty') or 0.0
588
- if not pid or qty <= 0:
589
- continue
590
- rows.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')),
591
- 'price': rev / qty, 'qty': qty, 'rev': rev})
592
- tot_rev += rev
593
- tot_qty += qty
594
- vwap = (tot_rev / tot_qty) if tot_qty else 0.0
595
- for r in rows:
596
- r['recoverable'] = max(0.0, vwap - r['price']) * r['qty']
597
- rows.sort(key=lambda x: -x['recoverable'])
598
- return {'vwap': vwap, 'n_buyers': len(rows),
599
- 'recoverable_total': sum(r['recoverable'] for r in rows), 'rows': rows[:cap]}
600
-
601
-
602
- def sku_top_buyers(code, t=None, team_id=None, top=15):
603
- """Ranked buyers of this SKU (YTD) with YoY β€” clickable to open the customer drawer."""
604
- t = t or P.today()
605
- pids = _sku_product_ids(code)
606
- yf, yt = P.ytd(t)
607
- lf, lt = P.ytd_last_year(t)
608
- this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
609
- rows = [{'pid': p, 'customer': v['name'], 'rev': v['rev'], 'qty': v['qty'],
610
- 'yoy_pct': P.yoy_pct(v['rev'], last.get(p, {}).get('rev', 0.0))} for p, v in this.items()]
611
- rows.sort(key=lambda x: -x['rev'])
612
- return rows[:top]
613
-
614
-
615
- _BUYER_TIERS = [('Whale (β‰₯$25k)', 25000.0), ('Large ($10–25k)', 10000.0),
616
- ('Mid ($2–10k)', 2000.0), ('Small (<$2k)', 0.0)]
617
-
618
-
619
- def sku_customer_analysis(code, t=None, team_id=None, top=15):
620
- """WHO buys this SKU (LTM), as customers: the value-tier mix of its buyers (by each buyer's TOTAL
621
- spend), and the accounts most DEPENDENT on it (this SKU as a share of their spend β€” who gets hurt
622
- most if it stocks out / who to protect)."""
623
- t = t or P.today()
624
- lf, lt = P.ltm(t)
625
- pids = _sku_product_ids(code)
626
- empty = {'segments': [], 'dependency': [], 'n_buyers': 0, 'avg_dependency': 0.0}
627
- if not pids:
628
- return empty
629
- buyers = _sku_buyers(pids, lf, lt, team_id) # {pid: {name, rev(this SKU), qty}}
630
- if not buyers:
631
- return empty
632
- bpids = list(buyers)
633
- g = O.read_group('sale.order', sales_mod.order_domain(lf, lt, team_id) + [('partner_id', 'in', bpids)],
634
- ['amount_untaxed:sum'], ['partner_id'], lazy=False)
635
- total = {O.m2o_id(r['partner_id']): (r.get('amount_untaxed') or 0.0) for r in g if r.get('partner_id')}
636
-
637
- def tier(rev):
638
- for nm, lo in _BUYER_TIERS:
639
- if rev >= lo:
640
- return nm
641
- return _BUYER_TIERS[-1][0]
642
- seg = {nm: {'tier': nm, 'buyers': 0, 'sku_rev': 0.0} for nm, _ in _BUYER_TIERS}
643
- dep = []
644
- for p, v in buyers.items():
645
- ct = total.get(p, v['rev']) or v['rev']
646
- s = seg[tier(ct)]
647
- s['buyers'] += 1
648
- s['sku_rev'] += v['rev']
649
- dep.append({'customer': v['name'], 'pid': p, 'sku_rev': v['rev'], 'cust_total': ct,
650
- 'dependency_pct': (v['rev'] / ct * 100) if ct else None})
651
- tot = sum(s['sku_rev'] for s in seg.values()) or 1.0
652
- segments = []
653
- for nm, _ in _BUYER_TIERS:
654
- s = seg[nm]
655
- s['rev_share'] = s['sku_rev'] / tot * 100
656
- s['avg_per_buyer'] = (s['sku_rev'] / s['buyers']) if s['buyers'] else 0.0
657
- segments.append(s)
658
- dep.sort(key=lambda x: -x['sku_rev'])
659
- deps_known = [d['dependency_pct'] for d in dep if d['dependency_pct'] is not None]
660
- return {'segments': segments, 'dependency': dep[:top], 'n_buyers': len(buyers),
661
- 'avg_dependency': (sum(deps_known) / len(deps_known)) if deps_known else 0.0}
662
-
663
-
664
- def sku_whitespace(code, t=None, team_id=None, top=15):
665
- """Customers who buy this SKU's category but NOT this SKU β€” ranked prospect list."""
666
- import modules.customers as cust_mod
667
- t = t or P.today()
668
- pids = _sku_product_ids(code)
669
- _name, category = _sku_name_category(pids)
670
- cat_buyers = cust_mod.category_buyers(category, team_id=team_id) or set()
671
- mf, mt = P.ltm(t)
672
- prospects = list(cat_buyers - set(_sku_buyers(pids, mf, mt, team_id)))
673
- if not prospects:
674
- return {'category': category, 'rows': []}
675
- g = O.read_group('sale.order', sales_mod.order_domain(mf, mt, team_id) + [('partner_id', 'in', prospects)],
676
- ['amount_untaxed:sum'], ['partner_id'], lazy=False)
677
- rows = [{'pid': O.m2o_id(r['partner_id']), 'customer': O.m2o_name(r['partner_id']),
678
- 'total_spend': r.get('amount_untaxed') or 0.0} for r in g if r.get('partner_id')]
679
- rows.sort(key=lambda x: -x['total_spend'])
680
- return {'category': category, 'rows': rows[:top]}
681
-
682
-
683
- def sku_drawer_bundle(code, t=None, team_id=None, allsku=None):
684
- """The whole SKU drawer's first paint in one cached unit: sku_detail first (the not-found gate),
685
- then the other six pulls CONCURRENTLY (O.parallel). Cuts a ~6-call cold open to ~max(call)."""
686
- detail = sku_detail(code, t=t, team_id=team_id, allsku=allsku)
687
- if detail is None:
688
- return {'detail': None}
689
- bridge, conc, price, buyers, white, ca = O.parallel([
690
- lambda: sku_buyer_bridge(code, t, team_id),
691
- lambda: sku_concentration(code, t, team_id),
692
- lambda: sku_price_dispersion(code, t, team_id),
693
- lambda: sku_top_buyers(code, t, team_id),
694
- lambda: sku_whitespace(code, t, team_id),
695
- lambda: sku_customer_analysis(code, t, team_id),
696
- ])
697
- return {'detail': detail, 'bridge': bridge, 'conc': conc, 'price': price,
698
- 'buyers': buyers, 'white': white, 'ca': ca}
699
-
700
-
701
- def validate(t=None, team_id=None):
702
- """Reconcile SKU metrics to independent Odoo aggregates. When team_id is set (a single BU
703
- selected) every check runs SCOPED to that BU, so the validation panel never reconciles
704
- against β€” or exposes β€” the other BU's numbers. All checks here are BU-scopeable (no cross-BU
705
- mirror), so team_id threads straight through."""
706
- t = t or P.today()
707
- yf, yt = P.ytd(t)
708
- checks = []
709
- sku = _sku_rev(yf, yt, team_id)
710
- sku_sum = sum(v['rev'] for v in sku.values())
711
- line_total = O.sum_field('sale.order.line',
712
- O.sale_line_domain(yf, yt, team_id, extra=_NO_SVC), 'price_subtotal')
713
- checks.append({'check': 'SKU rev: Ξ£(per-SKU) == total line revenue, ex-services (YTD)',
714
- 'a': round(sku_sum, 2), 'b': round(line_total, 2),
715
- 'gap': round(sku_sum - line_total, 2),
716
- 'ok': abs(sku_sum - line_total) <= 1.0})
717
-
718
- m = yoy_movers(t, limit=10**9, team_id=team_id)
719
- movers_sum = sum(r['change'] for r in m['risers']) + sum(r['change'] for r in m['decliners'])
720
- lf, lt = P.ytd_last_year(t)
721
- last_total = O.sum_field('sale.order.line',
722
- O.sale_line_domain(lf, lt, team_id, extra=_NO_SVC), 'price_subtotal')
723
- checks.append({'check': 'SKU movers: Ξ£(Ξ”) == (YTD βˆ’ LY) total',
724
- 'a': round(movers_sum, 2), 'b': round(line_total - last_total, 2),
725
- 'gap': round(movers_sum - (line_total - last_total), 2),
726
- 'ok': abs(movers_sum - (line_total - last_total)) <= 1.0})
727
-
728
- # SKU drawer: buyer-bridge reconciles β€” retained + new buyer revenue == this-period SKU revenue
729
- top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None
730
- if top_code:
731
- bb = sku_buyer_bridge(top_code, t, team_id=team_id)
732
- recon = bb['retained']['rev'] + bb['new']['rev']
733
- checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue',
734
- 'a': round(recon, 2), 'b': round(bb['this_total'], 2),
735
- 'gap': round(recon - bb['this_total'], 2),
736
- 'ok': abs(recon - bb['this_total']) <= 1.0})
737
- return checks
 
1
+ """Products / SKU module β€” SKU health: YoY movers (risers & decliners), zombie SKUs
2
+ (catalog rot β€” sellable, formerly selling, now dead), new winners, coverage collapse
3
+ (SKUs losing customer breadth β€” the FFS-recovery early-warning signal), and velocity leaders.
4
+
5
+ Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here.
6
+ Basket / co-purchase (MBA) is intentionally deferred β€” the existing client app computes it
7
+ runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the
8
+ project guardrails keep off the local PC. (See BACKLOG.)
9
+
10
+ All line-level (sale.order.line), RI+FFS scope, excluded accounts removed β€” reusing sale_line_domain.
11
+ Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/
12
+ window), so it's its own function the app calls lazily and caches.
13
+ """
14
+ import sys
15
+ import datetime as dt
16
+ from pathlib import Path
17
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
18
+ from functools import lru_cache
19
+ import core.odoo as O
20
+ import core.periods as P
21
+ import modules.sales as sales_mod
22
+
23
+ # SKU-health views are about real products β€” exclude service/delivery pseudo-SKUs
24
+ # (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby
25
+ # on the dot-path does not). Validation reconciles on this same non-service universe.
26
+ _NO_SVC = [('product_id.type', '!=', 'service')]
27
+
28
+
29
+ @lru_cache(maxsize=1)
30
+ def _code_map():
31
+ """product_id β†’ SKU code. Multiple product records can share a default_code (re-SKUing
32
+ /duplicates); keying by code merges them so a re-coded item doesn't read as a fake
33
+ decliner + fake riser. Products without a code fall back to a per-id key. Archived
34
+ products are INCLUDED β€” re-SKUing typically archives the old record and creates a new
35
+ one under the same code, and the old record still carries last-year sales."""
36
+ prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'],
37
+ limit=50000)
38
+ return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
39
+ for p in prods}
40
+
41
+
42
+ def _sku_rev(date_from, date_to, team_id=None):
43
+ """{sku_code: {'name','rev','qty','orders'}} over a window (services excluded,
44
+ duplicate product records merged by SKU code)."""
45
+ g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
46
+ ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
47
+ codes = _code_map()
48
+ out = {}
49
+ for r in g:
50
+ pid = O.m2o_id(r.get('product_id'))
51
+ if not pid:
52
+ continue
53
+ key = codes.get(pid, f"pid:{pid}")
54
+ e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')),
55
+ 'rev': 0.0, 'qty': 0.0, 'orders': 0})
56
+ e['rev'] += r.get('price_subtotal') or 0.0
57
+ e['qty'] += r.get('product_uom_qty') or 0.0
58
+ e['orders'] += r.get('__count') or 0
59
+ return out
60
+
61
+
62
+ def yoy_movers(t=None, limit=20, team_id=None):
63
+ """Top SKU risers and decliners by YTD-vs-same-period-LY revenue change."""
64
+ t = t or P.today()
65
+ yf, yt = P.ytd(t)
66
+ lf, lt = P.ytd_last_year(t)
67
+ this = _sku_rev(yf, yt, team_id)
68
+ last = _sku_rev(lf, lt, team_id)
69
+ rows = []
70
+ for pid in set(this) | set(last):
71
+ tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')})
72
+ lr = last.get(pid, {'rev': 0.0})
73
+ name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or ''
74
+ rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
75
+ 'change': tr['rev'] - lr['rev']})
76
+ risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit]
77
+ decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit]
78
+ return {'risers': risers, 'decliners': decliners}
79
+
80
+
81
+ def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None):
82
+ """Catalog rot: SKUs that sold materially last year but are ~dead this year."""
83
+ t = t or P.today()
84
+ yf, yt = P.ytd(t)
85
+ lf, lt = P.ytd_last_year(t)
86
+ this = _sku_rev(yf, yt, team_id)
87
+ last = _sku_rev(lf, lt, team_id)
88
+ rows = []
89
+ for pid, lr in last.items():
90
+ if lr['rev'] < min_prior:
91
+ continue
92
+ tr = this.get(pid, {'rev': 0.0})
93
+ if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior β†’ not a zombie
94
+ continue
95
+ rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'],
96
+ 'lost': lr['rev'] - tr['rev']})
97
+ rows.sort(key=lambda x: -x['lost'])
98
+ return rows[:limit]
99
+
100
+
101
+ def new_winners(t=None, limit=20, min_this=1500.0, team_id=None):
102
+ """SKUs that barely sold last year but are selling well this year."""
103
+ t = t or P.today()
104
+ yf, yt = P.ytd(t)
105
+ lf, lt = P.ytd_last_year(t)
106
+ this = _sku_rev(yf, yt, team_id)
107
+ last = _sku_rev(lf, lt, team_id)
108
+ rows = []
109
+ for pid, tr in this.items():
110
+ if tr['rev'] < min_this:
111
+ continue
112
+ lr = last.get(pid, {'rev': 0.0})
113
+ if lr['rev'] > 0.05 * tr['rev']:
114
+ continue
115
+ rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
116
+ 'gained': tr['rev'] - lr['rev']})
117
+ rows.sort(key=lambda x: -x['gained'])
118
+ return rows[:limit]
119
+
120
+
121
+ def velocity_leaders(t=None, limit=25, team_id=None):
122
+ """Top SKUs by LTM unit velocity (units/month) and revenue."""
123
+ t = t or P.today()
124
+ lf, lt = P.ltm(t)
125
+ sku = _sku_rev(lf, lt, team_id)
126
+ rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0,
127
+ 'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()]
128
+ rows.sort(key=lambda x: -x['units_ltm'])
129
+ return rows[:limit]
130
+
131
+
132
+ def _code_category():
133
+ """code -> category name. Built from _code_map (pid->code) + the sales category map
134
+ (pid->category); the first record carrying a code sets that code's category."""
135
+ codes = _code_map()
136
+ cats = sales_mod._product_cat()
137
+ out = {}
138
+ for pid, code in codes.items():
139
+ if code not in out and pid in cats:
140
+ out[code] = cats[pid]
141
+ return out
142
+
143
+
144
+ def directory(t=None, team_id=None):
145
+ """Every SKU that sold this YTD or the same period last year, with the fields the SKU
146
+ drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the
147
+ customer directory β€” the full list (filtering happens in the UI), sorted by YTD revenue."""
148
+ t = t or P.today()
149
+ yf, yt = P.ytd(t)
150
+ lf, lt = P.ytd_last_year(t)
151
+ this = _sku_rev(yf, yt, team_id)
152
+ last = _sku_rev(lf, lt, team_id)
153
+ code_cat = _code_category()
154
+ rows = []
155
+ for code in set(this) | set(last):
156
+ tr = this.get(code, {})
157
+ lr = last.get(code, {})
158
+ rev_ytd = tr.get('rev', 0.0)
159
+ rev_ly = lr.get('rev', 0.0)
160
+ rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code,
161
+ 'category': code_cat.get(code, '(uncategorized)'),
162
+ 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly,
163
+ 'yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
164
+ 'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)})
165
+ rows.sort(key=lambda r: -r['rev_ytd'])
166
+ return rows
167
+
168
+
169
+ def catalogue():
170
+ """`{sku_code: {'product', 'category', 'id'}}` β€” EVERY ACTIVE product, sold or not.
171
+
172
+ β›” THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s
173
+ row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU
174
+ that never sold cannot exist in it. That is CORRECT for its own callers β€” `yoy_movers`,
175
+ `zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe β€”
176
+ and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of
177
+ 5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in
178
+ the consumer, never a window removed from `directory()`: removing the window alone lands at
179
+ 3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all.
180
+
181
+ ⚠ NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one
182
+ catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than
183
+ merged β€” a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold.
184
+
185
+ Keyed exactly like `_code_map()` β€” the `default_code`, or a `pid:N` fallback for the 33 active
186
+ records that carry none β€” so the join against `directory()` is code-for-code with no
187
+ normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields
188
+ off a sale line, so a never-sold row wears the same format as a sold one.
189
+
190
+ β›” RAISES on a truncated read rather than returning a short catalogue. A silently short pull
191
+ would put the grid back at a plausible wrong number with every gate green β€” the exact failure
192
+ this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard
193
+ `modules/backorders.py:161` already uses).
194
+ """
195
+ dom = [('active', '=', True)]
196
+ prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'],
197
+ limit=50000)
198
+ n = O.get_odoo().search_count('product.product', dom)
199
+ if len(prods) != n:
200
+ raise ValueError(
201
+ f"products.catalogue: the product pull is TRUNCATED β€” read {len(prods)} rows against "
202
+ f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
203
+ f"nothing reporting it; raise the limit before shipping this.")
204
+ code_cat = _code_category()
205
+ out = {}
206
+ for p in prods:
207
+ code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
208
+ # First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero
209
+ # active products share a code, so this branch is a guard, not a merge policy.
210
+ # ⭐⭐ W33-T43 (R2 / amendment A2) β€” `id` RIDES THE ROW. Odoo's `product.product` id is
211
+ # already in hand (the `pid:{id}` fallback above uses it and then threw it away), and it is
212
+ # the ONE column `ut_odoo_products` had that `product_data` lacked. R2 merges onto the
213
+ # legacy key and keeps every data column, so retiring the twin without this would lose a
214
+ # real identifier rather than a navigational link.
215
+ # ⚠ A product's grid pid is `crc32(default_code)`, NOT the Odoo id β€” unlike a CUSTOMER row,
216
+ # whose pid IS the partner id. That asymmetry is exactly why this cannot be derived
217
+ # downstream in `aios_grid.py` and has to be carried from the source read.
218
+ out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code,
219
+ 'category': code_cat.get(code, '(uncategorized)'),
220
+ 'id': p['id']})
221
+ return out
222
+
223
+
224
+ #: ⭐⭐ WAVE 30 / W30-T34 (the carried W29-T52) β€” THE PRICELIST STRATUM, PER LIST.
225
+ #:
226
+ #: MEASURED LIVE 2026-08-12, and every one of these numbers shaped the design rather than
227
+ #: decorating it:
228
+ #:
229
+ #: Β· **Five pricelists, three of them material.** Date-valid fixed rules: Fisch 5,139,
230
+ #: Royal 1 2,702, Royal 2 2,605, Public Pricelist 17, Giftware Deals 1. At SKU level that
231
+ #: is Fisch 3,971 Β· Royal 1 1,920 Β· Royal 2 1,857 Β· Public 6 Β· Giftware 0.
232
+ #: Β· **3,988 of 5,871 active products carry a usable fixed rule; 1,883 carry NONE.**
233
+ #: Β· **1,853 SKUs are priced on all three lists AND THE LISTS DISAGREE** β€” median relative
234
+ #: spread 16%, max 82%. That is why this is three columns rather than one "the price" or a
235
+ #: low/high pair: neither can tell a Fisch rep what Fisch sells the SKU for, which is the
236
+ #: only question the column exists to answer.
237
+ #: Β· β›” **`list_price` is 1.00 on 5,817 of the 5,871** β€” so the single `3_global` rule
238
+ #: computed over it is meaningless, exactly as W29-T52 said. **This reader never falls
239
+ #: through to it**, and that is the ticket's own negative control: a SKU with no specific
240
+ #: item renders BLANK, never a fallback dressed as a price.
241
+ #:
242
+ #: ⚠ RESOLVED BY NAME, NOT BY ID. A pricelist renamed or deleted in Odoo makes its column go
243
+ #: blank and makes `report["lists_missing"]` name it β€” a hard-coded id would keep pointing at
244
+ #: whatever inherited it and mislabel every cell silently.
245
+ PRICELIST_COLUMNS = (
246
+ ("price_fisch", "Fisch"),
247
+ ("price_royal_1", "Royal 1"),
248
+ ("price_royal_2", "Royal 2"),
249
+ )
250
+
251
+
252
+ def pricelist_by_code():
253
+ """`({code: {column_key: price}}, report)` β€” the date-valid FIXED base-tier price for each
254
+ declared pricelist, keyed EXACTLY as `catalogue()` keys its rows.
255
+
256
+ β›” THE KEYING IS LOAD-BEARING, not a detail. `catalogue()` keys on `default_code` with a
257
+ `pid:{id}` fallback for the ~33 active records that carry none. Keying this map any other
258
+ way would leave those SKUs permanently blank while an oracle counting active products
259
+ counted them β€” a red gate on a working build, or worse, a silent hole nobody counts.
260
+
261
+ **The base tier, deliberately.** `pricecomp._tier_for` picks the highest `min_quantity` at
262
+ or below an order's quantity, because it is pricing a LINE that has one. A catalogue column
263
+ has no quantity in hand, so it takes the LOWEST `min_quantity` β€” the price at qty 1. Rules
264
+ above that break are a bulk price, and `report["qty_break_only"]` counts the SKUs whose only
265
+ rule sits on one (MEASURED: 10 rules of 10,464 carry a break at all).
266
+
267
+ **Variant rules beat template rules**, matching `pricecomp._tier_for`: a `0_product_variant`
268
+ rule is the more specific statement about this exact SKU.
269
+
270
+ ⚠ DEGRADES TO `({}, report)` on a read failure, matching `_inventory_by_code` rather than
271
+ `catalogue()`: these are COLUMNS, and a product grid that will not render because pricing is
272
+ momentarily unreachable is a worse failure than one with blank price columns. The blanks are
273
+ not silent β€” `product_data.validate()`'s coverage leg reconciles each column against a fresh
274
+ Odoo count and goes red at zero.
275
+
276
+ ⭐ **The report exists because R6's second sentence is law** (*"if it can't be done, you need
277
+ to explicitly tell me why and recommend a fix"*). Everything this reader CANNOT see is
278
+ counted rather than dropped: rules on pricelists the contract does not declare, non-fixed
279
+ (`formula`/`percent`) rules, the `3_global` fallback, and prices that only exist above a
280
+ quantity break.
281
+ """
282
+ # ⚠ THE COUNTERS ARE DISJOINT AND ORDER-DEPENDENT, and saying so is the difference between a
283
+ # report and a misleading one. A rule is classified ONCE, by the first reason it is skipped:
284
+ # undeclared list β†’ out of date β†’ not fixed β†’ global β†’ zero price. So `rules_not_fixed: 0`
285
+ # means "no formula rule on a list we declare", NOT "this Odoo has no formula rules"
286
+ # (MEASURED 2026-08-12: it has exactly one, and it sits on Public Pricelist, which the
287
+ # contract does not declare β€” so it lands in `rules_undeclared_list`).
288
+ report = {"lists_missing": [], "rules_total": 0, "rules_undeclared_list": 0,
289
+ "rules_not_fixed": 0, "rules_global": 0, "rules_out_of_date": 0,
290
+ "rules_zero_price": 0, "qty_break_only": 0}
291
+ try:
292
+ pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
293
+ by_name = {}
294
+ for p in pl_rows:
295
+ by_name.setdefault(str(p.get('name') or '').strip(), p['id'])
296
+ wanted = {}
297
+ for col, name in PRICELIST_COLUMNS:
298
+ pid = by_name.get(name)
299
+ if pid is None:
300
+ report["lists_missing"].append(name)
301
+ else:
302
+ wanted[pid] = col
303
+
304
+ # ⭐⭐ FILTERED SERVER-SIDE, AND THE REASON IS A MEASUREMENT, NOT A STYLE PREFERENCE.
305
+ # Reading all 10,464 rules with 9 fields and sorting them in Python costs **30.1s** on
306
+ # this connection; the same rules under a server-side domain with 6 fields cost
307
+ # **13.6s** (measured 2026-08-12, back to back). `pool()` runs this on a scope's
308
+ # first-ever build, so that 16.5s is 16.5s of somebody's page load.
309
+ #
310
+ # ⚠ The domain reproduces the Python predicate EXACTLY, and the null legs are the part
311
+ # that is easy to get wrong: an absent `date_start` is `False`, not a past date, so
312
+ # `('date_start','<=',today)` ALONE would drop every open-ended rule β€” which is almost
313
+ # all of them.
314
+ today = P.today().isoformat()
315
+ _declared = sorted(wanted)
316
+ _kinds = ['0_product_variant', '1_product']
317
+ _live = [('pricelist_id', 'in', _declared), ('compute_price', '=', 'fixed'),
318
+ ('applied_on', 'in', _kinds),
319
+ '|', ('date_start', '=', False), ('date_start', '<=', today),
320
+ '|', ('date_end', '=', False), ('date_end', '>=', today),
321
+ ('fixed_price', '>', 0)]
322
+ rules = O.search_read(
323
+ 'product.pricelist.item', _live,
324
+ ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
325
+ 'min_quantity']) if _declared else []
326
+
327
+ # ⭐ R6's SECOND SENTENCE, PAID FOR WITH `search_count` RATHER THAN A WIDER READ. Six
328
+ # counts cost ~1.2s together; the rules they count would cost 16s to read. What this
329
+ # reader cannot see is still REPORTED β€” it is just no longer transferred.
330
+ def _n(extra):
331
+ try:
332
+ return O.get_odoo().search_count('product.pricelist.item', extra)
333
+ except Exception:
334
+ return -1 # -1 reads as "not measured", never as zero
335
+ _dated = ['|', ('date_start', '=', False), ('date_start', '<=', today),
336
+ '|', ('date_end', '=', False), ('date_end', '>=', today)]
337
+ report["rules_total"] = _n([])
338
+ report["rules_undeclared_list"] = _n([('pricelist_id', 'not in', _declared)]) \
339
+ if _declared else report["rules_total"]
340
+ if _declared:
341
+ _on = [('pricelist_id', 'in', _declared)]
342
+ # `percent_price` and formula rules are read NOWHERE in this repo. A rule we cannot
343
+ # price is one the operator is TOLD about, never one that quietly becomes a blank.
344
+ report["rules_not_fixed"] = _n(_on + [('compute_price', '!=', 'fixed')])
345
+ report["rules_global"] = _n(_on + [('compute_price', '=', 'fixed'),
346
+ ('applied_on', 'not in', _kinds)])
347
+ report["rules_out_of_date"] = (
348
+ _n(_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)])
349
+ - _n(_on + [('compute_price', '=', 'fixed'),
350
+ ('applied_on', 'in', _kinds)] + _dated))
351
+ # A zero price is not "free" β€” it is an unset rule. Blank says so; 0 does not.
352
+ report["rules_zero_price"] = _n(
353
+ _on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)]
354
+ + _dated + [('fixed_price', '<=', 0)])
355
+
356
+ by_var, by_tmpl = {}, {}
357
+ for r in rules:
358
+ col = wanted[O.m2o_id(r.get('pricelist_id'))]
359
+ if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
360
+ by_var.setdefault((col, O.m2o_id(r['product_id'])), []).append(r)
361
+ elif r.get('product_tmpl_id'):
362
+ by_tmpl.setdefault((col, O.m2o_id(r['product_tmpl_id'])), []).append(r)
363
+
364
+ dom = [('active', '=', True)]
365
+ prods = O.search_read('product.product', dom,
366
+ ['id', 'default_code', 'product_tmpl_id'], limit=50000)
367
+ n = O.get_odoo().search_count('product.product', dom)
368
+ if len(prods) != n:
369
+ # Same guard, same reason as `catalogue()`: a short pull renders as a plausible
370
+ # smaller set of priced SKUs with nothing reporting it.
371
+ raise ValueError(
372
+ f"products.pricelist_by_code: the product pull is TRUNCATED β€” read {len(prods)} "
373
+ f"rows against a search_count of {n}.")
374
+ except Exception as e:
375
+ report["error"] = f"{type(e).__name__}: {str(e)[:200]}"
376
+ return {}, report
377
+
378
+ out = {}
379
+ for p in prods:
380
+ code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
381
+ tmpl = O.m2o_id(p.get('product_tmpl_id'))
382
+ cells = {}
383
+ for col, _name in PRICELIST_COLUMNS:
384
+ cands = by_var.get((col, p['id'])) or by_tmpl.get((col, tmpl))
385
+ if not cands:
386
+ continue
387
+ base = min(cands, key=lambda r: r.get('min_quantity') or 0.0)
388
+ if (base.get('min_quantity') or 0.0) > 1.0:
389
+ report["qty_break_only"] += 1
390
+ cells[col] = base.get('fixed_price')
391
+ if cells:
392
+ out.setdefault(code, {}).update(cells)
393
+ return out, report
394
+
395
+
396
+ def catalogue_count():
397
+ """The INDEPENDENT population oracle: Odoo's own count of active products.
398
+
399
+ Deliberately a bare `search_count` and not a `len()` over anything this module built β€” the
400
+ 2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides
401
+ of its reconciliation from `_sku_rev`, so the oracle could never see a missing row.
402
+ """
403
+ return O.get_odoo().search_count('product.product', [('active', '=', True)])
404
+
405
+
406
+ def categories(t=None, team_id=None):
407
+ """Sorted distinct category names present in the SKU directory (for the drill-down filter)."""
408
+ return sorted({r['category'] for r in directory(t, team_id)})
409
+
410
+
411
+ def _coverage(date_from, date_to, team_id=None):
412
+ """{sku_code: distinct_customer_count} via 2-level read_group (slow-ish), merged by code.
413
+ (A customer buying two records sharing a code can count twice; duplicates are rare, and
414
+ code-level avoids the bigger error of a re-SKUed product reading as full coverage loss.)"""
415
+ g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
416
+ ['__count'], ['product_id', 'order_partner_id'], lazy=False)
417
+ codes = _code_map()
418
+ cov = {}
419
+ names = {}
420
+ for r in g:
421
+ pid = O.m2o_id(r.get('product_id'))
422
+ if not pid:
423
+ continue
424
+ key = codes.get(pid, f"pid:{pid}")
425
+ cov[key] = cov.get(key, 0) + 1
426
+ names.setdefault(key, O.m2o_name(r.get('product_id')))
427
+ return cov, names
428
+
429
+
430
+ def coverage_collapse(t=None, limit=25, min_prior_custs=8, team_id=None):
431
+ """SKUs that lost the most customer breadth YoY β€” early warning a SKU is dying even if
432
+ revenue hasn't fully cratered yet."""
433
+ t = t or P.today()
434
+ yf, yt = P.ytd(t)
435
+ lf, lt = P.ytd_last_year(t)
436
+ cov_t, names_t = _coverage(yf, yt, team_id)
437
+ cov_l, names_l = _coverage(lf, lt, team_id)
438
+ rows = []
439
+ for pid, lc in cov_l.items():
440
+ if lc < min_prior_custs:
441
+ continue
442
+ tc = cov_t.get(pid, 0)
443
+ drop = lc - tc
444
+ if drop <= 0:
445
+ continue
446
+ rows.append({'code': pid, 'product': names_l.get(pid) or names_t.get(pid) or '',
447
+ 'custs_ly': lc, 'custs_ytd': tc, 'lost_custs': drop,
448
+ 'pct_drop': drop / lc * 100})
449
+ rows.sort(key=lambda x: (-x['lost_custs'], -x['pct_drop']))
450
+ return rows[:limit]
451
+
452
+
453
+ # ====================================================================== SKU DRAWER (mirror of customer drawer)
454
+ def _sku_product_ids(code):
455
+ """All product.product ids sharing this SKU code (merged variants / archived records)."""
456
+ return [pid for pid, c in _code_map().items() if c == code]
457
+
458
+
459
+ def _sku_dom(pids, date_from, date_to, team_id=None):
460
+ return O.sale_line_domain(date_from, date_to, team_id, extra=[('product_id', 'in', pids)] + _NO_SVC)
461
+
462
+
463
+ def _sku_name_category(pids):
464
+ rows = O.search_read('product.product', [('id', 'in', pids)], ['name', 'categ_id'])
465
+ name = rows[0]['name'] if rows else '(unknown)'
466
+ catmap = sales_mod._product_cat()
467
+ main = next((catmap.get(p) for p in pids if catmap.get(p)), '(uncategorized)')
468
+ return name, main
469
+
470
+
471
+ def _sku_buyers(pids, date_from, date_to, team_id=None):
472
+ """{partner_id: {'name','rev','qty'}} for buyers of this SKU over a window."""
473
+ g = O.read_group('sale.order.line', _sku_dom(pids, date_from, date_to, team_id),
474
+ ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
475
+ out = {}
476
+ for r in g:
477
+ pid = O.m2o_id(r.get('order_partner_id'))
478
+ if pid:
479
+ out[pid] = {'name': O.m2o_name(r.get('order_partner_id')),
480
+ 'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0}
481
+ return out
482
+
483
+
484
+ def sku_detail(code, t=None, team_id=None, n_months=13, allsku=None):
485
+ """KPIs (rev/qty/buyers YoY, GM%), monthly trend, rank & % of BU for one SKU code.
486
+ Pass `allsku` (a cached _sku_rev YTD map) to skip the ~all-SKU rank read."""
487
+ t = t or P.today()
488
+ pids = _sku_product_ids(code)
489
+ if not pids:
490
+ return None
491
+ yf, yt = P.ytd(t)
492
+ lf, lt = P.ytd_last_year(t)
493
+ mf, mt = P.ltm(t)
494
+ name, category = _sku_name_category(pids)
495
+
496
+ def s(df, dtt, field='price_subtotal'):
497
+ return O.sum_field('sale.order.line', _sku_dom(pids, df, dtt, team_id), field)
498
+ rev_ytd, rev_ly = s(yf, yt), s(lf, lt)
499
+ qty_ytd, qty_ly = s(yf, yt, 'product_uom_qty'), s(lf, lt, 'product_uom_qty')
500
+ buyers_ytd = len(_sku_buyers(pids, yf, yt, team_id))
501
+ buyers_ly = len(_sku_buyers(pids, lf, lt, team_id))
502
+ g = O.read_group('sale.order.line', _sku_dom(pids, mf, mt, team_id),
503
+ ['price_subtotal:sum', 'margin:sum'], [], lazy=False)
504
+ line_rev = (g[0].get('price_subtotal') if g else 0) or 0.0
505
+ margin = (g[0].get('margin') if g else 0) or 0.0
506
+
507
+ # Monthly trend: bin lines by their order's month in Python (dot-path month groupby is rejected
508
+ # on sale.order.line), in 2 reads instead of 26 point queries.
509
+ range_start = dt.date(t.year - 2, t.month, 1).isoformat()
510
+ lines = O.search_read('sale.order.line', _sku_dom(pids, range_start, t.isoformat(), team_id),
511
+ ['price_subtotal', 'order_id'])
512
+ oids = list({O.m2o_id(line['order_id']) for line in lines if line.get('order_id')})
513
+ odate = {}
514
+ for i in range(0, len(oids), 1000):
515
+ for o in O.search_read('sale.order', [('id', 'in', oids[i:i + 1000])], ['date_order']):
516
+ if o.get('date_order'):
517
+ odate[o['id']] = str(o['date_order'])[:7]
518
+ mrev = {}
519
+ for line in lines:
520
+ ym = odate.get(O.m2o_id(line.get('order_id')))
521
+ if ym:
522
+ mrev[ym] = mrev.get(ym, 0.0) + (line.get('price_subtotal') or 0.0)
523
+ monthly = []
524
+ for ym, start, end in P.month_starts(n_months, t):
525
+ y, m = int(ym[:4]) - 1, int(ym[5:7])
526
+ monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0),
527
+ 'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)})
528
+
529
+ allsku = allsku if allsku is not None else _sku_rev(yf, yt, team_id)
530
+ total = sum(v['rev'] for v in allsku.values()) or 1.0
531
+ rank = next((i + 1 for i, (c, _v) in enumerate(sorted(allsku.items(), key=lambda kv: -kv[1]['rev']))
532
+ if c == code), None)
533
+ return {
534
+ 'code': code, 'name': name, 'category': category,
535
+ 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'rev_yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
536
+ 'qty_ytd': qty_ytd, 'qty_ly': qty_ly, 'qty_yoy_pct': P.yoy_pct(qty_ytd, qty_ly),
537
+ 'buyers_ytd': buyers_ytd, 'buyers_ly': buyers_ly, 'buyers_delta': buyers_ytd - buyers_ly,
538
+ 'gm_pct': (margin / line_rev * 100) if line_rev else 0.0, 'gm_dollars': margin,
539
+ 'rank': rank, 'n_skus': len(allsku), 'pct_of_bu': rev_ytd / total * 100,
540
+ 'monthly': monthly,
541
+ }
542
+
543
+
544
+ def sku_buyer_bridge(code, t=None, team_id=None, top=12):
545
+ """Who drives the SKU's YoY: retained / new / churned buyers + a $-ranked churned call list."""
546
+ t = t or P.today()
547
+ pids = _sku_product_ids(code)
548
+ yf, yt = P.ytd(t)
549
+ lf, lt = P.ytd_last_year(t)
550
+ this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
551
+ tset, lset = set(this), set(last)
552
+ retained, new, churned = tset & lset, tset - lset, lset - tset
553
+ churned_list = sorted([{'pid': p, 'customer': last[p]['name'], 'ly_rev': last[p]['rev']}
554
+ for p in churned], key=lambda x: -x['ly_rev'])[:top]
555
+ return {'retained': {'n': len(retained), 'rev': sum(this[p]['rev'] for p in retained)},
556
+ 'new': {'n': len(new), 'rev': sum(this[p]['rev'] for p in new)},
557
+ 'churned': {'n': len(churned), 'rev': sum(last[p]['rev'] for p in churned)},
558
+ 'buyer_retention_pct': (len(retained) / len(lset) * 100) if lset else 0.0,
559
+ 'churned_buyers': churned_list, 'buyers_this': len(tset), 'buyers_last': len(lset),
560
+ 'this_total': sum(v['rev'] for v in this.values())}
561
+
562
+
563
+ def sku_concentration(code, t=None, team_id=None):
564
+ """Buyer-concentration risk: top-1/top-3 share, Herfindahl index, effective buyer count (LTM)."""
565
+ t = t or P.today()
566
+ mf, mt = P.ltm(t)
567
+ buyers = _sku_buyers(_sku_product_ids(code), mf, mt, team_id)
568
+ revs = sorted([v['rev'] for v in buyers.values()], reverse=True)
569
+ total = sum(revs) or 1.0
570
+ hhi = sum((r / total) ** 2 for r in revs)
571
+ return {'n_buyers': len(revs), 'top1_pct': (revs[0] / total * 100) if revs else 0.0,
572
+ 'top3_pct': (sum(revs[:3]) / total * 100) if revs else 0.0,
573
+ 'hhi': hhi, 'eff_buyers': (1 / hhi) if hhi else 0.0}
574
+
575
+
576
+ def sku_price_dispersion(code, t=None, team_id=None, cap=60):
577
+ """Realized $/unit per buyer (LTM) vs the volume-weighted average; recoverable $ on below-VWAP
578
+ accounts. list_price is unreliable, so the dispersion among actual buyers is the margin lever."""
579
+ t = t or P.today()
580
+ mf, mt = P.ltm(t)
581
+ g = O.read_group('sale.order.line', _sku_dom(_sku_product_ids(code), mf, mt, team_id),
582
+ ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
583
+ rows, tot_rev, tot_qty = [], 0.0, 0.0
584
+ for r in g:
585
+ pid = O.m2o_id(r.get('order_partner_id'))
586
+ rev = r.get('price_subtotal') or 0.0
587
+ qty = r.get('product_uom_qty') or 0.0
588
+ if not pid or qty <= 0:
589
+ continue
590
+ rows.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')),
591
+ 'price': rev / qty, 'qty': qty, 'rev': rev})
592
+ tot_rev += rev
593
+ tot_qty += qty
594
+ vwap = (tot_rev / tot_qty) if tot_qty else 0.0
595
+ for r in rows:
596
+ r['recoverable'] = max(0.0, vwap - r['price']) * r['qty']
597
+ rows.sort(key=lambda x: -x['recoverable'])
598
+ return {'vwap': vwap, 'n_buyers': len(rows),
599
+ 'recoverable_total': sum(r['recoverable'] for r in rows), 'rows': rows[:cap]}
600
+
601
+
602
+ def sku_top_buyers(code, t=None, team_id=None, top=15):
603
+ """Ranked buyers of this SKU (YTD) with YoY β€” clickable to open the customer drawer."""
604
+ t = t or P.today()
605
+ pids = _sku_product_ids(code)
606
+ yf, yt = P.ytd(t)
607
+ lf, lt = P.ytd_last_year(t)
608
+ this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
609
+ rows = [{'pid': p, 'customer': v['name'], 'rev': v['rev'], 'qty': v['qty'],
610
+ 'yoy_pct': P.yoy_pct(v['rev'], last.get(p, {}).get('rev', 0.0))} for p, v in this.items()]
611
+ rows.sort(key=lambda x: -x['rev'])
612
+ return rows[:top]
613
+
614
+
615
+ _BUYER_TIERS = [('Whale (β‰₯$25k)', 25000.0), ('Large ($10–25k)', 10000.0),
616
+ ('Mid ($2–10k)', 2000.0), ('Small (<$2k)', 0.0)]
617
+
618
+
619
+ def sku_customer_analysis(code, t=None, team_id=None, top=15):
620
+ """WHO buys this SKU (LTM), as customers: the value-tier mix of its buyers (by each buyer's TOTAL
621
+ spend), and the accounts most DEPENDENT on it (this SKU as a share of their spend β€” who gets hurt
622
+ most if it stocks out / who to protect)."""
623
+ t = t or P.today()
624
+ lf, lt = P.ltm(t)
625
+ pids = _sku_product_ids(code)
626
+ empty = {'segments': [], 'dependency': [], 'n_buyers': 0, 'avg_dependency': 0.0}
627
+ if not pids:
628
+ return empty
629
+ buyers = _sku_buyers(pids, lf, lt, team_id) # {pid: {name, rev(this SKU), qty}}
630
+ if not buyers:
631
+ return empty
632
+ bpids = list(buyers)
633
+ g = O.read_group('sale.order', sales_mod.order_domain(lf, lt, team_id) + [('partner_id', 'in', bpids)],
634
+ ['amount_untaxed:sum'], ['partner_id'], lazy=False)
635
+ total = {O.m2o_id(r['partner_id']): (r.get('amount_untaxed') or 0.0) for r in g if r.get('partner_id')}
636
+
637
+ def tier(rev):
638
+ for nm, lo in _BUYER_TIERS:
639
+ if rev >= lo:
640
+ return nm
641
+ return _BUYER_TIERS[-1][0]
642
+ seg = {nm: {'tier': nm, 'buyers': 0, 'sku_rev': 0.0} for nm, _ in _BUYER_TIERS}
643
+ dep = []
644
+ for p, v in buyers.items():
645
+ ct = total.get(p, v['rev']) or v['rev']
646
+ s = seg[tier(ct)]
647
+ s['buyers'] += 1
648
+ s['sku_rev'] += v['rev']
649
+ dep.append({'customer': v['name'], 'pid': p, 'sku_rev': v['rev'], 'cust_total': ct,
650
+ 'dependency_pct': (v['rev'] / ct * 100) if ct else None})
651
+ tot = sum(s['sku_rev'] for s in seg.values()) or 1.0
652
+ segments = []
653
+ for nm, _ in _BUYER_TIERS:
654
+ s = seg[nm]
655
+ s['rev_share'] = s['sku_rev'] / tot * 100
656
+ s['avg_per_buyer'] = (s['sku_rev'] / s['buyers']) if s['buyers'] else 0.0
657
+ segments.append(s)
658
+ dep.sort(key=lambda x: -x['sku_rev'])
659
+ deps_known = [d['dependency_pct'] for d in dep if d['dependency_pct'] is not None]
660
+ return {'segments': segments, 'dependency': dep[:top], 'n_buyers': len(buyers),
661
+ 'avg_dependency': (sum(deps_known) / len(deps_known)) if deps_known else 0.0}
662
+
663
+
664
+ def sku_whitespace(code, t=None, team_id=None, top=15):
665
+ """Customers who buy this SKU's category but NOT this SKU β€” ranked prospect list."""
666
+ import modules.customers as cust_mod
667
+ t = t or P.today()
668
+ pids = _sku_product_ids(code)
669
+ _name, category = _sku_name_category(pids)
670
+ cat_buyers = cust_mod.category_buyers(category, team_id=team_id) or set()
671
+ mf, mt = P.ltm(t)
672
+ prospects = list(cat_buyers - set(_sku_buyers(pids, mf, mt, team_id)))
673
+ if not prospects:
674
+ return {'category': category, 'rows': []}
675
+ g = O.read_group('sale.order', sales_mod.order_domain(mf, mt, team_id) + [('partner_id', 'in', prospects)],
676
+ ['amount_untaxed:sum'], ['partner_id'], lazy=False)
677
+ rows = [{'pid': O.m2o_id(r['partner_id']), 'customer': O.m2o_name(r['partner_id']),
678
+ 'total_spend': r.get('amount_untaxed') or 0.0} for r in g if r.get('partner_id')]
679
+ rows.sort(key=lambda x: -x['total_spend'])
680
+ return {'category': category, 'rows': rows[:top]}
681
+
682
+
683
+ def sku_drawer_bundle(code, t=None, team_id=None, allsku=None):
684
+ """The whole SKU drawer's first paint in one cached unit: sku_detail first (the not-found gate),
685
+ then the other six pulls CONCURRENTLY (O.parallel). Cuts a ~6-call cold open to ~max(call)."""
686
+ detail = sku_detail(code, t=t, team_id=team_id, allsku=allsku)
687
+ if detail is None:
688
+ return {'detail': None}
689
+ bridge, conc, price, buyers, white, ca = O.parallel([
690
+ lambda: sku_buyer_bridge(code, t, team_id),
691
+ lambda: sku_concentration(code, t, team_id),
692
+ lambda: sku_price_dispersion(code, t, team_id),
693
+ lambda: sku_top_buyers(code, t, team_id),
694
+ lambda: sku_whitespace(code, t, team_id),
695
+ lambda: sku_customer_analysis(code, t, team_id),
696
+ ])
697
+ return {'detail': detail, 'bridge': bridge, 'conc': conc, 'price': price,
698
+ 'buyers': buyers, 'white': white, 'ca': ca}
699
+
700
+
701
+ def validate(t=None, team_id=None):
702
+ """Reconcile SKU metrics to independent Odoo aggregates. When team_id is set (a single BU
703
+ selected) every check runs SCOPED to that BU, so the validation panel never reconciles
704
+ against β€” or exposes β€” the other BU's numbers. All checks here are BU-scopeable (no cross-BU
705
+ mirror), so team_id threads straight through."""
706
+ t = t or P.today()
707
+ yf, yt = P.ytd(t)
708
+ checks = []
709
+ sku = _sku_rev(yf, yt, team_id)
710
+ sku_sum = sum(v['rev'] for v in sku.values())
711
+ line_total = O.sum_field('sale.order.line',
712
+ O.sale_line_domain(yf, yt, team_id, extra=_NO_SVC), 'price_subtotal')
713
+ checks.append({'check': 'SKU rev: Ξ£(per-SKU) == total line revenue, ex-services (YTD)',
714
+ 'a': round(sku_sum, 2), 'b': round(line_total, 2),
715
+ 'gap': round(sku_sum - line_total, 2),
716
+ 'ok': abs(sku_sum - line_total) <= 1.0})
717
+
718
+ m = yoy_movers(t, limit=10**9, team_id=team_id)
719
+ movers_sum = sum(r['change'] for r in m['risers']) + sum(r['change'] for r in m['decliners'])
720
+ lf, lt = P.ytd_last_year(t)
721
+ last_total = O.sum_field('sale.order.line',
722
+ O.sale_line_domain(lf, lt, team_id, extra=_NO_SVC), 'price_subtotal')
723
+ checks.append({'check': 'SKU movers: Ξ£(Ξ”) == (YTD βˆ’ LY) total',
724
+ 'a': round(movers_sum, 2), 'b': round(line_total - last_total, 2),
725
+ 'gap': round(movers_sum - (line_total - last_total), 2),
726
+ 'ok': abs(movers_sum - (line_total - last_total)) <= 1.0})
727
+
728
+ # SKU drawer: buyer-bridge reconciles β€” retained + new buyer revenue == this-period SKU revenue
729
+ top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None
730
+ if top_code:
731
+ bb = sku_buyer_bridge(top_code, t, team_id=team_id)
732
+ recon = bb['retained']['rev'] + bb['new']['rev']
733
+ checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue',
734
+ 'a': round(recon, 2), 'b': round(bb['this_total'], 2),
735
+ 'gap': round(recon - bb['this_total'], 2),
736
+ 'ok': abs(recon - bb['this_total']) <= 1.0})
737
+ return checks
web/public/sample_customers.json CHANGED
@@ -1,558 +1,558 @@
1
- {
2
- "fields": [
3
- {
4
- "key": "customer",
5
- "label": "Customer",
6
- "type": "text",
7
- "source": "odoo",
8
- "pinned": true,
9
- "default": true,
10
- "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
11
- },
12
- {
13
- "key": "partner_id",
14
- "label": "Odoo ID",
15
- "type": "int",
16
- "source": "odoo",
17
- "derived": true,
18
- "default": false,
19
- "description": "The Odoo res.partner id β€” the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source."
20
- },
21
- {
22
- "key": "odoo_status",
23
- "label": "Odoo record",
24
- "type": "status",
25
- "source": "odoo",
26
- "default": false,
27
- "options": [
28
- "Active",
29
- "Archived"
30
- ],
31
- "description": "Whether this customer still exists in Odoo. Archived means deleted there."
32
- },
33
- {
34
- "key": "agent",
35
- "label": "Agent",
36
- "type": "text",
37
- "source": "odoo",
38
- "default": true,
39
- "description": "The sales agent who owns this account."
40
- },
41
- {
42
- "key": "dba",
43
- "label": "DBA",
44
- "type": "select",
45
- "source": "odoo",
46
- "default": false,
47
- "options": [
48
- "Fisch",
49
- "Royal",
50
- "Both"
51
- ],
52
- "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
53
- },
54
- {
55
- "key": "salesperson",
56
- "label": "Salesperson",
57
- "type": "text",
58
- "source": "odoo",
59
- "default": false,
60
- "description": "Who keyed in most of this customer's orders β€” not the Agent, who owns the account."
61
- },
62
- {
63
- "key": "street",
64
- "label": "Street",
65
- "type": "text",
66
- "source": "odoo",
67
- "default": false,
68
- "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
69
- },
70
- {
71
- "key": "street2",
72
- "label": "Street 2",
73
- "type": "text",
74
- "source": "odoo",
75
- "default": false,
76
- "description": "Second address line (suite, unit, floor) on the customer's Odoo address."
77
- },
78
- {
79
- "key": "city",
80
- "label": "City",
81
- "type": "text",
82
- "source": "odoo",
83
- "default": true,
84
- "description": "City on the customer's Odoo address."
85
- },
86
- {
87
- "key": "state",
88
- "label": "State",
89
- "type": "text",
90
- "source": "odoo",
91
- "default": true,
92
- "description": "State or province on the customer's Odoo address."
93
- },
94
- {
95
- "key": "country",
96
- "label": "Country",
97
- "type": "text",
98
- "source": "odoo",
99
- "default": false,
100
- "description": "Country on the customer's Odoo address."
101
- },
102
- {
103
- "key": "zip",
104
- "label": "ZIP",
105
- "type": "text",
106
- "source": "odoo",
107
- "default": false,
108
- "description": "Postal code on the customer's Odoo address."
109
- },
110
- {
111
- "key": "customer_since",
112
- "label": "Customer since",
113
- "type": "date",
114
- "source": "odoo",
115
- "default": false,
116
- "description": "When this customer was first set up in Odoo."
117
- },
118
- {
119
- "key": "tags",
120
- "label": "Tags",
121
- "type": "text",
122
- "source": "odoo",
123
- "default": false,
124
- "description": "Odoo labels on this customer, comma-separated."
125
- },
126
- {
127
- "key": "pricelist",
128
- "label": "Price list",
129
- "type": "text",
130
- "source": "odoo",
131
- "default": false,
132
- "description": "The price list this customer buys on."
133
- },
134
- {
135
- "key": "payment_terms",
136
- "label": "Payment terms",
137
- "type": "text",
138
- "source": "odoo",
139
- "default": false,
140
- "description": "Payment terms on this customer's account β€” Net 30, for example."
141
- },
142
- {
143
- "key": "last_order",
144
- "label": "Last order",
145
- "type": "date",
146
- "source": "odoo",
147
- "default": true,
148
- "description": "Date of the most recent confirmed order."
149
- },
150
- {
151
- "key": "overdue_days",
152
- "label": "Overdue days",
153
- "type": "int",
154
- "source": "odoo",
155
- "default": true,
156
- "description": "How many days late this customer is running against their own usual ordering rhythm."
157
- },
158
- {
159
- "_note": "filterable:false β€” DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule β€” see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
160
- "key": "est_missed",
161
- "label": "Est. missed $",
162
- "type": "currency",
163
- "source": "odoo",
164
- "default": true,
165
- "agg": "sum",
166
- "filterable": false,
167
- "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
168
- },
169
- {
170
- "_note": "wave 21 R1 β€” KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary β€” 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
171
- "key": "ar_open",
172
- "label": "AR current $",
173
- "type": "currency",
174
- "source": "odoo",
175
- "default": false,
176
- "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
177
- },
178
- {
179
- "key": "ar_overdue",
180
- "label": "AR overdue $",
181
- "type": "currency",
182
- "source": "odoo",
183
- "default": false,
184
- "description": "Invoiced money past due β€” same basis as the Collections page."
185
- },
186
- {
187
- "_note": "wave 21 R1 β€” the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie β€” no second oracle.",
188
- "key": "ar_outstanding",
189
- "label": "AR outstanding $",
190
- "type": "currency",
191
- "source": "odoo",
192
- "default": false,
193
- "description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
194
- },
195
- {
196
- "key": "ar_exposure",
197
- "label": "Credit exposure $",
198
- "type": "currency",
199
- "source": "odoo",
200
- "default": false,
201
- "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
202
- },
203
- {
204
- "key": "ar_aged_1_30",
205
- "label": "1-30 days $",
206
- "type": "currency",
207
- "source": "odoo",
208
- "default": false,
209
- "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
210
- },
211
- {
212
- "key": "ar_aged_31_60",
213
- "label": "31-60 days $",
214
- "type": "currency",
215
- "source": "odoo",
216
- "default": false,
217
- "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
218
- },
219
- {
220
- "key": "ar_aged_61_90",
221
- "label": "61-90 days $",
222
- "type": "currency",
223
- "source": "odoo",
224
- "default": false,
225
- "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
226
- },
227
- {
228
- "key": "ar_aged_90_plus",
229
- "label": "90+ days $",
230
- "type": "currency",
231
- "source": "odoo",
232
- "default": false,
233
- "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
234
- },
235
- {
236
- "key": "days_to_pay",
237
- "label": "Days to pay",
238
- "type": "int",
239
- "source": "odoo",
240
- "default": false,
241
- "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
242
- },
243
- {
244
- "key": "top_category",
245
- "label": "Top category",
246
- "type": "text",
247
- "source": "odoo",
248
- "default": false,
249
- "description": "The category this customer spent the most on in the last 12 months."
250
- },
251
- {
252
- "key": "top_category_pct",
253
- "label": "Top category %",
254
- "type": "pct",
255
- "source": "odoo",
256
- "default": false,
257
- "description": "Share of last-12-months spend that went to the top category."
258
- },
259
- {
260
- "key": "sku_count",
261
- "label": "SKUs bought",
262
- "type": "int",
263
- "source": "odoo",
264
- "default": false,
265
- "description": "Distinct products bought in the last 12 months."
266
- },
267
- {
268
- "key": "top_sku",
269
- "label": "Top SKU",
270
- "type": "text",
271
- "source": "odoo",
272
- "default": false,
273
- "description": "The product this customer spent the most on in the last 12 months."
274
- },
275
- {
276
- "key": "days_since",
277
- "label": "Days since order",
278
- "type": "int",
279
- "source": "odoo",
280
- "default": false,
281
- "description": "Days since the last confirmed order."
282
- },
283
- {
284
- "key": "typical_gap_days",
285
- "label": "Typical gap days",
286
- "type": "int",
287
- "source": "odoo",
288
- "default": false,
289
- "description": "Days this customer usually goes between orders, from their own history."
290
- },
291
- {
292
- "key": "notes",
293
- "label": "Notes",
294
- "type": "text",
295
- "source": "overlay",
296
- "default": false,
297
- "description": "Your notes on this customer. Saved in this app only, visible only to you."
298
- }
299
- ],
300
- "rows": [
301
- {
302
- "pid": 101,
303
- "customer": "Poppy Flowers",
304
- "status": "New",
305
- "agent": "Naomi Linnell Rivera",
306
- "city": "Charlottesville",
307
- "state": "Virginia (US)",
308
- "last_order": "2026-07-21",
309
- "est_missed": 0,
310
- "notes": "",
311
- "country": "United States",
312
- "zip": "02720",
313
- "payment_terms": "30 Days",
314
- "pricelist": "Fisch 1 (USD)",
315
- "tags": "Royal",
316
- "customer_since": "2023-01-10",
317
- "salesperson": "Jessica",
318
- "ar_open": 0,
319
- "ar_overdue": 0,
320
- "ar_exposure": 0,
321
- "top_category": "Styrofoam",
322
- "top_category_pct": 0.47,
323
- "sku_count": 93,
324
- "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
325
- "days_to_pay": 34,
326
- "_created": "2023-01-15 09:10:00",
327
- "lat": 25.7617,
328
- "lon": -80.1918,
329
- "odoo_status": "Archived",
330
- "dba": "Royal",
331
- "ar_outstanding": 0
332
- },
333
- {
334
- "pid": 102,
335
- "customer": "Meadow & Vine Wholesale",
336
- "status": "Growing",
337
- "agent": "Carla Jimenez",
338
- "city": "Portland",
339
- "state": "Oregon (US)",
340
- "last_order": "2026-07-19",
341
- "est_missed": 0,
342
- "notes": "Expanding to a second storefront.",
343
- "country": "United States",
344
- "zip": "77041",
345
- "payment_terms": "Immediate Payment",
346
- "pricelist": "Royal 1 (USD)",
347
- "tags": "Royal, Key account",
348
- "customer_since": "2024-02-11",
349
- "salesperson": "Naomi",
350
- "ar_open": 1240.5,
351
- "ar_overdue": 0,
352
- "ar_exposure": 1740.5,
353
- "top_category": "Ribbon",
354
- "top_category_pct": 0.95,
355
- "sku_count": 4,
356
- "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
357
- "days_to_pay": null,
358
- "_created": "2023-02-15 09:11:00",
359
- "lat": 27.9506,
360
- "lon": -82.4572,
361
- "odoo_status": "Active",
362
- "dba": "Fisch",
363
- "ar_outstanding": 1240.5
364
- },
365
- {
366
- "pid": 103,
367
- "customer": "Bluestem Floral Supply",
368
- "status": "Growing",
369
- "agent": "Naomi Linnell Rivera",
370
- "city": "Kansas City",
371
- "state": "Missouri (US)",
372
- "last_order": "2026-07-17",
373
- "est_missed": 0,
374
- "notes": "",
375
- "country": "United States",
376
- "zip": "11219",
377
- "payment_terms": "60 Days",
378
- "pricelist": "Royal 1 (USD)",
379
- "tags": "Fisch",
380
- "customer_since": "2025-03-12",
381
- "salesperson": "Karen",
382
- "ar_open": 0,
383
- "ar_overdue": 5120.25,
384
- "ar_exposure": 5120.25,
385
- "top_category": "Foams & Finishes",
386
- "top_category_pct": 0.31,
387
- "sku_count": 27,
388
- "top_sku": "SATIN RIBBON 2IN",
389
- "days_to_pay": 61,
390
- "_created": "2023-03-15 09:12:00",
391
- "lat": 28.5384,
392
- "lon": -81.3789,
393
- "odoo_status": "Active",
394
- "dba": "Both",
395
- "ar_outstanding": 5120.25
396
- },
397
- {
398
- "pid": 104,
399
- "customer": "Camellia Row Florist",
400
- "status": "Declining",
401
- "agent": "Devon Marsh",
402
- "city": "Savannah",
403
- "state": "Georgia (US)",
404
- "last_order": "2026-05-30",
405
- "est_missed": 41200,
406
- "notes": "Switched some volume to a local grower.",
407
- "country": "United States",
408
- "zip": "07649",
409
- "payment_terms": "30 Days",
410
- "pricelist": "Fisch 1 (USD)",
411
- "tags": "(none)",
412
- "customer_since": "2026-04-13",
413
- "salesperson": "(none)",
414
- "ar_open": 8300,
415
- "ar_overdue": 940,
416
- "ar_exposure": 11440,
417
- "top_category": "All",
418
- "top_category_pct": 1.0,
419
- "sku_count": 1,
420
- "top_sku": "(none)",
421
- "days_to_pay": 12,
422
- "_created": "2023-04-15 09:13:00",
423
- "lat": 30.3322,
424
- "lon": -81.6557,
425
- "odoo_status": "Active",
426
- "dba": "Royal",
427
- "ar_outstanding": 9240
428
- },
429
- {
430
- "pid": 105,
431
- "customer": "Harborlight Wholesale Blooms",
432
- "status": "Growing",
433
- "agent": "Carla Jimenez",
434
- "city": "Seattle",
435
- "state": "Washington (US)",
436
- "last_order": "2026-07-22",
437
- "est_missed": 0,
438
- "notes": "Top-10 account.",
439
- "country": "United States",
440
- "zip": "33125",
441
- "payment_terms": "Immediate Payment",
442
- "pricelist": "Royal 1 (USD)",
443
- "tags": "Royal",
444
- "customer_since": "2023-05-14",
445
- "salesperson": "Jessica",
446
- "ar_open": 0,
447
- "ar_overdue": 0,
448
- "ar_exposure": 0,
449
- "top_category": "Styrofoam",
450
- "top_category_pct": 0.47,
451
- "sku_count": 93,
452
- "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
453
- "days_to_pay": 34,
454
- "_created": "2023-05-15 09:14:00",
455
- "lat": 26.1224,
456
- "lon": -80.1373,
457
- "odoo_status": "Active",
458
- "dba": "",
459
- "ar_outstanding": 0
460
- },
461
- {
462
- "pid": 106,
463
- "customer": "Dogwood & Fern Co.",
464
- "status": "Dormant",
465
- "agent": "Devon Marsh",
466
- "city": "Asheville",
467
- "state": "North Carolina (US)",
468
- "last_order": "2026-02-11",
469
- "est_missed": 52400,
470
- "notes": "No spring order this year.",
471
- "country": "United States",
472
- "zip": "90210",
473
- "payment_terms": "60 Days",
474
- "pricelist": "Royal 1 (USD)",
475
- "tags": "Royal, Key account",
476
- "customer_since": "2024-06-15",
477
- "salesperson": "Naomi",
478
- "ar_open": 1240.5,
479
- "ar_overdue": 0,
480
- "ar_exposure": 1740.5,
481
- "top_category": "Ribbon",
482
- "top_category_pct": 0.95,
483
- "sku_count": 4,
484
- "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
485
- "days_to_pay": null,
486
- "_created": "2023-06-15 09:15:00",
487
- "lat": 27.3364,
488
- "lon": -82.5307,
489
- "odoo_status": "Active",
490
- "dba": "Fisch",
491
- "ar_outstanding": 1240.5
492
- },
493
- {
494
- "pid": 107,
495
- "customer": "Verbena Market Florals",
496
- "status": "Lost",
497
- "agent": "Naomi Linnell Rivera",
498
- "city": "Austin",
499
- "state": "Texas (US)",
500
- "last_order": "2025-11-04",
501
- "est_missed": 78300,
502
- "notes": "Went with a competitor on freight terms.",
503
- "country": "United States",
504
- "zip": "08701",
505
- "payment_terms": "30 Days",
506
- "pricelist": "Fisch 1 (USD)",
507
- "tags": "Fisch",
508
- "customer_since": "2025-07-16",
509
- "salesperson": "Karen",
510
- "ar_open": 0,
511
- "ar_overdue": 5120.25,
512
- "ar_exposure": 5120.25,
513
- "top_category": "Foams & Finishes",
514
- "top_category_pct": 0.31,
515
- "sku_count": 27,
516
- "top_sku": "SATIN RIBBON 2IN",
517
- "days_to_pay": 61,
518
- "_created": "2023-07-15 09:16:00",
519
- "lat": null,
520
- "lon": null,
521
- "odoo_status": "Active",
522
- "dba": "Both",
523
- "ar_outstanding": 5120.25
524
- },
525
- {
526
- "pid": 108,
527
- "customer": "Larkspur Lane Supply",
528
- "status": "New",
529
- "agent": "Carla Jimenez",
530
- "city": "Denver",
531
- "state": "Colorado (US)",
532
- "last_order": "2026-07-14",
533
- "est_missed": 0,
534
- "notes": "First order in April.",
535
- "country": "United States",
536
- "zip": "60614",
537
- "payment_terms": "Immediate Payment",
538
- "pricelist": "Royal 1 (USD)",
539
- "tags": "(none)",
540
- "customer_since": "2026-08-17",
541
- "salesperson": "(none)",
542
- "ar_open": 8300,
543
- "ar_overdue": 940,
544
- "ar_exposure": 11440,
545
- "top_category": "All",
546
- "top_category_pct": 1.0,
547
- "sku_count": 1,
548
- "top_sku": "(none)",
549
- "days_to_pay": 12,
550
- "_created": "2023-08-15 09:17:00",
551
- "lat": 33.749,
552
- "lon": -84.388,
553
- "odoo_status": "Active",
554
- "dba": "Royal",
555
- "ar_outstanding": 9240
556
- }
557
- ]
558
- }
 
1
+ {
2
+ "fields": [
3
+ {
4
+ "key": "customer",
5
+ "label": "Customer",
6
+ "type": "text",
7
+ "source": "odoo",
8
+ "pinned": true,
9
+ "default": true,
10
+ "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
11
+ },
12
+ {
13
+ "key": "partner_id",
14
+ "label": "Odoo ID",
15
+ "type": "int",
16
+ "source": "odoo",
17
+ "derived": true,
18
+ "default": false,
19
+ "description": "The Odoo res.partner id β€” the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source."
20
+ },
21
+ {
22
+ "key": "odoo_status",
23
+ "label": "Odoo record",
24
+ "type": "status",
25
+ "source": "odoo",
26
+ "default": false,
27
+ "options": [
28
+ "Active",
29
+ "Archived"
30
+ ],
31
+ "description": "Whether this customer still exists in Odoo. Archived means deleted there."
32
+ },
33
+ {
34
+ "key": "agent",
35
+ "label": "Agent",
36
+ "type": "text",
37
+ "source": "odoo",
38
+ "default": true,
39
+ "description": "The sales agent who owns this account."
40
+ },
41
+ {
42
+ "key": "dba",
43
+ "label": "DBA",
44
+ "type": "select",
45
+ "source": "odoo",
46
+ "default": false,
47
+ "options": [
48
+ "Fisch",
49
+ "Royal",
50
+ "Both"
51
+ ],
52
+ "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
53
+ },
54
+ {
55
+ "key": "salesperson",
56
+ "label": "Salesperson",
57
+ "type": "text",
58
+ "source": "odoo",
59
+ "default": false,
60
+ "description": "Who keyed in most of this customer's orders β€” not the Agent, who owns the account."
61
+ },
62
+ {
63
+ "key": "street",
64
+ "label": "Street",
65
+ "type": "text",
66
+ "source": "odoo",
67
+ "default": false,
68
+ "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
69
+ },
70
+ {
71
+ "key": "street2",
72
+ "label": "Street 2",
73
+ "type": "text",
74
+ "source": "odoo",
75
+ "default": false,
76
+ "description": "Second address line (suite, unit, floor) on the customer's Odoo address."
77
+ },
78
+ {
79
+ "key": "city",
80
+ "label": "City",
81
+ "type": "text",
82
+ "source": "odoo",
83
+ "default": true,
84
+ "description": "City on the customer's Odoo address."
85
+ },
86
+ {
87
+ "key": "state",
88
+ "label": "State",
89
+ "type": "text",
90
+ "source": "odoo",
91
+ "default": true,
92
+ "description": "State or province on the customer's Odoo address."
93
+ },
94
+ {
95
+ "key": "country",
96
+ "label": "Country",
97
+ "type": "text",
98
+ "source": "odoo",
99
+ "default": false,
100
+ "description": "Country on the customer's Odoo address."
101
+ },
102
+ {
103
+ "key": "zip",
104
+ "label": "ZIP",
105
+ "type": "text",
106
+ "source": "odoo",
107
+ "default": false,
108
+ "description": "Postal code on the customer's Odoo address."
109
+ },
110
+ {
111
+ "key": "customer_since",
112
+ "label": "Customer since",
113
+ "type": "date",
114
+ "source": "odoo",
115
+ "default": false,
116
+ "description": "When this customer was first set up in Odoo."
117
+ },
118
+ {
119
+ "key": "tags",
120
+ "label": "Tags",
121
+ "type": "text",
122
+ "source": "odoo",
123
+ "default": false,
124
+ "description": "Odoo labels on this customer, comma-separated."
125
+ },
126
+ {
127
+ "key": "pricelist",
128
+ "label": "Price list",
129
+ "type": "text",
130
+ "source": "odoo",
131
+ "default": false,
132
+ "description": "The price list this customer buys on."
133
+ },
134
+ {
135
+ "key": "payment_terms",
136
+ "label": "Payment terms",
137
+ "type": "text",
138
+ "source": "odoo",
139
+ "default": false,
140
+ "description": "Payment terms on this customer's account β€” Net 30, for example."
141
+ },
142
+ {
143
+ "key": "last_order",
144
+ "label": "Last order",
145
+ "type": "date",
146
+ "source": "odoo",
147
+ "default": true,
148
+ "description": "Date of the most recent confirmed order."
149
+ },
150
+ {
151
+ "key": "overdue_days",
152
+ "label": "Overdue days",
153
+ "type": "int",
154
+ "source": "odoo",
155
+ "default": true,
156
+ "description": "How many days late this customer is running against their own usual ordering rhythm."
157
+ },
158
+ {
159
+ "_note": "filterable:false β€” DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule β€” see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
160
+ "key": "est_missed",
161
+ "label": "Est. missed $",
162
+ "type": "currency",
163
+ "source": "odoo",
164
+ "default": true,
165
+ "agg": "sum",
166
+ "filterable": false,
167
+ "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
168
+ },
169
+ {
170
+ "_note": "wave 21 R1 β€” KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary β€” 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
171
+ "key": "ar_open",
172
+ "label": "AR current $",
173
+ "type": "currency",
174
+ "source": "odoo",
175
+ "default": false,
176
+ "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
177
+ },
178
+ {
179
+ "key": "ar_overdue",
180
+ "label": "AR overdue $",
181
+ "type": "currency",
182
+ "source": "odoo",
183
+ "default": false,
184
+ "description": "Invoiced money past due β€” same basis as the Collections page."
185
+ },
186
+ {
187
+ "_note": "wave 21 R1 β€” the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie β€” no second oracle.",
188
+ "key": "ar_outstanding",
189
+ "label": "AR outstanding $",
190
+ "type": "currency",
191
+ "source": "odoo",
192
+ "default": false,
193
+ "description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
194
+ },
195
+ {
196
+ "key": "ar_exposure",
197
+ "label": "Credit exposure $",
198
+ "type": "currency",
199
+ "source": "odoo",
200
+ "default": false,
201
+ "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
202
+ },
203
+ {
204
+ "key": "ar_aged_1_30",
205
+ "label": "1-30 days $",
206
+ "type": "currency",
207
+ "source": "odoo",
208
+ "default": false,
209
+ "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
210
+ },
211
+ {
212
+ "key": "ar_aged_31_60",
213
+ "label": "31-60 days $",
214
+ "type": "currency",
215
+ "source": "odoo",
216
+ "default": false,
217
+ "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
218
+ },
219
+ {
220
+ "key": "ar_aged_61_90",
221
+ "label": "61-90 days $",
222
+ "type": "currency",
223
+ "source": "odoo",
224
+ "default": false,
225
+ "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
226
+ },
227
+ {
228
+ "key": "ar_aged_90_plus",
229
+ "label": "90+ days $",
230
+ "type": "currency",
231
+ "source": "odoo",
232
+ "default": false,
233
+ "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
234
+ },
235
+ {
236
+ "key": "days_to_pay",
237
+ "label": "Days to pay",
238
+ "type": "int",
239
+ "source": "odoo",
240
+ "default": false,
241
+ "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
242
+ },
243
+ {
244
+ "key": "top_category",
245
+ "label": "Top category",
246
+ "type": "text",
247
+ "source": "odoo",
248
+ "default": false,
249
+ "description": "The category this customer spent the most on in the last 12 months."
250
+ },
251
+ {
252
+ "key": "top_category_pct",
253
+ "label": "Top category %",
254
+ "type": "pct",
255
+ "source": "odoo",
256
+ "default": false,
257
+ "description": "Share of last-12-months spend that went to the top category."
258
+ },
259
+ {
260
+ "key": "sku_count",
261
+ "label": "SKUs bought",
262
+ "type": "int",
263
+ "source": "odoo",
264
+ "default": false,
265
+ "description": "Distinct products bought in the last 12 months."
266
+ },
267
+ {
268
+ "key": "top_sku",
269
+ "label": "Top SKU",
270
+ "type": "text",
271
+ "source": "odoo",
272
+ "default": false,
273
+ "description": "The product this customer spent the most on in the last 12 months."
274
+ },
275
+ {
276
+ "key": "days_since",
277
+ "label": "Days since order",
278
+ "type": "int",
279
+ "source": "odoo",
280
+ "default": false,
281
+ "description": "Days since the last confirmed order."
282
+ },
283
+ {
284
+ "key": "typical_gap_days",
285
+ "label": "Typical gap days",
286
+ "type": "int",
287
+ "source": "odoo",
288
+ "default": false,
289
+ "description": "Days this customer usually goes between orders, from their own history."
290
+ },
291
+ {
292
+ "key": "notes",
293
+ "label": "Notes",
294
+ "type": "text",
295
+ "source": "overlay",
296
+ "default": false,
297
+ "description": "Your notes on this customer. Saved in this app only, visible only to you."
298
+ }
299
+ ],
300
+ "rows": [
301
+ {
302
+ "pid": 101,
303
+ "customer": "Poppy Flowers",
304
+ "status": "New",
305
+ "agent": "Naomi Linnell Rivera",
306
+ "city": "Charlottesville",
307
+ "state": "Virginia (US)",
308
+ "last_order": "2026-07-21",
309
+ "est_missed": 0,
310
+ "notes": "",
311
+ "country": "United States",
312
+ "zip": "02720",
313
+ "payment_terms": "30 Days",
314
+ "pricelist": "Fisch 1 (USD)",
315
+ "tags": "Royal",
316
+ "customer_since": "2023-01-10",
317
+ "salesperson": "Jessica",
318
+ "ar_open": 0,
319
+ "ar_overdue": 0,
320
+ "ar_exposure": 0,
321
+ "top_category": "Styrofoam",
322
+ "top_category_pct": 0.47,
323
+ "sku_count": 93,
324
+ "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
325
+ "days_to_pay": 34,
326
+ "_created": "2023-01-15 09:10:00",
327
+ "lat": 25.7617,
328
+ "lon": -80.1918,
329
+ "odoo_status": "Archived",
330
+ "dba": "Royal",
331
+ "ar_outstanding": 0
332
+ },
333
+ {
334
+ "pid": 102,
335
+ "customer": "Meadow & Vine Wholesale",
336
+ "status": "Growing",
337
+ "agent": "Carla Jimenez",
338
+ "city": "Portland",
339
+ "state": "Oregon (US)",
340
+ "last_order": "2026-07-19",
341
+ "est_missed": 0,
342
+ "notes": "Expanding to a second storefront.",
343
+ "country": "United States",
344
+ "zip": "77041",
345
+ "payment_terms": "Immediate Payment",
346
+ "pricelist": "Royal 1 (USD)",
347
+ "tags": "Royal, Key account",
348
+ "customer_since": "2024-02-11",
349
+ "salesperson": "Naomi",
350
+ "ar_open": 1240.5,
351
+ "ar_overdue": 0,
352
+ "ar_exposure": 1740.5,
353
+ "top_category": "Ribbon",
354
+ "top_category_pct": 0.95,
355
+ "sku_count": 4,
356
+ "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
357
+ "days_to_pay": null,
358
+ "_created": "2023-02-15 09:11:00",
359
+ "lat": 27.9506,
360
+ "lon": -82.4572,
361
+ "odoo_status": "Active",
362
+ "dba": "Fisch",
363
+ "ar_outstanding": 1240.5
364
+ },
365
+ {
366
+ "pid": 103,
367
+ "customer": "Bluestem Floral Supply",
368
+ "status": "Growing",
369
+ "agent": "Naomi Linnell Rivera",
370
+ "city": "Kansas City",
371
+ "state": "Missouri (US)",
372
+ "last_order": "2026-07-17",
373
+ "est_missed": 0,
374
+ "notes": "",
375
+ "country": "United States",
376
+ "zip": "11219",
377
+ "payment_terms": "60 Days",
378
+ "pricelist": "Royal 1 (USD)",
379
+ "tags": "Fisch",
380
+ "customer_since": "2025-03-12",
381
+ "salesperson": "Karen",
382
+ "ar_open": 0,
383
+ "ar_overdue": 5120.25,
384
+ "ar_exposure": 5120.25,
385
+ "top_category": "Foams & Finishes",
386
+ "top_category_pct": 0.31,
387
+ "sku_count": 27,
388
+ "top_sku": "SATIN RIBBON 2IN",
389
+ "days_to_pay": 61,
390
+ "_created": "2023-03-15 09:12:00",
391
+ "lat": 28.5384,
392
+ "lon": -81.3789,
393
+ "odoo_status": "Active",
394
+ "dba": "Both",
395
+ "ar_outstanding": 5120.25
396
+ },
397
+ {
398
+ "pid": 104,
399
+ "customer": "Camellia Row Florist",
400
+ "status": "Declining",
401
+ "agent": "Devon Marsh",
402
+ "city": "Savannah",
403
+ "state": "Georgia (US)",
404
+ "last_order": "2026-05-30",
405
+ "est_missed": 41200,
406
+ "notes": "Switched some volume to a local grower.",
407
+ "country": "United States",
408
+ "zip": "07649",
409
+ "payment_terms": "30 Days",
410
+ "pricelist": "Fisch 1 (USD)",
411
+ "tags": "(none)",
412
+ "customer_since": "2026-04-13",
413
+ "salesperson": "(none)",
414
+ "ar_open": 8300,
415
+ "ar_overdue": 940,
416
+ "ar_exposure": 11440,
417
+ "top_category": "All",
418
+ "top_category_pct": 1.0,
419
+ "sku_count": 1,
420
+ "top_sku": "(none)",
421
+ "days_to_pay": 12,
422
+ "_created": "2023-04-15 09:13:00",
423
+ "lat": 30.3322,
424
+ "lon": -81.6557,
425
+ "odoo_status": "Active",
426
+ "dba": "Royal",
427
+ "ar_outstanding": 9240
428
+ },
429
+ {
430
+ "pid": 105,
431
+ "customer": "Harborlight Wholesale Blooms",
432
+ "status": "Growing",
433
+ "agent": "Carla Jimenez",
434
+ "city": "Seattle",
435
+ "state": "Washington (US)",
436
+ "last_order": "2026-07-22",
437
+ "est_missed": 0,
438
+ "notes": "Top-10 account.",
439
+ "country": "United States",
440
+ "zip": "33125",
441
+ "payment_terms": "Immediate Payment",
442
+ "pricelist": "Royal 1 (USD)",
443
+ "tags": "Royal",
444
+ "customer_since": "2023-05-14",
445
+ "salesperson": "Jessica",
446
+ "ar_open": 0,
447
+ "ar_overdue": 0,
448
+ "ar_exposure": 0,
449
+ "top_category": "Styrofoam",
450
+ "top_category_pct": 0.47,
451
+ "sku_count": 93,
452
+ "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
453
+ "days_to_pay": 34,
454
+ "_created": "2023-05-15 09:14:00",
455
+ "lat": 26.1224,
456
+ "lon": -80.1373,
457
+ "odoo_status": "Active",
458
+ "dba": "",
459
+ "ar_outstanding": 0
460
+ },
461
+ {
462
+ "pid": 106,
463
+ "customer": "Dogwood & Fern Co.",
464
+ "status": "Dormant",
465
+ "agent": "Devon Marsh",
466
+ "city": "Asheville",
467
+ "state": "North Carolina (US)",
468
+ "last_order": "2026-02-11",
469
+ "est_missed": 52400,
470
+ "notes": "No spring order this year.",
471
+ "country": "United States",
472
+ "zip": "90210",
473
+ "payment_terms": "60 Days",
474
+ "pricelist": "Royal 1 (USD)",
475
+ "tags": "Royal, Key account",
476
+ "customer_since": "2024-06-15",
477
+ "salesperson": "Naomi",
478
+ "ar_open": 1240.5,
479
+ "ar_overdue": 0,
480
+ "ar_exposure": 1740.5,
481
+ "top_category": "Ribbon",
482
+ "top_category_pct": 0.95,
483
+ "sku_count": 4,
484
+ "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
485
+ "days_to_pay": null,
486
+ "_created": "2023-06-15 09:15:00",
487
+ "lat": 27.3364,
488
+ "lon": -82.5307,
489
+ "odoo_status": "Active",
490
+ "dba": "Fisch",
491
+ "ar_outstanding": 1240.5
492
+ },
493
+ {
494
+ "pid": 107,
495
+ "customer": "Verbena Market Florals",
496
+ "status": "Lost",
497
+ "agent": "Naomi Linnell Rivera",
498
+ "city": "Austin",
499
+ "state": "Texas (US)",
500
+ "last_order": "2025-11-04",
501
+ "est_missed": 78300,
502
+ "notes": "Went with a competitor on freight terms.",
503
+ "country": "United States",
504
+ "zip": "08701",
505
+ "payment_terms": "30 Days",
506
+ "pricelist": "Fisch 1 (USD)",
507
+ "tags": "Fisch",
508
+ "customer_since": "2025-07-16",
509
+ "salesperson": "Karen",
510
+ "ar_open": 0,
511
+ "ar_overdue": 5120.25,
512
+ "ar_exposure": 5120.25,
513
+ "top_category": "Foams & Finishes",
514
+ "top_category_pct": 0.31,
515
+ "sku_count": 27,
516
+ "top_sku": "SATIN RIBBON 2IN",
517
+ "days_to_pay": 61,
518
+ "_created": "2023-07-15 09:16:00",
519
+ "lat": null,
520
+ "lon": null,
521
+ "odoo_status": "Active",
522
+ "dba": "Both",
523
+ "ar_outstanding": 5120.25
524
+ },
525
+ {
526
+ "pid": 108,
527
+ "customer": "Larkspur Lane Supply",
528
+ "status": "New",
529
+ "agent": "Carla Jimenez",
530
+ "city": "Denver",
531
+ "state": "Colorado (US)",
532
+ "last_order": "2026-07-14",
533
+ "est_missed": 0,
534
+ "notes": "First order in April.",
535
+ "country": "United States",
536
+ "zip": "60614",
537
+ "payment_terms": "Immediate Payment",
538
+ "pricelist": "Royal 1 (USD)",
539
+ "tags": "(none)",
540
+ "customer_since": "2026-08-17",
541
+ "salesperson": "(none)",
542
+ "ar_open": 8300,
543
+ "ar_overdue": 940,
544
+ "ar_exposure": 11440,
545
+ "top_category": "All",
546
+ "top_category_pct": 1.0,
547
+ "sku_count": 1,
548
+ "top_sku": "(none)",
549
+ "days_to_pay": 12,
550
+ "_created": "2023-08-15 09:17:00",
551
+ "lat": 33.749,
552
+ "lon": -84.388,
553
+ "odoo_status": "Active",
554
+ "dba": "Royal",
555
+ "ar_outstanding": 9240
556
+ }
557
+ ]
558
+ }
web/src/alerts/alertsModel.ts CHANGED
@@ -1,418 +1,418 @@
1
- // ---------------------------------------------------------------------------
2
- // alerts/alertsModel.ts β€” WAVE 20 item 25 (contract C-ALERT): the inbox's PURE
3
- // half. React-free and fetch-free, so `verify_alerts.py` runs it under node.
4
- //
5
- // An alert says "tell me when a record ENTERS this view". The client half is
6
- // small, and every part of it fails silently when it is wrong:
7
- //
8
- // Β· an unread count taken from `items.length` rather than from the server's
9
- // own `unread` disagrees with the badge the moment a page is capped or a
10
- // read lands in another tab β€” and a badge that says 3 when the list shows 9
11
- // teaches the reader to ignore the badge;
12
- // Β· a notification whose `viewId` no longer resolves must open NOTHING rather
13
- // than a wrong view β€” alerts outlive the views they were made from;
14
- // Β· `topic` is the surface's scope key ("customer", "product", "ut_…"), and
15
- // the ROUTE is a registry key ("customer_data") β€” mapping one to the other
16
- // by guesswork sends every click to a page that does not exist.
17
- // ---------------------------------------------------------------------------
18
-
19
- /**
20
- * ⭐⭐ WAVE 32 Β· T20 Β· CONTRACT C3 β€” WHERE AN INBOX ITEM OPENS.
21
- *
22
- * `module` is the destination surface (`"database"` or `"automation"`) and `id` is what to open
23
- * in it. `tab` is the SUB-SELECTION inside that module: the literal `"runs"` for an automation's
24
- * run log, or the VIEW ID to select on a database. One optional key, two destinations.
25
- *
26
- * β›” EVERY FIELD IS A PLAIN `string`, NEVER A UNION, and that is alertsModel's wave-9 law rather
27
- * than laziness: the vocabulary is the SERVER's, and a client union over it turns "the server grew
28
- * a module" into "the client silently drops the row". An unknown module is refused by the
29
- * FRAME's dispatcher, out loud, which is a different thing from never arriving.
30
- */
31
- export interface NotificationTarget {
32
- module: string;
33
- id: string;
34
- tab?: string;
35
- }
36
-
37
- /** C3's `kind` vocabulary, mirroring `routes_alerts.NOTIF_KIND_*`. Held as constants so the
38
- * Inbox's branch and the gate's fixtures cannot disagree about the word. */
39
- export const NOTIF_KIND_ALERT = "alert";
40
- export const NOTIF_KIND_AUTOMATION = "automation";
41
- export const NOTIF_KIND_SHARE = "share";
42
-
43
- /** C3's `target.module` vocabulary, and the automation sub-selection. */
44
- export const TARGET_MODULE_DATABASE = "database";
45
- export const TARGET_MODULE_AUTOMATION = "automation";
46
- export const TARGET_TAB_RUNS = "runs";
47
-
48
- /** One notification, as `GET /api/v1/notifications` sends it. */
49
- export interface Notification {
50
- id: string;
51
- alertId: string;
52
- viewId: string;
53
- topic: string;
54
- rowId: string;
55
- /** What entered β€” the record's own label. */
56
- label: string;
57
- /** What the alert is called, so a row reads without opening anything. */
58
- alertLabel: string;
59
- /** UTC WITH OFFSET (D-18). Kept as the server's STRING: re-formatting it here
60
- * would re-introduce the browser-clock drift the offset exists to remove. */
61
- at: string;
62
- read: boolean;
63
- /**
64
- * WAVE 23 (contract C6) β€” WHAT KIND of notification this is.
65
- *
66
- * Absent (and every notification written before this wave) means the original one: a record
67
- * ENTERED a watched view, routed by `topic` + `viewId`. `"automation_review"` means a card
68
- * arrived at a review stage and routes by `autoId` instead β€” a different destination reached
69
- * from the same list.
70
- *
71
- * β›” A STRING, NEVER A UNION, and that is the wave-9 law rather than laziness: the vocabulary
72
- * is the SERVER's, and a client union over it turns "the server grew a kind" into "the client
73
- * silently drops the row". Unknown kinds fall through to the view route, which is exactly what
74
- * they did before this field existed.
75
- */
76
- kind?: string;
77
- /** `automation_review` only: the automation whose review stage a card reached. Absent on
78
- * every other kind β€” and an `automation_review` row that arrives WITHOUT one opens nothing
79
- * rather than guessing, the same posture `viewId` gets. */
80
- autoId?: string;
81
- /** Advisory. A surface that does not scroll to a stage simply selects the automation. */
82
- stageId?: string;
83
- /** How many cards arrived in the batch. C6 queues ONE notification per run naming the count,
84
- * never one per record β€” so this is the number the row's own text is built from. */
85
- count?: number;
86
- /**
87
- * ⭐ WAVE 32 Β· C3 β€” THE HEADER LINE, so the Inbox can be laid out like mail.
88
- *
89
- * `subject` is what the item is ABOUT (the alert's name, the automation's name, the database
90
- * that was shared); `label` stays what HAPPENED (the record that entered, the run summary).
91
- * They were one field, which is why the pane could only ever render a sentence with no sender.
92
- * Absent on a server that predates this wave β€” {@link subjectOf} falls back rather than
93
- * rendering a blank header.
94
- */
95
- subject?: string;
96
- /**
97
- * ⭐⭐ WAVE 33 Β· W33-T28 β€” WHO IT IS FROM. Mail has a sender; this list did not.
98
- *
99
- * A `verifier` reading the finished wave-32 Inbox found the row's sender POSITION occupied by
100
- * `kindLabel(n.kind)` β€” the literals "Alert" / "Automation" / "Shared with you", a CATEGORY
101
- * standing where a who belongs β€” and no sender anywhere on the wire. Derived by the server
102
- * (`routes_alerts.notification_view`), never here: only it knows that a share has a person
103
- * behind it and an alert has a machine.
104
- * ⚠ Optional, so a payload from a server that predates this wave keeps today's shape;
105
- * {@link senderOf} falls back rather than rendering a blank From column.
106
- */
107
- sender?: string;
108
- /** ⭐ WAVE 32 Β· C3 β€” where clicking it goes. ABSENT when this product cannot resolve a
109
- * destination (an alert on a table this account can no longer route to), and that absence is
110
- * load-bearing: the Inbox renders such a row as plainly unclickable rather than as a click
111
- * that silently does nothing. */
112
- target?: NotificationTarget;
113
- }
114
-
115
- /** One alert, as `GET /api/v1/alerts` sends it. */
116
- export interface Alert {
117
- id: string;
118
- viewId: string;
119
- topic: string;
120
- owner: string;
121
- label: string;
122
- createdAt: string;
123
- /** How many records are in its remembered set right now. */
124
- matched: number;
125
- seeded: boolean;
126
- lastRunAt: string;
127
- lastError: string;
128
- }
129
-
130
- export interface Inbox {
131
- unread: number;
132
- items: Notification[];
133
- /**
134
- * ⭐⭐ WAVE 33 Β· W33-T28 / D-208 β€” THE SERVER'S CLOCK, so a stamp can be mail-shaped.
135
- *
136
- * β›” It exists so {@link stampText} never reads the BROWSER's clock, which is D-208's exit
137
- * condition word for word. `at` is UTC with its offset (D-18) precisely so every reader sees
138
- * the same instant; asking "is this today?" of a local clock would put the drift back.
139
- * ⚠ Optional: absent, `stampText` returns the old absolute string rather than guessing.
140
- */
141
- now?: string;
142
- }
143
-
144
- export const EMPTY_INBOX: Inbox = { unread: 0, items: [] };
145
-
146
- /**
147
- * ⭐⭐ WAVE 31 Β· T23 (owner item 1) β€” WHAT THE PANE IS ENTITLED TO SAY, AS A FUNCTION.
148
- *
149
- * Owner, verbatim: *"Alerts shows notification, but when clicked it says nothing, and it doesn't
150
- * remove the notification number."* Both halves are ONE mechanism. `AlertsPane` held its own
151
- * `inbox` seeded to {@link EMPTY_INBOX} and had no pending state, so between opening the panel
152
- * and `GET /notifications` answering β€” measured at **3,280 ms live** β€” and for ever after a
153
- * failed fetch:
154
- * Β· `items` was `[]`, so the pane printed **"Nothing new."** β€” a claim, not a wait;
155
- * Β· `unread` was `0`, so **"Mark all read" was `disabled`**, and the badge the frame had
156
- * already loaded could never be cleared.
157
- * A confident sentence about somebody's inbox, and the one control that would fix it, both
158
- * switched off by the same uninitialised state.
159
- *
160
- * β›” IT IS A FUNCTION BECAUSE THE PANE IS TSX AND TSX IS NOT UNDER TEST HERE. `verify_alerts`
161
- * compiles and RUNS this module under node; markup it cannot reach is markup no control can
162
- * mutate. Deciding here means the truth table is asserted and each branch is load-bearing.
163
- */
164
- export type PaneView = "pending" | "rows" | "empty" | "error";
165
-
166
- /**
167
- * `pending` while the first read is in flight Β· `error` when it failed and we have nothing to
168
- * show Β· `rows` when there is something Β· `empty` ONLY when a successful read returned nothing.
169
- *
170
- * ⚠ ROWS WIN OVER AN ERROR, and that is deliberate rather than lax: a refresh that fails while
171
- * the pane already holds notifications should not blank them β€” the reader loses real information
172
- * to a transient. The error still reaches them as the pane's message line.
173
- */
174
- export function paneView(
175
- phase: "pending" | "ready" | "error",
176
- rowCount: number,
177
- seededUnread = 0
178
- ): PaneView {
179
- if (rowCount > 0) return "rows";
180
- if (phase === "pending") return "pending";
181
- if (phase === "error") return "error";
182
- // ⚠ A SUCCESSFUL READ WITH NO ROWS AND A NON-ZERO COUNT IS NOT "nothing new". The badge says
183
- // there is something; the page we were given does not contain it. Saying "Nothing new" there
184
- // is the same false confidence in a different costume.
185
- return seededUnread > 0 ? "error" : "empty";
186
- }
187
-
188
- /**
189
- * May "Mark all read" be pressed?
190
- *
191
- * β›” NOT `unread === 0`, WHICH IS THE SHIPPED BUG. That test asked the PANE's own state β€” zero
192
- * until its fetch lands, zero for ever if the fetch fails β€” so the control was dead in exactly
193
- * the situations the owner hit. The question is about the ACCOUNT, so it is asked of the count
194
- * the frame already holds, and a failed read does not take the verb away: `POST /notifications/
195
- * read` with `ids: null` clears the account's inbox whether or not we managed to list it.
196
- */
197
- export function canMarkAll(unread: number, busy = false): boolean {
198
- return !busy && Math.max(0, Math.floor(unread)) > 0;
199
- }
200
-
201
- const str = (v: unknown): string => (typeof v === "string" ? v : "");
202
- const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0);
203
-
204
- /**
205
- * ⭐ WAVE 32 Β· C3 β€” one wire `target` β†’ a {@link NotificationTarget}, or `null`.
206
- *
207
- * β›” BOTH `module` AND `id` ARE REQUIRED, and dropping either test is the failure this guard is
208
- * for: a target with a module and no id dispatches an open request naming NOTHING β€” a click that
209
- * appears to work and silently does not, which is this repo's most-repeated shape. `null` is
210
- * rendered as an unclickable row, which a reader can SEE.
211
- */
212
- export function parseTarget(raw: unknown): NotificationTarget | null {
213
- if (!raw || typeof raw !== "object") return null;
214
- const t = raw as Record<string, unknown>;
215
- const module = str(t.module).trim();
216
- const id = str(t.id).trim();
217
- if (!module || !id) return null;
218
- const tab = str(t.tab).trim();
219
- return { module, id, ...(tab ? { tab } : {}) };
220
- }
221
-
222
- /**
223
- * `GET /notifications` β†’ the inbox, fail-closed.
224
- *
225
- * ⚠ `unread` COMES FROM THE SERVER, and is not recounted from `items`. The two
226
- * can legitimately differ β€” the list is what this page holds, the count is what
227
- * the account has β€” and recomputing it here would make the badge a function of
228
- * whatever the last fetch happened to include.
229
- */
230
- export function parseInbox(body: unknown): Inbox {
231
- const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
232
- const raw = Array.isArray(b.items) ? b.items : [];
233
- const items: Notification[] = [];
234
- for (const item of raw) {
235
- if (!item || typeof item !== "object") continue;
236
- const n = item as Record<string, unknown>;
237
- const id = str(n.id);
238
- // An id-less notification cannot be marked read, so it would sit unread for
239
- // ever and hold the badge up. Dropped, not rendered.
240
- if (!id) continue;
241
- items.push({
242
- id,
243
- alertId: str(n.alertId),
244
- viewId: str(n.viewId),
245
- topic: str(n.topic),
246
- rowId: String(n.rowId ?? ""),
247
- label: str(n.label) || String(n.rowId ?? ""),
248
- alertLabel: str(n.alertLabel),
249
- at: str(n.at),
250
- read: n.read === true,
251
- // WAVE 23 C6 β€” ADDITIVE and spread-conditional, exactly like `NavPage`'s flags: a payload
252
- // that predates this wave keeps today's shape rather than gaining four `undefined` keys,
253
- // and an `automation_review` row missing its `autoId` is left WITHOUT one rather than
254
- // with an empty string that would render as a real destination.
255
- ...(str(n.kind) ? { kind: str(n.kind) } : {}),
256
- ...(str(n.autoId) ? { autoId: str(n.autoId) } : {}),
257
- ...(str(n.stageId) ? { stageId: str(n.stageId) } : {}),
258
- ...(num(n.count) > 0 ? { count: num(n.count) } : {}),
259
- // ⭐ WAVE 32 Β· C3 β€” additive and spread-conditional, exactly like the four above.
260
- ...(str(n.subject) ? { subject: str(n.subject) } : {}),
261
- // ⭐ WAVE 33 Β· W33-T28 β€” additive and spread-conditional, like every flag above it.
262
- ...(str(n.sender) ? { sender: str(n.sender) } : {}),
263
- ...(parseTarget(n.target) ? { target: parseTarget(n.target)! } : {}),
264
- });
265
- }
266
- // ⭐ W33-T28 / D-208 β€” the server's clock rides through, spread-conditionally like every other
267
- // additive key here, so a server that predates this wave yields the same object it always did.
268
- return {
269
- unread: Math.max(0, num(b.unread)),
270
- items,
271
- ...(str(b.now) ? { now: str(b.now) } : {}),
272
- };
273
- }
274
-
275
- /** `GET /alerts` β†’ the alert list, fail-closed. */
276
- export function parseAlerts(body: unknown): Alert[] {
277
- const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
278
- const raw = Array.isArray(b.alerts) ? b.alerts : [];
279
- const out: Alert[] = [];
280
- for (const item of raw) {
281
- if (!item || typeof item !== "object") continue;
282
- const a = item as Record<string, unknown>;
283
- const id = str(a.id);
284
- if (!id) continue;
285
- out.push({
286
- id,
287
- viewId: str(a.viewId),
288
- topic: str(a.topic),
289
- owner: str(a.owner),
290
- label: str(a.label) || "Untitled alert",
291
- createdAt: str(a.createdAt),
292
- matched: num(a.matched),
293
- seeded: a.seeded === true,
294
- lastRunAt: str(a.lastRunAt),
295
- lastError: str(a.lastError),
296
- });
297
- }
298
- return out;
299
- }
300
-
301
- /**
302
- * A topic (the grid's scope key) β†’ the hash route that renders it.
303
- *
304
- * The two built-ins are the only pair that differ, and they differ because the
305
- * REGISTRY names the surface while the GRID names the scope; a user table is its
306
- * own key in both. `null` for anything else: a notification for a topic this
307
- * client cannot route to must do nothing, not navigate somewhere plausible.
308
- */
309
- export function routeForTopic(topic: string): string | null {
310
- const t = str(topic).trim();
311
- if (t === "customer") return "customer_data";
312
- if (t === "product") return "product_data";
313
- if (/^ut_[A-Za-z0-9_]+$/.test(t)) return t;
314
- return null;
315
- }
316
-
317
-
318
-
319
-
320
-
321
-
322
- /**
323
- * The stamp, made readable WITHOUT touching a clock.
324
- *
325
- * β›” NO `new Date()`, NO `toLocaleString()`, and that is the whole design. The
326
- * server sends UTC WITH ITS OFFSET (D-18) precisely so every reader sees the
327
- * same instant; parsing it into a browser Date and formatting it back would
328
- * re-introduce the drift the offset exists to remove β€” a tenant a day ahead
329
- * being told an event happened tomorrow ([[date-window-vocabulary]]). This is
330
- * STRING SURGERY: keep the date and the minutes, drop the seconds and the `T`.
331
- * Anything that does not look like an ISO stamp passes through untouched, so a
332
- * format this function has never seen is shown as sent rather than mangled.
333
- */
334
- const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
335
- "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
336
-
337
- export function stampText(at: string, now?: string): string {
338
- const m = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})/.exec(str(at));
339
- if (!m) return str(at);
340
- const [, day, hhmm] = m;
341
- // ⭐⭐ W33-T28 / D-208 β€” MAIL-SHAPED, AND STILL STRING SURGERY.
342
- //
343
- // β›” `now` COMES FROM THE SERVER (`inbox.now`), NEVER FROM `new Date()`. That is D-208's exit
344
- // condition word for word β€” "without reading the browser clock" β€” and the reason is the same
345
- // one this function has always carried: `at` is sent as UTC WITH its offset (D-18) so every
346
- // reader sees the same instant, and deciding "is this today?" against a browser clock would
347
- // re-introduce exactly the drift the offset removes. Both operands are the server's.
348
- //
349
- // ⚠ AND WITHOUT `now` IT DEGRADES TO THE OLD FORMAT RATHER THAN GUESSING. A caller that has no
350
- // server clock gets `2026-08-13 09:41` β€” the pre-wave string, unambiguous and never wrong β€”
351
- // instead of a relative stamp computed from something we do not trust.
352
- const today = /^(\d{4}-\d{2}-\d{2})/.exec(str(now));
353
- if (!today) return `${day} ${hhmm}`;
354
- if (today[1] === day) return hhmm;
355
- const [y, mo, d] = day.split("-");
356
- const month = MONTHS[Number(mo) - 1] || mo;
357
- // The year only when it differs β€” a mail client does not print "2026" on a message from March.
358
- return today[1].slice(0, 4) === y
359
- ? `${month} ${Number(d)}`
360
- : `${month} ${Number(d)}, ${y}`;
361
- }
362
-
363
- /**
364
- * The badge's text. Never the raw number past 99: a nav row is 236px wide and a
365
- * four-digit badge pushes the label out of it.
366
- */
367
- export function badgeText(unread: number): string {
368
- const n = Math.max(0, Math.floor(unread));
369
- if (n <= 0) return "";
370
- return n > 99 ? "99+" : String(n);
371
- }
372
-
373
- /**
374
- * Apply a read/unread change LOCALLY, mirroring what the server just stored, and
375
- * return the new inbox with the count corrected.
376
- *
377
- * `ids === null` is "all of them" (the API's own convention for mark-all). The
378
- * count is derived from the ITEMS here β€” deliberately, and it is the one place
379
- * that is right to do so: the server's answer is in flight, and the alternative
380
- * is a badge that keeps its old number until the refetch lands.
381
- */
382
- export function applyRead(inbox: Inbox, ids: string[] | null, read: boolean): Inbox {
383
- const wanted = ids === null ? null : new Set(ids);
384
- const items = inbox.items.map((n) =>
385
- wanted === null || wanted.has(n.id) ? { ...n, read } : n
386
- );
387
- const seenUnread = items.filter((n) => !n.read).length;
388
- // A page can hold fewer notifications than the account has, so a partial read
389
- // must SUBTRACT from the server's count rather than replace it with this
390
- // page's tally β€” except when marking everything, where zero is the answer.
391
- if (wanted === null) return { unread: read ? 0 : items.length, items };
392
- const changed = inbox.items.filter(
393
- (n) => wanted.has(n.id) && n.read !== read
394
- ).length;
395
- const delta = read ? -changed : changed;
396
- return { unread: Math.max(seenUnread, inbox.unread + delta), items };
397
- }
398
-
399
- /**
400
- * The shell↔rail channel for "make an alert out of this view" β€” the view rail
401
- * raises it, the frame (which knows the current route, and therefore the topic)
402
- * answers. Same reason as C-SHARE's event: the rail is host-neutral and cannot
403
- * import the shell, and it does not know its own scope key.
404
- */
405
- export const ALERT_CREATE_EVENT = "aios:alert-create";
406
-
407
- export interface AlertCreateRequest {
408
- viewId: string;
409
- label: string;
410
- }
411
-
412
- export function parseAlertCreate(detail: unknown): AlertCreateRequest | null {
413
- if (!detail || typeof detail !== "object") return null;
414
- const d = detail as Record<string, unknown>;
415
- const viewId = str(d.viewId).trim();
416
- if (!viewId) return null;
417
- return { viewId, label: str(d.label).trim() || viewId };
418
- }
 
1
+ // ---------------------------------------------------------------------------
2
+ // alerts/alertsModel.ts β€” WAVE 20 item 25 (contract C-ALERT): the inbox's PURE
3
+ // half. React-free and fetch-free, so `verify_alerts.py` runs it under node.
4
+ //
5
+ // An alert says "tell me when a record ENTERS this view". The client half is
6
+ // small, and every part of it fails silently when it is wrong:
7
+ //
8
+ // Β· an unread count taken from `items.length` rather than from the server's
9
+ // own `unread` disagrees with the badge the moment a page is capped or a
10
+ // read lands in another tab β€” and a badge that says 3 when the list shows 9
11
+ // teaches the reader to ignore the badge;
12
+ // Β· a notification whose `viewId` no longer resolves must open NOTHING rather
13
+ // than a wrong view β€” alerts outlive the views they were made from;
14
+ // Β· `topic` is the surface's scope key ("customer", "product", "ut_…"), and
15
+ // the ROUTE is a registry key ("customer_data") β€” mapping one to the other
16
+ // by guesswork sends every click to a page that does not exist.
17
+ // ---------------------------------------------------------------------------
18
+
19
+ /**
20
+ * ⭐⭐ WAVE 32 Β· T20 Β· CONTRACT C3 β€” WHERE AN INBOX ITEM OPENS.
21
+ *
22
+ * `module` is the destination surface (`"database"` or `"automation"`) and `id` is what to open
23
+ * in it. `tab` is the SUB-SELECTION inside that module: the literal `"runs"` for an automation's
24
+ * run log, or the VIEW ID to select on a database. One optional key, two destinations.
25
+ *
26
+ * β›” EVERY FIELD IS A PLAIN `string`, NEVER A UNION, and that is alertsModel's wave-9 law rather
27
+ * than laziness: the vocabulary is the SERVER's, and a client union over it turns "the server grew
28
+ * a module" into "the client silently drops the row". An unknown module is refused by the
29
+ * FRAME's dispatcher, out loud, which is a different thing from never arriving.
30
+ */
31
+ export interface NotificationTarget {
32
+ module: string;
33
+ id: string;
34
+ tab?: string;
35
+ }
36
+
37
+ /** C3's `kind` vocabulary, mirroring `routes_alerts.NOTIF_KIND_*`. Held as constants so the
38
+ * Inbox's branch and the gate's fixtures cannot disagree about the word. */
39
+ export const NOTIF_KIND_ALERT = "alert";
40
+ export const NOTIF_KIND_AUTOMATION = "automation";
41
+ export const NOTIF_KIND_SHARE = "share";
42
+
43
+ /** C3's `target.module` vocabulary, and the automation sub-selection. */
44
+ export const TARGET_MODULE_DATABASE = "database";
45
+ export const TARGET_MODULE_AUTOMATION = "automation";
46
+ export const TARGET_TAB_RUNS = "runs";
47
+
48
+ /** One notification, as `GET /api/v1/notifications` sends it. */
49
+ export interface Notification {
50
+ id: string;
51
+ alertId: string;
52
+ viewId: string;
53
+ topic: string;
54
+ rowId: string;
55
+ /** What entered β€” the record's own label. */
56
+ label: string;
57
+ /** What the alert is called, so a row reads without opening anything. */
58
+ alertLabel: string;
59
+ /** UTC WITH OFFSET (D-18). Kept as the server's STRING: re-formatting it here
60
+ * would re-introduce the browser-clock drift the offset exists to remove. */
61
+ at: string;
62
+ read: boolean;
63
+ /**
64
+ * WAVE 23 (contract C6) β€” WHAT KIND of notification this is.
65
+ *
66
+ * Absent (and every notification written before this wave) means the original one: a record
67
+ * ENTERED a watched view, routed by `topic` + `viewId`. `"automation_review"` means a card
68
+ * arrived at a review stage and routes by `autoId` instead β€” a different destination reached
69
+ * from the same list.
70
+ *
71
+ * β›” A STRING, NEVER A UNION, and that is the wave-9 law rather than laziness: the vocabulary
72
+ * is the SERVER's, and a client union over it turns "the server grew a kind" into "the client
73
+ * silently drops the row". Unknown kinds fall through to the view route, which is exactly what
74
+ * they did before this field existed.
75
+ */
76
+ kind?: string;
77
+ /** `automation_review` only: the automation whose review stage a card reached. Absent on
78
+ * every other kind β€” and an `automation_review` row that arrives WITHOUT one opens nothing
79
+ * rather than guessing, the same posture `viewId` gets. */
80
+ autoId?: string;
81
+ /** Advisory. A surface that does not scroll to a stage simply selects the automation. */
82
+ stageId?: string;
83
+ /** How many cards arrived in the batch. C6 queues ONE notification per run naming the count,
84
+ * never one per record β€” so this is the number the row's own text is built from. */
85
+ count?: number;
86
+ /**
87
+ * ⭐ WAVE 32 Β· C3 β€” THE HEADER LINE, so the Inbox can be laid out like mail.
88
+ *
89
+ * `subject` is what the item is ABOUT (the alert's name, the automation's name, the database
90
+ * that was shared); `label` stays what HAPPENED (the record that entered, the run summary).
91
+ * They were one field, which is why the pane could only ever render a sentence with no sender.
92
+ * Absent on a server that predates this wave β€” {@link subjectOf} falls back rather than
93
+ * rendering a blank header.
94
+ */
95
+ subject?: string;
96
+ /**
97
+ * ⭐⭐ WAVE 33 Β· W33-T28 β€” WHO IT IS FROM. Mail has a sender; this list did not.
98
+ *
99
+ * A `verifier` reading the finished wave-32 Inbox found the row's sender POSITION occupied by
100
+ * `kindLabel(n.kind)` β€” the literals "Alert" / "Automation" / "Shared with you", a CATEGORY
101
+ * standing where a who belongs β€” and no sender anywhere on the wire. Derived by the server
102
+ * (`routes_alerts.notification_view`), never here: only it knows that a share has a person
103
+ * behind it and an alert has a machine.
104
+ * ⚠ Optional, so a payload from a server that predates this wave keeps today's shape;
105
+ * {@link senderOf} falls back rather than rendering a blank From column.
106
+ */
107
+ sender?: string;
108
+ /** ⭐ WAVE 32 Β· C3 β€” where clicking it goes. ABSENT when this product cannot resolve a
109
+ * destination (an alert on a table this account can no longer route to), and that absence is
110
+ * load-bearing: the Inbox renders such a row as plainly unclickable rather than as a click
111
+ * that silently does nothing. */
112
+ target?: NotificationTarget;
113
+ }
114
+
115
+ /** One alert, as `GET /api/v1/alerts` sends it. */
116
+ export interface Alert {
117
+ id: string;
118
+ viewId: string;
119
+ topic: string;
120
+ owner: string;
121
+ label: string;
122
+ createdAt: string;
123
+ /** How many records are in its remembered set right now. */
124
+ matched: number;
125
+ seeded: boolean;
126
+ lastRunAt: string;
127
+ lastError: string;
128
+ }
129
+
130
+ export interface Inbox {
131
+ unread: number;
132
+ items: Notification[];
133
+ /**
134
+ * ⭐⭐ WAVE 33 Β· W33-T28 / D-208 β€” THE SERVER'S CLOCK, so a stamp can be mail-shaped.
135
+ *
136
+ * β›” It exists so {@link stampText} never reads the BROWSER's clock, which is D-208's exit
137
+ * condition word for word. `at` is UTC with its offset (D-18) precisely so every reader sees
138
+ * the same instant; asking "is this today?" of a local clock would put the drift back.
139
+ * ⚠ Optional: absent, `stampText` returns the old absolute string rather than guessing.
140
+ */
141
+ now?: string;
142
+ }
143
+
144
+ export const EMPTY_INBOX: Inbox = { unread: 0, items: [] };
145
+
146
+ /**
147
+ * ⭐⭐ WAVE 31 Β· T23 (owner item 1) β€” WHAT THE PANE IS ENTITLED TO SAY, AS A FUNCTION.
148
+ *
149
+ * Owner, verbatim: *"Alerts shows notification, but when clicked it says nothing, and it doesn't
150
+ * remove the notification number."* Both halves are ONE mechanism. `AlertsPane` held its own
151
+ * `inbox` seeded to {@link EMPTY_INBOX} and had no pending state, so between opening the panel
152
+ * and `GET /notifications` answering β€” measured at **3,280 ms live** β€” and for ever after a
153
+ * failed fetch:
154
+ * Β· `items` was `[]`, so the pane printed **"Nothing new."** β€” a claim, not a wait;
155
+ * Β· `unread` was `0`, so **"Mark all read" was `disabled`**, and the badge the frame had
156
+ * already loaded could never be cleared.
157
+ * A confident sentence about somebody's inbox, and the one control that would fix it, both
158
+ * switched off by the same uninitialised state.
159
+ *
160
+ * β›” IT IS A FUNCTION BECAUSE THE PANE IS TSX AND TSX IS NOT UNDER TEST HERE. `verify_alerts`
161
+ * compiles and RUNS this module under node; markup it cannot reach is markup no control can
162
+ * mutate. Deciding here means the truth table is asserted and each branch is load-bearing.
163
+ */
164
+ export type PaneView = "pending" | "rows" | "empty" | "error";
165
+
166
+ /**
167
+ * `pending` while the first read is in flight Β· `error` when it failed and we have nothing to
168
+ * show Β· `rows` when there is something Β· `empty` ONLY when a successful read returned nothing.
169
+ *
170
+ * ⚠ ROWS WIN OVER AN ERROR, and that is deliberate rather than lax: a refresh that fails while
171
+ * the pane already holds notifications should not blank them β€” the reader loses real information
172
+ * to a transient. The error still reaches them as the pane's message line.
173
+ */
174
+ export function paneView(
175
+ phase: "pending" | "ready" | "error",
176
+ rowCount: number,
177
+ seededUnread = 0
178
+ ): PaneView {
179
+ if (rowCount > 0) return "rows";
180
+ if (phase === "pending") return "pending";
181
+ if (phase === "error") return "error";
182
+ // ⚠ A SUCCESSFUL READ WITH NO ROWS AND A NON-ZERO COUNT IS NOT "nothing new". The badge says
183
+ // there is something; the page we were given does not contain it. Saying "Nothing new" there
184
+ // is the same false confidence in a different costume.
185
+ return seededUnread > 0 ? "error" : "empty";
186
+ }
187
+
188
+ /**
189
+ * May "Mark all read" be pressed?
190
+ *
191
+ * β›” NOT `unread === 0`, WHICH IS THE SHIPPED BUG. That test asked the PANE's own state β€” zero
192
+ * until its fetch lands, zero for ever if the fetch fails β€” so the control was dead in exactly
193
+ * the situations the owner hit. The question is about the ACCOUNT, so it is asked of the count
194
+ * the frame already holds, and a failed read does not take the verb away: `POST /notifications/
195
+ * read` with `ids: null` clears the account's inbox whether or not we managed to list it.
196
+ */
197
+ export function canMarkAll(unread: number, busy = false): boolean {
198
+ return !busy && Math.max(0, Math.floor(unread)) > 0;
199
+ }
200
+
201
+ const str = (v: unknown): string => (typeof v === "string" ? v : "");
202
+ const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0);
203
+
204
+ /**
205
+ * ⭐ WAVE 32 Β· C3 β€” one wire `target` β†’ a {@link NotificationTarget}, or `null`.
206
+ *
207
+ * β›” BOTH `module` AND `id` ARE REQUIRED, and dropping either test is the failure this guard is
208
+ * for: a target with a module and no id dispatches an open request naming NOTHING β€” a click that
209
+ * appears to work and silently does not, which is this repo's most-repeated shape. `null` is
210
+ * rendered as an unclickable row, which a reader can SEE.
211
+ */
212
+ export function parseTarget(raw: unknown): NotificationTarget | null {
213
+ if (!raw || typeof raw !== "object") return null;
214
+ const t = raw as Record<string, unknown>;
215
+ const module = str(t.module).trim();
216
+ const id = str(t.id).trim();
217
+ if (!module || !id) return null;
218
+ const tab = str(t.tab).trim();
219
+ return { module, id, ...(tab ? { tab } : {}) };
220
+ }
221
+
222
+ /**
223
+ * `GET /notifications` β†’ the inbox, fail-closed.
224
+ *
225
+ * ⚠ `unread` COMES FROM THE SERVER, and is not recounted from `items`. The two
226
+ * can legitimately differ β€” the list is what this page holds, the count is what
227
+ * the account has β€” and recomputing it here would make the badge a function of
228
+ * whatever the last fetch happened to include.
229
+ */
230
+ export function parseInbox(body: unknown): Inbox {
231
+ const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
232
+ const raw = Array.isArray(b.items) ? b.items : [];
233
+ const items: Notification[] = [];
234
+ for (const item of raw) {
235
+ if (!item || typeof item !== "object") continue;
236
+ const n = item as Record<string, unknown>;
237
+ const id = str(n.id);
238
+ // An id-less notification cannot be marked read, so it would sit unread for
239
+ // ever and hold the badge up. Dropped, not rendered.
240
+ if (!id) continue;
241
+ items.push({
242
+ id,
243
+ alertId: str(n.alertId),
244
+ viewId: str(n.viewId),
245
+ topic: str(n.topic),
246
+ rowId: String(n.rowId ?? ""),
247
+ label: str(n.label) || String(n.rowId ?? ""),
248
+ alertLabel: str(n.alertLabel),
249
+ at: str(n.at),
250
+ read: n.read === true,
251
+ // WAVE 23 C6 β€” ADDITIVE and spread-conditional, exactly like `NavPage`'s flags: a payload
252
+ // that predates this wave keeps today's shape rather than gaining four `undefined` keys,
253
+ // and an `automation_review` row missing its `autoId` is left WITHOUT one rather than
254
+ // with an empty string that would render as a real destination.
255
+ ...(str(n.kind) ? { kind: str(n.kind) } : {}),
256
+ ...(str(n.autoId) ? { autoId: str(n.autoId) } : {}),
257
+ ...(str(n.stageId) ? { stageId: str(n.stageId) } : {}),
258
+ ...(num(n.count) > 0 ? { count: num(n.count) } : {}),
259
+ // ⭐ WAVE 32 Β· C3 β€” additive and spread-conditional, exactly like the four above.
260
+ ...(str(n.subject) ? { subject: str(n.subject) } : {}),
261
+ // ⭐ WAVE 33 Β· W33-T28 β€” additive and spread-conditional, like every flag above it.
262
+ ...(str(n.sender) ? { sender: str(n.sender) } : {}),
263
+ ...(parseTarget(n.target) ? { target: parseTarget(n.target)! } : {}),
264
+ });
265
+ }
266
+ // ⭐ W33-T28 / D-208 β€” the server's clock rides through, spread-conditionally like every other
267
+ // additive key here, so a server that predates this wave yields the same object it always did.
268
+ return {
269
+ unread: Math.max(0, num(b.unread)),
270
+ items,
271
+ ...(str(b.now) ? { now: str(b.now) } : {}),
272
+ };
273
+ }
274
+
275
+ /** `GET /alerts` β†’ the alert list, fail-closed. */
276
+ export function parseAlerts(body: unknown): Alert[] {
277
+ const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
278
+ const raw = Array.isArray(b.alerts) ? b.alerts : [];
279
+ const out: Alert[] = [];
280
+ for (const item of raw) {
281
+ if (!item || typeof item !== "object") continue;
282
+ const a = item as Record<string, unknown>;
283
+ const id = str(a.id);
284
+ if (!id) continue;
285
+ out.push({
286
+ id,
287
+ viewId: str(a.viewId),
288
+ topic: str(a.topic),
289
+ owner: str(a.owner),
290
+ label: str(a.label) || "Untitled alert",
291
+ createdAt: str(a.createdAt),
292
+ matched: num(a.matched),
293
+ seeded: a.seeded === true,
294
+ lastRunAt: str(a.lastRunAt),
295
+ lastError: str(a.lastError),
296
+ });
297
+ }
298
+ return out;
299
+ }
300
+
301
+ /**
302
+ * A topic (the grid's scope key) β†’ the hash route that renders it.
303
+ *
304
+ * The two built-ins are the only pair that differ, and they differ because the
305
+ * REGISTRY names the surface while the GRID names the scope; a user table is its
306
+ * own key in both. `null` for anything else: a notification for a topic this
307
+ * client cannot route to must do nothing, not navigate somewhere plausible.
308
+ */
309
+ export function routeForTopic(topic: string): string | null {
310
+ const t = str(topic).trim();
311
+ if (t === "customer") return "customer_data";
312
+ if (t === "product") return "product_data";
313
+ if (/^ut_[A-Za-z0-9_]+$/.test(t)) return t;
314
+ return null;
315
+ }
316
+
317
+
318
+
319
+
320
+
321
+
322
+ /**
323
+ * The stamp, made readable WITHOUT touching a clock.
324
+ *
325
+ * β›” NO `new Date()`, NO `toLocaleString()`, and that is the whole design. The
326
+ * server sends UTC WITH ITS OFFSET (D-18) precisely so every reader sees the
327
+ * same instant; parsing it into a browser Date and formatting it back would
328
+ * re-introduce the drift the offset exists to remove β€” a tenant a day ahead
329
+ * being told an event happened tomorrow ([[date-window-vocabulary]]). This is
330
+ * STRING SURGERY: keep the date and the minutes, drop the seconds and the `T`.
331
+ * Anything that does not look like an ISO stamp passes through untouched, so a
332
+ * format this function has never seen is shown as sent rather than mangled.
333
+ */
334
+ const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
335
+ "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
336
+
337
+ export function stampText(at: string, now?: string): string {
338
+ const m = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})/.exec(str(at));
339
+ if (!m) return str(at);
340
+ const [, day, hhmm] = m;
341
+ // ⭐⭐ W33-T28 / D-208 β€” MAIL-SHAPED, AND STILL STRING SURGERY.
342
+ //
343
+ // β›” `now` COMES FROM THE SERVER (`inbox.now`), NEVER FROM `new Date()`. That is D-208's exit
344
+ // condition word for word β€” "without reading the browser clock" β€” and the reason is the same
345
+ // one this function has always carried: `at` is sent as UTC WITH its offset (D-18) so every
346
+ // reader sees the same instant, and deciding "is this today?" against a browser clock would
347
+ // re-introduce exactly the drift the offset removes. Both operands are the server's.
348
+ //
349
+ // ⚠ AND WITHOUT `now` IT DEGRADES TO THE OLD FORMAT RATHER THAN GUESSING. A caller that has no
350
+ // server clock gets `2026-08-13 09:41` β€” the pre-wave string, unambiguous and never wrong β€”
351
+ // instead of a relative stamp computed from something we do not trust.
352
+ const today = /^(\d{4}-\d{2}-\d{2})/.exec(str(now));
353
+ if (!today) return `${day} ${hhmm}`;
354
+ if (today[1] === day) return hhmm;
355
+ const [y, mo, d] = day.split("-");
356
+ const month = MONTHS[Number(mo) - 1] || mo;
357
+ // The year only when it differs β€” a mail client does not print "2026" on a message from March.
358
+ return today[1].slice(0, 4) === y
359
+ ? `${month} ${Number(d)}`
360
+ : `${month} ${Number(d)}, ${y}`;
361
+ }
362
+
363
+ /**
364
+ * The badge's text. Never the raw number past 99: a nav row is 236px wide and a
365
+ * four-digit badge pushes the label out of it.
366
+ */
367
+ export function badgeText(unread: number): string {
368
+ const n = Math.max(0, Math.floor(unread));
369
+ if (n <= 0) return "";
370
+ return n > 99 ? "99+" : String(n);
371
+ }
372
+
373
+ /**
374
+ * Apply a read/unread change LOCALLY, mirroring what the server just stored, and
375
+ * return the new inbox with the count corrected.
376
+ *
377
+ * `ids === null` is "all of them" (the API's own convention for mark-all). The
378
+ * count is derived from the ITEMS here β€” deliberately, and it is the one place
379
+ * that is right to do so: the server's answer is in flight, and the alternative
380
+ * is a badge that keeps its old number until the refetch lands.
381
+ */
382
+ export function applyRead(inbox: Inbox, ids: string[] | null, read: boolean): Inbox {
383
+ const wanted = ids === null ? null : new Set(ids);
384
+ const items = inbox.items.map((n) =>
385
+ wanted === null || wanted.has(n.id) ? { ...n, read } : n
386
+ );
387
+ const seenUnread = items.filter((n) => !n.read).length;
388
+ // A page can hold fewer notifications than the account has, so a partial read
389
+ // must SUBTRACT from the server's count rather than replace it with this
390
+ // page's tally β€” except when marking everything, where zero is the answer.
391
+ if (wanted === null) return { unread: read ? 0 : items.length, items };
392
+ const changed = inbox.items.filter(
393
+ (n) => wanted.has(n.id) && n.read !== read
394
+ ).length;
395
+ const delta = read ? -changed : changed;
396
+ return { unread: Math.max(seenUnread, inbox.unread + delta), items };
397
+ }
398
+
399
+ /**
400
+ * The shell↔rail channel for "make an alert out of this view" β€” the view rail
401
+ * raises it, the frame (which knows the current route, and therefore the topic)
402
+ * answers. Same reason as C-SHARE's event: the rail is host-neutral and cannot
403
+ * import the shell, and it does not know its own scope key.
404
+ */
405
+ export const ALERT_CREATE_EVENT = "aios:alert-create";
406
+
407
+ export interface AlertCreateRequest {
408
+ viewId: string;
409
+ label: string;
410
+ }
411
+
412
+ export function parseAlertCreate(detail: unknown): AlertCreateRequest | null {
413
+ if (!detail || typeof detail !== "object") return null;
414
+ const d = detail as Record<string, unknown>;
415
+ const viewId = str(d.viewId).trim();
416
+ if (!viewId) return null;
417
+ return { viewId, label: str(d.label).trim() || viewId };
418
+ }
web/src/assistant/AssistantPage.tsx CHANGED
@@ -105,7 +105,17 @@ const KIND_LABEL: Record<string, string> = {
105
  * `modelStatus`: a model that IS offered and whose key this deployment does not hold.
106
  */
107
  const MODEL_LABEL: Record<string, string> = {
108
- auto: "Auto", cerebras: "Cerebras", groq: "Groq", openrouter: "OpenRouter",
 
 
 
 
 
 
 
 
 
 
109
  };
110
  const modelLabel = (key: string) => MODEL_LABEL[key] ?? key;
111
 
 
105
  * `modelStatus`: a model that IS offered and whose key this deployment does not hold.
106
  */
107
  const MODEL_LABEL: Record<string, string> = {
108
+ // ⭐ `anthropic` JOINS THE MAP (W36 · ASK E-11, ruling R4). Not a fifth option nobody wanted.
109
+ // E measured all three side rungs refusing live today (cerebras 402, groq 404, openrouter 402),
110
+ // so Anthropic is the only rung that answers at all, and C5 makes it the tool-calling path by
111
+ // ruling rather than by accident. The fall-through below still covers a provider added
112
+ // server-side; this row exists so the ONE rung that works is not the one rendered as a raw key.
113
+ // ⚠ NO `word` FOLLOWED BY A COLON IN THIS COMMENT, and that is not fussiness. `verify_query`
114
+ // reads this map with a regex over the block, comments included, so a phrase like "asked for"
115
+ // with a colon after it registered as a SIXTH key named `for` and reddened the check that
116
+ // exists to keep this map equal to the server's list [[prose-that-becomes-its-own-marker]].
117
+ auto: "Auto", anthropic: "Anthropic", cerebras: "Cerebras", groq: "Groq",
118
+ openrouter: "OpenRouter",
119
  };
120
  const modelLabel = (key: string) => MODEL_LABEL[key] ?? key;
121
 
web/src/automation/AutomationBuilder.tsx CHANGED
@@ -46,13 +46,19 @@ import type {
46
  FlowVocab,
47
  GraphNode,
48
  OAuthStatus,
 
49
  TriggerOption,
50
  UserTable,
51
  } from "./automationApi";
52
  // ⭐ WAVE 26 Β· ITEM 22 / D-70 β€” the Run guard, imported rather than re-derived. This file and
53
  // `AutomationDetail` each paint a Run button onto the same paid action; ONE function decides.
54
  // ITEM 9 / R11 β€” `createTable` + `AutomationError` for the in-place "+ New database".
55
- import { AutomationError, createTable, runBlock } from "./automationApi";
 
 
 
 
 
56
  import CondBuilder, { condComplete } from "./CondBuilder";
57
  import { groupActions, groupBranches, groupByPanel, numberActions, reorderList }
58
  from "./steps";
@@ -141,7 +147,27 @@ interface Props {
141
  * ⚠ REQUIRED. An optional flag defaulting to `true` would make an unmounted toggle look like
142
  * a working panel, which is the shape this repo keeps paying for.
143
  */
 
 
 
 
 
144
  showProperties: boolean;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
145
  /** The Properties/Run-history switch, OWNED by `AutomationDetail` and drawn in both panel
146
  * heads so it is in the same place whichever one is showing. */
147
  panelTabs: ReactNode;
@@ -420,7 +446,9 @@ export default function AutomationBuilder({
420
  onRunNow,
421
  renderNodeBody,
422
  scheduleFace,
 
423
  showProperties,
 
424
  panelTabs,
425
  onStepPicked,
426
  schedules,
@@ -1091,6 +1119,30 @@ export default function AutomationBuilder({
1091
  vocabulary β€” it would have gone on saying the old words after the catalog
1092
  changed, which is the one failure this file's header names twice. */}
1093
  {row?.label || a.kind}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1094
  </span>
1095
  {isGroup ? (
1096
  <span className="autox-card-sub">
@@ -1224,7 +1276,10 @@ export default function AutomationBuilder({
1224
 
1225
  return (
1226
  <>
1227
- <div className="autox-flow">
 
 
 
1228
  {/* ── TRIGGER ─────────────────────────────────────────────────────────────────── */}
1229
  <div className="autox-step">
1230
  <div className="autox-side">
@@ -1504,6 +1559,11 @@ export default function AutomationBuilder({
1504
  ) : (
1505
  <ActionProps
1506
  action={findAction(actions, sel.id)}
 
 
 
 
 
1507
  /* ⭐ THE SAME RULE `actionCard` DERIVES AND THE SERVER ENFORCES (`ig_action_pinned`),
1508
  passed down rather than re-derived a third time. The panel needs it because the
1509
  pinned step is the one action in the product whose value map the ENGINE NEVER
@@ -1938,7 +1998,8 @@ function TriggerProps({
1938
  <p className="auto-note">This server did not offer a view list for that database.</p>
1939
  ) : !table.views.length ? (
1940
  <p className="auto-note">
1941
- That database has no saved views yet β€” make one on its grid and it appears here.
 
1942
  </p>
1943
  ) : (
1944
  <select
@@ -2379,8 +2440,8 @@ export function WebActionConfig({
2379
  <div className="auto-field" key="field">
2380
  <label>{mark("field")} Column to write into</label>
2381
  <p className="auto-note">
2382
- This flow has no record to write to β€” give it a database on the trigger
2383
- first.
2384
  </p>
2385
  </div>
2386
  );
@@ -2462,6 +2523,379 @@ export function WebActionConfig({
2462
  );
2463
  }
2464
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2465
  /**
2466
  * ⭐ WAVE 28 β€” EXPORTED so the include-row render suite can mount the real panel.
2467
  *
@@ -2474,6 +2908,7 @@ export function WebActionConfig({
2474
  export function ActionProps({
2475
  action,
2476
  pinned,
 
2477
  catalog,
2478
  tables,
2479
  onTablesChanged,
@@ -2501,6 +2936,13 @@ export function ActionProps({
2501
  * refuses in five other places.
2502
  */
2503
  pinned: boolean;
 
 
 
 
 
 
 
2504
  catalog: ActionCatalogRow[];
2505
  tables: UserTable[];
2506
  /** ITEM 9 / R11 β€” re-read the list after this panel's picker creates one. REQUIRED; see Props. */
@@ -2556,6 +2998,17 @@ export function ActionProps({
2556
  * picker that reads as "this database has no views".
2557
  */
2558
  const walkViews = (tables.find((t) => t.key === walkTable) || null)?.views;
 
 
 
 
 
 
 
 
 
 
 
2559
 
2560
  return (
2561
  <>
@@ -2564,7 +3017,60 @@ export function ActionProps({
2564
 
2565
  <h3>Configuration</h3>
2566
 
2567
- {action.kind === "group" ? (
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2568
  <>
2569
  {/*
2570
  ⭐ ONE CONDITION EDITOR PER BRANCH (C-FORK). The group's own `config.cond` is GONE β€”
@@ -2606,8 +3112,8 @@ export function ActionProps({
2606
  </div>
2607
  ))}
2608
  <p className="auto-hint">
2609
- The first branch whose conditions match is the one that runs β€” the record does not go
2610
- down two of them.
2611
  </p>
2612
  </>
2613
  ) : null}
@@ -2665,9 +3171,9 @@ export function ActionProps({
2665
  the same lie the value map was. */}
2666
  {pinned ? (
2667
  <p className="auto-hint">
2668
- The search writes one row per profile it finds β€” handle, profile link, name, followers,
2669
- following, average engagement, bio, link in bio, verified and category β€” into that
2670
- database. It matches on the handle, so a profile found again updates its row instead of
2671
  adding another. There is nothing to map: the columns come from the search.
2672
  </p>
2673
  ) : null}
@@ -3254,7 +3760,7 @@ export function ActionProps({
3254
  start, what to do in words, and where to put the answer. Deliberately NOT routed through
3255
  `WebActionConfig`: that component walks `WEB_SEEDS`, and sharing it would mean adding
3256
  `ai_agent` to a table the browser job's runner does not implement. */}
3257
- {action.kind === "ai_agent" ? (
3258
  <>
3259
  <div className="auto-field">
3260
  <label htmlFor="autox-ai-url">
@@ -3296,7 +3802,8 @@ export function ActionProps({
3296
  <div className="auto-field">
3297
  <label>Column to write the answer into</label>
3298
  <p className="auto-note">
3299
- This flow has no record to write to β€” give it a database on the trigger first.
 
3300
  </p>
3301
  </div>
3302
  ) : (
@@ -3359,7 +3866,7 @@ export function ActionProps({
3359
  </>
3360
  ) : null}
3361
 
3362
- {isWeb(action.kind) ? (
3363
  <WebActionConfig
3364
  kind={action.kind}
3365
  cfg={cfg as Record<string, unknown>}
@@ -3371,6 +3878,13 @@ export function ActionProps({
3371
  />
3372
  ) : null}
3373
 
 
 
 
 
 
 
 
3374
  {/*
3375
  ⭐ WAVE 26 Β· ITEM 5 β€” OWNER RULINGS R9 AND R10, and they are two different fixes to one
3376
  symptom (*"the condition on action act_1 names no field"*).
@@ -3396,11 +3910,38 @@ export function ActionProps({
3396
  walking a database that genuinely has no columns yet keeps its (empty) picker, because
3397
  for that flow the picker is the right control and adding a column is the fix.
3398
  */}
3399
- {action.kind === "create_record" ? null : !walkTable ? (
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3400
  <>
3401
  <h3>Run this only when</h3>
3402
  <p className="auto-note">
3403
- This flow has no record to test β€” set conditions on the trigger instead.
3404
  </p>
3405
  </>
3406
  ) : (
 
46
  FlowVocab,
47
  GraphNode,
48
  OAuthStatus,
49
+ StatementWorld,
50
  TriggerOption,
51
  UserTable,
52
  } from "./automationApi";
53
  // ⭐ WAVE 26 Β· ITEM 22 / D-70 β€” the Run guard, imported rather than re-derived. This file and
54
  // `AutomationDetail` each paint a Run button onto the same paid action; ONE function decides.
55
  // ITEM 9 / R11 β€” `createTable` + `AutomationError` for the in-place "+ New database".
56
+ // ⭐ W36-T44 β€” the statements panel's three doors. They go to `routes_statements`, not
57
+ // `/automations`, for the reason `statementBatch`'s own note gives: that is where the send
58
+ // guardrails already are and R10's whole point is that they do not move.
59
+ import {
60
+ AutomationError, createTable, previewStatement, runBlock, statementWorld, testSendStatement,
61
+ } from "./automationApi";
62
  import CondBuilder, { condComplete } from "./CondBuilder";
63
  import { groupActions, groupBranches, groupByPanel, numberActions, reorderList }
64
  from "./steps";
 
147
  * ⚠ REQUIRED. An optional flag defaulting to `true` would make an unmounted toggle look like
148
  * a working panel, which is the shape this repo keeps paying for.
149
  */
150
+ /**
151
+ * ⭐ W36-T69 / R9 β€” `agentActions` off the wire, keyed by action id. Optional, and absent reads
152
+ * as "nothing is agent-owned" (the wire type carries the reasoning).
153
+ */
154
+ agentMarks?: Record<string, { agent: string; agentName?: string; created?: string; by?: string }>;
155
  showProperties: boolean;
156
+ /**
157
+ * ⭐⭐ W36-T32 (owner item 3) β€” HIDE THE FLOW, KEEP THE PANEL. `true` while the Chat view holds
158
+ * the work area.
159
+ *
160
+ * β›” WHY THE BUILDER STAYS MOUNTED AT ALL, WHICH REVISES A WAVE-34 TRADE RATHER THAN IGNORING
161
+ * IT. `AutomationDetail`'s own note recorded that keeping this component mounted behind
162
+ * `hidden` would "preserve the selection too, at the price of two live editors of one
163
+ * automation", and chose to unmount. The owner has since asked for the opposite in as many
164
+ * words: *"even when the Chat is shown first, we still need to be able to see the
165
+ * Properties/the new module/Run history tab on the right side."* The Properties panel is a
166
+ * child of THIS component, so an unmounted builder means no Properties tab during Chat.
167
+ * ⚠ The wave-34 concern is answered rather than overruled: what it feared was two live EDITORS
168
+ * of the flow, and the flow is exactly what this prop hides. The panel is the only live half.
169
+ */
170
+ flowHidden?: boolean;
171
  /** The Properties/Run-history switch, OWNED by `AutomationDetail` and drawn in both panel
172
  * heads so it is in the same place whichever one is showing. */
173
  panelTabs: ReactNode;
 
446
  onRunNow,
447
  renderNodeBody,
448
  scheduleFace,
449
+ agentMarks,
450
  showProperties,
451
+ flowHidden,
452
  panelTabs,
453
  onStepPicked,
454
  schedules,
 
1119
  vocabulary β€” it would have gone on saying the old words after the catalog
1120
  changed, which is the one failure this file's header names twice. */}
1121
  {row?.label || a.kind}
1122
+ {/*
1123
+ ⭐⭐ W36-T69 / R9 β€” THE LOCK, ON THE CARD, so the rule is visible BEFORE the panel
1124
+ is opened. The done-when asks for a lock that NAMES its agent; the glyph carries
1125
+ the name in its `title` and its `aria-label` rather than as a third line of card
1126
+ text, because every card in this column has exactly two lines and a third on one
1127
+ of them is the ragged edge the owner reads as unfinished.
1128
+ ⚠ `aria-hidden` on the SVG and the name on the wrapper: a screen reader should
1129
+ hear "set up by Odoo sync", never "image".
1130
+ */}
1131
+ {agentMarks?.[a.id] ? (
1132
+ <span
1133
+ className="autox-card-lock"
1134
+ title={`Set up by ${agentMarks[a.id].agentName || agentMarks[a.id].agent}. Its `
1135
+ + "configuration is edited by that agent."}
1136
+ aria-label={`Set up by ${agentMarks[a.id].agentName || agentMarks[a.id].agent}`}
1137
+ >
1138
+ <svg width="12" height="12" viewBox="0 0 16 16" fill="none" aria-hidden="true"
1139
+ stroke="currentColor" strokeWidth="1.4" strokeLinecap="round"
1140
+ strokeLinejoin="round">
1141
+ <rect x="3.4" y="7" width="9.2" height="6.4" rx="1.4" />
1142
+ <path d="M5.6 7V5.2a2.4 2.4 0 0 1 4.8 0V7" />
1143
+ </svg>
1144
+ </span>
1145
+ ) : null}
1146
  </span>
1147
  {isGroup ? (
1148
  <span className="autox-card-sub">
 
1276
 
1277
  return (
1278
  <>
1279
+ {/* ⚠ `hidden`, not unmounted (W36-T32). `.autox-flow` sets no `display`, so the UA's own
1280
+ `[hidden] { display: none }` applies and no `!important` override is needed here β€” unlike
1281
+ `.auto-panel[hidden]`, whose class rule sets `display: flex` and had to be beaten. */}
1282
+ <div className="autox-flow" hidden={flowHidden}>
1283
  {/* ── TRIGGER ─────────────────────────────────────────────────────────────────── */}
1284
  <div className="autox-step">
1285
  <div className="autox-side">
 
1559
  ) : (
1560
  <ActionProps
1561
  action={findAction(actions, sel.id)}
1562
+ /* β›” THE MARK FOR THIS ACTION, LOOKED UP ONCE HERE. The panel takes a mark rather than
1563
+ the whole map so it cannot accidentally read another step's ownership, which is the
1564
+ kind of off-by-one that renders a lock on the wrong card and is invisible to a
1565
+ source grep. */
1566
+ agentMark={agentMarks?.[sel.id] || null}
1567
  /* ⭐ THE SAME RULE `actionCard` DERIVES AND THE SERVER ENFORCES (`ig_action_pinned`),
1568
  passed down rather than re-derived a third time. The panel needs it because the
1569
  pinned step is the one action in the product whose value map the ENGINE NEVER
 
1998
  <p className="auto-note">This server did not offer a view list for that database.</p>
1999
  ) : !table.views.length ? (
2000
  <p className="auto-note">
2001
+ That database has no saved views yet. Make one on its grid and it
2002
+ appears here.
2003
  </p>
2004
  ) : (
2005
  <select
 
2440
  <div className="auto-field" key="field">
2441
  <label>{mark("field")} Column to write into</label>
2442
  <p className="auto-note">
2443
+ This flow has no record to write to. Give it a database on the
2444
+ trigger first.
2445
  </p>
2446
  </div>
2447
  );
 
2523
  );
2524
  }
2525
 
2526
+ /**
2527
+ * ⭐⭐ WAVE 36 Β· W36-T44 (D-316 + D-299) β€” THE STATEMENTS STEP'S CONFIGURATION, WHICH DID NOT EXIST.
2528
+ *
2529
+ * β›” THE DEFECT WAS NOT A WRONG CONTROL. IT WAS NO CONTROL, ON THE PUBLIC DOMAIN. The server half
2530
+ * has been complete since wave 35: `automation_engine` seeds `{tier, subject, intro, footer}`,
2531
+ * `_clean_action_config` validates all four, and `assemble_statements` reads all four. This file's
2532
+ * `kind ===` chain had no arm for the kind, so a Royal admin opening *Monthly statements* met a
2533
+ * heading, a hint and NOT ONE INPUT. QA measured it exactly: a page-wide `input, textarea` count
2534
+ * of **1**, the agent's own Name field. And the run's parked-batch banner said *"or narrow the
2535
+ * tier filter"*, naming a control that existed nowhere.
2536
+ * ⚠ THIS IS `ai_enrich` (D-277) A SECOND TIME. A step kind is a CONTRACT BETWEEN TWO TREES and
2537
+ * each end's gate was green because each end was internally consistent
2538
+ * [[artifact-with-no-importer]]. W36-T39 is the gate that makes the class unrepeatable; this is
2539
+ * the instance [[reachable-is-not-the-same-as-built]].
2540
+ *
2541
+ * β›” THE FIX IS THE RENDERER, NEVER A SEED. Blank tier means EVERY tier and blank template means
2542
+ * the sender's own default, and both are deliberate: the seed's own comment says the blank "is the
2543
+ * value the config panel shows as all of them rather than an empty box". Writing non-empty values
2544
+ * in would hide the missing panel and freeze today's wording into every stored agent.
2545
+ *
2546
+ * β›” A COMPONENT AND NOT AN ARM, for a reason the file enforces structurally: `ActionProps` opens
2547
+ * with `if (!action) return`, an early return BEFORE any hook, so nothing inside it may hold
2548
+ * state. This panel has to load the sender's own vocabulary, so it needs its own frame. That is
2549
+ * `WebActionConfig`'s shape too, and exported for `WebActionConfig`'s reason: a panel's defects
2550
+ * (a control bound to no key, a stored value that never reaches its box) are invisible to a grep.
2551
+ */
2552
+ export function StatementActionConfig({
2553
+ cfg,
2554
+ busy,
2555
+ setCfg,
2556
+ }: {
2557
+ cfg: Record<string, unknown>;
2558
+ busy: boolean;
2559
+ setCfg: (patch: Record<string, unknown>) => void;
2560
+ }) {
2561
+ /**
2562
+ * ⚠ THE WORKLIST IS A THIRD STATE AND IS KEPT AS ONE. `null` is "not answered yet", an error
2563
+ * string is "asked and refused", and a value is a value. Collapsing the first two would paint
2564
+ * "this tenant has no tiers" over a request still in flight [[empty-answer-vs-unfinished-answer]].
2565
+ */
2566
+ const [world, setWorld] = useState<StatementWorld | null>(null);
2567
+ const [worldErr, setWorldErr] = useState("");
2568
+ const [customer, setCustomer] = useState("");
2569
+ const [shot, setShot] = useState<{ html: string; to: string; subject: string } | null>(null);
2570
+ const [testTo, setTestTo] = useState("");
2571
+ const [outcome, setOutcome] = useState("");
2572
+ const [working, setWorking] = useState<"" | "preview" | "test">("");
2573
+
2574
+ /*
2575
+ β›” ONE READ, ON MOUNT, ABORTED ON UNMOUNT. `GET /admin/statements` reaches Odoo through
2576
+ `load_collection_list` (cached ~30 min server side), so it is the one slow thing on this panel
2577
+ and everything above it paints without waiting. Selecting a different step unmounts this, and
2578
+ an in-flight answer landing in a dead component is how a panel paints the previous step's
2579
+ customers over the current one's.
2580
+ ⚠ The header of this file forbids a fetch in the BUILDER's own render (C14 leg 1, the ghost
2581
+ fix). That rule is about the trigger/actions/graph painting synchronously from the list
2582
+ payload, and they still do. This is a leaf panel for one selected step, the same shape
2583
+ `NewDatabase`'s create call already has.
2584
+ */
2585
+ useEffect(() => {
2586
+ const ac = new AbortController();
2587
+ statementWorld(ac.signal)
2588
+ .then((w) => {
2589
+ if (!ac.signal.aborted) setWorld(w);
2590
+ })
2591
+ .catch((e: unknown) => {
2592
+ if (ac.signal.aborted) return;
2593
+ setWorldErr(e instanceof Error ? e.message : "the collection list could not be read");
2594
+ });
2595
+ return () => ac.abort();
2596
+ }, []);
2597
+
2598
+ const text = (v: unknown) => (v === undefined || v === null ? "" : String(v));
2599
+ const tier = text(cfg.tier);
2600
+ const templates = {
2601
+ subject: text(cfg.subject),
2602
+ intro: text(cfg.intro),
2603
+ footer: text(cfg.footer),
2604
+ };
2605
+ const tiers = world?.tiers || [];
2606
+ const rows = world?.rows || [];
2607
+
2608
+ /* Free text commits on BLUR, which is this file's idiom and its stated reason: `setCfg` merges
2609
+ into the action and re-renders the whole builder, so a controlled box would round-trip the
2610
+ tree on every keystroke of a footer. */
2611
+ const template = (
2612
+ key: "subject" | "intro" | "footer",
2613
+ label: string,
2614
+ rowsCount: number,
2615
+ hint: string
2616
+ ) => (
2617
+ <div className="auto-field" key={key}>
2618
+ <label htmlFor={`autox-stmt-${key}`}>{label}</label>
2619
+ {rowsCount > 1 ? (
2620
+ <textarea
2621
+ id={`autox-stmt-${key}`}
2622
+ className="auto-input"
2623
+ rows={rowsCount}
2624
+ defaultValue={templates[key]}
2625
+ /* ⚠ THE DEFAULT IS THE PLACEHOLDER, NOT THE VALUE. Blank means "use the sender's own",
2626
+ and the send door resolves it at send time, so painting it as a placeholder shows what
2627
+ blank actually does without storing a copy that stops tracking the sender. With no
2628
+ worklist yet there is nothing honest to show, so it shows nothing. */
2629
+ placeholder={world?.templates?.[key] || ""}
2630
+ disabled={busy}
2631
+ onBlur={(e) => setCfg({ [key]: e.target.value })}
2632
+ />
2633
+ ) : (
2634
+ <input
2635
+ id={`autox-stmt-${key}`}
2636
+ className="auto-input"
2637
+ defaultValue={templates[key]}
2638
+ placeholder={world?.templates?.[key] || ""}
2639
+ disabled={busy}
2640
+ onBlur={(e) => setCfg({ [key]: e.target.value })}
2641
+ />
2642
+ )}
2643
+ <p className="auto-hint">{hint}</p>
2644
+ </div>
2645
+ );
2646
+
2647
+ const canPreview = !!customer && !busy && working === "";
2648
+ const canTest = canPreview && !!testTo.trim();
2649
+
2650
+ return (
2651
+ <>
2652
+ <div className="auto-field">
2653
+ <label htmlFor="autox-stmt-tier">Customers to include</label>
2654
+ <select
2655
+ id="autox-stmt-tier"
2656
+ className="auto-input"
2657
+ value={tier}
2658
+ disabled={busy}
2659
+ onChange={(e) => setCfg({ tier: e.target.value })}
2660
+ >
2661
+ {/*
2662
+ β›”β›” THE BLANK OPTION IS THE STORED DEFAULT AND IT MUST EXIST. `tier: ""` is what the
2663
+ seed writes and it means EVERY tier. A `<select>` whose `value` matches no `<option>`
2664
+ renders the FIRST one, so without this row a freshly seeded agent would paint
2665
+ "A-Urgent", and the next edit would write that over the blank: blank-means-all would
2666
+ silently become a tier filter, and a batch of 200 would quietly become a batch of 40.
2667
+ Same guard the `find_records` database picker carries, and D-10 is why it carries it.
2668
+ */}
2669
+ <option value="">Every tier</option>
2670
+ {/*
2671
+ ⚠ AND THE STORED VALUE IS ALWAYS AN OPTION, with its wording decided by WHICH THIRD
2672
+ STATE we are in. Before the worklist answers, an unmatched stored tier is simply
2673
+ painted: calling it unknown while the list is still empty would be a lie about a
2674
+ request in flight. After it answers, an unmatched value is genuinely not a tier the
2675
+ sender knows, and saying so is the honest reading.
2676
+ */}
2677
+ {tier && !tiers.includes(tier) ? (
2678
+ <option value={tier}>{world ? `${tier} (not a tier the sender offers)` : tier}</option>
2679
+ ) : null}
2680
+ {tiers.map((t) => (
2681
+ <option key={t} value={t}>
2682
+ {t}
2683
+ </option>
2684
+ ))}
2685
+ </select>
2686
+ <p className="auto-hint">
2687
+ {world
2688
+ ? "Leave this on Every tier to statement the whole collection list. One batch holds "
2689
+ + "200 customers."
2690
+ : worldErr
2691
+ ? "The collection list could not be read, so the tier names are not available yet. "
2692
+ + "What is already saved here is kept."
2693
+ : "Reading the collection list to get the tier names."}
2694
+ </p>
2695
+ {worldErr ? <p className="auto-note">{worldErr}</p> : null}
2696
+ </div>
2697
+
2698
+ {template("subject", "Subject", 1,
2699
+ "Leave blank to use the sender's own subject, shown greyed above. {customer}, {company} "
2700
+ + "and {month} are filled in for each customer.")}
2701
+ {template("intro", "Opening", 4,
2702
+ "The paragraph above the invoice table. Blank uses the sender's own. HTML is allowed, and "
2703
+ + "the same three placeholders work here.")}
2704
+ {template("footer", "Closing", 4,
2705
+ "The paragraph below the total. Blank uses the sender's own.")}
2706
+
2707
+ {/*
2708
+ ⭐⭐ D-299 SHIPS IN THIS SAME PANEL, AND THAT IS THE POINT OF PUTTING IT HERE.
2709
+ `W35-T38` deleted `settings/StatementsPane.tsx` and left `routes_statements.preview` and
2710
+ the `overrideTo` branch of `.send` working, gated, and reachable from nowhere. A template
2711
+ editor with no preview and no test send is how somebody mails 200 real debtors to find out
2712
+ what the template looks like, so the editor above and these two controls are ONE piece of
2713
+ work rather than two tickets [[artifact-with-no-importer]].
2714
+ */}
2715
+ <h3>Check it before it goes</h3>
2716
+ <div className="auto-field">
2717
+ <label htmlFor="autox-stmt-customer">Try it on this customer</label>
2718
+ <select
2719
+ id="autox-stmt-customer"
2720
+ className="auto-input"
2721
+ value={customer}
2722
+ disabled={busy || !world}
2723
+ onChange={(e) => {
2724
+ setCustomer(e.target.value);
2725
+ setShot(null);
2726
+ setOutcome("");
2727
+ }}
2728
+ >
2729
+ <option value="">
2730
+ {world
2731
+ ? "Select a customer…"
2732
+ : worldErr
2733
+ ? "The collection list could not be read"
2734
+ : "Reading the collection list…"}
2735
+ </option>
2736
+ {rows.map((r) => (
2737
+ <option key={r.Customer} value={r.Customer}>
2738
+ {r.Customer}
2739
+ {r.Tier ? ` (${r.Tier})` : ""}
2740
+ {r.Email ? "" : ", no email on record"}
2741
+ </option>
2742
+ ))}
2743
+ </select>
2744
+ <p className="auto-hint">
2745
+ Whoever you pick here is only used for the preview and the test send below. It does not
2746
+ narrow what the agent sends.
2747
+ </p>
2748
+ </div>
2749
+
2750
+ <div className="autox-stmt-row">
2751
+ <button
2752
+ type="button"
2753
+ className="auto-btn"
2754
+ disabled={!canPreview}
2755
+ onClick={async () => {
2756
+ setWorking("preview");
2757
+ setOutcome("");
2758
+ try {
2759
+ setShot(await previewStatement(customer, templates));
2760
+ } catch (e: unknown) {
2761
+ setShot(null);
2762
+ setOutcome(e instanceof Error ? e.message : "the preview could not be rendered");
2763
+ } finally {
2764
+ setWorking("");
2765
+ }
2766
+ }}
2767
+ >
2768
+ {working === "preview" ? "Rendering…" : "Preview"}
2769
+ </button>
2770
+ </div>
2771
+
2772
+ {shot ? (
2773
+ <div className="autox-stmt-shot">
2774
+ <p className="auto-hint">
2775
+ To {shot.to || "no address on this customer's record"}. Subject: {shot.subject}
2776
+ </p>
2777
+ {/*
2778
+ β›” SANDBOXED, AND `sandbox=""` IS THE WHOLE POINT. This body is rendered by the SAME
2779
+ `render_statement_html` the send door uses, so what a person approves is what goes out.
2780
+ But it interpolates customer names and document numbers straight out of Odoo, and an
2781
+ empty sandbox attribute is the most restrictive value there is: no scripts, no forms,
2782
+ no same-origin. Painting it with `dangerouslySetInnerHTML` instead would run whatever a
2783
+ partner name happens to contain, inside our own origin.
2784
+ */}
2785
+ <iframe
2786
+ className="autox-stmt-frame"
2787
+ title="Statement preview"
2788
+ sandbox=""
2789
+ srcDoc={shot.html}
2790
+ />
2791
+ </div>
2792
+ ) : null}
2793
+
2794
+ <div className="auto-field">
2795
+ <label htmlFor="autox-stmt-testto">Send one test to this address</label>
2796
+ <input
2797
+ id="autox-stmt-testto"
2798
+ className="auto-input"
2799
+ value={testTo}
2800
+ disabled={busy || !world}
2801
+ onChange={(e) => setTestTo(e.target.value)}
2802
+ />
2803
+ <p className="auto-hint">
2804
+ {world?.safeMode
2805
+ ? "Safe mode is on, so mail can only reach "
2806
+ + (world.safeRecipients.length
2807
+ ? world.safeRecipients.join(", ")
2808
+ : "an allow-list nobody is on yet")
2809
+ + ". Anything else is refused."
2810
+ : "One statement, to this address only. The customer above is never mailed by this "
2811
+ + "button."}
2812
+ </p>
2813
+ </div>
2814
+
2815
+ <div className="autox-stmt-row">
2816
+ <button
2817
+ type="button"
2818
+ className="auto-btn"
2819
+ disabled={!canTest}
2820
+ onClick={async () => {
2821
+ setWorking("test");
2822
+ setOutcome("");
2823
+ try {
2824
+ const r = await testSendStatement(customer, testTo, templates);
2825
+ /* β›” THE OUTCOME IS READ OFF THE RESPONSE, never assumed from a 200. The send door
2826
+ answers `{sent, failed, skipped}` per customer precisely so that "we could not"
2827
+ and "there was nowhere to send" stay distinct, and SAFE_MODE's refusal arrives as
2828
+ a `failed` row with its own sentence rather than as an HTTP error. A caller that
2829
+ printed "sent" on a 200 would report a delivery the guardrail blocked. */
2830
+ const failed = r.failed?.[0];
2831
+ const skipped = r.skipped?.[0];
2832
+ setOutcome(
2833
+ failed
2834
+ ? `Not sent. ${failed.error}`
2835
+ : skipped
2836
+ ? `Not sent. ${skipped.reason}`
2837
+ : r.sent?.length
2838
+ ? `Sent one test to ${r.sent[0].to}.`
2839
+ : "The send door reported nothing sent and gave no reason."
2840
+ );
2841
+ } catch (e: unknown) {
2842
+ setOutcome(e instanceof Error ? e.message : "the test send was refused");
2843
+ } finally {
2844
+ setWorking("");
2845
+ }
2846
+ }}
2847
+ >
2848
+ {working === "test" ? "Sending…" : "Send one test"}
2849
+ </button>
2850
+ </div>
2851
+ {outcome ? <p className="auto-note">{outcome}</p> : null}
2852
+ </>
2853
+ );
2854
+ }
2855
+
2856
+ /**
2857
+ * ⭐⭐ WAVE 36 Β· W36-T69 (owner item 8, ruling R9) + E-13 β€” A CONFIG THAT CAN BE READ AND NOT TYPED.
2858
+ *
2859
+ * β›” ONE MECHANISM, THREE CALLERS, and that is the point rather than a tidiness. Three different
2860
+ * steps need the same thing β€” a configuration a person may READ but not EDIT:
2861
+ * Β· an agent-authored Action (R9: *"its configuration can only be touched by the agent"*),
2862
+ * Β· `ai_enrich`, whose settings belong to a COLUMN (D-277's exit condition: "a NAMED first step
2863
+ * with the Field and the Database under Config"),
2864
+ * Β· `odoo_sync`, configured on the connector rather than here.
2865
+ * A second read-only renderer per case is how three surfaces come to disagree about what
2866
+ * "read-only" looks like, and E booked the last two as PENDING precisely because they had no
2867
+ * mechanism yet. They have one now.
2868
+ *
2869
+ * β›” AND IT IS A DEFINITION LIST, NOT DISABLED INPUTS. A greyed-out `<input>` still reads as a
2870
+ * control somebody could enable, invites a click that does nothing, and is announced as a form
2871
+ * field to a screen reader. What this renders is what it is: the values, as text, labelled.
2872
+ */
2873
+ function ReadOnlyConfig({ rows, note }: {
2874
+ rows: { label: string; value: string }[];
2875
+ note: string;
2876
+ }) {
2877
+ const shown = rows.filter((r) => r.value);
2878
+ return (
2879
+ <>
2880
+ {shown.length ? (
2881
+ <dl className="autox-ro">
2882
+ {shown.map((r) => (
2883
+ <div key={r.label}>
2884
+ <dt>{r.label}</dt>
2885
+ <dd>{r.value}</dd>
2886
+ </div>
2887
+ ))}
2888
+ </dl>
2889
+ ) : (
2890
+ /* ⚠ NEVER AN EMPTY PANEL. A step whose config has not been filled in yet still has to say
2891
+ what it IS, or the reader meets the captioned-step-over-nothing that D-277 reported. */
2892
+ <p className="auto-note">This step has nothing configured yet.</p>
2893
+ )}
2894
+ <p className="auto-hint">{note}</p>
2895
+ </>
2896
+ );
2897
+ }
2898
+
2899
  /**
2900
  * ⭐ WAVE 28 β€” EXPORTED so the include-row render suite can mount the real panel.
2901
  *
 
2908
  export function ActionProps({
2909
  action,
2910
  pinned,
2911
+ agentMark,
2912
  catalog,
2913
  tables,
2914
  onTablesChanged,
 
2936
  * refuses in five other places.
2937
  */
2938
  pinned: boolean;
2939
+ /**
2940
+ * ⭐ W36-T69 / R9 β€” the `agentActions` entry for THIS action, or `null`.
2941
+ * ⚠ OPTIONAL, and absent reads as "not agent-owned" (see the wire type's own note): an older
2942
+ * server sends no map, and a client that defaulted the other way would lock every step in the
2943
+ * product behind a rule nobody had applied.
2944
+ */
2945
+ agentMark?: { agent: string; agentName?: string; created?: string; by?: string } | null;
2946
  catalog: ActionCatalogRow[];
2947
  tables: UserTable[];
2948
  /** ITEM 9 / R11 β€” re-read the list after this panel's picker creates one. REQUIRED; see Props. */
 
2998
  * picker that reads as "this database has no views".
2999
  */
3000
  const walkViews = (tables.find((t) => t.key === walkTable) || null)?.views;
3001
+ /**
3002
+ * ⭐⭐ W36-T69 β€” IS THIS STEP'S CONFIGURATION READ-ONLY? ONE PREDICATE, CONSULTED BY EVERY EDITOR.
3003
+ *
3004
+ * β›” THE FIRST DRAFT GUARDED ONLY THE `group`/`create_record`/`find_records` TERNARY CHAIN, and
3005
+ * the render suite caught it in one line: `isWeb(...)`, `ai_agent` and the condition editor are
3006
+ * SEPARATE JSX expressions, so an agent-owned `web_read` still painted seven inputs under a
3007
+ * notice saying it could not be edited. A rule spelled once per arm is a rule with a list to
3008
+ * maintain, and the arm added next wave will not be on it.
3009
+ */
3010
+ const readOnly = !!agentMark
3011
+ || action.kind === "ai_enrich" || action.kind === "odoo_sync";
3012
 
3013
  return (
3014
  <>
 
3017
 
3018
  <h3>Configuration</h3>
3019
 
3020
+ {/*
3021
+ β›”β›” W36-T69 β€” THE AGENT-OWNED BRANCH COMES FIRST AND RETURNS, so no per-kind editor below
3022
+ can paint a control over a config the server will refuse. Putting it after them would leave
3023
+ the wall depending on which arm happened to match, which is a wall with a list to maintain.
3024
+ ⚠ R9 IS A WRITE RULE. The user reads everything and may still DELETE the step; only the
3025
+ typing is gone. Today the server 409s a PATCH and no client knows, so a person configures,
3026
+ saves, and meets a refusal they were never warned about.
3027
+ */}
3028
+ {agentMark ? (
3029
+ <>
3030
+ <p className="auto-note">
3031
+ {agentMark.agentName || agentMark.agent} set this step up, so its configuration is
3032
+ edited by that agent rather than by hand. You can still read it here, and you can
3033
+ delete the step.
3034
+ </p>
3035
+ <ReadOnlyConfig
3036
+ rows={Object.entries(cfg as Record<string, unknown>)
3037
+ .filter(([, v]) => typeof v !== "object" || v === null)
3038
+ .map(([k, v]) => ({ label: k, value: v === null ? "" : String(v) }))}
3039
+ note="Ask the agent to change any of this."
3040
+ />
3041
+ </>
3042
+ ) : action.kind === "ai_enrich" ? (
3043
+ /* ⭐ D-277's exit condition, verbatim: "a NAMED first step with the Field and the Database
3044
+ under Config". The NAME arrives from E's catalog row; these are the two lines under it.
3045
+ An enrichment is configured on the COLUMN, which is why it is read-only here rather than
3046
+ missing: `patch_automation` already refuses a `field:` id. */
3047
+ <ReadOnlyConfig
3048
+ rows={[
3049
+ { label: "Column", value: String((cfg as { field?: string }).field || "") },
3050
+ { label: "Database",
3051
+ value: String((cfg as { tableLabel?: string; table?: string }).tableLabel
3052
+ || (cfg as { table?: string }).table || "") },
3053
+ ]}
3054
+ /* ⚠ A TEMPLATE, NOT A WRAPPED ATTRIBUTE STRING. JSX collapses whitespace in TEXT
3055
+ children but preserves it verbatim inside an attribute, so the wrapped literal that
3056
+ was here rendered a newline and fourteen spaces into the middle of the sentence. It
3057
+ looked right in the source and wrong on the screen, which is the only kind of typo
3058
+ a type-checker cannot help with. */
3059
+ note={"An enrichment belongs to its column. Change its prompt, model or schedule "
3060
+ + "on the column itself."}
3061
+ />
3062
+ ) : action.kind === "odoo_sync" ? (
3063
+ <ReadOnlyConfig
3064
+ rows={[
3065
+ { label: "Connector",
3066
+ value: String((cfg as { connector?: string }).connector || "") },
3067
+ { label: "Runs", value: String((cfg as { every?: string }).every || "") },
3068
+ ]}
3069
+ note="This schedule is configured on the connector, under Keychains."
3070
+ />
3071
+ ) : null}
3072
+
3073
+ {readOnly ? null : action.kind === "group" ? (
3074
  <>
3075
  {/*
3076
  ⭐ ONE CONDITION EDITOR PER BRANCH (C-FORK). The group's own `config.cond` is GONE β€”
 
3112
  </div>
3113
  ))}
3114
  <p className="auto-hint">
3115
+ The first branch whose conditions match is the one that runs. The record
3116
+ does not go down two of them.
3117
  </p>
3118
  </>
3119
  ) : null}
 
3171
  the same lie the value map was. */}
3172
  {pinned ? (
3173
  <p className="auto-hint">
3174
+ The search writes one row per profile it finds, into that database: handle,
3175
+ profile link, name, followers, following, average engagement, bio, link in
3176
+ bio, verified and category. It matches on the handle, so a profile found again updates its row instead of
3177
  adding another. There is nothing to map: the columns come from the search.
3178
  </p>
3179
  ) : null}
 
3760
  start, what to do in words, and where to put the answer. Deliberately NOT routed through
3761
  `WebActionConfig`: that component walks `WEB_SEEDS`, and sharing it would mean adding
3762
  `ai_agent` to a table the browser job's runner does not implement. */}
3763
+ {action.kind === "ai_agent" && !readOnly ? (
3764
  <>
3765
  <div className="auto-field">
3766
  <label htmlFor="autox-ai-url">
 
3802
  <div className="auto-field">
3803
  <label>Column to write the answer into</label>
3804
  <p className="auto-note">
3805
+ This flow has no record to write to. Give it a database on the trigger
3806
+ first.
3807
  </p>
3808
  </div>
3809
  ) : (
 
3866
  </>
3867
  ) : null}
3868
 
3869
+ {isWeb(action.kind) && !readOnly ? (
3870
  <WebActionConfig
3871
  kind={action.kind}
3872
  cfg={cfg as Record<string, unknown>}
 
3878
  />
3879
  ) : null}
3880
 
3881
+ {/* ⭐⭐ W36-T44 (D-316) β€” the arm whose absence was the whole defect. See
3882
+ `StatementActionConfig`: the server has seeded, validated and consumed these four keys
3883
+ since wave 35, and this line is what lets anybody type them. */}
3884
+ {action.kind === "send_statement" && !readOnly ? (
3885
+ <StatementActionConfig cfg={cfg as Record<string, unknown>} busy={busy} setCfg={setCfg} />
3886
+ ) : null}
3887
+
3888
  {/*
3889
  ⭐ WAVE 26 Β· ITEM 5 β€” OWNER RULINGS R9 AND R10, and they are two different fixes to one
3890
  symptom (*"the condition on action act_1 names no field"*).
 
3910
  walking a database that genuinely has no columns yet keeps its (empty) picker, because
3911
  for that flow the picker is the right control and adding a column is the fix.
3912
  */}
3913
+ {/*
3914
+ ⭐⭐ W36-T44 β€” AND A `send_statement` HAS NO CONDITIONS EITHER, for R9's reason arriving at
3915
+ a different kind. R9 removed the editor from `create_record` because that step MAKES the
3916
+ record there is nothing to test yet. This step is not record-scoped at all: a statements
3917
+ flow is BATCH-scoped, it binds no database (its customers come from the Odoo collection
3918
+ list, not a `ut_*` grid), and `assemble_statements` reads the step's `config` and NOTHING
3919
+ else. `is_statement_flow` requires exactly ONE enabled action, so there is no walk to
3920
+ condition and no second step to condition it against.
3921
+ β›” AND THE STORED KEY SURVIVES, WHICH IS WHY THE EDITOR MUST NOT. `clean_actions` drops
3922
+ `when` for `create_record` alone; on this kind it is kept, stored, reloaded, and read by
3923
+ nobody. That is a control whose value appears to be ignored, which is exactly the state R9
3924
+ called out as indefensible and exactly what the `discover_instagram` values editor was
3925
+ [[a-flag-can-ship-without-its-writer]]. The sentence points at the control that DOES narrow
3926
+ the batch, which is the tier filter above, and which is the control the run's own
3927
+ parked-batch banner has been naming all along.
3928
+ */}
3929
+ {/* β›” A CONDITION IS CONFIGURATION TOO. `clean_actions` keeps `when` on an agent-authored
3930
+ step exactly as it keeps its config, so leaving this editor live would be a control the
3931
+ server refuses on the same PATCH β€” the whole defect one field over. */}
3932
+ {readOnly ? null : action.kind === "send_statement" ? (
3933
+ <>
3934
+ <h3>Run this only when</h3>
3935
+ <p className="auto-note">
3936
+ A statements step runs over the whole collection list at once, not once per record, so
3937
+ there is no record here to test. Narrow it with Customers to include, above.
3938
+ </p>
3939
+ </>
3940
+ ) : action.kind === "create_record" ? null : !walkTable ? (
3941
  <>
3942
  <h3>Run this only when</h3>
3943
  <p className="auto-note">
3944
+ This flow has no record to test. Set conditions on the trigger instead.
3945
  </p>
3946
  </>
3947
  ) : (